Date: Sat, 4 Jun 2011 14:00:28 -0600
Subject: [PATCH 14/31] Add an action to remove all formats to the remove books
button
---
src/calibre/gui2/actions/delete.py | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/calibre/gui2/actions/delete.py b/src/calibre/gui2/actions/delete.py
index 43465512e0..bef456033b 100644
--- a/src/calibre/gui2/actions/delete.py
+++ b/src/calibre/gui2/actions/delete.py
@@ -94,6 +94,9 @@ class DeleteAction(InterfaceAction):
self.delete_menu.addAction(
_('Remove all formats from selected books, except...'),
self.delete_all_but_selected_formats)
+ self.delete_menu.addAction(
+ _('Remove all formats from selected books'),
+ self.delete_all_formats)
self.delete_menu.addAction(
_('Remove covers from selected books'), self.delete_covers)
self.delete_menu.addSeparator()
@@ -174,6 +177,28 @@ class DeleteAction(InterfaceAction):
if ids:
self.gui.tags_view.recount()
+ def delete_all_formats(self, *args):
+ ids = self._get_selected_ids()
+ if not ids:
+ return
+ if not confirm(''+_('All formats for the selected books will '
+ 'be deleted from your library.
'
+ 'The book metadata will be kept. Are you sure?')
+ +'
', 'delete_all_formats', self.gui):
+ return
+ db = self.gui.library_view.model().db
+ for id in ids:
+ fmts = db.formats(id, index_is_id=True, verify_formats=False)
+ if fmts:
+ for fmt in fmts.split(','):
+ self.gui.library_view.model().db.remove_format(id, fmt,
+ index_is_id=True, notify=False)
+ self.gui.library_view.model().refresh_ids(ids)
+ self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(),
+ self.gui.library_view.currentIndex())
+ if ids:
+ self.gui.tags_view.recount()
+
def remove_matching_books_from_device(self, *args):
if not self.gui.device_manager.is_device_connected:
d = error_dialog(self.gui, _('Cannot delete books'),
From c6733399f796c3a5f10d8ca5f7bff44f25d521f9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 4 Jun 2011 14:14:20 -0600
Subject: [PATCH 15/31] Fix Newsweek
---
recipes/newsweek.recipe | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe
index a31706e257..0cae4275b0 100644
--- a/recipes/newsweek.recipe
+++ b/recipes/newsweek.recipe
@@ -69,7 +69,11 @@ class Newsweek(BasicNewsRecipe):
for section, shref in self.newsweek_sections():
self.log('Processing section', section, shref)
articles = []
- soups = [self.index_to_soup(shref)]
+ try:
+ soups = [self.index_to_soup(shref)]
+ except:
+ self.log.warn('Section %s not found, skipping'%section)
+ continue
na = soups[0].find('a', rel='next')
if na:
soups.append(self.index_to_soup(self.BASE_URL+na['href']))
From 5e4e45ec92523f4e7b145d6f845160743600efbc Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 4 Jun 2011 19:26:15 -0600
Subject: [PATCH 16/31] Remove unneccessary import of PyQt
---
src/calibre/utils/config_base.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py
index 7660370353..ae2806176b 100644
--- a/src/calibre/utils/config_base.py
+++ b/src/calibre/utils/config_base.py
@@ -223,8 +223,7 @@ class OptionSet(object):
if val is val is True or val is False or val is None or \
isinstance(val, (int, float, long, basestring)):
return repr(val)
- from PyQt4.QtCore import QString
- if isinstance(val, QString):
+ if val.__class__.__name__ == 'QString' or hasattr(val, 'isNull'):
return repr(unicode(val))
pickle = cPickle.dumps(val, -1)
return 'cPickle.loads(%s)'%repr(pickle)
From 3fb57e64f29eabf875fe8b6343df04d7817d75f7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 4 Jun 2011 19:55:19 -0600
Subject: [PATCH 17/31] ...
---
src/calibre/utils/config_base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py
index ae2806176b..345caae384 100644
--- a/src/calibre/utils/config_base.py
+++ b/src/calibre/utils/config_base.py
@@ -223,7 +223,7 @@ class OptionSet(object):
if val is val is True or val is False or val is None or \
isinstance(val, (int, float, long, basestring)):
return repr(val)
- if val.__class__.__name__ == 'QString' or hasattr(val, 'isNull'):
+ if val.__class__.__name__ == 'QString':
return repr(unicode(val))
pickle = cPickle.dumps(val, -1)
return 'cPickle.loads(%s)'%repr(pickle)
From 4a8d33a4552535d88c5d4456287bbec32be73599 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sat, 4 Jun 2011 22:25:46 -0600
Subject: [PATCH 18/31] Store translations compressed in the calibre install
dir. Also remove the broken librarything executable.
---
setup/translations.py | 17 ++++++++---
src/calibre/linux.py | 1 -
src/calibre/translations/dynamic.py | 18 +++++++-----
src/calibre/utils/localization.py | 45 ++++++++++++++++-------------
4 files changed, 49 insertions(+), 32 deletions(-)
diff --git a/setup/translations.py b/setup/translations.py
index 1f026555ec..3a33b7bcc4 100644
--- a/setup/translations.py
+++ b/setup/translations.py
@@ -85,7 +85,7 @@ class Translations(POT):
def mo_file(self, po_file):
locale = os.path.splitext(os.path.basename(po_file))[0]
- return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo')
+ return locale, os.path.join(self.DEST, locale, 'messages.mo')
def run(self, opts):
@@ -94,9 +94,8 @@ class Translations(POT):
base = os.path.dirname(dest)
if not os.path.exists(base):
os.makedirs(base)
- if self.newer(dest, f):
- self.info('\tCompiling translations for', locale)
- subprocess.check_call(['msgfmt', '-o', dest, f])
+ self.info('\tCompiling translations for', locale)
+ subprocess.check_call(['msgfmt', '-o', dest, f])
if locale in ('en_GB', 'nds', 'te', 'yi'):
continue
pycountry = self.j(sysconfig.get_python_lib(), 'pycountry',
@@ -123,6 +122,16 @@ class Translations(POT):
shutil.copy2(f, dest)
self.write_stats()
+ self.freeze_locales()
+
+ def freeze_locales(self):
+ zf = self.DEST + '.zip'
+ from calibre import CurrentDir
+ from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
+ with ZipFile(zf, 'w', ZIP_DEFLATED) as zf:
+ with CurrentDir(self.DEST):
+ zf.add_dir('.')
+ shutil.rmtree(self.DEST)
@property
def stats(self):
diff --git a/src/calibre/linux.py b/src/calibre/linux.py
index 9e58d4f638..1686f66b22 100644
--- a/src/calibre/linux.py
+++ b/src/calibre/linux.py
@@ -23,7 +23,6 @@ entry_points = {
'calibre-server = calibre.library.server.main:main',
'lrf2lrs = calibre.ebooks.lrf.lrfparser:main',
'lrs2lrf = calibre.ebooks.lrf.lrs.convert_from:main',
- 'librarything = calibre.ebooks.metadata.library_thing:main',
'calibre-debug = calibre.debug:main',
'calibredb = calibre.library.cli:main',
'calibre-parallel = calibre.utils.ipc.worker:main',
diff --git a/src/calibre/translations/dynamic.py b/src/calibre/translations/dynamic.py
index c1f368ff5a..4d65475ac0 100644
--- a/src/calibre/translations/dynamic.py
+++ b/src/calibre/translations/dynamic.py
@@ -5,10 +5,9 @@ Dynamic language lookup of translations for user-visible strings.
__license__ = 'GPL v3'
__copyright__ = '2008, Marshall T. Vandegrift '
-import os
-
+import cStringIO
from gettext import GNUTranslations
-from calibre.utils.localization import get_lc_messages_path
+from calibre.utils.localization import get_lc_messages_path, ZipFile
__all__ = ['translate']
@@ -21,10 +20,15 @@ def translate(lang, text):
else:
mpath = get_lc_messages_path(lang)
if mpath is not None:
- p = os.path.join(mpath, 'messages.mo')
- if os.path.exists(p):
- trans = GNUTranslations(open(p, 'rb'))
- _CACHE[lang] = trans
+ with ZipFile(P('localization/locales.zip',
+ allow_user_override=False), 'r') as zf:
+ try:
+ buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+ except:
+ pass
+ else:
+ trans = GNUTranslations(buf)
+ _CACHE[lang] = trans
if trans is None:
return getattr(__builtins__, '_', lambda x: x)(text)
return trans.ugettext(text)
diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index 92e6ea9b5e..a8285ad5a2 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
+from __future__ import absolute_import
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal '
@@ -8,13 +8,14 @@ __docformat__ = 'restructuredtext en'
import os, locale, re, cStringIO, cPickle
from gettext import GNUTranslations
+from zipfile import ZipFile
_available_translations = None
def available_translations():
global _available_translations
if _available_translations is None:
- stats = P('localization/stats.pickle')
+ stats = P('localization/stats.pickle', allow_user_override=False)
if os.path.exists(stats):
stats = cPickle.load(open(stats, 'rb'))
else:
@@ -49,21 +50,20 @@ def get_lang():
lang = 'en'
return lang
-def messages_path(lang):
- return P('localization/locales/%s/LC_MESSAGES'%lang)
-
def get_lc_messages_path(lang):
hlang = None
- if lang in available_translations():
- hlang = lang
- else:
- xlang = lang.split('_')[0]
- if xlang in available_translations():
- hlang = xlang
- if hlang is not None:
- return messages_path(hlang)
- return None
+ if zf_exists():
+ if lang in available_translations():
+ hlang = lang
+ else:
+ xlang = lang.split('_')[0]
+ if xlang in available_translations():
+ hlang = xlang
+ return hlang
+def zf_exists():
+ return os.path.exists(P('localization/locales.zip',
+ allow_user_override=False))
def set_translators():
# To test different translations invoke as
@@ -79,12 +79,17 @@ def set_translators():
mpath = get_lc_messages_path(lang)
if mpath is not None:
- if buf is None:
- buf = open(os.path.join(mpath, 'messages.mo'), 'rb')
- mpath = mpath.replace(os.sep+'nds'+os.sep, os.sep+'de'+os.sep)
- isof = os.path.join(mpath, 'iso639.mo')
- if os.path.exists(isof):
- iso639 = open(isof, 'rb')
+ with ZipFile(P('localization/locales.zip',
+ allow_user_override=False), 'r') as zf:
+ if buf is None:
+ buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+ if mpath == 'nds':
+ mpath = 'de'
+ isof = mpath + '/iso639.mo'
+ try:
+ iso639 = cStringIO.StringIO(zf.read(isof))
+ except:
+ pass # No iso639 translations for this lang
if buf is not None:
t = GNUTranslations(buf)
From 14dc0006d0c6a86e7d680b6284444d2cdb279741 Mon Sep 17 00:00:00 2001
From: koehler
Date: Sun, 5 Jun 2011 14:15:09 +0200
Subject: [PATCH 19/31] cssutils has recently renamed CSSValueList to
PropertyValue. Fix resulting breakage in src/calibre/ebooks/oeb/stylizer.py
---
src/calibre/ebooks/oeb/stylizer.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index dc73862022..ed1d2a576c 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -13,7 +13,11 @@ from weakref import WeakKeyDictionary
from xml.dom import SyntaxErr as CSSSyntaxError
import cssutils
from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
- CSSValueList, CSSFontFaceRule, cssproperties)
+ CSSFontFaceRule, cssproperties)
+try:
+ from cssutils.css import CSSValueList
+except ImportError:
+ from cssutils.css import PropertyValue as CSSValueList
from cssutils import profile as cssprofiles
from lxml import etree
from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError
From 4018fb48a5eaf44da96cccf5c145d37d0a2f0d41 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 08:09:52 -0600
Subject: [PATCH 20/31] Heise Online by schuster and improved express.de and
max planck
---
recipes/express_de.recipe | 10 +++----
recipes/heise_online.recipe | 52 +++++++++++++++++++++++++++++++++++++
recipes/max_planck.recipe | 8 +++---
3 files changed, 61 insertions(+), 9 deletions(-)
create mode 100644 recipes/heise_online.recipe
diff --git a/recipes/express_de.recipe b/recipes/express_de.recipe
index 255538b08e..10595b9d92 100644
--- a/recipes/express_de.recipe
+++ b/recipes/express_de.recipe
@@ -1,5 +1,4 @@
from calibre.web.feeds.news import BasicNewsRecipe
-
class AdvancedUserRecipe1303841067(BasicNewsRecipe):
title = u'Express.de'
@@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
extra_css = '''
h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
h1{ font-family:Arial,Helvetica,sans-serif; font-size:x-large; font-weight:bold;}
-
'''
remove_javascript = True
remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})]
@@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
dict(id='Logo'),
dict(id='MainLinkSpacer'),
dict(id='MainLinks'),
+ dict(id='ContainerPfad'), #neu
dict(title='Diese Seite Bookmarken'),
dict(name='span'),
@@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
dict(name='div', attrs={'class':'HeaderSearch'}),
dict(name='div', attrs={'class':'sbutton'}),
dict(name='div', attrs={'class':'active'}),
-
+ dict(name='div', attrs={'class':'MoreNews'}), #neu
+ dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu
]
@@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
(u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'),
(u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'),
(u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'),
+ ]
-
-
-]
diff --git a/recipes/heise_online.recipe b/recipes/heise_online.recipe
new file mode 100644
index 0000000000..f83ff8126b
--- /dev/null
+++ b/recipes/heise_online.recipe
@@ -0,0 +1,52 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+ title = 'Heise-online'
+ description = 'News vom Heise-Verlag'
+ __author__ = 'schuster'
+ use_embedded_content = False
+ language = 'de'
+ oldest_article = 2
+ max_articles_per_feed = 35
+ rescale_images = True
+ remove_empty_feeds = True
+ timeout = 5
+ no_stylesheets = True
+
+
+ remove_tags_after = dict(name ='p', attrs={'class':'editor'})
+ remove_tags = [dict(id='navi_top_container'),
+ dict(id='navi_bottom'),
+ dict(id='mitte_rechts'),
+ dict(id='navigation'),
+ dict(id='subnavi'),
+ dict(id='social_bookmarks'),
+ dict(id='permalink'),
+ dict(id='content_foren'),
+ dict(id='seiten_navi'),
+ dict(id='adbottom'),
+ dict(id='sitemap')]
+
+ feeds = [
+ ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'),
+ ('Auto', 'http://www.heise.de/autos/rss/news.rdf'),
+ ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'),
+ ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'),
+ ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'),
+ ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'),
+ ('Open ', 'http://www.heise.de/open/news/news-atom.xml'),
+ ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'),
+ ('Security ', 'http://www.heise.de/security/news/news-atom.xml'),
+ ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'),
+ ('iX', 'http://www.heise.de/ix/news/news.rdf'),
+ ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'),
+ ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'),
+ ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'),
+ ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'),
+ ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'),
+ ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf')
+]
+
+ def print_version(self, url):
+ return url + '?view=print'
+
diff --git a/recipes/max_planck.recipe b/recipes/max_planck.recipe
index e9bf62008a..cf778a7374 100644
--- a/recipes/max_planck.recipe
+++ b/recipes/max_planck.recipe
@@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
title = u'Max-Planck-Inst.'
__author__ = 'schuster'
- remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}),
- dict(id=['ie_clearing', 'col2', 'col2_content']),
- dict(name=['script', 'noscript', 'style'])]
oldest_article = 30
max_articles_per_feed = 100
no_stylesheets = True
@@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe):
language = 'de'
remove_javascript = True
+ remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}),
+ dict(id=['skiplinks'])]
+
+
+
def print_version(self, url):
split_url = url.split("/")
print_url = 'http://www.mpg.de/print/' + split_url[3]
From 542f468465e96ac58275964703c8cf4aa914900a Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 08:12:16 -0600
Subject: [PATCH 21/31] Driver for Samsung Galaxy S2
---
src/calibre/devices/android/driver.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index b557ac3526..554ea3c698 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -53,6 +53,7 @@ class ANDROID(USBMS):
0x681c : [0x0222, 0x0224, 0x0400],
0x6640 : [0x0100],
0x685e : [0x0400],
+ 0x6860 : [0x0400],
0x6877 : [0x0400],
},
From 37be144dfec849a09b2136f379e036c2b86c0b64 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 08:16:46 -0600
Subject: [PATCH 22/31] Amazon metadata plugin: Fix parsing of published date
from amazon.de when it has februar in it
---
src/calibre/ebooks/metadata/sources/amazon.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py
index 7da37ce5af..6220f29020 100644
--- a/src/calibre/ebooks/metadata/sources/amazon.py
+++ b/src/calibre/ebooks/metadata/sources/amazon.py
@@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{
months = {
'de': {
1 : ['jän'],
+ 2 : ['februar'],
3 : ['märz'],
5 : ['mai'],
6 : ['juni'],
From 06c76ae36dc5ed4e89d54c87b35daf230541a85b Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 11:45:45 -0600
Subject: [PATCH 23/31] Brigitte.de by schuster
---
recipes/brigitte_de.recipe | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
create mode 100644 recipes/brigitte_de.recipe
diff --git a/recipes/brigitte_de.recipe b/recipes/brigitte_de.recipe
new file mode 100644
index 0000000000..860d5176ac
--- /dev/null
+++ b/recipes/brigitte_de.recipe
@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe(BasicNewsRecipe):
+
+ title = u'Brigitte.de'
+ __author__ = 'schuster'
+ oldest_article = 14
+ max_articles_per_feed = 100
+ no_stylesheets = True
+ use_embedded_content = False
+ language = 'de'
+ remove_javascript = True
+ remove_empty_feeds = True
+ timeout = 10
+ cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+ masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg'
+
+
+ remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}),
+ dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']),
+ dict(name=['hjtrs', 'kud'])]
+
+ feeds = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'),
+ (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'),
+ (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'),
+ (u'Figur', u'http://www.brigitte.de/figur/feed.rss'),
+ (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'),
+ (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'),
+ (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'),
+ (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'),
+ (u'Reise', u'http://www.brigitte.de/reise/feed.rss'),
+ (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'),
+ (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'),
+ (u'Job', u'http://www.brigitte.de/job/feed.rss'),
+ (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'),
+]
From 70e0cd6ba43b379183e1b2f6a02196c55c5a0c66 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 13:38:50 -0600
Subject: [PATCH 24/31] When writing files to zipfile, reset timestamp if it
doesn't fit in 1980's vintage storage structures
---
src/calibre/utils/zipfile.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py
index fa15bff4b4..394b667bbf 100644
--- a/src/calibre/utils/zipfile.py
+++ b/src/calibre/utils/zipfile.py
@@ -340,7 +340,14 @@ class ZipInfo (object):
"""Return the per-file header as a string."""
dt = self.date_time
dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]
+ # Added by Kovid to reset timestamp to default if it overflows the DOS
+ # limits
+ if dosdate > 0xffff or dosdate < 0:
+ dosdate = 0
dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)
+ if dostime > 0xffff or dostime < 0:
+ dostime = 0
+
if self.flag_bits & 0x08:
# Set these to zero because we write them after the file data
CRC = compress_size = file_size = 0
From 4eb7ef507c1c69499dc5521f13e19760176b81de Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 17:44:12 -0600
Subject: [PATCH 25/31] ...
---
src/calibre/utils/zipfile.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py
index 394b667bbf..868445d15a 100644
--- a/src/calibre/utils/zipfile.py
+++ b/src/calibre/utils/zipfile.py
@@ -148,6 +148,12 @@ def decode_arcname(name):
name = name.decode('utf-8', 'replace')
return name
+# Added by Kovid to reset timestamp to default if it overflows the DOS
+# limits
+def fixtimevar(val):
+ if val < 0 or val > 0xffff:
+ val = 0
+ return val
def _check_zipfile(fp):
try:
@@ -340,13 +346,7 @@ class ZipInfo (object):
"""Return the per-file header as a string."""
dt = self.date_time
dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]
- # Added by Kovid to reset timestamp to default if it overflows the DOS
- # limits
- if dosdate > 0xffff or dosdate < 0:
- dosdate = 0
dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)
- if dostime > 0xffff or dostime < 0:
- dostime = 0
if self.flag_bits & 0x08:
# Set these to zero because we write them after the file data
@@ -372,7 +372,7 @@ class ZipInfo (object):
filename, flag_bits = self._encodeFilenameFlags()
header = struct.pack(structFileHeader, stringFileHeader,
self.extract_version, self.reserved, flag_bits,
- self.compress_type, dostime, dosdate, CRC,
+ self.compress_type, fixtimevar(dostime), fixtimevar(dosdate), CRC,
compress_size, file_size,
len(filename), len(extra))
return header + filename + extra
@@ -1328,8 +1328,8 @@ class ZipFile:
for zinfo in self.filelist: # write central directory
count = count + 1
dt = zinfo.date_time
- dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]
- dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)
+ dosdate = fixtimevar((dt[0] - 1980) << 9 | dt[1] << 5 | dt[2])
+ dostime = fixtimevar(dt[3] << 11 | dt[4] << 5 | (dt[5] // 2))
extra = []
if zinfo.file_size > ZIP64_LIMIT \
or zinfo.compress_size > ZIP64_LIMIT:
From 571cc551c2b82ebc7c519939b120323dfe957a7b Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 17:48:28 -0600
Subject: [PATCH 26/31] ...
---
src/calibre/gui2/preferences/toolbar.ui | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/src/calibre/gui2/preferences/toolbar.ui b/src/calibre/gui2/preferences/toolbar.ui
index 0e601f74a2..51819b0df2 100644
--- a/src/calibre/gui2/preferences/toolbar.ui
+++ b/src/calibre/gui2/preferences/toolbar.ui
@@ -16,13 +16,28 @@
-
+
+
+ 75
+ true
+
+
- Customize the actions in:
+ Choose the &toolbar to customize:
+
+
+ what
-
+
+
+ 75
+ true
+
+
QComboBox::AdjustToMinimumContentsLengthWithIcon
From 796dc93450a9c3129fd3713198dd5e8a77508de5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 17:49:06 -0600
Subject: [PATCH 27/31] ...
---
src/calibre/gui2/dialogs/tweak_epub.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py
index 732d74b77d..e0be9fa1e9 100755
--- a/src/calibre/gui2/dialogs/tweak_epub.py
+++ b/src/calibre/gui2/dialogs/tweak_epub.py
@@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal '
__docformat__ = 'restructuredtext en'
import os, shutil
-from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
+from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED
from PyQt4.Qt import QDialog
From a5d1d6b196d51febce03099e3bccd07528c4d2d5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 18:54:05 -0600
Subject: [PATCH 28/31] ...
---
src/calibre/manual/faq.rst | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 99c53e5a37..c1aa4e5614 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -562,6 +562,16 @@ You have two choices:
1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_.
2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
+Why doesn't |app| have an automatic update?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For many reasons:
+
+ * *There is no need to update every week*. If you are happy with how |app| works turn off the update notification and be on your merry way. Check back to see if you want to update once a year or so.
+ * Pre downloading the updates for all users in the background would mean require about 80TB of bandwidth *every week*. That costs thousands of dollars a month. And |app| is currently growing at 300,000 new users every month.
+ * If I implement a dialog that downloads the update and launches it, instead of going to the website as it does now, that would save the most ardent |app| updater, *at most five clicks a week*. There are far higher priority things to do in |app| development.
+ * If you really, really hate downloading |app| every week but still want to be upto the latest, I encourage you to run from source, which makes updating trivial. Instructions are :ref:`here `.
+
How is |app| licensed?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode `_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 `_.
From 9ea27629c26db1cc2ee97160fe3ed0a546006420 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 19:17:28 -0600
Subject: [PATCH 29/31] Driver for the Notion Ink Adam
---
src/calibre/customize/builtins.py | 9 +++++----
src/calibre/devices/misc.py | 22 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 9ebec5e7e8..33685ea254 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE
from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER
from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \
- GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \
- TREKSTOR, EEEREADER, NEXTBOOK
+from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
+ GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
+ TREKSTOR, EEEREADER, NEXTBOOK, ADAM)
from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
from calibre.devices.kobo.driver import KOBO
from calibre.devices.bambook.driver import BAMBOOK
@@ -744,6 +744,7 @@ plugins += [
TREKSTOR,
EEEREADER,
NEXTBOOK,
+ ADAM,
ITUNES,
BOEYE_BEX,
BOEYE_BDX,
@@ -1231,7 +1232,7 @@ class StoreEpubBudStore(StoreBase):
name = 'ePub Bud'
description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.'
actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore'
-
+
drm_free_only = True
headquarters = 'US'
formats = ['EPUB']
diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py
index 936faeb32d..2a6a76719d 100644
--- a/src/calibre/devices/misc.py
+++ b/src/calibre/devices/misc.py
@@ -255,6 +255,28 @@ class EEEREADER(USBMS):
VENDOR_NAME = 'LINUX'
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'
+class ADAM(USBMS):
+
+ name = 'Notion Ink Adam device interface'
+ gui_name = 'Adam'
+
+ description = _('Communicate with the Adam tablet')
+ author = 'Kovid Goyal'
+ supported_platforms = ['windows', 'osx', 'linux']
+
+ # Ordered list of supported formats
+ FORMATS = ['epub', 'pdf', 'doc']
+
+ VENDOR_ID = [0x0955]
+ PRODUCT_ID = [0x7100]
+ BCD = [0x9999]
+
+ EBOOK_DIR_MAIN = 'eBooks'
+
+ VENDOR_NAME = 'NI'
+ WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM']
+ SUPPORTS_SUB_DIRS = True
+
class NEXTBOOK(USBMS):
name = 'Nextbook device interface'
From b22d5ac5fb4501975c3c5bf4e9e0e003a56b1243 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 20:21:09 -0600
Subject: [PATCH 30/31] Windows build: Add code to load .pyd python extensions
from a zip file. This allows many more files in the calibre installation to
be zipped up, speeding up the installer.
---
setup/installer/windows/MemoryModule.c | 689 +++++++++++++++++++++++
setup/installer/windows/MemoryModule.h | 58 ++
setup/installer/windows/freeze.py | 92 ++-
setup/installer/windows/site.py | 109 ++--
setup/installer/windows/util.c | 126 ++++-
setup/installer/windows/wix-template.xml | 4 -
src/calibre/debug.py | 5 +
src/calibre/test_build.py | 103 ++++
8 files changed, 1089 insertions(+), 97 deletions(-)
create mode 100644 setup/installer/windows/MemoryModule.c
create mode 100644 setup/installer/windows/MemoryModule.h
create mode 100644 src/calibre/test_build.py
diff --git a/setup/installer/windows/MemoryModule.c b/setup/installer/windows/MemoryModule.c
new file mode 100644
index 0000000000..253c8d7d9f
--- /dev/null
+++ b/setup/installer/windows/MemoryModule.c
@@ -0,0 +1,689 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2 with additions from Thomas Heller
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.c
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ * Portions Copyright (C) 2005 Thomas Heller.
+ *
+ */
+
+// disable warnings about pointer <-> DWORD conversions
+#pragma warning( disable : 4311 4312 )
+
+#include
+#include
+#if DEBUG_OUTPUT
+#include
+#endif
+
+#ifndef IMAGE_SIZEOF_BASE_RELOCATION
+// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
+# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
+#endif
+#include "MemoryModule.h"
+
+/*
+ XXX We need to protect at least walking the 'loaded' linked list with a lock!
+*/
+
+/******************************************************************/
+FINDPROC findproc;
+void *findproc_data = NULL;
+
+struct NAME_TABLE {
+ char *name;
+ DWORD ordinal;
+};
+
+typedef struct tagMEMORYMODULE {
+ PIMAGE_NT_HEADERS headers;
+ unsigned char *codeBase;
+ HMODULE *modules;
+ int numModules;
+ int initialized;
+
+ struct NAME_TABLE *name_table;
+
+ char *name;
+ int refcount;
+ struct tagMEMORYMODULE *next, *prev;
+} MEMORYMODULE, *PMEMORYMODULE;
+
+typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
+
+#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx]
+
+MEMORYMODULE *loaded; /* linked list of loaded memory modules */
+
+/* private - insert a loaded library in a linked list */
+static void _Register(char *name, MEMORYMODULE *module)
+{
+ module->next = loaded;
+ if (loaded)
+ loaded->prev = module;
+ module->prev = NULL;
+ loaded = module;
+}
+
+/* private - remove a loaded library from a linked list */
+static void _Unregister(MEMORYMODULE *module)
+{
+ free(module->name);
+ if (module->prev)
+ module->prev->next = module->next;
+ if (module->next)
+ module->next->prev = module->prev;
+ if (module == loaded)
+ loaded = module->next;
+}
+
+/* public - replacement for GetModuleHandle() */
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName)
+{
+ MEMORYMODULE *p = loaded;
+ while (p) {
+ // If already loaded, only increment the reference count
+ if (0 == stricmp(lpModuleName, p->name)) {
+ return (HMODULE)p;
+ }
+ p = p->next;
+ }
+ return GetModuleHandle(lpModuleName);
+}
+
+/* public - replacement for LoadLibrary, but searches FIRST for memory
+ libraries, then for normal libraries. So, it will load libraries AS memory
+ module if they are found by findproc().
+*/
+HMODULE MyLoadLibrary(char *lpFileName)
+{
+ MEMORYMODULE *p = loaded;
+ HMODULE hMod;
+
+ while (p) {
+ // If already loaded, only increment the reference count
+ if (0 == stricmp(lpFileName, p->name)) {
+ p->refcount++;
+ return (HMODULE)p;
+ }
+ p = p->next;
+ }
+ if (findproc && findproc_data) {
+ void *pdata = findproc(lpFileName, findproc_data);
+ if (pdata) {
+ hMod = MemoryLoadLibrary(lpFileName, pdata);
+ free(p);
+ return hMod;
+ }
+ }
+ hMod = LoadLibrary(lpFileName);
+ return hMod;
+}
+
+/* public - replacement for GetProcAddress() */
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName)
+{
+ MEMORYMODULE *p = loaded;
+ while (p) {
+ if ((HMODULE)p == hModule)
+ return MemoryGetProcAddress(p, lpProcName);
+ p = p->next;
+ }
+ return GetProcAddress(hModule, lpProcName);
+}
+
+/* public - replacement for FreeLibrary() */
+BOOL MyFreeLibrary(HMODULE hModule)
+{
+ MEMORYMODULE *p = loaded;
+ while (p) {
+ if ((HMODULE)p == hModule) {
+ if (--p->refcount == 0) {
+ _Unregister(p);
+ MemoryFreeLibrary(p);
+ }
+ return TRUE;
+ }
+ p = p->next;
+ }
+ return FreeLibrary(hModule);
+}
+
+#if DEBUG_OUTPUT
+static void
+OutputLastError(const char *msg)
+{
+ LPVOID tmp;
+ char *tmpmsg;
+ FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+ NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
+ tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
+ sprintf(tmpmsg, "%s: %s", msg, tmp);
+ OutputDebugString(tmpmsg);
+ LocalFree(tmpmsg);
+ LocalFree(tmp);
+}
+#endif
+
+/*
+static int dprintf(char *fmt, ...)
+{
+ char Buffer[4096];
+ va_list marker;
+ int result;
+
+ va_start(marker, fmt);
+ result = vsprintf(Buffer, fmt, marker);
+ OutputDebugString(Buffer);
+ return result;
+}
+*/
+
+static void
+CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
+{
+ int i, size;
+ unsigned char *codeBase = module->codeBase;
+ unsigned char *dest;
+ PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+ for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++)
+ {
+ if (section->SizeOfRawData == 0)
+ {
+ // section doesn't contain data in the dll itself, but may define
+ // uninitialized data
+ size = old_headers->OptionalHeader.SectionAlignment;
+ if (size > 0)
+ {
+ dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+ size,
+ MEM_COMMIT,
+ PAGE_READWRITE);
+
+ section->Misc.PhysicalAddress = (DWORD)dest;
+ memset(dest, 0, size);
+ }
+
+ // section is empty
+ continue;
+ }
+
+ // commit memory block and copy data from dll
+ dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
+ section->SizeOfRawData,
+ MEM_COMMIT,
+ PAGE_READWRITE);
+ memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
+ section->Misc.PhysicalAddress = (DWORD)dest;
+ }
+}
+
+// Protection flags for memory pages (Executable, Readable, Writeable)
+static int ProtectionFlags[2][2][2] = {
+ {
+ // not executable
+ {PAGE_NOACCESS, PAGE_WRITECOPY},
+ {PAGE_READONLY, PAGE_READWRITE},
+ }, {
+ // executable
+ {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
+ {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
+ },
+};
+
+static void
+FinalizeSections(PMEMORYMODULE module)
+{
+ int i;
+ PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
+
+ // loop through all sections and change access flags
+ for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++)
+ {
+ DWORD protect, oldProtect, size;
+ int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
+ int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
+ int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
+
+ if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
+ {
+ // section is not needed any more and can safely be freed
+ VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT);
+ continue;
+ }
+
+ // determine protection flags based on characteristics
+ protect = ProtectionFlags[executable][readable][writeable];
+ if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED)
+ protect |= PAGE_NOCACHE;
+
+ // determine size of region
+ size = section->SizeOfRawData;
+ if (size == 0)
+ {
+ if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA)
+ size = module->headers->OptionalHeader.SizeOfInitializedData;
+ else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)
+ size = module->headers->OptionalHeader.SizeOfUninitializedData;
+ }
+
+ if (size > 0)
+ {
+ // change memory access flags
+ if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0)
+#if DEBUG_OUTPUT
+ OutputLastError("Error protecting memory page")
+#endif
+ ;
+ }
+ }
+}
+
+static void
+PerformBaseRelocation(PMEMORYMODULE module, DWORD delta)
+{
+ DWORD i;
+ unsigned char *codeBase = module->codeBase;
+
+ PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
+ if (directory->Size > 0)
+ {
+ PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress);
+ for (; relocation->VirtualAddress > 0; )
+ {
+ unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress);
+ unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
+ for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++)
+ {
+ DWORD *patchAddrHL;
+ int type, offset;
+
+ // the upper 4 bits define the type of relocation
+ type = *relInfo >> 12;
+ // the lower 12 bits define the offset
+ offset = *relInfo & 0xfff;
+
+ switch (type)
+ {
+ case IMAGE_REL_BASED_ABSOLUTE:
+ // skip relocation
+ break;
+
+ case IMAGE_REL_BASED_HIGHLOW:
+ // change complete 32 bit address
+ patchAddrHL = (DWORD *)(dest + offset);
+ *patchAddrHL += delta;
+ break;
+
+ default:
+ //printf("Unknown relocation: %d\n", type);
+ break;
+ }
+ }
+
+ // advance to next relocation block
+ relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock);
+ }
+ }
+}
+
+static int
+BuildImportTable(PMEMORYMODULE module)
+{
+ int result=1;
+ unsigned char *codeBase = module->codeBase;
+
+ PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
+ if (directory->Size > 0)
+ {
+ PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress);
+ for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++)
+ {
+ DWORD *thunkRef, *funcRef;
+ HMODULE handle;
+
+ handle = MyLoadLibrary(codeBase + importDesc->Name);
+ if (handle == INVALID_HANDLE_VALUE)
+ {
+ //LastError should already be set
+#if DEBUG_OUTPUT
+ OutputLastError("Can't load library");
+#endif
+ result = 0;
+ break;
+ }
+
+ module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
+ if (module->modules == NULL)
+ {
+ SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+ result = 0;
+ break;
+ }
+
+ module->modules[module->numModules++] = handle;
+ if (importDesc->OriginalFirstThunk)
+ {
+ thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk);
+ funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+ } else {
+ // no hint table
+ thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+ funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
+ }
+ for (; *thunkRef; thunkRef++, funcRef++)
+ {
+ if IMAGE_SNAP_BY_ORDINAL(*thunkRef) {
+ *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
+ } else {
+ PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef);
+ *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name);
+ }
+ if (*funcRef == 0)
+ {
+ SetLastError(ERROR_PROC_NOT_FOUND);
+ result = 0;
+ break;
+ }
+ }
+
+ if (!result)
+ break;
+ }
+ }
+
+ return result;
+}
+
+/*
+ MemoryLoadLibrary - load a library AS MEMORY MODULE, or return
+ existing MEMORY MODULE with increased refcount.
+
+ This allows to load a library AGAIN as memory module which is
+ already loaded as HMODULE!
+
+*/
+HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data)
+{
+ PMEMORYMODULE result;
+ PIMAGE_DOS_HEADER dos_header;
+ PIMAGE_NT_HEADERS old_header;
+ unsigned char *code, *headers;
+ DWORD locationDelta;
+ DllEntryProc DllEntry;
+ BOOL successfull;
+ MEMORYMODULE *p = loaded;
+
+ while (p) {
+ // If already loaded, only increment the reference count
+ if (0 == stricmp(name, p->name)) {
+ p->refcount++;
+ return (HMODULE)p;
+ }
+ p = p->next;
+ }
+
+ /* Do NOT check for GetModuleHandle here! */
+
+ dos_header = (PIMAGE_DOS_HEADER)data;
+ if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
+ {
+ SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+ OutputDebugString("Not a valid executable file.\n");
+#endif
+ return NULL;
+ }
+
+ old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
+ if (old_header->Signature != IMAGE_NT_SIGNATURE)
+ {
+ SetLastError(ERROR_BAD_FORMAT);
+#if DEBUG_OUTPUT
+ OutputDebugString("No PE header found.\n");
+#endif
+ return NULL;
+ }
+
+ // reserve memory for image of library
+ code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
+ old_header->OptionalHeader.SizeOfImage,
+ MEM_RESERVE,
+ PAGE_READWRITE);
+
+ if (code == NULL)
+ // try to allocate memory at arbitrary position
+ code = (unsigned char *)VirtualAlloc(NULL,
+ old_header->OptionalHeader.SizeOfImage,
+ MEM_RESERVE,
+ PAGE_READWRITE);
+
+ if (code == NULL)
+ {
+ SetLastError(ERROR_NOT_ENOUGH_MEMORY);
+#if DEBUG_OUTPUT
+ OutputLastError("Can't reserve memory");
+#endif
+ return NULL;
+ }
+
+ result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
+ result->codeBase = code;
+ result->numModules = 0;
+ result->modules = NULL;
+ result->initialized = 0;
+ result->next = result->prev = NULL;
+ result->refcount = 1;
+ result->name = strdup(name);
+ result->name_table = NULL;
+
+ // XXX: is it correct to commit the complete memory region at once?
+ // calling DllEntry raises an exception if we don't...
+ VirtualAlloc(code,
+ old_header->OptionalHeader.SizeOfImage,
+ MEM_COMMIT,
+ PAGE_READWRITE);
+
+ // commit memory for headers
+ headers = (unsigned char *)VirtualAlloc(code,
+ old_header->OptionalHeader.SizeOfHeaders,
+ MEM_COMMIT,
+ PAGE_READWRITE);
+
+ // copy PE header to code
+ memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
+ result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
+
+ // update position
+ result->headers->OptionalHeader.ImageBase = (DWORD)code;
+
+ // copy sections from DLL file block to new memory location
+ CopySections(data, old_header, result);
+
+ // adjust base address of imported data
+ locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase);
+ if (locationDelta != 0)
+ PerformBaseRelocation(result, locationDelta);
+
+ // load required dlls and adjust function table of imports
+ if (!BuildImportTable(result))
+ goto error;
+
+ // mark memory pages depending on section headers and release
+ // sections that are marked as "discardable"
+ FinalizeSections(result);
+
+ // get entry point of loaded library
+ if (result->headers->OptionalHeader.AddressOfEntryPoint != 0)
+ {
+ DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint);
+ if (DllEntry == 0)
+ {
+ SetLastError(ERROR_BAD_FORMAT); /* XXX ? */
+#if DEBUG_OUTPUT
+ OutputDebugString("Library has no entry point.\n");
+#endif
+ goto error;
+ }
+
+ // notify library about attaching to process
+ successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
+ if (!successfull)
+ {
+#if DEBUG_OUTPUT
+ OutputDebugString("Can't attach library.\n");
+#endif
+ goto error;
+ }
+ result->initialized = 1;
+ }
+
+ _Register(name, result);
+
+ return (HMEMORYMODULE)result;
+
+error:
+ // cleanup
+ free(result->name);
+ MemoryFreeLibrary(result);
+ return NULL;
+}
+
+int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2)
+{
+ return stricmp(p1->name, p2->name);
+}
+
+int _find(const char **name, const struct NAME_TABLE *p)
+{
+ return stricmp(*name, p->name);
+}
+
+struct NAME_TABLE *GetNameTable(PMEMORYMODULE module)
+{
+ unsigned char *codeBase;
+ PIMAGE_EXPORT_DIRECTORY exports;
+ PIMAGE_DATA_DIRECTORY directory;
+ DWORD i, *nameRef;
+ WORD *ordinal;
+ struct NAME_TABLE *p, *ptab;
+
+ if (module->name_table)
+ return module->name_table;
+
+ codeBase = module->codeBase;
+ directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+ exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+
+ nameRef = (DWORD *)(codeBase + exports->AddressOfNames);
+ ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals);
+
+ p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE)
+ * exports->NumberOfNames);
+ if (p == NULL)
+ return NULL;
+ ptab = p;
+ for (i=0; iNumberOfNames; ++i) {
+ p->name = (char *)(codeBase + *nameRef++);
+ p->ordinal = *ordinal++;
+ ++p;
+ }
+ qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare);
+ return ptab;
+}
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
+{
+ unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
+ int idx=-1;
+ PIMAGE_EXPORT_DIRECTORY exports;
+ PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
+
+ if (directory->Size == 0)
+ // no export table found
+ return NULL;
+
+ exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
+ if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0)
+ // DLL doesn't export anything
+ return NULL;
+
+ if (HIWORD(name)) {
+ struct NAME_TABLE *ptab;
+ struct NAME_TABLE *found;
+ ptab = GetNameTable((PMEMORYMODULE)module);
+ if (ptab == NULL)
+ // some failure
+ return NULL;
+ found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find);
+ if (found == NULL)
+ // exported symbol not found
+ return NULL;
+
+ idx = found->ordinal;
+ }
+ else
+ idx = LOWORD(name) - exports->Base;
+
+ if ((DWORD)idx > exports->NumberOfFunctions)
+ // name <-> ordinal number don't match
+ return NULL;
+
+ // AddressOfFunctions contains the RVAs to the "real" functions
+ return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4)));
+}
+
+void MemoryFreeLibrary(HMEMORYMODULE mod)
+{
+ int i;
+ PMEMORYMODULE module = (PMEMORYMODULE)mod;
+
+ if (module != NULL)
+ {
+ if (module->initialized != 0)
+ {
+ // notify library about detaching from process
+ DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
+ (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
+ module->initialized = 0;
+ }
+
+ if (module->modules != NULL)
+ {
+ // free previously opened libraries
+ for (i=0; inumModules; i++)
+ if (module->modules[i] != INVALID_HANDLE_VALUE)
+ MyFreeLibrary(module->modules[i]);
+
+ free(module->modules);
+ }
+
+ if (module->codeBase != NULL)
+ // release memory of library
+ VirtualFree(module->codeBase, 0, MEM_RELEASE);
+
+ if (module->name_table != NULL)
+ free(module->name_table);
+
+ HeapFree(GetProcessHeap(), 0, module);
+ }
+}
diff --git a/setup/installer/windows/MemoryModule.h b/setup/installer/windows/MemoryModule.h
new file mode 100644
index 0000000000..601d4c50df
--- /dev/null
+++ b/setup/installer/windows/MemoryModule.h
@@ -0,0 +1,58 @@
+/*
+ * Memory DLL loading code
+ * Version 0.0.2
+ *
+ * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
+ * http://www.joachim-bauch.de
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is MemoryModule.h
+ *
+ * The Initial Developer of the Original Code is Joachim Bauch.
+ *
+ * Portions created by Joachim Bauch are Copyright (C) 2004-2005
+ * Joachim Bauch. All Rights Reserved.
+ *
+ */
+
+#ifndef __MEMORY_MODULE_HEADER
+#define __MEMORY_MODULE_HEADER
+
+#include
+
+typedef void *HMEMORYMODULE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *(*FINDPROC)();
+
+extern FINDPROC findproc;
+extern void *findproc_data;
+
+HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
+
+FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
+
+void MemoryFreeLibrary(HMEMORYMODULE);
+
+BOOL MyFreeLibrary(HMODULE hModule);
+HMODULE MyLoadLibrary(char *lpFileName);
+FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
+HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // __MEMORY_MODULE_HEADER
diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py
index 7fb60968e7..35840c8de8 100644
--- a/setup/installer/windows/freeze.py
+++ b/setup/installer/windows/freeze.py
@@ -71,11 +71,13 @@ class Win32Freeze(Command, WixMixIn):
self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
self.py_ver = ''.join(map(str, sys.version_info[:2]))
self.lib_dir = self.j(self.base, 'Lib')
- self.pydlib = self.j(self.base, 'pydlib')
self.pylib = self.j(self.base, 'pylib.zip')
+ self.dll_dir = self.j(self.base, 'DLLs')
+ self.plugins_dir = os.path.join(self.base, 'plugins')
self.initbase()
self.build_launchers()
+ self.add_plugins()
self.freeze()
self.embed_manifests()
self.install_site_py()
@@ -87,18 +89,20 @@ class Win32Freeze(Command, WixMixIn):
shutil.rmtree(self.base)
os.makedirs(self.base)
- def freeze(self):
- shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
-
+ def add_plugins(self):
self.info('Adding plugins...')
- tgt = os.path.join(self.base, 'plugins')
- if not os.path.exists(tgt):
- os.mkdir(tgt)
+ tgt = self.plugins_dir
+ if os.path.exists(tgt):
+ shutil.rmtree(tgt)
+ os.mkdir(tgt)
base = self.j(self.SRC, 'calibre', 'plugins')
for pat in ('*.pyd', '*.manifest'):
for f in glob.glob(self.j(base, pat)):
shutil.copy2(f, tgt)
+ def freeze(self):
+ shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
+
self.info('Adding resources...')
tgt = self.j(self.base, 'resources')
if os.path.exists(tgt):
@@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn):
shutil.copytree(self.j(self.src_root, 'resources'), tgt)
self.info('Adding Qt and python...')
- self.dll_dir = self.j(self.base, 'DLLs')
shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
@@ -318,8 +321,8 @@ class Win32Freeze(Command, WixMixIn):
if not os.path.exists(self.obj_dir):
os.makedirs(self.obj_dir)
base = self.j(self.src_root, 'setup', 'installer', 'windows')
- sources = [self.j(base, x) for x in ['util.c']]
- headers = [self.j(base, x) for x in ['util.h']]
+ sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
+ headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
@@ -371,43 +374,49 @@ class Win32Freeze(Command, WixMixIn):
def archive_lib_dir(self):
self.info('Putting all python code into a zip file for performance')
- if os.path.exists(self.pydlib):
- shutil.rmtree(self.pydlib)
- os.makedirs(self.pydlib)
self.zf_timestamp = time.localtime(time.time())[:6]
self.zf_names = set()
with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
+ # Add the .pyds from python and calibre to the zip file
+ for x in (self.plugins_dir, self.dll_dir):
+ for pyd in os.listdir(x):
+ if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
+ # sqlite_custom has to be a file for
+ # sqlite_load_extension to work
+ self.add_to_zipfile(zf, pyd, x)
+ os.remove(self.j(x, pyd))
+
+ # Add everything in Lib except site-packages to the zip file
for x in os.listdir(self.lib_dir):
if x == 'site-packages':
continue
self.add_to_zipfile(zf, x, self.lib_dir)
sp = self.j(self.lib_dir, 'site-packages')
- handled = set(['site.pyo'])
- for pth in ('PIL.pth', 'pywin32.pth'):
- handled.add(pth)
- shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth))
- for d in self.get_pth_dirs(self.j(sp, pth)):
- shutil.copytree(d, self.j(self.pydlib, self.b(d)), True)
- handled.add(self.b(d))
+ # Special handling for PIL and pywin32
+ handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
+ self.add_to_zipfile(zf, 'PIL', sp)
+ base = self.j(sp, 'win32', 'lib')
+ for x in os.listdir(base):
+ if os.path.splitext(x)[1] not in ('.exe',):
+ self.add_to_zipfile(zf, x, base)
+ base = self.d(base)
+ for x in os.listdir(base):
+ if not os.path.isdir(self.j(base, x)):
+ if os.path.splitext(x)[1] not in ('.exe',):
+ self.add_to_zipfile(zf, x, base)
handled.add('easy-install.pth')
for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
handled.add(self.b(d))
- zip_safe = self.is_zip_safe(d)
for x in os.listdir(d):
if x == 'EGG-INFO':
continue
- if zip_safe:
- self.add_to_zipfile(zf, x, d)
- else:
- absp = self.j(d, x)
- dest = self.j(self.pydlib, x)
- if os.path.isdir(absp):
- shutil.copytree(absp, dest, True)
- else:
- shutil.copy2(absp, dest)
+ self.add_to_zipfile(zf, x, d)
+ # The rest of site-packages
+ # We dont want the site.py from site-packages
+ handled.add('site.pyo')
for x in os.listdir(sp):
if x in handled or x.endswith('.egg-info'):
continue
@@ -415,33 +424,18 @@ class Win32Freeze(Command, WixMixIn):
if os.path.isdir(absp):
if not os.listdir(absp):
continue
- if self.is_zip_safe(absp):
- self.add_to_zipfile(zf, x, sp)
- else:
- shutil.copytree(absp, self.j(self.pydlib, x), True)
+ self.add_to_zipfile(zf, x, sp)
else:
- if x.endswith('.pyd'):
- shutil.copy2(absp, self.j(self.pydlib, x))
- else:
- self.add_to_zipfile(zf, x, sp)
+ self.add_to_zipfile(zf, x, sp)
shutil.rmtree(self.lib_dir)
- def is_zip_safe(self, path):
- for f in walk(path):
- ext = os.path.splitext(f)[1].lower()
- if ext in ('.pyd', '.dll', '.exe'):
- return False
- return True
-
def get_pth_dirs(self, pth):
base = os.path.dirname(pth)
for line in open(pth).readlines():
line = line.strip()
if not line or line.startswith('#') or line.startswith('import'):
continue
- if line == 'win32\\lib':
- continue
candidate = self.j(base, line)
if os.path.exists(candidate):
yield candidate
@@ -463,10 +457,10 @@ class Win32Freeze(Command, WixMixIn):
self.add_to_zipfile(zf, name + os.sep + x, base)
else:
ext = os.path.splitext(name)[1].lower()
- if ext in ('.pyd', '.dll', '.exe'):
+ if ext in ('.dll',):
raise ValueError('Cannot add %r to zipfile'%abspath)
zinfo.external_attr = 0600 << 16
- if ext in ('.py', '.pyc', '.pyo'):
+ if ext in ('.py', '.pyc', '.pyo', '.pyd'):
with open(abspath, 'rb') as f:
zf.writestr(zinfo, f.read())
diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py
index 5610ff197e..33f2e63585 100644
--- a/setup/installer/windows/site.py
+++ b/setup/installer/windows/site.py
@@ -1,12 +1,72 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import sys, os, linecache
+import sys
+import os
+import zipimport
+import _memimporter
+
+DEBUG_ZIPIMPORT = False
+
+class ZipExtensionImporter(zipimport.zipimporter):
+ '''
+ Taken, with thanks, from the py2exe source code
+ '''
+
+ def __init__(self, *args, **kwargs):
+ zipimport.zipimporter.__init__(self, *args, **kwargs)
+ # We know there are no dlls in the zip file, so dont set findproc
+ # (performance optimization)
+ #_memimporter.set_find_proc(self.locate_dll_image)
+
+ def find_module(self, fullname, path=None):
+ result = zipimport.zipimporter.find_module(self, fullname, path)
+ if result:
+ return result
+ fullname = fullname.replace(".", "\\")
+ if (fullname + '.pyd') in self._files:
+ return self
+ return None
+
+ def locate_dll_image(self, name):
+ # A callback function for_memimporter.import_module. Tries to
+ # locate additional dlls. Returns the image as Python string,
+ # or None if not found.
+ if name in self._files:
+ return self.get_data(name)
+ return None
+
+ def load_module(self, fullname):
+ if sys.modules.has_key(fullname):
+ mod = sys.modules[fullname]
+ if DEBUG_ZIPIMPORT:
+ sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
+ return mod
+ try:
+ return zipimport.zipimporter.load_module(self, fullname)
+ except zipimport.ZipImportError:
+ pass
+ initname = "init" + fullname.split(".")[-1] # name of initfunction
+ filename = fullname.replace(".", "\\")
+ path = filename + '.pyd'
+ if path in self._files:
+ if DEBUG_ZIPIMPORT:
+ sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
+ code = self.get_data(path)
+ mod = _memimporter.import_module(code, initname, fullname, path)
+ mod.__file__ = "%s\\%s" % (self.archive, path)
+ mod.__loader__ = self
+ if DEBUG_ZIPIMPORT:
+ sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
+ return mod
+ raise zipimport.ZipImportError, "can't find module %s" % fullname
+
+ def __repr__(self):
+ return "<%s object %r>" % (self.__class__.__name__, self.archive)
def abs__file__():
@@ -42,42 +102,6 @@ def makepath(*paths):
dir = os.path.abspath(os.path.join(*paths))
return dir, os.path.normcase(dir)
-def addpackage(sitedir, name):
- """Process a .pth file within the site-packages directory:
- For each line in the file, either combine it with sitedir to a path,
- or execute it if it starts with 'import '.
- """
- fullname = os.path.join(sitedir, name)
- try:
- f = open(fullname, "rU")
- except IOError:
- return
- with f:
- for line in f:
- if line.startswith("#"):
- continue
- if line.startswith(("import ", "import\t")):
- exec line
- continue
- line = line.rstrip()
- dir, dircase = makepath(sitedir, line)
- if os.path.exists(dir):
- sys.path.append(dir)
-
-
-def addsitedir(sitedir):
- """Add 'sitedir' argument to sys.path if missing and handle .pth files in
- 'sitedir'"""
- sitedir, sitedircase = makepath(sitedir)
- try:
- names = os.listdir(sitedir)
- except os.error:
- return
- dotpth = os.extsep + "pth"
- names = [name for name in names if name.endswith(dotpth)]
- for name in sorted(names):
- addpackage(sitedir, name)
-
def run_entry_point():
bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
sys.argv[0] = bname+'.exe'
@@ -89,6 +113,10 @@ def main():
sys.setdefaultencoding('utf-8')
aliasmbcs()
+ sys.path_hooks.insert(0, ZipExtensionImporter)
+ sys.path_importer_cache.clear()
+
+ import linecache
def fake_getline(filename, lineno, module_globals=None):
return ''
linecache.orig_getline = linecache.getline
@@ -96,10 +124,11 @@ def main():
abs__file__()
- addsitedir(os.path.join(sys.app_dir, 'pydlib'))
-
add_calibre_vars()
+ # Needed for pywintypes to be able to load its DLL
+ sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
+
return run_entry_point()
diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c
index 329e3bf8c3..4075d7e123 100644
--- a/setup/installer/windows/util.c
+++ b/setup/installer/windows/util.c
@@ -1,18 +1,130 @@
/*
* Copyright 2009 Kovid Goyal
+ * The memimporter code is taken from the py2exe project
*/
#include "util.h"
+
#include
#include
#include
+
static char GUI_APP = 0;
static char python_dll[] = PYDLL;
void set_gui_app(char yes) { GUI_APP = yes; }
char is_gui_app() { return GUI_APP; }
+
+// memimporter {{{
+
+#include "MemoryModule.h"
+
+static char **DLL_Py_PackageContext = NULL;
+static PyObject **DLL_ImportError = NULL;
+static char module_doc[] =
+"Importer which can load extension modules from memory";
+
+
+static void *memdup(void *ptr, Py_ssize_t size)
+{
+ void *p = malloc(size);
+ if (p == NULL)
+ return NULL;
+ memcpy(p, ptr, size);
+ return p;
+}
+
+/*
+ Be sure to detect errors in FindLibrary - undetected errors lead to
+ very strange behaviour.
+*/
+static void* FindLibrary(char *name, PyObject *callback)
+{
+ PyObject *result;
+ char *p;
+ Py_ssize_t size;
+
+ if (callback == NULL)
+ return NULL;
+ result = PyObject_CallFunction(callback, "s", name);
+ if (result == NULL) {
+ PyErr_Clear();
+ return NULL;
+ }
+ if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
+ PyErr_Clear();
+ Py_DECREF(result);
+ return NULL;
+ }
+ p = memdup(p, size);
+ Py_DECREF(result);
+ return p;
+}
+
+static PyObject *set_find_proc(PyObject *self, PyObject *args)
+{
+ PyObject *callback = NULL;
+ if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
+ return NULL;
+ Py_DECREF((PyObject *)findproc_data);
+ Py_INCREF(callback);
+ findproc_data = (void *)callback;
+ return Py_BuildValue("i", 1);
+}
+
+static PyObject *
+import_module(PyObject *self, PyObject *args)
+{
+ char *data;
+ int size;
+ char *initfuncname;
+ char *modname;
+ char *pathname;
+ HMEMORYMODULE hmem;
+ FARPROC do_init;
+
+ char *oldcontext;
+
+ /* code, initfuncname, fqmodulename, path */
+ if (!PyArg_ParseTuple(args, "s#sss:import_module",
+ &data, &size,
+ &initfuncname, &modname, &pathname))
+ return NULL;
+ hmem = MemoryLoadLibrary(pathname, data);
+ if (!hmem) {
+ PyErr_Format(*DLL_ImportError,
+ "MemoryLoadLibrary() failed loading %s", pathname);
+ return NULL;
+ }
+ do_init = MemoryGetProcAddress(hmem, initfuncname);
+ if (!do_init) {
+ MemoryFreeLibrary(hmem);
+ PyErr_Format(*DLL_ImportError,
+ "Could not find function %s in memory loaded pyd", initfuncname);
+ return NULL;
+ }
+
+ oldcontext = *DLL_Py_PackageContext;
+ *DLL_Py_PackageContext = modname;
+ do_init();
+ *DLL_Py_PackageContext = oldcontext;
+ if (PyErr_Occurred())
+ return NULL;
+ /* Retrieve from sys.modules */
+ return PyImport_ImportModule(modname);
+}
+
+static PyMethodDef methods[] = {
+ { "import_module", import_module, METH_VARARGS,
+ "import_module(code, initfunc, dllname[, finder]) -> module" },
+ { "set_find_proc", set_find_proc, METH_VARARGS },
+ { NULL, NULL }, /* Sentinel */
+};
+
+// }}}
+
static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
wchar_t *buf, *cbuf;
buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
@@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
char *dummy_argv[1] = {""};
buf = (char*)calloc(MAX_PATH, sizeof(char));
- path = (char*)calloc(3*MAX_PATH, sizeof(char));
+ path = (char*)calloc(MAX_PATH, sizeof(char));
if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1));
sz = GetModuleFileNameA(NULL, buf, MAX_PATH);
@@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
buf[strlen(buf)-1] = '\0';
_snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
- _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs",
- buf, buf, buf);
+ _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
free(buf);
@@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
//*flag = 1;
-
+ DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
+ if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
+ DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
+ if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
Py_SetProgramName(program_name);
Py_SetPythonHome(python_home);
@@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
PyList_SetItem(argv, i, v);
}
PySys_SetObject("argv", argv);
+
+ findproc = FindLibrary;
+ Py_InitModule3("_memimporter", methods, module_doc);
+
}
diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml
index 0a85b6fb81..3ebe0882e0 100644
--- a/setup/installer/windows/wix-template.xml
+++ b/setup/installer/windows/wix-template.xml
@@ -164,10 +164,6 @@
-
-
-
-
diff --git a/src/calibre/debug.py b/src/calibre/debug.py
index 8d65c37bbf..79110d9585 100644
--- a/src/calibre/debug.py
+++ b/src/calibre/debug.py
@@ -53,6 +53,8 @@ Run an embedded python interpreter.
default=False, action='store_true')
parser.add_option('-m', '--inspect-mobi',
help='Inspect the MOBI file at the specified path', default=None)
+ parser.add_option('--test-build', help='Test binary modules in build',
+ action='store_true', default=False)
return parser
@@ -232,6 +234,9 @@ def main(args=sys.argv):
elif opts.inspect_mobi is not None:
from calibre.ebooks.mobi.debug import inspect_mobi
inspect_mobi(opts.inspect_mobi)
+ elif opts.test_build:
+ from calibre.test_build import test
+ test()
else:
from calibre import ipython
ipython()
diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py
new file mode 100644
index 0000000000..0610f01805
--- /dev/null
+++ b/src/calibre/test_build.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+ print_function)
+from future_builtins import map
+
+__license__ = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal '
+__docformat__ = 'restructuredtext en'
+
+'''
+Test a binary calibre build to ensure that all needed binary images/libraries have loaded.
+'''
+
+import cStringIO
+from calibre.constants import plugins, iswindows
+
+def test_plugins():
+ for name in plugins:
+ mod, err = plugins[name]
+ if err or not mod:
+ raise RuntimeError('Plugin %s failed to load with error: %s' %
+ (name, err))
+ print (mod, 'loaded')
+
+def test_lxml():
+ from lxml import etree
+ raw = ''
+ root = etree.fromstring(raw)
+ if etree.tostring(root) == raw:
+ print ('lxml OK!')
+ else:
+ raise RuntimeError('lxml failed')
+
+def test_fontconfig():
+ from calibre.utils.fonts import fontconfig
+ families = fontconfig.find_font_families()
+ num = len(families)
+ if num < 10:
+ raise RuntimeError('Fontconfig found only %d font families'%num)
+ print ('Fontconfig OK! (%d families)'%num)
+
+def test_winutil():
+ from calibre.devices.scanner import win_pnp_drives
+ matches = win_pnp_drives.scanner()
+ if len(matches) < 1:
+ raise RuntimeError('win_pnp_drives returned no drives')
+ print ('win_pnp_drives OK!')
+
+def test_win32():
+ from calibre.utils.winshell import desktop
+ d = desktop()
+ if not d:
+ raise RuntimeError('winshell failed')
+ print ('winshell OK! (%s is the desktop)'%d)
+
+def test_sqlite():
+ import sqlite3
+ conn = sqlite3.connect(':memory:')
+ from calibre.library.sqlite import load_c_extensions
+ if not load_c_extensions(conn, True):
+ raise RuntimeError('Failed to load sqlite extension')
+ print ('sqlite OK!')
+
+def test_qt():
+ from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager)
+ fmts = set(map(unicode, QImageReader.supportedImageFormats()))
+ if 'jpg' not in fmts or 'png' not in fmts:
+ raise RuntimeError(
+ "Qt doesn't seem to be able to load its image plugins")
+ QWebView, QDialog
+ na = QNetworkAccessManager()
+ if not hasattr(na, 'sslErrors'):
+ raise RuntimeError('Qt not compiled with openssl')
+ print ('Qt OK!')
+
+def test_imaging():
+ from calibre.utils.magick.draw import create_canvas, Image
+ im = create_canvas(20, 20, '#ffffff')
+ jpg = im.export('jpg')
+ Image().load(jpg)
+ im.export('png')
+ print ('ImageMagick OK!')
+ from PIL import Image
+ i = Image.open(cStringIO.StringIO(jpg))
+ if i.size != (20, 20):
+ raise RuntimeError('PIL choked!')
+ print ('PIL OK!')
+
+def test():
+ test_plugins()
+ test_lxml()
+ test_fontconfig()
+ test_sqlite()
+ if iswindows:
+ test_winutil()
+ test_win32()
+ test_qt()
+ test_imaging()
+
+if __name__ == '__main__':
+ test()
+
From 51c4fe162c3ca2c62507ce21c56280ee76818480 Mon Sep 17 00:00:00 2001
From: Kovid Goyal
Date: Sun, 5 Jun 2011 23:01:32 -0600
Subject: [PATCH 31/31] No longer ship libusb in the windows binary
---
setup/installer/windows/freeze.py | 13 ++++---------
setup/installer/windows/notes.rst | 4 +++-
2 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py
index 35840c8de8..0fe494e831 100644
--- a/setup/installer/windows/freeze.py
+++ b/setup/installer/windows/freeze.py
@@ -16,7 +16,6 @@ from setup.installer.windows.wix import WixMixIn
OPENSSL_DIR = r'Q:\openssl'
QT_DIR = 'Q:\\Qt\\4.7.3'
QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUSB_DIR = 'C:\\libusb'
LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
SW = r'C:\cygwin\home\kovid\sw'
IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
@@ -96,9 +95,10 @@ class Win32Freeze(Command, WixMixIn):
shutil.rmtree(tgt)
os.mkdir(tgt)
base = self.j(self.SRC, 'calibre', 'plugins')
- for pat in ('*.pyd', '*.manifest'):
- for f in glob.glob(self.j(base, pat)):
- shutil.copy2(f, tgt)
+ for f in glob.glob(self.j(base, '*.pyd')):
+ # We dont want the manifests as the manifest in the exe will be
+ # used instead
+ shutil.copy2(f, tgt)
def freeze(self):
shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
@@ -200,11 +200,6 @@ class Win32Freeze(Command, WixMixIn):
print
print 'Adding third party dependencies'
- tdir = os.path.join(self.base, 'driver')
- os.makedirs(tdir)
- for pat in ('*.dll', '*.sys', '*.cat', '*.inf'):
- for f in glob.glob(os.path.join(LIBUSB_DIR, pat)):
- shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
print '\tAdding unrar'
shutil.copyfile(LIBUNRAR,
os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))
diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst
index 11b5bccf79..0bb8b7b15b 100644
--- a/setup/installer/windows/notes.rst
+++ b/setup/installer/windows/notes.rst
@@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open
Now, run configure and make::
- configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
+-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly
+
+ configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
SIP
-----