From 14dc0006d0c6a86e7d680b6284444d2cdb279741 Mon Sep 17 00:00:00 2001 From: koehler Date: Sun, 5 Jun 2011 14:15:09 +0200 Subject: [PATCH 01/35] cssutils has recently renamed CSSValueList to PropertyValue. Fix resulting breakage in src/calibre/ebooks/oeb/stylizer.py --- src/calibre/ebooks/oeb/stylizer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index dc73862022..ed1d2a576c 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -13,7 +13,11 @@ from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration, - CSSValueList, CSSFontFaceRule, cssproperties) + CSSFontFaceRule, cssproperties) +try: + from cssutils.css import CSSValueList +except ImportError: + from cssutils.css import PropertyValue as CSSValueList from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError From 4018fb48a5eaf44da96cccf5c145d37d0a2f0d41 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 08:09:52 -0600 Subject: [PATCH 02/35] Heise Online by schuster and improved express.de and max planck --- recipes/express_de.recipe | 10 +++---- recipes/heise_online.recipe | 52 +++++++++++++++++++++++++++++++++++++ recipes/max_planck.recipe | 8 +++--- 3 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 recipes/heise_online.recipe diff --git a/recipes/express_de.recipe b/recipes/express_de.recipe index 255538b08e..10595b9d92 100644 --- a/recipes/express_de.recipe +++ b/recipes/express_de.recipe @@ -1,5 +1,4 @@ from calibre.web.feeds.news import BasicNewsRecipe - class AdvancedUserRecipe1303841067(BasicNewsRecipe): title = u'Express.de' @@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): extra_css = ''' h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;} h1{ font-family:Arial,Helvetica,sans-serif; font-size:x-large; font-weight:bold;} - ''' remove_javascript = True remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})] @@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): dict(id='Logo'), dict(id='MainLinkSpacer'), dict(id='MainLinks'), + dict(id='ContainerPfad'), #neu dict(title='Diese Seite Bookmarken'), dict(name='span'), @@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): dict(name='div', attrs={'class':'HeaderSearch'}), dict(name='div', attrs={'class':'sbutton'}), dict(name='div', attrs={'class':'active'}), - + dict(name='div', attrs={'class':'MoreNews'}), #neu + dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu ] @@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'), (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'), (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'), + ] - - -] diff --git a/recipes/heise_online.recipe b/recipes/heise_online.recipe new file mode 100644 index 0000000000..f83ff8126b --- /dev/null +++ b/recipes/heise_online.recipe @@ -0,0 +1,52 @@ +from calibre.web.feeds.news import BasicNewsRecipe +class AdvancedUserRecipe(BasicNewsRecipe): + + title = 'Heise-online' + description = 'News vom Heise-Verlag' + __author__ = 'schuster' + use_embedded_content = False + language = 'de' + oldest_article = 2 + max_articles_per_feed = 35 + rescale_images = True + remove_empty_feeds = True + timeout = 5 + no_stylesheets = True + + + remove_tags_after = dict(name ='p', attrs={'class':'editor'}) + remove_tags = [dict(id='navi_top_container'), + dict(id='navi_bottom'), + dict(id='mitte_rechts'), + dict(id='navigation'), + dict(id='subnavi'), + dict(id='social_bookmarks'), + dict(id='permalink'), + dict(id='content_foren'), + dict(id='seiten_navi'), + dict(id='adbottom'), + dict(id='sitemap')] + + feeds = [ + ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'), + ('Auto', 'http://www.heise.de/autos/rss/news.rdf'), + ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'), + ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'), + ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'), + ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'), + ('Open ', 'http://www.heise.de/open/news/news-atom.xml'), + ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'), + ('Security ', 'http://www.heise.de/security/news/news-atom.xml'), + ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'), + ('iX', 'http://www.heise.de/ix/news/news.rdf'), + ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'), + ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'), + ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'), + ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'), + ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'), + ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf') +] + + def print_version(self, url): + return url + '?view=print' + diff --git a/recipes/max_planck.recipe b/recipes/max_planck.recipe index e9bf62008a..cf778a7374 100644 --- a/recipes/max_planck.recipe +++ b/recipes/max_planck.recipe @@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): title = u'Max-Planck-Inst.' __author__ = 'schuster' - remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}), - dict(id=['ie_clearing', 'col2', 'col2_content']), - dict(name=['script', 'noscript', 'style'])] oldest_article = 30 max_articles_per_feed = 100 no_stylesheets = True @@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): language = 'de' remove_javascript = True + remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}), + dict(id=['skiplinks'])] + + + def print_version(self, url): split_url = url.split("/") print_url = 'http://www.mpg.de/print/' + split_url[3] From 542f468465e96ac58275964703c8cf4aa914900a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 08:12:16 -0600 Subject: [PATCH 03/35] Driver for Samsung Galaxy S2 --- src/calibre/devices/android/driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index b557ac3526..554ea3c698 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -53,6 +53,7 @@ class ANDROID(USBMS): 0x681c : [0x0222, 0x0224, 0x0400], 0x6640 : [0x0100], 0x685e : [0x0400], + 0x6860 : [0x0400], 0x6877 : [0x0400], }, From 37be144dfec849a09b2136f379e036c2b86c0b64 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 08:16:46 -0600 Subject: [PATCH 04/35] Amazon metadata plugin: Fix parsing of published date from amazon.de when it has februar in it --- src/calibre/ebooks/metadata/sources/amazon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 7da37ce5af..6220f29020 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{ months = { 'de': { 1 : ['jän'], + 2 : ['februar'], 3 : ['märz'], 5 : ['mai'], 6 : ['juni'], From 06c76ae36dc5ed4e89d54c87b35daf230541a85b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 11:45:45 -0600 Subject: [PATCH 05/35] Brigitte.de by schuster --- recipes/brigitte_de.recipe | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 recipes/brigitte_de.recipe diff --git a/recipes/brigitte_de.recipe b/recipes/brigitte_de.recipe new file mode 100644 index 0000000000..860d5176ac --- /dev/null +++ b/recipes/brigitte_de.recipe @@ -0,0 +1,36 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe(BasicNewsRecipe): + + title = u'Brigitte.de' + __author__ = 'schuster' + oldest_article = 14 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + language = 'de' + remove_javascript = True + remove_empty_feeds = True + timeout = 10 + cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg' + masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg' + + + remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}), + dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']), + dict(name=['hjtrs', 'kud'])] + + feeds = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'), + (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'), + (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'), + (u'Figur', u'http://www.brigitte.de/figur/feed.rss'), + (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'), + (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'), + (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'), + (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'), + (u'Reise', u'http://www.brigitte.de/reise/feed.rss'), + (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'), + (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'), + (u'Job', u'http://www.brigitte.de/job/feed.rss'), + (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'), +] From 70e0cd6ba43b379183e1b2f6a02196c55c5a0c66 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 13:38:50 -0600 Subject: [PATCH 06/35] When writing files to zipfile, reset timestamp if it doesn't fit in 1980's vintage storage structures --- src/calibre/utils/zipfile.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index fa15bff4b4..394b667bbf 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -340,7 +340,14 @@ class ZipInfo (object): """Return the per-file header as a string.""" dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + # Added by Kovid to reset timestamp to default if it overflows the DOS + # limits + if dosdate > 0xffff or dosdate < 0: + dosdate = 0 dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + if dostime > 0xffff or dostime < 0: + dostime = 0 + if self.flag_bits & 0x08: # Set these to zero because we write them after the file data CRC = compress_size = file_size = 0 From 4a236943d04849622a810e731d3b8339e47553b9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 5 Jun 2011 18:13:47 -0400 Subject: [PATCH 07/35] Store: Update declined stores list. --- src/calibre/gui2/store/declined.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/declined.txt b/src/calibre/gui2/store/declined.txt index 2186303d4b..ada839662d 100644 --- a/src/calibre/gui2/store/declined.txt +++ b/src/calibre/gui2/store/declined.txt @@ -2,7 +2,8 @@ This is a list of stores that objected, declined or asked not to be included in the store integration. * Borders (http://www.borders.com/) -* WH Smith (http://www.whsmith.co.uk/) - Refused to permit signing up for the affiliate program +* Indigo (http://www.chapters.indigo.ca/) * Libraria Rizzoli (http://libreriarizzoli.corriere.it/). - No reply with two attempts over 2 weeks \ No newline at end of file + No reply with two attempts over 2 weeks +* WH Smith (http://www.whsmith.co.uk/) + Refused to permit signing up for the affiliate program \ No newline at end of file From c3cec681720b8f202c6b46fcfd15071767c5b6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Mon, 6 Jun 2011 00:19:06 +0200 Subject: [PATCH 08/35] Zixo Store, improved Nexto --- src/calibre/customize/builtins.py | 12 +++- src/calibre/gui2/store/nexto_plugin.py | 2 +- src/calibre/gui2/store/zixo_plugin.py | 80 ++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 src/calibre/gui2/store/zixo_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9ebec5e7e8..4c7e03e89d 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1417,6 +1417,15 @@ class StoreWoblinkStore(StoreBase): headquarters = 'PL' formats = ['EPUB'] +class StoreZixoStore(StoreBase): + name = 'Zixo' + author = u'Tomasz Długosz' + description = u'Księgarnia z ebookami oraz książkami audio' + actual_plugin = 'calibre.gui2.store.zixo_plugin:ZixoStore' + + headquarters = 'PL' + formats = ['PDF, ZIXO'] + plugins += [ StoreArchiveOrgStore, StoreAmazonKindleStore, @@ -1451,7 +1460,8 @@ plugins += [ StoreWeightlessBooksStore, StoreWHSmithUKStore, StoreWizardsTowerBooksStore, - StoreWoblinkStore + StoreWoblinkStore, + StoreZixoStore ] # }}} diff --git a/src/calibre/gui2/store/nexto_plugin.py b/src/calibre/gui2/store/nexto_plugin.py index fa152f958c..16004908df 100644 --- a/src/calibre/gui2/store/nexto_plugin.py +++ b/src/calibre/gui2/store/nexto_plugin.py @@ -71,7 +71,7 @@ class NextoStore(BasicStoreConfig, StorePlugin): author = '' with closing(br.open('http://www.nexto.pl/' + id.strip(), timeout=timeout/4)) as nf: idata = html.fromstring(nf.read()) - author = ''.join(idata.xpath('//div[@class="basic_data"]/p[1]/b/a/text()')) + author = ', '.join(idata.xpath('//div[@class="basic_data"]/p[1]/b/a/text()')) counter -= 1 diff --git a/src/calibre/gui2/store/zixo_plugin.py b/src/calibre/gui2/store/zixo_plugin.py new file mode 100644 index 0000000000..419b87137a --- /dev/null +++ b/src/calibre/gui2/store/zixo_plugin.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, Tomasz Długosz ' +__docformat__ = 'restructuredtext en' + +import re +import urllib +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class ZixoStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + + url = 'http://zixo.pl/e_ksiazki/start/' + + if external or self.config.get('open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://zixo.pl/wyszukiwarka/?search=' + urllib.quote(query.encode('utf-8')) + '&product_type=0' + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="productInline"]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//a[@class="productThumb"]/@href')) + if not id: + continue + + cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()')) + price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()')) + price = re.sub('\.', ',', price) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = 'http://zixo.pl' + id.strip() + s.drm = SearchResult.DRM_LOCKED + + yield s + + def get_details(self, search_result, timeout): + br = browser() + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + formats = ''.join(idata.xpath('//ul[@class="prop"]/li[3]/text()')) + formats = re.sub(r'\(.*\)', '', formats) + formats = re.sub('Zixo Reader', 'ZIXO', formats) + search_result.formats = formats + return True From 83895be2f813b39d0c09fe56aabb6313936edef9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 5 Jun 2011 19:43:22 -0400 Subject: [PATCH 09/35] Store: Fix B&N store to work with new B&N website layout. --- src/calibre/gui2/store/bn_plugin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index 62826e825d..eec8e1690d 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -47,26 +47,26 @@ class BNStore(BasicStoreConfig, StorePlugin): d.exec_() def search(self, query, max_results=10, timeout=60): - url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results - url += urllib.quote_plus(query) + query = query.replace(' ', '-') + url = 'http://www.barnesandnoble.com/s/%s?store=ebook&sze=%s' % (query, max_results) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'): + for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'): if counter <= 0: break - id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href')) + id = ''.join(data.xpath('.//div[contains(@class, "image")]/a/@href')) if not id: continue - cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src')) + cover_url = ''.join(data.xpath('.//div[contains(@class, "image")]//img/@src')) - title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()')) - author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()')) - price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()')) + title = ''.join(data.xpath('.//p[@class="title"]//span[@class="name"]/text()')) + author = ', '.join(data.xpath('.//ul[@class="contributors"]//li[position()>1]//a/text()')) + price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[@class="subtle"]/text()')) counter -= 1 @@ -74,9 +74,9 @@ class BNStore(BasicStoreConfig, StorePlugin): s.cover_url = cover_url s.title = title.strip() s.author = author.strip() - s.price = price + s.price = price.strip() s.detail_item = id.strip() s.drm = SearchResult.DRM_UNKNOWN s.formats = 'Nook' - + yield s From 4eb7ef507c1c69499dc5521f13e19760176b81de Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 17:44:12 -0600 Subject: [PATCH 10/35] ... --- src/calibre/utils/zipfile.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index 394b667bbf..868445d15a 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -148,6 +148,12 @@ def decode_arcname(name): name = name.decode('utf-8', 'replace') return name +# Added by Kovid to reset timestamp to default if it overflows the DOS +# limits +def fixtimevar(val): + if val < 0 or val > 0xffff: + val = 0 + return val def _check_zipfile(fp): try: @@ -340,13 +346,7 @@ class ZipInfo (object): """Return the per-file header as a string.""" dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - # Added by Kovid to reset timestamp to default if it overflows the DOS - # limits - if dosdate > 0xffff or dosdate < 0: - dosdate = 0 dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if dostime > 0xffff or dostime < 0: - dostime = 0 if self.flag_bits & 0x08: # Set these to zero because we write them after the file data @@ -372,7 +372,7 @@ class ZipInfo (object): filename, flag_bits = self._encodeFilenameFlags() header = struct.pack(structFileHeader, stringFileHeader, self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, + self.compress_type, fixtimevar(dostime), fixtimevar(dosdate), CRC, compress_size, file_size, len(filename), len(extra)) return header + filename + extra @@ -1328,8 +1328,8 @@ class ZipFile: for zinfo in self.filelist: # write central directory count = count + 1 dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + dosdate = fixtimevar((dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]) + dostime = fixtimevar(dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)) extra = [] if zinfo.file_size > ZIP64_LIMIT \ or zinfo.compress_size > ZIP64_LIMIT: From 907d96e1f344a99dd65b7ce8f92cdd8529bd7bb4 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 5 Jun 2011 19:44:58 -0400 Subject: [PATCH 11/35] ... --- src/calibre/gui2/store/bn_plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index eec8e1690d..94e498bb44 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -8,7 +8,6 @@ __docformat__ = 'restructuredtext en' import random import re -import urllib from contextlib import closing from lxml import html From 571cc551c2b82ebc7c519939b120323dfe957a7b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 17:48:28 -0600 Subject: [PATCH 12/35] ... --- src/calibre/gui2/preferences/toolbar.ui | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/toolbar.ui b/src/calibre/gui2/preferences/toolbar.ui index 0e601f74a2..51819b0df2 100644 --- a/src/calibre/gui2/preferences/toolbar.ui +++ b/src/calibre/gui2/preferences/toolbar.ui @@ -16,13 +16,28 @@ + + + 75 + true + + - Customize the actions in: + Choose the &toolbar to customize: + + + what + + + 75 + true + + QComboBox::AdjustToMinimumContentsLengthWithIcon From 796dc93450a9c3129fd3713198dd5e8a77508de5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 17:49:06 -0600 Subject: [PATCH 13/35] ... --- src/calibre/gui2/dialogs/tweak_epub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index 732d74b77d..e0be9fa1e9 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, shutil -from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED +from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog From a66fec40912f05c8c033e477434ce443de4b4799 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 5 Jun 2011 20:15:20 -0400 Subject: [PATCH 14/35] Store: Fix Amazon plugin to account for changes in Amazon website. --- src/calibre/gui2/store/amazon_plugin.py | 28 +++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index b70d03ad0a..a9ec17e694 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -131,16 +131,22 @@ class AmazonKindleStore(StorePlugin): # Amazon has two results pages. is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') - # Horizontal grid of books. + # Horizontal grid of books. Search "Paolo Bacigalupi" if is_shot: data_xpath = '//div[contains(@class, "result")]' - format_xpath = './/div[@class="productTitle"]/text()' + format_xpath = './/div[@class="productTitle"]//text()' + asin_xpath = './/div[@class="productTitle"]//a' cover_xpath = './/div[@class="productTitle"]//img/@src' - # Vertical list of books. + title_xpath = './/div[@class="productTitle"]/a//text()' + price_xpath = './/div[@class="newPrice"]/span/text()' + # Vertical list of books. Search "martin" else: - data_xpath = '//div[@class="productData"]' - format_xpath = './/span[@class="format"]/text()' - cover_xpath = '../div[@class="productImage"]/a/img/@src' + data_xpath = '//div[contains(@class, "results")]//div[contains(@class, "result")]' + format_xpath = './/span[@class="binding"]//text()' + asin_xpath = './/div[@class="image"]/a[1]' + cover_xpath = './/img[@class="productImage"]/@src' + title_xpath = './/a[@class="title"]/text()' + price_xpath = './/span[@class="price"]/text()' for data in doc.xpath(data_xpath): if counter <= 0: @@ -157,7 +163,7 @@ class AmazonKindleStore(StorePlugin): # We must have an asin otherwise we can't easily reference the # book later. asin_href = None - asin_a = data.xpath('.//div[@class="productTitle"]/a[1]') + asin_a = data.xpath(asin_xpath) if asin_a: asin_href = asin_a[0].get('href', '') m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) @@ -170,14 +176,14 @@ class AmazonKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath(title_xpath)) + price = ''.join(data.xpath(price_xpath)) if is_shot: author = format.split(' by ')[-1] else: - author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()')) - author = author.split(' by ')[-1] + author = ''.join(data.xpath('.//span[@class="ptBrand"]/text()')) + author = author.split('by ')[-1] counter -= 1 From a5d1d6b196d51febce03099e3bccd07528c4d2d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 18:54:05 -0600 Subject: [PATCH 15/35] ... --- src/calibre/manual/faq.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 99c53e5a37..c1aa4e5614 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -562,6 +562,16 @@ You have two choices: 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_. 2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it. +Why doesn't |app| have an automatic update? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For many reasons: + + * *There is no need to update every week*. If you are happy with how |app| works turn off the update notification and be on your merry way. Check back to see if you want to update once a year or so. + * Pre downloading the updates for all users in the background would mean require about 80TB of bandwidth *every week*. That costs thousands of dollars a month. And |app| is currently growing at 300,000 new users every month. + * If I implement a dialog that downloads the update and launches it, instead of going to the website as it does now, that would save the most ardent |app| updater, *at most five clicks a week*. There are far higher priority things to do in |app| development. + * If you really, really hate downloading |app| every week but still want to be upto the latest, I encourage you to run from source, which makes updating trivial. Instructions are :ref:`here `. + How is |app| licensed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode `_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 `_. From 9ea27629c26db1cc2ee97160fe3ed0a546006420 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 19:17:28 -0600 Subject: [PATCH 16/35] Driver for the Notion Ink Adam --- src/calibre/customize/builtins.py | 9 +++++---- src/calibre/devices/misc.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9ebec5e7e8..33685ea254 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \ SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER from calibre.devices.sne.driver import SNE -from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \ - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \ - TREKSTOR, EEEREADER, NEXTBOOK +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM) from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK @@ -744,6 +744,7 @@ plugins += [ TREKSTOR, EEEREADER, NEXTBOOK, + ADAM, ITUNES, BOEYE_BEX, BOEYE_BDX, @@ -1231,7 +1232,7 @@ class StoreEpubBudStore(StoreBase): name = 'ePub Bud' description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.' actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore' - + drm_free_only = True headquarters = 'US' formats = ['EPUB'] diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 936faeb32d..2a6a76719d 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -255,6 +255,28 @@ class EEEREADER(USBMS): VENDOR_NAME = 'LINUX' WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET' +class ADAM(USBMS): + + name = 'Notion Ink Adam device interface' + gui_name = 'Adam' + + description = _('Communicate with the Adam tablet') + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + + # Ordered list of supported formats + FORMATS = ['epub', 'pdf', 'doc'] + + VENDOR_ID = [0x0955] + PRODUCT_ID = [0x7100] + BCD = [0x9999] + + EBOOK_DIR_MAIN = 'eBooks' + + VENDOR_NAME = 'NI' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM'] + SUPPORTS_SUB_DIRS = True + class NEXTBOOK(USBMS): name = 'Nextbook device interface' From b22d5ac5fb4501975c3c5bf4e9e0e003a56b1243 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 20:21:09 -0600 Subject: [PATCH 17/35] Windows build: Add code to load .pyd python extensions from a zip file. This allows many more files in the calibre installation to be zipped up, speeding up the installer. --- setup/installer/windows/MemoryModule.c | 689 +++++++++++++++++++++++ setup/installer/windows/MemoryModule.h | 58 ++ setup/installer/windows/freeze.py | 92 ++- setup/installer/windows/site.py | 109 ++-- setup/installer/windows/util.c | 126 ++++- setup/installer/windows/wix-template.xml | 4 - src/calibre/debug.py | 5 + src/calibre/test_build.py | 103 ++++ 8 files changed, 1089 insertions(+), 97 deletions(-) create mode 100644 setup/installer/windows/MemoryModule.c create mode 100644 setup/installer/windows/MemoryModule.h create mode 100644 src/calibre/test_build.py diff --git a/setup/installer/windows/MemoryModule.c b/setup/installer/windows/MemoryModule.c new file mode 100644 index 0000000000..253c8d7d9f --- /dev/null +++ b/setup/installer/windows/MemoryModule.c @@ -0,0 +1,689 @@ +/* + * Memory DLL loading code + * Version 0.0.2 with additions from Thomas Heller + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.c + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + * Portions Copyright (C) 2005 Thomas Heller. + * + */ + +// disable warnings about pointer <-> DWORD conversions +#pragma warning( disable : 4311 4312 ) + +#include +#include +#if DEBUG_OUTPUT +#include +#endif + +#ifndef IMAGE_SIZEOF_BASE_RELOCATION +// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? +# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) +#endif +#include "MemoryModule.h" + +/* + XXX We need to protect at least walking the 'loaded' linked list with a lock! +*/ + +/******************************************************************/ +FINDPROC findproc; +void *findproc_data = NULL; + +struct NAME_TABLE { + char *name; + DWORD ordinal; +}; + +typedef struct tagMEMORYMODULE { + PIMAGE_NT_HEADERS headers; + unsigned char *codeBase; + HMODULE *modules; + int numModules; + int initialized; + + struct NAME_TABLE *name_table; + + char *name; + int refcount; + struct tagMEMORYMODULE *next, *prev; +} MEMORYMODULE, *PMEMORYMODULE; + +typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); + +#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] + +MEMORYMODULE *loaded; /* linked list of loaded memory modules */ + +/* private - insert a loaded library in a linked list */ +static void _Register(char *name, MEMORYMODULE *module) +{ + module->next = loaded; + if (loaded) + loaded->prev = module; + module->prev = NULL; + loaded = module; +} + +/* private - remove a loaded library from a linked list */ +static void _Unregister(MEMORYMODULE *module) +{ + free(module->name); + if (module->prev) + module->prev->next = module->next; + if (module->next) + module->next->prev = module->prev; + if (module == loaded) + loaded = module->next; +} + +/* public - replacement for GetModuleHandle() */ +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName) +{ + MEMORYMODULE *p = loaded; + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpModuleName, p->name)) { + return (HMODULE)p; + } + p = p->next; + } + return GetModuleHandle(lpModuleName); +} + +/* public - replacement for LoadLibrary, but searches FIRST for memory + libraries, then for normal libraries. So, it will load libraries AS memory + module if they are found by findproc(). +*/ +HMODULE MyLoadLibrary(char *lpFileName) +{ + MEMORYMODULE *p = loaded; + HMODULE hMod; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpFileName, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + if (findproc && findproc_data) { + void *pdata = findproc(lpFileName, findproc_data); + if (pdata) { + hMod = MemoryLoadLibrary(lpFileName, pdata); + free(p); + return hMod; + } + } + hMod = LoadLibrary(lpFileName); + return hMod; +} + +/* public - replacement for GetProcAddress() */ +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) + return MemoryGetProcAddress(p, lpProcName); + p = p->next; + } + return GetProcAddress(hModule, lpProcName); +} + +/* public - replacement for FreeLibrary() */ +BOOL MyFreeLibrary(HMODULE hModule) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) { + if (--p->refcount == 0) { + _Unregister(p); + MemoryFreeLibrary(p); + } + return TRUE; + } + p = p->next; + } + return FreeLibrary(hModule); +} + +#if DEBUG_OUTPUT +static void +OutputLastError(const char *msg) +{ + LPVOID tmp; + char *tmpmsg; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL); + tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3); + sprintf(tmpmsg, "%s: %s", msg, tmp); + OutputDebugString(tmpmsg); + LocalFree(tmpmsg); + LocalFree(tmp); +} +#endif + +/* +static int dprintf(char *fmt, ...) +{ + char Buffer[4096]; + va_list marker; + int result; + + va_start(marker, fmt); + result = vsprintf(Buffer, fmt, marker); + OutputDebugString(Buffer); + return result; +} +*/ + +static void +CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +{ + int i, size; + unsigned char *codeBase = module->codeBase; + unsigned char *dest; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + if (section->SizeOfRawData == 0) + { + // section doesn't contain data in the dll itself, but may define + // uninitialized data + size = old_headers->OptionalHeader.SectionAlignment; + if (size > 0) + { + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + size, + MEM_COMMIT, + PAGE_READWRITE); + + section->Misc.PhysicalAddress = (DWORD)dest; + memset(dest, 0, size); + } + + // section is empty + continue; + } + + // commit memory block and copy data from dll + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + section->SizeOfRawData, + MEM_COMMIT, + PAGE_READWRITE); + memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); + section->Misc.PhysicalAddress = (DWORD)dest; + } +} + +// Protection flags for memory pages (Executable, Readable, Writeable) +static int ProtectionFlags[2][2][2] = { + { + // not executable + {PAGE_NOACCESS, PAGE_WRITECOPY}, + {PAGE_READONLY, PAGE_READWRITE}, + }, { + // executable + {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY}, + {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE}, + }, +}; + +static void +FinalizeSections(PMEMORYMODULE module) +{ + int i; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + + // loop through all sections and change access flags + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + DWORD protect, oldProtect, size; + int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; + int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; + int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; + + if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) + { + // section is not needed any more and can safely be freed + VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT); + continue; + } + + // determine protection flags based on characteristics + protect = ProtectionFlags[executable][readable][writeable]; + if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) + protect |= PAGE_NOCACHE; + + // determine size of region + size = section->SizeOfRawData; + if (size == 0) + { + if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfInitializedData; + else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfUninitializedData; + } + + if (size > 0) + { + // change memory access flags + if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0) +#if DEBUG_OUTPUT + OutputLastError("Error protecting memory page") +#endif + ; + } + } +} + +static void +PerformBaseRelocation(PMEMORYMODULE module, DWORD delta) +{ + DWORD i; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); + if (directory->Size > 0) + { + PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress); + for (; relocation->VirtualAddress > 0; ) + { + unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress); + unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) + { + DWORD *patchAddrHL; + int type, offset; + + // the upper 4 bits define the type of relocation + type = *relInfo >> 12; + // the lower 12 bits define the offset + offset = *relInfo & 0xfff; + + switch (type) + { + case IMAGE_REL_BASED_ABSOLUTE: + // skip relocation + break; + + case IMAGE_REL_BASED_HIGHLOW: + // change complete 32 bit address + patchAddrHL = (DWORD *)(dest + offset); + *patchAddrHL += delta; + break; + + default: + //printf("Unknown relocation: %d\n", type); + break; + } + } + + // advance to next relocation block + relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock); + } + } +} + +static int +BuildImportTable(PMEMORYMODULE module) +{ + int result=1; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); + if (directory->Size > 0) + { + PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress); + for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) + { + DWORD *thunkRef, *funcRef; + HMODULE handle; + + handle = MyLoadLibrary(codeBase + importDesc->Name); + if (handle == INVALID_HANDLE_VALUE) + { + //LastError should already be set +#if DEBUG_OUTPUT + OutputLastError("Can't load library"); +#endif + result = 0; + break; + } + + module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE))); + if (module->modules == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + result = 0; + break; + } + + module->modules[module->numModules++] = handle; + if (importDesc->OriginalFirstThunk) + { + thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } else { + // no hint table + thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } + for (; *thunkRef; thunkRef++, funcRef++) + { + if IMAGE_SNAP_BY_ORDINAL(*thunkRef) { + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef)); + } else { + PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef); + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name); + } + if (*funcRef == 0) + { + SetLastError(ERROR_PROC_NOT_FOUND); + result = 0; + break; + } + } + + if (!result) + break; + } + } + + return result; +} + +/* + MemoryLoadLibrary - load a library AS MEMORY MODULE, or return + existing MEMORY MODULE with increased refcount. + + This allows to load a library AGAIN as memory module which is + already loaded as HMODULE! + +*/ +HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data) +{ + PMEMORYMODULE result; + PIMAGE_DOS_HEADER dos_header; + PIMAGE_NT_HEADERS old_header; + unsigned char *code, *headers; + DWORD locationDelta; + DllEntryProc DllEntry; + BOOL successfull; + MEMORYMODULE *p = loaded; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(name, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + + /* Do NOT check for GetModuleHandle here! */ + + dos_header = (PIMAGE_DOS_HEADER)data; + if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("Not a valid executable file.\n"); +#endif + return NULL; + } + + old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; + if (old_header->Signature != IMAGE_NT_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("No PE header found.\n"); +#endif + return NULL; + } + + // reserve memory for image of library + code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + // try to allocate memory at arbitrary position + code = (unsigned char *)VirtualAlloc(NULL, + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); +#if DEBUG_OUTPUT + OutputLastError("Can't reserve memory"); +#endif + return NULL; + } + + result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); + result->codeBase = code; + result->numModules = 0; + result->modules = NULL; + result->initialized = 0; + result->next = result->prev = NULL; + result->refcount = 1; + result->name = strdup(name); + result->name_table = NULL; + + // XXX: is it correct to commit the complete memory region at once? + // calling DllEntry raises an exception if we don't... + VirtualAlloc(code, + old_header->OptionalHeader.SizeOfImage, + MEM_COMMIT, + PAGE_READWRITE); + + // commit memory for headers + headers = (unsigned char *)VirtualAlloc(code, + old_header->OptionalHeader.SizeOfHeaders, + MEM_COMMIT, + PAGE_READWRITE); + + // copy PE header to code + memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders); + result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; + + // update position + result->headers->OptionalHeader.ImageBase = (DWORD)code; + + // copy sections from DLL file block to new memory location + CopySections(data, old_header, result); + + // adjust base address of imported data + locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase); + if (locationDelta != 0) + PerformBaseRelocation(result, locationDelta); + + // load required dlls and adjust function table of imports + if (!BuildImportTable(result)) + goto error; + + // mark memory pages depending on section headers and release + // sections that are marked as "discardable" + FinalizeSections(result); + + // get entry point of loaded library + if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) + { + DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint); + if (DllEntry == 0) + { + SetLastError(ERROR_BAD_FORMAT); /* XXX ? */ +#if DEBUG_OUTPUT + OutputDebugString("Library has no entry point.\n"); +#endif + goto error; + } + + // notify library about attaching to process + successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); + if (!successfull) + { +#if DEBUG_OUTPUT + OutputDebugString("Can't attach library.\n"); +#endif + goto error; + } + result->initialized = 1; + } + + _Register(name, result); + + return (HMEMORYMODULE)result; + +error: + // cleanup + free(result->name); + MemoryFreeLibrary(result); + return NULL; +} + +int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2) +{ + return stricmp(p1->name, p2->name); +} + +int _find(const char **name, const struct NAME_TABLE *p) +{ + return stricmp(*name, p->name); +} + +struct NAME_TABLE *GetNameTable(PMEMORYMODULE module) +{ + unsigned char *codeBase; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory; + DWORD i, *nameRef; + WORD *ordinal; + struct NAME_TABLE *p, *ptab; + + if (module->name_table) + return module->name_table; + + codeBase = module->codeBase; + directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT); + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + + nameRef = (DWORD *)(codeBase + exports->AddressOfNames); + ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals); + + p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE) + * exports->NumberOfNames); + if (p == NULL) + return NULL; + ptab = p; + for (i=0; iNumberOfNames; ++i) { + p->name = (char *)(codeBase + *nameRef++); + p->ordinal = *ordinal++; + ++p; + } + qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare); + return ptab; +} + +FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name) +{ + unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; + int idx=-1; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); + + if (directory->Size == 0) + // no export table found + return NULL; + + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) + // DLL doesn't export anything + return NULL; + + if (HIWORD(name)) { + struct NAME_TABLE *ptab; + struct NAME_TABLE *found; + ptab = GetNameTable((PMEMORYMODULE)module); + if (ptab == NULL) + // some failure + return NULL; + found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find); + if (found == NULL) + // exported symbol not found + return NULL; + + idx = found->ordinal; + } + else + idx = LOWORD(name) - exports->Base; + + if ((DWORD)idx > exports->NumberOfFunctions) + // name <-> ordinal number don't match + return NULL; + + // AddressOfFunctions contains the RVAs to the "real" functions + return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4))); +} + +void MemoryFreeLibrary(HMEMORYMODULE mod) +{ + int i; + PMEMORYMODULE module = (PMEMORYMODULE)mod; + + if (module != NULL) + { + if (module->initialized != 0) + { + // notify library about detaching from process + DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); + (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); + module->initialized = 0; + } + + if (module->modules != NULL) + { + // free previously opened libraries + for (i=0; inumModules; i++) + if (module->modules[i] != INVALID_HANDLE_VALUE) + MyFreeLibrary(module->modules[i]); + + free(module->modules); + } + + if (module->codeBase != NULL) + // release memory of library + VirtualFree(module->codeBase, 0, MEM_RELEASE); + + if (module->name_table != NULL) + free(module->name_table); + + HeapFree(GetProcessHeap(), 0, module); + } +} diff --git a/setup/installer/windows/MemoryModule.h b/setup/installer/windows/MemoryModule.h new file mode 100644 index 0000000000..601d4c50df --- /dev/null +++ b/setup/installer/windows/MemoryModule.h @@ -0,0 +1,58 @@ +/* + * Memory DLL loading code + * Version 0.0.2 + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.h + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + */ + +#ifndef __MEMORY_MODULE_HEADER +#define __MEMORY_MODULE_HEADER + +#include + +typedef void *HMEMORYMODULE; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*FINDPROC)(); + +extern FINDPROC findproc; +extern void *findproc_data; + +HMEMORYMODULE MemoryLoadLibrary(char *, const void *); + +FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *); + +void MemoryFreeLibrary(HMEMORYMODULE); + +BOOL MyFreeLibrary(HMODULE hModule); +HMODULE MyLoadLibrary(char *lpFileName); +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName); +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName); + +#ifdef __cplusplus +} +#endif + +#endif // __MEMORY_MODULE_HEADER diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 7fb60968e7..35840c8de8 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -71,11 +71,13 @@ class Win32Freeze(Command, WixMixIn): self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc') self.py_ver = ''.join(map(str, sys.version_info[:2])) self.lib_dir = self.j(self.base, 'Lib') - self.pydlib = self.j(self.base, 'pydlib') self.pylib = self.j(self.base, 'pylib.zip') + self.dll_dir = self.j(self.base, 'DLLs') + self.plugins_dir = os.path.join(self.base, 'plugins') self.initbase() self.build_launchers() + self.add_plugins() self.freeze() self.embed_manifests() self.install_site_py() @@ -87,18 +89,20 @@ class Win32Freeze(Command, WixMixIn): shutil.rmtree(self.base) os.makedirs(self.base) - def freeze(self): - shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) - + def add_plugins(self): self.info('Adding plugins...') - tgt = os.path.join(self.base, 'plugins') - if not os.path.exists(tgt): - os.mkdir(tgt) + tgt = self.plugins_dir + if os.path.exists(tgt): + shutil.rmtree(tgt) + os.mkdir(tgt) base = self.j(self.SRC, 'calibre', 'plugins') for pat in ('*.pyd', '*.manifest'): for f in glob.glob(self.j(base, pat)): shutil.copy2(f, tgt) + def freeze(self): + shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) + self.info('Adding resources...') tgt = self.j(self.base, 'resources') if os.path.exists(tgt): @@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn): shutil.copytree(self.j(self.src_root, 'resources'), tgt) self.info('Adding Qt and python...') - self.dll_dir = self.j(self.base, 'DLLs') shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): @@ -318,8 +321,8 @@ class Win32Freeze(Command, WixMixIn): if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, 'setup', 'installer', 'windows') - sources = [self.j(base, x) for x in ['util.c']] - headers = [self.j(base, x) for x in ['util.h']] + sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']] + headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']] objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources] cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split() cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver] @@ -371,43 +374,49 @@ class Win32Freeze(Command, WixMixIn): def archive_lib_dir(self): self.info('Putting all python code into a zip file for performance') - if os.path.exists(self.pydlib): - shutil.rmtree(self.pydlib) - os.makedirs(self.pydlib) self.zf_timestamp = time.localtime(time.time())[:6] self.zf_names = set() with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf: + # Add the .pyds from python and calibre to the zip file + for x in (self.plugins_dir, self.dll_dir): + for pyd in os.listdir(x): + if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd': + # sqlite_custom has to be a file for + # sqlite_load_extension to work + self.add_to_zipfile(zf, pyd, x) + os.remove(self.j(x, pyd)) + + # Add everything in Lib except site-packages to the zip file for x in os.listdir(self.lib_dir): if x == 'site-packages': continue self.add_to_zipfile(zf, x, self.lib_dir) sp = self.j(self.lib_dir, 'site-packages') - handled = set(['site.pyo']) - for pth in ('PIL.pth', 'pywin32.pth'): - handled.add(pth) - shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth)) - for d in self.get_pth_dirs(self.j(sp, pth)): - shutil.copytree(d, self.j(self.pydlib, self.b(d)), True) - handled.add(self.b(d)) + # Special handling for PIL and pywin32 + handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32']) + self.add_to_zipfile(zf, 'PIL', sp) + base = self.j(sp, 'win32', 'lib') + for x in os.listdir(base): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) + base = self.d(base) + for x in os.listdir(base): + if not os.path.isdir(self.j(base, x)): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) handled.add('easy-install.pth') for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')): handled.add(self.b(d)) - zip_safe = self.is_zip_safe(d) for x in os.listdir(d): if x == 'EGG-INFO': continue - if zip_safe: - self.add_to_zipfile(zf, x, d) - else: - absp = self.j(d, x) - dest = self.j(self.pydlib, x) - if os.path.isdir(absp): - shutil.copytree(absp, dest, True) - else: - shutil.copy2(absp, dest) + self.add_to_zipfile(zf, x, d) + # The rest of site-packages + # We dont want the site.py from site-packages + handled.add('site.pyo') for x in os.listdir(sp): if x in handled or x.endswith('.egg-info'): continue @@ -415,33 +424,18 @@ class Win32Freeze(Command, WixMixIn): if os.path.isdir(absp): if not os.listdir(absp): continue - if self.is_zip_safe(absp): - self.add_to_zipfile(zf, x, sp) - else: - shutil.copytree(absp, self.j(self.pydlib, x), True) + self.add_to_zipfile(zf, x, sp) else: - if x.endswith('.pyd'): - shutil.copy2(absp, self.j(self.pydlib, x)) - else: - self.add_to_zipfile(zf, x, sp) + self.add_to_zipfile(zf, x, sp) shutil.rmtree(self.lib_dir) - def is_zip_safe(self, path): - for f in walk(path): - ext = os.path.splitext(f)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): - return False - return True - def get_pth_dirs(self, pth): base = os.path.dirname(pth) for line in open(pth).readlines(): line = line.strip() if not line or line.startswith('#') or line.startswith('import'): continue - if line == 'win32\\lib': - continue candidate = self.j(base, line) if os.path.exists(candidate): yield candidate @@ -463,10 +457,10 @@ class Win32Freeze(Command, WixMixIn): self.add_to_zipfile(zf, name + os.sep + x, base) else: ext = os.path.splitext(name)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): + if ext in ('.dll',): raise ValueError('Cannot add %r to zipfile'%abspath) zinfo.external_attr = 0600 << 16 - if ext in ('.py', '.pyc', '.pyo'): + if ext in ('.py', '.pyc', '.pyo', '.pyd'): with open(abspath, 'rb') as f: zf.writestr(zinfo, f.read()) diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index 5610ff197e..33f2e63585 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -1,12 +1,72 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, linecache +import sys +import os +import zipimport +import _memimporter + +DEBUG_ZIPIMPORT = False + +class ZipExtensionImporter(zipimport.zipimporter): + ''' + Taken, with thanks, from the py2exe source code + ''' + + def __init__(self, *args, **kwargs): + zipimport.zipimporter.__init__(self, *args, **kwargs) + # We know there are no dlls in the zip file, so dont set findproc + # (performance optimization) + #_memimporter.set_find_proc(self.locate_dll_image) + + def find_module(self, fullname, path=None): + result = zipimport.zipimporter.find_module(self, fullname, path) + if result: + return result + fullname = fullname.replace(".", "\\") + if (fullname + '.pyd') in self._files: + return self + return None + + def locate_dll_image(self, name): + # A callback function for_memimporter.import_module. Tries to + # locate additional dlls. Returns the image as Python string, + # or None if not found. + if name in self._files: + return self.get_data(name) + return None + + def load_module(self, fullname): + if sys.modules.has_key(fullname): + mod = sys.modules[fullname] + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive)) + return mod + try: + return zipimport.zipimporter.load_module(self, fullname) + except zipimport.ZipImportError: + pass + initname = "init" + fullname.split(".")[-1] # name of initfunction + filename = fullname.replace(".", "\\") + path = filename + '.pyd' + if path in self._files: + if DEBUG_ZIPIMPORT: + sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive)) + code = self.get_data(path) + mod = _memimporter.import_module(code, initname, fullname, path) + mod.__file__ = "%s\\%s" % (self.archive, path) + mod.__loader__ = self + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__)) + return mod + raise zipimport.ZipImportError, "can't find module %s" % fullname + + def __repr__(self): + return "<%s object %r>" % (self.__class__.__name__, self.archive) def abs__file__(): @@ -42,42 +102,6 @@ def makepath(*paths): dir = os.path.abspath(os.path.join(*paths)) return dir, os.path.normcase(dir) -def addpackage(sitedir, name): - """Process a .pth file within the site-packages directory: - For each line in the file, either combine it with sitedir to a path, - or execute it if it starts with 'import '. - """ - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "rU") - except IOError: - return - with f: - for line in f: - if line.startswith("#"): - continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if os.path.exists(dir): - sys.path.append(dir) - - -def addsitedir(sitedir): - """Add 'sitedir' argument to sys.path if missing and handle .pth files in - 'sitedir'""" - sitedir, sitedircase = makepath(sitedir) - try: - names = os.listdir(sitedir) - except os.error: - return - dotpth = os.extsep + "pth" - names = [name for name in names if name.endswith(dotpth)] - for name in sorted(names): - addpackage(sitedir, name) - def run_entry_point(): bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function sys.argv[0] = bname+'.exe' @@ -89,6 +113,10 @@ def main(): sys.setdefaultencoding('utf-8') aliasmbcs() + sys.path_hooks.insert(0, ZipExtensionImporter) + sys.path_importer_cache.clear() + + import linecache def fake_getline(filename, lineno, module_globals=None): return '' linecache.orig_getline = linecache.getline @@ -96,10 +124,11 @@ def main(): abs__file__() - addsitedir(os.path.join(sys.app_dir, 'pydlib')) - add_calibre_vars() + # Needed for pywintypes to be able to load its DLL + sys.path.append(os.path.join(sys.app_dir, 'DLLs')) + return run_entry_point() diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c index 329e3bf8c3..4075d7e123 100644 --- a/setup/installer/windows/util.c +++ b/setup/installer/windows/util.c @@ -1,18 +1,130 @@ /* * Copyright 2009 Kovid Goyal + * The memimporter code is taken from the py2exe project */ #include "util.h" + #include #include #include + static char GUI_APP = 0; static char python_dll[] = PYDLL; void set_gui_app(char yes) { GUI_APP = yes; } char is_gui_app() { return GUI_APP; } + +// memimporter {{{ + +#include "MemoryModule.h" + +static char **DLL_Py_PackageContext = NULL; +static PyObject **DLL_ImportError = NULL; +static char module_doc[] = +"Importer which can load extension modules from memory"; + + +static void *memdup(void *ptr, Py_ssize_t size) +{ + void *p = malloc(size); + if (p == NULL) + return NULL; + memcpy(p, ptr, size); + return p; +} + +/* + Be sure to detect errors in FindLibrary - undetected errors lead to + very strange behaviour. +*/ +static void* FindLibrary(char *name, PyObject *callback) +{ + PyObject *result; + char *p; + Py_ssize_t size; + + if (callback == NULL) + return NULL; + result = PyObject_CallFunction(callback, "s", name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (-1 == PyString_AsStringAndSize(result, &p, &size)) { + PyErr_Clear(); + Py_DECREF(result); + return NULL; + } + p = memdup(p, size); + Py_DECREF(result); + return p; +} + +static PyObject *set_find_proc(PyObject *self, PyObject *args) +{ + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback)) + return NULL; + Py_DECREF((PyObject *)findproc_data); + Py_INCREF(callback); + findproc_data = (void *)callback; + return Py_BuildValue("i", 1); +} + +static PyObject * +import_module(PyObject *self, PyObject *args) +{ + char *data; + int size; + char *initfuncname; + char *modname; + char *pathname; + HMEMORYMODULE hmem; + FARPROC do_init; + + char *oldcontext; + + /* code, initfuncname, fqmodulename, path */ + if (!PyArg_ParseTuple(args, "s#sss:import_module", + &data, &size, + &initfuncname, &modname, &pathname)) + return NULL; + hmem = MemoryLoadLibrary(pathname, data); + if (!hmem) { + PyErr_Format(*DLL_ImportError, + "MemoryLoadLibrary() failed loading %s", pathname); + return NULL; + } + do_init = MemoryGetProcAddress(hmem, initfuncname); + if (!do_init) { + MemoryFreeLibrary(hmem); + PyErr_Format(*DLL_ImportError, + "Could not find function %s in memory loaded pyd", initfuncname); + return NULL; + } + + oldcontext = *DLL_Py_PackageContext; + *DLL_Py_PackageContext = modname; + do_init(); + *DLL_Py_PackageContext = oldcontext; + if (PyErr_Occurred()) + return NULL; + /* Retrieve from sys.modules */ + return PyImport_ImportModule(modname); +} + +static PyMethodDef methods[] = { + { "import_module", import_module, METH_VARARGS, + "import_module(code, initfunc, dllname[, finder]) -> module" }, + { "set_find_proc", set_find_proc, METH_VARARGS }, + { NULL, NULL }, /* Sentinel */ +}; + +// }}} + static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) { wchar_t *buf, *cbuf; buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)* @@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, char *dummy_argv[1] = {""}; buf = (char*)calloc(MAX_PATH, sizeof(char)); - path = (char*)calloc(3*MAX_PATH, sizeof(char)); + path = (char*)calloc(MAX_PATH, sizeof(char)); if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1)); sz = GetModuleFileNameA(NULL, buf, MAX_PATH); @@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, buf[strlen(buf)-1] = '\0'; _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf); - _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs", - buf, buf, buf); + _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf); free(buf); @@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1)); //*flag = 1; - + DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext"); + if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1)); + DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError"); + if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1)); Py_SetProgramName(program_name); Py_SetPythonHome(python_home); @@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, PyList_SetItem(argv, i, v); } PySys_SetObject("argv", argv); + + findproc = FindLibrary; + Py_InitModule3("_memimporter", methods, module_doc); + } diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index 0a85b6fb81..3ebe0882e0 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -164,10 +164,6 @@ - - - - diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 8d65c37bbf..79110d9585 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -53,6 +53,8 @@ Run an embedded python interpreter. default=False, action='store_true') parser.add_option('-m', '--inspect-mobi', help='Inspect the MOBI file at the specified path', default=None) + parser.add_option('--test-build', help='Test binary modules in build', + action='store_true', default=False) return parser @@ -232,6 +234,9 @@ def main(args=sys.argv): elif opts.inspect_mobi is not None: from calibre.ebooks.mobi.debug import inspect_mobi inspect_mobi(opts.inspect_mobi) + elif opts.test_build: + from calibre.test_build import test + test() else: from calibre import ipython ipython() diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py new file mode 100644 index 0000000000..0610f01805 --- /dev/null +++ b/src/calibre/test_build.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from future_builtins import map + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +''' +Test a binary calibre build to ensure that all needed binary images/libraries have loaded. +''' + +import cStringIO +from calibre.constants import plugins, iswindows + +def test_plugins(): + for name in plugins: + mod, err = plugins[name] + if err or not mod: + raise RuntimeError('Plugin %s failed to load with error: %s' % + (name, err)) + print (mod, 'loaded') + +def test_lxml(): + from lxml import etree + raw = '' + root = etree.fromstring(raw) + if etree.tostring(root) == raw: + print ('lxml OK!') + else: + raise RuntimeError('lxml failed') + +def test_fontconfig(): + from calibre.utils.fonts import fontconfig + families = fontconfig.find_font_families() + num = len(families) + if num < 10: + raise RuntimeError('Fontconfig found only %d font families'%num) + print ('Fontconfig OK! (%d families)'%num) + +def test_winutil(): + from calibre.devices.scanner import win_pnp_drives + matches = win_pnp_drives.scanner() + if len(matches) < 1: + raise RuntimeError('win_pnp_drives returned no drives') + print ('win_pnp_drives OK!') + +def test_win32(): + from calibre.utils.winshell import desktop + d = desktop() + if not d: + raise RuntimeError('winshell failed') + print ('winshell OK! (%s is the desktop)'%d) + +def test_sqlite(): + import sqlite3 + conn = sqlite3.connect(':memory:') + from calibre.library.sqlite import load_c_extensions + if not load_c_extensions(conn, True): + raise RuntimeError('Failed to load sqlite extension') + print ('sqlite OK!') + +def test_qt(): + from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager) + fmts = set(map(unicode, QImageReader.supportedImageFormats())) + if 'jpg' not in fmts or 'png' not in fmts: + raise RuntimeError( + "Qt doesn't seem to be able to load its image plugins") + QWebView, QDialog + na = QNetworkAccessManager() + if not hasattr(na, 'sslErrors'): + raise RuntimeError('Qt not compiled with openssl') + print ('Qt OK!') + +def test_imaging(): + from calibre.utils.magick.draw import create_canvas, Image + im = create_canvas(20, 20, '#ffffff') + jpg = im.export('jpg') + Image().load(jpg) + im.export('png') + print ('ImageMagick OK!') + from PIL import Image + i = Image.open(cStringIO.StringIO(jpg)) + if i.size != (20, 20): + raise RuntimeError('PIL choked!') + print ('PIL OK!') + +def test(): + test_plugins() + test_lxml() + test_fontconfig() + test_sqlite() + if iswindows: + test_winutil() + test_win32() + test_qt() + test_imaging() + +if __name__ == '__main__': + test() + From 51c4fe162c3ca2c62507ce21c56280ee76818480 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Jun 2011 23:01:32 -0600 Subject: [PATCH 18/35] No longer ship libusb in the windows binary --- setup/installer/windows/freeze.py | 13 ++++--------- setup/installer/windows/notes.rst | 4 +++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 35840c8de8..0fe494e831 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -16,7 +16,6 @@ from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = r'Q:\openssl' QT_DIR = 'Q:\\Qt\\4.7.3' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] -LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.6.6', @@ -96,9 +95,10 @@ class Win32Freeze(Command, WixMixIn): shutil.rmtree(tgt) os.mkdir(tgt) base = self.j(self.SRC, 'calibre', 'plugins') - for pat in ('*.pyd', '*.manifest'): - for f in glob.glob(self.j(base, pat)): - shutil.copy2(f, tgt) + for f in glob.glob(self.j(base, '*.pyd')): + # We dont want the manifests as the manifest in the exe will be + # used instead + shutil.copy2(f, tgt) def freeze(self): shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) @@ -200,11 +200,6 @@ class Win32Freeze(Command, WixMixIn): print print 'Adding third party dependencies' - tdir = os.path.join(self.base, 'driver') - os.makedirs(tdir) - for pat in ('*.dll', '*.sys', '*.cat', '*.inf'): - for f in glob.glob(os.path.join(LIBUSB_DIR, pat)): - shutil.copyfile(f, os.path.join(tdir, os.path.basename(f))) print '\tAdding unrar' shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir, os.path.basename(LIBUNRAR))) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 11b5bccf79..0bb8b7b15b 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open Now, run configure and make:: - configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake +-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly + + configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake SIP ----- From ae724b9ef686a3e314a6e8307adb5688dd610a64 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 6 Jun 2011 12:20:05 +0100 Subject: [PATCH 19/35] Split amazon UK and DE away from US, making each a separate plugin --- src/calibre/gui2/store/amazon_de_plugin.py | 107 +++++++++++++++++++-- src/calibre/gui2/store/amazon_uk_plugin.py | 12 +-- 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/store/amazon_de_plugin.py b/src/calibre/gui2/store/amazon_de_plugin.py index f7b17a2e83..88ccbdbded 100644 --- a/src/calibre/gui2/store/amazon_de_plugin.py +++ b/src/calibre/gui2/store/amazon_de_plugin.py @@ -6,21 +6,23 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re, urllib +from contextlib import closing + +from lxml import html + from PyQt4.Qt import QUrl +from calibre import browser from calibre.gui2 import open_url -from calibre.gui2.store.amazon_plugin import AmazonKindleStore +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult -class AmazonDEKindleStore(AmazonKindleStore): +class AmazonDEKindleStore(StorePlugin): ''' For comments on the implementation, please see amazon_plugin.py ''' - search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords=' - details_url = 'http://amazon.de/dp/' - drm_search_text = u'Gleichzeitige Verwendung von Geräten' - drm_free_text = u'Keine Einschränkung' - def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'charhale0a-21'} store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8&site-redirect=de' @@ -32,3 +34,94 @@ class AmazonDEKindleStore(AmazonKindleStore): '&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de' '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742') % aff_id open_url(QUrl(store_link)) + + def search(self, query, max_results=10, timeout=60): + search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords=' + url = search_url + urllib.quote_plus(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + + # Amazon has two results pages. + is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') + # Horizontal grid of books. + if is_shot: + data_xpath = '//div[contains(@class, "result")]' + format_xpath = './/div[@class="productTitle"]/text()' + cover_xpath = './/div[@class="productTitle"]//img/@src' + # Vertical list of books. + else: + data_xpath = '//div[@class="productData"]' + format_xpath = './/span[@class="format"]/text()' + cover_xpath = '../div[@class="productImage"]/a/img/@src' + + for data in doc.xpath(data_xpath): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). Se we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + format = ''.join(data.xpath(format_xpath)) + if 'kindle' not in format.lower(): + continue + + # We must have an asin otherwise we can't easily reference the + # book later. + asin_href = None + asin_a = data.xpath('.//div[@class="productTitle"]/a[1]') + if asin_a: + asin_href = asin_a[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) + if m: + asin = m.group('asin') + else: + continue + else: + continue + + cover_url = ''.join(data.xpath(cover_xpath)) + + title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()')) + price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + + if is_shot: + author = format.split(' von ')[-1] + else: + author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split(' von ')[-1] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url.strip() + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = asin.strip() + s.formats = 'Kindle' + + yield s + + def get_details(self, search_result, timeout): + drm_search_text = u'Gleichzeitige Verwendung von Geräten' + drm_free_text = u'Keine Einschränkung' + url = 'http://amazon.de/dp/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + + drm_search_text + '")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + + drm_free_text + '") and contains(b, "' + + drm_search_text + '")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + return True diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py index a922f0516b..fcc0c02e01 100644 --- a/src/calibre/gui2/store/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -15,17 +15,14 @@ from PyQt4.Qt import QUrl from calibre import browser from calibre.gui2 import open_url -from calibre.gui2.store.amazon_plugin import AmazonKindleStore +from calibre.gui2.store import StorePlugin from calibre.gui2.store.search_result import SearchResult -class AmazonUKKindleStore(AmazonKindleStore): +class AmazonUKKindleStore(StorePlugin): ''' For comments on the implementation, please see amazon_plugin.py ''' - search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords=' - details_url = 'http://amazon.co.uk/dp/' - def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'calcharles-21'} store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/Kindle-eBooks/b?ie=UTF8&node=341689031&ref_=sa_menu_kbo2&tag=%(tag)s&linkCode=ur2&camp=1634&creative=19450' % aff_id @@ -36,7 +33,8 @@ class AmazonUKKindleStore(AmazonKindleStore): open_url(QUrl(store_link)) def search(self, query, max_results=10, timeout=60): - url = self.search_url + urllib.quote_plus(query) + search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords=' + url = search_url + urllib.quote_plus(query) br = browser() counter = max_results @@ -95,7 +93,7 @@ class AmazonUKKindleStore(AmazonKindleStore): if search_result.drm: return - url = self.details_url + url = 'http://amazon.co.uk/dp/' br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: From 5617ddeb731fde3ea160084f0e1f77beb5efd43f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 6 Jun 2011 12:23:21 +0100 Subject: [PATCH 20/35] ... --- src/calibre/gui2/store/amazon_uk_plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py index fcc0c02e01..f8686d19fe 100644 --- a/src/calibre/gui2/store/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -94,6 +94,8 @@ class AmazonUKKindleStore(StorePlugin): return url = 'http://amazon.co.uk/dp/' + drm_search_text = u'Simultaneous Device Usage' + drm_free_text = u'Unlimited' br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: @@ -104,10 +106,10 @@ class AmazonUKKindleStore(StorePlugin): if is_kindle: search_result.formats = 'Kindle' if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + - self.drm_search_text + '")])'): + drm_search_text + '")])'): if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + - self.drm_free_text + '") and contains(b, "' + - self.drm_search_text + '")])'): + drm_free_text + '") and contains(b, "' + + drm_search_text + '")])'): search_result.drm = SearchResult.DRM_UNLOCKED else: search_result.drm = SearchResult.DRM_UNKNOWN From 1f837ca129b0f5a0b7e0e311362ed907a7754224 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 06:34:52 -0600 Subject: [PATCH 21/35] ... --- setup/installer/windows/freeze.py | 2 +- setup/installer/windows/site.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 0fe494e831..e67f75f9b0 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -72,7 +72,7 @@ class Win32Freeze(Command, WixMixIn): self.lib_dir = self.j(self.base, 'Lib') self.pylib = self.j(self.base, 'pylib.zip') self.dll_dir = self.j(self.base, 'DLLs') - self.plugins_dir = os.path.join(self.base, 'plugins') + self.plugins_dir = os.path.join(self.base, 'plugins2') self.initbase() self.build_launchers() diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index 33f2e63585..d2212037ee 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -92,7 +92,7 @@ def aliasmbcs(): def add_calibre_vars(): sys.resources_location = os.path.join(sys.app_dir, 'resources') - sys.extensions_location = os.path.join(sys.app_dir, 'plugins') + sys.extensions_location = os.path.join(sys.app_dir, 'plugins2') dv = os.environ.get('CALIBRE_DEVELOP_FROM', None) if dv and os.path.exists(dv): From 1620dee54830d88f769dfa2aa9d34eff2161f156 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 06:56:34 -0600 Subject: [PATCH 22/35] Clean up Qt plugins dir in windows build --- setup/installer/windows/freeze.py | 4 ++++ src/calibre/test_build.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index e67f75f9b0..59d08cea64 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -197,6 +197,10 @@ class Win32Freeze(Command, WixMixIn): if os.path.exists(tg): shutil.rmtree(tg) shutil.copytree(imfd, tg) + for dirpath, dirnames, filenames in os.walk(tdir): + for x in filenames: + if not x.endswith('.dll'): + os.remove(self.j(dirpath, x)) print print 'Adding third party dependencies' diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 0610f01805..4d45077b93 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -65,7 +65,8 @@ def test_sqlite(): def test_qt(): from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager) fmts = set(map(unicode, QImageReader.supportedImageFormats())) - if 'jpg' not in fmts or 'png' not in fmts: + testf = set(['jpg', 'png', 'mng', 'svg', 'ico', 'gif']) + if testf.intersection(fmts) != testf: raise RuntimeError( "Qt doesn't seem to be able to load its image plugins") QWebView, QDialog From 9ba38f6585a80aa667c45b4a74ad994d41532fe4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 08:40:29 -0600 Subject: [PATCH 23/35] Allow has_device_jobs to also check for queued device jobs --- src/calibre/gui2/jobs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 51c54843a4..6aae892d61 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -197,10 +197,12 @@ class JobManager(QAbstractTableModel): # {{{ def row_to_job(self, row): return self.jobs[row] - def has_device_jobs(self): + def has_device_jobs(self, queued_also=False): for job in self.jobs: - if job.is_running and isinstance(job, DeviceJob): - return True + if isinstance(job, DeviceJob): + if job.duration is None: # Running or waiting + if (job.is_running or queued_also): + return True return False def has_jobs(self): From 9baceb011a624448165cd275a3735020ca67a833 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 08:53:27 -0600 Subject: [PATCH 24/35] Make the Get Books dialog larger by default --- src/calibre/gui2/store/search/search.ui | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/store/search/search.ui b/src/calibre/gui2/store/search/search.ui index 1451aa09f1..ba5cb2b975 100644 --- a/src/calibre/gui2/store/search/search.ui +++ b/src/calibre/gui2/store/search/search.ui @@ -6,15 +6,15 @@ 0 0 - 584 - 533 + 872 + 610 Get Books - + :/images/store.png:/images/store.png @@ -82,8 +82,8 @@ 0 0 - 125 - 127 + 173 + 106 @@ -255,7 +255,7 @@ - + From 6f087f87d85bfc3bef95475a1f0c2d7c5087b1c6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 09:19:24 -0600 Subject: [PATCH 25/35] ... --- src/calibre/gui2/store/search/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 8289c89b96..c802238a35 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -281,11 +281,11 @@ class SearchDialog(QDialog, Ui_Dialog): tab_widget.setCurrentIndex(tab_index) d.exec_() - + # Save dialog state. self.config['config_dialog_geometry'] = bytearray(d.saveGeometry()) self.config['config_dialog_tab_index'] = tab_widget.currentIndex() - + search_config_widget.save_settings() self.config_changed() self.gui.load_store_plugins() From 95731c3be07553f2754f03772ab142b9d666a853 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 09:48:37 -0600 Subject: [PATCH 26/35] Add a cancel callback to ProceedNotification --- src/calibre/gui2/dialogs/message_box.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/message_box.py b/src/calibre/gui2/dialogs/message_box.py index fdec19dc69..f7cd43bdbd 100644 --- a/src/calibre/gui2/dialogs/message_box.py +++ b/src/calibre/gui2/dialogs/message_box.py @@ -154,13 +154,16 @@ _proceed_memory = [] class ProceedNotification(MessageBox): # {{{ def __init__(self, callback, payload, html_log, log_viewer_title, title, msg, - det_msg='', show_copy_button=False, parent=None): + det_msg='', show_copy_button=False, parent=None, + cancel_callback=None): ''' A non modal popup that notifies the user that a background task has been completed. :param callback: A callable that is called with payload if the user - asks to proceed. Note that this is always called in the GUI thread + asks to proceed. Note that this is always called in the GUI thread. + :param cancel_callback: A callable that is called with the payload if + the users asks not to proceed. :param payload: Arbitrary object, passed to callback :param html_log: An HTML or plain text log :param log_viewer_title: The title for the log viewer window @@ -181,7 +184,7 @@ class ProceedNotification(MessageBox): # {{{ self.vlb.clicked.connect(self.show_log) self.det_msg_toggle.setVisible(bool(det_msg)) self.setModal(False) - self.callback = callback + self.callback, self.cancel_callback = callback, cancel_callback _proceed_memory.append(self) def show_log(self): @@ -192,9 +195,11 @@ class ProceedNotification(MessageBox): # {{{ try: if result == self.Accepted: self.callback(self.payload) + elif self.cancel_callback is not None: + self.cancel_callback(self.payload) finally: # Ensure this notification is garbage collected - self.callback = None + self.callback = self.cancel_callback = None self.setParent(None) self.finished.disconnect() self.vlb.clicked.disconnect() From b5a166753b02eea1712131ec004adfe2564b8db4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 11:33:32 -0600 Subject: [PATCH 27/35] Windows build: No longer install CRT in the system SxS folder. Makes calibre fully relocatable. --- setup/installer/windows/en-us.xml | 2 +- setup/installer/windows/freeze.py | 33 ++++++++++++++++++++++-- setup/installer/windows/wix-template.xml | 16 ++++++------ setup/installer/windows/wix.py | 1 - src/calibre/test_build.py | 7 +++++ 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/setup/installer/windows/en-us.xml b/setup/installer/windows/en-us.xml index 89cc25f0a2..d3cdac2a41 100644 --- a/setup/installer/windows/en-us.xml +++ b/setup/installer/windows/en-us.xml @@ -1,6 +1,6 @@ - If you are upgrading from a {app} version older than 0.6.17, please uninstall {app} first. Click Advanced to change installation settings. + Click Advanced to change installation settings. Computing space requirements, this may take upto five minutes... Computing space requirements, this may take upto five minutes... Computing space requirements, this may take upto five minutes... diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 59d08cea64..63993c19f0 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -8,8 +8,8 @@ __docformat__ = 'restructuredtext en' import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time -from setup import Command, modules, functions, basenames, __version__, \ - __appname__ +from setup import (Command, modules, functions, basenames, __version__, + __appname__) from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn @@ -20,6 +20,7 @@ LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.6.6', 'VisualMagick', 'bin') +CRT = r'C:\Microsoft.VC90.CRT' VERSION = re.sub('[a-z]\d+', '', __version__) WINVER = VERSION+'.0' @@ -81,8 +82,33 @@ class Win32Freeze(Command, WixMixIn): self.embed_manifests() self.install_site_py() self.archive_lib_dir() + self.remove_CRT_from_manifests() self.create_installer() + def remove_CRT_from_manifests(self): + ''' + The dependency on the CRT is removed from the manifests of all DLLs. + This allows the CRT loaded by the .exe files to be used instead. + ''' + search_pat = re.compile(r'(?is).*Microsoft\.VC\d+\.CRT') + repl_pat = re.compile( + r'(?is).*?Microsoft\.VC\d+\.CRT.*?') + + for dll in glob.glob(self.j(self.dll_dir, '*.dll')): + bn = self.b(dll) + with open(dll, 'rb') as f: + raw = f.read() + match = search_pat.search(raw) + if match is None: + continue + self.info('Removing CRT dependency from manifest of: %s'%bn) + # Blank out the bytes corresponding to the dependency specification + nraw = repl_pat.sub(lambda m: b' '*len(m.group()), raw) + if len(nraw) != len(raw): + raise Exception('Something went wrong with %s'%bn) + with open(dll, 'wb') as f: + f.write(nraw) + def initbase(self): if self.e(self.base): shutil.rmtree(self.base) @@ -103,6 +129,9 @@ class Win32Freeze(Command, WixMixIn): def freeze(self): shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) + self.info('Adding CRT') + shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT))) + self.info('Adding resources...') tgt = self.j(self.base, 'resources') if os.path.exists(tgt): diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index 3ebe0882e0..46b5a07ead 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -11,6 +11,10 @@ SummaryCodepage='1252' /> + + - @@ -100,10 +103,6 @@ - - - - @@ -149,12 +148,13 @@ Set default folder name and allow only per machine installs. For a per-machine installation, the default installation location will be [ProgramFilesFolder][ApplicationFolderName] and the user - will be able to change it in the setup UI. This is because the installer - has to install the VC90 merge module into the system winsxs folder for python - to work, so per user installs are impossible anyway. + will be able to change it in the setup UI. This is no longer necessary + (i.e. per user installs should work) but left this way as I + dont want to deal with the complications --> + diff --git a/setup/installer/windows/wix.py b/setup/installer/windows/wix.py index 857a667a9e..b43e14a711 100644 --- a/setup/installer/windows/wix.py +++ b/setup/installer/windows/wix.py @@ -35,7 +35,6 @@ class WixMixIn: exe_map = self.smap, main_icon = self.j(self.src_root, 'icons', 'library.ico'), web_icon = self.j(self.src_root, 'icons', 'web.ico'), - crt_msm = self.j(self.SW, 'Microsoft_VC90_CRT_x86.msm') ) template = open(self.j(self.d(__file__), 'en-us.xml'), 'rb').read() diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 4d45077b93..b987658d72 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -88,6 +88,12 @@ def test_imaging(): raise RuntimeError('PIL choked!') print ('PIL OK!') +def test_unrar(): + from calibre.libunrar import _libunrar + if not _libunrar: + raise RuntimeError('Failed to load libunrar') + print ('Unrar OK!') + def test(): test_plugins() test_lxml() @@ -98,6 +104,7 @@ def test(): test_win32() test_qt() test_imaging() + test_unrar() if __name__ == '__main__': test() From 29c0449d38f9efd9ac165af3e69ae7f37b1890d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Mon, 6 Jun 2011 20:13:21 +0200 Subject: [PATCH 28/35] Zixo description --- src/calibre/customize/builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4c7e03e89d..0968e752a7 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1420,7 +1420,7 @@ class StoreWoblinkStore(StoreBase): class StoreZixoStore(StoreBase): name = 'Zixo' author = u'Tomasz Długosz' - description = u'Księgarnia z ebookami oraz książkami audio' + description = u'Księgarnia z ebookami oraz książkami audio. Aby otwierać książki w formacie Zixo należy zainstalować program dostępny na stronie księgarni. Umożliwia on m.in. dodawanie zakładek i dostosowywanie rozmiaru czcionki.' actual_plugin = 'calibre.gui2.store.zixo_plugin:ZixoStore' headquarters = 'PL' From 88e8e9db7eb9354f397c33203ae535972fe2f7ac Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Jun 2011 12:14:03 -0600 Subject: [PATCH 29/35] Content server: Show stack trace on javascript errors --- resources/content_server/browse/browse.html | 4 ++-- resources/content_server/browse/browse.js | 8 +++++++- src/calibre/library/server/browse.py | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/resources/content_server/browse/browse.html b/resources/content_server/browse/browse.html index de78e432d7..6a9697dc06 100644 --- a/resources/content_server/browse/browse.html +++ b/resources/content_server/browse/browse.html @@ -20,8 +20,8 @@ - - + +