From fbd7f787c275813099b38239ab724bc6974ec74a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 11:53:40 +0530 Subject: [PATCH 01/28] E-book viewer: Make the detection of full screen layouts like covers a little more robust --- resources/compiled_coffeescript.zip | Bin 57018 -> 56964 bytes src/calibre/ebooks/oeb/display/paged.coffee | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 9f8bccfd5e35a0494759378e0f8a291174a34875..573a8128ac70556e1a7468ad34ee82f7d3157c76 100644 GIT binary patch delta 174 zcmdnBm$_vxvqXS5Gm8iV2&{-xc1-o@Onv6Y!0^m%vRsJd=0`jS_$S|xkz+L9{9Q(p zo6&f)pvJW=8<{rpXfiU1Fk{olz#t7a4Ngo}3=x}Lc-M$Y-hJ}LMA6B+fXs6tK&HZE szI$3slBr-(|9b{Zn#U$L-cz0Y{Ei0Gzw?s??t*KUr51%}rj%qTXe$_*DyXSV<~I|a{E=Ud(P*=PfFw7I5lF02N@26F z^yV#-Rqts`zIjJ{vi=1wod9n}CJ|;41`xos5-bcQHu69mV8p~00%8TYb4+%;tHrc6 r5ybl+!acbc$YUr1@y^{fU=p}I>7J?xD;r2V8xZbiWMFuD2gCyaYEDRu diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 4f912513a9..286945bfb6 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -79,7 +79,7 @@ class PagedDisplay if not this.in_paged_mode # Check if the current document is a full screen layout like # cover, if so we treat it specially. - single_screen = (document.body.scrollWidth < window.innerWidth + 25 and document.body.scrollHeight < window.innerHeight + 25) + single_screen = (document.body.scrollHeight < window.innerHeight + 75) first_layout = true ww = window.innerWidth @@ -149,7 +149,7 @@ class PagedDisplay # current page (when cols_per_screen == 1). Similarly img elements # with height=100% overflow the first column has_svg = document.getElementsByTagName('svg').length > 0 - only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 2 and document.getElementsByTagName('p').length < 2 + only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 3 and document.getElementsByTagName('p').length < 2 this.is_full_screen_layout = (only_img or has_svg) and single_screen and document.body.scrollWidth > document.body.clientWidth this.in_paged_mode = true From 0e8b5d6bb4cc70c8e553086ba74607568d80f2c3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 12:09:18 +0530 Subject: [PATCH 02/28] Updated various Romanian news sources --- recipes/catavencu.recipe | 16 ++--- recipes/dilemaveche.recipe | 112 ++++++++++++++--------------------- recipes/timesnewroman.recipe | 8 ++- 3 files changed, 59 insertions(+), 77 deletions(-) diff --git a/recipes/catavencu.recipe b/recipes/catavencu.recipe index db4057cd6d..d5ba7593e2 100644 --- a/recipes/catavencu.recipe +++ b/recipes/catavencu.recipe @@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class AcademiaCatavencu(BasicNewsRecipe): title = u'Academia Ca\u0163avencu' __author__ = u'Silviu Cotoar\u0103' - description = 'Tagma cum laude' + description = 'Academia Catavencu. Pamflete!' publisher = u'Ca\u0163avencu' oldest_article = 5 language = 'ro' @@ -21,7 +21,7 @@ class AcademiaCatavencu(BasicNewsRecipe): use_embedded_content = False category = 'Ziare' encoding = 'utf-8' - cover_url = 'http://www.academiacatavencu.info/images/logo.png' + cover_url = 'http://www.inpolitics.ro/Uploads/Articles/academia_catavencu.jpg' conversion_options = { 'comments' : description @@ -31,21 +31,21 @@ class AcademiaCatavencu(BasicNewsRecipe): } keep_only_tags = [ - dict(name='h1', attrs={'class':'art_title'}), - dict(name='div', attrs={'class':'art_text'}) + dict(name='h1', attrs={'class':'entry-title'}), + dict(name='div', attrs={'class':'entry-content'}) ] remove_tags = [ - dict(name='div', attrs={'class':['desp_m']}) - , dict(name='div', attrs={'id':['tags']}) + dict(name='div', attrs={'class':['mr_social_sharing_wrapper']}) + , dict(name='div', attrs={'id':['fb_share_1']}) ] remove_tags_after = [ - dict(name='div', attrs={'class':['desp_m']}) + dict(name='div', attrs={'id':['fb_share_1']}) ] feeds = [ - (u'Feeds', u'http://www.academiacatavencu.info/rss.xml') + (u'Feeds', u'http://www.academiacatavencu.info/feed') ] def preprocess_html(self, soup): diff --git a/recipes/dilemaveche.recipe b/recipes/dilemaveche.recipe index 8ba75c4123..72920600f7 100644 --- a/recipes/dilemaveche.recipe +++ b/recipes/dilemaveche.recipe @@ -1,71 +1,51 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +dilemaveche.ro +''' + from calibre.web.feeds.news import BasicNewsRecipe class DilemaVeche(BasicNewsRecipe): - title = u'Dilema Veche' # apare vinerea, mai pe dupa-masa,depinde de Luiza cred (care se semneaza ca fiind creatorul fiecarui articol in feed-ul RSS) - __author__ = 'song2' # inspirat din scriptul pentru Le Monde. Inspired from the Le Monde script - description = '"Sint vechi, domnule!" (I.L. Caragiale)' - publisher = 'Adevarul Holding' - oldest_article = 7 - max_articles_per_feed = 200 - encoding = 'utf8' - language = 'ro' - masthead_url = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' - publication_type = 'magazine' - feeds = [ - ('Editoriale si opinii - Situatiunea', 'http://www.dilemaveche.ro/taxonomy/term/37/0/feed'), - ('Editoriale si opinii - Pe ce lume traim', 'http://www.dilemaveche.ro/taxonomy/term/38/0/feed'), - ('Editoriale si opinii - Bordeie si obiceie', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'), - ('Editoriale si opinii - Talc Show', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'), - ('Tema saptamanii', 'http://www.dilemaveche.ro/taxonomy/term/19/0/feed'), - ('La zi in cultura - Dilema va recomanda', 'http://www.dilemaveche.ro/taxonomy/term/58/0/feed'), - ('La zi in cultura - Carte', 'http://www.dilemaveche.ro/taxonomy/term/14/0/feed'), - ('La zi in cultura - Film', 'http://www.dilemaveche.ro/taxonomy/term/13/0/feed'), - ('La zi in cultura - Muzica', 'http://www.dilemaveche.ro/taxonomy/term/1341/0/feed'), - ('La zi in cultura - Arte performative', 'http://www.dilemaveche.ro/taxonomy/term/1342/0/feed'), - ('La zi in cultura - Arte vizuale', 'http://www.dilemaveche.ro/taxonomy/term/1512/0/feed'), - ('Societate - Ieri cu vedere spre azi', 'http://www.dilemaveche.ro/taxonomy/term/15/0/feed'), - ('Societate - Din polul opus', 'http://www.dilemaveche.ro/taxonomy/term/41/0/feed'), - ('Societate - Mass comedia', 'http://www.dilemaveche.ro/taxonomy/term/43/0/feed'), - ('Societate - La singular si la plural', 'http://www.dilemaveche.ro/taxonomy/term/42/0/feed'), - ('Oameni si idei - Educatie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'), - ('Oameni si idei - Polemici si dezbateri', 'http://www.dilemaveche.ro/taxonomy/term/48/0/feed'), - ('Oameni si idei - Stiinta si tehnologie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'), - ('Dileme on-line', 'http://www.dilemaveche.ro/taxonomy/term/005/0/feed') - ] - remove_tags_before = dict(name='div',attrs={'class':'spacer_10'}) - remove_tags = [ - dict(name='div', attrs={'class':'art_related_left'}), - dict(name='div', attrs={'class':'controale'}), - dict(name='div', attrs={'class':'simple_overlay'}), - ] - remove_tags_after = [dict(id='facebookLike')] - remove_javascript = True + title = u'Dilema Veche' + __author__ = u'Silviu Cotoar\u0103' + description = 'Sint vechi, domnule! (I.L. Caragiale)' + publisher = u'Adev\u0103rul Holding' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 no_stylesheets = True - remove_empty_feeds = True - extra_css = """ - body{font-family: Georgia,Times,serif } - img{margin-bottom: 0.4em; display:block} - """ - def get_cover_url(self): - cover_url = None - soup = self.index_to_soup('http://dilemaveche.ro') - link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'}) - if link_item and link_item.a: - cover_url = link_item.a['href'] - br = BasicNewsRecipe.get_browser() - try: - br.open(cover_url) - except: #daca nu gaseste pdf-ul - self.log("\nPDF indisponibil") - link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'}) - if link_item and link_item.img: - cover_url = link_item.img['src'] - br = BasicNewsRecipe.get_browser() - try: - br.open(cover_url) - except: #daca nu gaseste nici imaginea mica mica - print('Mama lor de nenorociti! nu este nici pdf nici imagine') - cover_url ='http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' - return cover_url - cover_margins = (10, 15, '#ffffff') + use_embedded_content = False + category = 'Ziare' + encoding = 'utf-8' + cover_url = 'http://dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'c_left_column'}) + ] + + remove_tags = [ + dict(name='div', attrs={'id':['adshop_widget_428x60']}) , + dict(name='div', attrs={'id':['gallery']}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':['adshop_widget_428x60']}) + ] + + feeds = [ + (u'Feeds', u'http://dilemaveche.ro/rss.xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/timesnewroman.recipe b/recipes/timesnewroman.recipe index 12672aa888..79c2a9628a 100644 --- a/recipes/timesnewroman.recipe +++ b/recipes/timesnewroman.recipe @@ -36,12 +36,14 @@ class TimesNewRoman(BasicNewsRecipe): remove_tags = [ dict(name='p', attrs={'class':['articleinfo']}) - , dict(name='div',attrs={'class':['vergefacebooklike']}) - , dict(name='div', attrs={'class':'cleared'}) + , dict(name='div', attrs={'class':['shareTools']}) + , dict(name='div', attrs={'class':'fb_iframe_widget'}) + , dict(name='div', attrs={'id':'jc'}) ] remove_tags_after = [ - dict(name='div', attrs={'class':'cleared'}) + dict(name='div', attrs={'class':'fb_iframe_widget'}), + dict(name='div', attrs={'id':'jc'}) ] feeds = [ From ed05dbaa2d9696debf0d596720b6fb1e2694a8d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 13:04:51 +0530 Subject: [PATCH 03/28] Android driver: Add an extra customization option to configure the directory to which eboks are sent on the storage cards. Fixes #1045045 (Android save-to directory list honored only for main memory) --- src/calibre/devices/android/driver.py | 44 +++++++++++++++++++-------- src/calibre/devices/usbms/device.py | 30 ++++++++++-------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 9e8aa5fe17..efe979158d 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -186,10 +186,15 @@ class ANDROID(USBMS): } EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books', 'sdcard/ebooks'] - EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' - 'send e-books to on the device. The first one that exists will ' + EXTRA_CUSTOMIZATION_MESSAGE = [_('Comma separated list of directories to ' + 'send e-books to on the device\'s main memory. The first one that exists will ' + 'be used'), + _('Comma separated list of directories to ' + 'send e-books to on the device\'s storage cards. The first one that exists will ' 'be used') - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN) + ] + + EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(EBOOK_DIR_MAIN), ''] VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', @@ -237,23 +242,35 @@ class ANDROID(USBMS): def post_open_callback(self): opts = self.settings() - dirs = opts.extra_customization - if not dirs: - dirs = self.EBOOK_DIR_MAIN - else: - dirs = [x.strip() for x in dirs.split(',')] - self.EBOOK_DIR_MAIN = dirs + opts = opts.extra_customization + if not opts: + opts = [self.EBOOK_DIR_MAIN, ''] + + def strtolist(x): + if isinstance(x, basestring): + x = [y.strip() for y in x.split(',')] + return x or [] + + opts = [strtolist(x) for x in opts] + self._android_main_ebook_dir = opts[0] + self._android_card_ebook_dir = opts[1] def get_main_ebook_dir(self, for_upload=False): - dirs = self.EBOOK_DIR_MAIN + dirs = self._android_main_ebook_dir if not for_upload: def aldiko_tweak(x): return 'eBooks' if x == 'eBooks/import' else x - if isinstance(dirs, basestring): - dirs = [dirs] dirs = list(map(aldiko_tweak, dirs)) return dirs + def get_carda_ebook_dir(self, for_upload=False): + if not for_upload: + return '' + return self._android_card_ebook_dir + + def get_cardb_ebook_dir(self, for_upload=False): + return self.get_carda_ebook_dir() + def windows_sort_drives(self, drives): try: vid, pid, bcd = self.device_being_opened[:3] @@ -271,7 +288,8 @@ class ANDROID(USBMS): proxy = cls._configProxy() proxy['format_map'] = ['mobi', 'azw', 'azw1', 'azw4', 'pdf'] proxy['use_subdirs'] = False - proxy['extra_customization'] = ','.join(['kindle']+cls.EBOOK_DIR_MAIN) + proxy['extra_customization'] = [ + ','.join(['kindle']+cls.EBOOK_DIR_MAIN), ''] @classmethod def configure_for_generic_epub_app(cls): diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 4d4b198de0..025a7e2d95 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -991,24 +991,28 @@ class Device(DeviceConfig, DevicePlugin): elif on_card and on_card not in ('carda', 'cardb'): raise DeviceError(_('Selected slot: %s is not supported.') % on_card) - if on_card == 'carda': - path = os.path.join(self._card_a_prefix, - *(self.get_carda_ebook_dir(for_upload=True).split('/'))) - elif on_card == 'cardb': - path = os.path.join(self._card_b_prefix, - *(self.EBOOK_DIR_CARD_B.split('/'))) - else: - candidates = self.get_main_ebook_dir(for_upload=True) + def get_dest_dir(prefix, candidates): if isinstance(candidates, basestring): candidates = [candidates] + if not candidates: + candidates = [''] candidates = [ - ((os.path.join(self._main_prefix, *(x.split('/')))) if x else - self._main_prefix) for x - in candidates] + ((os.path.join(prefix, *(x.split('/')))) if x else prefix) + for x in candidates] existing = [x for x in candidates if os.path.exists(x)] if not existing: - existing = candidates[:1] - path = existing[0] + existing = candidates + return existing[0] + + if on_card == 'carda': + candidates = self.get_carda_ebook_dir(for_upload=True) + path = get_dest_dir(self._carda_prefix, candidates) + elif on_card == 'cardb': + candidates = self.get_cardb_ebook_dir(for_upload=True) + path = get_dest_dir(self._cardb_prefix, candidates) + else: + candidates = self.get_main_ebook_dir(for_upload=True) + path = get_dest_dir(self._main_prefix, candidates) def get_size(obj): path = getattr(obj, 'name', obj) From f9dec96a168cec5f8cce4d32e85a45d78ba31a0c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 13:08:34 +0530 Subject: [PATCH 04/28] ... --- src/calibre/gui2/device_drivers/configwidget.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index b47a80b6ad..9624efac96 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -89,6 +89,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setBuddy(self.opt_extra_customization[i]) l.setWordWrap(True) self.opt_extra_customization[i].setText(settings.extra_customization[i]) + self.opt_extra_customization[i].setCursorPosition(0) self.extra_layout.addWidget(l, row_func(i, 0), col_func(i)) self.extra_layout.addWidget(self.opt_extra_customization[i], row_func(i, 1), col_func(i)) @@ -101,6 +102,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setWordWrap(True) if settings.extra_customization: self.opt_extra_customization.setText(settings.extra_customization) + self.opt_extra_customization.setCursorPosition(0) self.opt_extra_customization.setCursorPosition(0) self.extra_layout.addWidget(l, 0, 0) self.extra_layout.addWidget(self.opt_extra_customization, 1, 0) From bc7de03717b2308916fe988dd38108f1e3ad81f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 13:44:24 +0530 Subject: [PATCH 05/28] Fix getting device GUI name in the GUI to not use the classmethod, as some drivers (MTP) require it to be an instance method. --- src/calibre/gui2/device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 98e42f4178..8eb4df3fbd 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -977,7 +977,7 @@ class DeviceMixin(object): # {{{ self.set_default_thumbnail(\ self.device_manager.device.THUMBNAIL_HEIGHT) self.status_bar.show_message(_('Device: ')+\ - self.device_manager.device.__class__.get_gui_name()+\ + self.device_manager.device.get_gui_name()+\ _(' detected.'), 3000) self.device_connected = device_kind self.library_view.set_device_connected(self.device_connected) From 4016e852d87da755e3664c778523fc49ece1f031 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 13:58:43 +0530 Subject: [PATCH 06/28] ... --- src/calibre/devices/mtp/driver.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 8f8f4d119b..31679062d2 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -32,6 +32,7 @@ class MTP_DEVICE(BASE): CAN_SET_METADATA = [] BACKLOADING_ERROR_MESSAGE = None MANAGES_DEVICE_PRESENCE = True + FORMATS = ['epub', 'azw3', 'mobi', 'pdf'] def open(self, devices, library_uuid): self.current_library_uuid = library_uuid @@ -225,6 +226,8 @@ class MTP_DEVICE(BASE): return ans # }}} + # Sending files to the device {{{ + def create_upload_path(self, path, mdata, fname): from calibre.devices import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize @@ -237,6 +240,19 @@ class MTP_DEVICE(BASE): ) return tuple(x.lower() for x in filepath.split('/')) + # }}} + + # Settings {{{ + @classmethod + def settings(self): + # TODO: Implement this + class Opts(object): + def __init__(s): + s.format_map = self.FORMATS + return Opts() + + # }}} + if __name__ == '__main__': dev = MTP_DEVICE(None) dev.startup() From e7e3b86573c7c40c9dce2479c53335c3a1945ffa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 14:40:31 +0530 Subject: [PATCH 07/28] Refactor out sanity_check as a utility method and re-organize utility methods into their own module --- src/calibre/devices/__init__.py | 108 +------------------- src/calibre/devices/mtp/base.py | 2 +- src/calibre/devices/mtp/driver.py | 14 ++- src/calibre/devices/usbms/device.py | 34 +------ src/calibre/devices/usbms/driver.py | 2 +- src/calibre/devices/utils.py | 148 ++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 140 deletions(-) create mode 100644 src/calibre/devices/utils.py diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 89d0e4e026..2c1d628566 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint, operator, re, os +import sys, time, pprint, operator from functools import partial from StringIO import StringIO @@ -27,112 +27,6 @@ def strftime(epoch, zone=time.gmtime): src[2] = INVERSE_MONTH_MAP[int(src[2])] return ' '.join(src) -def build_template_regexp(template): - from calibre import prints - - def replfunc(match, seen=None): - v = match.group(1) - if v in ['authors', 'author_sort']: - v = 'author' - if v in ('title', 'series', 'series_index', 'isbn', 'author'): - if v not in seen: - seen.add(v) - return '(?P<' + v + '>.+?)' - return '(.+?)' - s = set() - f = partial(replfunc, seen=s) - - try: - template = template.rpartition('/')[2] - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - except: - prints(u'Failed to parse template: %r'%template) - template = u'{title} - {authors}' - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - -def create_upload_path(mdata, fname, template, sanitize, - prefix_path='', - path_type=os.path, - maxlen=250, - use_subdirs=True, - news_in_folder=True, - filename_callback=lambda x, y:x, - sanitize_path_components=lambda x: x - ): - from calibre.library.save_to_disk import get_components, config - from calibre.utils.filenames import shorten_components_to - - special_tag = None - if mdata.tags: - for t in mdata.tags: - if t.startswith(_('News')) or t.startswith('/'): - special_tag = t - break - - if mdata.tags and _('News') in mdata.tags: - try: - p = mdata.pubdate - date = (p.year, p.month, p.day) - except: - today = time.localtime() - date = (today[0], today[1], today[2]) - template = u"{title}_%d-%d-%d" % date - - fname = sanitize(fname) - ext = path_type.splitext(fname)[1] - - opts = config().parse() - if not isinstance(template, unicode): - template = template.decode('utf-8') - app_id = str(getattr(mdata, 'application_id', '')) - id_ = mdata.get('id', fname) - extra_components = get_components(template, mdata, id_, - timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) - if not extra_components: - extra_components.append(sanitize(filename_callback(fname, - mdata))) - else: - extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) - - if extra_components[-1] and extra_components[-1][0] in ('.', '_'): - extra_components[-1] = 'x' + extra_components[-1][1:] - - if special_tag is not None: - name = extra_components[-1] - extra_components = [] - tag = special_tag - if tag.startswith(_('News')): - if news_in_folder: - extra_components.append('News') - else: - for c in tag.split('/'): - c = sanitize(c) - if not c: continue - extra_components.append(c) - extra_components.append(name) - - if not use_subdirs: - extra_components = extra_components[-1:] - - def remove_trailing_periods(x): - ans = x - while ans.endswith('.'): - ans = ans[:-1].strip() - if not ans: - ans = 'x' - return ans - - extra_components = list(map(remove_trailing_periods, extra_components)) - components = shorten_components_to(maxlen - len(prefix_path), extra_components) - components = sanitize_path_components(components) - if prefix_path: - filepath = path_type.join(prefix_path, *components) - else: - filepath = path_type.join(*components) - - return filepath - - def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 516b68ae1e..a5922a4d21 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -55,7 +55,7 @@ class MTPDeviceBase(DevicePlugin): return False def build_template_regexp(self): - from calibre.devices import build_template_regexp + from calibre.devices.utils import build_template_regexp return build_template_regexp(self.save_template) @property diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 31679062d2..2d414e7e5a 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -33,9 +33,11 @@ class MTP_DEVICE(BASE): BACKLOADING_ERROR_MESSAGE = None MANAGES_DEVICE_PRESENCE = True FORMATS = ['epub', 'azw3', 'mobi', 'pdf'] + DEVICE_PLUGBOARD_NAME = 'MTP_DEVICE' def open(self, devices, library_uuid): self.current_library_uuid = library_uuid + self.plugboards = self.plugboard_func = None BASE.open(self, devices, library_uuid) # Device information {{{ @@ -228,8 +230,12 @@ class MTP_DEVICE(BASE): # Sending files to the device {{{ + def set_plugboards(self, plugboards, pb_func): + self.plugboards = plugboards + self.plugboard_func = pb_func + def create_upload_path(self, path, mdata, fname): - from calibre.devices import create_upload_path + from calibre.devices.utils import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize filepath = create_upload_path(mdata, fname, self.save_template, sanitize, prefix_path=path, @@ -240,6 +246,12 @@ class MTP_DEVICE(BASE): ) return tuple(x.lower() for x in filepath.split('/')) + def upload_books(self, files, names, on_card=None, end_session=True, + metadata=None): + from calibre.devices.utils import sanity_check + sanity_check(on_card, files, self.card_prefix(), self.free_space()) + raise NotImplementedError() + # }}} # Settings {{{ diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 025a7e2d95..795a22888f 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -15,8 +15,7 @@ import os, subprocess, time, re, sys, glob from itertools import repeat from calibre.devices.interface import DevicePlugin -from calibre.devices.errors import (DeviceError, FreeSpaceError, - WrongDestinationError) +from calibre.devices.errors import DeviceError from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins from calibre.utils.filenames import ascii_filename as sanitize @@ -976,20 +975,8 @@ class Device(DeviceConfig, DevicePlugin): return self.EBOOK_DIR_CARD_A def _sanity_check(self, on_card, files): - if on_card == 'carda' and not self._card_a_prefix: - raise WrongDestinationError(_( - 'The reader has no storage card %s. You may have changed ' - 'the default send to device action. Right click on the send ' - 'to device button and reset the default action to be ' - '"Send to main memory".')%'A') - elif on_card == 'cardb' and not self._card_b_prefix: - raise WrongDestinationError(_( - 'The reader has no storage card %s. You may have changed ' - 'the default send to device action. Right click on the send ' - 'to device button and reset the default action to be ' - '"Send to main memory".')%'B') - elif on_card and on_card not in ('carda', 'cardb'): - raise DeviceError(_('Selected slot: %s is not supported.') % on_card) + from calibre.devices.utils import sanity_check + sanity_check(on_card, files, self.card_prefix(), self.free_space()) def get_dest_dir(prefix, candidates): if isinstance(candidates, basestring): @@ -1014,19 +1001,6 @@ class Device(DeviceConfig, DevicePlugin): candidates = self.get_main_ebook_dir(for_upload=True) path = get_dest_dir(self._main_prefix, candidates) - def get_size(obj): - path = getattr(obj, 'name', obj) - return os.path.getsize(path) - - sizes = [get_size(f) for f in files] - size = sum(sizes) - - if not on_card and size > self.free_space()[0] - 2*1024*1024: - raise FreeSpaceError(_("There is insufficient free space in main memory")) - if on_card == 'carda' and size > self.free_space()[1] - 1024*1024: - raise FreeSpaceError(_("There is insufficient free space on the storage card")) - if on_card == 'cardb' and size > self.free_space()[2] - 1024*1024: - raise FreeSpaceError(_("There is insufficient free space on the storage card")) return path def filename_callback(self, default, mi): @@ -1056,7 +1030,7 @@ class Device(DeviceConfig, DevicePlugin): pass def create_upload_path(self, path, mdata, fname, create_dirs=True): - from calibre.devices import create_upload_path + from calibre.devices.utils import create_upload_path settings = self.settings() filepath = create_upload_path(mdata, fname, self.save_template(), sanitize, prefix_path=os.path.abspath(path), diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index f6c7556fd8..5f6bdd9402 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -404,7 +404,7 @@ class USBMS(CLI, Device): @classmethod def build_template_regexp(cls): - from calibre.devices import build_template_regexp + from calibre.devices.utils import build_template_regexp return build_template_regexp(cls.save_template()) @classmethod diff --git a/src/calibre/devices/utils.py b/src/calibre/devices/utils.py new file mode 100644 index 0000000000..114e7e4e13 --- /dev/null +++ b/src/calibre/devices/utils.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, re +from functools import partial + +from calibre.devices.errors import DeviceError, WrongDestinationError, FreeSpaceError + +def sanity_check(on_card, files, card_prefixes, free_space): + if on_card == 'carda' and not card_prefixes[0]: + raise WrongDestinationError(_( + 'The reader has no storage card %s. You may have changed ' + 'the default send to device action. Right click on the send ' + 'to device button and reset the default action to be ' + '"Send to main memory".')%'A') + elif on_card == 'cardb' and not card_prefixes[1]: + raise WrongDestinationError(_( + 'The reader has no storage card %s. You may have changed ' + 'the default send to device action. Right click on the send ' + 'to device button and reset the default action to be ' + '"Send to main memory".')%'B') + elif on_card and on_card not in ('carda', 'cardb'): + raise DeviceError(_('Selected slot: %s is not supported.') % on_card) + + size = 0 + for f in files: + size += os.path.getsize(getattr(f, 'name', f)) + + if not on_card and size > free_space[0] - 2*1024*1024: + raise FreeSpaceError(_("There is insufficient free space in main memory")) + if on_card == 'carda' and size > free_space[1] - 1024*1024: + raise FreeSpaceError(_("There is insufficient free space on the storage card")) + if on_card == 'cardb' and size > free_space[2] - 1024*1024: + raise FreeSpaceError(_("There is insufficient free space on the storage card")) + +def build_template_regexp(template): + from calibre import prints + + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + + try: + template = template.rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + +def create_upload_path(mdata, fname, template, sanitize, + prefix_path='', + path_type=os.path, + maxlen=250, + use_subdirs=True, + news_in_folder=True, + filename_callback=lambda x, y:x, + sanitize_path_components=lambda x: x + ): + from calibre.library.save_to_disk import get_components, config + from calibre.utils.filenames import shorten_components_to + + special_tag = None + if mdata.tags: + for t in mdata.tags: + if t.startswith(_('News')) or t.startswith('/'): + special_tag = t + break + + if mdata.tags and _('News') in mdata.tags: + try: + p = mdata.pubdate + date = (p.year, p.month, p.day) + except: + today = time.localtime() + date = (today[0], today[1], today[2]) + template = u"{title}_%d-%d-%d" % date + + fname = sanitize(fname) + ext = path_type.splitext(fname)[1] + + opts = config().parse() + if not isinstance(template, unicode): + template = template.decode('utf-8') + app_id = str(getattr(mdata, 'application_id', '')) + id_ = mdata.get('id', fname) + extra_components = get_components(template, mdata, id_, + timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) + if not extra_components: + extra_components.append(sanitize(filename_callback(fname, + mdata))) + else: + extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) + + if extra_components[-1] and extra_components[-1][0] in ('.', '_'): + extra_components[-1] = 'x' + extra_components[-1][1:] + + if special_tag is not None: + name = extra_components[-1] + extra_components = [] + tag = special_tag + if tag.startswith(_('News')): + if news_in_folder: + extra_components.append('News') + else: + for c in tag.split('/'): + c = sanitize(c) + if not c: continue + extra_components.append(c) + extra_components.append(name) + + if not use_subdirs: + extra_components = extra_components[-1:] + + def remove_trailing_periods(x): + ans = x + while ans.endswith('.'): + ans = ans[:-1].strip() + if not ans: + ans = 'x' + return ans + + extra_components = list(map(remove_trailing_periods, extra_components)) + components = shorten_components_to(maxlen - len(prefix_path), extra_components) + components = sanitize_path_components(components) + if prefix_path: + filepath = path_type.join(prefix_path, *components) + else: + filepath = path_type.join(*components) + + return filepath + + + From 598f3bd17c3caa3200b935edea12811437bae661 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 14:43:32 +0530 Subject: [PATCH 08/28] ... --- src/calibre/devices/mtp/driver.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 2d414e7e5a..2706cba32c 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -77,12 +77,7 @@ class MTP_DEVICE(BASE): return tuple( list(dinfo) + [self.driveinfo] ) def card_prefix(self, end_session=True): - ans = [None, None] - if self._carda_id is not None: - ans[0] = self.filesystem_cache.storage(self._carda_id).storage_prefix - if self._cardb_id is not None: - ans[1] = self.filesystem_cache.storage(self._cardb_id).storage_prefix - return tuple(ans) + return (self._carda_id, self._cardb_id) def set_driveinfo_name(self, location_code, name): sid = {'main':self._main_id, 'A':self._carda_id, From 66d01629c377b171c5261b9dca0ad90ee5686eed Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 14:45:59 +0530 Subject: [PATCH 09/28] ... --- src/calibre/devices/mtp/driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 2706cba32c..8bc2481205 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -35,9 +35,12 @@ class MTP_DEVICE(BASE): FORMATS = ['epub', 'azw3', 'mobi', 'pdf'] DEVICE_PLUGBOARD_NAME = 'MTP_DEVICE' + def __init__(self, *args, **kwargs): + BASE.__init__(self, *args, **kwargs) + self.plugboards = self.plugboard_func = None + def open(self, devices, library_uuid): self.current_library_uuid = library_uuid - self.plugboards = self.plugboard_func = None BASE.open(self, devices, library_uuid) # Device information {{{ From 0b02c1f593a3f0c6abc6f3660cce845f44b10395 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 14:59:31 +0530 Subject: [PATCH 10/28] ... --- src/calibre/devices/mtp/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index a5922a4d21..26523362ed 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -25,7 +25,7 @@ def synchronous(func): return synchronizer class MTPDeviceBase(DevicePlugin): - name = 'SmartDevice App Interface' + name = 'MTP Device Interface' gui_name = _('MTP Device') icon = I('devices/galaxy_s3.png') description = _('Communicate with MTP devices') From 7a1d05199e7813d17ce4f59f38f5ed0fa96f6870 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 15:01:35 +0530 Subject: [PATCH 11/28] ... --- src/calibre/devices/mtp/base.py | 5 +++-- src/calibre/devices/mtp/driver.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 26523362ed..bad027baa1 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -45,8 +45,9 @@ class MTPDeviceBase(DevicePlugin): def set_progress_reporter(self, report_progress): self.report_progress = report_progress - def get_gui_name(self): - return self.current_friendly_name or self.name + @classmethod + def get_gui_name(cls): + return getattr(cls, 'current_friendly_name', cls.gui_name) def is_usb_connected(self, devices_on_system, debug=False, only_presence=False): diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 8bc2481205..f36b068a30 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' import json, traceback, posixpath, importlib, os from io import BytesIO +from itertools import izip from calibre import prints from calibre.constants import iswindows, numeric_version @@ -244,10 +245,18 @@ class MTP_DEVICE(BASE): ) return tuple(x.lower() for x in filepath.split('/')) + def prefix_for_location(self, on_card): + # TODO: Implement this + return 'calibre' + def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): from calibre.devices.utils import sanity_check sanity_check(on_card, files, self.card_prefix(), self.free_space()) + prefix = self.prefix_for_location(on_card) + for infile, fname, mi in izip(files, names, metadata): + path = self.create_upload_path(prefix, mi, fname) + print (1111111, path) raise NotImplementedError() # }}} From 86726f44d8fcc25ee9a18e0991fedf7871f28c99 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 15:02:05 +0530 Subject: [PATCH 12/28] ... --- src/calibre/devices/mtp/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index f36b068a30..21419960ae 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -243,7 +243,7 @@ class MTP_DEVICE(BASE): use_subdirs = True, news_in_folder = self.NEWS_IN_FOLDER, ) - return tuple(x.lower() for x in filepath.split('/')) + return tuple(x for x in filepath.split('/')) def prefix_for_location(self, on_card): # TODO: Implement this From 4d7ed1b4c6e696d5888dd1fef40aec13c9be1fe4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 17:23:12 +0530 Subject: [PATCH 13/28] ... --- src/calibre/devices/mtp/filesystem_cache.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index 216e06031f..da0445962b 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -80,6 +80,10 @@ class FileOrFolder(object): __str__ = __repr__ __unicode__ = __repr__ + @property + def empty(self): + return not self.files and not self.folders + @property def id_map(self): return self.fs_cache().id_map @@ -217,6 +221,8 @@ class FilesystemCache(object): def iterebooks(self, storage_id): for x in self.id_map.itervalues(): if x.storage_id == storage_id and x.is_ebook: + if x.parent_id == storage_id and x.name.lower().endswith('.txt'): + continue # Ignore .txt files in the root yield x def resolve_mtp_id_path(self, path): From 92c46d2dedc0b335d6b53b17c7fa558672b4859f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 17:36:45 +0530 Subject: [PATCH 14/28] MTP: Sending books to device and deleting books from device implemented --- src/calibre/devices/mtp/books.py | 27 ++++++ src/calibre/devices/mtp/driver.py | 108 +++++++++++++++++++++- src/calibre/devices/mtp/unix/driver.py | 1 + src/calibre/devices/mtp/windows/driver.py | 1 + src/calibre/gui2/device.py | 8 +- 5 files changed, 141 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py index 73e483f19e..a72fc1f84e 100644 --- a/src/calibre/devices/mtp/books.py +++ b/src/calibre/devices/mtp/books.py @@ -22,6 +22,22 @@ class BookList(BL): def supports_collections(self): return False + def add_book(self, book, replace_metadata=True): + try: + b = self.index(book) + except (ValueError, IndexError): + b = None + if b is None: + self.append(book) + return book + if replace_metadata: + self[b].smart_update(book, replace_metadata=True) + return self[b] + return None + + def remove_book(self, book): + self.remove(book) + class Book(Metadata): def __init__(self, storage_id, lpath, other=None): @@ -36,6 +52,17 @@ class Book(Metadata): return (self.storage_id == mtp_file.storage_id and self.mtp_relpath == mtp_file.mtp_relpath) + def __eq__(self, other): + return (isinstance(other, self.__class__) and (self.storage_id == + other.storage_id and self.mtp_relpath == other.mtp_relpath)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.storage_id, self.mtp_relpath)) + + class JSONCodec(JsonCodec): pass diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 21419960ae..68539f334b 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -191,6 +191,7 @@ class MTP_DEVICE(BASE): self.put_file(storage, self.METADATA_CACHE, stream, size) def sync_booklists(self, booklists, end_session=True): + debug('sync_booklists() called') for bl in booklists: if getattr(bl, 'storage_id', None) is None: continue @@ -198,6 +199,7 @@ class MTP_DEVICE(BASE): if storage is None: continue self.write_metadata_cache(storage, bl) + debug('sync_booklists() ended') # }}} @@ -249,15 +251,117 @@ class MTP_DEVICE(BASE): # TODO: Implement this return 'calibre' + def ensure_parent(self, storage, path): + parent = storage + pos = list(path)[:-1] + while pos: + name = pos[0] + pos = pos[1:] + parent = self.create_folder(parent, name) + return parent + def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): + debug('upload_books() called') from calibre.devices.utils import sanity_check sanity_check(on_card, files, self.card_prefix(), self.free_space()) prefix = self.prefix_for_location(on_card) + sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(on_card, + self._main_id) + bl_idx = {'carda':1, 'cardb':2}.get(on_card, 0) + storage = self.filesystem_cache.storage(sid) + + ans = [] + self.report_progress(0, _('Transferring books to device...')) + i, total = 0, len(files) + for infile, fname, mi in izip(files, names, metadata): path = self.create_upload_path(prefix, mi, fname) - print (1111111, path) - raise NotImplementedError() + parent = self.ensure_parent(storage, path) + if hasattr(infile, 'read'): + pos = infile.tell() + infile.seek(0, 2) + sz = infile.tell() + infile.seek(pos) + stream = infile + close = False + else: + sz = os.path.getsize(infile) + stream = lopen(infile, 'rb') + close = True + try: + mtp_file = self.put_file(parent, path[-1], stream, sz) + finally: + if close: + stream.close() + ans.append((mtp_file, bl_idx)) + i += 1 + self.report_progress(i/total, _('Transferred %s to device')%mi.title) + + self.report_progress(1, _('Transfer to device finished...')) + debug('upload_books() ended') + return ans + + def add_books_to_metadata(self, mtp_files, metadata, booklists): + debug('add_books_to_metadata() called') + from calibre.devices.mtp.books import Book + + i, total = 0, len(mtp_files) + self.report_progress(0, _('Adding books to device metadata listing...')) + for x, mi in izip(mtp_files, metadata): + mtp_file, bl_idx = x + bl = booklists[bl_idx] + book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath), + other=mi) + book = bl.add_book(book, replace_metadata=True) + if book is not None: + book.size = mtp_file.size + book.datetime = mtp_file.last_modified.timetuple() + book.path = mtp_file.mtp_id_path + i += 1 + self.report_progress(i/total, _('Added %s')%mi.title) + + self.report_progress(1, _('Adding complete')) + debug('add_books_to_metadata() ended') + + # }}} + + # Removing books from the device {{{ + def recursive_delete(self, obj): + parent = self.delete_file_or_folder(obj) + if parent.empty and parent.can_delete and not parent.is_system: + try: + self.recursive_delete(parent) + except: + prints('Failed to delete parent: %s, ignoring'%( + '/'.join(parent.full_path))) + + def delete_books(self, paths, end_session=True): + self.report_progress(0, _('Deleting books from device...')) + + for i, path in enumerate(paths): + f = self.filesystem_cache.resolve_mtp_id_path(path) + self.recursive_delete(f) + self.report_progress((i+1) / float(len(paths)), + _('Deleted %s')%path) + self.report_progress(1, _('All books deleted')) + + def remove_books_from_metadata(self, paths, booklists): + self.report_progress(0, _('Removing books from metadata')) + class NextPath(Exception): pass + + for i, path in enumerate(paths): + try: + for bl in booklists: + for book in bl: + if book.path == path: + bl.remove_book(book) + raise NextPath('') + except NextPath: + pass + self.report_progress((i+1)/len(paths), _('Removed %s')%path) + + self.report_progress(1, _('All books removed')) # }}} diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 2f215f6353..b59ec22110 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -306,6 +306,7 @@ class MTP_DEVICE(MTPDeviceBase): raise DeviceError('Failed to delete %s with error: %s'% (obj.full_path, self.format_errorstack(errs))) parent.remove_child(obj) + return parent def develop(): from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 7c15797ef6..63fedfaf66 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -338,6 +338,7 @@ class MTP_DEVICE(MTPDeviceBase): parent = obj.parent self.dev.delete_object(obj.object_id) parent.remove_child(obj) + return parent @same_thread def put_file(self, parent, name, stream, size, callback=None, replace=True): diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 8eb4df3fbd..4cc4c0fb5f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1495,8 +1495,12 @@ class DeviceMixin(object): # {{{ self.device_job_exception(job) return - self.device_manager.add_books_to_metadata(job.result, - metadata, self.booklists()) + try: + self.device_manager.add_books_to_metadata(job.result, + metadata, self.booklists()) + except: + traceback.print_exc() + raise books_to_be_deleted = [] if memory and memory[1]: From 4fcbf630e59d8fd5cb50857f687b89f3c5d6bcf2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 17:56:15 +0530 Subject: [PATCH 15/28] MTP: Print last modified date when dumping filesystem --- src/calibre/devices/mtp/driver.py | 2 +- src/calibre/devices/mtp/filesystem_cache.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 68539f334b..d716d15de5 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -389,7 +389,7 @@ if __name__ == '__main__': raise ValueError('Failed to detect MTP device') dev.set_progress_reporter(prints) dev.open(cd, None) - dev.books() + dev.filesystem_cache.dump() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index da0445962b..d8c5170e59 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -40,6 +40,7 @@ class FileOrFolder(object): self.last_modified = datetime.fromtimestamp(md, local_tz) except: self.last_modified = datetime.fromtimestamp(0, local_tz) + self.last_mod_string = self.last_modified.strftime('%Y/%m/%d %H:%M') self.last_modified = as_utc(self.last_modified) if self.storage_id not in self.all_storage_ids: @@ -74,8 +75,8 @@ class FileOrFolder(object): datum = 'size=%s'%(self.size) if self.is_folder: datum = 'children=%s'%(len(self.files) + len(self.folders)) - return '%s(id=%s, storage_id=%s, %s, path=%s)'%(name, self.object_id, - self.storage_id, datum, path) + return '%s(id=%s, storage_id=%s, %s, path=%s, modified=%s)'%(name, self.object_id, + self.storage_id, datum, path, self.last_mod_string) __str__ = __repr__ __unicode__ = __repr__ @@ -127,6 +128,7 @@ class FileOrFolder(object): c = '+' if self.is_folder else '-' data = ('%s children'%(sum(map(len, (self.files, self.folders)))) if self.is_folder else human_readable(self.size)) + data += ' modified=%s'%self.last_mod_string line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data) prints(line, file=out) for c in (self.folders, self.files): From 75f127168130f9a452aade714c8754d041b65b82 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 19:57:39 +0530 Subject: [PATCH 16/28] Fix #1045046 (Houston Chronicle news fetch failing) --- recipes/houston_chronicle.recipe | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/houston_chronicle.recipe b/recipes/houston_chronicle.recipe index b8171467ec..639d5c2042 100644 --- a/recipes/houston_chronicle.recipe +++ b/recipes/houston_chronicle.recipe @@ -7,18 +7,19 @@ class HoustonChronicle(BasicNewsRecipe): title = u'The Houston Chronicle' description = 'News from Houston, Texas' - __author__ = 'Kovid Goyal' + __author__ = 'Kovid Goyal' language = 'en' timefmt = ' [%a, %d %b, %Y]' no_stylesheets = True use_embedded_content = False remove_attributes = ['style'] + auto_cleanup = True oldest_article = 2.0 - keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or - 'hst-articletext' in x or 'hst-galleryitem' in x)} - remove_attributes = ['xmlns'] + #keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or + #'hst-articletext' in x or 'hst-galleryitem' in x)} + #remove_attributes = ['xmlns'] feeds = [ ('News', "http://www.chron.com/rss/feed/News-270.php"), @@ -37,3 +38,4 @@ class HoustonChronicle(BasicNewsRecipe): ] + From f36feac5baae59d291c1f7a1e19de17795fc70eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 21:38:08 +0530 Subject: [PATCH 17/28] ... --- session.vim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/session.vim b/session.vim index 1a94d6bf07..4b9dcb72c1 100644 --- a/session.vim +++ b/session.vim @@ -13,6 +13,8 @@ let g:syntastic_cpp_include_dirs = [ \] let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs +set wildignore+=resources/viewer/mathjax/** + fun! CalibreLog() " Setup buffers to edit the calibre changelog and version info prior to " making a release. From f07002fdd226fc735e93a6b5979ed0849e1cd89f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 3 Sep 2012 21:42:31 +0530 Subject: [PATCH 18/28] MTP: Fix last modified date not getting read correctly on windows --- setup/extensions.py | 2 +- src/calibre/devices/mtp/filesystem_cache.py | 5 ++++- .../mtp/windows/content_enumeration.cpp | 12 +++++++++-- src/calibre/devices/mtp/windows/remote.py | 20 +++++++++---------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/setup/extensions.py b/setup/extensions.py index f4ed22687b..f7d40ca72c 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -187,7 +187,7 @@ if iswindows: headers=[ 'calibre/devices/mtp/windows/global.h', ], - libraries=['ole32', 'portabledeviceguids', 'user32'], + libraries=['ole32', 'oleaut32', 'portabledeviceguids', 'user32'], # needs_ddk=True, cflags=['/X'] ), diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index d8c5170e59..8f4d20ae18 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -37,7 +37,10 @@ class FileOrFolder(object): self.size = entry.get('size', 0) md = entry.get('modified', 0) try: - self.last_modified = datetime.fromtimestamp(md, local_tz) + if isinstance(md, tuple): + self.last_modified = datetime(*(list(md)+[local_tz])) + else: + self.last_modified = datetime.fromtimestamp(md, local_tz) except: self.last_modified = datetime.fromtimestamp(0, local_tz) self.last_mod_string = self.last_modified.strftime('%Y/%m/%d %H:%M') diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 7186bbdcdb..580f77f9b0 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -84,11 +84,19 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { FLOAT val = 0; + SYSTEMTIME st; + unsigned int microseconds; PyObject *t; if (SUCCEEDED(properties->GetFloatValue(key, &val))) { - t = Py_BuildValue("d", (double)val); - if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + if (VariantTimeToSystemTime(val, &st)) { + microseconds = 1000 * st.wMilliseconds; + t = Py_BuildValue("H H H H H H I", (unsigned short)st.wYear, + (unsigned short)st.wMonth, (unsigned short)st.wDay, + (unsigned short)st.wHour, (unsigned short)st.wMinute, + (unsigned short)st.wSecond, microseconds); + if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + } } } diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index cbc23978d2..f1dfa92767 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -54,9 +54,9 @@ def main(): plugins._plugins['wpd'] = (wpd, '') sys.path.pop(0) - from calibre.devices.mtp.test import run - run() - return + # from calibre.devices.mtp.test import run + # run() + # return from calibre.devices.scanner import win_scanner from calibre.devices.mtp.windows.driver import MTP_DEVICE @@ -81,13 +81,13 @@ def main(): # print ('Fetching file: oFF (198214 bytes)') # stream = dev.get_file('oFF') # print ("Fetched size: ", stream.tell()) - size = 4 - stream = io.BytesIO(b'a'*size) - name = 'zzz-test-file.txt' - stream.seek(0) - f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size) - print ('Put file:', f) - # dev.filesystem_cache.dump() + # size = 4 + # stream = io.BytesIO(b'a'*size) + # name = 'zzz-test-file.txt' + # stream.seek(0) + # f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size) + # print ('Put file:', f) + dev.filesystem_cache.dump() finally: dev.shutdown() From 7fb0ef82561cc8f267f66c7ac3efac9481842346 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 4 Sep 2012 09:44:37 +0530 Subject: [PATCH 19/28] ... --- src/calibre/devices/mtp/windows/wpd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 15cdd51e22..867ce6bcee 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -120,14 +120,14 @@ wpd_enumerate_devices(PyObject *self, PyObject *args) { hresult_set_exc("Failed to get list of portable devices", hr); } + Py_BEGIN_ALLOW_THREADS; for (i = 0; i < num_of_devices; i++) { - Py_BEGIN_ALLOW_THREADS; CoTaskMemFree(pnp_device_ids[i]); - Py_END_ALLOW_THREADS; pnp_device_ids[i] = NULL; } free(pnp_device_ids); pnp_device_ids = NULL; + Py_END_ALLOW_THREADS; return Py_BuildValue("N", ans); } // }}} From 6ec107a92886635419c64e0bfa65a606a81032e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 4 Sep 2012 13:22:49 +0530 Subject: [PATCH 20/28] Switch to using psutil to measure memory consumption --- setup/installer/linux/freeze2.py | 3 +- setup/installer/windows/notes.rst | 9 ++ src/calibre/utils/mem.py | 191 ++---------------------------- 3 files changed, 24 insertions(+), 179 deletions(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 13c02cca12..0ed49c9fef 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -15,7 +15,8 @@ from setup import Command, modules, basenames, functions, __version__, \ SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py', - '_dbus_glib_bindings.so', 'netifaces.so'] + '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so', + '_psutil_linux.so', 'psutil'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index d0f6eb67ba..aa330095d0 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -360,6 +360,15 @@ Run python setup.py build cp build/lib.win32-2.7/netifaces.pyd /cygdrive/c/Python27/Lib/site-packages/ +psutil +-------- + +Download the source tarball + +Run + +Python setup.py build +cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/ calibre --------- diff --git a/src/calibre/utils/mem.py b/src/calibre/utils/mem.py index 4358ec7522..3bc1f29636 100644 --- a/src/calibre/utils/mem.py +++ b/src/calibre/utils/mem.py @@ -13,188 +13,23 @@ You can pass a number to memory and it will be subtracted from the returned value. ''' -import gc, os, re +import gc, os from calibre.constants import iswindows, islinux -if islinux: - # Taken, with thanks, from: - # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications - - def permute(args): - ret = [] - if args: - first = args.pop(0) - for y in permute(args): - for x in first: - ret.append(x + y) - else: - ret.append('') - return ret - - def parsed_groups(match, *types): - groups = match.groups() - assert len(groups) == len(types) - return tuple([type(group) for group, type in zip(groups, types)]) - - class VMA(dict): - def __init__(self, *args): - (self.start, self.end, self.perms, self.offset, - self.major, self.minor, self.inode, self.filename) = args - - def parse_smaps(pid): - with open('/proc/%s/smaps'%pid, 'r') as maps: - hex = lambda s: int(s, 16) - - ret = [] - header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) ' - r'(..):(..) (\d+) *(.*)$') - detail = re.compile(r'^(.*): +(\d+) kB') - for line in maps: - m = header.match(line) - if m: - vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str)) - ret.append(vma) - else: - m = detail.match(line) - if m: - k, v = parsed_groups(m, str, int) - assert k not in vma - vma[k] = v - else: - print 'unparseable line:', line - return ret - - perms = permute(['r-', 'w-', 'x-', 'ps']) - - def make_summary_dicts(vmas): - mapped = {} - anon = {} - for d in mapped, anon: - # per-perm - for k in perms: - d[k] = {} - d[k]['Size'] = 0 - for y in 'Shared', 'Private': - d[k][y] = {} - for z in 'Clean', 'Dirty': - d[k][y][z] = 0 - # totals - for y in 'Shared', 'Private': - d[y] = {} - for z in 'Clean', 'Dirty': - d[y][z] = 0 - - for vma in vmas: - if vma.major == '00' and vma.minor == '00': - d = anon - else: - d = mapped - for y in 'Shared', 'Private': - for z in 'Clean', 'Dirty': - d[vma.perms][y][z] += vma.get(y + '_' + z, 0) - d[y][z] += vma.get(y + '_' + z, 0) - d[vma.perms]['Size'] += vma.get('Size', 0) - return mapped, anon - - def values(d, args): - if args: - ret = () - first = args[0] - for k in first: - ret += values(d[k], args[1:]) - return ret - else: - return (d,) - - def print_summary(dicts_and_titles): - def desc(title, perms): - ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)', - ('Anonymous', 'rwxp'): 'Writable code (stack)', - ('Mapped', 'r-xp'): 'Code', - ('Mapped', 'rwxp'): 'Writable code (jump tables)', - ('Mapped', 'r--p'): 'Read-only data', - ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None) - if ret: - return ' -- ' + ret - else: - return '' - - for d, title in dicts_and_titles: - print title, 'memory:' - print ' Shared Private' - print ' Clean Dirty Clean Dirty' - for k in perms: - if d[k]['Size']: - print (' %s %7d %7d %7d %7d%s' - % ((k,) - + values(d[k], (('Shared', 'Private'), - ('Clean', 'Dirty'))) - + (desc(title, k),))) - print (' total %7d %7d %7d %7d' - % values(d, (('Shared', 'Private'), - ('Clean', 'Dirty')))) - - print ' ' + '-' * 40 - print (' total %7d %7d %7d %7d' - % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'), - ('Clean', 'Dirty'))) - for d, title in dicts_and_titles])))) - - def print_stats(pid=None): - if pid is None: - pid = os.getpid() - vmas = parse_smaps(pid) - mapped, anon = make_summary_dicts(vmas) - print_summary(((mapped, "Mapped"), (anon, "Anonymous"))) - - def linux_memory(since=0.0): - vmas = parse_smaps(os.getpid()) - mapped, anon = make_summary_dicts(vmas) - dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous")) - totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'), - ('Clean', 'Dirty'))) - for d, title in dicts_and_titles]))) - return (totals[-1]/1024.) - since - - memory = linux_memory - -elif iswindows: - import win32process - import win32con - import win32api - - # See http://msdn.microsoft.com/en-us/library/ms684877.aspx - # for details on the info returned by get_meminfo - - def get_handle(pid): - return win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0, - pid) - - def listprocesses(self): - for process in win32process.EnumProcesses(): - try: - han = get_handle(process) - procmeminfo = meminfo(han) - procmemusage = procmeminfo["WorkingSetSize"] - yield process, procmemusage - except: - pass - - def get_meminfo(pid): - han = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0, - pid) - return meminfo(han) - - def meminfo(handle): - return win32process.GetProcessMemoryInfo(handle) - - def win_memory(since=0.0): - info = meminfo(get_handle(os.getpid())) - return (info['WorkingSetSize']/1024.**2) - since - - memory = win_memory +def get_memory(): + 'Return memory usage in bytes' + import psutil + p = psutil.Process(os.getpid()) + mem = p.get_ext_memory_info() + attr = 'wset' if iswindows else 'data' if islinux else 'rss' + return getattr(mem, attr) +def memory(since=0.0): + 'Return memory used in MB. The value of since is subtracted from the used memory' + ans = get_memory() + ans /= float(1024**2) + return ans - since def gc_histogram(): """Returns per-class counts of existing objects.""" From 206af4c041a905f069835bc7c8bd46cd9848aa10 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 09:28:06 +0530 Subject: [PATCH 21/28] Device drivers: Ignore corrupted metadata.calibre, instead of raising an error --- src/calibre/devices/usbms/driver.py | 2 +- src/calibre/ebooks/metadata/book/json_codec.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 5f6bdd9402..94309e747e 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -166,7 +166,7 @@ class USBMS(CLI, Device): # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} - for idx,b in enumerate(bl): + for idx, b in enumerate(bl): bl_cache[b.lpath] = idx all_formats = self.formats_to_scan_for() diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index cc9b6f252d..e9cec8acc7 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -161,7 +161,9 @@ class JsonCodec(object): try: js = json.load(file_, encoding='utf-8') for item in js: - booklist.append(self.raw_to_book(item, book_class, prefix)) + entry = self.raw_to_book(item, book_class, prefix) + if entry is not None: + booklist.append(entry) except: print 'exception during JSON decode_from_file' traceback.print_exc() From 288e71de6f581a5fda511ff47de45528aa4e62b9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 09:33:11 +0530 Subject: [PATCH 22/28] ... --- recipes/scmp.recipe | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/recipes/scmp.recipe b/recipes/scmp.recipe index 1da7b9e1bc..a4f4bf497c 100644 --- a/recipes/scmp.recipe +++ b/recipes/scmp.recipe @@ -39,10 +39,10 @@ class SCMP(BasicNewsRecipe): #br.set_debug_responses(True) #br.set_debug_redirects(True) if self.username is not None and self.password is not None: - br.open('http://www.scmp.com/portal/site/SCMP/') - br.select_form(name='loginForm') - br['Login' ] = self.username - br['Password'] = self.password + br.open('http://www.scmp.com/') + br.select_form(nr=1) + br['name'] = self.username + br['pass'] = self.password br.submit() return br From 2d23f6d493b15bfc4e7d575c9383fbcccd58503e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 10:05:55 +0530 Subject: [PATCH 23/28] Update published signatures when re-uploading an installer --- setup/upload.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/setup/upload.py b/setup/upload.py index a73d0d2c31..f4e8706d1d 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -47,6 +47,21 @@ def installer_description(fname): return 'Calibre Portable' return 'Unknown file' +def upload_signatures(): + tdir = mkdtemp() + for installer in installers(): + if not os.path.exists(installer): + continue + with open(installer, 'rb') as f: + raw = f.read() + fingerprint = hashlib.sha512(raw).hexdigest() + fname = os.path.basename(installer+'.sha512') + with open(os.path.join(tdir, fname), 'wb') as f: + f.write(fingerprint) + check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS), + shell=True) + shutil.rmtree(tdir) + class ReUpload(Command): # {{{ description = 'Re-uplaod any installers present in dist/' @@ -57,6 +72,7 @@ class ReUpload(Command): # {{{ opts.replace = True def run(self, opts): + upload_signatures() for x in installers(): if os.path.exists(x): os.remove(x) @@ -223,19 +239,7 @@ class UploadToServer(Command): # {{{ %(__version__, DOWNLOADS), shell=True) check_call('ssh divok /etc/init.d/apache2 graceful', shell=True) - tdir = mkdtemp() - for installer in installers(): - if not os.path.exists(installer): - continue - with open(installer, 'rb') as f: - raw = f.read() - fingerprint = hashlib.sha512(raw).hexdigest() - fname = os.path.basename(installer+'.sha512') - with open(os.path.join(tdir, fname), 'wb') as f: - f.write(fingerprint) - check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS), - shell=True) - shutil.rmtree(tdir) + upload_signatures() # }}} # Testing {{{ From a9e104dda625b09b3e1712c54ec1e54ab6ba2ff4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 10:29:13 +0530 Subject: [PATCH 24/28] ... --- src/calibre/devices/android/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index efe979158d..9ae6f4ab9c 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -187,10 +187,10 @@ class ANDROID(USBMS): EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books', 'sdcard/ebooks'] EXTRA_CUSTOMIZATION_MESSAGE = [_('Comma separated list of directories to ' - 'send e-books to on the device\'s main memory. The first one that exists will ' + 'send e-books to on the device\'s main memory. The first one that exists will ' 'be used'), _('Comma separated list of directories to ' - 'send e-books to on the device\'s storage cards. The first one that exists will ' + 'send e-books to on the device\'s storage cards. The first one that exists will ' 'be used') ] From ecd5ec11e9c1225264f08a47266bf8cc0e92a278 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Sep 2012 12:22:55 +0200 Subject: [PATCH 25/28] Added the option to the wireless driver to force the IP address that the wireless driver listens on. This address is also the one that is advertized by zeroconf. --- .../devices/smart_device_app/driver.py | 54 ++++++++++++------- src/calibre/utils/mdns.py | 9 ++-- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 2d2ddaf568..a6f78facc0 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -33,7 +33,7 @@ from calibre.utils.config import from_json, tweaks from calibre.utils.date import isoformat, now from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as - unpublish_zeroconf) + unpublish_zeroconf, get_all_ips) def synchronous(tlockname): """A decorator to place an instance based lock around a method """ @@ -46,10 +46,6 @@ def synchronous(tlockname): return _synchronizer return _synched -def do_zeroconf(f, port): - f('calibre smart device client', - '_calibresmartdeviceapp._tcp', port, {}) - class SDBook(Book): def __init__(self, prefix, lpath, size=None, other=None): @@ -80,7 +76,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): CAN_DO_DEVICE_DB_PLUGBOARD = False SUPPORTS_SUB_DIRS = True MUST_READ_METADATA = True - NEWS_IN_FOLDER = False + NEWS_IN_FOLDER = True SUPPORTS_USE_AUTHOR_SORT = False WANTS_UPDATED_THUMBNAILS = True MAX_PATH_LEN = 250 @@ -97,7 +93,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): SEND_NOOP_EVERY_NTH_PROBE = 5 DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes - ZEROCONF_CLIENT_STRING = b'calibre smart device client' + ZEROCONF_CLIENT_STRING = b'calibre wireless device client' # A few "random" port numbers to use for detecting clients using broadcast # The clients are expected to broadcast a UDP 'hi there' on all of these @@ -130,8 +126,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): } reverse_opcodes = dict([(v, k) for k,v in opcodes.iteritems()]) - ALL_BY_TITLE = _('All by title') - ALL_BY_AUTHOR = _('All by author') + ALL_BY_TITLE = _('All by title') + ALL_BY_AUTHOR = _('All by author') + ALL_BY_SOMETHING = _('All by something') EXTRA_CUSTOMIZATION_MESSAGE = [ _('Enable connections at startup') + ':::

' + @@ -149,18 +146,25 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): _('Check this box if requested when reporting problems') + '

', '', _('Comma separated list of metadata fields ' - 'to turn into collections on the device. Possibilities include: ')+\ - 'series, tags, authors' +\ - _('. Two special collections are available: %(abt)s:%(abtv)s and %(aba)s:%(abav)s. Add ' - 'these values to the list to enable them. The collections will be ' - 'given the name provided after the ":" character.')%dict( - abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR), + 'to turn into collections on the device.') + ':::

' + + _('Possibilities include: series, tags, authors, etc' + + '. Three special collections are available: %(abt)s:%(abtv)s, ' + '%(aba)s:%(abav)s, and %(abs)s:%(absv)s. Add ' + 'these values to the list to enable them. The collections will be ' + 'given the name provided after the ":" character.')%dict( + abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR, + abs='abs', absv=ALL_BY_SOMETHING), '', _('Enable the no-activity timeout') + ':::

' + _('If this box is checked, calibre will automatically disconnect if ' 'a connected device does nothing for %d minutes. Unchecking this ' ' box disables this timeout, so calibre will never automatically ' 'disconnect.')%(DISCONNECT_AFTER_N_SECONDS/60,) + '

', + _('Use this IP address') + ':::

' + + _('Use this option if you want to force the driver to listen on a ' + 'particular IP address. The driver will listen only on the ' + 'entered address, and this address will be the one advertized ' + 'over mDNS (bonjour).') + '

', ] EXTRA_CUSTOMIZATION_DEFAULT = [ False, @@ -173,6 +177,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): '', '', True, + '' ] OPT_AUTOSTART = 0 OPT_PASSWORD = 2 @@ -181,6 +186,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): OPT_EXTRA_DEBUG = 6 OPT_COLLECTIONS = 8 OPT_AUTODISCONNECT = 10 + OPT_FORCE_IP_ADDRESS = 11 def __init__(self, path): @@ -527,8 +533,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def _attach_to_port(self, sock, port): try: - self._debug('try port', port) - sock.bind(('', port)) + ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS] + self._debug('try ip address "'+ ip_addr + '"', 'on port', port) + if ip_addr: + sock.bind((ip_addr, port)) + else: + sock.bind(('', port)) except socket.error: self._debug('socket error on port', port) port = 0 @@ -996,6 +1006,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.client_can_stream_books = False self.client_can_stream_metadata = False + self._debug("All IP addresses", get_all_ips()) + message = None try: self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -1044,7 +1056,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return message try: - do_zeroconf(publish_zeroconf, port) + ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS] + publish_zeroconf('calibre smart device client', + '_calibresmartdeviceapp._tcp', port, {}, + use_ip_address=ip_addr) except: message = 'registration with bonjour failed' self._debug(message) @@ -1080,7 +1095,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def shutdown(self): if getattr(self, 'listen_socket', None) is not None: - do_zeroconf(unpublish_zeroconf, self.port) + unpublish_zeroconf('calibre smart device client', + '_calibresmartdeviceapp._tcp', self.port, {}) self._close_listen_socket() # Methods for dynamic control diff --git a/src/calibre/utils/mdns.py b/src/calibre/utils/mdns.py index 6140435e46..abbd6c2247 100644 --- a/src/calibre/utils/mdns.py +++ b/src/calibre/utils/mdns.py @@ -60,7 +60,7 @@ def start_server(): return _server -def create_service(desc, type, port, properties, add_hostname): +def create_service(desc, type, port, properties, add_hostname, use_ip_address=None): port = int(port) try: hostname = socket.gethostname().partition('.')[0] @@ -69,7 +69,10 @@ def create_service(desc, type, port, properties, add_hostname): if add_hostname: desc += ' (on %s)'%hostname - local_ip = get_external_ip() + if use_ip_address: + local_ip = use_ip_address + else: + local_ip = get_external_ip() type = type+'.local.' from calibre.utils.Zeroconf import ServiceInfo return ServiceInfo(type, desc+'.'+type, @@ -79,7 +82,7 @@ def create_service(desc, type, port, properties, add_hostname): server=hostname+'.local.') -def publish(desc, type, port, properties=None, add_hostname=True): +def publish(desc, type, port, properties=None, add_hostname=True, use_ip_address=None): ''' Publish a service. From fd80b279cd436f9dd205b91239ef9577530d3ef2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Sep 2012 13:06:41 +0200 Subject: [PATCH 26/28] Display the forced IP address in the Connect/Share menu and the Wireless device dialog. --- src/calibre/devices/smart_device_app/driver.py | 2 ++ src/calibre/gui2/actions/device.py | 17 +++++++++++------ src/calibre/gui2/dialogs/smartdevice.py | 6 +++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index a6f78facc0..b1cd1e635b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -505,6 +505,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return self.OPT_USE_PORT elif opt_string == 'port_number': return self.OPT_PORT_NUMBER + elif opt_string == 'force_ip_address': + return self.OPT_FORCE_IP_ADDRESS else: return None diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index a8475c3a3e..6e215661c8 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -240,13 +240,18 @@ class ConnectShareAction(InterfaceAction): from calibre.gui2.dialogs.smartdevice import get_all_ip_addresses dm = self.gui.device_manager - all_ips = get_all_ip_addresses() - if len(all_ips) > 3: - formatted_addresses = _('Many IP addresses. See Start/Stop dialog.') - show_port = False - else: - formatted_addresses = ' or '.join(get_all_ip_addresses()) + forced_ip = dm.get_option('smartdevice', 'force_ip_address') + if forced_ip: + formatted_addresses = forced_ip show_port = True + else: + all_ips = get_all_ip_addresses() + if len(all_ips) > 3: + formatted_addresses = _('Many IP addresses. See Start/Stop dialog.') + show_port = False + else: + formatted_addresses = ' or '.join(get_all_ip_addresses()) + show_port = True running = dm.is_running('smartdevice') if not running: diff --git a/src/calibre/gui2/dialogs/smartdevice.py b/src/calibre/gui2/dialogs/smartdevice.py index ba58b71048..745125d409 100644 --- a/src/calibre/gui2/dialogs/smartdevice.py +++ b/src/calibre/gui2/dialogs/smartdevice.py @@ -115,7 +115,11 @@ class SmartdeviceDialog(QDialog, Ui_Dialog): self.auto_mgmt_button.setText(_('Automatic metadata management is enabled')) self.auto_mgmt_button.setEnabled(False) - self.ip_addresses.setText(', '.join(get_all_ip_addresses())) + forced_ip = self.device_manager.get_option('smartdevice', 'force_ip_address') + if forced_ip: + self.ip_addresses.setText(forced_ip) + else: + self.ip_addresses.setText(', '.join(get_all_ip_addresses())) self.resize(self.sizeHint()) From 8d8915c0d764e7692ae8d8aca05567e75cefc978 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 17:41:29 +0530 Subject: [PATCH 27/28] ... --- src/calibre/devices/interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index d0b2611ead..2af6b63e3a 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -215,7 +215,9 @@ class DevicePlugin(Plugin): Scan for devices that this driver can handle. Should return a device object if a device is found. This object will be passed to the open() - method as the connected_device. If no device is found, return None. + method as the connected_device. If no device is found, return None. The + returned object can be anything, calibre does not use it, it is only + passed to open(). This method is called periodically by the GUI, so make sure it is not too resource intensive. Use a cache to avoid repeatedly scanning the From 4d5f4558d55c40493e3f2aa2165e84446213d620 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 5 Sep 2012 20:54:54 +0530 Subject: [PATCH 28/28] MTP: Get serial number from the device --- src/calibre/devices/mtp/base.py | 1 + src/calibre/devices/mtp/unix/driver.py | 4 ++++ src/calibre/devices/mtp/windows/driver.py | 7 ++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index bad027baa1..90369221d1 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -37,6 +37,7 @@ class MTPDeviceBase(DevicePlugin): self.progress_reporter = None self.current_friendly_name = None self.report_progress = lambda x, y: None + self.current_serial_num = None def reset(self, key='-1', log_packets=False, report_progress=None, detected_device=None): diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index b59ec22110..3792bb2fcc 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -129,6 +129,7 @@ class MTP_DEVICE(MTPDeviceBase): def post_yank_cleanup(self): self.dev = self._filesystem_cache = self.current_friendly_name = None self.currently_connected_dev = None + self.current_serial_num = None @synchronous def startup(self): @@ -173,6 +174,9 @@ class MTP_DEVICE(MTPDeviceBase): if len(storage) > 2: self._cardb_id = storage[2]['id'] self.current_friendly_name = self.dev.friendly_name + if not self.current_friendly_name: + self.current_friendly_name = self.dev.model_name or _('Unknown MTP device') + self.current_serial_num = self.dev.serial_number @property def filesystem_cache(self): diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 63fedfaf66..2f606b42d1 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -231,10 +231,12 @@ class MTP_DEVICE(MTPDeviceBase): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None self.dev = self._filesystem_cache = None + self.current_serial_num = None def eject(self): if self.currently_connected_pnp_id is None: return self.eject_dev_on_next_scan = True + self.current_serial_num = None @same_thread def open(self, connected_device, library_uuid): @@ -259,9 +261,12 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = devdata.get('friendly_name', + self.current_friendly_name = devdata.get('friendly_name', '') + if not self.current_friendly_name: + self.current_friendly_name = devdata.get('model_name', _('Unknown MTP device')) self.currently_connected_pnp_id = connected_device + self.current_serial_num = devdata.get('serial_number', None) @same_thread def get_basic_device_information(self):