diff --git a/recipes/catavencu.recipe b/recipes/catavencu.recipe index db4057cd6d..d5ba7593e2 100644 --- a/recipes/catavencu.recipe +++ b/recipes/catavencu.recipe @@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class AcademiaCatavencu(BasicNewsRecipe): title = u'Academia Ca\u0163avencu' __author__ = u'Silviu Cotoar\u0103' - description = 'Tagma cum laude' + description = 'Academia Catavencu. Pamflete!' publisher = u'Ca\u0163avencu' oldest_article = 5 language = 'ro' @@ -21,7 +21,7 @@ class AcademiaCatavencu(BasicNewsRecipe): use_embedded_content = False category = 'Ziare' encoding = 'utf-8' - cover_url = 'http://www.academiacatavencu.info/images/logo.png' + cover_url = 'http://www.inpolitics.ro/Uploads/Articles/academia_catavencu.jpg' conversion_options = { 'comments' : description @@ -31,21 +31,21 @@ class AcademiaCatavencu(BasicNewsRecipe): } keep_only_tags = [ - dict(name='h1', attrs={'class':'art_title'}), - dict(name='div', attrs={'class':'art_text'}) + dict(name='h1', attrs={'class':'entry-title'}), + dict(name='div', attrs={'class':'entry-content'}) ] remove_tags = [ - dict(name='div', attrs={'class':['desp_m']}) - , dict(name='div', attrs={'id':['tags']}) + dict(name='div', attrs={'class':['mr_social_sharing_wrapper']}) + , dict(name='div', attrs={'id':['fb_share_1']}) ] remove_tags_after = [ - dict(name='div', attrs={'class':['desp_m']}) + dict(name='div', attrs={'id':['fb_share_1']}) ] feeds = [ - (u'Feeds', u'http://www.academiacatavencu.info/rss.xml') + (u'Feeds', u'http://www.academiacatavencu.info/feed') ] def preprocess_html(self, soup): diff --git a/recipes/dilemaveche.recipe b/recipes/dilemaveche.recipe index 8ba75c4123..72920600f7 100644 --- a/recipes/dilemaveche.recipe +++ b/recipes/dilemaveche.recipe @@ -1,71 +1,51 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +dilemaveche.ro +''' + from calibre.web.feeds.news import BasicNewsRecipe class DilemaVeche(BasicNewsRecipe): - title = u'Dilema Veche' # apare vinerea, mai pe dupa-masa,depinde de Luiza cred (care se semneaza ca fiind creatorul fiecarui articol in feed-ul RSS) - __author__ = 'song2' # inspirat din scriptul pentru Le Monde. Inspired from the Le Monde script - description = '"Sint vechi, domnule!" (I.L. Caragiale)' - publisher = 'Adevarul Holding' - oldest_article = 7 - max_articles_per_feed = 200 - encoding = 'utf8' - language = 'ro' - masthead_url = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' - publication_type = 'magazine' - feeds = [ - ('Editoriale si opinii - Situatiunea', 'http://www.dilemaveche.ro/taxonomy/term/37/0/feed'), - ('Editoriale si opinii - Pe ce lume traim', 'http://www.dilemaveche.ro/taxonomy/term/38/0/feed'), - ('Editoriale si opinii - Bordeie si obiceie', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'), - ('Editoriale si opinii - Talc Show', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'), - ('Tema saptamanii', 'http://www.dilemaveche.ro/taxonomy/term/19/0/feed'), - ('La zi in cultura - Dilema va recomanda', 'http://www.dilemaveche.ro/taxonomy/term/58/0/feed'), - ('La zi in cultura - Carte', 'http://www.dilemaveche.ro/taxonomy/term/14/0/feed'), - ('La zi in cultura - Film', 'http://www.dilemaveche.ro/taxonomy/term/13/0/feed'), - ('La zi in cultura - Muzica', 'http://www.dilemaveche.ro/taxonomy/term/1341/0/feed'), - ('La zi in cultura - Arte performative', 'http://www.dilemaveche.ro/taxonomy/term/1342/0/feed'), - ('La zi in cultura - Arte vizuale', 'http://www.dilemaveche.ro/taxonomy/term/1512/0/feed'), - ('Societate - Ieri cu vedere spre azi', 'http://www.dilemaveche.ro/taxonomy/term/15/0/feed'), - ('Societate - Din polul opus', 'http://www.dilemaveche.ro/taxonomy/term/41/0/feed'), - ('Societate - Mass comedia', 'http://www.dilemaveche.ro/taxonomy/term/43/0/feed'), - ('Societate - La singular si la plural', 'http://www.dilemaveche.ro/taxonomy/term/42/0/feed'), - ('Oameni si idei - Educatie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'), - ('Oameni si idei - Polemici si dezbateri', 'http://www.dilemaveche.ro/taxonomy/term/48/0/feed'), - ('Oameni si idei - Stiinta si tehnologie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'), - ('Dileme on-line', 'http://www.dilemaveche.ro/taxonomy/term/005/0/feed') - ] - remove_tags_before = dict(name='div',attrs={'class':'spacer_10'}) - remove_tags = [ - dict(name='div', attrs={'class':'art_related_left'}), - dict(name='div', attrs={'class':'controale'}), - dict(name='div', attrs={'class':'simple_overlay'}), - ] - remove_tags_after = [dict(id='facebookLike')] - remove_javascript = True + title = u'Dilema Veche' + __author__ = u'Silviu Cotoar\u0103' + description = 'Sint vechi, domnule! (I.L. Caragiale)' + publisher = u'Adev\u0103rul Holding' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 no_stylesheets = True - remove_empty_feeds = True - extra_css = """ - body{font-family: Georgia,Times,serif } - img{margin-bottom: 0.4em; display:block} - """ - def get_cover_url(self): - cover_url = None - soup = self.index_to_soup('http://dilemaveche.ro') - link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'}) - if link_item and link_item.a: - cover_url = link_item.a['href'] - br = BasicNewsRecipe.get_browser() - try: - br.open(cover_url) - except: #daca nu gaseste pdf-ul - self.log("\nPDF indisponibil") - link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'}) - if link_item and link_item.img: - cover_url = link_item.img['src'] - br = BasicNewsRecipe.get_browser() - try: - br.open(cover_url) - except: #daca nu gaseste nici imaginea mica mica - print('Mama lor de nenorociti! nu este nici pdf nici imagine') - cover_url ='http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' - return cover_url - cover_margins = (10, 15, '#ffffff') + use_embedded_content = False + category = 'Ziare' + encoding = 'utf-8' + cover_url = 'http://dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png' + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'c_left_column'}) + ] + + remove_tags = [ + dict(name='div', attrs={'id':['adshop_widget_428x60']}) , + dict(name='div', attrs={'id':['gallery']}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':['adshop_widget_428x60']}) + ] + + feeds = [ + (u'Feeds', u'http://dilemaveche.ro/rss.xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/houston_chronicle.recipe b/recipes/houston_chronicle.recipe index b8171467ec..639d5c2042 100644 --- a/recipes/houston_chronicle.recipe +++ b/recipes/houston_chronicle.recipe @@ -7,18 +7,19 @@ class HoustonChronicle(BasicNewsRecipe): title = u'The Houston Chronicle' description = 'News from Houston, Texas' - __author__ = 'Kovid Goyal' + __author__ = 'Kovid Goyal' language = 'en' timefmt = ' [%a, %d %b, %Y]' no_stylesheets = True use_embedded_content = False remove_attributes = ['style'] + auto_cleanup = True oldest_article = 2.0 - keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or - 'hst-articletext' in x or 'hst-galleryitem' in x)} - remove_attributes = ['xmlns'] + #keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or + #'hst-articletext' in x or 'hst-galleryitem' in x)} + #remove_attributes = ['xmlns'] feeds = [ ('News', "http://www.chron.com/rss/feed/News-270.php"), @@ -37,3 +38,4 @@ class HoustonChronicle(BasicNewsRecipe): ] + diff --git a/recipes/scmp.recipe b/recipes/scmp.recipe index 1da7b9e1bc..a4f4bf497c 100644 --- a/recipes/scmp.recipe +++ b/recipes/scmp.recipe @@ -39,10 +39,10 @@ class SCMP(BasicNewsRecipe): #br.set_debug_responses(True) #br.set_debug_redirects(True) if self.username is not None and self.password is not None: - br.open('http://www.scmp.com/portal/site/SCMP/') - br.select_form(name='loginForm') - br['Login' ] = self.username - br['Password'] = self.password + br.open('http://www.scmp.com/') + br.select_form(nr=1) + br['name'] = self.username + br['pass'] = self.password br.submit() return br diff --git a/recipes/timesnewroman.recipe b/recipes/timesnewroman.recipe index 12672aa888..79c2a9628a 100644 --- a/recipes/timesnewroman.recipe +++ b/recipes/timesnewroman.recipe @@ -36,12 +36,14 @@ class TimesNewRoman(BasicNewsRecipe): remove_tags = [ dict(name='p', attrs={'class':['articleinfo']}) - , dict(name='div',attrs={'class':['vergefacebooklike']}) - , dict(name='div', attrs={'class':'cleared'}) + , dict(name='div', attrs={'class':['shareTools']}) + , dict(name='div', attrs={'class':'fb_iframe_widget'}) + , dict(name='div', attrs={'id':'jc'}) ] remove_tags_after = [ - dict(name='div', attrs={'class':'cleared'}) + dict(name='div', attrs={'class':'fb_iframe_widget'}), + dict(name='div', attrs={'id':'jc'}) ] feeds = [ diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index 9f8bccfd5e..573a8128ac 100644 Binary files a/resources/compiled_coffeescript.zip and b/resources/compiled_coffeescript.zip differ diff --git a/session.vim b/session.vim index 1a94d6bf07..4b9dcb72c1 100644 --- a/session.vim +++ b/session.vim @@ -13,6 +13,8 @@ let g:syntastic_cpp_include_dirs = [ \] let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs +set wildignore+=resources/viewer/mathjax/** + fun! CalibreLog() " Setup buffers to edit the calibre changelog and version info prior to " making a release. diff --git a/setup/extensions.py b/setup/extensions.py index f4ed22687b..f7d40ca72c 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -187,7 +187,7 @@ if iswindows: headers=[ 'calibre/devices/mtp/windows/global.h', ], - libraries=['ole32', 'portabledeviceguids', 'user32'], + libraries=['ole32', 'oleaut32', 'portabledeviceguids', 'user32'], # needs_ddk=True, cflags=['/X'] ), diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 13c02cca12..0ed49c9fef 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -15,7 +15,8 @@ from setup import Command, modules, basenames, functions, __version__, \ SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize', 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py', - '_dbus_glib_bindings.so', 'netifaces.so'] + '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so', + '_psutil_linux.so', 'psutil'] QTDIR = '/usr/lib/qt4' QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus') diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index d0f6eb67ba..aa330095d0 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -360,6 +360,15 @@ Run python setup.py build cp build/lib.win32-2.7/netifaces.pyd /cygdrive/c/Python27/Lib/site-packages/ +psutil +-------- + +Download the source tarball + +Run + +Python setup.py build +cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/ calibre --------- diff --git a/setup/upload.py b/setup/upload.py index a73d0d2c31..f4e8706d1d 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -47,6 +47,21 @@ def installer_description(fname): return 'Calibre Portable' return 'Unknown file' +def upload_signatures(): + tdir = mkdtemp() + for installer in installers(): + if not os.path.exists(installer): + continue + with open(installer, 'rb') as f: + raw = f.read() + fingerprint = hashlib.sha512(raw).hexdigest() + fname = os.path.basename(installer+'.sha512') + with open(os.path.join(tdir, fname), 'wb') as f: + f.write(fingerprint) + check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS), + shell=True) + shutil.rmtree(tdir) + class ReUpload(Command): # {{{ description = 'Re-uplaod any installers present in dist/' @@ -57,6 +72,7 @@ class ReUpload(Command): # {{{ opts.replace = True def run(self, opts): + upload_signatures() for x in installers(): if os.path.exists(x): os.remove(x) @@ -223,19 +239,7 @@ class UploadToServer(Command): # {{{ %(__version__, DOWNLOADS), shell=True) check_call('ssh divok /etc/init.d/apache2 graceful', shell=True) - tdir = mkdtemp() - for installer in installers(): - if not os.path.exists(installer): - continue - with open(installer, 'rb') as f: - raw = f.read() - fingerprint = hashlib.sha512(raw).hexdigest() - fname = os.path.basename(installer+'.sha512') - with open(os.path.join(tdir, fname), 'wb') as f: - f.write(fingerprint) - check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS), - shell=True) - shutil.rmtree(tdir) + upload_signatures() # }}} # Testing {{{ diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 89d0e4e026..2c1d628566 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint, operator, re, os +import sys, time, pprint, operator from functools import partial from StringIO import StringIO @@ -27,112 +27,6 @@ def strftime(epoch, zone=time.gmtime): src[2] = INVERSE_MONTH_MAP[int(src[2])] return ' '.join(src) -def build_template_regexp(template): - from calibre import prints - - def replfunc(match, seen=None): - v = match.group(1) - if v in ['authors', 'author_sort']: - v = 'author' - if v in ('title', 'series', 'series_index', 'isbn', 'author'): - if v not in seen: - seen.add(v) - return '(?P<' + v + '>.+?)' - return '(.+?)' - s = set() - f = partial(replfunc, seen=s) - - try: - template = template.rpartition('/')[2] - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - except: - prints(u'Failed to parse template: %r'%template) - template = u'{title} - {authors}' - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - -def create_upload_path(mdata, fname, template, sanitize, - prefix_path='', - path_type=os.path, - maxlen=250, - use_subdirs=True, - news_in_folder=True, - filename_callback=lambda x, y:x, - sanitize_path_components=lambda x: x - ): - from calibre.library.save_to_disk import get_components, config - from calibre.utils.filenames import shorten_components_to - - special_tag = None - if mdata.tags: - for t in mdata.tags: - if t.startswith(_('News')) or t.startswith('/'): - special_tag = t - break - - if mdata.tags and _('News') in mdata.tags: - try: - p = mdata.pubdate - date = (p.year, p.month, p.day) - except: - today = time.localtime() - date = (today[0], today[1], today[2]) - template = u"{title}_%d-%d-%d" % date - - fname = sanitize(fname) - ext = path_type.splitext(fname)[1] - - opts = config().parse() - if not isinstance(template, unicode): - template = template.decode('utf-8') - app_id = str(getattr(mdata, 'application_id', '')) - id_ = mdata.get('id', fname) - extra_components = get_components(template, mdata, id_, - timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) - if not extra_components: - extra_components.append(sanitize(filename_callback(fname, - mdata))) - else: - extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) - - if extra_components[-1] and extra_components[-1][0] in ('.', '_'): - extra_components[-1] = 'x' + extra_components[-1][1:] - - if special_tag is not None: - name = extra_components[-1] - extra_components = [] - tag = special_tag - if tag.startswith(_('News')): - if news_in_folder: - extra_components.append('News') - else: - for c in tag.split('/'): - c = sanitize(c) - if not c: continue - extra_components.append(c) - extra_components.append(name) - - if not use_subdirs: - extra_components = extra_components[-1:] - - def remove_trailing_periods(x): - ans = x - while ans.endswith('.'): - ans = ans[:-1].strip() - if not ans: - ans = 'x' - return ans - - extra_components = list(map(remove_trailing_periods, extra_components)) - components = shorten_components_to(maxlen - len(prefix_path), extra_components) - components = sanitize_path_components(components) - if prefix_path: - filepath = path_type.join(prefix_path, *components) - else: - filepath = path_type.join(*components) - - return filepath - - def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 9e8aa5fe17..9ae6f4ab9c 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -186,10 +186,15 @@ class ANDROID(USBMS): } EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books', 'sdcard/ebooks'] - EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' - 'send e-books to on the device. The first one that exists will ' + EXTRA_CUSTOMIZATION_MESSAGE = [_('Comma separated list of directories to ' + 'send e-books to on the device\'s main memory. The first one that exists will ' + 'be used'), + _('Comma separated list of directories to ' + 'send e-books to on the device\'s storage cards. The first one that exists will ' 'be used') - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN) + ] + + EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(EBOOK_DIR_MAIN), ''] VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', @@ -237,23 +242,35 @@ class ANDROID(USBMS): def post_open_callback(self): opts = self.settings() - dirs = opts.extra_customization - if not dirs: - dirs = self.EBOOK_DIR_MAIN - else: - dirs = [x.strip() for x in dirs.split(',')] - self.EBOOK_DIR_MAIN = dirs + opts = opts.extra_customization + if not opts: + opts = [self.EBOOK_DIR_MAIN, ''] + + def strtolist(x): + if isinstance(x, basestring): + x = [y.strip() for y in x.split(',')] + return x or [] + + opts = [strtolist(x) for x in opts] + self._android_main_ebook_dir = opts[0] + self._android_card_ebook_dir = opts[1] def get_main_ebook_dir(self, for_upload=False): - dirs = self.EBOOK_DIR_MAIN + dirs = self._android_main_ebook_dir if not for_upload: def aldiko_tweak(x): return 'eBooks' if x == 'eBooks/import' else x - if isinstance(dirs, basestring): - dirs = [dirs] dirs = list(map(aldiko_tweak, dirs)) return dirs + def get_carda_ebook_dir(self, for_upload=False): + if not for_upload: + return '' + return self._android_card_ebook_dir + + def get_cardb_ebook_dir(self, for_upload=False): + return self.get_carda_ebook_dir() + def windows_sort_drives(self, drives): try: vid, pid, bcd = self.device_being_opened[:3] @@ -271,7 +288,8 @@ class ANDROID(USBMS): proxy = cls._configProxy() proxy['format_map'] = ['mobi', 'azw', 'azw1', 'azw4', 'pdf'] proxy['use_subdirs'] = False - proxy['extra_customization'] = ','.join(['kindle']+cls.EBOOK_DIR_MAIN) + proxy['extra_customization'] = [ + ','.join(['kindle']+cls.EBOOK_DIR_MAIN), ''] @classmethod def configure_for_generic_epub_app(cls): diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index d0b2611ead..2af6b63e3a 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -215,7 +215,9 @@ class DevicePlugin(Plugin): Scan for devices that this driver can handle. Should return a device object if a device is found. This object will be passed to the open() - method as the connected_device. If no device is found, return None. + method as the connected_device. If no device is found, return None. The + returned object can be anything, calibre does not use it, it is only + passed to open(). This method is called periodically by the GUI, so make sure it is not too resource intensive. Use a cache to avoid repeatedly scanning the diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 516b68ae1e..90369221d1 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -25,7 +25,7 @@ def synchronous(func): return synchronizer class MTPDeviceBase(DevicePlugin): - name = 'SmartDevice App Interface' + name = 'MTP Device Interface' gui_name = _('MTP Device') icon = I('devices/galaxy_s3.png') description = _('Communicate with MTP devices') @@ -37,6 +37,7 @@ class MTPDeviceBase(DevicePlugin): self.progress_reporter = None self.current_friendly_name = None self.report_progress = lambda x, y: None + self.current_serial_num = None def reset(self, key='-1', log_packets=False, report_progress=None, detected_device=None): @@ -45,8 +46,9 @@ class MTPDeviceBase(DevicePlugin): def set_progress_reporter(self, report_progress): self.report_progress = report_progress - def get_gui_name(self): - return self.current_friendly_name or self.name + @classmethod + def get_gui_name(cls): + return getattr(cls, 'current_friendly_name', cls.gui_name) def is_usb_connected(self, devices_on_system, debug=False, only_presence=False): @@ -55,7 +57,7 @@ class MTPDeviceBase(DevicePlugin): return False def build_template_regexp(self): - from calibre.devices import build_template_regexp + from calibre.devices.utils import build_template_regexp return build_template_regexp(self.save_template) @property diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py index 73e483f19e..a72fc1f84e 100644 --- a/src/calibre/devices/mtp/books.py +++ b/src/calibre/devices/mtp/books.py @@ -22,6 +22,22 @@ class BookList(BL): def supports_collections(self): return False + def add_book(self, book, replace_metadata=True): + try: + b = self.index(book) + except (ValueError, IndexError): + b = None + if b is None: + self.append(book) + return book + if replace_metadata: + self[b].smart_update(book, replace_metadata=True) + return self[b] + return None + + def remove_book(self, book): + self.remove(book) + class Book(Metadata): def __init__(self, storage_id, lpath, other=None): @@ -36,6 +52,17 @@ class Book(Metadata): return (self.storage_id == mtp_file.storage_id and self.mtp_relpath == mtp_file.mtp_relpath) + def __eq__(self, other): + return (isinstance(other, self.__class__) and (self.storage_id == + other.storage_id and self.mtp_relpath == other.mtp_relpath)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.storage_id, self.mtp_relpath)) + + class JSONCodec(JsonCodec): pass diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 8f8f4d119b..d716d15de5 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en' import json, traceback, posixpath, importlib, os from io import BytesIO +from itertools import izip from calibre import prints from calibre.constants import iswindows, numeric_version @@ -32,6 +33,12 @@ class MTP_DEVICE(BASE): CAN_SET_METADATA = [] BACKLOADING_ERROR_MESSAGE = None MANAGES_DEVICE_PRESENCE = True + FORMATS = ['epub', 'azw3', 'mobi', 'pdf'] + DEVICE_PLUGBOARD_NAME = 'MTP_DEVICE' + + def __init__(self, *args, **kwargs): + BASE.__init__(self, *args, **kwargs) + self.plugboards = self.plugboard_func = None def open(self, devices, library_uuid): self.current_library_uuid = library_uuid @@ -74,12 +81,7 @@ class MTP_DEVICE(BASE): return tuple( list(dinfo) + [self.driveinfo] ) def card_prefix(self, end_session=True): - ans = [None, None] - if self._carda_id is not None: - ans[0] = self.filesystem_cache.storage(self._carda_id).storage_prefix - if self._cardb_id is not None: - ans[1] = self.filesystem_cache.storage(self._cardb_id).storage_prefix - return tuple(ans) + return (self._carda_id, self._cardb_id) def set_driveinfo_name(self, location_code, name): sid = {'main':self._main_id, 'A':self._carda_id, @@ -189,6 +191,7 @@ class MTP_DEVICE(BASE): self.put_file(storage, self.METADATA_CACHE, stream, size) def sync_booklists(self, booklists, end_session=True): + debug('sync_booklists() called') for bl in booklists: if getattr(bl, 'storage_id', None) is None: continue @@ -196,6 +199,7 @@ class MTP_DEVICE(BASE): if storage is None: continue self.write_metadata_cache(storage, bl) + debug('sync_booklists() ended') # }}} @@ -225,8 +229,14 @@ class MTP_DEVICE(BASE): return ans # }}} + # Sending files to the device {{{ + + def set_plugboards(self, plugboards, pb_func): + self.plugboards = plugboards + self.plugboard_func = pb_func + def create_upload_path(self, path, mdata, fname): - from calibre.devices import create_upload_path + from calibre.devices.utils import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize filepath = create_upload_path(mdata, fname, self.save_template, sanitize, prefix_path=path, @@ -235,7 +245,136 @@ class MTP_DEVICE(BASE): use_subdirs = True, news_in_folder = self.NEWS_IN_FOLDER, ) - return tuple(x.lower() for x in filepath.split('/')) + return tuple(x for x in filepath.split('/')) + + def prefix_for_location(self, on_card): + # TODO: Implement this + return 'calibre' + + def ensure_parent(self, storage, path): + parent = storage + pos = list(path)[:-1] + while pos: + name = pos[0] + pos = pos[1:] + parent = self.create_folder(parent, name) + return parent + + def upload_books(self, files, names, on_card=None, end_session=True, + metadata=None): + debug('upload_books() called') + from calibre.devices.utils import sanity_check + sanity_check(on_card, files, self.card_prefix(), self.free_space()) + prefix = self.prefix_for_location(on_card) + sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(on_card, + self._main_id) + bl_idx = {'carda':1, 'cardb':2}.get(on_card, 0) + storage = self.filesystem_cache.storage(sid) + + ans = [] + self.report_progress(0, _('Transferring books to device...')) + i, total = 0, len(files) + + for infile, fname, mi in izip(files, names, metadata): + path = self.create_upload_path(prefix, mi, fname) + parent = self.ensure_parent(storage, path) + if hasattr(infile, 'read'): + pos = infile.tell() + infile.seek(0, 2) + sz = infile.tell() + infile.seek(pos) + stream = infile + close = False + else: + sz = os.path.getsize(infile) + stream = lopen(infile, 'rb') + close = True + try: + mtp_file = self.put_file(parent, path[-1], stream, sz) + finally: + if close: + stream.close() + ans.append((mtp_file, bl_idx)) + i += 1 + self.report_progress(i/total, _('Transferred %s to device')%mi.title) + + self.report_progress(1, _('Transfer to device finished...')) + debug('upload_books() ended') + return ans + + def add_books_to_metadata(self, mtp_files, metadata, booklists): + debug('add_books_to_metadata() called') + from calibre.devices.mtp.books import Book + + i, total = 0, len(mtp_files) + self.report_progress(0, _('Adding books to device metadata listing...')) + for x, mi in izip(mtp_files, metadata): + mtp_file, bl_idx = x + bl = booklists[bl_idx] + book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath), + other=mi) + book = bl.add_book(book, replace_metadata=True) + if book is not None: + book.size = mtp_file.size + book.datetime = mtp_file.last_modified.timetuple() + book.path = mtp_file.mtp_id_path + i += 1 + self.report_progress(i/total, _('Added %s')%mi.title) + + self.report_progress(1, _('Adding complete')) + debug('add_books_to_metadata() ended') + + # }}} + + # Removing books from the device {{{ + def recursive_delete(self, obj): + parent = self.delete_file_or_folder(obj) + if parent.empty and parent.can_delete and not parent.is_system: + try: + self.recursive_delete(parent) + except: + prints('Failed to delete parent: %s, ignoring'%( + '/'.join(parent.full_path))) + + def delete_books(self, paths, end_session=True): + self.report_progress(0, _('Deleting books from device...')) + + for i, path in enumerate(paths): + f = self.filesystem_cache.resolve_mtp_id_path(path) + self.recursive_delete(f) + self.report_progress((i+1) / float(len(paths)), + _('Deleted %s')%path) + self.report_progress(1, _('All books deleted')) + + def remove_books_from_metadata(self, paths, booklists): + self.report_progress(0, _('Removing books from metadata')) + class NextPath(Exception): pass + + for i, path in enumerate(paths): + try: + for bl in booklists: + for book in bl: + if book.path == path: + bl.remove_book(book) + raise NextPath('') + except NextPath: + pass + self.report_progress((i+1)/len(paths), _('Removed %s')%path) + + self.report_progress(1, _('All books removed')) + + # }}} + + # Settings {{{ + @classmethod + def settings(self): + # TODO: Implement this + class Opts(object): + def __init__(s): + s.format_map = self.FORMATS + return Opts() + + # }}} if __name__ == '__main__': dev = MTP_DEVICE(None) @@ -250,7 +389,7 @@ if __name__ == '__main__': raise ValueError('Failed to detect MTP device') dev.set_progress_reporter(prints) dev.open(cd, None) - dev.books() + dev.filesystem_cache.dump() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index 216e06031f..8f4d20ae18 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -37,9 +37,13 @@ class FileOrFolder(object): self.size = entry.get('size', 0) md = entry.get('modified', 0) try: - self.last_modified = datetime.fromtimestamp(md, local_tz) + if isinstance(md, tuple): + self.last_modified = datetime(*(list(md)+[local_tz])) + else: + self.last_modified = datetime.fromtimestamp(md, local_tz) except: self.last_modified = datetime.fromtimestamp(0, local_tz) + self.last_mod_string = self.last_modified.strftime('%Y/%m/%d %H:%M') self.last_modified = as_utc(self.last_modified) if self.storage_id not in self.all_storage_ids: @@ -74,12 +78,16 @@ class FileOrFolder(object): datum = 'size=%s'%(self.size) if self.is_folder: datum = 'children=%s'%(len(self.files) + len(self.folders)) - return '%s(id=%s, storage_id=%s, %s, path=%s)'%(name, self.object_id, - self.storage_id, datum, path) + return '%s(id=%s, storage_id=%s, %s, path=%s, modified=%s)'%(name, self.object_id, + self.storage_id, datum, path, self.last_mod_string) __str__ = __repr__ __unicode__ = __repr__ + @property + def empty(self): + return not self.files and not self.folders + @property def id_map(self): return self.fs_cache().id_map @@ -123,6 +131,7 @@ class FileOrFolder(object): c = '+' if self.is_folder else '-' data = ('%s children'%(sum(map(len, (self.files, self.folders)))) if self.is_folder else human_readable(self.size)) + data += ' modified=%s'%self.last_mod_string line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data) prints(line, file=out) for c in (self.folders, self.files): @@ -217,6 +226,8 @@ class FilesystemCache(object): def iterebooks(self, storage_id): for x in self.id_map.itervalues(): if x.storage_id == storage_id and x.is_ebook: + if x.parent_id == storage_id and x.name.lower().endswith('.txt'): + continue # Ignore .txt files in the root yield x def resolve_mtp_id_path(self, path): diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 2f215f6353..3792bb2fcc 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -129,6 +129,7 @@ class MTP_DEVICE(MTPDeviceBase): def post_yank_cleanup(self): self.dev = self._filesystem_cache = self.current_friendly_name = None self.currently_connected_dev = None + self.current_serial_num = None @synchronous def startup(self): @@ -173,6 +174,9 @@ class MTP_DEVICE(MTPDeviceBase): if len(storage) > 2: self._cardb_id = storage[2]['id'] self.current_friendly_name = self.dev.friendly_name + if not self.current_friendly_name: + self.current_friendly_name = self.dev.model_name or _('Unknown MTP device') + self.current_serial_num = self.dev.serial_number @property def filesystem_cache(self): @@ -306,6 +310,7 @@ class MTP_DEVICE(MTPDeviceBase): raise DeviceError('Failed to delete %s with error: %s'% (obj.full_path, self.format_errorstack(errs))) parent.remove_child(obj) + return parent def develop(): from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 7186bbdcdb..580f77f9b0 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -84,11 +84,19 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { FLOAT val = 0; + SYSTEMTIME st; + unsigned int microseconds; PyObject *t; if (SUCCEEDED(properties->GetFloatValue(key, &val))) { - t = Py_BuildValue("d", (double)val); - if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + if (VariantTimeToSystemTime(val, &st)) { + microseconds = 1000 * st.wMilliseconds; + t = Py_BuildValue("H H H H H H I", (unsigned short)st.wYear, + (unsigned short)st.wMonth, (unsigned short)st.wDay, + (unsigned short)st.wHour, (unsigned short)st.wMinute, + (unsigned short)st.wSecond, microseconds); + if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + } } } diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 7c15797ef6..2f606b42d1 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -231,10 +231,12 @@ class MTP_DEVICE(MTPDeviceBase): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None self.dev = self._filesystem_cache = None + self.current_serial_num = None def eject(self): if self.currently_connected_pnp_id is None: return self.eject_dev_on_next_scan = True + self.current_serial_num = None @same_thread def open(self, connected_device, library_uuid): @@ -259,9 +261,12 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = devdata.get('friendly_name', + self.current_friendly_name = devdata.get('friendly_name', '') + if not self.current_friendly_name: + self.current_friendly_name = devdata.get('model_name', _('Unknown MTP device')) self.currently_connected_pnp_id = connected_device + self.current_serial_num = devdata.get('serial_number', None) @same_thread def get_basic_device_information(self): @@ -338,6 +343,7 @@ class MTP_DEVICE(MTPDeviceBase): parent = obj.parent self.dev.delete_object(obj.object_id) parent.remove_child(obj) + return parent @same_thread def put_file(self, parent, name, stream, size, callback=None, replace=True): diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index cbc23978d2..f1dfa92767 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -54,9 +54,9 @@ def main(): plugins._plugins['wpd'] = (wpd, '') sys.path.pop(0) - from calibre.devices.mtp.test import run - run() - return + # from calibre.devices.mtp.test import run + # run() + # return from calibre.devices.scanner import win_scanner from calibre.devices.mtp.windows.driver import MTP_DEVICE @@ -81,13 +81,13 @@ def main(): # print ('Fetching file: oFF (198214 bytes)') # stream = dev.get_file('oFF') # print ("Fetched size: ", stream.tell()) - size = 4 - stream = io.BytesIO(b'a'*size) - name = 'zzz-test-file.txt' - stream.seek(0) - f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size) - print ('Put file:', f) - # dev.filesystem_cache.dump() + # size = 4 + # stream = io.BytesIO(b'a'*size) + # name = 'zzz-test-file.txt' + # stream.seek(0) + # f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size) + # print ('Put file:', f) + dev.filesystem_cache.dump() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/windows/wpd.cpp b/src/calibre/devices/mtp/windows/wpd.cpp index 15cdd51e22..867ce6bcee 100644 --- a/src/calibre/devices/mtp/windows/wpd.cpp +++ b/src/calibre/devices/mtp/windows/wpd.cpp @@ -120,14 +120,14 @@ wpd_enumerate_devices(PyObject *self, PyObject *args) { hresult_set_exc("Failed to get list of portable devices", hr); } + Py_BEGIN_ALLOW_THREADS; for (i = 0; i < num_of_devices; i++) { - Py_BEGIN_ALLOW_THREADS; CoTaskMemFree(pnp_device_ids[i]); - Py_END_ALLOW_THREADS; pnp_device_ids[i] = NULL; } free(pnp_device_ids); pnp_device_ids = NULL; + Py_END_ALLOW_THREADS; return Py_BuildValue("N", ans); } // }}} diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 2d2ddaf568..b1cd1e635b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -33,7 +33,7 @@ from calibre.utils.config import from_json, tweaks from calibre.utils.date import isoformat, now from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as - unpublish_zeroconf) + unpublish_zeroconf, get_all_ips) def synchronous(tlockname): """A decorator to place an instance based lock around a method """ @@ -46,10 +46,6 @@ def synchronous(tlockname): return _synchronizer return _synched -def do_zeroconf(f, port): - f('calibre smart device client', - '_calibresmartdeviceapp._tcp', port, {}) - class SDBook(Book): def __init__(self, prefix, lpath, size=None, other=None): @@ -80,7 +76,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): CAN_DO_DEVICE_DB_PLUGBOARD = False SUPPORTS_SUB_DIRS = True MUST_READ_METADATA = True - NEWS_IN_FOLDER = False + NEWS_IN_FOLDER = True SUPPORTS_USE_AUTHOR_SORT = False WANTS_UPDATED_THUMBNAILS = True MAX_PATH_LEN = 250 @@ -97,7 +93,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): SEND_NOOP_EVERY_NTH_PROBE = 5 DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes - ZEROCONF_CLIENT_STRING = b'calibre smart device client' + ZEROCONF_CLIENT_STRING = b'calibre wireless device client' # A few "random" port numbers to use for detecting clients using broadcast # The clients are expected to broadcast a UDP 'hi there' on all of these @@ -130,8 +126,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): } reverse_opcodes = dict([(v, k) for k,v in opcodes.iteritems()]) - ALL_BY_TITLE = _('All by title') - ALL_BY_AUTHOR = _('All by author') + ALL_BY_TITLE = _('All by title') + ALL_BY_AUTHOR = _('All by author') + ALL_BY_SOMETHING = _('All by something') EXTRA_CUSTOMIZATION_MESSAGE = [ _('Enable connections at startup') + ':::

' + @@ -149,18 +146,25 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): _('Check this box if requested when reporting problems') + '

', '', _('Comma separated list of metadata fields ' - 'to turn into collections on the device. Possibilities include: ')+\ - 'series, tags, authors' +\ - _('. Two special collections are available: %(abt)s:%(abtv)s and %(aba)s:%(abav)s. Add ' - 'these values to the list to enable them. The collections will be ' - 'given the name provided after the ":" character.')%dict( - abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR), + 'to turn into collections on the device.') + ':::

' + + _('Possibilities include: series, tags, authors, etc' + + '. Three special collections are available: %(abt)s:%(abtv)s, ' + '%(aba)s:%(abav)s, and %(abs)s:%(absv)s. Add ' + 'these values to the list to enable them. The collections will be ' + 'given the name provided after the ":" character.')%dict( + abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR, + abs='abs', absv=ALL_BY_SOMETHING), '', _('Enable the no-activity timeout') + ':::

' + _('If this box is checked, calibre will automatically disconnect if ' 'a connected device does nothing for %d minutes. Unchecking this ' ' box disables this timeout, so calibre will never automatically ' 'disconnect.')%(DISCONNECT_AFTER_N_SECONDS/60,) + '

', + _('Use this IP address') + ':::

' + + _('Use this option if you want to force the driver to listen on a ' + 'particular IP address. The driver will listen only on the ' + 'entered address, and this address will be the one advertized ' + 'over mDNS (bonjour).') + '

', ] EXTRA_CUSTOMIZATION_DEFAULT = [ False, @@ -173,6 +177,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): '', '', True, + '' ] OPT_AUTOSTART = 0 OPT_PASSWORD = 2 @@ -181,6 +186,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): OPT_EXTRA_DEBUG = 6 OPT_COLLECTIONS = 8 OPT_AUTODISCONNECT = 10 + OPT_FORCE_IP_ADDRESS = 11 def __init__(self, path): @@ -499,6 +505,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return self.OPT_USE_PORT elif opt_string == 'port_number': return self.OPT_PORT_NUMBER + elif opt_string == 'force_ip_address': + return self.OPT_FORCE_IP_ADDRESS else: return None @@ -527,8 +535,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def _attach_to_port(self, sock, port): try: - self._debug('try port', port) - sock.bind(('', port)) + ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS] + self._debug('try ip address "'+ ip_addr + '"', 'on port', port) + if ip_addr: + sock.bind((ip_addr, port)) + else: + sock.bind(('', port)) except socket.error: self._debug('socket error on port', port) port = 0 @@ -996,6 +1008,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.client_can_stream_books = False self.client_can_stream_metadata = False + self._debug("All IP addresses", get_all_ips()) + message = None try: self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -1044,7 +1058,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return message try: - do_zeroconf(publish_zeroconf, port) + ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS] + publish_zeroconf('calibre smart device client', + '_calibresmartdeviceapp._tcp', port, {}, + use_ip_address=ip_addr) except: message = 'registration with bonjour failed' self._debug(message) @@ -1080,7 +1097,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def shutdown(self): if getattr(self, 'listen_socket', None) is not None: - do_zeroconf(unpublish_zeroconf, self.port) + unpublish_zeroconf('calibre smart device client', + '_calibresmartdeviceapp._tcp', self.port, {}) self._close_listen_socket() # Methods for dynamic control diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 4d4b198de0..795a22888f 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -15,8 +15,7 @@ import os, subprocess, time, re, sys, glob from itertools import repeat from calibre.devices.interface import DevicePlugin -from calibre.devices.errors import (DeviceError, FreeSpaceError, - WrongDestinationError) +from calibre.devices.errors import DeviceError from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins from calibre.utils.filenames import ascii_filename as sanitize @@ -976,53 +975,32 @@ class Device(DeviceConfig, DevicePlugin): return self.EBOOK_DIR_CARD_A def _sanity_check(self, on_card, files): - if on_card == 'carda' and not self._card_a_prefix: - raise WrongDestinationError(_( - 'The reader has no storage card %s. You may have changed ' - 'the default send to device action. Right click on the send ' - 'to device button and reset the default action to be ' - '"Send to main memory".')%'A') - elif on_card == 'cardb' and not self._card_b_prefix: - raise WrongDestinationError(_( - 'The reader has no storage card %s. You may have changed ' - 'the default send to device action. Right click on the send ' - 'to device button and reset the default action to be ' - '"Send to main memory".')%'B') - elif on_card and on_card not in ('carda', 'cardb'): - raise DeviceError(_('Selected slot: %s is not supported.') % on_card) + from calibre.devices.utils import sanity_check + sanity_check(on_card, files, self.card_prefix(), self.free_space()) - if on_card == 'carda': - path = os.path.join(self._card_a_prefix, - *(self.get_carda_ebook_dir(for_upload=True).split('/'))) - elif on_card == 'cardb': - path = os.path.join(self._card_b_prefix, - *(self.EBOOK_DIR_CARD_B.split('/'))) - else: - candidates = self.get_main_ebook_dir(for_upload=True) + def get_dest_dir(prefix, candidates): if isinstance(candidates, basestring): candidates = [candidates] + if not candidates: + candidates = [''] candidates = [ - ((os.path.join(self._main_prefix, *(x.split('/')))) if x else - self._main_prefix) for x - in candidates] + ((os.path.join(prefix, *(x.split('/')))) if x else prefix) + for x in candidates] existing = [x for x in candidates if os.path.exists(x)] if not existing: - existing = candidates[:1] - path = existing[0] + existing = candidates + return existing[0] - def get_size(obj): - path = getattr(obj, 'name', obj) - return os.path.getsize(path) + if on_card == 'carda': + candidates = self.get_carda_ebook_dir(for_upload=True) + path = get_dest_dir(self._carda_prefix, candidates) + elif on_card == 'cardb': + candidates = self.get_cardb_ebook_dir(for_upload=True) + path = get_dest_dir(self._cardb_prefix, candidates) + else: + candidates = self.get_main_ebook_dir(for_upload=True) + path = get_dest_dir(self._main_prefix, candidates) - sizes = [get_size(f) for f in files] - size = sum(sizes) - - if not on_card and size > self.free_space()[0] - 2*1024*1024: - raise FreeSpaceError(_("There is insufficient free space in main memory")) - if on_card == 'carda' and size > self.free_space()[1] - 1024*1024: - raise FreeSpaceError(_("There is insufficient free space on the storage card")) - if on_card == 'cardb' and size > self.free_space()[2] - 1024*1024: - raise FreeSpaceError(_("There is insufficient free space on the storage card")) return path def filename_callback(self, default, mi): @@ -1052,7 +1030,7 @@ class Device(DeviceConfig, DevicePlugin): pass def create_upload_path(self, path, mdata, fname, create_dirs=True): - from calibre.devices import create_upload_path + from calibre.devices.utils import create_upload_path settings = self.settings() filepath = create_upload_path(mdata, fname, self.save_template(), sanitize, prefix_path=os.path.abspath(path), diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index f6c7556fd8..94309e747e 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -166,7 +166,7 @@ class USBMS(CLI, Device): # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} - for idx,b in enumerate(bl): + for idx, b in enumerate(bl): bl_cache[b.lpath] = idx all_formats = self.formats_to_scan_for() @@ -404,7 +404,7 @@ class USBMS(CLI, Device): @classmethod def build_template_regexp(cls): - from calibre.devices import build_template_regexp + from calibre.devices.utils import build_template_regexp return build_template_regexp(cls.save_template()) @classmethod diff --git a/src/calibre/devices/utils.py b/src/calibre/devices/utils.py new file mode 100644 index 0000000000..114e7e4e13 --- /dev/null +++ b/src/calibre/devices/utils.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, re +from functools import partial + +from calibre.devices.errors import DeviceError, WrongDestinationError, FreeSpaceError + +def sanity_check(on_card, files, card_prefixes, free_space): + if on_card == 'carda' and not card_prefixes[0]: + raise WrongDestinationError(_( + 'The reader has no storage card %s. You may have changed ' + 'the default send to device action. Right click on the send ' + 'to device button and reset the default action to be ' + '"Send to main memory".')%'A') + elif on_card == 'cardb' and not card_prefixes[1]: + raise WrongDestinationError(_( + 'The reader has no storage card %s. You may have changed ' + 'the default send to device action. Right click on the send ' + 'to device button and reset the default action to be ' + '"Send to main memory".')%'B') + elif on_card and on_card not in ('carda', 'cardb'): + raise DeviceError(_('Selected slot: %s is not supported.') % on_card) + + size = 0 + for f in files: + size += os.path.getsize(getattr(f, 'name', f)) + + if not on_card and size > free_space[0] - 2*1024*1024: + raise FreeSpaceError(_("There is insufficient free space in main memory")) + if on_card == 'carda' and size > free_space[1] - 1024*1024: + raise FreeSpaceError(_("There is insufficient free space on the storage card")) + if on_card == 'cardb' and size > free_space[2] - 1024*1024: + raise FreeSpaceError(_("There is insufficient free space on the storage card")) + +def build_template_regexp(template): + from calibre import prints + + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + + try: + template = template.rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + +def create_upload_path(mdata, fname, template, sanitize, + prefix_path='', + path_type=os.path, + maxlen=250, + use_subdirs=True, + news_in_folder=True, + filename_callback=lambda x, y:x, + sanitize_path_components=lambda x: x + ): + from calibre.library.save_to_disk import get_components, config + from calibre.utils.filenames import shorten_components_to + + special_tag = None + if mdata.tags: + for t in mdata.tags: + if t.startswith(_('News')) or t.startswith('/'): + special_tag = t + break + + if mdata.tags and _('News') in mdata.tags: + try: + p = mdata.pubdate + date = (p.year, p.month, p.day) + except: + today = time.localtime() + date = (today[0], today[1], today[2]) + template = u"{title}_%d-%d-%d" % date + + fname = sanitize(fname) + ext = path_type.splitext(fname)[1] + + opts = config().parse() + if not isinstance(template, unicode): + template = template.decode('utf-8') + app_id = str(getattr(mdata, 'application_id', '')) + id_ = mdata.get('id', fname) + extra_components = get_components(template, mdata, id_, + timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) + if not extra_components: + extra_components.append(sanitize(filename_callback(fname, + mdata))) + else: + extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) + + if extra_components[-1] and extra_components[-1][0] in ('.', '_'): + extra_components[-1] = 'x' + extra_components[-1][1:] + + if special_tag is not None: + name = extra_components[-1] + extra_components = [] + tag = special_tag + if tag.startswith(_('News')): + if news_in_folder: + extra_components.append('News') + else: + for c in tag.split('/'): + c = sanitize(c) + if not c: continue + extra_components.append(c) + extra_components.append(name) + + if not use_subdirs: + extra_components = extra_components[-1:] + + def remove_trailing_periods(x): + ans = x + while ans.endswith('.'): + ans = ans[:-1].strip() + if not ans: + ans = 'x' + return ans + + extra_components = list(map(remove_trailing_periods, extra_components)) + components = shorten_components_to(maxlen - len(prefix_path), extra_components) + components = sanitize_path_components(components) + if prefix_path: + filepath = path_type.join(prefix_path, *components) + else: + filepath = path_type.join(*components) + + return filepath + + + diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index cc9b6f252d..e9cec8acc7 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -161,7 +161,9 @@ class JsonCodec(object): try: js = json.load(file_, encoding='utf-8') for item in js: - booklist.append(self.raw_to_book(item, book_class, prefix)) + entry = self.raw_to_book(item, book_class, prefix) + if entry is not None: + booklist.append(entry) except: print 'exception during JSON decode_from_file' traceback.print_exc() diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 4f912513a9..286945bfb6 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -79,7 +79,7 @@ class PagedDisplay if not this.in_paged_mode # Check if the current document is a full screen layout like # cover, if so we treat it specially. - single_screen = (document.body.scrollWidth < window.innerWidth + 25 and document.body.scrollHeight < window.innerHeight + 25) + single_screen = (document.body.scrollHeight < window.innerHeight + 75) first_layout = true ww = window.innerWidth @@ -149,7 +149,7 @@ class PagedDisplay # current page (when cols_per_screen == 1). Similarly img elements # with height=100% overflow the first column has_svg = document.getElementsByTagName('svg').length > 0 - only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 2 and document.getElementsByTagName('p').length < 2 + only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 3 and document.getElementsByTagName('p').length < 2 this.is_full_screen_layout = (only_img or has_svg) and single_screen and document.body.scrollWidth > document.body.clientWidth this.in_paged_mode = true diff --git a/src/calibre/gui2/actions/device.py b/src/calibre/gui2/actions/device.py index a8475c3a3e..6e215661c8 100644 --- a/src/calibre/gui2/actions/device.py +++ b/src/calibre/gui2/actions/device.py @@ -240,13 +240,18 @@ class ConnectShareAction(InterfaceAction): from calibre.gui2.dialogs.smartdevice import get_all_ip_addresses dm = self.gui.device_manager - all_ips = get_all_ip_addresses() - if len(all_ips) > 3: - formatted_addresses = _('Many IP addresses. See Start/Stop dialog.') - show_port = False - else: - formatted_addresses = ' or '.join(get_all_ip_addresses()) + forced_ip = dm.get_option('smartdevice', 'force_ip_address') + if forced_ip: + formatted_addresses = forced_ip show_port = True + else: + all_ips = get_all_ip_addresses() + if len(all_ips) > 3: + formatted_addresses = _('Many IP addresses. See Start/Stop dialog.') + show_port = False + else: + formatted_addresses = ' or '.join(get_all_ip_addresses()) + show_port = True running = dm.is_running('smartdevice') if not running: diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 98e42f4178..4cc4c0fb5f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -977,7 +977,7 @@ class DeviceMixin(object): # {{{ self.set_default_thumbnail(\ self.device_manager.device.THUMBNAIL_HEIGHT) self.status_bar.show_message(_('Device: ')+\ - self.device_manager.device.__class__.get_gui_name()+\ + self.device_manager.device.get_gui_name()+\ _(' detected.'), 3000) self.device_connected = device_kind self.library_view.set_device_connected(self.device_connected) @@ -1495,8 +1495,12 @@ class DeviceMixin(object): # {{{ self.device_job_exception(job) return - self.device_manager.add_books_to_metadata(job.result, - metadata, self.booklists()) + try: + self.device_manager.add_books_to_metadata(job.result, + metadata, self.booklists()) + except: + traceback.print_exc() + raise books_to_be_deleted = [] if memory and memory[1]: diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index b47a80b6ad..9624efac96 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -89,6 +89,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setBuddy(self.opt_extra_customization[i]) l.setWordWrap(True) self.opt_extra_customization[i].setText(settings.extra_customization[i]) + self.opt_extra_customization[i].setCursorPosition(0) self.extra_layout.addWidget(l, row_func(i, 0), col_func(i)) self.extra_layout.addWidget(self.opt_extra_customization[i], row_func(i, 1), col_func(i)) @@ -101,6 +102,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setWordWrap(True) if settings.extra_customization: self.opt_extra_customization.setText(settings.extra_customization) + self.opt_extra_customization.setCursorPosition(0) self.opt_extra_customization.setCursorPosition(0) self.extra_layout.addWidget(l, 0, 0) self.extra_layout.addWidget(self.opt_extra_customization, 1, 0) diff --git a/src/calibre/gui2/dialogs/smartdevice.py b/src/calibre/gui2/dialogs/smartdevice.py index ba58b71048..745125d409 100644 --- a/src/calibre/gui2/dialogs/smartdevice.py +++ b/src/calibre/gui2/dialogs/smartdevice.py @@ -115,7 +115,11 @@ class SmartdeviceDialog(QDialog, Ui_Dialog): self.auto_mgmt_button.setText(_('Automatic metadata management is enabled')) self.auto_mgmt_button.setEnabled(False) - self.ip_addresses.setText(', '.join(get_all_ip_addresses())) + forced_ip = self.device_manager.get_option('smartdevice', 'force_ip_address') + if forced_ip: + self.ip_addresses.setText(forced_ip) + else: + self.ip_addresses.setText(', '.join(get_all_ip_addresses())) self.resize(self.sizeHint()) diff --git a/src/calibre/utils/mdns.py b/src/calibre/utils/mdns.py index 6140435e46..abbd6c2247 100644 --- a/src/calibre/utils/mdns.py +++ b/src/calibre/utils/mdns.py @@ -60,7 +60,7 @@ def start_server(): return _server -def create_service(desc, type, port, properties, add_hostname): +def create_service(desc, type, port, properties, add_hostname, use_ip_address=None): port = int(port) try: hostname = socket.gethostname().partition('.')[0] @@ -69,7 +69,10 @@ def create_service(desc, type, port, properties, add_hostname): if add_hostname: desc += ' (on %s)'%hostname - local_ip = get_external_ip() + if use_ip_address: + local_ip = use_ip_address + else: + local_ip = get_external_ip() type = type+'.local.' from calibre.utils.Zeroconf import ServiceInfo return ServiceInfo(type, desc+'.'+type, @@ -79,7 +82,7 @@ def create_service(desc, type, port, properties, add_hostname): server=hostname+'.local.') -def publish(desc, type, port, properties=None, add_hostname=True): +def publish(desc, type, port, properties=None, add_hostname=True, use_ip_address=None): ''' Publish a service. diff --git a/src/calibre/utils/mem.py b/src/calibre/utils/mem.py index 4358ec7522..3bc1f29636 100644 --- a/src/calibre/utils/mem.py +++ b/src/calibre/utils/mem.py @@ -13,188 +13,23 @@ You can pass a number to memory and it will be subtracted from the returned value. ''' -import gc, os, re +import gc, os from calibre.constants import iswindows, islinux -if islinux: - # Taken, with thanks, from: - # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications - - def permute(args): - ret = [] - if args: - first = args.pop(0) - for y in permute(args): - for x in first: - ret.append(x + y) - else: - ret.append('') - return ret - - def parsed_groups(match, *types): - groups = match.groups() - assert len(groups) == len(types) - return tuple([type(group) for group, type in zip(groups, types)]) - - class VMA(dict): - def __init__(self, *args): - (self.start, self.end, self.perms, self.offset, - self.major, self.minor, self.inode, self.filename) = args - - def parse_smaps(pid): - with open('/proc/%s/smaps'%pid, 'r') as maps: - hex = lambda s: int(s, 16) - - ret = [] - header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) ' - r'(..):(..) (\d+) *(.*)$') - detail = re.compile(r'^(.*): +(\d+) kB') - for line in maps: - m = header.match(line) - if m: - vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str)) - ret.append(vma) - else: - m = detail.match(line) - if m: - k, v = parsed_groups(m, str, int) - assert k not in vma - vma[k] = v - else: - print 'unparseable line:', line - return ret - - perms = permute(['r-', 'w-', 'x-', 'ps']) - - def make_summary_dicts(vmas): - mapped = {} - anon = {} - for d in mapped, anon: - # per-perm - for k in perms: - d[k] = {} - d[k]['Size'] = 0 - for y in 'Shared', 'Private': - d[k][y] = {} - for z in 'Clean', 'Dirty': - d[k][y][z] = 0 - # totals - for y in 'Shared', 'Private': - d[y] = {} - for z in 'Clean', 'Dirty': - d[y][z] = 0 - - for vma in vmas: - if vma.major == '00' and vma.minor == '00': - d = anon - else: - d = mapped - for y in 'Shared', 'Private': - for z in 'Clean', 'Dirty': - d[vma.perms][y][z] += vma.get(y + '_' + z, 0) - d[y][z] += vma.get(y + '_' + z, 0) - d[vma.perms]['Size'] += vma.get('Size', 0) - return mapped, anon - - def values(d, args): - if args: - ret = () - first = args[0] - for k in first: - ret += values(d[k], args[1:]) - return ret - else: - return (d,) - - def print_summary(dicts_and_titles): - def desc(title, perms): - ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)', - ('Anonymous', 'rwxp'): 'Writable code (stack)', - ('Mapped', 'r-xp'): 'Code', - ('Mapped', 'rwxp'): 'Writable code (jump tables)', - ('Mapped', 'r--p'): 'Read-only data', - ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None) - if ret: - return ' -- ' + ret - else: - return '' - - for d, title in dicts_and_titles: - print title, 'memory:' - print ' Shared Private' - print ' Clean Dirty Clean Dirty' - for k in perms: - if d[k]['Size']: - print (' %s %7d %7d %7d %7d%s' - % ((k,) - + values(d[k], (('Shared', 'Private'), - ('Clean', 'Dirty'))) - + (desc(title, k),))) - print (' total %7d %7d %7d %7d' - % values(d, (('Shared', 'Private'), - ('Clean', 'Dirty')))) - - print ' ' + '-' * 40 - print (' total %7d %7d %7d %7d' - % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'), - ('Clean', 'Dirty'))) - for d, title in dicts_and_titles])))) - - def print_stats(pid=None): - if pid is None: - pid = os.getpid() - vmas = parse_smaps(pid) - mapped, anon = make_summary_dicts(vmas) - print_summary(((mapped, "Mapped"), (anon, "Anonymous"))) - - def linux_memory(since=0.0): - vmas = parse_smaps(os.getpid()) - mapped, anon = make_summary_dicts(vmas) - dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous")) - totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'), - ('Clean', 'Dirty'))) - for d, title in dicts_and_titles]))) - return (totals[-1]/1024.) - since - - memory = linux_memory - -elif iswindows: - import win32process - import win32con - import win32api - - # See http://msdn.microsoft.com/en-us/library/ms684877.aspx - # for details on the info returned by get_meminfo - - def get_handle(pid): - return win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0, - pid) - - def listprocesses(self): - for process in win32process.EnumProcesses(): - try: - han = get_handle(process) - procmeminfo = meminfo(han) - procmemusage = procmeminfo["WorkingSetSize"] - yield process, procmemusage - except: - pass - - def get_meminfo(pid): - han = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0, - pid) - return meminfo(han) - - def meminfo(handle): - return win32process.GetProcessMemoryInfo(handle) - - def win_memory(since=0.0): - info = meminfo(get_handle(os.getpid())) - return (info['WorkingSetSize']/1024.**2) - since - - memory = win_memory +def get_memory(): + 'Return memory usage in bytes' + import psutil + p = psutil.Process(os.getpid()) + mem = p.get_ext_memory_info() + attr = 'wset' if iswindows else 'data' if islinux else 'rss' + return getattr(mem, attr) +def memory(since=0.0): + 'Return memory used in MB. The value of since is subtracted from the used memory' + ans = get_memory() + ans /= float(1024**2) + return ans - since def gc_histogram(): """Returns per-class counts of existing objects."""