0.8.67+

recipes/catavencu.recipe

@@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AcademiaCatavencu(BasicNewsRecipe):
     title                 = u'Academia Ca\u0163avencu'
     __author__            = u'Silviu Cotoar\u0103'
-    description           = 'Tagma cum laude'
+    description           = 'Academia Catavencu. Pamflete!'
     publisher             = u'Ca\u0163avencu'
     oldest_article        = 5
     language              = 'ro'
@@ -21,7 +21,7 @@ class AcademiaCatavencu(BasicNewsRecipe):
     use_embedded_content  = False
     category              = 'Ziare'
     encoding              = 'utf-8'
-    cover_url         = 'http://www.academiacatavencu.info/images/logo.png'
+    cover_url         = 'http://www.inpolitics.ro/Uploads/Articles/academia_catavencu.jpg'
 
     conversion_options = {
                              'comments'   : description
@@ -31,21 +31,21 @@ class AcademiaCatavencu(BasicNewsRecipe):
                          }
 
     keep_only_tags = [
-            dict(name='h1', attrs={'class':'art_title'}),
+            dict(name='h1', attrs={'class':'entry-title'}),
-			dict(name='div', attrs={'class':'art_text'})
+			dict(name='div', attrs={'class':'entry-content'})
                      ]
 
     remove_tags = [
-             dict(name='div', attrs={'class':['desp_m']})
+             dict(name='div', attrs={'class':['mr_social_sharing_wrapper']})
-           , dict(name='div', attrs={'id':['tags']})          
+           , dict(name='div', attrs={'id':['fb_share_1']})          
                   ]
 
     remove_tags_after = [
-              dict(name='div', attrs={'class':['desp_m']})
+              dict(name='div', attrs={'id':['fb_share_1']})
             ]
 
     feeds          = [
-            (u'Feeds', u'http://www.academiacatavencu.info/rss.xml')
+            (u'Feeds', u'http://www.academiacatavencu.info/feed')
                  ]
 
     def preprocess_html(self, soup):

recipes/dilemaveche.recipe

@@ -1,71 +1,51 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+dilemaveche.ro
+'''
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class DilemaVeche(BasicNewsRecipe):
-    title          = u'Dilema Veche' # apare vinerea, mai pe dupa-masa,depinde de Luiza cred (care se semneaza ca fiind creatorul fiecarui articol in feed-ul RSS)
+    title                 = u'Dilema Veche'
-    __author__            = 'song2' # inspirat din scriptul pentru Le Monde. Inspired from the Le Monde script
+    __author__            = u'Silviu Cotoar\u0103'
-    description           = '"Sint vechi, domnule!" (I.L. Caragiale)'
+    description           = 'Sint vechi, domnule! (I.L. Caragiale)'
-    publisher             = 'Adevarul Holding'
+    publisher             = u'Adev\u0103rul Holding'
-    oldest_article        = 7
+    oldest_article        = 5
-    max_articles_per_feed = 200
-    encoding              = 'utf8'
     language              = 'ro'
-    masthead_url = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
+    max_articles_per_feed = 100
-    publication_type = 'magazine'
-    feeds = [
-                ('Editoriale si opinii - Situatiunea', 'http://www.dilemaveche.ro/taxonomy/term/37/0/feed'),
-                ('Editoriale si opinii - Pe ce lume traim', 'http://www.dilemaveche.ro/taxonomy/term/38/0/feed'),
-                ('Editoriale si opinii - Bordeie si obiceie', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
-                ('Editoriale si opinii - Talc Show', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
-                ('Tema saptamanii', 'http://www.dilemaveche.ro/taxonomy/term/19/0/feed'),
-                ('La zi in cultura - Dilema va recomanda', 'http://www.dilemaveche.ro/taxonomy/term/58/0/feed'),
-                ('La zi in cultura - Carte', 'http://www.dilemaveche.ro/taxonomy/term/14/0/feed'),
-                ('La zi in cultura - Film', 'http://www.dilemaveche.ro/taxonomy/term/13/0/feed'),
-                ('La zi in cultura - Muzica', 'http://www.dilemaveche.ro/taxonomy/term/1341/0/feed'),
-                ('La zi in cultura - Arte performative', 'http://www.dilemaveche.ro/taxonomy/term/1342/0/feed'),
-                ('La zi in cultura - Arte vizuale', 'http://www.dilemaveche.ro/taxonomy/term/1512/0/feed'),
-                ('Societate - Ieri cu vedere spre azi', 'http://www.dilemaveche.ro/taxonomy/term/15/0/feed'),
-                ('Societate - Din polul opus', 'http://www.dilemaveche.ro/taxonomy/term/41/0/feed'),
-                ('Societate - Mass comedia', 'http://www.dilemaveche.ro/taxonomy/term/43/0/feed'),
-                ('Societate - La singular si la plural', 'http://www.dilemaveche.ro/taxonomy/term/42/0/feed'),
-                ('Oameni si idei - Educatie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
-                ('Oameni si idei - Polemici si dezbateri', 'http://www.dilemaveche.ro/taxonomy/term/48/0/feed'),
-                ('Oameni si idei - Stiinta si tehnologie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
-                ('Dileme on-line', 'http://www.dilemaveche.ro/taxonomy/term/005/0/feed')
-                 ]
-    remove_tags_before = dict(name='div',attrs={'class':'spacer_10'})
-    remove_tags = [
-        dict(name='div', attrs={'class':'art_related_left'}),
-        dict(name='div', attrs={'class':'controale'}),
-		dict(name='div', attrs={'class':'simple_overlay'}),
-    ]
-    remove_tags_after = [dict(id='facebookLike')]
-    remove_javascript = True
     no_stylesheets        = True
-    remove_empty_feeds = True
+    use_embedded_content  = False
-    extra_css             = """
+    category              = 'Ziare'
-        body{font-family: Georgia,Times,serif }
+    encoding              = 'utf-8'
-        img{margin-bottom: 0.4em; display:block}
+    cover_url             = 'http://dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
-                            """
-    def get_cover_url(self):
-        cover_url = None
-        soup = self.index_to_soup('http://dilemaveche.ro')
-        link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
-        if link_item and link_item.a:
-           cover_url = link_item.a['href']
-        br = BasicNewsRecipe.get_browser()
-        try:
-            br.open(cover_url)
-        except: #daca nu gaseste pdf-ul
-            self.log("\nPDF indisponibil")
-            link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
-            if link_item and link_item.img:
-                cover_url = link_item.img['src']
-            br = BasicNewsRecipe.get_browser()
-            try:
-                 br.open(cover_url)
-            except: #daca nu gaseste nici imaginea mica mica
-                print('Mama lor de nenorociti! nu este nici pdf nici imagine')
-                cover_url ='http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
-        return cover_url
-    cover_margins = (10, 15, '#ffffff')
 
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+							,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+            dict(name='div', attrs={'class':'c_left_column'})
+                     ]
+
+    remove_tags = [
+             dict(name='div', attrs={'id':['adshop_widget_428x60']}) ,
+			 dict(name='div', attrs={'id':['gallery']})			 
+                  ]
+
+    remove_tags_after = [
+              dict(name='div', attrs={'id':['adshop_widget_428x60']})    
+            ]
+
+    feeds          = [
+            (u'Feeds', u'http://dilemaveche.ro/rss.xml')
+                 ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/houston_chronicle.recipe

@@ -13,12 +13,13 @@ class HoustonChronicle(BasicNewsRecipe):
     no_stylesheets = True
     use_embedded_content = False
     remove_attributes = ['style']
+    auto_cleanup = True
 
     oldest_article = 2.0
 
-    keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or
+    #keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or
-        'hst-articletext' in x or 'hst-galleryitem' in x)}
+        #'hst-articletext' in x or 'hst-galleryitem' in x)}
-    remove_attributes = ['xmlns']
+    #remove_attributes = ['xmlns']
 
     feeds = [
             ('News', "http://www.chron.com/rss/feed/News-270.php"),
@@ -37,3 +38,4 @@ class HoustonChronicle(BasicNewsRecipe):
         ]
 
 
+ 

recipes/scmp.recipe

@@ -39,10 +39,10 @@ class SCMP(BasicNewsRecipe):
         #br.set_debug_responses(True)
         #br.set_debug_redirects(True)
         if self.username is not None and self.password is not None:
-           br.open('http://www.scmp.com/portal/site/SCMP/')
+           br.open('http://www.scmp.com/')
-           br.select_form(name='loginForm')
+           br.select_form(nr=1)
-           br['Login'   ] = self.username
+           br['name'] = self.username
-           br['Password'] = self.password
+           br['pass'] = self.password
            br.submit()
         return br
 

recipes/timesnewroman.recipe

@@ -36,12 +36,14 @@ class TimesNewRoman(BasicNewsRecipe):
 
     remove_tags = [
                 dict(name='p', attrs={'class':['articleinfo']})
-              , dict(name='div',attrs={'class':['vergefacebooklike']})
+              , dict(name='div', attrs={'class':['shareTools']})
-                  , dict(name='div', attrs={'class':'cleared'})
+			  ,  dict(name='div', attrs={'class':'fb_iframe_widget'})
+			  , dict(name='div', attrs={'id':'jc'})
                   ]
 
     remove_tags_after = [
-               dict(name='div', attrs={'class':'cleared'})
+               dict(name='div', attrs={'class':'fb_iframe_widget'}), 
+			   dict(name='div', attrs={'id':'jc'})
                     ]
 
     feeds          = [

session.vim

@@ -13,6 +13,8 @@ let g:syntastic_cpp_include_dirs = [
             \]
 let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs
 
+set wildignore+=resources/viewer/mathjax/**
+
 fun! CalibreLog()
     " Setup buffers to edit the calibre changelog and version info prior to
     " making a release.

setup/extensions.py

@@ -187,7 +187,7 @@ if iswindows:
             headers=[
                 'calibre/devices/mtp/windows/global.h',
             ],
-            libraries=['ole32', 'portabledeviceguids', 'user32'],
+            libraries=['ole32', 'oleaut32', 'portabledeviceguids', 'user32'],
             # needs_ddk=True,
             cflags=['/X']
             ),

setup/installer/linux/freeze2.py

@@ -15,7 +15,8 @@ from setup import Command, modules, basenames, functions, __version__, \
 SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize',
         'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml',
         'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py',
-        '_dbus_glib_bindings.so', 'netifaces.so']
+        '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so',
+        '_psutil_linux.so', 'psutil']
 
 QTDIR          = '/usr/lib/qt4'
 QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus')

setup/installer/windows/notes.rst

@@ -360,6 +360,15 @@ Run
 python setup.py build
 cp build/lib.win32-2.7/netifaces.pyd /cygdrive/c/Python27/Lib/site-packages/
 
+psutil
+--------
+
+Download the source tarball
+
+Run
+
+Python setup.py build
+cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/
 
 calibre
 ---------

setup/upload.py

@@ -47,6 +47,21 @@ def installer_description(fname):
         return 'Calibre Portable'
     return 'Unknown file'
 
+def upload_signatures():
+    tdir = mkdtemp()
+    for installer in installers():
+        if not os.path.exists(installer):
+            continue
+        with open(installer, 'rb') as f:
+            raw = f.read()
+        fingerprint = hashlib.sha512(raw).hexdigest()
+        fname = os.path.basename(installer+'.sha512')
+        with open(os.path.join(tdir, fname), 'wb') as f:
+            f.write(fingerprint)
+    check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS),
+            shell=True)
+    shutil.rmtree(tdir)
+
 class ReUpload(Command): # {{{
 
     description = 'Re-uplaod any installers present in dist/'
@@ -57,6 +72,7 @@ class ReUpload(Command): # {{{
         opts.replace = True
 
     def run(self, opts):
+        upload_signatures()
         for x in installers():
             if os.path.exists(x):
                 os.remove(x)
@@ -223,19 +239,7 @@ class UploadToServer(Command): # {{{
                    %(__version__, DOWNLOADS), shell=True)
         check_call('ssh divok /etc/init.d/apache2 graceful',
                    shell=True)
-        tdir = mkdtemp()
+        upload_signatures()
-        for installer in installers():
-            if not os.path.exists(installer):
-                continue
-            with open(installer, 'rb') as f:
-                raw = f.read()
-            fingerprint = hashlib.sha512(raw).hexdigest()
-            fname = os.path.basename(installer+'.sha512')
-            with open(os.path.join(tdir, fname), 'wb') as f:
-                f.write(fingerprint)
-        check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS),
-                shell=True)
-        shutil.rmtree(tdir)
 # }}}
 
 # Testing {{{

src/calibre/devices/__init__.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Device drivers.
 '''
 
-import sys, time, pprint, operator, re, os
+import sys, time, pprint, operator
 from functools import partial
 from StringIO import StringIO
 
@@ -27,112 +27,6 @@ def strftime(epoch, zone=time.gmtime):
     src[2] = INVERSE_MONTH_MAP[int(src[2])]
     return ' '.join(src)
 
-def build_template_regexp(template):
-    from calibre import prints
-
-    def replfunc(match, seen=None):
-        v = match.group(1)
-        if v in ['authors', 'author_sort']:
-            v = 'author'
-        if v in ('title', 'series', 'series_index', 'isbn', 'author'):
-            if v not in seen:
-                seen.add(v)
-                return '(?P<' + v + '>.+?)'
-        return '(.+?)'
-    s = set()
-    f = partial(replfunc, seen=s)
-
-    try:
-        template = template.rpartition('/')[2]
-        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
-    except:
-        prints(u'Failed to parse template: %r'%template)
-        template = u'{title} - {authors}'
-        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
-
-def create_upload_path(mdata, fname, template, sanitize,
-        prefix_path='',
-        path_type=os.path,
-        maxlen=250,
-        use_subdirs=True,
-        news_in_folder=True,
-        filename_callback=lambda x, y:x,
-        sanitize_path_components=lambda x: x
-        ):
-    from calibre.library.save_to_disk import get_components, config
-    from calibre.utils.filenames import shorten_components_to
-
-    special_tag = None
-    if mdata.tags:
-        for t in mdata.tags:
-            if t.startswith(_('News')) or t.startswith('/'):
-                special_tag = t
-                break
-
-    if mdata.tags and _('News') in mdata.tags:
-        try:
-            p = mdata.pubdate
-            date  = (p.year, p.month, p.day)
-        except:
-            today = time.localtime()
-            date = (today[0], today[1], today[2])
-        template = u"{title}_%d-%d-%d" % date
-
-    fname = sanitize(fname)
-    ext = path_type.splitext(fname)[1]
-
-    opts = config().parse()
-    if not isinstance(template, unicode):
-        template = template.decode('utf-8')
-    app_id = str(getattr(mdata, 'application_id', ''))
-    id_ = mdata.get('id', fname)
-    extra_components = get_components(template, mdata, id_,
-            timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
-    if not extra_components:
-        extra_components.append(sanitize(filename_callback(fname,
-            mdata)))
-    else:
-        extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata))
-
-    if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
-        extra_components[-1] = 'x' + extra_components[-1][1:]
-
-    if special_tag is not None:
-        name = extra_components[-1]
-        extra_components = []
-        tag = special_tag
-        if tag.startswith(_('News')):
-            if news_in_folder:
-                extra_components.append('News')
-        else:
-            for c in tag.split('/'):
-                c = sanitize(c)
-                if not c: continue
-                extra_components.append(c)
-        extra_components.append(name)
-
-    if not use_subdirs:
-        extra_components = extra_components[-1:]
-
-    def remove_trailing_periods(x):
-        ans = x
-        while ans.endswith('.'):
-            ans = ans[:-1].strip()
-        if not ans:
-            ans = 'x'
-        return ans
-
-    extra_components = list(map(remove_trailing_periods, extra_components))
-    components = shorten_components_to(maxlen - len(prefix_path), extra_components)
-    components = sanitize_path_components(components)
-    if prefix_path:
-        filepath = path_type.join(prefix_path, *components)
-    else:
-        filepath = path_type.join(*components)
-
-    return filepath
-
-
 def get_connected_device():
     from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner

src/calibre/devices/android/driver.py

@@ -186,10 +186,15 @@ class ANDROID(USBMS):
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books',
             'sdcard/ebooks']
-    EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
+    EXTRA_CUSTOMIZATION_MESSAGE = [_('Comma separated list of directories to '
-            'send e-books to on the device. The first one that exists will '
+            'send e-books to on the device\'s <b>main memory</b>. The first one that exists will '
+            'be used'),
+            _('Comma separated list of directories to '
+            'send e-books to on the device\'s <b>storage cards</b>. The first one that exists will '
             'be used')
-    EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN)
+            ]
+
+    EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(EBOOK_DIR_MAIN), '']
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
@@ -237,23 +242,35 @@ class ANDROID(USBMS):
 
     def post_open_callback(self):
         opts = self.settings()
-        dirs = opts.extra_customization
+        opts = opts.extra_customization
-        if not dirs:
+        if not opts:
-            dirs = self.EBOOK_DIR_MAIN
+            opts = [self.EBOOK_DIR_MAIN, '']
-        else:
+
-            dirs = [x.strip() for x in dirs.split(',')]
+        def strtolist(x):
-        self.EBOOK_DIR_MAIN = dirs
+            if isinstance(x, basestring):
+                x = [y.strip() for y in x.split(',')]
+            return x or []
+
+        opts = [strtolist(x) for x in opts]
+        self._android_main_ebook_dir = opts[0]
+        self._android_card_ebook_dir = opts[1]
 
     def get_main_ebook_dir(self, for_upload=False):
-        dirs = self.EBOOK_DIR_MAIN
+        dirs = self._android_main_ebook_dir
         if not for_upload:
             def aldiko_tweak(x):
                 return 'eBooks' if x == 'eBooks/import' else x
-            if isinstance(dirs, basestring):
-                dirs = [dirs]
             dirs = list(map(aldiko_tweak, dirs))
         return dirs
 
+    def get_carda_ebook_dir(self, for_upload=False):
+        if not for_upload:
+            return ''
+        return self._android_card_ebook_dir
+
+    def get_cardb_ebook_dir(self, for_upload=False):
+        return self.get_carda_ebook_dir()
+
     def windows_sort_drives(self, drives):
         try:
             vid, pid, bcd = self.device_being_opened[:3]
@@ -271,7 +288,8 @@ class ANDROID(USBMS):
         proxy = cls._configProxy()
         proxy['format_map'] = ['mobi', 'azw', 'azw1', 'azw4', 'pdf']
         proxy['use_subdirs'] = False
-        proxy['extra_customization'] = ','.join(['kindle']+cls.EBOOK_DIR_MAIN)
+        proxy['extra_customization'] = [
+                ','.join(['kindle']+cls.EBOOK_DIR_MAIN), '']
 
     @classmethod
     def configure_for_generic_epub_app(cls):

src/calibre/devices/interface.py

@@ -215,7 +215,9 @@ class DevicePlugin(Plugin):
 
         Scan for devices that this driver can handle. Should return a device
         object if a device is found. This object will be passed to the open()
-        method as the connected_device. If no device is found, return None.
+        method as the connected_device. If no device is found, return None. The
+        returned object can be anything, calibre does not use it, it is only
+        passed to open().
 
         This method is called periodically by the GUI, so make sure it is not
         too resource intensive. Use a cache to avoid repeatedly scanning the

src/calibre/devices/mtp/base.py

@@ -25,7 +25,7 @@ def synchronous(func):
     return synchronizer
 
 class MTPDeviceBase(DevicePlugin):
-    name = 'SmartDevice App Interface'
+    name = 'MTP Device Interface'
     gui_name = _('MTP Device')
     icon = I('devices/galaxy_s3.png')
     description = _('Communicate with MTP devices')
@@ -37,6 +37,7 @@ class MTPDeviceBase(DevicePlugin):
         self.progress_reporter = None
         self.current_friendly_name = None
         self.report_progress = lambda x, y: None
+        self.current_serial_num = None
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
@@ -45,8 +46,9 @@ class MTPDeviceBase(DevicePlugin):
     def set_progress_reporter(self, report_progress):
         self.report_progress = report_progress
 
-    def get_gui_name(self):
+    @classmethod
-        return self.current_friendly_name or self.name
+    def get_gui_name(cls):
+        return getattr(cls, 'current_friendly_name', cls.gui_name)
 
     def is_usb_connected(self, devices_on_system, debug=False,
             only_presence=False):
@@ -55,7 +57,7 @@ class MTPDeviceBase(DevicePlugin):
         return False
 
     def build_template_regexp(self):
-        from calibre.devices import build_template_regexp
+        from calibre.devices.utils import build_template_regexp
         return build_template_regexp(self.save_template)
 
     @property

src/calibre/devices/mtp/books.py

@@ -22,6 +22,22 @@ class BookList(BL):
     def supports_collections(self):
         return False
 
+    def add_book(self, book, replace_metadata=True):
+        try:
+            b = self.index(book)
+        except (ValueError, IndexError):
+            b = None
+        if b is None:
+            self.append(book)
+            return book
+        if replace_metadata:
+            self[b].smart_update(book, replace_metadata=True)
+            return self[b]
+        return None
+
+    def remove_book(self, book):
+        self.remove(book)
+
 class Book(Metadata):
 
     def __init__(self, storage_id, lpath, other=None):
@@ -36,6 +52,17 @@ class Book(Metadata):
         return (self.storage_id == mtp_file.storage_id and
                 self.mtp_relpath == mtp_file.mtp_relpath)
 
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and (self.storage_id ==
+            other.storage_id and self.mtp_relpath == other.mtp_relpath))
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash((self.storage_id, self.mtp_relpath))
+
+
 class JSONCodec(JsonCodec):
     pass
 

src/calibre/devices/mtp/driver.py

@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import json, traceback, posixpath, importlib, os
 from io import BytesIO
+from itertools import izip
 
 from calibre import prints
 from calibre.constants import iswindows, numeric_version
@@ -32,6 +33,12 @@ class MTP_DEVICE(BASE):
     CAN_SET_METADATA = []
     BACKLOADING_ERROR_MESSAGE = None
     MANAGES_DEVICE_PRESENCE = True
+    FORMATS = ['epub', 'azw3', 'mobi', 'pdf']
+    DEVICE_PLUGBOARD_NAME = 'MTP_DEVICE'
+
+    def __init__(self, *args, **kwargs):
+        BASE.__init__(self, *args, **kwargs)
+        self.plugboards = self.plugboard_func = None
 
     def open(self, devices, library_uuid):
         self.current_library_uuid = library_uuid
@@ -74,12 +81,7 @@ class MTP_DEVICE(BASE):
         return tuple( list(dinfo) + [self.driveinfo] )
 
     def card_prefix(self, end_session=True):
-        ans = [None, None]
+        return (self._carda_id, self._cardb_id)
-        if self._carda_id is not None:
-            ans[0] = self.filesystem_cache.storage(self._carda_id).storage_prefix
-        if self._cardb_id is not None:
-            ans[1] = self.filesystem_cache.storage(self._cardb_id).storage_prefix
-        return tuple(ans)
 
     def set_driveinfo_name(self, location_code, name):
         sid = {'main':self._main_id, 'A':self._carda_id,
@@ -189,6 +191,7 @@ class MTP_DEVICE(BASE):
         self.put_file(storage, self.METADATA_CACHE, stream, size)
 
     def sync_booklists(self, booklists, end_session=True):
+        debug('sync_booklists() called')
         for bl in booklists:
             if getattr(bl, 'storage_id', None) is None:
                 continue
@@ -196,6 +199,7 @@ class MTP_DEVICE(BASE):
             if storage is None:
                 continue
             self.write_metadata_cache(storage, bl)
+        debug('sync_booklists() ended')
 
     # }}}
 
@@ -225,8 +229,14 @@ class MTP_DEVICE(BASE):
         return ans
     # }}}
 
+    # Sending files to the device {{{
+
+    def set_plugboards(self, plugboards, pb_func):
+        self.plugboards = plugboards
+        self.plugboard_func = pb_func
+
     def create_upload_path(self, path, mdata, fname):
-        from calibre.devices import create_upload_path
+        from calibre.devices.utils import create_upload_path
         from calibre.utils.filenames import ascii_filename as sanitize
         filepath = create_upload_path(mdata, fname, self.save_template, sanitize,
                 prefix_path=path,
@@ -235,7 +245,136 @@ class MTP_DEVICE(BASE):
                 use_subdirs = True,
                 news_in_folder = self.NEWS_IN_FOLDER,
                 )
-        return tuple(x.lower() for x in filepath.split('/'))
+        return tuple(x for x in filepath.split('/'))
+
+    def prefix_for_location(self, on_card):
+        # TODO: Implement this
+        return 'calibre'
+
+    def ensure_parent(self, storage, path):
+        parent = storage
+        pos = list(path)[:-1]
+        while pos:
+            name = pos[0]
+            pos = pos[1:]
+            parent = self.create_folder(parent, name)
+        return parent
+
+    def upload_books(self, files, names, on_card=None, end_session=True,
+                     metadata=None):
+        debug('upload_books() called')
+        from calibre.devices.utils import sanity_check
+        sanity_check(on_card, files, self.card_prefix(), self.free_space())
+        prefix = self.prefix_for_location(on_card)
+        sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(on_card,
+                self._main_id)
+        bl_idx = {'carda':1, 'cardb':2}.get(on_card, 0)
+        storage = self.filesystem_cache.storage(sid)
+
+        ans = []
+        self.report_progress(0, _('Transferring books to device...'))
+        i, total = 0, len(files)
+
+        for infile, fname, mi in izip(files, names, metadata):
+            path = self.create_upload_path(prefix, mi, fname)
+            parent = self.ensure_parent(storage, path)
+            if hasattr(infile, 'read'):
+                pos = infile.tell()
+                infile.seek(0, 2)
+                sz = infile.tell()
+                infile.seek(pos)
+                stream = infile
+                close = False
+            else:
+                sz = os.path.getsize(infile)
+                stream = lopen(infile, 'rb')
+                close = True
+            try:
+                mtp_file = self.put_file(parent, path[-1], stream, sz)
+            finally:
+                if close:
+                    stream.close()
+            ans.append((mtp_file, bl_idx))
+            i += 1
+            self.report_progress(i/total, _('Transferred %s to device')%mi.title)
+
+        self.report_progress(1, _('Transfer to device finished...'))
+        debug('upload_books() ended')
+        return ans
+
+    def add_books_to_metadata(self, mtp_files, metadata, booklists):
+        debug('add_books_to_metadata() called')
+        from calibre.devices.mtp.books import Book
+
+        i, total = 0, len(mtp_files)
+        self.report_progress(0, _('Adding books to device metadata listing...'))
+        for x, mi in izip(mtp_files, metadata):
+            mtp_file, bl_idx = x
+            bl = booklists[bl_idx]
+            book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath),
+                    other=mi)
+            book = bl.add_book(book, replace_metadata=True)
+            if book is not None:
+                book.size = mtp_file.size
+                book.datetime = mtp_file.last_modified.timetuple()
+                book.path = mtp_file.mtp_id_path
+            i += 1
+            self.report_progress(i/total, _('Added %s')%mi.title)
+
+        self.report_progress(1, _('Adding complete'))
+        debug('add_books_to_metadata() ended')
+
+    # }}}
+
+    # Removing books from the device {{{
+    def recursive_delete(self, obj):
+        parent = self.delete_file_or_folder(obj)
+        if parent.empty and parent.can_delete and not parent.is_system:
+            try:
+                self.recursive_delete(parent)
+            except:
+                prints('Failed to delete parent: %s, ignoring'%(
+                    '/'.join(parent.full_path)))
+
+    def delete_books(self, paths, end_session=True):
+        self.report_progress(0, _('Deleting books from device...'))
+
+        for i, path in enumerate(paths):
+            f = self.filesystem_cache.resolve_mtp_id_path(path)
+            self.recursive_delete(f)
+            self.report_progress((i+1) / float(len(paths)),
+                    _('Deleted %s')%path)
+        self.report_progress(1, _('All books deleted'))
+
+    def remove_books_from_metadata(self, paths, booklists):
+        self.report_progress(0, _('Removing books from metadata'))
+        class NextPath(Exception): pass
+
+        for i, path in enumerate(paths):
+            try:
+                for bl in booklists:
+                    for book in bl:
+                        if book.path == path:
+                            bl.remove_book(book)
+                            raise NextPath('')
+            except NextPath:
+                pass
+            self.report_progress((i+1)/len(paths), _('Removed %s')%path)
+
+        self.report_progress(1, _('All books removed'))
+
+    # }}}
+
+    # Settings {{{
+    @classmethod
+    def settings(self):
+        # TODO: Implement this
+        class Opts(object):
+            def __init__(s):
+                s.format_map = self.FORMATS
+        return Opts()
+
+    # }}}
 
 if __name__ == '__main__':
     dev = MTP_DEVICE(None)
@@ -250,7 +389,7 @@ if __name__ == '__main__':
             raise ValueError('Failed to detect MTP device')
         dev.set_progress_reporter(prints)
         dev.open(cd, None)
-        dev.books()
+        dev.filesystem_cache.dump()
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -37,9 +37,13 @@ class FileOrFolder(object):
         self.size = entry.get('size', 0)
         md = entry.get('modified', 0)
         try:
+            if isinstance(md, tuple):
+                self.last_modified = datetime(*(list(md)+[local_tz]))
+            else:
                 self.last_modified = datetime.fromtimestamp(md, local_tz)
         except:
             self.last_modified = datetime.fromtimestamp(0, local_tz)
+        self.last_mod_string = self.last_modified.strftime('%Y/%m/%d %H:%M')
         self.last_modified = as_utc(self.last_modified)
 
         if self.storage_id not in self.all_storage_ids:
@@ -74,12 +78,16 @@ class FileOrFolder(object):
         datum = 'size=%s'%(self.size)
         if self.is_folder:
             datum = 'children=%s'%(len(self.files) + len(self.folders))
-        return '%s(id=%s, storage_id=%s, %s, path=%s)'%(name, self.object_id,
+        return '%s(id=%s, storage_id=%s, %s, path=%s, modified=%s)'%(name, self.object_id,
-                self.storage_id, datum, path)
+                self.storage_id, datum, path, self.last_mod_string)
 
     __str__ = __repr__
     __unicode__ = __repr__
 
+    @property
+    def empty(self):
+        return not self.files and not self.folders
+
     @property
     def id_map(self):
         return self.fs_cache().id_map
@@ -123,6 +131,7 @@ class FileOrFolder(object):
         c = '+' if self.is_folder else '-'
         data = ('%s children'%(sum(map(len, (self.files, self.folders))))
             if self.is_folder else human_readable(self.size))
+        data += ' modified=%s'%self.last_mod_string
         line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data)
         prints(line, file=out)
         for c in (self.folders, self.files):
@@ -217,6 +226,8 @@ class FilesystemCache(object):
     def iterebooks(self, storage_id):
         for x in self.id_map.itervalues():
             if x.storage_id == storage_id and x.is_ebook:
+                if x.parent_id == storage_id and x.name.lower().endswith('.txt'):
+                    continue # Ignore .txt files in the root
                 yield x
 
     def resolve_mtp_id_path(self, path):

src/calibre/devices/mtp/unix/driver.py

@@ -129,6 +129,7 @@ class MTP_DEVICE(MTPDeviceBase):
     def post_yank_cleanup(self):
         self.dev = self._filesystem_cache = self.current_friendly_name = None
         self.currently_connected_dev = None
+        self.current_serial_num = None
 
     @synchronous
     def startup(self):
@@ -173,6 +174,9 @@ class MTP_DEVICE(MTPDeviceBase):
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
         self.current_friendly_name = self.dev.friendly_name
+        if not self.current_friendly_name:
+            self.current_friendly_name = self.dev.model_name or _('Unknown MTP device')
+        self.current_serial_num = self.dev.serial_number
 
     @property
     def filesystem_cache(self):
@@ -306,6 +310,7 @@ class MTP_DEVICE(MTPDeviceBase):
             raise DeviceError('Failed to delete %s with error: %s'%
                 (obj.full_path, self.format_errorstack(errs)))
         parent.remove_child(obj)
+        return parent
 
 def develop():
     from calibre.devices.scanner import DeviceScanner

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -84,13 +84,21 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py
 
 static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) {
     FLOAT val = 0;
+    SYSTEMTIME st;
+    unsigned int microseconds;
     PyObject *t;
 
     if (SUCCEEDED(properties->GetFloatValue(key, &val))) {
-        t = Py_BuildValue("d", (double)val);
+        if (VariantTimeToSystemTime(val, &st)) {
+            microseconds = 1000 * st.wMilliseconds;
+            t = Py_BuildValue("H H H H H H I", (unsigned short)st.wYear,
+                    (unsigned short)st.wMonth, (unsigned short)st.wDay,
+                    (unsigned short)st.wHour, (unsigned short)st.wMinute,
+                    (unsigned short)st.wSecond, microseconds);
             if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); }
         }
     }
+}
 
 static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) {
     GUID guid = GUID_NULL;

src/calibre/devices/mtp/windows/driver.py

@@ -231,10 +231,12 @@ class MTP_DEVICE(MTPDeviceBase):
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
         self.dev = self._filesystem_cache = None
+        self.current_serial_num = None
 
     def eject(self):
         if self.currently_connected_pnp_id is None: return
         self.eject_dev_on_next_scan = True
+        self.current_serial_num = None
 
     @same_thread
     def open(self, connected_device, library_uuid):
@@ -259,9 +261,12 @@ class MTP_DEVICE(MTPDeviceBase):
             self._carda_id = storage[1]['id']
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
-        self.current_friendly_name = devdata.get('friendly_name',
+        self.current_friendly_name = devdata.get('friendly_name', '')
+        if not self.current_friendly_name:
+            self.current_friendly_name = devdata.get('model_name',
                 _('Unknown MTP device'))
         self.currently_connected_pnp_id = connected_device
+        self.current_serial_num = devdata.get('serial_number', None)
 
     @same_thread
     def get_basic_device_information(self):
@@ -338,6 +343,7 @@ class MTP_DEVICE(MTPDeviceBase):
         parent = obj.parent
         self.dev.delete_object(obj.object_id)
         parent.remove_child(obj)
+        return parent
 
     @same_thread
     def put_file(self, parent, name, stream, size, callback=None, replace=True):

src/calibre/devices/mtp/windows/remote.py

@@ -54,9 +54,9 @@ def main():
     plugins._plugins['wpd'] = (wpd, '')
     sys.path.pop(0)
 
-    from calibre.devices.mtp.test import run
+    # from calibre.devices.mtp.test import run
-    run()
+    # run()
-    return
+    # return
 
     from calibre.devices.scanner import win_scanner
     from calibre.devices.mtp.windows.driver import MTP_DEVICE
@@ -81,13 +81,13 @@ def main():
         # print ('Fetching file: oFF (198214 bytes)')
         # stream = dev.get_file('oFF')
         # print ("Fetched size: ", stream.tell())
-        size = 4
+        # size = 4
-        stream = io.BytesIO(b'a'*size)
+        # stream = io.BytesIO(b'a'*size)
-        name = 'zzz-test-file.txt'
+        # name = 'zzz-test-file.txt'
-        stream.seek(0)
+        # stream.seek(0)
-        f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size)
+        # f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size)
-        print ('Put file:', f)
+        # print ('Put file:', f)
-        # dev.filesystem_cache.dump()
+        dev.filesystem_cache.dump()
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/windows/wpd.cpp

@@ -120,14 +120,14 @@ wpd_enumerate_devices(PyObject *self, PyObject *args) {
         hresult_set_exc("Failed to get list of portable devices", hr);
     }
 
-    for (i = 0; i < num_of_devices; i++) {
     Py_BEGIN_ALLOW_THREADS;
+    for (i = 0; i < num_of_devices; i++) {
         CoTaskMemFree(pnp_device_ids[i]);
-        Py_END_ALLOW_THREADS;
         pnp_device_ids[i] = NULL;
     }
     free(pnp_device_ids);
     pnp_device_ids = NULL;
+    Py_END_ALLOW_THREADS;
 
     return Py_BuildValue("N", ans);
 } // }}}

src/calibre/devices/smart_device_app/driver.py

@@ -33,7 +33,7 @@ from calibre.utils.config import from_json, tweaks
 from calibre.utils.date import isoformat, now
 from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
 from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
-        unpublish_zeroconf)
+        unpublish_zeroconf, get_all_ips)
 
 def synchronous(tlockname):
     """A decorator to place an instance based lock around a method """
@@ -46,10 +46,6 @@ def synchronous(tlockname):
         return _synchronizer
     return _synched
 
-def do_zeroconf(f, port):
-    f('calibre smart device client',
-        '_calibresmartdeviceapp._tcp', port, {})
-
 
 class SDBook(Book):
     def __init__(self, prefix, lpath, size=None, other=None):
@@ -80,7 +76,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     CAN_DO_DEVICE_DB_PLUGBOARD  = False
     SUPPORTS_SUB_DIRS           = True
     MUST_READ_METADATA          = True
-    NEWS_IN_FOLDER              = False
+    NEWS_IN_FOLDER              = True
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
     MAX_PATH_LEN                = 250
@@ -97,7 +93,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     SEND_NOOP_EVERY_NTH_PROBE   = 5
     DISCONNECT_AFTER_N_SECONDS  = 30*60 # 30 minutes
 
-    ZEROCONF_CLIENT_STRING      = b'calibre smart device client'
+    ZEROCONF_CLIENT_STRING      = b'calibre wireless device client'
 
     # A few "random" port numbers to use for detecting clients using broadcast
     # The clients are expected to broadcast a UDP 'hi there' on all of these
@@ -132,6 +128,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     ALL_BY_TITLE     = _('All by title')
     ALL_BY_AUTHOR    = _('All by author')
+    ALL_BY_SOMETHING = _('All by something')
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
         _('Enable connections at startup') + ':::<p>' +
@@ -149,18 +146,25 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             _('Check this box if requested when reporting problems') + '</p>',
         '',
         _('Comma separated list of metadata fields '
-            'to turn into collections on the device. Possibilities include: ')+\
+            'to turn into collections on the device.') + ':::<p>' +
-                    'series, tags, authors' +\
+            _('Possibilities include: series, tags, authors, etc' +
-            _('. Two special collections are available: %(abt)s:%(abtv)s and %(aba)s:%(abav)s. Add  '
+              '. Three special collections are available: %(abt)s:%(abtv)s, '
+              '%(aba)s:%(abav)s, and %(abs)s:%(absv)s. Add  '
               'these values to the list to enable them. The collections will be '
               'given the name provided after the ":" character.')%dict(
-                            abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR),
+                    abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR,
+                    abs='abs', absv=ALL_BY_SOMETHING),
         '',
         _('Enable the no-activity timeout') + ':::<p>' +
             _('If this box is checked, calibre will automatically disconnect if '
               'a connected device does nothing for %d minutes. Unchecking this '
               ' box disables this timeout, so calibre will never automatically '
               'disconnect.')%(DISCONNECT_AFTER_N_SECONDS/60,) + '</p>',
+        _('Use this IP address') + ':::<p>' +
+            _('Use this option if you want to force the driver to listen on a '
+              'particular IP address. The driver will listen only on the '
+              'entered address, and this address will be the one advertized '
+              'over mDNS (bonjour).') + '</p>',
         ]
     EXTRA_CUSTOMIZATION_DEFAULT = [
                 False,
@@ -173,6 +177,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 '',
                 '',
                 True,
+                ''
     ]
     OPT_AUTOSTART               = 0
     OPT_PASSWORD                = 2
@@ -181,6 +186,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     OPT_EXTRA_DEBUG             = 6
     OPT_COLLECTIONS             = 8
     OPT_AUTODISCONNECT          = 10
+    OPT_FORCE_IP_ADDRESS        = 11
 
 
     def __init__(self, path):
@@ -499,6 +505,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return self.OPT_USE_PORT
         elif opt_string == 'port_number':
             return self.OPT_PORT_NUMBER
+        elif opt_string == 'force_ip_address':
+            return self.OPT_FORCE_IP_ADDRESS
         else:
             return None
 
@@ -527,7 +535,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     def _attach_to_port(self, sock, port):
         try:
-            self._debug('try port', port)
+            ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS]
+            self._debug('try ip address "'+ ip_addr + '"', 'on port', port)
+            if ip_addr:
+                sock.bind((ip_addr, port))
+            else:
                 sock.bind(('', port))
         except socket.error:
             self._debug('socket error on port', port)
@@ -996,6 +1008,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         self.client_can_stream_books = False
         self.client_can_stream_metadata = False
 
+        self._debug("All IP addresses", get_all_ips())
+
         message = None
         try:
             self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -1044,7 +1058,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return message
 
         try:
-            do_zeroconf(publish_zeroconf, port)
+            ip_addr = self.settings().extra_customization[self.OPT_FORCE_IP_ADDRESS]
+            publish_zeroconf('calibre smart device client',
+                             '_calibresmartdeviceapp._tcp', port, {},
+                             use_ip_address=ip_addr)
         except:
             message = 'registration with bonjour failed'
             self._debug(message)
@@ -1080,7 +1097,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     @synchronous('sync_lock')
     def shutdown(self):
         if getattr(self, 'listen_socket', None) is not None:
-            do_zeroconf(unpublish_zeroconf, self.port)
+            unpublish_zeroconf('calibre smart device client',
+                             '_calibresmartdeviceapp._tcp', self.port, {})
             self._close_listen_socket()
 
     # Methods for dynamic control

src/calibre/devices/usbms/device.py

@@ -15,8 +15,7 @@ import os, subprocess, time, re, sys, glob
 from itertools import repeat
 
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.errors import (DeviceError, FreeSpaceError,
+from calibre.devices.errors import DeviceError
-        WrongDestinationError)
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins
 from calibre.utils.filenames import ascii_filename as sanitize
@@ -976,53 +975,32 @@ class Device(DeviceConfig, DevicePlugin):
         return self.EBOOK_DIR_CARD_A
 
     def _sanity_check(self, on_card, files):
-        if on_card == 'carda' and not self._card_a_prefix:
+        from calibre.devices.utils import sanity_check
-            raise WrongDestinationError(_(
+        sanity_check(on_card, files, self.card_prefix(), self.free_space())
-                'The reader has no storage card %s. You may have changed '
-                'the default send to device action. Right click on the send '
-                'to device button and reset the default action to be '
-                '"Send to main memory".')%'A')
-        elif on_card == 'cardb' and not self._card_b_prefix:
-            raise WrongDestinationError(_(
-                'The reader has no storage card %s. You may have changed '
-                'the default send to device action. Right click on the send '
-                'to device button and reset the default action to be '
-                '"Send to main memory".')%'B')
-        elif on_card and on_card not in ('carda', 'cardb'):
-            raise DeviceError(_('Selected slot: %s is not supported.') % on_card)
 
-        if on_card == 'carda':
+        def get_dest_dir(prefix, candidates):
-            path = os.path.join(self._card_a_prefix,
-                    *(self.get_carda_ebook_dir(for_upload=True).split('/')))
-        elif on_card == 'cardb':
-            path = os.path.join(self._card_b_prefix,
-                    *(self.EBOOK_DIR_CARD_B.split('/')))
-        else:
-            candidates = self.get_main_ebook_dir(for_upload=True)
             if isinstance(candidates, basestring):
                 candidates = [candidates]
+            if not candidates:
+                candidates = ['']
             candidates = [
-                    ((os.path.join(self._main_prefix, *(x.split('/')))) if x else
+                ((os.path.join(prefix, *(x.split('/')))) if x else prefix)
-                    self._main_prefix) for x
+                for x in candidates]
-                    in candidates]
             existing = [x for x in candidates if os.path.exists(x)]
             if not existing:
-                existing = candidates[:1]
+                existing = candidates
-            path = existing[0]
+            return existing[0]
 
-        def get_size(obj):
+        if on_card == 'carda':
-            path = getattr(obj, 'name', obj)
+            candidates = self.get_carda_ebook_dir(for_upload=True)
-            return os.path.getsize(path)
+            path = get_dest_dir(self._carda_prefix, candidates)
+        elif on_card == 'cardb':
+            candidates = self.get_cardb_ebook_dir(for_upload=True)
+            path = get_dest_dir(self._cardb_prefix, candidates)
+        else:
+            candidates = self.get_main_ebook_dir(for_upload=True)
+            path = get_dest_dir(self._main_prefix, candidates)
 
-        sizes = [get_size(f) for f in files]
-        size = sum(sizes)
-
-        if not on_card and size > self.free_space()[0] - 2*1024*1024:
-            raise FreeSpaceError(_("There is insufficient free space in main memory"))
-        if on_card == 'carda' and size > self.free_space()[1] - 1024*1024:
-            raise FreeSpaceError(_("There is insufficient free space on the storage card"))
-        if on_card == 'cardb' and size > self.free_space()[2] - 1024*1024:
-            raise FreeSpaceError(_("There is insufficient free space on the storage card"))
         return path
 
     def filename_callback(self, default, mi):
@@ -1052,7 +1030,7 @@ class Device(DeviceConfig, DevicePlugin):
         pass
 
     def create_upload_path(self, path, mdata, fname, create_dirs=True):
-        from calibre.devices import create_upload_path
+        from calibre.devices.utils import create_upload_path
         settings = self.settings()
         filepath = create_upload_path(mdata, fname, self.save_template(), sanitize,
                 prefix_path=os.path.abspath(path),

src/calibre/devices/usbms/driver.py

@@ -404,7 +404,7 @@ class USBMS(CLI, Device):
 
     @classmethod
     def build_template_regexp(cls):
-        from calibre.devices import build_template_regexp
+        from calibre.devices.utils import build_template_regexp
         return build_template_regexp(cls.save_template())
 
     @classmethod

src/calibre/devices/utils.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, time, re
+from functools import partial
+
+from calibre.devices.errors import DeviceError, WrongDestinationError, FreeSpaceError
+
+def sanity_check(on_card, files, card_prefixes, free_space):
+    if on_card == 'carda' and not card_prefixes[0]:
+        raise WrongDestinationError(_(
+            'The reader has no storage card %s. You may have changed '
+            'the default send to device action. Right click on the send '
+            'to device button and reset the default action to be '
+            '"Send to main memory".')%'A')
+    elif on_card == 'cardb' and not card_prefixes[1]:
+        raise WrongDestinationError(_(
+            'The reader has no storage card %s. You may have changed '
+            'the default send to device action. Right click on the send '
+            'to device button and reset the default action to be '
+            '"Send to main memory".')%'B')
+    elif on_card and on_card not in ('carda', 'cardb'):
+        raise DeviceError(_('Selected slot: %s is not supported.') % on_card)
+
+    size = 0
+    for f in files:
+        size += os.path.getsize(getattr(f, 'name', f))
+
+    if not on_card and size > free_space[0] - 2*1024*1024:
+        raise FreeSpaceError(_("There is insufficient free space in main memory"))
+    if on_card == 'carda' and size > free_space[1] - 1024*1024:
+        raise FreeSpaceError(_("There is insufficient free space on the storage card"))
+    if on_card == 'cardb' and size > free_space[2] - 1024*1024:
+        raise FreeSpaceError(_("There is insufficient free space on the storage card"))
+
+def build_template_regexp(template):
+    from calibre import prints
+
+    def replfunc(match, seen=None):
+        v = match.group(1)
+        if v in ['authors', 'author_sort']:
+            v = 'author'
+        if v in ('title', 'series', 'series_index', 'isbn', 'author'):
+            if v not in seen:
+                seen.add(v)
+                return '(?P<' + v + '>.+?)'
+        return '(.+?)'
+    s = set()
+    f = partial(replfunc, seen=s)
+
+    try:
+        template = template.rpartition('/')[2]
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+    except:
+        prints(u'Failed to parse template: %r'%template)
+        template = u'{title} - {authors}'
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+
+def create_upload_path(mdata, fname, template, sanitize,
+        prefix_path='',
+        path_type=os.path,
+        maxlen=250,
+        use_subdirs=True,
+        news_in_folder=True,
+        filename_callback=lambda x, y:x,
+        sanitize_path_components=lambda x: x
+        ):
+    from calibre.library.save_to_disk import get_components, config
+    from calibre.utils.filenames import shorten_components_to
+
+    special_tag = None
+    if mdata.tags:
+        for t in mdata.tags:
+            if t.startswith(_('News')) or t.startswith('/'):
+                special_tag = t
+                break
+
+    if mdata.tags and _('News') in mdata.tags:
+        try:
+            p = mdata.pubdate
+            date  = (p.year, p.month, p.day)
+        except:
+            today = time.localtime()
+            date = (today[0], today[1], today[2])
+        template = u"{title}_%d-%d-%d" % date
+
+    fname = sanitize(fname)
+    ext = path_type.splitext(fname)[1]
+
+    opts = config().parse()
+    if not isinstance(template, unicode):
+        template = template.decode('utf-8')
+    app_id = str(getattr(mdata, 'application_id', ''))
+    id_ = mdata.get('id', fname)
+    extra_components = get_components(template, mdata, id_,
+            timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
+    if not extra_components:
+        extra_components.append(sanitize(filename_callback(fname,
+            mdata)))
+    else:
+        extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata))
+
+    if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
+        extra_components[-1] = 'x' + extra_components[-1][1:]
+
+    if special_tag is not None:
+        name = extra_components[-1]
+        extra_components = []
+        tag = special_tag
+        if tag.startswith(_('News')):
+            if news_in_folder:
+                extra_components.append('News')
+        else:
+            for c in tag.split('/'):
+                c = sanitize(c)
+                if not c: continue
+                extra_components.append(c)
+        extra_components.append(name)
+
+    if not use_subdirs:
+        extra_components = extra_components[-1:]
+
+    def remove_trailing_periods(x):
+        ans = x
+        while ans.endswith('.'):
+            ans = ans[:-1].strip()
+        if not ans:
+            ans = 'x'
+        return ans
+
+    extra_components = list(map(remove_trailing_periods, extra_components))
+    components = shorten_components_to(maxlen - len(prefix_path), extra_components)
+    components = sanitize_path_components(components)
+    if prefix_path:
+        filepath = path_type.join(prefix_path, *components)
+    else:
+        filepath = path_type.join(*components)
+
+    return filepath
+
+
+

src/calibre/ebooks/metadata/book/json_codec.py

@@ -161,7 +161,9 @@ class JsonCodec(object):
         try:
             js = json.load(file_, encoding='utf-8')
             for item in js:
-                booklist.append(self.raw_to_book(item, book_class, prefix))
+                entry = self.raw_to_book(item, book_class, prefix)
+                if entry is not None:
+                    booklist.append(entry)
         except:
             print 'exception during JSON decode_from_file'
             traceback.print_exc()

src/calibre/ebooks/oeb/display/paged.coffee

@@ -79,7 +79,7 @@ class PagedDisplay
         if not this.in_paged_mode
             # Check if the current document is a full screen layout like
             # cover, if so we treat it specially.
-            single_screen = (document.body.scrollWidth < window.innerWidth + 25 and document.body.scrollHeight < window.innerHeight + 25)
+            single_screen = (document.body.scrollHeight < window.innerHeight + 75)
             first_layout = true
 
         ww = window.innerWidth
@@ -149,7 +149,7 @@ class PagedDisplay
             # current page (when cols_per_screen == 1). Similarly img elements
             # with height=100% overflow the first column
             has_svg = document.getElementsByTagName('svg').length > 0
-            only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 2 and document.getElementsByTagName('p').length < 2
+            only_img = document.getElementsByTagName('img').length == 1 and document.getElementsByTagName('div').length < 3 and document.getElementsByTagName('p').length < 2
             this.is_full_screen_layout = (only_img or has_svg) and single_screen and document.body.scrollWidth > document.body.clientWidth
 
         this.in_paged_mode = true

src/calibre/gui2/actions/device.py

@@ -240,6 +240,11 @@ class ConnectShareAction(InterfaceAction):
         from calibre.gui2.dialogs.smartdevice import get_all_ip_addresses
         dm = self.gui.device_manager
 
+        forced_ip = dm.get_option('smartdevice', 'force_ip_address')
+        if forced_ip:
+            formatted_addresses = forced_ip
+            show_port = True
+        else:
             all_ips = get_all_ip_addresses()
             if len(all_ips) > 3:
                 formatted_addresses = _('Many IP addresses. See Start/Stop dialog.')

src/calibre/gui2/device.py

@@ -977,7 +977,7 @@ class DeviceMixin(object): # {{{
             self.set_default_thumbnail(\
                     self.device_manager.device.THUMBNAIL_HEIGHT)
             self.status_bar.show_message(_('Device: ')+\
-                self.device_manager.device.__class__.get_gui_name()+\
+                self.device_manager.device.get_gui_name()+\
                         _(' detected.'), 3000)
             self.device_connected = device_kind
             self.library_view.set_device_connected(self.device_connected)
@@ -1495,8 +1495,12 @@ class DeviceMixin(object): # {{{
                 self.device_job_exception(job)
             return
 
+        try:
             self.device_manager.add_books_to_metadata(job.result,
                     metadata, self.booklists())
+        except:
+            traceback.print_exc()
+            raise
 
         books_to_be_deleted = []
         if memory and memory[1]:

src/calibre/gui2/device_drivers/configwidget.py

@@ -89,6 +89,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
                         l.setBuddy(self.opt_extra_customization[i])
                         l.setWordWrap(True)
                         self.opt_extra_customization[i].setText(settings.extra_customization[i])
+                        self.opt_extra_customization[i].setCursorPosition(0)
                         self.extra_layout.addWidget(l, row_func(i, 0), col_func(i))
                     self.extra_layout.addWidget(self.opt_extra_customization[i],
                                                 row_func(i, 1), col_func(i))
@@ -102,6 +103,7 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
                 if settings.extra_customization:
                     self.opt_extra_customization.setText(settings.extra_customization)
                     self.opt_extra_customization.setCursorPosition(0)
+                self.opt_extra_customization.setCursorPosition(0)
                 self.extra_layout.addWidget(l, 0, 0)
                 self.extra_layout.addWidget(self.opt_extra_customization, 1, 0)
         self.opt_save_template.setText(settings.save_template)

src/calibre/gui2/dialogs/smartdevice.py

@@ -115,6 +115,10 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
             self.auto_mgmt_button.setText(_('Automatic metadata management is enabled'))
             self.auto_mgmt_button.setEnabled(False)
 
+        forced_ip = self.device_manager.get_option('smartdevice', 'force_ip_address')
+        if forced_ip:
+            self.ip_addresses.setText(forced_ip)
+        else:
             self.ip_addresses.setText(', '.join(get_all_ip_addresses()))
 
         self.resize(self.sizeHint())

src/calibre/utils/mdns.py

@@ -60,7 +60,7 @@ def start_server():
 
     return _server
 
-def create_service(desc, type, port, properties, add_hostname):
+def create_service(desc, type, port, properties, add_hostname, use_ip_address=None):
     port = int(port)
     try:
         hostname = socket.gethostname().partition('.')[0]
@@ -69,6 +69,9 @@ def create_service(desc, type, port, properties, add_hostname):
 
     if add_hostname:
         desc += ' (on %s)'%hostname
+    if use_ip_address:
+        local_ip = use_ip_address
+    else:
         local_ip = get_external_ip()
     type = type+'.local.'
     from calibre.utils.Zeroconf import ServiceInfo
@@ -79,7 +82,7 @@ def create_service(desc, type, port, properties, add_hostname):
                           server=hostname+'.local.')
 
 
-def publish(desc, type, port, properties=None, add_hostname=True):
+def publish(desc, type, port, properties=None, add_hostname=True, use_ip_address=None):
     '''
     Publish a service.
 

src/calibre/utils/mem.py

@@ -13,188 +13,23 @@ You can pass a number to memory and it will be subtracted from the returned
 value.
 '''
 
-import gc, os, re
+import gc, os
 
 from calibre.constants import iswindows, islinux
 
-if islinux:
+def get_memory():
-    # Taken, with thanks, from:
+    'Return memory usage in bytes'
-    # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
+    import psutil
-
+    p = psutil.Process(os.getpid())
-    def permute(args):
+    mem = p.get_ext_memory_info()
-        ret = []
+    attr = 'wset' if iswindows else 'data' if islinux else 'rss'
-        if args:
+    return getattr(mem, attr)
-            first = args.pop(0)
-            for y in permute(args):
-                for x in first:
-                    ret.append(x + y)
-        else:
-            ret.append('')
-        return ret
-
-    def parsed_groups(match, *types):
-        groups = match.groups()
-        assert len(groups) == len(types)
-        return tuple([type(group) for group, type in zip(groups, types)])
-
-    class VMA(dict):
-        def __init__(self, *args):
-            (self.start, self.end, self.perms, self.offset,
-            self.major, self.minor, self.inode, self.filename) = args
-
-    def parse_smaps(pid):
-        with open('/proc/%s/smaps'%pid, 'r') as maps:
-            hex = lambda s: int(s, 16)
-
-            ret = []
-            header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
-                                r'(..):(..) (\d+) *(.*)$')
-            detail = re.compile(r'^(.*): +(\d+) kB')
-            for line in maps:
-                m = header.match(line)
-                if m:
-                    vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
-                    ret.append(vma)
-                else:
-                    m = detail.match(line)
-                    if m:
-                        k, v = parsed_groups(m, str, int)
-                        assert k not in vma
-                        vma[k] = v
-                    else:
-                        print 'unparseable line:', line
-            return ret
-
-    perms = permute(['r-', 'w-', 'x-', 'ps'])
-
-    def make_summary_dicts(vmas):
-        mapped = {}
-        anon = {}
-        for d in mapped, anon:
-            # per-perm
-            for k in perms:
-                d[k] = {}
-                d[k]['Size'] = 0
-                for y in 'Shared', 'Private':
-                    d[k][y] = {}
-                    for z in 'Clean', 'Dirty':
-                        d[k][y][z] = 0
-            # totals
-            for y in 'Shared', 'Private':
-                d[y] = {}
-                for z in 'Clean', 'Dirty':
-                    d[y][z] = 0
-
-        for vma in vmas:
-            if vma.major == '00' and vma.minor == '00':
-                d = anon
-            else:
-                d = mapped
-            for y in 'Shared', 'Private':
-                for z in 'Clean', 'Dirty':
-                    d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
-                    d[y][z] += vma.get(y + '_' + z, 0)
-            d[vma.perms]['Size'] += vma.get('Size', 0)
-        return mapped, anon
-
-    def values(d, args):
-        if args:
-            ret = ()
-            first = args[0]
-            for k in first:
-                ret += values(d[k], args[1:])
-            return ret
-        else:
-            return (d,)
-
-    def print_summary(dicts_and_titles):
-        def desc(title, perms):
-            ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
-                ('Anonymous', 'rwxp'): 'Writable code (stack)',
-                ('Mapped', 'r-xp'): 'Code',
-                ('Mapped', 'rwxp'): 'Writable code (jump tables)',
-                ('Mapped', 'r--p'): 'Read-only data',
-                ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
-            if ret:
-                return '  -- ' + ret
-            else:
-                return ''
-
-        for d, title in dicts_and_titles:
-            print title, 'memory:'
-            print '               Shared            Private'
-            print '           Clean    Dirty    Clean    Dirty'
-            for k in perms:
-                if d[k]['Size']:
-                    print ('    %s %7d  %7d  %7d  %7d%s'
-                        % ((k,)
-                            + values(d[k], (('Shared', 'Private'),
-                                            ('Clean', 'Dirty')))
-                            + (desc(title, k),)))
-            print ('   total %7d  %7d  %7d  %7d'
-                % values(d, (('Shared', 'Private'),
-                                ('Clean', 'Dirty'))))
-
-        print '   ' + '-' * 40
-        print ('   total %7d  %7d  %7d  %7d'
-            % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
-                                                ('Clean', 'Dirty')))
-                                    for d, title in dicts_and_titles]))))
-
-    def print_stats(pid=None):
-        if pid is None:
-            pid = os.getpid()
-        vmas = parse_smaps(pid)
-        mapped, anon = make_summary_dicts(vmas)
-        print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
-
-    def linux_memory(since=0.0):
-        vmas = parse_smaps(os.getpid())
-        mapped, anon = make_summary_dicts(vmas)
-        dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
-        totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
-                                                ('Clean', 'Dirty')))
-                                    for d, title in dicts_and_titles])))
-        return (totals[-1]/1024.) - since
-
-    memory = linux_memory
-
-elif iswindows:
-    import win32process
-    import win32con
-    import win32api
-
-    # See http://msdn.microsoft.com/en-us/library/ms684877.aspx
-    # for details on the info returned by get_meminfo
-
-    def get_handle(pid):
-        return win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0,
-                pid)
-
-    def listprocesses(self):
-        for process in win32process.EnumProcesses():
-            try:
-                han = get_handle(process)
-                procmeminfo = meminfo(han)
-                procmemusage = procmeminfo["WorkingSetSize"]
-                yield process, procmemusage
-            except:
-                pass
-
-    def get_meminfo(pid):
-        han = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, 0,
-                pid)
-        return meminfo(han)
-
-    def meminfo(handle):
-        return win32process.GetProcessMemoryInfo(handle)
-
-    def win_memory(since=0.0):
-        info = meminfo(get_handle(os.getpid()))
-        return (info['WorkingSetSize']/1024.**2) - since
-
-    memory = win_memory
 
+def memory(since=0.0):
+    'Return memory used in MB. The value of since is subtracted from the used memory'
+    ans = get_memory()
+    ans /= float(1024**2)
+    return ans - since
 
 def gc_histogram():
     """Returns per-class counts of existing objects."""