0.8.67

Changelog.yaml

@@ -19,6 +19,56 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.67
+  date: 2012-08-31
+
+  new features:
+    - title: "PDF Output: Generate a PDF Outline based on the Table of Contents of the input document"
+
+    - title: "Conversion: Add an option under Structure Detection to set the 'Start reading at' metadata with an XPath expression."
+      tickets: [1043233]
+
+    - title: "Speed up changing the title and author of files with books larger than 3MB by avoiding an unnecessary extra copy."
+
+    - title: "Wireless device driver: Make detecting and connecting to devices easier on networks where mdns is disabled"
+
+    - title: "PDF Output: Allow choosing the default font family and size when generating PDF files (under PDF Options) in the conversion dialog"
+
+    - title: "Metadata dialog: Comments editor: Allow specifying the name of a link when using the insert link button."
+      tickets: [1042683]
+
+    - title: "Remove the unmaintained pdfmanipulate command line utility. There are many other tools that provide similar functionality, for example, pdftk and podofo"
+ 
+  bug fixes:
+    - title: "Catalogs: Fix regression that broke sorting of non series titles before series titles"
+
+    - title: "PDF Output: Do not create duplicate embedded fonts in the PDF for every individual HTML file in the input document"
+
+    - title: "Fix regression that broke DnD of files having a # character in their names to the book details panel"
+
+    - title: "PDF Output: Allow generating PDF files with more than 512 pages on windows."
+      tickets: [1041614]
+
+    - title: "Fix minor bug in handling of the completion popups when using the next/previous buttons in the edit metadata dialog"
+      ticket: [1041389]
+
+  improved recipes:
+    - Coding Horror
+    - TIME Magazine
+
+  new recipes:
+    - title: Cumhuriyet Yzarlar 
+      author: Sethi Eksi
+
+    - title: Arcadia 
+      author: Masahiro Hasegawa
+
+    - title: Business Week Magazine and Chronicle of Higher Education 
+      author: Rick Shang
+
+    - title: CIPER Chile 
+      author: Darko Miletic
+
 - version: 0.8.66
   date: 2012-08-24
 

setup/translations.py

@@ -152,7 +152,7 @@ class Translations(POT): # {{{
                     subprocess.check_call(['msgfmt', '-o', dest, iso639])
             elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc',
                     'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku',
-                    'fr_CA', 'him'):
+                    'fr_CA', 'him', 'jv', 'ka'):
                 self.warn('No ISO 639 translations for locale:', locale)
 
         self.write_stats()

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 66)
+numeric_version = (0, 8, 67)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/devices/mtp/base.py

@@ -35,13 +35,14 @@ class MTPDeviceBase(DevicePlugin):
         DevicePlugin.__init__(self, *args, **kwargs)
         self.progress_reporter = None
         self.current_friendly_name = None
+        self.report_progress = lambda x, y: None
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None):
         pass
 
     def set_progress_reporter(self, report_progress):
-        self.progress_reporter = report_progress
+        self.report_progress = report_progress
 
     def get_gui_name(self):
         return self.current_friendly_name or self.name

src/calibre/devices/mtp/driver.py

@@ -7,14 +7,94 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from calibre.constants import iswindows
+import json, pprint
+from io import BytesIO
+
+from calibre.constants import iswindows, numeric_version
+from calibre.utils.config import from_json, to_json
+from calibre.utils.date import now, isoformat
 
 if iswindows:
     from calibre.devices.mtp.windows.driver import MTP_DEVICE as BASE
     BASE
 else:
     from calibre.devices.mtp.unix.driver import MTP_DEVICE as BASE
+pprint
 
 class MTP_DEVICE(BASE):
-    pass
+
+    METADATA_CACHE = 'metadata.calibre'
+    DRIVEINFO = 'driveinfo.calibre'
+
+    def _update_drive_info(self, storage, location_code, name=None):
+        import uuid
+        f = storage.find_path((self.DRIVEINFO,))
+        dinfo = {}
+        if f is not None:
+            stream = self.get_file(f)
+            try:
+                dinfo = json.load(stream, object_hook=from_json)
+            except:
+                dinfo = None
+        if dinfo.get('device_store_uuid', None) is None:
+            dinfo['device_store_uuid'] = unicode(uuid.uuid4())
+        if dinfo.get('device_name', None) is None:
+            dinfo['device_name'] = self.current_friendly_name
+        if name is not None:
+            dinfo['device_name'] = name
+        dinfo['location_code'] = location_code
+        dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
+        dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
+        dinfo['date_last_connected'] = isoformat(now())
+        dinfo['mtp_prefix'] = storage.storage_prefix
+        raw = json.dumps(dinfo, default=to_json)
+        self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw))
+        self.driveinfo = dinfo
+
+    def open(self, devices, library_uuid):
+        self.current_library_uuid = library_uuid
+        BASE.open(self, devices, library_uuid)
+
+    def get_device_information(self, end_session=True):
+        self.report_progress(1.0, _('Get device information...'))
+        self.driveinfo = {}
+        for sid, location_code in ( (self._main_id, 'main'), (self._carda_id,
+            'A'), (self._cardb_id, 'B')):
+            if sid is None: continue
+            self._update_drive_info(self.filesystem_cache.storage(sid), location_code)
+        dinfo = self.get_basic_device_information()
+        return tuple( list(dinfo) + [self.driveinfo] )
+
+    def card_prefix(self, end_session=True):
+        ans = [None, None]
+        if self._carda_id is not None:
+            ans[0] = self.filesystem_cache.storage(self._carda_id).storage_prefix
+        if self._cardb_id is not None:
+            ans[1] = self.filesystem_cache.storage(self._cardb_id).storage_prefix
+        return tuple(ans)
+
+    def set_driveinfo_name(self, location_code, name):
+        sid = {'main':self._main_id, 'A':self._carda_id,
+                'B':self._cardb_id}.get(location_code, None)
+        if sid is None:
+            return
+        self._update_drive_info(self.filesystem_cache.storage(sid),
+                location_code, name=name)
+
+if __name__ == '__main__':
+    dev = MTP_DEVICE(None)
+    dev.startup()
+    try:
+        from calibre.devices.scanner import DeviceScanner
+        scanner = DeviceScanner()
+        scanner.scan()
+        devs = scanner.devices
+        cd = dev.detect_managed_devices(devs)
+        if cd is None:
+            raise ValueError('Failed to detect MTP device')
+        dev.open(cd, None)
+        pprint.pprint(dev.get_device_information())
+    finally:
+        dev.shutdown()
+
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -47,6 +47,9 @@ class FileOrFolder(object):
         self.fs_cache = weakref.ref(fs_cache)
         self.deleted = False
 
+        if self.storage_id == self.object_id:
+            self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
+
     def __repr__(self):
         name = 'Folder' if self.is_folder else 'File'
         try:
@@ -125,6 +128,26 @@ class FileOrFolder(object):
                 return e
         return None
 
+    def find_path(self, path):
+        '''
+        Find a path in this folder, where path is a
+        tuple of folder and file names like ('eBooks', 'newest',
+        'calibre.epub'). Finding is case-insensitive.
+        '''
+        parent = self
+        components = list(path)
+        while components:
+            child = components[0]
+            components = components[1:]
+            c = parent.folder_named(child)
+            if c is None:
+                c = parent.file_named(child)
+            if c is None:
+                return None
+            parent = c
+        return parent
+
+
 class FilesystemCache(object):
 
     def __init__(self, all_storage, entries):
@@ -164,4 +187,9 @@ class FilesystemCache(object):
         for e in self.entries:
             e.dump(out=out)
 
+    def storage(self, storage_id):
+        for e in self.entries:
+            if e.storage_id == storage_id:
+                return e
+
 

src/calibre/devices/mtp/unix/driver.py

@@ -46,14 +46,6 @@ class MTP_DEVICE(MTPDeviceBase):
     def set_debug_level(self, lvl):
         self.libmtp.set_debug_level(lvl)
 
-    def report_progress(self, sent, total):
-        try:
-            p = int(sent/total * 100)
-        except ZeroDivisionError:
-            p = 100
-        if self.progress_reporter is not None:
-            self.progress_reporter(p)
-
     @synchronous
     def detect_managed_devices(self, devices_on_system, force_refresh=False):
         if self.libmtp is None: return None
@@ -212,19 +204,10 @@ class MTP_DEVICE(MTPDeviceBase):
         return self._filesystem_cache
 
     @synchronous
-    def get_device_information(self, end_session=True):
+    def get_basic_device_information(self):
         d = self.dev
         return (self.current_friendly_name, d.device_version, d.device_version, '')
 
-    @synchronous
-    def card_prefix(self, end_session=True):
-        ans = [None, None]
-        if self._carda_id is not None:
-            ans[0] = 'mtp:::%d:::'%self._carda_id
-        if self._cardb_id is not None:
-            ans[1] = 'mtp:::%d:::'%self._cardb_id
-        return tuple(ans)
-
     @synchronous
     def total_space(self, end_session=True):
         ans = [0, 0, 0]
@@ -298,6 +281,7 @@ class MTP_DEVICE(MTPDeviceBase):
         if not ok:
             raise DeviceError('Failed to get file: %s with errors: %s'%(
                 f.full_path, self.format_errorstack(errs)))
+        stream.seek(0)
         return stream
 
     @synchronous

src/calibre/devices/mtp/windows/driver.py

@@ -203,24 +203,11 @@ class MTP_DEVICE(MTPDeviceBase):
         self.current_friendly_name = devdata.get('friendly_name', None)
 
     @same_thread
-    def get_device_information(self, end_session=True):
+    def get_basic_device_information(self):
         d = self.dev.data
         dv = d.get('device_version', '')
-        for sid, location_code in ( (self._main_id, 'main'), (self._carda_id,
-            'A'), (self._cardb_id, 'B')):
-            if sid is None: continue
-            # TODO: Implement the drive info dict
         return (self.current_friendly_name, dv, dv, '')
 
-    @same_thread
-    def card_prefix(self, end_session=True):
-        ans = [None, None]
-        if self._carda_id is not None:
-            ans[0] = 'mtp:::%s:::'%self._carda_id
-        if self._cardb_id is not None:
-            ans[1] = 'mtp:::%s:::'%self._cardb_id
-        return tuple(ans)
-
     @same_thread
     def total_space(self, end_session=True):
         ans = [0, 0, 0]
@@ -260,6 +247,7 @@ class MTP_DEVICE(MTPDeviceBase):
         except Exception as e:
             raise DeviceError('Failed to fetch the file %s with error: %s'%
                     f.full_path, as_unicode(e))
+        stream.seek(0)
         return stream
 
     @same_thread

src/calibre/devices/usbms/driver.py

@@ -63,7 +63,7 @@ class USBMS(CLI, Device):
             dinfo = {}
         if dinfo.get('device_store_uuid', None) is None:
             dinfo['device_store_uuid'] = unicode(uuid.uuid4())
-        if dinfo.get('device_name') is None:
+        if dinfo.get('device_name', None) is None:
             dinfo['device_name'] = self.get_gui_name()
         if name is not None:
             dinfo['device_name'] = name

src/calibre/ebooks/conversion/cli.py

@@ -170,7 +170,7 @@ def add_pipeline_options(parser, plumber):
                       'chapter', 'chapter_mark',
                       'prefer_metadata_cover', 'remove_first_image',
                       'insert_metadata', 'page_breaks_before',
-                      'remove_fake_margins',
+                      'remove_fake_margins', 'start_reading_at',
                   ]
                   ),
 

src/calibre/ebooks/conversion/plumber.py

@@ -304,6 +304,16 @@ OptionRecommendation(name='chapter_mark',
                     'to mark chapters.')
         ),
 
+OptionRecommendation(name='start_reading_at',
+        recommended_value=None, level=OptionRecommendation.LOW,
+        help=_('An XPath expression to detect the location in the document'
+            ' at which to start reading. Some ebook reading programs'
+            ' (most prominently the Kindle) use this location as the'
+            ' position at which to open the book. See the XPath tutorial'
+            ' in the calibre User Manual for further help using this'
+            ' feature.')
+        ),
+
 OptionRecommendation(name='extra_css',
             recommended_value=None, level=OptionRecommendation.LOW,
             help=_('Either the path to a CSS stylesheet or raw CSS. '

src/calibre/ebooks/oeb/transforms/structure.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, uuid
 
 from lxml import etree
 from urlparse import urlparse
@@ -80,6 +80,35 @@ class DetectStructure(object):
             if not node.title or not node.title.strip():
                 node.title = _('Unnamed')
 
+        if self.opts.start_reading_at:
+            self.detect_start_reading()
+
+    def detect_start_reading(self):
+        expr = self.opts.start_reading_at
+        try:
+            expr = XPath(expr)
+        except:
+            self.log.warn(
+                'Invalid start reading at XPath expression, ignoring: %s'%expr)
+            return
+        for item in self.oeb.spine:
+            if not hasattr(item.data, 'xpath'): continue
+            matches = expr(item.data)
+            if matches:
+                elem = matches[0]
+                eid = elem.get('id', None)
+                if not eid:
+                    eid = u'start_reading_at_'+unicode(uuid.uuid4()).replace(u'-', u'')
+                    elem.set('id', eid)
+                if u'text' in self.oeb.guide:
+                    self.oeb.guide.remove(u'text')
+                self.oeb.guide.add(u'text', u'Start', item.href+u'#'+eid)
+                self.log('Setting start reading at position to %s in %s'%(
+                    self.opts.start_reading_at, item.href))
+                return
+        self.log.warn("Failed to find start reading at position: %s"%
+                self.opts.start_reading_at)
+
     def detect_chapters(self):
         self.detected_chapters = []
 

src/calibre/gui2/convert/structure_detection.py

@@ -20,7 +20,7 @@ class StructureDetectionWidget(Widget, Ui_Form):
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
-                ['chapter', 'chapter_mark',
+                ['chapter', 'chapter_mark', 'start_reading_at',
                 'remove_first_image', 'remove_fake_margins',
                 'insert_metadata', 'page_breaks_before']
                 )
@@ -31,15 +31,18 @@ class StructureDetectionWidget(Widget, Ui_Form):
         self.opt_chapter.set_msg(_('Detect chapters at (XPath expression):'))
         self.opt_page_breaks_before.set_msg(_('Insert page breaks before '
             '(XPath expression):'))
+        self.opt_start_reading_at.set_msg(
+                _('Start reading at (XPath expression):'))
 
     def break_cycles(self):
         Widget.break_cycles(self)
 
     def pre_commit_check(self):
-        for x in ('chapter', 'page_breaks_before'):
+        for x in ('chapter', 'page_breaks_before', 'start_reading_at'):
             x = getattr(self, 'opt_'+x)
             if not x.check():
                 error_dialog(self, _('Invalid XPath'),
                 _('The XPath expression %s is invalid.')%x.text).exec_()
                 return False
         return True
+

src/calibre/gui2/convert/structure_detection.ui

@@ -14,10 +14,40 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" colspan="3">
+   <item row="2" column="3">
+    <widget class="QCheckBox" name="opt_remove_fake_margins">
+     <property name="text">
+      <string>Remove &amp;fake margins</string>
+     </property>
+    </widget>
+   </item>
+   <item row="4" column="0" colspan="4">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>The header and footer removal options have been replaced by the Search &amp; Replace options. Click the Search &amp; Replace category in the bar to the left to use these options. Leave the replace field blank and enter your header/footer removal regexps into the search field.</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="5" column="0" rowspan="2" colspan="4">
+    <widget class="XPathEdit" name="opt_page_breaks_before" native="true"/>
+   </item>
+   <item row="3" column="0" colspan="4">
+    <widget class="QCheckBox" name="opt_insert_metadata">
+     <property name="text">
+      <string>Insert &amp;metadata as page at start of book</string>
+     </property>
+    </widget>
+   </item>
+   <item row="7" column="0" colspan="4">
+    <widget class="XPathEdit" name="opt_start_reading_at" native="true"/>
+   </item>
+   <item row="0" column="0" colspan="4">
     <widget class="XPathEdit" name="opt_chapter" native="true"/>
    </item>
-   <item row="1" column="0">
+   <item row="1" column="0" colspan="2">
     <widget class="QLabel" name="label">
      <property name="text">
       <string>Chapter &amp;mark:</string>
@@ -27,44 +57,14 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="1">
+   <item row="1" column="2">
     <widget class="QComboBox" name="opt_chapter_mark">
      <property name="minimumContentsLength">
       <number>20</number>
      </property>
     </widget>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="1" column="3">
-    <widget class="QCheckBox" name="opt_remove_first_image">
-     <property name="text">
-      <string>Remove first &amp;image</string>
-     </property>
-    </widget>
-   </item>
-   <item row="3" column="0" colspan="2">
-    <widget class="QCheckBox" name="opt_insert_metadata">
-     <property name="text">
-      <string>Insert &amp;metadata as page at start of book</string>
-     </property>
-    </widget>
-   </item>
-   <item row="7" column="0" colspan="3">
-    <widget class="XPathEdit" name="opt_page_breaks_before" native="true"/>
-   </item>
-   <item row="8" column="0" colspan="3">
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>40</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="1" column="2">
     <spacer name="horizontalSpacer">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -77,22 +77,25 @@
      </property>
     </spacer>
    </item>
-   <item row="5" column="0" colspan="3">
+   <item row="2" column="0" colspan="3">
-    <widget class="QLabel" name="label_2">
+    <widget class="QCheckBox" name="opt_remove_first_image">
      <property name="text">
-      <string>The header and footer removal options have been replaced by the Search &amp; Replace options. Click the Search &amp; Replace category in the bar to the left to use these options. Leave the replace field blank and enter your header/footer removal regexps into the search field.</string>
+      <string>Remove first &amp;image</string>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="2" column="2">
+   <item row="8" column="0">
-    <widget class="QCheckBox" name="opt_remove_fake_margins">
+    <spacer name="verticalSpacer">
-     <property name="text">
+     <property name="orientation">
-      <string>Remove &amp;fake margins</string>
+      <enum>Qt::Vertical</enum>
      </property>
-    </widget>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
    </item>
   </layout>
  </widget>

src/calibre/utils/filenames.py

@@ -203,6 +203,11 @@ def samefile_windows(src, dst):
     import win32file
     from pywintypes import error
 
+    samestring = (os.path.normcase(os.path.abspath(src)) ==
+            os.path.normcase(os.path.abspath(dst)))
+    if samestring:
+        return True
+
     def get_fileid(x):
         if isbytestring(x): x = x.decode(filesystem_encoding)
         try:

src/calibre/utils/icu.py

@@ -82,6 +82,17 @@ def icu_sort_key(collator, obj):
             obj = obj.replace(b'\0', b'')
         return _secondary_collator.sort_key(obj)
 
+def icu_change_case(upper, locale, obj):
+    func = _icu.upper if upper else _icu.lower
+    try:
+        return func(locale, obj)
+    except TypeError:
+        if isinstance(obj, unicode):
+            obj = obj.replace(u'\0', u'')
+        else:
+            obj = obj.replace(b'\0', b'')
+        return func(locale, obj)
+
 def py_find(pattern, source):
     pos = source.find(pattern)
     if pos > -1:
@@ -163,10 +174,10 @@ case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \
 case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp
 
 upper = (lambda s: s.upper()) if _icu_not_ok else \
-    partial(_icu.upper, get_locale())
+    partial(icu_change_case, True, get_locale())
 
 lower = (lambda s: s.lower()) if _icu_not_ok else \
-    partial(_icu.lower, get_locale())
+    partial(icu_change_case, False, get_locale())
 
 title_case = (lambda s: s.title()) if _icu_not_ok else \
     partial(_icu.title, get_locale())