KG revisions

resources/recipes/oldnewthing.recipe

@@ -28,7 +28,7 @@ class OldNewThing(BasicNewsRecipe):
                         }
 
     remove_attributes = ['width','height']
-    keep_only_tags    = [dict(attrs={'class':['postsub','comment']})]
+    keep_only_tags    = [dict(attrs={'class':'full-post'})]
-
+    remove_tags       = [dict(attrs={'class':['post-attributes','post-tags','post-actions']})]
     feeds = [(u'Posts', u'http://blogs.msdn.com/oldnewthing/rss.xml')]
 

setup/installer/windows/wix-template.xml

@@ -153,6 +153,18 @@
                 <Property Id="WixShellExecTarget" Value="[#{exe_map[calibre]}]" />
                 <CustomAction Id="LaunchApplication" BinaryKey="WixCA"
                     DllEntry="WixShellExec" Impersonate="yes"/>
+                <InstallExecuteSequence>
+                    <FileCost Suppress="yes" />
+                </InstallExecuteSequence>
+                <InstallUISequence>
+                    <FileCost Suppress="yes" />
+                </InstallUISequence>
+                <AdminExecuteSequence>
+                    <FileCost Suppress="yes" />
+                </AdminExecuteSequence>
+                <AdminUISequence>
+                    <FileCost Suppress="yes" />
+                </AdminUISequence>
 
 		</Product>
 </Wix>

src/calibre/devices/apple/driver.py

@@ -2395,7 +2395,7 @@ class ITUNES(DriverBase):
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
 
                 # Force the language declaration for iBooks 1.1
-                metadata.language = get_lang()
+                metadata.language = get_lang().replace('_', '-')
                 if DEBUG:
                     self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
 

src/calibre/devices/prs505/sony_cache.py

@@ -144,51 +144,72 @@ class XMLCache(object):
                         if title+str(i) not in seen:
                             title = title+str(i)
                             playlist.set('title', title)
+                            seen.add(title)
                             break
                 else:
                     seen.add(title)
 
-    def build_playlist_id_map(self):
-        debug_print('Start build_playlist_id_map')
-        ans = {}
-        self.ensure_unique_playlist_titles()
-        debug_print('after ensure_unique_playlist_titles')
-        self.prune_empty_playlists()
-        for i, root in self.record_roots.items():
-            debug_print('build_playlist_id_map loop', i)
-            id_map = self.build_id_map(root)
-            ans[i] = []
-            for playlist in root.xpath('//*[local-name()="playlist"]'):
-                items = []
-                for item in playlist:
-                    id_ = item.get('id', None)
-                    record = id_map.get(id_, None)
-                    if record is not None:
-                        items.append(record)
-                ans[i].append((playlist.get('title'), items))
-        debug_print('end build_playlist_id_map')
-        return ans
-
     def build_id_playlist_map(self, bl_index):
+        '''
+        Return a map of the collections in books: {lpaths: [collection names]}
+        '''
         debug_print('Start build_id_playlist_map')
-        pmap = self.build_playlist_id_map()[bl_index]
+        self.ensure_unique_playlist_titles()
+        self.prune_empty_playlists()
+        debug_print('after cleaning playlists')
+        root = self.record_roots[bl_index]
+        if root is None:
+            return
+        id_map = self.build_id_map(root)
         playlist_map = {}
-        for title, records in pmap:
+        # foreach playlist, get the lpaths for the ids in it, then add to dict
-            for record in records:
+        for playlist in root.xpath('//*[local-name()="playlist"]'):
-                path = record.get('path', None)
+            name = playlist.get('title')
-                if path:
+            if name is None:
-                    if path not in playlist_map:
+                debug_print('build_id_playlist_map: unnamed playlist!')
-                        playlist_map[path] = []
+                continue
-                    playlist_map[path].append(title)
+            for item in playlist:
+                # translate each id into its lpath
+                id_ = item.get('id', None)
+                if id_ is None:
+                    debug_print('build_id_playlist_map: id_ is None!')
+                    continue
+                bk = id_map.get(id_, None)
+                if bk is None:
+                    debug_print('build_id_playlist_map: book is None!', id_)
+                    continue
+                lpath = bk.get('path', None)
+                if lpath is None:
+                    debug_print('build_id_playlist_map: lpath is None!', id_)
+                    continue
+                if lpath not in playlist_map:
+                    playlist_map[lpath] = []
+                playlist_map[lpath].append(name)
         debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
         return playlist_map
 
+    def reset_existing_playlists_map(self):
+        '''
+        Call this method before calling get_or_create_playlist in the context of
+        a given job. Call it again after deleting any playlists. The current
+        implementation adds all new playlists before deleting any, so that
+        constraint is respected.
+        '''
+        self._playlist_to_playlist_id_map = {}
+
     def get_or_create_playlist(self, bl_idx, title):
+        # maintain a private map of playlists to their ids. Don't check if it
+        # exists, because reset_existing_playlist_map must be called before it
+        # is used to ensure that deleted playlists are taken into account
         root = self.record_roots[bl_idx]
+        if bl_idx not in self._playlist_to_playlist_id_map:
+            self._playlist_to_playlist_id_map[bl_idx] = {}
             for playlist in root.xpath('//*[local-name()="playlist"]'):
-            if playlist.get('title', None) == title:
+                pl_title = playlist.get('title', None)
-                return playlist
+                if pl_title is not None:
-        if DEBUG:
+                    self._playlist_to_playlist_id_map[bl_idx][pl_title] = playlist
+        if title in self._playlist_to_playlist_id_map[bl_idx]:
+            return self._playlist_to_playlist_id_map[bl_idx][title]
         debug_print('Creating playlist:', title)
         ans = root.makeelement('{%s}playlist'%self.namespaces[bl_idx],
                 nsmap=root.nsmap, attrib={
@@ -198,6 +219,7 @@ class XMLCache(object):
                     'sourceid': '1'
                     })
         root.append(ans)
+        self._playlist_to_playlist_id_map[bl_idx][title] = ans
         return ans
     # }}}
 
@@ -260,6 +282,8 @@ class XMLCache(object):
                 ensure_media_xml_base_ids(root)
 
             idmap = ensure_numeric_ids(root)
+            if len(idmap) > 0:
+                debug_print('fix_ids: found some non-numeric ids')
                 remap_playlist_references(root, idmap)
             if i == 0:
                 sourceid, playlist_sid = 1, 0
@@ -326,7 +350,9 @@ class XMLCache(object):
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
-                self.update_text_record(record, book, path, i)
+                date = self.check_timestamp(record, book, path)
+                if date is not None:
+                    self.update_text_record(record, book, date, path, i)
                 # Ensure the collections in the XML database are recorded for
                 # this book
                 if book.device_collections is None:
@@ -352,8 +378,10 @@ class XMLCache(object):
 
     def update_playlists(self, bl_index, root, booklist, collections_attributes):
         debug_print('Starting update_playlists', collections_attributes, bl_index)
+        self.reset_existing_playlists_map()
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
+        debug_print('update_playlists: finished building maps')
         for category, books in collections.items():
             records = [lpath_map.get(b.lpath, None) for b in books]
             # Remove any books that were not found, although this
@@ -362,24 +390,23 @@ class XMLCache(object):
                 debug_print('WARNING: Some elements in the JSON cache were not'
                         ' found in the XML cache')
             records = [x for x in records if x is not None]
+            ids = set()
             for rec in records:
-                if rec.get('id', None) is None:
+                id = rec.get('id', None)
+                if id is None:
                     rec.set('id', str(self.max_id(root)+1))
-            ids = [x.get('id', None) for x in records]
+                    id = rec.get('id', None)
-            if None in ids:
+                ids.add(id)
-                debug_print('WARNING: Some <text> elements do not have ids')
+            # ids cannot contain None, so no reason to check
-                ids = [x for x in ids if x is not None]
+
             playlist = self.get_or_create_playlist(bl_index, category)
-            playlist_ids = []
+            # Reduce ids to books not already in the playlist
             for item in playlist:
                 id_ = item.get('id', None)
                 if id_ is not None:
-                    playlist_ids.append(id_)
+                    ids.discard(id_)
-            for item in list(playlist):
+            # Add the books in ids that were not already in the playlist
-                playlist.remove(item)
+            for id_ in ids:
-
-            extra_ids = [x for x in playlist_ids if x not in ids]
-            for id_ in ids + extra_ids:
                 item = playlist.makeelement(
                         '{%s}item'%self.namespaces[bl_index],
                         nsmap=playlist.nsmap, attrib={'id':id_})
@@ -416,10 +443,22 @@ class XMLCache(object):
         root.append(ans)
         return ans
 
-    def update_text_record(self, record, book, path, bl_index):
+    def check_timestamp(self, record, book, path):
+        '''
+        Checks the timestamp in the Sony DB against the file. If different,
+        return the file timestamp. Otherwise return None.
+        '''
         timestamp = os.path.getmtime(path)
         date = strftime(timestamp)
         if date != record.get('date', None):
+            return date
+        return None
+
+    def update_text_record(self, record, book, date, path, bl_index):
+        '''
+        Update the Sony database from the book. This is done if the timestamp in
+        the db differs from the timestamp on the file.
+        '''
         record.set('date', date)
         record.set('size', str(os.stat(path).st_size))
         title = book.title if book.title else _('Unknown')

src/calibre/devices/usbms/books.py

@@ -134,9 +134,16 @@ class CollectionsBookList(BookList):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
+        # This map of sets is used to avoid linear searches when testing for
+        # book equality
+        collections_lpaths = {}
         for book in self:
-            # The default: leave the book in all existing collections. Do not
+            # Make sure we can identify this book via the lpath
-            # add any new ones.
+            lpath = getattr(book, 'lpath', None)
+            if lpath is None:
+                continue
+            # Decide how we will build the collections. The default: leave the
+            # book in all existing collections. Do not add any new ones.
             attrs = ['device_collections']
             if getattr(book, '_new_book', False):
                 if prefs['preserve_user_collections']:
@@ -163,11 +170,12 @@ class CollectionsBookList(BookList):
                         continue
                     if category not in collections:
                         collections[category] = []
-                    if book not in collections[category]:
+                        collections_lpaths[category] = set()
+                    if lpath not in collections_lpaths[category]:
+                        collections_lpaths[category].add(lpath)
                         collections[category].append(book)
                         if attr == 'series':
                             series_categories.add(category)
-
         # Sort collections
         for category, books in collections.items():
             def tgetter(x):

src/calibre/ebooks/chm/input.py

@@ -92,7 +92,7 @@ class CHMInput(InputFormatPlugin):
             metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'})
         if not metadata.language:
             oeb.logger.warn(u'Language not specified')
-            metadata.add('language', get_lang())
+            metadata.add('language', get_lang().replace('_', '-'))
         if not metadata.creator:
             oeb.logger.warn('Creator not specified')
             metadata.add('creator', _('Unknown'))

src/calibre/ebooks/compression/palmdoc.c

@@ -151,6 +151,7 @@ cpalmdoc_do_compress(buffer *b, char *output) {
             for (j=0; j < temp.len; j++) *(output++) = (char)temp.data[j];
         }
     }
+    PyMem_Free(temp.data);
     return output - head;
 }
 
@@ -168,7 +169,9 @@ cpalmdoc_compress(PyObject *self, PyObject *args) {
     for (j = 0; j < input_len; j++) 
         b.data[j] = (_input[j] < 0) ? _input[j]+256 : _input[j];
     b.len = input_len;
-    output = (char *)PyMem_Malloc(sizeof(char) * b.len);
+    // Make the output buffer larger than the input as sometimes
+    // compression results in a larger block
+    output = (char *)PyMem_Malloc(sizeof(char) * (int)(1.25*b.len));
     if (output == NULL) return PyErr_NoMemory();
     j = cpalmdoc_do_compress(&b, output);
     if ( j == 0) return PyErr_NoMemory();

src/calibre/ebooks/html/input.py

@@ -329,7 +329,7 @@ class HTMLInput(InputFormatPlugin):
             metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'})
         if not metadata.language:
             oeb.logger.warn(u'Language not specified')
-            metadata.add('language', get_lang())
+            metadata.add('language', get_lang().replace('_', '-'))
         if not metadata.creator:
             oeb.logger.warn('Creator not specified')
             metadata.add('creator', self.oeb.translate(__('Unknown')))

src/calibre/ebooks/metadata/fetch.py

@@ -313,6 +313,8 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
     def sort_func(x, y):
 
         def cleanup_title(s):
+            if s is None:
+                s = _('Unknown')
             s = s.strip().lower()
             s = prefix_pat.sub(' ', s)
             s = trailing_paren_pat.sub('', s)

src/calibre/ebooks/metadata/opf2.py

@@ -1069,7 +1069,8 @@ class OPFCreator(MetaInformation):
             dc_attrs={'id':__appname__+'_id'}))
         if getattr(self, 'pubdate', None) is not None:
             a(DC_ELEM('date', self.pubdate.isoformat()))
-        a(DC_ELEM('language', self.language if self.language else get_lang()))
+        a(DC_ELEM('language', self.language if self.language else
+            get_lang().replace('_', '-')))
         if self.comments:
             a(DC_ELEM('description', self.comments))
         if self.publisher:
@@ -1194,7 +1195,8 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('identifier'), mi.isbn, scheme='ISBN')
     if mi.rights:
         factory(DC('rights'), mi.rights)
-    factory(DC('language'), mi.language if mi.language and mi.language.lower() != 'und' else get_lang())
+    factory(DC('language'), mi.language if mi.language and mi.language.lower()
+            != 'und' else get_lang().replace('_', '-'))
     if mi.tags:
         for tag in mi.tags:
             factory(DC('subject'), tag)

src/calibre/ebooks/oeb/reader.py

@@ -131,7 +131,7 @@ class OEBReader(object):
         stream = cStringIO.StringIO(etree.tostring(opf))
         mi = MetaInformation(OPF(stream))
         if not mi.language:
-            mi.language = get_lang()
+            mi.language = get_lang().replace('_', '-')
         self.oeb.metadata.add('language', mi.language)
         if not mi.title:
             mi.title = self.oeb.translate(__('Unknown'))

src/calibre/gui2/device.py

@@ -1416,7 +1416,6 @@ class DeviceMixin(object): # {{{
         # the application_id, which is really the db key, but as this can
         # accidentally match across libraries we also verify the title. The
         # db_id exists on Sony devices. Fallback is title and author match
-        resend_metadata = False
         for booklist in booklists:
             for book in booklist:
                 if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
@@ -1433,12 +1432,10 @@ class DeviceMixin(object): # {{{
                     if getattr(book, 'application_id', None) in d['db_ids']:
                         book.in_library = True
                         book.smart_update(d['db_ids'][book.application_id])
-                        resend_metadata = True
                         continue
                     if book.db_id in d['db_ids']:
                         book.in_library = True
                         book.smart_update(d['db_ids'][book.db_id])
-                        resend_metadata = True
                         continue
                     if book.authors:
                         # Compare against both author and author sort, because
@@ -1448,21 +1445,13 @@ class DeviceMixin(object): # {{{
                         if book_authors in d['authors']:
                             book.in_library = True
                             book.smart_update(d['authors'][book_authors])
-                            resend_metadata = True
                         elif book_authors in d['author_sort']:
                             book.in_library = True
                             book.smart_update(d['author_sort'][book_authors])
-                            resend_metadata = True
                 # Set author_sort if it isn't already
                 asort = getattr(book, 'author_sort', None)
                 if not asort and book.authors:
                     book.author_sort = self.library_view.model().db.author_sort_from_authors(book.authors)
-                    resend_metadata = True
-
-        if resend_metadata:
-            # Correct the metadata cache on device.
-            if self.device_manager.is_device_connected:
-                self.device_manager.sync_booklists(None, booklists)
 
     # }}}
 

src/calibre/gui2/dialogs/metadata_single.py

@@ -103,7 +103,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         if _file:
             _file = os.path.abspath(_file)
             if not os.access(_file, os.R_OK):
-                d = error_dialog(self.window, _('Cannot read'),
+                d = error_dialog(self, _('Cannot read'),
                         _('You do not have permission to read the file: ') + _file)
                 d.exec_()
                 return
@@ -112,14 +112,14 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                 cf = open(_file, "rb")
                 cover = cf.read()
             except IOError, e:
-                d = error_dialog(self.window, _('Error reading file'),
+                d = error_dialog(self, _('Error reading file'),
                         _("<p>There was an error reading from file: <br /><b>") + _file + "</b></p><br />"+str(e))
                 d.exec_()
             if cover:
                 pix = QPixmap()
                 pix.loadFromData(cover)
                 if pix.isNull():
-                    d = error_dialog(self.window,
+                    d = error_dialog(self,
                         _("Not a valid picture"),
                             _file + _(" is not a valid picture"))
                     d.exec_()
@@ -162,7 +162,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             self.formats_changed = True
             added = True
         if bad_perms:
-            error_dialog(self.window, _('No permission'),
+            error_dialog(self, _('No permission'),
                     _('You do not have '
                 'permission to read the following files:'),
                 det_msg='\n'.join(bad_perms), show=True)