From 55a7e1393b9a31d85e5fde5d8db84b93a0bd68b8 Mon Sep 17 00:00:00 2001 From: Kolenka Date: Sat, 15 Oct 2011 09:05:26 -0700 Subject: [PATCH 01/10] Sony T1: Driver Updates - Support switching between Calibre and Sony T1 style metadata in books.db (Defaults to Calibre style) - Attempt to calculate the device's time offset from books.db when scanning. --- src/calibre/devices/prst1/driver.py | 49 ++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index a3e3e8d0a0..fc72c99ec7 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -20,6 +20,7 @@ from calibre.devices.usbms.device import USBDevice from calibre.devices.usbms.books import CollectionsBookList from calibre.devices.usbms.books import BookList from calibre.constants import islinux +from calibre.ebooks.metadata import authors_to_string, authors_to_sort_string DBPATH = 'Sony_Reader/database/books.db' THUMBPATH = 'Sony_Reader/database/cache/books/%s/thumbnail/main_thumbnail.jpg' @@ -82,18 +83,26 @@ class PRST1(USBMS): 'the same aspect ratio (width to height) as the cover. ' 'Unset it if you want the thumbnail to be the maximum size, ' 'ignoring aspect ratio.'), + _('Use SONY Author Format (First Author Only)') + + ':::' + + _('Set this option if you want the author on the Sony to ' + 'appear the same way the T1 sets it. This means it will ' + 'only show the first author for books with multiple authors. ' + 'Leave this disabled if you use Metadata Plugboards.') ] EXTRA_CUSTOMIZATION_DEFAULT = [ ', '.join(['series', 'tags']), True, False, True, + False, ] OPT_COLLECTIONS = 0 OPT_UPLOAD_COVERS = 1 OPT_REFRESH_COVERS = 2 OPT_PRESERVE_ASPECT_RATIO = 3 + OPT_USE_SONY_AUTHORS = 4 plugboards = None plugboard_func = None @@ -103,6 +112,8 @@ class PRST1(USBMS): # that we do not preserve aspect ratio if not self.settings().extra_customization[self.OPT_PRESERVE_ASPECT_RATIO]: self.THUMBNAIL_WIDTH = 108 + # Make sure the date offset is set to none, we'll calculate it in books. + self.device_offset = None def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id or '_SETTING' in pnp_id @@ -168,6 +179,27 @@ class PRST1(USBMS): bl_collections.setdefault(row[0], []) bl_collections[row[0]].append(row[1]) + # collect information on offsets, but assume any + # offset we already calculated is correct + if self.device_offset is None: + query = 'SELECT file_path, modified_date FROM books' + cursor.execute(query) + + time_offsets = {} + for i, row in enumerate(cursor): + comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000); + device_date = int(row[1]); + offset = device_date - comp_date + time_offsets.setdefault(offset, 0) + time_offsets[offset] = time_offsets[offset] + 1 + + try: + device_offset = max(time_offsets,key = lambda a: time_offsets.get(a)) + debug_print("Device Offset: %d ms"%device_offset) + self.device_offset = device_offset + except ValueError: + debug_print("No Books To Detect Device Offset.") + for idx, book in enumerate(bl): query = 'SELECT _id, thumbnail FROM books WHERE file_path = ?' t = (book.lpath,) @@ -237,6 +269,7 @@ class PRST1(USBMS): opts = self.settings() upload_covers = opts.extra_customization[self.OPT_UPLOAD_COVERS] refresh_covers = opts.extra_customization[self.OPT_REFRESH_COVERS] + use_sony_authors = opts.extra_customization[self.OPT_USE_SONY_AUTHORS] cursor = connection.cursor() @@ -261,20 +294,26 @@ class PRST1(USBMS): lpath = book.lpath try: if opts.use_author_sort: - if newmi.author_sort : + if newmi.author_sort: author = newmi.author_sort else: author = authors_to_sort_string(newmi.authors) else: - author = newmi.authors[0] + if use_sony_authors: + author = newmi.authors[0] + else: + author = authors_to_string(newmi.authors) except: author = _('Unknown') title = newmi.title or _('Unknown') # Get modified date - modified_date = os.path.getmtime(book.path) - time_offset = time.altzone if time.daylight else time.timezone - modified_date = (modified_date - time_offset) * 1000 + modified_date = os.path.getmtime(book.path) * 1000 + if self.device_offset is not None: + modified_date = modified_date + self.device_offset + else: + time_offset = -time.altzone if time.daylight else -time.timezone + modified_date = modified_date + (time_offset * 1000) if lpath not in db_books: query = ''' From e078a7ca802226fab7a53c6996b01779335ac31d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 Oct 2011 18:36:22 +0200 Subject: [PATCH 02/10] Improve performance of list_ formatter functions. Make list_re not add a value more than once. --- src/calibre/utils/formatter_functions.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index f72e26b19d..27ded23eeb 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -909,12 +909,10 @@ class BuiltinListUnion(BuiltinFormatterFunction): l2 = [l.strip() for l in list2.split(separator) if l.strip()] lcl1 = set([icu_lower(l) for l in l1]) - res = [] - for i in l1: - res.append(i) + res = set(l1) for i in l2: if icu_lower(i) not in lcl1: - res.append(i) + res.add(i) if separator == ',': return ', '.join(res) return separator.join(res) @@ -930,7 +928,7 @@ class BuiltinListDifference(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator): l1 = [l.strip() for l in list1.split(separator) if l.strip()] - l2 = [icu_lower(l.strip()) for l in list2.split(separator) if l.strip()] + l2 = set([icu_lower(l.strip()) for l in list2.split(separator) if l.strip()]) res = [] for i in l1: @@ -951,12 +949,12 @@ class BuiltinListIntersection(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, list1, list2, separator): l1 = [l.strip() for l in list1.split(separator) if l.strip()] - l2 = [icu_lower(l.strip()) for l in list2.split(separator) if l.strip()] + l2 = set([icu_lower(l.strip()) for l in list2.split(separator) if l.strip()]) - res = [] + res = set() for i in l1: if icu_lower(i) in l2: - res.append(i) + res.add(i) if separator == ',': return ', '.join(res) return separator.join(res) @@ -1007,12 +1005,12 @@ class BuiltinListRe(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, src_list, separator, search_re, opt_replace): l = [l.strip() for l in src_list.split(separator) if l.strip()] - res = [] + res = set() for item in l: if re.search(search_re, item, flags=re.I) is not None: if opt_replace: item = re.sub(search_re, opt_replace, item) - res.append(item) + res.add(item) if separator == ',': return ', '.join(res) return separator.join(res) From 2a4c9dadc9b445893c89b388ec2e9a7a67bfbd82 Mon Sep 17 00:00:00 2001 From: Kolenka Date: Sat, 15 Oct 2011 09:43:32 -0700 Subject: [PATCH 03/10] Sony T1: Periodical Support (Rough) --- src/calibre/devices/prst1/driver.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 95747e7d23..9ec0ad7094 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -14,6 +14,7 @@ Device driver for the SONY T1 devices import os, time, re import sqlite3 as sqlite from contextlib import closing +from datetime import date from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.usbms.device import USBDevice @@ -345,6 +346,9 @@ class PRST1(USBMS): self.upload_book_cover(connection, book, source_id) db_books[lpath] = None + if self.is_sony_periodical(book): + self.periodicalize_book(connection, book) + for book, bookId in db_books.items(): if bookId is not None: # Remove From Collections @@ -518,3 +522,54 @@ class PRST1(USBMS): connection.commit() cursor.close() + + def is_sony_periodical(self, book): + if _('News') not in book.tags: + return False + if not book.lpath.lower().endswith('.epub'): + return False + if book.pubdate.date() < date(2010, 10, 17): + return False + return True + + def periodicalize_book(self, connection, book): + if not self.is_sony_periodical(book): + return + + name = None + if '[' in book.title: + name = book.title.split('[')[0].strip() + if len(name) < 4: + name = None + if not name: + try: + name = [t for t in book.tags if t != _('News')][0] + except: + name = None + + if not name: + name = book.title + + pubdate = None + try: + pubdate = int(time.mktime(book.pubdate.timetuple()) * 1000) + except: + pass + + description = '' + + cursor = connection.cursor() + + query = ''' + UPDATE books + SET conforms_to = 'http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0', + periodical_name = ?, + description = ?, + publication_date = ? + WHERE _id = ? + ''' + t = (name, description, pubdate, book.bookId,) + cursor.execute(query, t) + + connection.commit() + cursor.close() From 0329eb833e56c1640dee7d36fc072ca5414eff5d Mon Sep 17 00:00:00 2001 From: Kolenka Date: Sat, 15 Oct 2011 09:50:59 -0700 Subject: [PATCH 04/10] Sony T1: Updated Periodical Support --- src/calibre/devices/prst1/driver.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 9ec0ad7094..12d6b3ddef 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -556,8 +556,6 @@ class PRST1(USBMS): except: pass - description = '' - cursor = connection.cursor() query = ''' @@ -568,7 +566,7 @@ class PRST1(USBMS): publication_date = ? WHERE _id = ? ''' - t = (name, description, pubdate, book.bookId,) + t = (name, None, pubdate, book.bookId,) cursor.execute(query, t) connection.commit() From 809ea8c1a7f0a9649b51ce2a27fceb53d7f566de Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 Oct 2011 18:52:48 +0200 Subject: [PATCH 05/10] prevent list_re from adding empty items to the set. Permit the replacement to create list items (add commas). --- src/calibre/utils/formatter_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 27ded23eeb..38fcbe8a1e 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -1010,7 +1010,8 @@ class BuiltinListRe(BuiltinFormatterFunction): if re.search(search_re, item, flags=re.I) is not None: if opt_replace: item = re.sub(search_re, opt_replace, item) - res.add(item) + for i in [l.strip() for l in item.split(',') if l.strip()]: + res.add(i) if separator == ',': return ', '.join(res) return separator.join(res) From 48d68a6f694d132feab78bbed680638f9433e60d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Oct 2011 06:00:22 +0530 Subject: [PATCH 06/10] Fix #875287 (Remove recipe for Technet.hu) --- recipes/huntechnet.recipe | 41 --------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 recipes/huntechnet.recipe diff --git a/recipes/huntechnet.recipe b/recipes/huntechnet.recipe deleted file mode 100644 index 3dea03ba34..0000000000 --- a/recipes/huntechnet.recipe +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -from calibre.web.feeds.news import BasicNewsRecipe - -class HunTechNet(BasicNewsRecipe): - title = u'TechNet' - oldest_article = 3 - description = u'Az ut\xf3bbi 3 nap TechNet h\xedrei' - language = 'hu' - - lang = 'hu' - encoding = 'utf-8' - __author__ = 'Devilinside' - max_articles_per_feed = 30 - timefmt = ' [%Y, %b %d, %a]' - - - - - remove_tags_before = dict(name='div', attrs={'id':'c-main'}) - remove_tags = [dict(name='div', attrs={'class':'wrp clr'}), - {'class' : ['screenrdr','forum','print','startlap','text_small','text_normal','text_big','email']}, - ] - keep_only_tags = [dict(name='div', attrs={'class':'cikk_head box'}),dict(name='div', attrs={'class':'cikk_txt box'})] - - - - feeds = [(u'C\xedmlap', - u'http://www.technet.hu/rss/cimoldal/'), (u'TechTud', - u'http://www.technet.hu/rss/techtud/'), (u'PDA M\xe1nia', - u'http://www.technet.hu/rss/pdamania/'), (u'Telefon', - u'http://www.technet.hu/rss/telefon/'), (u'Sz\xe1m\xedt\xf3g\xe9p', - u'http://www.technet.hu/rss/notebook/'), (u'GPS', - u'http://www.technet.hu/rss/gps/')] - From 89c05e0acf9f71ecf2c96d715b9cdadb530df9e3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Oct 2011 06:00:56 +0530 Subject: [PATCH 07/10] Fix #875264 (Remove invalid recipe Honvedelem.hu) --- recipes/honvedelem.recipe | 50 --------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 recipes/honvedelem.recipe diff --git a/recipes/honvedelem.recipe b/recipes/honvedelem.recipe deleted file mode 100644 index f2cab5d640..0000000000 --- a/recipes/honvedelem.recipe +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - - -from calibre.web.feeds.news import BasicNewsRecipe - -class HunMilNews(BasicNewsRecipe): - title = u'Honvedelem.hu' - oldest_article = 3 - description = u'Katonah\xedrek' - language = 'hu' - - lang = 'hu' - encoding = 'windows-1250' - category = 'news, military' - - no_stylesheets = True - - - __author__ = 'Devilinside' - max_articles_per_feed = 16 - no_stylesheets = True - - - - keep_only_tags = [dict(name='div', attrs={'class':'cikkoldal_cikk_cim'}), - dict(name='div', attrs={'class':'cikkoldal_cikk_alcim'}), - dict(name='div', attrs={'class':'cikkoldal_datum'}), - dict(name='div', attrs={'class':'cikkoldal_lead'}), - dict(name='div', attrs={'class':'cikkoldal_szoveg'}), - dict(name='img', attrs={'class':'ajanlo_kep_keretes'}), - ] - - - - feeds = [(u'Misszi\xf3k', u'http://www.honvedelem.hu/rss_b?c=22'), - (u'Aktu\xe1lis hazai h\xedrek', u'http://www.honvedelem.hu/rss_b?c=3'), - (u'K\xfclf\xf6ldi h\xedrek', u'http://www.honvedelem.hu/rss_b?c=4'), - (u'A h\xf3nap t\xe9m\xe1ja', u'http://www.honvedelem.hu/rss_b?c=6'), - (u'Riport', u'http://www.honvedelem.hu/rss_b?c=5'), - (u'Portr\xe9k', u'http://www.honvedelem.hu/rss_b?c=7'), - (u'Haditechnika', u'http://www.honvedelem.hu/rss_b?c=8'), - (u'Programok, esem\xe9nyek', u'http://www.honvedelem.hu/rss_b?c=12') - ] - From 68448106a6edb43b2b7702a497c1243c494e8fae Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Oct 2011 07:25:38 +0530 Subject: [PATCH 08/10] Fix regression that broke updating of covers inside ebook files when saving to disk --- src/calibre/library/save_to_disk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 7b8ff11d16..2dcdd252bc 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -374,6 +374,8 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, newmi.template_to_attribute(mi, cpb) else: newmi = mi + if cover: + newmi.cover_data = ('jpg', cover) set_metadata(stream, newmi, fmt) except: if DEBUG: From 12fffc051e45e640d6b3cbfdabf56f74361d6126 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Oct 2011 17:18:26 +0530 Subject: [PATCH 09/10] ... --- src/calibre/devices/prs505/driver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 4d9c66aaa8..bfce4fa1be 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -207,8 +207,11 @@ class PRS505(USBMS): c = self.initialize_XML_cache() blists = {} for i in c.paths: - if booklists[i] is not None: - blists[i] = booklists[i] + try: + if booklists[i] is not None: + blists[i] = booklists[i] + except IndexError: + pass opts = self.settings() if opts.extra_customization: collections = [x.strip() for x in From 557f46fd47136946a375d0570d803c67e6fe8273 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Oct 2011 17:20:04 +0530 Subject: [PATCH 10/10] Fix a regression that broke downloading certain periodicals for the Kindle. Fixes #875595 (Calibre- Unable to download news - UnicodeEncodeError) --- src/calibre/ebooks/mobi/writer2/serializer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index eeef720144..041a9922cb 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -212,7 +212,11 @@ class Serializer(object): if tocref.klass == "periodical": buf.write('
') else: - buf.write('

'+tocref.title+'

') + t = tocref.title + if isinstance(t, unicode): + t = t.encode('utf-8') + buf.write('

' + +t+'

') buf.write('
    ') @@ -221,14 +225,17 @@ class Serializer(object): itemhref = tocitem.href if tocref.klass == 'periodical': # This is a section node. - # For periodical toca, the section urls are like r'feed_\d+/index.html' + # For periodical tocs, the section urls are like r'feed_\d+/index.html' # We dont want to point to the start of the first article # so we change the href. itemhref = re.sub(r'article_\d+/', '', itemhref) self.href_offsets[itemhref].append(buf.tell()) buf.write('0000000000') buf.write(' >') - buf.write(tocitem.title) + t = tocitem.title + if isinstance(t, unicode): + t = t.encode('utf-8') + buf.write(t) buf.write('') buf.write('
')