From 150acaffef9aa90aa55d9e42bee2e3a2e4f35c7b Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Wed, 2 Jun 2010 21:43:09 -0300 Subject: [PATCH 01/30] New KOBO driver. Displays and allows kobo books to be deleted from the sqlite database --- src/calibre/__init__.py | 1 + src/calibre/devices/kobo/books.py | 112 +++++++++++ src/calibre/devices/kobo/driver.py | 302 ++++++++++++++++++++++++++++- 3 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 src/calibre/devices/kobo/books.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index ff4bab6a9a..65618aaff4 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -41,6 +41,7 @@ mimetypes.add_type('application/vnd.palm', '.pdb') mimetypes.add_type('application/x-mobipocket-ebook', '.mobi') mimetypes.add_type('application/x-mobipocket-ebook', '.prc') mimetypes.add_type('application/x-mobipocket-ebook', '.azw') +mimetypes.add_type('application/x-koboreader-ebook', '.kobo') mimetypes.add_type('image/wmf', '.wmf') guess_type = mimetypes.guess_type import cssutils diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py new file mode 100644 index 0000000000..95daec58a9 --- /dev/null +++ b/src/calibre/devices/kobo/books.py @@ -0,0 +1,112 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Timothy Legge ' +''' +''' + +import os +import re +import time + +from calibre.ebooks.metadata import MetaInformation +from calibre.devices.interface import BookList as _BookList +from calibre.constants import filesystem_encoding, preferred_encoding +from calibre import isbytestring + +class Book(MetaInformation): + + BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections'] + + JSON_ATTRS = [ + 'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort', + 'title_sort', 'comments', 'category', 'publisher', 'series', + 'series_index', 'rating', 'isbn', 'language', 'application_id', + 'book_producer', 'lccn', 'lcc', 'ddc', 'rights', 'publication_type', + 'uuid', + ] + + def __init__(self, mountpath, path, title, authors, mime, date, ContentType, ImageID, other=None): + + MetaInformation.__init__(self, '') + self.device_collections = [] + + self.title = title + if not authors: + self.authors = [''] + else: + self.authors = [authors] + self.mime = mime + self.path = path + try: + self.size = os.path.getsize(path) + except OSError: + self.size = 0 + try: + if ContentType == '6': + self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") + else: + self.datetime = time.gmtime(os.path.getctime(path)) + except ValueError: + self.datetime = time.gmtime() + except OSError: + self.datetime = time.gmtime() + self.lpath = path + + self.thumbnail = ImageWrapper(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + self.tags = [] + if other: + self.smart_update(other) + + def __eq__(self, other): + return self.path == getattr(other, 'path', None) + + @dynamic_property + def db_id(self): + doc = '''The database id in the application database that this file corresponds to''' + def fget(self): + match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0]) + if match: + return int(match.group(1)) + return None + return property(fget=fget, doc=doc) + + @dynamic_property + def title_sorter(self): + doc = '''String to sort the title. If absent, title is returned''' + def fget(self): + return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip() + return property(doc=doc, fget=fget) + + @dynamic_property + def thumbnail(self): + return None + + def smart_update(self, other): + ''' + Merge the information in C{other} into self. In case of conflicts, the information + in C{other} takes precedence, unless the information in C{other} is NULL. + ''' + + MetaInformation.smart_update(self, other) + + for attr in self.BOOK_ATTRS: + if hasattr(other, attr): + val = getattr(other, attr, None) + setattr(self, attr, val) + + def to_json(self): + json = {} + for attr in self.JSON_ATTRS: + val = getattr(self, attr) + if isbytestring(val): + enc = filesystem_encoding if attr == 'lpath' else preferred_encoding + val = val.decode(enc, 'replace') + elif isinstance(val, (list, tuple)): + val = [x.decode(preferred_encoding, 'replace') if + isbytestring(x) else x for x in val] + json[attr] = val + return json + +class ImageWrapper(object): + def __init__(self, image_path): + self.image_path = image_path + diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 4b14b2bf8e..a480b8de2a 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -2,17 +2,26 @@ # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai __license__ = 'GPL v3' -__copyright__ = '2010, Kovid Goyal ' +__copyright__ = '2010, Timothy Legge and Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os + +import cStringIO +import sqlite3 as sqlite +from calibre.devices.usbms.books import BookList +from calibre.devices.kobo.books import Book +from calibre.devices.kobo.books import ImageWrapper +from calibre.devices.mime import mime_type_ext from calibre.devices.usbms.driver import USBMS + class KOBO(USBMS): name = 'Kobo Reader Device Interface' gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') - author = 'Kovid Goyal' + author = 'Timothy Legge and Kovid Goyal' supported_platforms = ['windows', 'osx', 'linux'] @@ -29,3 +38,292 @@ class KOBO(USBMS): EBOOK_DIR_MAIN = '' SUPPORTS_SUB_DIRS = True + def initialize(self): + USBMS.initialize(self) + self.book_class = Book + + def books(self, oncard=None, end_session=True): + from calibre.ebooks.metadata.meta import path_to_ext + + dummy_bl = BookList(None, None, None) + + if oncard == 'carda' and not self._card_a_prefix: + self.report_progress(1.0, _('Getting list of books on device...')) + return dummy_bl + elif oncard == 'cardb' and not self._card_b_prefix: + self.report_progress(1.0, _('Getting list of books on device...')) + return dummy_bl + elif oncard and oncard != 'carda' and oncard != 'cardb': + self.report_progress(1.0, _('Getting list of books on device...')) + return dummy_bl + + prefix = self._card_a_prefix if oncard == 'carda' else \ + self._card_b_prefix if oncard == 'cardb' \ + else self._main_prefix + + ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ + self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ + self.get_main_ebook_dir() + + # get the metadata cache + bl = self.booklist_class(oncard, prefix, self.settings) + need_sync = self.parse_metadata_cache(bl, prefix, self.METADATA_CACHE) + + # make a dict cache of paths so the lookup in the loop below is faster. + bl_cache = {} + for idx,b in enumerate(bl): + bl_cache[b.lpath] = idx + + def update_booklist(mountpath, ContentID, filename, title, authors, mime, date, ContentType, ImageID): + changed = False + # if path_to_ext(filename) in self.FORMATS: + try: + # lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] + # if lpath.startswith(os.sep): + # lpath = lpath[len(os.sep):] + # lpath = lpath.replace('\\', '/') + idx = bl_cache.get(filename, None) + if idx is not None: + bl[idx].thumbnail = ImageWrapper(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + bl_cache[filename] = None + if ContentType != '6': + if self.update_metadata_item(bl[idx]): + # print 'update_metadata_item returned true' + changed = True + else: + book = Book(mountpath, filename, title, authors, mime, date, ContentType, ImageID) + # print 'Update booklist' + if bl.add_book(book, replace_metadata=False): + changed = True + except: # Probably a filename encoding error + import traceback + traceback.print_exc() + return changed + + connection = sqlite.connect(self._main_prefix + '.kobo/KoboReader.sqlite') + cursor = connection.cursor() + + query = 'select count(distinct volumeId) from volume_shortcovers' + cursor.execute(query) + for row in (cursor): + numrows = row[0] + cursor.close() + + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID from content where ContentID in (select distinct volumeId from volume_shortcovers)' + + cursor.execute (query) + + changed = False + + for i, row in enumerate(cursor): + self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) + + filename = row[3] + if row[5] == "6": + filename = filename + '.kobo' + mime = mime_type_ext(path_to_ext(row[3])) + + if oncard != 'carda' and oncard != 'cardb': + if row[5] == '6': + # print "shortbook: " + filename + changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + if changed: + need_sync = True + else: + if filename.startswith("file:///mnt/onboard/"): + filename = filename.replace("file:///mnt/onboard/", self._main_prefix) + # print "Internal: " + filename + changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + if changed: + need_sync = True + elif oncard == 'carda': + if filename.startswith("file:///mnt/sd/"): + filename = filename.replace("file:///mnt/sd/", self._card_a_prefix) + # print "SD Card: " + filename + changed = update_booklist(self._card_a_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + if changed: + need_sync = True + else: + print "Add card b support" + + #FIXME - NOT NEEDED flist.append({'filename': filename, 'path':row[3]}) + #bl.append(book) + + cursor.close() + connection.close() + + # Remove books that are no longer in the filesystem. Cache contains + # indices into the booklist if book not in filesystem, None otherwise + # Do the operation in reverse order so indices remain valid + for idx in sorted(bl_cache.itervalues(), reverse=True): + if idx is not None: + need_sync = True + del bl[idx] + + print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ + (len(bl_cache), len(bl), need_sync) + if need_sync: #self.count_found_in_bl != len(bl) or need_sync: + if oncard == 'cardb': + self.sync_booklists((None, None, bl)) + elif oncard == 'carda': + self.sync_booklists((None, bl, None)) + else: + self.sync_booklists((bl, None, None)) + + self.report_progress(1.0, _('Getting list of books on device...')) + return bl + + def delete_via_sql(self, ContentID, ContentType): + # Delete Order: + # 1) shortcover_page + # 2) volume_shorcover + # 2) content + + connection = sqlite.connect(self._main_prefix + '.kobo/KoboReader.sqlite') + cursor = connection.cursor() + t = (ContentID,) + cursor.execute('select ImageID from content where ContentID = ?', t) + + for row in cursor: + # First get the ImageID to delete the images + ImageID = row[0] + cursor.close() + + cursor = connection.cursor() + if ContentType == 6: + # Delete the shortcover_pages first + cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) + + #Delete the volume_shortcovers second + cursor.execute('delete from volume_shortcovers where volumeid = ?', t) + + # Delete the chapters associated with the book next + t = (ContentID,ContentID,) + cursor.execute('delete from content where BookID = ? or ContentID = ?', t) + + connection.commit() + + cursor.close() + connection.close() + # If all this succeeds we need to delete the images files via the ImageID + return ImageID + + def delete_images(self, ImageID): + path_prefix = '.kobo/images/' + path = self._main_prefix + path_prefix + ImageID + + file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',) + + for ending in file_endings: + fpath = path + ending + fpath = self.normalize_path(fpath) + + if os.path.exists(fpath): + # print 'Image File Exists: ' + fpath + os.unlink(fpath) + + def delete_books(self, paths, end_session=True): + for i, path in enumerate(paths): + self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) + path = self.normalize_path(path) + extension = os.path.splitext(path)[1] + + if extension == '.kobo': + # Kobo books do not have book files. They do have some images though + #print "kobo book" + ContentType = 6 + ContentID = os.path.splitext(path)[0] + # Remove the prefix on the file. it could be either + ContentID = ContentID.replace(self._main_prefix, '') + if self._card_a_prefix is not None: + ContentID = ContentID.replace(self._card_a_prefix, '') + + ImageID = self.delete_via_sql(ContentID, ContentType) + #print " We would now delete the Images for" + ImageID + self.delete_images(ImageID) + if extension == '.pdf' or extension == '.epub': + # print "ePub or pdf" + ContentType = 16 + #print "Path: " + path + ContentID = path + ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") + if self._card_a_prefix is not None: + ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") + # print "ContentID: " + ContentID + ImageID = self.delete_via_sql(ContentID, ContentType) + #print " We would now delete the Images for" + ImageID + self.delete_images(ImageID) + + if os.path.exists(path): + # Delete the ebook + # print "Delete the ebook: " + path + os.unlink(path) + + filepath = os.path.splitext(path)[0] + for ext in self.DELETE_EXTS: + if os.path.exists(filepath + ext): + # print "Filename: " + filename + os.unlink(filepath + ext) + if os.path.exists(path + ext): + # print "Filename: " + filename + os.unlink(path + ext) + + if self.SUPPORTS_SUB_DIRS: + try: + # print "removed" + os.removedirs(os.path.dirname(path)) + except: + pass + self.report_progress(1.0, _('Removing books from device...')) + + def remove_books_from_metadata(self, paths, booklists): + for i, path in enumerate(paths): + self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...')) + for bl in booklists: + for book in bl: + #print "Book Path: " + book.path + if path.endswith(book.path): + #print " Remove: " + book.path + bl.remove_book(book) + self.report_progress(1.0, _('Removing books from device metadata listing...')) + + def add_books_to_metadata(self, locations, metadata, booklists): + metadata = iter(metadata) + for i, location in enumerate(locations): + self.report_progress((i+1) / float(len(locations)), _('Adding books to device metadata listing...')) + info = metadata.next() + blist = 2 if location[1] == 'cardb' else 1 if location[1] == 'carda' else 0 + + # Extract the correct prefix from the pathname. To do this correctly, + # we must ensure that both the prefix and the path are normalized + # so that the comparison will work. Book's __init__ will fix up + # lpath, so we don't need to worry about that here. + path = self.normalize_path(location[0]) + if self._main_prefix: + prefix = self._main_prefix if \ + path.startswith(self.normalize_path(self._main_prefix)) else None + if not prefix and self._card_a_prefix: + prefix = self._card_a_prefix if \ + path.startswith(self.normalize_path(self._card_a_prefix)) else None + if not prefix and self._card_b_prefix: + prefix = self._card_b_prefix if \ + path.startswith(self.normalize_path(self._card_b_prefix)) else None + if prefix is None: + prints('in add_books_to_metadata. Prefix is None!', path, + self._main_prefix) + continue + lpath = path.partition(prefix)[2] + if lpath.startswith('/') or lpath.startswith('\\'): + lpath = lpath[1:] + #book = self.book_class(prefix, lpath, other=info) + book = Book(prefix, lpath, '', '', '', '', '', '', other=info) + if book.size is None: + book.size = os.stat(self.normalize_path(path)).st_size + booklists[blist].add_book(book, replace_metadata=True) + self.report_progress(1.0, _('Adding books to device metadata listing...')) + +#class ImageWrapper(object): +# def __init__(self, image_path): +# self.image_path = image_path + From bb0c44693a8e43a6dfcc72eabfe04660dc41c6a7 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 8 Jun 2010 19:38:27 -0300 Subject: [PATCH 02/30] Few changes for silly path differences in Windows and darn tabs versus spaces --- src/calibre/devices/kobo/books.py | 34 ++++----- src/calibre/devices/kobo/driver.py | 117 ++++++++++++++++------------- 2 files changed, 83 insertions(+), 68 deletions(-) diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index 95daec58a9..2ed95a4116 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -24,34 +24,34 @@ class Book(MetaInformation): 'uuid', ] - def __init__(self, mountpath, path, title, authors, mime, date, ContentType, ImageID, other=None): + def __init__(self, mountpath, path, title, authors, mime, date, ContentType, thumbnail_name, other=None): MetaInformation.__init__(self, '') self.device_collections = [] - self.title = title - if not authors: - self.authors = [''] - else: - self.authors = [authors] + self.title = title + if not authors: + self.authors = [''] + else: + self.authors = [authors] self.mime = mime - self.path = path - try: - self.size = os.path.getsize(path) - except OSError: - self.size = 0 + self.path = path try: - if ContentType == '6': - self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") - else: + self.size = os.path.getsize(path) + except OSError: + self.size = 0 + try: + if ContentType == '6': + self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") + else: self.datetime = time.gmtime(os.path.getctime(path)) except ValueError: self.datetime = time.gmtime() - except OSError: + except OSError: self.datetime = time.gmtime() self.lpath = path - self.thumbnail = ImageWrapper(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + self.thumbnail = ImageWrapper(thumbnail_name) self.tags = [] if other: self.smart_update(other) @@ -108,5 +108,5 @@ class Book(MetaInformation): class ImageWrapper(object): def __init__(self, image_path): - self.image_path = image_path + self.image_path = image_path diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index a480b8de2a..35847cdbf8 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -82,14 +82,20 @@ class KOBO(USBMS): # if lpath.startswith(os.sep): # lpath = lpath[len(os.sep):] # lpath = lpath.replace('\\', '/') + print "Filename: " + filename + filename = self.normalize_path(filename) + print "Normalized FileName: " + filename + idx = bl_cache.get(filename, None) if idx is not None: - bl[idx].thumbnail = ImageWrapper(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + imagename = self.normalize_path(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + print "Image name Normalized: " + imagename + bl[idx].thumbnail = ImageWrapper(imagename) bl_cache[filename] = None if ContentType != '6': - if self.update_metadata_item(bl[idx]): - # print 'update_metadata_item returned true' - changed = True + if self.update_metadata_item(bl[idx]): + # print 'update_metadata_item returned true' + changed = True else: book = Book(mountpath, filename, title, authors, mime, date, ContentType, ImageID) # print 'Update booklist' @@ -103,21 +109,21 @@ class KOBO(USBMS): connection = sqlite.connect(self._main_prefix + '.kobo/KoboReader.sqlite') cursor = connection.cursor() - query = 'select count(distinct volumeId) from volume_shortcovers' - cursor.execute(query) - for row in (cursor): - numrows = row[0] - cursor.close() + #query = 'select count(distinct volumeId) from volume_shortcovers' + #cursor.execute(query) + #for row in (cursor): + # numrows = row[0] + #cursor.close() query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID from content where ContentID in (select distinct volumeId from volume_shortcovers)' + 'ImageID from content where ContentID in (select distinct volumeId from volume_shortcovers)' cursor.execute (query) changed = False for i, row in enumerate(cursor): - self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) + # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) filename = row[3] if row[5] == "6": @@ -125,18 +131,18 @@ class KOBO(USBMS): mime = mime_type_ext(path_to_ext(row[3])) if oncard != 'carda' and oncard != 'cardb': - if row[5] == '6': - # print "shortbook: " + filename - changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + if row[5] == '6': + # print "shortbook: " + filename + changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + if changed: + need_sync = True + else: + if filename.startswith("file:///mnt/onboard/"): + filename = filename.replace("file:///mnt/onboard/", self._main_prefix) + # print "Internal: " + filename + changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) if changed: need_sync = True - else: - if filename.startswith("file:///mnt/onboard/"): - filename = filename.replace("file:///mnt/onboard/", self._main_prefix) - # print "Internal: " + filename - changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) - if changed: - need_sync = True elif oncard == 'carda': if filename.startswith("file:///mnt/sd/"): filename = filename.replace("file:///mnt/sd/", self._card_a_prefix) @@ -145,14 +151,14 @@ class KOBO(USBMS): if changed: need_sync = True else: - print "Add card b support" + print "Add card b support" #FIXME - NOT NEEDED flist.append({'filename': filename, 'path':row[3]}) #bl.append(book) - + cursor.close() connection.close() - + # Remove books that are no longer in the filesystem. Cache contains # indices into the booklist if book not in filesystem, None otherwise # Do the operation in reverse order so indices remain valid @@ -185,48 +191,56 @@ class KOBO(USBMS): t = (ContentID,) cursor.execute('select ImageID from content where ContentID = ?', t) + ImageID = None for row in cursor: # First get the ImageID to delete the images ImageID = row[0] cursor.close() - - cursor = connection.cursor() - if ContentType == 6: - # Delete the shortcover_pages first - cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) + + if ImageID != None: + cursor = connection.cursor() + if ContentType == 6: + # Delete the shortcover_pages first + cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) - #Delete the volume_shortcovers second - cursor.execute('delete from volume_shortcovers where volumeid = ?', t) + #Delete the volume_shortcovers second + cursor.execute('delete from volume_shortcovers where volumeid = ?', t) - # Delete the chapters associated with the book next - t = (ContentID,ContentID,) - cursor.execute('delete from content where BookID = ? or ContentID = ?', t) + # Delete the chapters associated with the book next + t = (ContentID,ContentID,) + cursor.execute('delete from content where BookID = ? or ContentID = ?', t) - connection.commit() + connection.commit() + + cursor.close() + else: + print "Error condition ImageID was not found" + print "You likely tried to delete a book that the kobo has not yet added to the database" - cursor.close() connection.close() # If all this succeeds we need to delete the images files via the ImageID return ImageID def delete_images(self, ImageID): - path_prefix = '.kobo/images/' - path = self._main_prefix + path_prefix + ImageID + if ImageID == None: + path_prefix = '.kobo/images/' + path = self._main_prefix + path_prefix + ImageID - file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',) + file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',) - for ending in file_endings: - fpath = path + ending - fpath = self.normalize_path(fpath) + for ending in file_endings: + fpath = path + ending + fpath = self.normalize_path(fpath) - if os.path.exists(fpath): - # print 'Image File Exists: ' + fpath - os.unlink(fpath) + if os.path.exists(fpath): + # print 'Image File Exists: ' + fpath + os.unlink(fpath) def delete_books(self, paths, end_session=True): for i, path in enumerate(paths): self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) path = self.normalize_path(path) + print "Delete file normalized path: " + path extension = os.path.splitext(path)[1] if extension == '.kobo': @@ -238,7 +252,7 @@ class KOBO(USBMS): ContentID = ContentID.replace(self._main_prefix, '') if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, '') - + ContentID = ContentID.replace("\\", '/') ImageID = self.delete_via_sql(ContentID, ContentType) #print " We would now delete the Images for" + ImageID self.delete_images(ImageID) @@ -250,6 +264,7 @@ class KOBO(USBMS): ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") + ContentID = ContentID.replace("\\", '/') # print "ContentID: " + ContentID ImageID = self.delete_via_sql(ContentID, ContentType) #print " We would now delete the Images for" + ImageID @@ -313,17 +328,17 @@ class KOBO(USBMS): prints('in add_books_to_metadata. Prefix is None!', path, self._main_prefix) continue + print "Add book to metatdata: " + print "prefix: " + prefix lpath = path.partition(prefix)[2] if lpath.startswith('/') or lpath.startswith('\\'): lpath = lpath[1:] + print "path: " + lpath #book = self.book_class(prefix, lpath, other=info) - book = Book(prefix, lpath, '', '', '', '', '', '', other=info) + lpath = self.normalize_path(prefix + lpath) + book = Book(prefix, lpath, '', '', '', '', '', '', other=info) if book.size is None: book.size = os.stat(self.normalize_path(path)).st_size booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) -#class ImageWrapper(object): -# def __init__(self, image_path): -# self.image_path = image_path - From 7b9823e80d8e4c7f96b2ced9752c3a4e1c991546 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 8 Jun 2010 21:07:13 -0300 Subject: [PATCH 03/30] Fix silly bug --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 35847cdbf8..42d6cdc295 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -222,7 +222,7 @@ class KOBO(USBMS): return ImageID def delete_images(self, ImageID): - if ImageID == None: + if ImageID != None: path_prefix = '.kobo/images/' path = self._main_prefix + path_prefix + ImageID From 330c9ef1ba0196d3a1ff41bfd3cce30aff48b922 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Tue, 8 Jun 2010 21:12:35 -0300 Subject: [PATCH 04/30] remove debugging statements --- src/calibre/devices/kobo/driver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 42d6cdc295..10261351a8 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -82,14 +82,14 @@ class KOBO(USBMS): # if lpath.startswith(os.sep): # lpath = lpath[len(os.sep):] # lpath = lpath.replace('\\', '/') - print "Filename: " + filename + # print "Filename: " + filename filename = self.normalize_path(filename) - print "Normalized FileName: " + filename + # print "Normalized FileName: " + filename idx = bl_cache.get(filename, None) if idx is not None: imagename = self.normalize_path(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') - print "Image name Normalized: " + imagename + # print "Image name Normalized: " + imagename bl[idx].thumbnail = ImageWrapper(imagename) bl_cache[filename] = None if ContentType != '6': @@ -240,7 +240,7 @@ class KOBO(USBMS): for i, path in enumerate(paths): self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) path = self.normalize_path(path) - print "Delete file normalized path: " + path + # print "Delete file normalized path: " + path extension = os.path.splitext(path)[1] if extension == '.kobo': From 54d360e95acf37d22c9f51b0ec2082d6875e7bd7 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Mon, 14 Jun 2010 23:01:16 -0300 Subject: [PATCH 05/30] Simplify path to ContentID --- src/calibre/devices/kobo/driver.py | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 10261351a8..dce72d0fba 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -74,7 +74,7 @@ class KOBO(USBMS): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx - def update_booklist(mountpath, ContentID, filename, title, authors, mime, date, ContentType, ImageID): + def update_booklist(mountpath, filename, title, authors, mime, date, ContentType, ImageID): changed = False # if path_to_ext(filename) in self.FORMATS: try: @@ -89,7 +89,7 @@ class KOBO(USBMS): idx = bl_cache.get(filename, None) if idx is not None: imagename = self.normalize_path(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') - # print "Image name Normalized: " + imagename + print "Image name Normalized: " + imagename bl[idx].thumbnail = ImageWrapper(imagename) bl_cache[filename] = None if ContentType != '6': @@ -133,21 +133,21 @@ class KOBO(USBMS): if oncard != 'carda' and oncard != 'cardb': if row[5] == '6': # print "shortbook: " + filename - changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + changed = update_booklist(self._main_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) if changed: need_sync = True else: if filename.startswith("file:///mnt/onboard/"): filename = filename.replace("file:///mnt/onboard/", self._main_prefix) # print "Internal: " + filename - changed = update_booklist(self._main_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + changed = update_booklist(self._main_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) if changed: need_sync = True elif oncard == 'carda': if filename.startswith("file:///mnt/sd/"): filename = filename.replace("file:///mnt/sd/", self._card_a_prefix) # print "SD Card: " + filename - changed = update_booklist(self._card_a_prefix, row[3], filename, row[0], row[1], mime, row[2], row[5], row[6]) + changed = update_booklist(self._card_a_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) if changed: need_sync = True else: @@ -247,28 +247,16 @@ class KOBO(USBMS): # Kobo books do not have book files. They do have some images though #print "kobo book" ContentType = 6 - ContentID = os.path.splitext(path)[0] - # Remove the prefix on the file. it could be either - ContentID = ContentID.replace(self._main_prefix, '') - if self._card_a_prefix is not None: - ContentID = ContentID.replace(self._card_a_prefix, '') - ContentID = ContentID.replace("\\", '/') - ImageID = self.delete_via_sql(ContentID, ContentType) - #print " We would now delete the Images for" + ImageID - self.delete_images(ImageID) + ContentID = self.contentid_from_path(path, ContentType) if extension == '.pdf' or extension == '.epub': # print "ePub or pdf" ContentType = 16 #print "Path: " + path - ContentID = path - ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") - if self._card_a_prefix is not None: - ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") - ContentID = ContentID.replace("\\", '/') + ContentID = self.contentid_from_path(path, ContentType) # print "ContentID: " + ContentID - ImageID = self.delete_via_sql(ContentID, ContentType) - #print " We would now delete the Images for" + ImageID - self.delete_images(ImageID) + ImageID = self.delete_via_sql(ContentID, ContentType) + #print " We would now delete the Images for" + ImageID + self.delete_images(ImageID) if os.path.exists(path): # Delete the ebook @@ -342,3 +330,17 @@ class KOBO(USBMS): booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) + def contentid_from_path(self, path, ContentType): + if ContentType == 6: + ContentID = os.path.splitext(path)[0] + # Remove the prefix on the file. it could be either + ContentID = ContentID.replace(self._main_prefix, '') + if self._card_a_prefix is not None: + ContentID = ContentID.replace(self._card_a_prefix, '') + else: # ContentType = 16 + ContentID = path + ContentID = ContentID.replace(self._main_prefix, "file:///mnt/onboard/") + if self._card_a_prefix is not None: + ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") + ContentID = ContentID.replace("\\", '/') + return ContentType From 5c093f846eab08fb84c5a15e29e2db2ce5c12aa4 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Thu, 17 Jun 2010 22:01:08 -0300 Subject: [PATCH 06/30] Remove debugging --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index dce72d0fba..bb1e01283c 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -89,7 +89,7 @@ class KOBO(USBMS): idx = bl_cache.get(filename, None) if idx is not None: imagename = self.normalize_path(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') - print "Image name Normalized: " + imagename + #print "Image name Normalized: " + imagename bl[idx].thumbnail = ImageWrapper(imagename) bl_cache[filename] = None if ContentType != '6': From ff3433cde9767f9d2dd3aee4a8fab43533105e9f Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Thu, 17 Jun 2010 22:58:58 -0300 Subject: [PATCH 07/30] Simplify ContentID to path conversion --- src/calibre/devices/kobo/driver.py | 49 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index bb1e01283c..1b2630c06f 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,6 +22,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' + version = (1, 0, 1) supported_platforms = ['windows', 'osx', 'linux'] @@ -121,40 +122,22 @@ class KOBO(USBMS): cursor.execute (query) changed = False - for i, row in enumerate(cursor): # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) - filename = row[3] - if row[5] == "6": - filename = filename + '.kobo' + path = self.path_from_contentid(row[3], row[5], oncard) mime = mime_type_ext(path_to_ext(row[3])) if oncard != 'carda' and oncard != 'cardb': - if row[5] == '6': - # print "shortbook: " + filename - changed = update_booklist(self._main_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) - if changed: - need_sync = True - else: - if filename.startswith("file:///mnt/onboard/"): - filename = filename.replace("file:///mnt/onboard/", self._main_prefix) - # print "Internal: " + filename - changed = update_booklist(self._main_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) - if changed: - need_sync = True + # print "shortbook: " + filename + changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) elif oncard == 'carda': - if filename.startswith("file:///mnt/sd/"): - filename = filename.replace("file:///mnt/sd/", self._card_a_prefix) - # print "SD Card: " + filename - changed = update_booklist(self._card_a_prefix, filename, row[0], row[1], mime, row[2], row[5], row[6]) - if changed: - need_sync = True + changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) else: print "Add card b support" - #FIXME - NOT NEEDED flist.append({'filename': filename, 'path':row[3]}) - #bl.append(book) + if changed: + need_sync = True cursor.close() connection.close() @@ -344,3 +327,21 @@ class KOBO(USBMS): ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") ContentID = ContentID.replace("\\", '/') return ContentType + + + def path_from_contentid(self, ContentID, ContentType, oncard): + path = ContentID + + if oncard != 'carda' and oncard != 'cardb': + if ContentType == "6": + path = path + '.kobo' + else: + if path.startswith("file:///mnt/onboard/"): + path = path.replace("file:///mnt/onboard/", self._main_prefix) + # print "Internal: " + filename + elif oncard == 'carda': + if path.startswith("file:///mnt/sd/"): + path = path.replace("file:///mnt/sd/", self._card_a_prefix) + # print "SD Card: " + filename + + return path From da83447900d61d10f14d2f13f6286fc220309745 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Fri, 18 Jun 2010 22:37:16 -0300 Subject: [PATCH 08/30] Additional clean up and conver to using path and lpath correctly - more review of lpath and path needed --- src/calibre/devices/kobo/books.py | 15 ++++++++---- src/calibre/devices/kobo/driver.py | 38 +++++++++++++++++------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index 2ed95a4116..f2ec8e2bf4 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -24,32 +24,37 @@ class Book(MetaInformation): 'uuid', ] - def __init__(self, mountpath, path, title, authors, mime, date, ContentType, thumbnail_name, other=None): + def __init__(self, prefix, lpath, title, authors, mime, date, ContentType, thumbnail_name, other=None): MetaInformation.__init__(self, '') self.device_collections = [] + self.path = os.path.join(prefix, lpath) + if os.sep == '\\': + self.path = self.path.replace('/', '\\') + self.lpath = lpath.replace('\\', '/') + else: + self.lpath = lpath + self.title = title if not authors: self.authors = [''] else: self.authors = [authors] self.mime = mime - self.path = path try: - self.size = os.path.getsize(path) + self.size = os.path.getsize(self.path) except OSError: self.size = 0 try: if ContentType == '6': self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") else: - self.datetime = time.gmtime(os.path.getctime(path)) + self.datetime = time.gmtime(os.path.getctime(self.path)) except ValueError: self.datetime = time.gmtime() except OSError: self.datetime = time.gmtime() - self.lpath = path self.thumbnail = ImageWrapper(thumbnail_name) self.tags = [] diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 1b2630c06f..2a3ebfaea5 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,7 +22,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' - version = (1, 0, 1) + version = (1, 0, 2) supported_platforms = ['windows', 'osx', 'linux'] @@ -75,34 +75,35 @@ class KOBO(USBMS): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx - def update_booklist(mountpath, filename, title, authors, mime, date, ContentType, ImageID): + def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID): changed = False - # if path_to_ext(filename) in self.FORMATS: + # if path_to_ext(path) in self.FORMATS: try: - # lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] - # if lpath.startswith(os.sep): - # lpath = lpath[len(os.sep):] - # lpath = lpath.replace('\\', '/') - # print "Filename: " + filename - filename = self.normalize_path(filename) - # print "Normalized FileName: " + filename + lpath = path.partition(self.normalize_path(prefix))[2] + if lpath.startswith(os.sep): + lpath = lpath[len(os.sep):] + lpath = lpath.replace('\\', '/') +# print "LPATH: " + lpath - idx = bl_cache.get(filename, None) + path = self.normalize_path(path) + # print "Normalized FileName: " + path + + idx = bl_cache.get(lpath, None) if idx is not None: - imagename = self.normalize_path(mountpath + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + imagename = self.normalize_path(prefix + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') #print "Image name Normalized: " + imagename bl[idx].thumbnail = ImageWrapper(imagename) - bl_cache[filename] = None + bl_cache[lpath] = None if ContentType != '6': if self.update_metadata_item(bl[idx]): # print 'update_metadata_item returned true' changed = True else: - book = Book(mountpath, filename, title, authors, mime, date, ContentType, ImageID) + book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID) # print 'Update booklist' if bl.add_book(book, replace_metadata=False): changed = True - except: # Probably a filename encoding error + except: # Probably a path encoding error import traceback traceback.print_exc() return changed @@ -129,7 +130,7 @@ class KOBO(USBMS): mime = mime_type_ext(path_to_ext(row[3])) if oncard != 'carda' and oncard != 'cardb': - # print "shortbook: " + filename + # print "shortbook: " + path changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) elif oncard == 'carda': changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) @@ -334,7 +335,10 @@ class KOBO(USBMS): if oncard != 'carda' and oncard != 'cardb': if ContentType == "6": - path = path + '.kobo' + # This is a hack as the kobo files do not exist + # but the path is required to make a unique id + # for calibre's reference + path = self._main_prefix + os.sep + path + '.kobo' else: if path.startswith("file:///mnt/onboard/"): path = path.replace("file:///mnt/onboard/", self._main_prefix) From b722530e455454653190983bc75cee5893cac91c Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 20 Jun 2010 15:59:37 -0300 Subject: [PATCH 09/30] Use color pic where available - kobo --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 2a3ebfaea5..cd95247d13 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -90,7 +90,7 @@ class KOBO(USBMS): idx = bl_cache.get(lpath, None) if idx is not None: - imagename = self.normalize_path(prefix + '.kobo/images/' + ImageID + ' - iPhoneThumbnail.parsed') + imagename = self.normalize_path(prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed') #print "Image name Normalized: " + imagename bl[idx].thumbnail = ImageWrapper(imagename) bl_cache[lpath] = None From a36216fdd1be65fd621d6344c8cace1084f0f929 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 20 Jun 2010 20:24:39 -0300 Subject: [PATCH 10/30] Fix contentid_from_path --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index cd95247d13..4fcdb95c37 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -327,7 +327,7 @@ class KOBO(USBMS): if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, "file:///mnt/sd/") ContentID = ContentID.replace("\\", '/') - return ContentType + return ContentID def path_from_contentid(self, ContentID, ContentType, oncard): From 7044e5811681064b7453106ce09747af1c5aa0e8 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 20 Jun 2010 23:25:44 -0300 Subject: [PATCH 11/30] Remove some extra prints used for debugging --- src/calibre/devices/kobo/driver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 4fcdb95c37..159e8f484c 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -151,8 +151,8 @@ class KOBO(USBMS): need_sync = True del bl[idx] - print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ - (len(bl_cache), len(bl), need_sync) + #print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \ + # (len(bl_cache), len(bl), need_sync) if need_sync: #self.count_found_in_bl != len(bl) or need_sync: if oncard == 'cardb': self.sync_booklists((None, None, bl)) @@ -300,12 +300,12 @@ class KOBO(USBMS): prints('in add_books_to_metadata. Prefix is None!', path, self._main_prefix) continue - print "Add book to metatdata: " - print "prefix: " + prefix + #print "Add book to metatdata: " + #print "prefix: " + prefix lpath = path.partition(prefix)[2] if lpath.startswith('/') or lpath.startswith('\\'): lpath = lpath[1:] - print "path: " + lpath + #print "path: " + lpath #book = self.book_class(prefix, lpath, other=info) lpath = self.normalize_path(prefix + lpath) book = Book(prefix, lpath, '', '', '', '', '', '', other=info) From 4e2bae4e01f3fe3a5027c5d1f6aaf67a27624ab4 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 20 Jun 2010 23:27:31 -0300 Subject: [PATCH 12/30] Increment the version - Sucessful testing on Linux Development Machine --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 159e8f484c..d73ed97bf9 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,7 +22,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' - version = (1, 0, 2) + version = (1, 0, 3) supported_platforms = ['windows', 'osx', 'linux'] From 1824dfe2b9cb9fa70a1550d925daeddd0b814d84 Mon Sep 17 00:00:00 2001 From: GRiker Date: Wed, 23 Jun 2010 02:09:36 -0600 Subject: [PATCH 13/30] GwR wip --- src/calibre/web/feeds/templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index 26d4cbdc9d..81eda8abef 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -231,7 +231,6 @@ class TouchscreenIndexTemplate(Template): div = DIV( masthead_p, PT(date, style='text-align:center'), - #DIV(style="border-color:gray;border-top-style:solid;border-width:thin"), DIV(style="border-top:1px solid gray;border-bottom:1em solid white"), toc) self.root = HTML(head, BODY(div)) From db406fab2dfa0f13f41235e999d9f676f6918a26 Mon Sep 17 00:00:00 2001 From: GRiker Date: Wed, 23 Jun 2010 07:41:25 -0600 Subject: [PATCH 14/30] Gwr revisions --- src/calibre/customize/profiles.py | 10 ++++++++- src/calibre/devices/apple/driver.py | 34 ++++++++++++++++------------- src/calibre/web/feeds/templates.py | 6 ++--- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index a16520410f..d352ce5de7 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -292,7 +292,7 @@ class iPadOutput(OutputProfile): .touchscreen_navbar td { background:#fff; font-family:Helvetica; - font-size:80%; + font-size:90%; padding: 5px; text-align:center; } @@ -309,6 +309,14 @@ class iPadOutput(OutputProfile): font-style: italic; } + /* Index formatting */ + .publish_date { + text-align:center; + } + .divider { + border-bottom:1em solid white; + border-top:1px solid gray; + } /* Feed summary formatting */ .feed_title { diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 77f33ccf3d..40ffe20b4c 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -18,6 +18,7 @@ from calibre.ebooks.metadata.epub import set_metadata from calibre.library.server.utils import strftime from calibre.utils.config import config_dir from calibre.utils.date import isoformat, now, parse_date +from calibre.utils.localization import get_lang from calibre.utils.logging import Log from calibre.utils.zipfile import ZipFile @@ -93,10 +94,13 @@ class ITUNES(DriverBase): # Product IDs: - # 0x1292:iPhone 3G - # 0x129a:iPad + # 0x1291 iPod Touch + # 0x1292 iPhone 3G + # 0x???? iPhone 3GS + # 0x129a iPad + # 0x???? iPhone 4 VENDOR_ID = [0x05ac] - PRODUCT_ID = [0x129a] + PRODUCT_ID = [0x1291,0x1292,0x129a] BCD = [0x01] # iTunes enumerations @@ -528,7 +532,7 @@ class ITUNES(DriverBase): cw.opt_save_template.setVisible(False) cw.label.setVisible(False) # Repurpose the checkbox - cw.opt_read_metadata.setText(_("Use Series as Genre in iTunes/iBooks")) + cw.opt_read_metadata.setText(_("Use Series as Category in iTunes/iBooks")) return cw def delete_books(self, paths, end_session=True): @@ -837,6 +841,7 @@ class ITUNES(DriverBase): self.log.info("ITUNES.upload_books()") self._dump_files(files, header='upload_books()',indent=2) self._dump_update_list(header='upload_books()',indent=2) + #self.log.info(" self.settings().format_map: %s" % self.settings().format_map) if isosx: for (i,file) in enumerate(files): @@ -1201,13 +1206,13 @@ class ITUNES(DriverBase): try: this_book.datetime = parse_date(str(lb_added.date_added())).timetuple() except: - pass + this_book.datetime = time.gmtime() elif db_added: this_book.size = self._get_device_book_size(fpath, db_added.size()) try: this_book.datetime = parse_date(str(db_added.date_added())).timetuple() except: - pass + this_book.datetime = time.gmtime() elif iswindows: if lb_added: @@ -1215,13 +1220,13 @@ class ITUNES(DriverBase): try: this_book.datetime = parse_date(str(lb_added.DateAdded)).timetuple() except: - pass + this_book.datetime = time.gmtime() elif db_added: this_book.size = self._get_device_book_size(fpath, db_added.Size) try: this_book.datetime = parse_date(str(db_added.DateAdded)).timetuple() except: - pass + this_book.datetime = time.gmtime() return this_book @@ -2062,6 +2067,7 @@ class ITUNES(DriverBase): (self.iTunes.name(), self.iTunes.version(), self.initial_status, self.version[0],self.version[1],self.version[2])) self.log.info(" iTunes_media: %s" % self.iTunes_media) + if iswindows: ''' Launch iTunes if not already running @@ -2305,12 +2311,11 @@ class ITUNES(DriverBase): self.log.info(" add timestamp: %s" % metadata.timestamp) # Fix the language declaration for iBooks 1.1 - patched_language = 'en-US' + patched_language = get_lang() language = md.find('dc:language') - if language: - self.log.info(" changing from '%s' to '%s'" % - (language.renderContents(),patched_language)) metadata.language = patched_language + if DEBUG: + self.log.info(" updating %s from localization settings" % patched_language) zf_opf.close() @@ -2447,7 +2452,8 @@ class ITUNES(DriverBase): elif metadata.tags: if DEBUG: - self.log.info(" using Tag as Genre") + self.log.info(" %susing Tag as Genre" % + "no Series name available, " if self.settings().read_metadata else '') for tag in metadata.tags: if self._is_alpha(tag[0]): if lb_added: @@ -2589,8 +2595,6 @@ class BookList(list): class Book(MetaInformation): ''' A simple class describing a book in the iTunes Books Library. - Q's: - - Should thumbnail come from calibre if available? - See ebooks.metadata.__init__ for all fields ''' def __init__(self,title,author): diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index 81eda8abef..df414e2e52 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -8,7 +8,7 @@ import copy from lxml import html, etree from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \ - STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, IMG, P as PT, \ + STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, H3, IMG, P as PT, \ TABLE, TD, TR from calibre import preferred_encoding, strftime, isbytestring @@ -230,8 +230,8 @@ class TouchscreenIndexTemplate(Template): toc.append(tr) div = DIV( masthead_p, - PT(date, style='text-align:center'), - DIV(style="border-top:1px solid gray;border-bottom:1em solid white"), + H3(CLASS('publish_date'),date), + DIV(CLASS('divider')), toc) self.root = HTML(head, BODY(div)) From bd0475fd46ef6b36a6726bc15dc8f18435ac3da1 Mon Sep 17 00:00:00 2001 From: GRiker Date: Thu, 24 Jun 2010 15:57:46 -0600 Subject: [PATCH 15/30] GwR revisions --- src/calibre/devices/apple/driver.py | 67 ++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 40ffe20b4c..f58e4ec27a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -96,11 +96,13 @@ class ITUNES(DriverBase): # Product IDs: # 0x1291 iPod Touch # 0x1292 iPhone 3G - # 0x???? iPhone 3GS + # 0x1293 iPod Touch 2G + # 0x1294 iPhone 3GS + # 0x1297 iPhone 4 + # 0x1299 iPod Touch 3G # 0x129a iPad - # 0x???? iPhone 4 VENDOR_ID = [0x05ac] - PRODUCT_ID = [0x1291,0x1292,0x129a] + PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a] BCD = [0x01] # iTunes enumerations @@ -1905,10 +1907,14 @@ class ITUNES(DriverBase): # Collect calibre orphans - remnants of recipe uploads path = self.path_template % (book.name(), book.artist()) if str(book.description()).startswith(self.description_prefix): - if book.location() == appscript.k.missing_value: - library_orphans[path] = book - if False: - self.log.info(" found iTunes PTF '%s' in Library|Books" % book.name()) + try: + if book.location() == appscript.k.missing_value: + library_orphans[path] = book + if False: + self.log.info(" found iTunes PTF '%s' in Library|Books" % book.name()) + except: + if DEBUG: + self.log.error(" iTunes returned an error returning .location() with %s" % book.name()) library_books[path] = book if DEBUG: @@ -2072,6 +2078,17 @@ class ITUNES(DriverBase): ''' Launch iTunes if not already running Assumes pythoncom wrapper + + *** Current implementation doesn't handle UNC paths correctly, + and python has two incompatible methods to parse UNCs: + os.path.splitdrive() and os.path.splitunc() + need to use os.path.normpath on result of splitunc() + + Once you have the //server/share, convert with os.path.normpath('//server/share') + os.path.splitdrive doesn't work as advertised, so use os.path.splitunc + os.path.splitunc("//server/share") returns ('//server/share','') + os.path.splitunc("C:/Documents") returns ('c:','/documents') + os.path.normpath("//server/share") returns "\\\\server\\share" ''' # Instantiate iTunes self.iTunes = win32com.client.Dispatch("iTunes.Application") @@ -2080,6 +2097,8 @@ class ITUNES(DriverBase): self.initial_status = 'launched' # Read the current storage path for iTunes media from the XML file + media_dir = '' + string = None with open(self.iTunes.LibraryXMLPath, 'r') as xml: for line in xml: if line.strip().startswith('Music Folder'): @@ -2089,10 +2108,12 @@ class ITUNES(DriverBase): break if os.path.exists(media_dir): self.iTunes_media = media_dir - else: + elif hasattr(string,'parent'): self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath) self.log.error(" %s" % string.parent.prettify()) self.log.error(" '%s' not found" % media_dir) + else: + self.log.error(" no media dir found: string: %s" % string) if DEBUG: self.log.info(" %s %s" % (__appname__, __version__)) @@ -2254,8 +2275,8 @@ class ITUNES(DriverBase): path = book.Location if book: - storage_path = os.path.split(path) - if path.startswith(self.iTunes_media): + if self.iTunes_media and path.startswith(self.iTunes_media): + storage_path = os.path.split(path) if DEBUG: self.log.info(" removing '%s' at %s" % (cached_book['title'], path)) @@ -2299,23 +2320,27 @@ class ITUNES(DriverBase): # Touch existing calibre timestamp md = soup.find('metadata') - ts = md.find('meta',attrs={'name':'calibre:timestamp'}) - if ts: - timestamp = ts['content'] - old_ts = parse_date(timestamp) - metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, - old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) + if md: + ts = md.find('meta',attrs={'name':'calibre:timestamp'}) + if ts: + timestamp = ts['content'] + old_ts = parse_date(timestamp) + metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, + old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) + else: + metadata.timestamp = isoformat(now()) + if DEBUG: + self.log.info(" add timestamp: %s" % metadata.timestamp) else: metadata.timestamp = isoformat(now()) if DEBUG: + self.log.warning(" missing block in OPF file") self.log.info(" add timestamp: %s" % metadata.timestamp) - # Fix the language declaration for iBooks 1.1 - patched_language = get_lang() - language = md.find('dc:language') - metadata.language = patched_language + # Force the language declaration for iBooks 1.1 + metadata.language = get_lang() if DEBUG: - self.log.info(" updating %s from localization settings" % patched_language) + self.log.info(" rewriting language: %s" % metadata.language) zf_opf.close() From 908d0fd6ce87c076ef5f2389a222a84aa3ca664e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 16:54:53 -0600 Subject: [PATCH 16/30] Fix LibraryThing metadata download plugin --- src/calibre/ebooks/metadata/fetch.py | 22 ++--- src/calibre/ebooks/metadata/library_thing.py | 93 +++++++++++++++----- 2 files changed, 78 insertions(+), 37 deletions(-) diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 0fd671f86a..66e994f24f 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -210,31 +210,19 @@ class LibraryThing(MetadataSource): # {{{ name = 'LibraryThing' metadata_type = 'social' - description = _('Downloads series information from librarything.com') + description = _('Downloads series/tags/rating information from librarything.com') def fetch(self): if not self.isbn: return - from calibre.ebooks.metadata import MetaInformation - import json - br = browser() + from calibre.ebooks.metadata.library_thing import get_social_metadata try: - raw = br.open( - 'http://status.calibre-ebook.com/library_thing/metadata/'+self.isbn - ).read() - data = json.loads(raw) - if not data: - return - if 'error' in data: - raise Exception(data['error']) - if 'series' in data and 'series_index' in data: - mi = MetaInformation(self.title, []) - mi.series = data['series'] - mi.series_index = data['series_index'] - self.results = mi + self.results = get_social_metadata(self.title, self.book_author, + self.publisher, self.isbn) except Exception, e: self.exception = e self.tb = traceback.format_exc() + # }}} diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index d10d80bc61..eaff881ede 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -6,10 +6,11 @@ Fetch cover from LibraryThing.com based on ISBN number. import sys, socket, os, re -from calibre import browser as _browser +from lxml import html + +from calibre import browser, prints from calibre.utils.config import OptionParser from calibre.ebooks.BeautifulSoup import BeautifulSoup -browser = None OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false' @@ -22,31 +23,28 @@ class ISBNNotFound(LibraryThingError): class ServerBusy(LibraryThingError): pass -def login(username, password, force=True): - global browser - if browser is not None and not force: - return - browser = _browser() - browser.open('http://www.librarything.com') - browser.select_form('signup') - browser['formusername'] = username - browser['formpassword'] = password - browser.submit() +def login(br, username, password, force=True): + br.open('http://www.librarything.com') + br.select_form('signup') + br['formusername'] = username + br['formpassword'] = password + br.submit() def cover_from_isbn(isbn, timeout=5., username=None, password=None): - global browser - if browser is None: - browser = _browser() src = None + br = browser() try: - return browser.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg' + return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg' except: pass # Cover not found if username and password: - login(username, password, force=False) + try: + login(br, username, password, force=False) + except: + pass try: - src = browser.open('http://www.librarything.com/isbn/'+isbn, + src = br.open_novisit('http://www.librarything.com/isbn/'+isbn, timeout=timeout).read().decode('utf-8', 'replace') except Exception, err: if isinstance(getattr(err, 'args', [None])[0], socket.timeout): @@ -63,7 +61,7 @@ def cover_from_isbn(isbn, timeout=5., username=None, password=None): if url is None: raise LibraryThingError(_('LibraryThing.com server error. Try again later.')) url = re.sub(r'_S[XY]\d+', '', url['src']) - cover_data = browser.open(url).read() + cover_data = br.open_novisit(url).read() return cover_data, url.rpartition('.')[-1] def option_parser(): @@ -71,7 +69,7 @@ def option_parser(): _(''' %prog [options] ISBN -Fetch a cover image for the book identified by ISBN from LibraryThing.com +Fetch a cover image/social metadata for the book identified by ISBN from LibraryThing.com ''')) parser.add_option('-u', '--username', default=None, help='Username for LibraryThing.com') @@ -79,6 +77,59 @@ Fetch a cover image for the book identified by ISBN from LibraryThing.com help='Password for LibraryThing.com') return parser +def get_social_metadata(title, authors, publisher, isbn, username=None, + password=None): + from calibre.ebooks.metadata import MetaInformation + mi = MetaInformation(title, authors) + if isbn: + br = browser() + if username and password: + try: + login(br, username, password, force=False) + except: + pass + + raw = br.open_novisit('http://www.librarything.com/isbn/' + +isbn).read() + root = html.fromstring(raw) + h1 = root.xpath('//div[@class="headsummary"]/h1') + if h1 and not mi.title: + mi.title = html.tostring(h1[0], method='text', encoding=unicode) + h2 = root.xpath('//div[@class="headsummary"]/h2/a') + if h2 and not mi.authors: + mi.authors = [html.tostring(x, method='text', encoding=unicode) for + x in h2] + h3 = root.xpath('//div[@class="headsummary"]/h3/a') + if h3: + match = None + for h in h3: + series = html.tostring(h, method='text', encoding=unicode) + match = re.search(r'(.+) \((.+)\)', series) + if match is not None: + break + if match is not None: + mi.series = match.group(1).strip() + match = re.search(r'[0-9.]+', match.group(2)) + si = 1.0 + if match is not None: + si = float(match.group()) + mi.series_index = si + tags = root.xpath('//div[@class="tags"]/span[@class="tag"]/a') + if tags: + mi.tags = [html.tostring(x, method='text', encoding=unicode) for x + in tags] + span = root.xpath( + '//table[@class="wsltable"]/tr[@class="wslcontent"]/td[4]//span') + if span: + raw = html.tostring(span[0], method='text', encoding=unicode) + match = re.search(r'([0-9.]+)', raw) + if match is not None: + rating = float(match.group()) + if rating > 0 and rating <= 5: + mi.rating = rating + return mi + + def main(args=sys.argv): parser = option_parser() opts, args = parser.parse_args(args) @@ -86,6 +137,8 @@ def main(args=sys.argv): parser.print_help() return 1 isbn = args[1] + mi = get_social_metadata('', [], '', isbn) + prints(mi) cover_data, ext = cover_from_isbn(isbn, username=opts.username, password=opts.password) if not ext: From 4b965a8f421336035dcb440be5405803573ada33 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 17:08:52 -0600 Subject: [PATCH 17/30] ... --- src/calibre/ebooks/metadata/library_thing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index eaff881ede..3a78204e8e 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -91,6 +91,8 @@ def get_social_metadata(title, authors, publisher, isbn, username=None, raw = br.open_novisit('http://www.librarything.com/isbn/' +isbn).read() + if not raw: + return mi root = html.fromstring(raw) h1 = root.xpath('//div[@class="headsummary"]/h1') if h1 and not mi.title: From 8bd036d9f44d4818afca95bba2646a43d58af73a Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Thu, 24 Jun 2010 22:48:29 -0300 Subject: [PATCH 18/30] Fix annoying little bug related to using an SD card. Caused by path from content id fix --- src/calibre/devices/kobo/driver.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index d73ed97bf9..007072f1e8 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -129,14 +129,12 @@ class KOBO(USBMS): path = self.path_from_contentid(row[3], row[5], oncard) mime = mime_type_ext(path_to_ext(row[3])) - if oncard != 'carda' and oncard != 'cardb': - # print "shortbook: " + path + if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) - elif oncard == 'carda': + # print "shortbook: " + path + elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"): changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6]) - else: - print "Add card b support" - + if changed: need_sync = True @@ -333,19 +331,21 @@ class KOBO(USBMS): def path_from_contentid(self, ContentID, ContentType, oncard): path = ContentID - if oncard != 'carda' and oncard != 'cardb': + if oncard == 'cardb': + print 'path from_contentid cardb' + elif oncard == 'carda': + path = path.replace("file:///mnt/sd/", self._card_a_prefix) + # print "SD Card: " + filename + else: if ContentType == "6": # This is a hack as the kobo files do not exist # but the path is required to make a unique id # for calibre's reference - path = self._main_prefix + os.sep + path + '.kobo' + path = self._main_prefix + path + '.kobo' + # print "Path: " + path else: - if path.startswith("file:///mnt/onboard/"): - path = path.replace("file:///mnt/onboard/", self._main_prefix) + # if path.startswith("file:///mnt/onboard/"): + path = path.replace("file:///mnt/onboard/", self._main_prefix) # print "Internal: " + filename - elif oncard == 'carda': - if path.startswith("file:///mnt/sd/"): - path = path.replace("file:///mnt/sd/", self._card_a_prefix) - # print "SD Card: " + filename return path From b8ff047ef4fa9ede0c849d04a982c03f1612d826 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Thu, 24 Jun 2010 22:49:37 -0300 Subject: [PATCH 19/30] Increment driver version --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 007072f1e8..4071b10f6e 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,7 +22,7 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' - version = (1, 0, 3) + version = (1, 0, 4) supported_platforms = ['windows', 'osx', 'linux'] From 72b3053c56f510329f43fb46056f6210b40ebe6a Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Thu, 24 Jun 2010 23:31:45 -0300 Subject: [PATCH 20/30] More efficient select query --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 4071b10f6e..2a1e7bda89 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -118,7 +118,7 @@ class KOBO(USBMS): #cursor.close() query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID from content where ContentID in (select distinct volumeId from volume_shortcovers)' + 'ImageID from content where BookID is Null' cursor.execute (query) From 2bdfa64c74267d51f2fad1636f77e07c50591e4e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 21:27:49 -0600 Subject: [PATCH 21/30] Restore workaround for ADE buggy rendering of anchors as links. However, make it overridable by extra CSS --- resources/templates/html.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/resources/templates/html.css b/resources/templates/html.css index 9e80d54f88..e9b683ca34 100644 --- a/resources/templates/html.css +++ b/resources/templates/html.css @@ -406,3 +406,8 @@ img, object, svg|svg { width: auto; height: auto; } + +/* These are needed because ADE renders anchors the same as links */ + +a { text-decoration: inherit; color: inherit; cursor: inherit } +a[href] { text-decoration: underline; color: blue; cursor: pointer } From d54f2b6d148394efe056e7e806a05f90eb4b7863 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 21:34:52 -0600 Subject: [PATCH 22/30] Metadata download: If any results have a published date, ensure they all do --- src/calibre/ebooks/metadata/fetch.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 66e994f24f..36aec75853 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -357,6 +357,16 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None, if title.lower() == r.title[:len(title)].lower() and r.comments and len(r.comments): results[0].comments = r.comments break + # Find a pubdate + pubdate = None + for r in results: + if r.pubdate is not None: + pubdate = r.pubdate + break + if pubdate is not None: + for r in results: + if r.pubdate is None: + r.pubdate = pubdate # for r in results: # print "{0:14.14} {1:30.30} {2:20.20} {3:6} {4}".format(r.isbn, r.title, r.publisher, len(r.comments if r.comments else ''), r.has_cover) From f3a38882bbcf21953b15536fc4b3d670efce1fa3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 21:47:55 -0600 Subject: [PATCH 23/30] Edit meta information dialog: Remember last used size and splitter position. Fixes #5908 (Remember size of Metadata window, and splitter position (patch included)) --- src/calibre/gui2/dialogs/metadata_single.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 84b601776e..817b3a7197 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -11,11 +11,10 @@ import re import time import traceback -import sip from PyQt4.Qt import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread, QDate, \ - QPixmap, QListWidgetItem, QDialog, QHBoxLayout, QGridLayout + QPixmap, QListWidgetItem, QDialog -from calibre.gui2 import error_dialog, file_icon_provider, \ +from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \ choose_files, choose_images, ResizableDialog, \ warning_dialog from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog @@ -301,6 +300,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.connect(self.__abort_button, SIGNAL('clicked()'), self.do_cancel_all) self.splitter.setStretchFactor(100, 1) + self.read_state() self.db = db self.pi = ProgressIndicator(self) self.accepted_callback = accepted_callback @@ -716,7 +716,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): _('Could not open %s. Is it being used by another' ' program?')%fname, show=True) raise - + self.save_state() QDialog.accept(self) if callable(self.accepted_callback): self.accepted_callback(self.id) @@ -728,3 +728,16 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): cf.wait() QDialog.reject(self, *args) + + def read_state(self): + wg = dynamic.get('metasingle_window_geometry', None) + ss = dynamic.get('metasingle_splitter_state', None) + if wg is not None: + self.restoreGeometry(wg) + if ss is not None: + self.splitter.restoreState(ss) + + def save_state(self): + dynamic.set('metasingle_window_geometry', bytes(self.saveGeometry())) + dynamic.set('metasingle_splitter_state', + bytes(self.splitter.saveState())) From 9c12494807f52e50ed0e6af7d22f624f5067aaaa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 21:56:45 -0600 Subject: [PATCH 24/30] Implement #5909 (Feature Request: Option for disabling book cover animation) --- src/calibre/gui2/__init__.py | 2 ++ src/calibre/gui2/book_details.py | 3 ++- src/calibre/gui2/dialogs/config/__init__.py | 2 ++ src/calibre/gui2/dialogs/config/config.ui | 14 ++++++++++++-- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 1056f6ced6..3d50b35ec4 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -103,6 +103,8 @@ def _config(): help=_('The layout of the user interface'), default='wide') c.add_opt('show_avg_rating', default=True, help=_('Show the average rating per item indication in the tag browser')) + c.add_opt('disable_animations', default=False, + help=_('Disable UI animations')) return ConfigProxy(c) config = _config() diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index f87f8886a5..eb8dc0d064 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -16,6 +16,7 @@ from calibre.gui2.widgets import IMAGE_EXTENSIONS from calibre.ebooks import BOOK_EXTENSIONS from calibre.constants import preferred_encoding from calibre.library.comments import comments_to_html +from calibre.gui2 import config # render_rows(data) {{{ WEIGHTS = collections.defaultdict(lambda : 100) @@ -133,7 +134,7 @@ class CoverView(QWidget): # {{{ self.pixmap = self.default_pixmap self.do_layout() self.update() - if not same_item: + if not same_item and not config['disable_animations']: self.animation.start() def paintEvent(self, event): diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index ad49848b7b..f17c0083ec 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -493,6 +493,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): if x == config['gui_layout']: li = i self.opt_gui_layout.setCurrentIndex(li) + self.opt_disable_animations.setChecked(config['disable_animations']) def check_port_value(self, *args): port = self.port.value() @@ -868,6 +869,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): config['get_social_metadata'] = self.opt_get_social_metadata.isChecked() config['overwrite_author_title_metadata'] = self.opt_overwrite_author_title_metadata.isChecked() config['enforce_cpu_limit'] = bool(self.opt_enforce_cpu_limit.isChecked()) + config['disable_animations'] = bool(self.opt_disable_animations.isChecked()) gprefs['show_splash_screen'] = bool(self.show_splash_screen.isChecked()) fmts = [] for i in range(self.viewer.count()): diff --git a/src/calibre/gui2/dialogs/config/config.ui b/src/calibre/gui2/dialogs/config/config.ui index efda00fc97..191b8def80 100644 --- a/src/calibre/gui2/dialogs/config/config.ui +++ b/src/calibre/gui2/dialogs/config/config.ui @@ -89,8 +89,8 @@ 0 0 - 720 - 679 + 724 + 683 @@ -655,6 +655,16 @@ + + + + Disable all animations. Useful if you have a slow/old computer. + + + Disable &animations + + + From 519469740d8c12bea0fac35caf6e71d2020c1e0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 21:58:00 -0600 Subject: [PATCH 25/30] ... --- src/calibre/gui2/custom_column_widgets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 6bb481ddec..ab3354497d 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, sys +import sys from functools import partial from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \ @@ -548,4 +548,4 @@ bulk_widgets = { 'datetime': BulkDateTime, 'text' : BulkText, 'series': BulkSeries, -} \ No newline at end of file +} From 216d74493ed89e9fafb77440d73940fb03f32966 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 22:00:32 -0600 Subject: [PATCH 26/30] Fix #5902 (Device Support - Dell Streak) --- src/calibre/devices/android/driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 49c41b4e57..b65c497038 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -34,6 +34,9 @@ class ANDROID(USBMS): # Acer 0x502 : { 0x3203 : [0x0100]}, + + # Dell + 0x413c : { 0xb007 : [0x0100]}, } EBOOK_DIR_MAIN = ['wordplayer/calibretransfer', 'eBooks/import', 'Books'] EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' @@ -42,7 +45,7 @@ class ANDROID(USBMS): EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN) VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER', - 'GT-I5700', 'SAMSUNG'] + 'GT-I5700', 'SAMSUNG', 'DELL'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'PR OD_GT-I9000'] From 247dd0cb77dc4d480205e3d8bf5d6d4facdca426 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 22:41:46 -0600 Subject: [PATCH 27/30] Fix #5877 (ebook-viewer not rendering ordered lists as ordered) --- src/calibre/ebooks/epub/output.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/epub/output.py index 75739e6a69..4146031cd2 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/epub/output.py @@ -380,10 +380,9 @@ class EPUBOutput(OutputFormatPlugin): sel = '.'+lb.get('class') for rule in stylesheet.data.cssRules.rulesOfType(CSSRule.STYLE_RULE): if sel == rule.selectorList.selectorText: - val = rule.style.removeProperty('margin-left') - pval = rule.style.getProperty('padding-left') - if val and not pval: - rule.style.setProperty('padding-left', val) + rule.style.removeProperty('margin-left') + # padding-left breaks rendering in webkit and gecko + rule.style.removeProperty('padding-left') # }}} From 80548645debbc5cbd8f8a1720fb9ef71b03ff0f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 23:02:30 -0600 Subject: [PATCH 28/30] Fix #5886 (Device driver for Astak Mentor EB600) --- src/calibre/customize/builtins.py | 3 ++- src/calibre/devices/eb600/driver.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 93344f4616..9c13e0062e 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -436,7 +436,7 @@ from calibre.devices.blackberry.driver import BLACKBERRY from calibre.devices.cybook.driver import CYBOOK from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \ POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, \ - BOOQ, ELONEX, POCKETBOOK301 + BOOQ, ELONEX, POCKETBOOK301, MENTOR from calibre.devices.iliad.driver import ILIAD from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 from calibre.devices.jetbook.driver import JETBOOK @@ -550,6 +550,7 @@ plugins += [ AZBOOKA, FOLDER_DEVICE_FOR_CONFIG, AVANT, + MENTOR, ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ x.__name__.endswith('MetadataReader')] diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index 9b7a21a3bb..bf03b1e4c2 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -186,6 +186,15 @@ class BOOQ(EB600): WINDOWS_MAIN_MEM = 'EB600' WINDOWS_CARD_A_MEM = 'EB600' +class MENTOR(EB600): + + name = 'Astak Mentor EB600' + gui_name = 'Mentor' + description = _('Communicate with the Astak Mentor EB600') + FORMATS = ['epub', 'fb2', 'mobi', 'prc', 'pdf', 'txt'] + + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'MENTOR' + class ELONEX(EB600): name = 'Elonex 600EB' From 5f41fde25f76cc9b447e53d813664a57bc580bcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 24 Jun 2010 23:56:56 -0600 Subject: [PATCH 29/30] Fix #5931 (Calibre does not convert whole LRF to EPUB) --- src/calibre/ebooks/lrf/input.py | 2 +- src/calibre/ebooks/oeb/base.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/calibre/ebooks/lrf/input.py b/src/calibre/ebooks/lrf/input.py index f511ba7f09..e9e6c502ec 100644 --- a/src/calibre/ebooks/lrf/input.py +++ b/src/calibre/ebooks/lrf/input.py @@ -367,7 +367,7 @@ class LRFInput(InputFormatPlugin): xml = d.to_xml(write_files=True) if options.verbose > 2: open('lrs.xml', 'wb').write(xml.encode('utf-8')) - parser = etree.XMLParser(recover=True, no_network=True) + parser = etree.XMLParser(recover=True, no_network=True, huge_tree=True) doc = etree.fromstring(xml, parser=parser) char_button_map = {} for x in doc.xpath('//CharButton[@refobj]'): diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 54549ac415..f4a76808ae 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Marshall T. Vandegrift ' __docformat__ = 'restructuredtext en' -import os, re, uuid, logging +import os, re, uuid, logging, functools from mimetypes import types_map from collections import defaultdict from itertools import count @@ -26,6 +26,8 @@ from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.oeb.entitydefs import ENTITYDEFS from calibre.ebooks.conversion.preprocess import CSSPreProcessor +RECOVER_PARSER = etree.XMLParser(recover=True, no_network=True, huge_tree=True) + XML_NS = 'http://www.w3.org/XML/1998/namespace' XHTML_NS = 'http://www.w3.org/1999/xhtml' OEB_DOC_NS = 'http://openebook.org/namespaces/oeb-document/1.0/' @@ -233,8 +235,6 @@ PREFIXNAME_RE = re.compile(r'^[^:]+[:][^:]+') XMLDECL_RE = re.compile(r'^\s*<[?]xml.*?[?]>') CSSURL_RE = re.compile(r'''url[(](?P["']?)(?P[^)]+)(?P=q)[)]''') -RECOVER_PARSER = etree.XMLParser(recover=True) - def element(parent, *args, **kwargs): if parent is not None: @@ -780,8 +780,7 @@ class Manifest(object): assume_utf8=True, resolve_entities=True)[0] if not data: return None - parser = etree.XMLParser(recover=True) - return etree.fromstring(data, parser=parser) + return etree.fromstring(data, parser=RECOVER_PARSER) def _parse_xhtml(self, data): self.oeb.log.debug('Parsing', self.href, '...') @@ -809,16 +808,17 @@ class Manifest(object): pat = re.compile(r'&(%s);'%('|'.join(user_entities.keys()))) data = pat.sub(lambda m:user_entities[m.group(1)], data) + fromstring = functools.partial(etree.fromstring, parser=RECOVER_PARSER) # Try with more & more drastic measures to parse def first_pass(data): try: - data = etree.fromstring(data) + data = fromstring(data) except etree.XMLSyntaxError, err: self.oeb.log.exception('Initial parse failed:') repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0)) data = ENTITY_RE.sub(repl, data) try: - data = etree.fromstring(data) + data = fromstring(data) except etree.XMLSyntaxError, err: self.oeb.logger.warn('Parsing file %r as HTML' % self.href) if err.args and err.args[0].startswith('Excessive depth'): @@ -832,9 +832,9 @@ class Manifest(object): elem.text = elem.text.strip('-') data = etree.tostring(data, encoding=unicode) try: - data = etree.fromstring(data) + data = fromstring(data) except etree.XMLSyntaxError: - data = etree.fromstring(data, parser=RECOVER_PARSER) + data = fromstring(data) return data data = first_pass(data) @@ -866,12 +866,12 @@ class Manifest(object): data = etree.tostring(data, encoding=unicode) try: - data = etree.fromstring(data) + data = fromstring(data) except: data = data.replace(':=', '=').replace(':>', '>') data = data.replace('', '') try: - data = etree.fromstring(data) + data = fromstring(data) except etree.XMLSyntaxError: self.oeb.logger.warn('Stripping comments and meta tags from %s'% self.href) @@ -882,7 +882,7 @@ class Manifest(object): "", '') data = data.replace("", '') - data = etree.fromstring(data) + data = fromstring(data) elif namespace(data.tag) != XHTML_NS: # OEB_DOC_NS, but possibly others ns = namespace(data.tag) From 4e4a69cdc5792e996cbfe1504272c781f644bb0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 25 Jun 2010 02:42:24 -0600 Subject: [PATCH 30/30] Updated zaobao --- resources/recipes/zaobao.recipe | 68 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/resources/recipes/zaobao.recipe b/resources/recipes/zaobao.recipe index bce594bafa..91a5459e18 100644 --- a/resources/recipes/zaobao.recipe +++ b/resources/recipes/zaobao.recipe @@ -15,22 +15,22 @@ class ZAOBAO(BasicNewsRecipe): no_stylesheets = True recursions = 1 language = 'zh' - encoding = 'gbk' # multithreaded_fetch = True keep_only_tags = [ - dict(name='table', attrs={'cellpadding':'9'}), - dict(name='table', attrs={'class':'cont'}), - dict(name='div', attrs={'id':'content'}), + dict(name='td', attrs={'class':'text'}), dict(name='span', attrs={'class':'page'}), + dict(name='div', attrs={'id':'content'}) ] remove_tags = [ dict(name='table', attrs={'cellspacing':'9'}), + dict(name='fieldset'), + dict(name='div', attrs={'width':'30%'}), ] - extra_css = '\ + extra_css = '\n\ @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}\n\ body{font-family: serif1, serif}\n\ .article_description{font-family: serif1, serif}\n\ @@ -41,7 +41,10 @@ class ZAOBAO(BasicNewsRecipe): .article {font-size:medium}\n\ .navbar {font-size: small}\n\ .feed{font-size: medium}\n\ - .small{font-size: small; padding-right: 8%}\n' + .small{font-size: small;padding-right: 8pt}\n\ + .text{padding-right: 8pt}\n\ + p{text-indent: 0cm}\n\ + div#content{padding-right: 10pt}' INDEXES = [ (u'\u65b0\u95fb\u56fe\u7247', u'http://www.zaobao.com/photoweb/photoweb_idx.shtml') @@ -51,27 +54,35 @@ class ZAOBAO(BasicNewsRecipe): DESC_SENSE = u'\u8054\u5408\u65e9\u62a5\u7f51' feeds = [ - (u'\u5373\u65f6\u62a5\u9053', u'http://realtime.zaobao.com/news.xml'), - (u'\u4e2d\u56fd\u65b0\u95fb', u'http://www.zaobao.com/zg/zg.xml'), - (u'\u56fd\u9645\u65b0\u95fb', u'http://www.zaobao.com/gj/gj.xml'), - (u'\u4e16\u754c\u62a5\u520a\u6587\u8403', u'http://www.zaobao.com/wencui/wencui.xml'), - (u'\u4e1c\u5357\u4e9a\u65b0\u95fb', u'http://www.zaobao.com/yx/yx.xml'), - (u'\u65b0\u52a0\u5761\u65b0\u95fb', u'http://www.zaobao.com/sp/sp.xml'), - (u'\u4eca\u65e5\u89c2\u70b9', u'http://www.zaobao.com/yl/yl.xml'), - (u'\u4e2d\u56fd\u8d22\u7ecf', u'http://www.zaobao.com/cz/cz.xml'), - (u'\u72ee\u57ce\u8d22\u7ecf', u'http://www.zaobao.com/cs/cs.xml'), - (u'\u5168\u7403\u8d22\u7ecf', u'http://www.zaobao.com/cg/cg.xml'), - (u'\u65e9\u62a5\u4f53\u80b2', u'http://www.zaobao.com/ty/ty.xml'), - (u'\u65e9\u62a5\u526f\u520a', u'http://www.zaobao.com/fk/fk.xml'), + (u'\u5373\u65f6\u62a5\u9053', u'http://realtime.zaobao.com/news.xml'), + (u'\u4e2d\u56fd\u65b0\u95fb', u'http://www.zaobao.com/zg/zg.xml'), + (u'\u56fd\u9645\u65b0\u95fb', u'http://www.zaobao.com/gj/gj.xml'), + (u'\u4e16\u754c\u62a5\u520a\u6587\u8403', u'http://www.zaobao.com/wencui/wencui.xml'), + (u'\u4e1c\u5357\u4e9a\u65b0\u95fb', u'http://www.zaobao.com/yx/yx.xml'), + (u'\u65b0\u52a0\u5761\u65b0\u95fb', u'http://www.zaobao.com/sp/sp.xml'), + (u'\u4eca\u65e5\u89c2\u70b9', u'http://www.zaobao.com/yl/yl.xml'), + (u'\u4e2d\u56fd\u8d22\u7ecf', u'http://www.zaobao.com/cz/cz.xml'), + (u'\u72ee\u57ce\u8d22\u7ecf', u'http://www.zaobao.com/cs/cs.xml'), + (u'\u5168\u7403\u8d22\u7ecf', u'http://www.zaobao.com/cg/cg.xml'), + (u'\u65e9\u62a5\u4f53\u80b2', u'http://www.zaobao.com/ty/ty.xml'), + (u'\u65e9\u62a5\u526f\u520a', u'http://www.zaobao.com/fk/fk.xml'), ] + def preprocess_html(self, soup): + for tag in soup.findAll(name='a'): + if tag.has_key('href'): + tag_url = tag['href'] + if tag_url.find('http://') != -1 and tag_url.find('zaobao.com') == -1: + del tag['href'] + return soup + def postprocess_html(self, soup, first): for tag in soup.findAll(name=['table', 'tr', 'td']): tag.name = 'div' return soup def parse_feeds(self): - self.log.debug('ZAOBAO overrided parse_feeds()') + self.log_debug(_('ZAOBAO overrided parse_feeds()')) parsed_feeds = BasicNewsRecipe.parse_feeds(self) for id, obj in enumerate(self.INDEXES): @@ -88,7 +99,7 @@ class ZAOBAO(BasicNewsRecipe): a_title = self.tag_to_string(a) date = '' description = '' - self.log.debug('adding %s at %s'%(a_title,a_url)) + self.log_debug(_('adding %s at %s')%(a_title,a_url)) articles.append({ 'title':a_title, 'date':date, @@ -97,26 +108,25 @@ class ZAOBAO(BasicNewsRecipe): }) pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article, - max_articles_per_feed=self.max_articles_per_feed, - log=self.log) + max_articles_per_feed=self.max_articles_per_feed) - self.log.debug('adding %s to feed'%(title)) + self.log_debug(_('adding %s to feed')%(title)) for feed in pfeeds: - self.log.debug('adding feed: %s'%(feed.title)) + self.log_debug(_('adding feed: %s')%(feed.title)) feed.description = self.DESC_SENSE parsed_feeds.append(feed) for a, article in enumerate(feed): - self.log.debug('added article %s from %s'%(article.title, article.url)) - self.log.debug('added feed %s'%(feed.title)) + self.log_debug(_('added article %s from %s')%(article.title, article.url)) + self.log_debug(_('added feed %s')%(feed.title)) for i, feed in enumerate(parsed_feeds): # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse() weired_encoding_detected = False if not isinstance(feed.description, unicode) and self.encoding and feed.description: - self.log.debug('Feed %s is not encoded correctly, manually replace it'%(feed.title)) + self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title)) feed.description = feed.description.decode(self.encoding, 'replace') elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description: - self.log.debug('Feed %s is strangely encoded, manually redo all'%(feed.title)) + self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title)) feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace') weired_encoding_detected = True @@ -138,7 +148,7 @@ class ZAOBAO(BasicNewsRecipe): article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace') if article.title == "Untitled article": - self.log.debug('Removing empty article %s from %s'%(article.title, article.url)) + self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url)) # remove the article feed.articles[a:a+1] = [] return parsed_feeds