diff --git a/resources/recipes/danas.recipe b/resources/recipes/danas.recipe index 3543acd684..1e0e319334 100644 --- a/resources/recipes/danas.recipe +++ b/resources/recipes/danas.recipe @@ -49,7 +49,11 @@ class Danas(BasicNewsRecipe): , 'language' : language } - preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] + preprocess_regexps = [ + (re.compile(u'\u0110'), lambda match: u'\u00D0') + ,(re.compile(u'\u201c'), lambda match: '"') + ,(re.compile(u'\u201e'), lambda match: '"') + ] keep_only_tags = [dict(name='div', attrs={'id':'left'})] remove_tags = [ diff --git a/resources/recipes/novosti.recipe b/resources/recipes/novosti.recipe index 0465b59e17..d66e7d28b7 100644 --- a/resources/recipes/novosti.recipe +++ b/resources/recipes/novosti.recipe @@ -21,8 +21,8 @@ class Novosti(BasicNewsRecipe): encoding = 'utf-8' language = 'sr' publication_type = 'newspaper' - extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} - .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif} + extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} + .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif} .author{font-size: small} .articleLead{font-size: large; font-weight: bold} """ @@ -47,6 +47,8 @@ class Novosti(BasicNewsRecipe): item.name='p' for item in soup.findAll('img'): if not item.has_key('alt'): - item['alt'] = 'image' + item['alt'] = 'image' return soup - \ No newline at end of file + + + diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 7a451112c0..c9c0827759 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -29,7 +29,9 @@ class ANDROID(USBMS): # Sony Ericsson 0xfce : { 0xd12e : [0x0100]}, - 0x18d1 : { 0x4e11 : [0x0100, 0x226], 0x4e12: [0x0100, 0x226]}, + # Google + 0x18d1 : { 0x4e11 : [0x0100, 0x226, 0x227], 0x4e12: [0x0100, 0x226, + 0x227]}, # Samsung 0x04e8 : { 0x681d : [0x0222, 0x0400], diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index 75728a94ea..6291864b86 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -11,6 +11,10 @@ import re from calibre.devices.usbms.driver import USBMS +def is_alex(device_info): + return device_info[3] == u'Linux 2.6.28 with pxa3xx_u2d' and \ + device_info[4] == u'Seleucia Disk' + class N516(USBMS): name = 'N516 driver' @@ -34,6 +38,9 @@ class N516(USBMS): EBOOK_DIR_MAIN = 'e_book' SUPPORTS_SUB_DIRS = True + def can_handle(self, device_info, debug=False): + return not is_alex(device_info) + class THEBOOK(N516): name = 'The Book driver' gui_name = 'The Book' @@ -61,6 +68,9 @@ class ALEX(N516): EBOOK_DIR_MAIN = 'eBooks' SUPPORTS_SUB_DIRS = True + def can_handle(self, device_info, debug=False): + return is_alex(device_info) + class AZBOOKA(ALEX): name = 'Azbooka driver' @@ -74,6 +84,9 @@ class AZBOOKA(ALEX): EBOOK_DIR_MAIN = '' + def can_handle(self, device_info, debug=False): + return not is_alex(device_info) + class EB511(USBMS): name = 'Elonex EB 511 driver' diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 1171b74f5c..104553b675 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -98,6 +98,8 @@ class KOBO(USBMS): if readstatus == 1: playlist_map[lpath]= "Im_Reading" + elif readstatus == 2: + playlist_map[lpath]= "Read" path = self.normalize_path(path) # print "Normalized FileName: " + path @@ -441,43 +443,99 @@ class KOBO(USBMS): connection = sqlite.connect(self._main_prefix + '.kobo/KoboReader.sqlite') cursor = connection.cursor() - # Reset Im_Reading list in the database - if oncard == 'carda': - query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID like \'file:///mnt/sd/%\'' - elif oncard != 'carda' and oncard != 'cardb': - query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID not like \'file:///mnt/sd/%\'' + + if collections: + # Process any collections that exist + for category, books in collections.items(): + if category == 'Im_Reading': + # Reset Im_Reading list in the database + if oncard == 'carda': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID like \'file:///mnt/sd/%\'' + elif oncard != 'carda' and oncard != 'cardb': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 1 and ContentID not like \'file:///mnt/sd/%\'' - try: - cursor.execute (query) - except: - debug_print('Database Exception: Unable to reset Im_Reading list') - raise - else: -# debug_print('Commit: Reset Im_Reading list') - connection.commit() - - for category, books in collections.items(): - if category == 'Im_Reading': - for book in books: -# debug_print('Title:', book.title, 'lpath:', book.path) - book.device_collections = ['Im_Reading'] - - extension = os.path.splitext(book.path)[1] - ContentType = self.get_content_type_from_extension(extension) - - ContentID = self.contentid_from_path(book.path, ContentType) - datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) - - t = (datelastread,ContentID,) - try: - cursor.execute('update content set ReadStatus=1,FirstTimeReading=\'false\',DateLastRead=? where BookID is Null and ContentID = ?', t) + cursor.execute (query) except: - debug_print('Database Exception: Unable create Im_Reading list') + debug_print('Database Exception: Unable to reset Im_Reading list') raise else: +# debug_print('Commit: Reset Im_Reading list') connection.commit() - # debug_print('Database: Commit create Im_Reading list') + + for book in books: +# debug_print('Title:', book.title, 'lpath:', book.path) + book.device_collections = ['Im_Reading'] + + extension = os.path.splitext(book.path)[1] + ContentType = self.get_content_type_from_extension(extension) + + ContentID = self.contentid_from_path(book.path, ContentType) + datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + + t = (datelastread,ContentID,) + + try: + cursor.execute('update content set ReadStatus=1,FirstTimeReading=\'false\',DateLastRead=? where BookID is Null and ContentID = ?', t) + except: + debug_print('Database Exception: Unable create Im_Reading list') + raise + else: + connection.commit() + # debug_print('Database: Commit create Im_Reading list') + if category == 'Read': + # Reset Im_Reading list in the database + if oncard == 'carda': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID like \'file:///mnt/sd/%\'' + elif oncard != 'carda' and oncard != 'cardb': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ReadStatus = 2 and ContentID not like \'file:///mnt/sd/%\'' + + try: + cursor.execute (query) + except: + debug_print('Database Exception: Unable to reset Im_Reading list') + raise + else: +# debug_print('Commit: Reset Im_Reading list') + connection.commit() + + for book in books: +# debug_print('Title:', book.title, 'lpath:', book.path) + book.device_collections = ['Read'] + + extension = os.path.splitext(book.path)[1] + ContentType = self.get_content_type_from_extension(extension) + + ContentID = self.contentid_from_path(book.path, ContentType) +# datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + + t = (ContentID,) + + try: + cursor.execute('update content set ReadStatus=2,FirstTimeReading=\'true\' where BookID is Null and ContentID = ?', t) + except: + debug_print('Database Exception: Unable set book as Rinished') + raise + else: + connection.commit() +# debug_print('Database: Commit set ReadStatus as Finished') + else: # No collections + # Since no collections exist the ReadStatus needs to be reset to 0 (Unread) + print "Reseting ReadStatus to 0" + # Reset Im_Reading list in the database + if oncard == 'carda': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID like \'file:///mnt/sd/%\'' + elif oncard != 'carda' and oncard != 'cardb': + query= 'update content set ReadStatus=0, FirstTimeReading = \'true\' where BookID is Null and ContentID not like \'file:///mnt/sd/%\'' + + try: + cursor.execute (query) + except: + debug_print('Database Exception: Unable to reset Im_Reading list') + raise + else: +# debug_print('Commit: Reset Im_Reading list') + connection.commit() cursor.close() connection.close() diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 5d5de7b153..23f92d1fd2 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -1574,14 +1574,15 @@ class MobiWriter(object): id = unicode(oeb.metadata.cover[0]) item = oeb.manifest.ids[id] href = item.href - index = self._images[href] - 1 - exth.write(pack('>III', 0xc9, 0x0c, index)) - exth.write(pack('>III', 0xcb, 0x0c, 0)) - nrecs += 2 - index = self._add_thumbnail(item) - if index is not None: - exth.write(pack('>III', 0xca, 0x0c, index - 1)) - nrecs += 1 + if href in self._images: + index = self._images[href] - 1 + exth.write(pack('>III', 0xc9, 0x0c, index)) + exth.write(pack('>III', 0xcb, 0x0c, 0)) + nrecs += 2 + index = self._add_thumbnail(item) + if index is not None: + exth.write(pack('>III', 0xca, 0x0c, index - 1)) + nrecs += 1 exth = exth.getvalue() trail = len(exth) % 4 diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index c284900734..66e199b8a0 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -1,7 +1,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' """ The GUI """ -import os, sys +import os, sys, Queue from threading import RLock from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ @@ -296,6 +296,34 @@ class Dispatcher(QObject): def dispatch(self, args, kwargs): self.func(*args, **kwargs) +class FunctionDispatcher(QObject): + ''' + Convenience class to use Qt signals with arbitrary python functions. + By default, ensures that a function call always happens in the + thread this Dispatcher was created in. + ''' + dispatch_signal = pyqtSignal(object, object, object) + + def __init__(self, func, queued=True, parent=None): + QObject.__init__(self, parent) + self.func = func + typ = Qt.QueuedConnection + if not queued: + typ = Qt.AutoConnection if queued is None else Qt.DirectConnection + self.dispatch_signal.connect(self.dispatch, type=typ) + + def __call__(self, *args, **kwargs): + q = Queue.Queue() + self.dispatch_signal.emit(q, args, kwargs) + return q.get() + + def dispatch(self, q, args, kwargs): + try: + res = self.func(*args, **kwargs) + except: + res = None + q.put(res) + class GetMetadata(QObject): ''' Convenience class to ensure that metadata readers are used only in the diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 3370fd4b75..53f701386b 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -12,7 +12,7 @@ from operator import attrgetter from PyQt4.Qt import QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, \ QModelIndex, QVariant, QDate -from calibre.gui2 import NONE, config, UNDEFINED_QDATE +from calibre.gui2 import NONE, config, UNDEFINED_QDATE, FunctionDispatcher from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors from calibre.ptempfile import PersistentTemporaryFile @@ -151,7 +151,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.database_changed.emit(db) if self.cover_cache is not None: self.cover_cache.stop() - self.cover_cache = CoverCache(db) + self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover)) self.cover_cache.start() def refresh_cover(event, ids): if event == 'cover' and self.cover_cache is not None: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 211baeb634..58edd89cb2 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, itertools +import re, itertools, time from itertools import repeat from datetime import timedelta from threading import Thread, RLock @@ -23,10 +23,11 @@ from calibre import fit_image class CoverCache(Thread): - def __init__(self, db): + def __init__(self, db, cover_func): Thread.__init__(self) self.daemon = True self.db = db + self.cover_func = cover_func self.load_queue = Queue() self.keep_running = True self.cache = {} @@ -37,7 +38,8 @@ class CoverCache(Thread): self.keep_running = False def _image_for_id(self, id_): - img = self.db.cover(id_, index_is_id=True, as_image=True) + time.sleep(0.050) # Limit 20/second to not overwhelm the GUI + img = self.cover_func(id_, index_is_id=True, as_image=True) if img is None: img = QImage() if not img.isNull(): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index eb6e8336f9..1a2eef2c81 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -402,7 +402,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path = path.lower() return path - def set_path(self, index, index_is_id=False, commit=True): + def set_path(self, index, index_is_id=False): ''' Set the path to the directory containing this books files based on its current title and author. If there was a previous directory, its contents @@ -432,7 +432,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if current_path and os.path.exists(spath): # Migrate existing files cdata = self.cover(id, index_is_id=True) if cdata is not None: - open(os.path.join(tpath, 'cover.jpg'), 'wb').write(cdata) + with open(os.path.join(tpath, 'cover.jpg'), 'wb') as f: + f.write(cdata) for format in formats: # Get data as string (can't use file as source and target files may be the same) f = self.format(id, format, index_is_id=True, as_file=False) @@ -442,8 +443,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.add_format(id, format, stream, index_is_id=True, path=tpath, notify=False) self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id)) - if commit: - self.conn.commit() self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True) # Delete not needed directories if current_path and os.path.exists(spath): @@ -1131,7 +1130,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_authors(self, id, authors, notify=True, commit=True): ''' - `authors`: A list of authors. + Note that even if commit is False, the db will still be committed to + because this causes the location of files to change + + :param authors: A list of authors. ''' if not authors: authors = [_('Unknown')] @@ -1163,11 +1165,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ','.join([a.replace(',', '|') for a in authors]), row_is_id=True) self.data.set(id, self.FIELD_MAP['author_sort'], ss, row_is_id=True) - self.set_path(id, index_is_id=True, commit=commit) + self.set_path(id, index_is_id=True) if notify: self.notify('metadata', [id]) def set_title(self, id, title, notify=True, commit=True): + ''' + Note that even if commit is False, the db will still be committed to + because this causes the location of files to change + ''' if not title: return if not isinstance(title, unicode): @@ -1178,7 +1184,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.data.set(id, self.FIELD_MAP['sort'], title_sort(title), row_is_id=True) else: self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True) - self.set_path(id, index_is_id=True, commit=commit) + self.set_path(id, index_is_id=True) if commit: self.conn.commit() if notify: diff --git a/src/calibre/utils/magick/draw.py b/src/calibre/utils/magick/draw.py index ed9e3d3d83..dcf9d7b671 100644 --- a/src/calibre/utils/magick/draw.py +++ b/src/calibre/utils/magick/draw.py @@ -60,15 +60,15 @@ def identify(path): data = open(path, 'rb').read() return identify_data(data) -def add_borders_to_image(path_to_image, left=0, top=0, right=0, bottom=0, - border_color='#ffffff'): +def add_borders_to_image(img_data, left=0, top=0, right=0, bottom=0, + border_color='#ffffff', fmt='jpg'): img = Image() - img.open(path_to_image) + img.load(img_data) lwidth, lheight = img.size canvas = create_canvas(lwidth+left+right, lheight+top+bottom, border_color) canvas.compose(img, left, top) - canvas.save(path_to_image) + return canvas.export(fmt) def create_text_wand(font_size, font_path=None): if font_path is None: diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index a140dfbf05..d1e7866198 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful __docformat__ = "restructuredtext en" -import os, time, traceback, re, urlparse, sys +import os, time, traceback, re, urlparse, sys, cStringIO from collections import defaultdict from functools import partial from contextlib import nested, closing @@ -27,6 +27,7 @@ from calibre.web.fetch.simple import RecursiveFetcher from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.date import now as nowf +from calibre.utils.magick.draw import save_cover_data_to, add_borders_to_image class LoginFailed(ValueError): pass @@ -948,38 +949,36 @@ class BasicNewsRecipe(Recipe): try: cu = self.get_cover_url() except Exception, err: - cu = None self.log.error(_('Could not download cover: %s')%str(err)) self.log.debug(traceback.format_exc()) - if cu is not None: - ext = cu.split('/')[-1].rpartition('.')[-1] - if '?' in ext: - ext = '' - ext = ext.lower() if ext and '/' not in ext else 'jpg' - cpath = os.path.join(self.output_dir, 'cover.'+ext) + else: + cdata = None if os.access(cu, os.R_OK): - with open(cpath, 'wb') as cfile: - cfile.write(open(cu, 'rb').read()) + cdata = open(cu, 'rb').read() else: self.report_progress(1, _('Downloading cover from %s')%cu) - with nested(open(cpath, 'wb'), closing(self.browser.open(cu))) as (cfile, r): - cfile.write(r.read()) - if self.cover_margins[0] or self.cover_margins[1]: - from calibre.utils.magick.draw import add_borders_to_image - add_borders_to_image(cpath, - left=self.cover_margins[0],right=self.cover_margins[0], - top=self.cover_margins[1],bottom=self.cover_margins[1], - border_color=self.cover_margins[2]) - if ext.lower() == 'pdf': + with closing(self.browser.open(cu)) as r: + cdata = r.read() + if not cdata: + return + ext = cu.split('/')[-1].rpartition('.')[-1].lower().strip() + if ext == 'pdf': from calibre.ebooks.metadata.pdf import get_metadata - stream = open(cpath, 'rb') + stream = cStringIO.StringIO(cdata) + cdata = None mi = get_metadata(stream) - cpath = None if mi.cover_data and mi.cover_data[1]: - cpath = os.path.join(self.output_dir, - 'cover.'+mi.cover_data[0]) - with open(cpath, 'wb') as f: - f.write(mi.cover_data[1]) + cdata = mi.cover_data[1] + if not cdata: + return + if self.cover_margins[0] or self.cover_margins[1]: + cdata = add_borders_to_image(cdata, + left=self.cover_margins[0],right=self.cover_margins[0], + top=self.cover_margins[1],bottom=self.cover_margins[1], + border_color=self.cover_margins[2]) + + cpath = os.path.join(self.output_dir, 'cover.jpg') + save_cover_data_to(cdata, cpath) self.cover_path = cpath def download_cover(self): @@ -1422,7 +1421,6 @@ class CalibrePeriodical(BasicNewsRecipe): return br def download(self): - import cStringIO self.log('Fetching downloaded recipe') try: raw = self.browser.open_novisit(