From e6d8302fcb881d0e3cc64354d89e4429dce4bd86 Mon Sep 17 00:00:00 2001 From: GRiker Date: Wed, 30 Jun 2010 08:35:12 -0600 Subject: [PATCH 01/14] GwR revisions --- src/calibre/devices/apple/driver.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 6c9235c0d4..19b6c6710b 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -13,7 +13,7 @@ from calibre.devices.errors import UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin from calibre.ebooks.BeautifulSoup import BeautifulSoup -from calibre.ebooks.metadata import MetaInformation +from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.ebooks.metadata.epub import set_metadata from calibre.library.server.utils import strftime from calibre.utils.config import config_dir @@ -84,7 +84,7 @@ class ITUNES(DriverBase): name = 'Apple device interface' gui_name = 'Apple device' icon = I('devices/ipad.png') - description = _('Communicate with iBooks through iTunes.') + description = _('Communicate with iTunes/iBooks.') supported_platforms = ['osx','windows'] author = 'GRiker' #: The version of this plugin as a 3-tuple (major, minor, revision) @@ -93,7 +93,6 @@ class ITUNES(DriverBase): OPEN_FEEDBACK_MESSAGE = _( 'Apple device detected, launching iTunes, please wait ...') - # Product IDs: # 0x1291 iPod Touch # 0x1292 iPhone 3G @@ -1242,8 +1241,7 @@ class ITUNES(DriverBase): if DEBUG: self.log.info(" ITUNES._create_new_book()") - #this_book = Book(metadata.title, metadata.author[0]) - this_book = Book(metadata.title, ' & '.join(metadata.author)) + this_book = Book(metadata.title, authors_to_string(metadata.author)) this_book.datetime = time.gmtime() this_book.db_id = None this_book.device_collections = [] @@ -2548,8 +2546,7 @@ class ITUNES(DriverBase): if isosx: if lb_added: lb_added.album.set(metadata.title) - #lb_added.artist.set(metadata.authors[0]) - lb_added.artist.set(' & '.join(metadata.authors)) + lb_added.artist.set(authors_to_string(metadata.authors)) lb_added.composer.set(metadata.uuid) lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.enabled.set(True) @@ -2560,8 +2557,7 @@ class ITUNES(DriverBase): if db_added: db_added.album.set(metadata.title) - #db_added.artist.set(metadata.authors[0]) - db_added.artist.set(' & '.join(metadata.authors)) + db_added.artist.set(authors_to_string(metadata.authors)) db_added.composer.set(metadata.uuid) db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.enabled.set(True) @@ -2618,8 +2614,7 @@ class ITUNES(DriverBase): elif iswindows: if lb_added: lb_added.Album = metadata.title - #lb_added.Artist = metadata.authors[0] - lb_added.Artist = ' & '.join(metadata.authors) + lb_added.Artist = authors_to_string(metadata.authors) lb_added.Composer = metadata.uuid lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.Enabled = True @@ -2630,8 +2625,7 @@ class ITUNES(DriverBase): if db_added: db_added.Album = metadata.title - #db_added.Artist = metadata.authors[0] - db_added.Artist = ' & '.join(metadata.authors) + db_added.Artist = authors_to_string(metadata.authors) db_added.Composer = metadata.uuid db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.Enabled = True From 71c8f750e7c3e44357be06bb704385a7c1a6f175 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 13:39:18 -0600 Subject: [PATCH 02/14] Tester for PoDoFo --- src/calibre/utils/podofo/test.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/calibre/utils/podofo/test.cpp diff --git a/src/calibre/utils/podofo/test.cpp b/src/calibre/utils/podofo/test.cpp new file mode 100644 index 0000000000..fb719fd0cc --- /dev/null +++ b/src/calibre/utils/podofo/test.cpp @@ -0,0 +1,26 @@ +#define USING_SHARED_PODOFO +#include +#include + +using namespace PoDoFo; +using namespace std; + + +int main(int argc, char **argv) { + if (argc < 2) return 1; + char *fname = argv[1]; + + PdfMemDocument doc(fname); + PdfInfo* info = doc.GetInfo(); + cout << endl; + cout << "is encrypted: " << doc.GetEncrypted() << endl; + PdfString old_title = info->GetTitle(); + cout << "is hex: " << old_title.IsHex() << endl; + PdfString new_title(reinterpret_cast("\0z\0z\0z"), 3); + cout << "is new unicode: " << new_title.IsUnicode() << endl; + info->SetTitle(new_title); + + doc.Write("/t/x.pdf"); + cout << "Output written to: " << "/t/x.pdf" << endl; + return 0; +} From 8e00c3bc9c20476d3a51018774fb6131e843e0eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 14:25:16 -0600 Subject: [PATCH 03/14] Cleanup --- src/calibre/ebooks/metadata/pdf.py | 62 ++---------------------------- 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/src/calibre/ebooks/metadata/pdf.py b/src/calibre/ebooks/metadata/pdf.py index b4bc6f962f..2d1935539e 100644 --- a/src/calibre/ebooks/metadata/pdf.py +++ b/src/calibre/ebooks/metadata/pdf.py @@ -8,7 +8,7 @@ from functools import partial from calibre import prints from calibre.constants import plugins -from calibre.ebooks.metadata import MetaInformation, string_to_authors, authors_to_string +from calibre.ebooks.metadata import MetaInformation, string_to_authors pdfreflow, pdfreflow_error = plugins['pdfreflow'] @@ -56,66 +56,10 @@ def get_metadata(stream, cover=True): get_quick_metadata = partial(get_metadata, cover=False) -import cStringIO -from threading import Thread - -from calibre.utils.pdftk import set_metadata as pdftk_set_metadata -from calibre.utils.podofo import set_metadata as podofo_set_metadata, Unavailable +from calibre.utils.podofo import set_metadata as podofo_set_metadata def set_metadata(stream, mi): stream.seek(0) - try: - return podofo_set_metadata(stream, mi) - except Unavailable: - pass - try: - return pdftk_set_metadata(stream, mi) - except: - pass - set_metadata_pypdf(stream, mi) - - -class MetadataWriter(Thread): - - def __init__(self, out_pdf, buf): - self.out_pdf = out_pdf - self.buf = buf - Thread.__init__(self) - self.daemon = True - - def run(self): - try: - self.out_pdf.write(self.buf) - except RuntimeError: - pass - -def set_metadata_pypdf(stream, mi): - # Use a StringIO object for the pdf because we will want to over - # write it later and if we are working on the stream directly it - # could cause some issues. - - from pyPdf import PdfFileReader, PdfFileWriter - raw = cStringIO.StringIO(stream.read()) - orig_pdf = PdfFileReader(raw) - title = mi.title if mi.title else orig_pdf.documentInfo.title - author = authors_to_string(mi.authors) if mi.authors else orig_pdf.documentInfo.author - out_pdf = PdfFileWriter(title=title, author=author) - out_str = cStringIO.StringIO() - writer = MetadataWriter(out_pdf, out_str) - for page in orig_pdf.pages: - out_pdf.addPage(page) - writer.start() - writer.join(10) # Wait 10 secs for writing to complete - out_pdf.killed = True - writer.join() - if out_pdf.killed: - print 'Failed to set metadata: took too long' - return - - stream.seek(0) - stream.truncate() - out_str.seek(0) - stream.write(out_str.read()) - stream.seek(0) + return podofo_set_metadata(stream, mi) From e280121bec00e20397f2d111a37cf511c6884d88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 14:39:19 -0600 Subject: [PATCH 04/14] ... --- src/calibre/gui2/add.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 64743e914b..74f5a2148e 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -15,7 +15,7 @@ from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding, filesystem_encoding from calibre.utils.config import prefs -class DuplicatesAdder(QThread): +class DuplicatesAdder(QThread): # {{{ # Add duplicate books def __init__(self, parent, db, duplicates, db_adder): QThread.__init__(self, parent) @@ -34,9 +34,9 @@ class DuplicatesAdder(QThread): self.emit(SIGNAL('added(PyQt_PyObject)'), count) count += 1 self.emit(SIGNAL('adding_done()')) +# }}} - -class RecursiveFind(QThread): +class RecursiveFind(QThread): # {{{ def __init__(self, parent, db, root, single): QThread.__init__(self, parent) @@ -79,7 +79,9 @@ class RecursiveFind(QThread): if not self.canceled: self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) -class DBAdder(Thread): +# }}} + +class DBAdder(Thread): # {{{ def __init__(self, db, ids, nmap): self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap) @@ -219,8 +221,9 @@ class DBAdder(Thread): self.db.add_format(id, fmt, f, index_is_id=True, notify=False, replace=replace) +# }}} -class Adder(QObject): +class Adder(QObject): # {{{ ADD_TIMEOUT = 600 # seconds @@ -410,6 +413,7 @@ class Adder(QObject): return getattr(getattr(self, 'db_adder', None), 'infos', []) +# }}} ############################################################################### ############################## END ADDER ###################################### From c43277eae08ddbc9815792c3f3a8c38172fa6303 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 16:34:08 -0600 Subject: [PATCH 05/14] Simplify implementation of cover caching and ensure cover browser is updated when covers are changed --- src/calibre/gui2/actions.py | 5 +- src/calibre/gui2/library/models.py | 26 ++--- src/calibre/gui2/ui.py | 10 +- src/calibre/gui2/widgets.py | 6 +- src/calibre/library/caches.py | 167 +++++++++++------------------ src/calibre/library/database2.py | 34 ++++-- 6 files changed, 114 insertions(+), 134 deletions(-) diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py index 43a657ae67..5dde2f745b 100644 --- a/src/calibre/gui2/actions.py +++ b/src/calibre/gui2/actions.py @@ -645,6 +645,8 @@ class EditMetadataAction(object): # {{{ if x.exception is None: self.library_view.model().refresh_ids( x.updated, cr) + if self.cover_flow: + self.cover_flow.dataChanged() if x.failures: details = ['%s: %s'%(title, reason) for title, reason in x.failures.values()] @@ -689,7 +691,6 @@ class EditMetadataAction(object): # {{{ if rows: current = self.library_view.currentIndex() m = self.library_view.model() - m.refresh_cover_cache(map(m.id, rows)) if self.cover_flow: self.cover_flow.dataChanged() m.current_changed(current, previous) @@ -711,6 +712,8 @@ class EditMetadataAction(object): # {{{ self.library_view.model().resort(reset=False) self.library_view.model().research() self.tags_view.recount() + if self.cover_flow: + self.cover_flow.dataChanged() # Merge books {{{ def merge_books(self, safe_merge=False): diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 508b3e591c..c487a8a252 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -20,7 +20,8 @@ from calibre.utils.config import tweaks, prefs from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser -from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ + REGEXP_MATCH, CoverCache from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml from calibre.constants import filesystem_encoding @@ -149,21 +150,22 @@ class BooksModel(QAbstractTableModel): # {{{ self.build_data_convertors() self.reset() self.database_changed.emit(db) + if self.cover_cache is not None: + self.cover_cache.stop() + self.cover_cache = CoverCache(db) + self.cover_cache.start() + def refresh_cover(event, ids): + if event == 'cover' and self.cover_cache is not None: + self.cover_cache.refresh(ids) + db.add_listener(refresh_cover) def refresh_ids(self, ids, current_row=-1): rows = self.db.refresh_ids(ids) if rows: self.refresh_rows(rows, current_row=current_row) - def refresh_cover_cache(self, ids): - if self.cover_cache: - self.cover_cache.refresh(ids) - def refresh_rows(self, rows, current_row=-1): for row in rows: - if self.cover_cache: - id = self.db.id(row) - self.cover_cache.refresh([id]) if row == current_row: self.new_bookdisplay_data.emit( self.get_book_display_info(row)) @@ -326,7 +328,7 @@ class BooksModel(QAbstractTableModel): # {{{ def set_cache(self, idx): l, r = 0, self.count()-1 - if self.cover_cache: + if self.cover_cache is not None: l = max(l, idx-self.buffer_size) r = min(r, idx+self.buffer_size) k = min(r-idx, idx-l) @@ -494,11 +496,9 @@ class BooksModel(QAbstractTableModel): # {{{ data = None try: id = self.db.id(row_number) - if self.cover_cache: + if self.cover_cache is not None: img = self.cover_cache.cover(id) - if img: - if img.isNull(): - img = self.default_image + if not img.isNull(): return img if not data: data = self.db.cover(row_number) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index c2ff99932d..16a754fab7 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -38,7 +38,6 @@ from calibre.gui2.dialogs.config import ConfigDialog from calibre.gui2.dialogs.book_info import BookInfo from calibre.library.database2 import LibraryDatabase2 -from calibre.library.caches import CoverCache from calibre.gui2.init import ToolbarMixin, LibraryViewMixin, LayoutMixin from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin @@ -138,6 +137,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ self.restriction_in_effect = False self.progress_indicator = ProgressIndicator(self) + self.progress_indicator.pos = (0, 20) self.verbose = opts.verbose self.get_metadata = GetMetadata() self.upload_memory = {} @@ -230,9 +230,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ if self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() - self.cover_cache = CoverCache(self.library_path) - self.cover_cache.start() - self.library_view.model().cover_cache = self.cover_cache self.library_view.model().count_changed_signal.connect \ (self.location_view.count_changed) if not gprefs.get('quick_start_guide_added', False): @@ -606,9 +603,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False - self.cover_cache.stop() + cc = self.library_view.model().cover_cache + if cc is not None: + cc.stop() self.hide_windows() - self.cover_cache.terminate() self.emailer.stop() try: try: diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index 8857945027..97538d9a63 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -38,12 +38,16 @@ class ProgressIndicator(QWidget): self.status.setWordWrap(True) self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop) self.setVisible(False) + self.pos = None def start(self, msg=''): view = self.parent() pwidth, pheight = view.size().width(), view.size().height() self.resize(pwidth, min(pheight, 250)) - self.move(0, (pheight-self.size().height())/2.) + if self.pos is None: + self.move(0, (pheight-self.size().height())/2.) + else: + self.move(self.pos[0], self.pos[1]) self.pi.resize(self.pi.sizeHint()) self.pi.move(int((self.size().width()-self.pi.size().width())/2.), 0) self.status.resize(self.size().width(), self.size().height()-self.pi.size().height()-10) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 6716c1c491..d46ae23d90 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,11 +6,13 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import collections, glob, os, re, itertools, functools +import re, itertools, functools from itertools import repeat from datetime import timedelta +from threading import Thread, RLock +from Queue import Queue, Empty -from PyQt4.Qt import QThread, QReadWriteLock, QImage, Qt +from PyQt4.Qt import QImage, Qt from calibre.utils.config import tweaks from calibre.utils.date import parse_date, now, UNDEFINED_DATE @@ -19,120 +21,73 @@ from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort from calibre import fit_image -class CoverCache(QThread): +class CoverCache(Thread): - def __init__(self, library_path, parent=None): - QThread.__init__(self, parent) - self.library_path = library_path - self.id_map = None - self.id_map_lock = QReadWriteLock() - self.load_queue = collections.deque() - self.load_queue_lock = QReadWriteLock(QReadWriteLock.Recursive) - self.cache = {} - self.cache_lock = QReadWriteLock() - self.id_map_stale = True + def __init__(self, db): + Thread.__init__(self) + self.daemon = True + self.db = db + self.load_queue = Queue() self.keep_running = True - - def build_id_map(self): - self.id_map_lock.lockForWrite() - self.id_map = {} - for f in glob.glob(os.path.join(self.library_path, '*', '* (*)', 'cover.jpg')): - c = os.path.basename(os.path.dirname(f)) - try: - id = int(re.search(r'\((\d+)\)', c[c.rindex('('):]).group(1)) - self.id_map[id] = f - except: - continue - self.id_map_lock.unlock() - self.id_map_stale = False - - - def set_cache(self, ids): - self.cache_lock.lockForWrite() - already_loaded = set([]) - for id in self.cache.keys(): - if id in ids: - already_loaded.add(id) - else: - self.cache.pop(id) - self.cache_lock.unlock() - ids = [i for i in ids if i not in already_loaded] - self.load_queue_lock.lockForWrite() - self.load_queue = collections.deque(ids) - self.load_queue_lock.unlock() - - - def run(self): - while self.keep_running: - if self.id_map is None or self.id_map_stale: - self.build_id_map() - while True: # Load images from the load queue - self.load_queue_lock.lockForWrite() - try: - id = self.load_queue.popleft() - except IndexError: - break - finally: - self.load_queue_lock.unlock() - - self.cache_lock.lockForRead() - need = True - if id in self.cache.keys(): - need = False - self.cache_lock.unlock() - if not need: - continue - path = None - self.id_map_lock.lockForRead() - if id in self.id_map.keys(): - path = self.id_map[id] - else: - self.id_map_stale = True - self.id_map_lock.unlock() - if path and os.access(path, os.R_OK): - try: - img = QImage() - data = open(path, 'rb').read() - img.loadFromData(data) - if img.isNull(): - continue - scaled, nwidth, nheight = fit_image(img.width(), - img.height(), 600, 800) - if scaled: - img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio, - Qt.SmoothTransformation) - except: - continue - self.cache_lock.lockForWrite() - self.cache[id] = img - self.cache_lock.unlock() - - self.sleep(1) + self.cache = {} + self.lock = RLock() + self.null_image = QImage() def stop(self): self.keep_running = False - def cover(self, id): - val = None - if self.cache_lock.tryLockForRead(50): - val = self.cache.get(id, None) - self.cache_lock.unlock() - return val + def _image_for_id(self, id_): + img = self.db.cover(id_, index_is_id=True, as_image=True) + if img is None: + img = QImage() + if not img.isNull(): + scaled, nwidth, nheight = fit_image(img.width(), + img.height(), 600, 800) + if scaled: + img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio, + Qt.SmoothTransformation) + + return img + + def run(self): + while self.keep_running: + try: + id_ = self.load_queue.get(True, 1) + except Empty: + continue + try: + img = self._image_for_id(id_) + except: + import traceback + traceback.print_exc() + continue + with self.lock: + self.cache[id_] = img + + def set_cache(self, ids): + with self.lock: + already_loaded = set([]) + for id in self.cache.keys(): + if id in ids: + already_loaded.add(id) + else: + self.cache.pop(id) + for id_ in set(ids) - already_loaded: + self.load_queue.put(id_) + + def cover(self, id_): + with self.lock: + return self.cache.get(id_, self.null_image) def clear_cache(self): - self.cache_lock.lockForWrite() - self.cache = {} - self.cache_lock.unlock() + with self.lock: + self.cache = {} def refresh(self, ids): - self.cache_lock.lockForWrite() - for id in ids: - self.cache.pop(id, None) - self.cache_lock.unlock() - self.load_queue_lock.lockForWrite() - for id in ids: - self.load_queue.appendleft(id) - self.load_queue_lock.unlock() + with self.lock: + for id_ in ids: + self.cache.pop(id_, None) + self.load_queue.put(id_) ### Global utility function for get_match here and in gui2/library.py CONTAINS_MATCH = 0 diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 9f9488a052..1534d3ffbf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en' ''' The database used to store ebook metadata ''' -import os, sys, shutil, cStringIO, glob,functools, traceback +import os, sys, shutil, cStringIO, glob, time, functools, traceback from itertools import repeat from math import floor @@ -440,12 +440,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if os.access(path, os.R_OK): if as_path: return path - f = open(path, 'rb') + try: + f = open(path, 'rb') + except (IOError, OSError): + time.sleep(0.2) + f = open(path, 'rb') if as_image: img = QImage() img.loadFromData(f.read()) + f.close() return img - return f if as_file else f.read() + ans = f if as_file else f.read() + if ans is not f: + f.close() + return ans def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' @@ -492,12 +500,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') return os.access(path, os.R_OK) - def remove_cover(self, id): + def remove_cover(self, id, notify=True): path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.path.exists(path): - os.remove(path) + try: + os.remove(path) + except (IOError, OSError): + time.sleep(0.2) + os.remove(path) + if notify: + self.notify('cover', [id]) - def set_cover(self, id, data): + def set_cover(self, id, data, notify=True): ''' Set the cover for this book. @@ -509,7 +523,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: if callable(getattr(data, 'read', None)): data = data.read() - save_cover_data_to(data, path) + try: + save_cover_data_to(data, path) + except (IOError, OSError): + time.sleep(0.2) + save_cover_data_to(data, path) + if notify: + self.notify('cover', [id]) def book_on_device(self, id): if callable(self.book_on_device_func): From 75e6bd1452ad3b80ec8e4d49fdfaaddfbaa91ffa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 21:35:48 -0600 Subject: [PATCH 06/14] ... --- src/calibre/ebooks/metadata/opf2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 0bb0c570ed..36588471f2 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1069,8 +1069,10 @@ class OPFCreator(MetaInformation): dc_attrs={'id':__appname__+'_id'})) if getattr(self, 'pubdate', None) is not None: a(DC_ELEM('date', self.pubdate.isoformat())) - a(DC_ELEM('language', self.language if self.language else - get_lang().replace('_', '-'))) + lang = self.language + if not lang or lang.lower() == 'und': + lang = get_lang().replace('_', '-') + a(DC_ELEM('language', lang)) if self.comments: a(DC_ELEM('description', self.comments)) if self.publisher: From 536bd9a31bcb97f667c83baf17cf09d74738be50 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 22:18:40 -0600 Subject: [PATCH 07/14] Fix #5949 (Writing PDF metadata should spin Jobs spinner prior to upload) --- src/calibre/gui2/device.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index c5e042d3e3..4acde6089b 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -31,6 +31,8 @@ from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.devices.apple.driver import ITUNES_ASYNC from calibre.devices.folder_device.driver import FOLDER_DEVICE +from calibre.ebooks.metadata.meta import set_metadata +from calibre.constants import DEBUG # }}} @@ -304,6 +306,21 @@ class DeviceManager(Thread): # {{{ def _upload_books(self, files, names, on_card=None, metadata=None): '''Upload books to device: ''' + if metadata and files and len(metadata) == len(files): + for f, mi in zip(files, metadata): + if isinstance(f, unicode): + ext = f.rpartition('.')[-1].lower() + if ext: + try: + if DEBUG: + prints('Setting metadata in:', mi.title, 'at:', + f, file=sys.__stdout__) + with open(f, 'r+b') as stream: + set_metadata(stream, mi, stream_type=ext) + except: + if DEBUG: + prints(traceback.format_exc(), file=sys.__stdout__) + return self.device.upload_books(files, names, on_card, metadata=metadata, end_session=False) @@ -1145,7 +1162,6 @@ class DeviceMixin(object): # {{{ _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids, settings.format_map, - set_metadata=True, specific_format=specific_format, exclude_auto=do_auto_convert) if do_auto_convert: From 5a1731a38f92f775546da458aa90a8c6cce2cfcc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 22:43:40 -0600 Subject: [PATCH 08/14] PoDoFo cleanups --- setup/installer/linux/freeze.py | 1 + src/calibre/utils/podofo/__init__.py | 9 ++++++++- src/calibre/utils/podofo/podofo.cpp | 6 ++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup/installer/linux/freeze.py b/setup/installer/linux/freeze.py index 08237b83fa..8c56ed4fb7 100644 --- a/setup/installer/linux/freeze.py +++ b/setup/installer/linux/freeze.py @@ -40,6 +40,7 @@ class LinuxFreeze(Command): '/usr/bin/pdftohtml', '/usr/lib/libwmflite-0.2.so.7', '/usr/lib/liblcms.so.1', + '/usr/lib/libstlport.so.5.1', '/tmp/calibre-mount-helper', '/usr/lib/libunrar.so', '/usr/lib/libchm.so.0', diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 9fc0c981e6..284deb7c43 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -14,6 +14,7 @@ from calibre.ebooks.metadata import MetaInformation, string_to_authors, \ from calibre.utils.ipc.job import ParallelJob from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryFile +from calibre import prints podofo, podofo_err = plugins['podofo'] @@ -117,12 +118,18 @@ def set_metadata(stream, mi): job.update() server.close() - if job.result is not None: + if job.failed: + prints(job.details) + elif job.result is not None: stream.seek(0) stream.truncate() stream.write(job.result) stream.flush() stream.seek(0) + try: + os.remove(pt.name) + except: + pass diff --git a/src/calibre/utils/podofo/podofo.cpp b/src/calibre/utils/podofo/podofo.cpp index ace8c58c70..ea982167d3 100644 --- a/src/calibre/utils/podofo/podofo.cpp +++ b/src/calibre/utils/podofo/podofo.cpp @@ -55,8 +55,7 @@ podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { } else return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject * @@ -73,8 +72,7 @@ podofo_PDFDoc_open(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { } else return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject * From 1483703aa02e5ca1e25254da8c89ea514a414374 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 23:21:59 -0600 Subject: [PATCH 09/14] Set correct LD_LIBRARY_PATH onlinux before launching urls --- src/calibre/__init__.py | 7 ------ src/calibre/gui2/__init__.py | 28 ++++++++++++++++----- src/calibre/gui2/actions.py | 25 +++++++----------- src/calibre/gui2/book_details.py | 6 ++--- src/calibre/gui2/dialogs/book_info.py | 13 +++++----- src/calibre/gui2/dialogs/config/__init__.py | 11 ++++---- src/calibre/gui2/dialogs/user_profiles.py | 6 ++--- src/calibre/gui2/ui.py | 8 +++--- src/calibre/gui2/update.py | 6 ++--- src/calibre/gui2/viewer/main.py | 6 ++--- 10 files changed, 60 insertions(+), 56 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 285b2d35b4..92ee2ca6d2 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -342,13 +342,6 @@ def detect_ncpus(): return ans -def launch(path_or_url): - from PyQt4.QtCore import QUrl - from PyQt4.QtGui import QDesktopServices - if os.path.exists(path_or_url): - path_or_url = 'file:'+path_or_url - QDesktopServices.openUrl(QUrl(path_or_url)) - relpath = os.path.relpath _spat = re.compile(r'^the\s+|^a\s+|^an\s+', re.IGNORECASE) def english_sort(x, y): diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index f286402a37..9face1d9e9 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -1,18 +1,18 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' """ The GUI """ -import os +import os, sys from threading import RLock -from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ +from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ QByteArray, QTranslator, QCoreApplication, QThread, \ - QEvent, QTimer, pyqtSignal, QDate -from PyQt4.QtGui import QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ - QIcon, QApplication, QDialog, QPushButton + QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ + QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ + QIcon, QApplication, QDialog, QPushButton, QUrl ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' -from calibre import islinux, iswindows, isosx, isfreebsd +from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats @@ -579,6 +579,22 @@ class Application(QApplication): _store_app = None +def open_url(qurl): + paths = os.environ.get('LD_LIBRARY_PATH', + '').split(os.pathsep) + paths = [x for x in paths if x] + if isfrozen and islinux and paths: + npaths = [x for x in paths if x != sys.frozen_path] + os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(npaths) + QDesktopServices.openUrl(qurl) + if isfrozen and islinux and paths: + os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths) + + +def open_local_file(path): + url = QUrl.fromLocalFile(path) + open_url(url) + def is_ok_to_use_qt(): global gui_thread, _store_app if (islinux or isfreebsd) and ':' not in os.environ.get('DISPLAY', ''): diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py index 5dde2f745b..f36df397f9 100644 --- a/src/calibre/gui2/actions.py +++ b/src/calibre/gui2/actions.py @@ -5,17 +5,18 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, os, datetime, sys, time +import shutil, os, datetime, time from functools import partial from PyQt4.Qt import QInputDialog, pyqtSignal, QModelIndex, QThread, Qt, \ - SIGNAL, QPixmap, QTimer, QDesktopServices, QUrl, QDialog + SIGNAL, QPixmap, QTimer, QDialog from calibre import strftime from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.gui2 import error_dialog, Dispatcher, gprefs, choose_files, \ - choose_dir, warning_dialog, info_dialog, question_dialog, config + choose_dir, warning_dialog, info_dialog, question_dialog, config, \ + open_local_file from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString from calibre.utils.filenames import ascii_filename from calibre.gui2.widgets import IMAGE_EXTENSIONS @@ -25,7 +26,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \ fetch_scheduled_recipe, generate_catalog from calibre.constants import preferred_encoding, filesystem_encoding, \ - isosx, isfrozen, islinux + isosx from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2.dialogs.confirm_delete import confirm @@ -920,7 +921,7 @@ class SaveToDiskAction(object): # {{{ _('Could not save some books') + ', ' + _('Click the show details button to see which ones.'), u'\n\n'.join(failures), show=True) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def books_saved(self, job): if job.failed: @@ -1186,15 +1187,7 @@ class ViewAction(object): # {{{ self.job_manager.launch_gui_app(viewer, kwargs=dict(args=args)) else: - paths = os.environ.get('LD_LIBRARY_PATH', - '').split(os.pathsep) - paths = [x for x in paths if x] - if isfrozen and islinux and paths: - npaths = [x for x in paths if x != sys.frozen_path] - os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(npaths) - QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name) - if isfrozen and islinux and paths: - os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths) + open_local_file(name) time.sleep(2) # User feedback finally: self.unsetCursor() @@ -1240,11 +1233,11 @@ class ViewAction(object): # {{{ return for row in rows: path = self.library_view.model().db.abspath(row.row()) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def view_folder_for_id(self, id_): path = self.library_view.model().db.abspath(id_, index_is_id=True) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def view_book(self, triggered): rows = self.current_view().selectionModel().selectedRows() diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 4deadbc857..f08dd09429 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -9,14 +9,14 @@ import os, collections from PyQt4.Qt import QLabel, QPixmap, QSize, QWidget, Qt, pyqtSignal, \ QVBoxLayout, QScrollArea, QPropertyAnimation, QEasingCurve, \ - QSizePolicy, QPainter, QRect, pyqtProperty, QDesktopServices, QUrl + QSizePolicy, QPainter, QRect, pyqtProperty from calibre import fit_image, prepare_string_for_xml from calibre.gui2.widgets import IMAGE_EXTENSIONS from calibre.ebooks import BOOK_EXTENSIONS from calibre.constants import preferred_encoding from calibre.library.comments import comments_to_html -from calibre.gui2 import config +from calibre.gui2 import config, open_local_file # render_rows(data) {{{ WEIGHTS = collections.defaultdict(lambda : 100) @@ -294,7 +294,7 @@ class BookDetails(QWidget): # {{{ id_, fmt = val.split(':') self.view_specific_format.emit(int(id_), fmt) elif typ == 'devpath': - QDesktopServices.openUrl(QUrl.fromLocalFile(val)) + open_local_file(val) def mouseReleaseEvent(self, ev): diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index 20ddfae0b4..9770ef864f 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -5,11 +5,11 @@ __docformat__ = 'restructuredtext en' import textwrap, os, re -from PyQt4.QtCore import QCoreApplication, SIGNAL, QModelIndex, QUrl, QTimer, Qt -from PyQt4.QtGui import QDialog, QPixmap, QGraphicsScene, QIcon, QDesktopServices +from PyQt4.QtCore import QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt +from PyQt4.QtGui import QDialog, QPixmap, QGraphicsScene, QIcon from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo -from calibre.gui2 import dynamic +from calibre.gui2 import dynamic, open_local_file from calibre import fit_image from calibre.library.comments import comments_to_html @@ -49,12 +49,12 @@ class BookInfo(QDialog, Ui_BookInfo): def open_book_path(self, path): if os.sep in unicode(path): - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) else: format = unicode(path) path = self.view.model().db.format_abspath(self.current_row, format) if path is not None: - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def next(self): @@ -123,6 +123,7 @@ class BookInfo(QDialog, Ui_BookInfo): for key in info.keys(): if key == 'id': continue txt = info[key] - txt = u'
\n'.join(textwrap.wrap(txt, 120)) + if key != _('Path'): + txt = u'
\n'.join(textwrap.wrap(txt, 120)) rows += u'%s:%s'%(key, txt) self.text.setText(u''+rows+'
') diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index f17c0083ec..144d8f8586 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import os, re, time, textwrap, copy, sys from PyQt4.Qt import QDialog, QListWidgetItem, QIcon, \ - QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \ + QVBoxLayout, QLabel, QPlainTextEdit, \ QStringListModel, QAbstractItemModel, QFont, \ SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \ QModelIndex, QAbstractTableModel, \ @@ -15,8 +15,9 @@ from calibre.constants import iswindows, isosx from calibre.gui2.dialogs.config.config_ui import Ui_Dialog from calibre.gui2.dialogs.config.create_custom_column import CreateCustomColumn from calibre.gui2 import choose_dir, error_dialog, config, gprefs, \ - ALL_COLUMNS, NONE, info_dialog, choose_files, \ - warning_dialog, ResizableDialog, question_dialog + open_url, open_local_file, \ + ALL_COLUMNS, NONE, info_dialog, choose_files, \ + warning_dialog, ResizableDialog, question_dialog from calibre.utils.config import prefs from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.oeb.iterator import is_supported @@ -512,7 +513,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): def open_config_dir(self): from calibre.utils.config import config_dir - QDesktopServices.openUrl(QUrl.fromLocalFile(config_dir)) + open_local_file(config_dir) def create_symlinks(self): from calibre.utils.osx_symlinks import create_symlinks @@ -805,7 +806,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): self.stop.setEnabled(False) def test_server(self): - QDesktopServices.openUrl(QUrl('http://127.0.0.1:'+str(self.port.value()))) + open_url(QUrl('http://127.0.0.1:'+str(self.port.value()))) def compact(self, toggled): d = CheckIntegrity(self.db, self) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 7b26fea0ae..16f5d383ed 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal ' import time, os -from PyQt4.Qt import SIGNAL, QUrl, QDesktopServices, QAbstractListModel, Qt, \ +from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ QVariant, QInputDialog from calibre.web.feeds.recipes import compile_recipe from calibre.web.feeds.news import AutomaticNewsRecipe from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog -from calibre.gui2 import error_dialog, question_dialog, \ +from calibre.gui2 import error_dialog, question_dialog, open_url, \ choose_files, ResizableDialog, NONE from calibre.gui2.widgets import PythonHighlighter from calibre.ptempfile import PersistentTemporaryFile @@ -135,7 +135,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): url.addQueryItem('subject', subject) url.addQueryItem('body', body) url.addQueryItem('attachment', pt.name) - QDesktopServices.openUrl(url) + open_url(url) def current_changed(self, current, previous): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 16a754fab7..756e375e23 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -12,9 +12,9 @@ __docformat__ = 'restructuredtext en' import collections, os, sys, textwrap, time from Queue import Queue, Empty from threading import Thread -from PyQt4.Qt import Qt, SIGNAL, QObject, QUrl, QTimer, \ +from PyQt4.Qt import Qt, SIGNAL, QObject, QTimer, \ QPixmap, QMenu, QIcon, pyqtSignal, \ - QDialog, QDesktopServices, \ + QDialog, \ QSystemTrayIcon, QApplication, QKeySequence, QAction, \ QMessageBox, QHelpEvent @@ -23,7 +23,7 @@ from calibre.constants import __version__, __appname__, isosx from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server -from calibre.gui2 import error_dialog, GetMetadata, \ +from calibre.gui2 import error_dialog, GetMetadata, open_local_file, \ gprefs, max_available_height, config, info_dialog from calibre.gui2.cover_flow import CoverFlowMixin from calibre.gui2.widgets import ProgressIndicator @@ -572,7 +572,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ pt = PersistentTemporaryFile('_donate.htm') pt.write(HTML.encode('utf-8')) pt.close() - QDesktopServices.openUrl(QUrl.fromLocalFile(pt.name)) + open_local_file(pt.name) def confirm_quit(self): diff --git a/src/calibre/gui2/update.py b/src/calibre/gui2/update.py index 9dcd4d9084..84168d17b5 100644 --- a/src/calibre/gui2/update.py +++ b/src/calibre/gui2/update.py @@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal ' import traceback -from PyQt4.Qt import QThread, pyqtSignal, QDesktopServices, QUrl, Qt +from PyQt4.Qt import QThread, pyqtSignal, Qt, QUrl import mechanize from calibre.constants import __appname__, __version__, iswindows, isosx from calibre import browser from calibre.utils.config import prefs -from calibre.gui2 import config, dynamic, question_dialog +from calibre.gui2 import config, dynamic, question_dialog, open_url URL = 'http://status.calibre-ebook.com/latest' @@ -64,7 +64,7 @@ class UpdateMixin(object): 'ge?')%(__appname__, version)): url = 'http://calibre-ebook.com/download_'+\ ('windows' if iswindows else 'osx' if isosx else 'linux') - QDesktopServices.openUrl(QUrl(url)) + open_url(QUrl(url)) dynamic.set('update to version %s'%version, False) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index fca3586f9d..ec88c3f886 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -6,7 +6,7 @@ from functools import partial from threading import Thread from PyQt4.Qt import QApplication, Qt, QIcon, QTimer, SIGNAL, QByteArray, \ - QDesktopServices, QDoubleSpinBox, QLabel, QTextBrowser, \ + QDoubleSpinBox, QLabel, QTextBrowser, \ QPainter, QBrush, QColor, QStandardItemModel, QPalette, \ QStandardItem, QUrl, QRegExpValidator, QRegExp, QLineEdit, \ QToolButton, QMenu, QInputDialog, QAction, QKeySequence @@ -17,7 +17,7 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager from calibre.gui2.widgets import ProgressIndicator from calibre.gui2.main_window import MainWindow from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \ - info_dialog, error_dialog + info_dialog, error_dialog, open_url from calibre.ebooks.oeb.iterator import EbookIterator from calibre.ebooks import DRMError from calibre.constants import islinux, isfreebsd @@ -472,7 +472,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): elif frag: self.view.scroll_to(frag) else: - QDesktopServices.openUrl(url) + open_url(url) def load_started(self): self.open_progress_indicator(_('Loading flow...')) From 9235d98428783a9ef773d692290ec71182963104 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 23:24:31 -0600 Subject: [PATCH 10/14] Make the comments to HTML transform faster and more robust --- src/calibre/library/comments.py | 73 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/src/calibre/library/comments.py b/src/calibre/library/comments.py index 018c39bcf7..d4ed1908f5 100644 --- a/src/calibre/library/comments.py +++ b/src/calibre/library/comments.py @@ -8,9 +8,14 @@ __docformat__ = 'restructuredtext en' import re from calibre.constants import preferred_encoding -from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString, \ + CData, Comment, Declaration, ProcessingInstruction from calibre import prepare_string_for_xml +# Hackish - ignoring sentences ending or beginning in numbers to avoid +# confusion with decimal points. +lost_cr_pat = re.compile('([a-z])([\.\?!])([A-Z])') + def comments_to_html(comments): ''' Convert random comment text to normalized, xml-legal block of

s @@ -41,36 +46,25 @@ def comments_to_html(comments): if '<' not in comments: comments = prepare_string_for_xml(comments) - comments = comments.replace(u'\n', u'
') - return u'

%s

'%comments - - # Hackish - ignoring sentences ending or beginning in numbers to avoid - # confusion with decimal points. + parts = [u'

%s

'%x.replace(u'\n', u'
') + for x in comments.split('\n\n')] + return '\n'.join(parts) # Explode lost CRs to \n\n - for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])', comments): + for lost_cr in lost_cr_pat.finditer(comments): comments = comments.replace(lost_cr.group(), '%s%s\n\n%s' % (lost_cr.group(1), lost_cr.group(2), lost_cr.group(3))) + comments = comments.replace(u'\r', u'') # Convert \n\n to

s - if re.search('\n\n', comments): - soup = BeautifulSoup() - split_ps = comments.split(u'\n\n') - tsc = 0 - for p in split_ps: - pTag = Tag(soup,'p') - pTag.insert(0,p) - soup.insert(tsc,pTag) - tsc += 1 - comments = soup.renderContents(None) - + comments = comments.replace(u'\n\n', u'

') # Convert solo returns to
- comments = re.sub('[\r\n]','
', comments) - + comments = comments.replace(u'\n', '
') # Convert two hyphens to emdash - comments = re.sub('--', '—', comments) + comments = comments.replace('--', '—') + soup = BeautifulSoup(comments) result = BeautifulSoup() rtc = 0 @@ -85,35 +79,52 @@ def comments_to_html(comments): ptc = 0 pTag.insert(ptc,prepare_string_for_xml(token)) ptc += 1 - - elif token.name in ['br','b','i','em']: + elif type(token) in (CData, Comment, Declaration, + ProcessingInstruction): + continue + elif token.name in ['br', 'b', 'i', 'em', 'strong', 'span', 'font', 'a', + 'hr']: if not open_pTag: pTag = Tag(result,'p') open_pTag = True ptc = 0 pTag.insert(ptc, token) ptc += 1 - else: if open_pTag: result.insert(rtc, pTag) rtc += 1 open_pTag = False ptc = 0 - # Clean up NavigableStrings for xml - sub_tokens = list(token.contents) - for sub_token in sub_tokens: - if type(sub_token) is NavigableString: - sub_token.replaceWith(prepare_string_for_xml(sub_token)) result.insert(rtc, token) rtc += 1 if open_pTag: result.insert(rtc, pTag) - paras = result.findAll('p') - for p in paras: + for p in result.findAll('p'): p['class'] = 'description' + for t in result.findAll(text=True): + t.replaceWith(prepare_string_for_xml(unicode(t))) + return result.renderContents(encoding=None) +def test(): + for pat, val in [ + ('lineone\n\nlinetwo', + '

lineone

\n

linetwo

'), + ('a b&c\nf', '

a b&c;
f

'), + ('a b\n\ncd', '

a b

cd

'), + ]: + print + print 'Testing: %r'%pat + cval = comments_to_html(pat) + print 'Value: %r'%cval + if comments_to_html(pat) != val: + print 'FAILED' + break + +if __name__ == '__main__': + test() + From f16bc59dd0b1b999120dace3cba55ad7769c9691 Mon Sep 17 00:00:00 2001 From: Li Fanxi Date: Thu, 1 Jul 2010 14:55:32 +0800 Subject: [PATCH 11/14] Refine the Douban.com plugin. According to Douban API usage guide, hard code the pre-applied API key for calibre metadata download plugin. --- src/calibre/ebooks/metadata/douban.py | 33 +++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/calibre/ebooks/metadata/douban.py b/src/calibre/ebooks/metadata/douban.py index c881721fcc..c6a34b6162 100644 --- a/src/calibre/ebooks/metadata/douban.py +++ b/src/calibre/ebooks/metadata/douban.py @@ -15,7 +15,6 @@ from calibre.utils.config import OptionParser from calibre.ebooks.metadata.fetch import MetadataSource from calibre.utils.date import parse_date, utcnow -DOUBAN_API_KEY = None NAMESPACES = { 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', 'atom' : 'http://www.w3.org/2005/Atom', @@ -35,13 +34,15 @@ date = XPath("descendant::db:attribute[@name='pubdate']") creator = XPath("descendant::db:attribute[@name='author']") tag = XPath("descendant::db:tag") +CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d' + class DoubanBooks(MetadataSource): name = 'Douban Books' description = _('Downloads metadata from Douban.com') supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on author = 'Li Fanxi ' # The author of this plugin - version = (1, 0, 0) # The version number of this plugin + version = (1, 0, 1) # The version number of this plugin def fetch(self): try: @@ -65,7 +66,7 @@ class Query(object): type = "search" def __init__(self, title=None, author=None, publisher=None, isbn=None, - max_results=20, start_index=1): + max_results=20, start_index=1, api_key=''): assert not(title is None and author is None and publisher is None and \ isbn is None) assert (int(max_results) < 21) @@ -89,16 +90,16 @@ class Query(object): if self.type == "isbn": self.url = self.ISBN_URL + q - if DOUBAN_API_KEY is not None: - self.url = self.url + "?apikey=" + DOUBAN_API_KEY + if api_key != '': + self.url = self.url + "?apikey=" + api_key else: self.url = self.SEARCH_URL+urlencode({ 'q':q, 'max-results':max_results, 'start-index':start_index, }) - if DOUBAN_API_KEY is not None: - self.url = self.url + "&apikey=" + DOUBAN_API_KEY + if api_key != '': + self.url = self.url + "&apikey=" + api_key def __call__(self, browser, verbose): if verbose: @@ -177,7 +178,7 @@ class ResultList(list): d = None return d - def populate(self, entries, browser, verbose=False): + def populate(self, entries, browser, verbose=False, api_key=''): for x in entries: try: id_url = entry_id(x)[0].text @@ -186,8 +187,8 @@ class ResultList(list): report(verbose) mi = MetaInformation(title, self.get_authors(x)) try: - if DOUBAN_API_KEY is not None: - id_url = id_url + "?apikey=" + DOUBAN_API_KEY + if api_key != '': + id_url = id_url + "?apikey=" + api_key raw = browser.open(id_url).read() feed = etree.fromstring(raw) x = entry(feed)[0] @@ -203,12 +204,16 @@ class ResultList(list): self.append(mi) def search(title=None, author=None, publisher=None, isbn=None, - verbose=False, max_results=40): + verbose=False, max_results=40, api_key=None): br = browser() start, entries = 1, [] + + if api_key is None: + api_key = CALIBRE_DOUBAN_API_KEY + while start > 0 and len(entries) <= max_results: - new, start = Query(title=title, author=author, publisher=publisher, - isbn=isbn, max_results=max_results, start_index=start)(br, verbose) + new, start = Query(title=title, author=author, publisher=publisher, + isbn=isbn, max_results=max_results, start_index=start, api_key=api_key)(br, verbose) if not new: break entries.extend(new) @@ -216,7 +221,7 @@ def search(title=None, author=None, publisher=None, isbn=None, entries = entries[:max_results] ans = ResultList() - ans.populate(entries, br, verbose) + ans.populate(entries, br, verbose, api_key) return ans def option_parser(): From e05dedfa0ff4c3d79c241cb19c7ed1c7a2fe45a2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Jul 2010 08:46:53 -0600 Subject: [PATCH 12/14] ifzm by rty. Fixes #6043 (Wrong Chinese Category in News) --- resources/recipes/china_press.recipe | 2 +- resources/recipes/ifzm.recipe | 50 +++++++++++++++++++++++++++ src/calibre/devices/apple/driver.py | 3 +- src/calibre/ebooks/metadata/covers.py | 15 ++++++++ src/calibre/web/feeds/templates.py | 2 +- 5 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 resources/recipes/ifzm.recipe create mode 100644 src/calibre/ebooks/metadata/covers.py diff --git a/resources/recipes/china_press.recipe b/resources/recipes/china_press.recipe index 9ceea1a423..502ebfd41c 100644 --- a/resources/recipes/china_press.recipe +++ b/resources/recipes/china_press.recipe @@ -7,7 +7,7 @@ class AdvancedUserRecipe1277228948(BasicNewsRecipe): __author__ = 'rty' __version__ = '1.0' - language = 'zh_CN' + language = 'zh' pubisher = 'www.chinapressusa.com' description = 'Overseas Chinese Network Newspaper in the USA' category = 'News in Chinese, USA' diff --git a/resources/recipes/ifzm.recipe b/resources/recipes/ifzm.recipe new file mode 100644 index 0000000000..407acefdd4 --- /dev/null +++ b/resources/recipes/ifzm.recipe @@ -0,0 +1,50 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1277305250(BasicNewsRecipe): + title = u'infzm - China Southern Weekly' + oldest_article = 14 + max_articles_per_feed = 100 + + feeds = [(u'\u5357\u65b9\u5468\u672b-\u70ed\u70b9\u65b0\u95fb', u'http://www.infzm.com/rss/home/rss2.0.xml'), + (u'\u5357\u65b9\u5468\u672b-\u7ecf\u6d4e\u65b0\u95fb', u'http://www.infzm.com/rss/economic.xml'), + (u'\u5357\u65b9\u5468\u672b-\u6587\u5316\u65b0\u95fb', u'http://www.infzm.com/rss/culture.xml'), + (u'\u5357\u65b9\u5468\u672b-\u751f\u6d3b\u65f6\u5c1a', u'http://www.infzm.com/rss/lifestyle.xml'), + (u'\u5357\u65b9\u5468\u672b-\u89c2\u70b9', u'http://www.infzm.com/rss/opinion.xml') + ] + __author__ = 'rty' + __version__ = '1.0' + language = 'zh' + pubisher = 'http://www.infzm.com' + description = 'Chinese Weekly Tabloid' + category = 'News, China' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + #encoding = 'GB2312' + encoding = 'UTF-8' + conversion_options = {'linearize_tables':True} + masthead_url = 'http://i50.tinypic.com/2qmfb7l.jpg' + + extra_css = ''' + @font-face { font-family: "DroidFont", serif, sans-serif; src: url(res:///system/fonts/DroidSansFallback.ttf); }\n + body { + margin-right: 8pt; + font-family: 'DroidFont', serif;} + .detailContent {font-family: 'DroidFont', serif, sans-serif} + ''' + + keep_only_tags = [ + dict(name='div', attrs={'id':'detailContent'}), + ] + remove_tags = [ + dict(name='div', attrs={'id':['detailTools', 'detailSideL', 'pageNum']}), + ] + remove_tags_after = [ + dict(name='div', attrs={'id':'pageNum'}), + ] + def preprocess_html(self, soup): + for item in soup.findAll(color=True): + del item['font'] + for item in soup.findAll(style=True): + del item['style'] + return soup diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 9028347c4b..3156542a92 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -20,7 +20,7 @@ from calibre.utils.config import config_dir from calibre.utils.date import isoformat, now, parse_date from calibre.utils.localization import get_lang from calibre.utils.logging import Log -from calibre.utils.zipfile import ZipFile, safe_replace +from calibre.utils.zipfile import ZipFile from PIL import Image as PILImage @@ -1727,7 +1727,6 @@ class ITUNES(DriverBase): return thumb_data thumb_path = book_path.rpartition('.')[0] + '.jpg' - format = book_path.rpartition('.')[2].lower() if isosx: title = book.name() elif iswindows: diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py new file mode 100644 index 0000000000..af213d1a6c --- /dev/null +++ b/src/calibre/ebooks/metadata/covers.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +from calibre.customize import Plugin + +class CoverDownload(Plugin): + + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Kovid Goyal' + type = _('Cover download') diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index 4f26a35a02..1b2ba11e9c 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -8,7 +8,7 @@ import copy from lxml import html, etree from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \ - STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, H3, IMG, P as PT, \ + STRONG, BR, SPAN, A, HR, UL, LI, H2, H3, IMG, P as PT, \ TABLE, TD, TR from calibre import preferred_encoding, strftime, isbytestring From 23a11b991ec82c9279152f574e4d1e130f89c540 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Jul 2010 09:00:47 -0600 Subject: [PATCH 13/14] Fix #6019 (Unwanted conversion of Norwegian letters) --- src/calibre/customize/builtins.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 2944035182..b1be1a62a9 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -30,6 +30,7 @@ every time you add an HTML file to the library.\ with TemporaryDirectory('_plugin_html2zip') as tdir: recs =[('debug_pipeline', tdir, OptionRecommendation.HIGH)] + recs.append(['keep_ligatures', True, OptionRecommendation.HIGH]) if self.site_customization and self.site_customization.strip(): recs.append(['input_encoding', self.site_customization.strip(), OptionRecommendation.HIGH]) From c295436c1bacb55889db7282075ae32e849ea9cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 1 Jul 2010 17:32:43 -0600 Subject: [PATCH 14/14] Support for the Nokia E52 --- src/calibre/customize/builtins.py | 3 ++- src/calibre/devices/nokia/driver.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index b1be1a62a9..194cf8a30c 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -445,7 +445,7 @@ from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX from calibre.devices.nook.driver import NOOK from calibre.devices.prs505.driver import PRS505 from calibre.devices.android.driver import ANDROID, S60 -from calibre.devices.nokia.driver import N770, N810, E71X +from calibre.devices.nokia.driver import N770, N810, E71X, E52 from calibre.devices.eslick.driver import ESLICK, EBK52 from calibre.devices.nuut2.driver import NUUT2 from calibre.devices.iriver.driver import IRIVER_STORY @@ -520,6 +520,7 @@ plugins += [ S60, N770, E71X, + E52, N810, COOL_ER, ESLICK, diff --git a/src/calibre/devices/nokia/driver.py b/src/calibre/devices/nokia/driver.py index 66a4243f2b..f378a656fb 100644 --- a/src/calibre/devices/nokia/driver.py +++ b/src/calibre/devices/nokia/driver.py @@ -67,3 +67,24 @@ class E71X(USBMS): VENDOR_NAME = 'NOKIA' WINDOWS_MAIN_MEM = 'S60' +class E52(USBMS): + + name = 'Nokia E52 device interface' + gui_name = 'Nokia E52' + description = _('Communicate with the Nokia E52') + author = 'David Ignjic' + supported_platforms = ['windows', 'linux', 'osx'] + + VENDOR_ID = [0x421] + PRODUCT_ID = [0x1CD] + BCD = [0x100] + + + FORMATS = ['mobi', 'prc'] + + EBOOK_DIR_MAIN = 'eBooks' + SUPPORTS_SUB_DIRS = True + + VENDOR_NAME = 'NOKIA' + WINDOWS_MAIN_MEM = 'S60' +