From dcd258b52bd00408aad84bfc8a55454ad83a2246 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 9 Feb 2009 16:15:16 -0800 Subject: [PATCH 01/21] Fix nasty typo that broke cover fetching in 0.4.135 --- src/calibre/ebooks/metadata/library_thing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/library_thing.py b/src/calibre/ebooks/metadata/library_thing.py index 8cc1282067..ef41d5e937 100644 --- a/src/calibre/ebooks/metadata/library_thing.py +++ b/src/calibre/ebooks/metadata/library_thing.py @@ -43,8 +43,9 @@ def cover_from_isbn(isbn, timeout=5.): src = browser.open('http://www.librarything.com/isbn/'+isbn).read().decode('utf-8', 'replace') except Exception, err: if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - raise LibraryThingError(_('LibraryThing.com timed out. Try again later.')) - raise + err = LibraryThingError(_('LibraryThing.com timed out. Try again later.')) + raise err + else: s = BeautifulSoup(src) url = s.find('td', attrs={'class':'left'}) if url is None: From 3392459c2737f0a66bb1e0268469cdf78fd2f243 Mon Sep 17 00:00:00 2001 From: "Marshall T. Vandegrift" Date: Tue, 10 Feb 2009 12:54:13 -0500 Subject: [PATCH 02/21] Fix #1815. Don't re-decode unicode metadata.opf2 @hrefs. --- src/calibre/ebooks/epub/from_html.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/epub/from_html.py b/src/calibre/ebooks/epub/from_html.py index bd9b59cfbd..fd94c9ee69 100644 --- a/src/calibre/ebooks/epub/from_html.py +++ b/src/calibre/ebooks/epub/from_html.py @@ -74,7 +74,9 @@ def check_links(opf_path, pretty_print): html_files = [] for item in opf.itermanifest(): if 'html' in item.get('media-type', '').lower(): - f = item.get('href').split('/')[-1].decode('utf-8') + f = item.get('href').split('/')[-1] + if isinstance(f, str): + f = f.decode('utf-8') html_files.append(os.path.abspath(content(f))) for path in html_files: From bc4b6fdf486b48a2bf771f57025bd3cd2795176f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 10:29:09 -0800 Subject: [PATCH 03/21] New recipe for Pobjeda by Darko Miletic --- src/calibre/gui2/__init__.py | 39 ++- src/calibre/gui2/add.py | 224 ++++++++++++++++++ src/calibre/gui2/dialogs/confirm_delete.py | 3 +- src/calibre/gui2/dialogs/progress.py | 1 + src/calibre/gui2/images/news/pobjeda.png | Bin 0 -> 285 bytes src/calibre/gui2/main.py | 145 +++++------- src/calibre/library/database.py | 114 +-------- src/calibre/library/database2.py | 112 ++++++++- src/calibre/web/feeds/recipes/__init__.py | 1 + .../web/feeds/recipes/recipe_pobjeda.py | 102 ++++++++ 10 files changed, 532 insertions(+), 209 deletions(-) create mode 100644 src/calibre/gui2/add.py create mode 100644 src/calibre/gui2/images/news/pobjeda.png create mode 100644 src/calibre/web/feeds/recipes/recipe_pobjeda.py diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 084b352f48..c33036e183 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -1,7 +1,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' """ The GUI """ -import sys, os, re, StringIO, traceback +import sys, os, re, StringIO, traceback, time from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ QByteArray, QLocale, QUrl, QTranslator, QCoreApplication, \ QModelIndex @@ -14,6 +14,9 @@ from calibre import __author__, islinux, iswindows, isosx from calibre.startup import get_lang from calibre.utils.config import Config, ConfigProxy, dynamic import calibre.resources as resources +from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats +from calibre.ebooks.metadata import MetaInformation + NONE = QVariant() #: Null value to return from the data function of item models @@ -148,7 +151,41 @@ class Dispatcher(QObject): def dispatch(self, args, kwargs): self.func(*args, **kwargs) + +class GetMetadata(QObject): + ''' + Convenience class to ensure that metadata readers are used only in the + GUI thread. Must be instantiated in the GUI thread. + ''' + + def __init__(self): + QObject.__init__(self) + self.connect(self, SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), + self._get_metadata, Qt.QueuedConnection) + self.connect(self, SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), + self._from_formats, Qt.QueuedConnection) + def __call__(self, id, *args, **kwargs): + self.emit(SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), + id, args, kwargs) + + def from_formats(self, id, *args, **kwargs): + self.emit(SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), + id, args, kwargs) + + def _from_formats(self, id, args, kwargs): + try: + mi = metadata_from_formats(*args, **kwargs) + except: + mi = MetaInformation('', [_('Unknown')]) + self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi) + + def _get_metadata(self, id, args, kwargs): + try: + mi = get_metadata(*args, **kwargs) + except: + mi = MetaInformation('', [_('Unknown')]) + self.emit(SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), id, mi) class TableView(QTableView): def __init__(self, parent): diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py new file mode 100644 index 0000000000..a6c6c699e8 --- /dev/null +++ b/src/calibre/gui2/add.py @@ -0,0 +1,224 @@ +''' +UI for adding books to the database +''' +import os + +from PyQt4.Qt import QThread, SIGNAL, QMutex, QWaitCondition, Qt + +from calibre.gui2.dialogs.progress import ProgressDialog +from calibre.constants import preferred_encoding +from calibre.gui2.widgets import WarningDialog + +class Add(QThread): + + def __init__(self): + QThread.__init__(self) + self._lock = QMutex() + self._waiting = QWaitCondition() + + def is_canceled(self): + if self.pd.canceled: + self.canceled = True + return self.canceled + + def wait_for_condition(self): + self._lock.lock() + self._waiting.wait(self._lock) + self._lock.unlock() + + def wake_up(self): + self._waiting.wakeAll() + +class AddFiles(Add): + + def __init__(self, paths, default_thumbnail, get_metadata, db=None): + Add.__init__(self) + self.paths = paths + self.get_metadata = get_metadata + self.default_thumbnail = default_thumbnail + self.db = db + self.formats, self.metadata, self.names, self.infos = [], [], [], [] + self.duplicates = [] + self.number_of_books_added = 0 + self.connect(self.get_metadata, + SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), + self.metadata_delivered) + + def metadata_delivered(self, id, mi): + if self.is_canceled(): + self.reading.wakeAll() + return + if not mi.title: + mi.title = os.path.splitext(self.names[id])[0] + mi.title = mi.title if isinstance(mi.title, unicode) else \ + mi.title.decode(preferred_encoding, 'replace') + self.metadata.append(mi) + self.infos.append({'title':mi.title, + 'authors':', '.join(mi.authors), + 'cover':self.default_thumbnail, 'tags':[]}) + if self.db is not None: + duplicates, num = self.db.add_books(self.paths[id:id+1], + self.formats[id:id+1], [mi], + add_duplicates=False) + self.number_of_books_added += num + if duplicates: + if not self.duplicates: + self.duplicates = [[], [], [], []] + for i in range(4): + self.duplicates[i] += duplicates[i] + self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), + mi.title, id) + self.wake_up() + + def create_progress_dialog(self, title, msg, parent): + self._parent = parent + self.pd = ProgressDialog(title, msg, -1, len(self.paths)-1, parent) + self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), + self.update_progress_dialog) + self.pd.setModal(True) + self.pd.show() + self.connect(self, SIGNAL('finished()'), self.pd.hide) + return self.pd + + + def update_progress_dialog(self, title, count): + self.pd.set_value(count) + if self.db is not None: + self.pd.set_msg(_('Added %s to library')%title) + else: + self.pd.set_msg(_('Read metadata from ')+title) + + + def run(self): + self.canceled = False + for c, book in enumerate(self.paths): + if self.pd.canceled: + self.canceled = True + break + format = os.path.splitext(book)[1] + format = format[1:] if format else None + stream = open(book, 'rb') + self.formats.append(format) + self.names.append(os.path.basename(book)) + self.get_metadata(c, stream, stream_type=format, + use_libprs_metadata=True) + self.wait_for_condition() + + + def process_duplicates(self): + if self.duplicates: + files = _('

Books with the same title as the following already ' + 'exist in the database. Add them anyway?

    ') + for mi in self.duplicates[2]: + files += '
  • '+mi.title+'
    • \n' + d = WarningDialog (_('Duplicates found!'), + _('Duplicates found!'), + files+'

', parent=self._parent) + if d.exec_() == d.Accepted: + num = self.db.add_books(*self.duplicates, + **dict(add_duplicates=True))[1] + self.number_of_books_added += num + + +class AddRecursive(Add): + + def __init__(self, path, db, get_metadata, single_book_per_directory, parent): + self.path = path + self.db = db + self.get_metadata = get_metadata + self.single_book_per_directory = single_book_per_directory + self.duplicates, self.books, self.metadata = [], [], [] + self.number_of_books_added = 0 + self.canceled = False + Add.__init__(self) + self.connect(self.get_metadata, + SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), + self.metadata_delivered, Qt.QueuedConnection) + self.connect(self, SIGNAL('searching_done()'), self.searching_done, + Qt.QueuedConnection) + self._parent = parent + self.pd = ProgressDialog(_('Adding books recursively...'), + _('Searching for books in all sub-directories...'), + 0, 0, parent) + self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), + self.update_progress_dialog) + self.connect(self, SIGNAL('update(PyQt_PyObject)'), self.pd.set_msg, + Qt.QueuedConnection) + self.connect(self, SIGNAL('pupdate(PyQt_PyObject)'), self.pd.set_value, + Qt.QueuedConnection) + self.pd.setModal(True) + self.pd.show() + self.connect(self, SIGNAL('finished()'), self.pd.hide) + + def update_progress_dialog(self, title, count): + self.pd.set_value(count) + if title: + self.pd.set_msg(_('Read metadata from ')+title) + + def metadata_delivered(self, id, mi): + if self.is_canceled(): + self.reading.wakeAll() + return + self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), + mi.title, id) + self.metadata.append((mi if mi.title else None, self.books[id])) + if len(self.metadata) >= len(self.books): + self.metadata = [x for x in self.metadata if x[0] is not None] + self.pd.set_min(-1) + self.pd.set_max(len(self.metadata)-1) + self.pd.set_value(-1) + self.pd.set_msg(_('Adding books to database...')) + self.wake_up() + + def searching_done(self): + self.pd.set_min(-1) + self.pd.set_max(len(self.books)-1) + self.pd.set_value(-1) + self.pd.set_msg(_('Reading metadata...')) + + + def run(self): + root = os.path.abspath(self.path) + for dirpath in os.walk(root): + if self.is_canceled(): + return + self.emit(SIGNAL('update(PyQt_PyObject)'), + _('Searching in')+' '+dirpath[0]) + self.books += list(self.db.find_books_in_directory(dirpath[0], + self.single_book_per_directory)) + self.books = [formats for formats in self.books if formats] + # Reset progress bar + self.emit(SIGNAL('searching_done()')) + + for c, formats in enumerate(self.books): + self.get_metadata.from_formats(c, formats) + self.wait_for_condition() + + # Add books to database + for c, x in enumerate(self.metadata): + mi, formats = x + if self.is_canceled(): + break + if self.db.has_book(mi): + self.duplicates.append((mi, formats)) + else: + self.db.import_book(mi, formats, notify=False) + self.number_of_books_added += 1 + self.emit(SIGNAL('pupdate(PyQt_PyObject)'), c) + + + def process_duplicates(self): + if self.duplicates: + files = _('

Books with the same title as the following already ' + 'exist in the database. Add them anyway?

    ') + for mi in self.duplicates: + files += '
  • '+mi[0].title+'
    • \n' + d = WarningDialog (_('Duplicates found!'), + _('Duplicates found!'), + files+'

', parent=self._parent) + if d.exec_() == d.Accepted: + for mi, formats in self.duplicates: + self.db.import_book(mi, formats, notify=False) + self.number_of_books_added += 1 + + \ No newline at end of file diff --git a/src/calibre/gui2/dialogs/confirm_delete.py b/src/calibre/gui2/dialogs/confirm_delete.py index 08db53e9a7..8c496987fb 100644 --- a/src/calibre/gui2/dialogs/confirm_delete.py +++ b/src/calibre/gui2/dialogs/confirm_delete.py @@ -5,7 +5,7 @@ __docformat__ = 'restructuredtext en' from calibre.gui2 import dynamic from calibre.gui2.dialogs.confirm_delete_ui import Ui_Dialog -from PyQt4.Qt import QDialog, SIGNAL +from PyQt4.Qt import QDialog, SIGNAL, Qt def _config_name(name): return name + '_again' @@ -19,6 +19,7 @@ class Dialog(QDialog, Ui_Dialog): self.msg.setText(msg) self.name = name self.connect(self.again, SIGNAL('stateChanged(int)'), self.toggle) + self.buttonBox.setFocus(Qt.OtherFocusReason) def toggle(self, x): diff --git a/src/calibre/gui2/dialogs/progress.py b/src/calibre/gui2/dialogs/progress.py index 2543cefb4d..0f64d7b041 100644 --- a/src/calibre/gui2/dialogs/progress.py +++ b/src/calibre/gui2/dialogs/progress.py @@ -20,6 +20,7 @@ class ProgressDialog(QDialog, Ui_Dialog): self.setWindowModality(Qt.ApplicationModal) self.set_min(min) self.set_max(max) + self.bar.setValue(min) self.canceled = False self.connect(self.button_box, SIGNAL('rejected()'), self._canceled) diff --git a/src/calibre/gui2/images/news/pobjeda.png b/src/calibre/gui2/images/news/pobjeda.png new file mode 100644 index 0000000000000000000000000000000000000000..d7612b4e9ef1866298a80491456c439b1f36d225 GIT binary patch literal 285 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*VR95+uF#}JFt$uf6Z@XI>3~C!+wYBq+`uo zSv(-%Gv$V#D6_=G!wfg24b+7uNmuCiNICd2UzluUBtQSof$GMIm)`I&Y`redYv0m( z1!$sbiEBhja#3nxNvduNkYF$}FtF4$Fw!+N2r;yBooks with the same title as the following already exist in the database. Add them anyway?
    ') - for mi, formats in duplicates: - files += '
  • '+mi.title+'
    • \n' - d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), - files+'

', self) - if d.exec_() == QDialog.Accepted: - for mi, formats in duplicates: - self.library_view.model().db.import_book(mi, formats ) - - self.library_view.model().resort() - self.library_view.model().research() - + from calibre.gui2.add import AddRecursive + self._add_recursive_thread = AddRecursive(root, + self.library_view.model().db, self.get_metadata, + single, self) + self.connect(self._add_recursive_thread, SIGNAL('finished()'), + self._recursive_files_added) + self._add_recursive_thread.start() + + def _recursive_files_added(self): + self._add_recursive_thread.process_duplicates() + if self._add_recursive_thread.number_of_books_added > 0: + self.library_view.model().resort(reset=False) + self.library_view.model().research() + self.library_view.model().count_changed() + self._add_recursive_thread = None + def add_recursive_single(self, checked): ''' Add books from the local filesystem to either the library or the device @@ -686,66 +676,41 @@ class Main(MainWindow, Ui_MainWindow): return to_device = self.stack.currentIndex() != 0 self._add_books(books, to_device) - if to_device: - self.status_bar.showMessage(_('Uploading books to device.'), 2000) + def _add_books(self, paths, to_device, on_card=None): if on_card is None: on_card = self.stack.currentIndex() == 2 if not paths: return - # Get format and metadata information - formats, metadata, names, infos = [], [], [], [] - progress = ProgressDialog(_('Adding books...'), _('Reading metadata...'), - min=0, max=len(paths), parent=self) - progress.show() - try: - for c, book in enumerate(paths): - progress.set_value(c+1) - if progress.canceled: - return - format = os.path.splitext(book)[1] - format = format[1:] if format else None - stream = open(book, 'rb') - try: - mi = get_metadata(stream, stream_type=format, use_libprs_metadata=True) - except: - mi = MetaInformation(None, None) - if not mi.title: - mi.title = os.path.splitext(os.path.basename(book))[0] - if not mi.authors: - mi.authors = [_('Unknown')] - formats.append(format) - metadata.append(mi) - names.append(os.path.basename(book)) - infos.append({'title':mi.title, 'authors':', '.join(mi.authors), - 'cover':self.default_thumbnail, 'tags':[]}) - title = mi.title if isinstance(mi.title, unicode) else mi.title.decode(preferred_encoding, 'replace') - progress.set_msg(_('Read metadata from ')+title) - QApplication.processEvents() - - if not to_device: - progress.set_msg(_('Adding books to database...')) - QApplication.processEvents() - model = self.library_view.model() - - paths = list(paths) - duplicates, number_added = model.add_books(paths, formats, metadata) - if duplicates: - files = _('

Books with the same title as the following already exist in the database. Add them anyway?

    ') - for mi in duplicates[2]: - files += '
  • '+mi.title+'
    • \n' - d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), files+'

', parent=self) - if d.exec_() == QDialog.Accepted: - num = model.add_books(*duplicates, **dict(add_duplicates=True))[1] - number_added += num - model.books_added(number_added) + from calibre.gui2.add import AddFiles + self._add_files_thread = AddFiles(paths, self.default_thumbnail, + self.get_metadata, + None if to_device else \ + self.library_view.model().db + ) + self._add_files_thread.send_to_device = to_device + self._add_files_thread.on_card = on_card + self._add_files_thread.create_progress_dialog(_('Adding books...'), + _('Reading metadata...'), self) + self.connect(self._add_files_thread, SIGNAL('finished()'), + self._files_added) + self._add_files_thread.start() + + def _files_added(self): + t = self._add_files_thread + self._add_files_thread = None + if not t.canceled: + if t.send_to_device: + self.upload_books(t.paths, + list(map(sanitize_file_name, t.names)), + t.infos, on_card=t.on_card) + self.status_bar.showMessage(_('Uploading books to device.'), 2000) else: - self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card) - finally: - QApplication.processEvents() - progress.hide() - + t.process_duplicates() + if t.number_of_books_added > 0: + self.library_view.model().books_added(t.number_of_books_added) + def upload_books(self, files, names, metadata, on_card=False, memory=None): ''' Upload books to device. @@ -801,7 +766,10 @@ class Main(MainWindow, Ui_MainWindow): if not rows or len(rows) == 0: return if self.stack.currentIndex() == 0: - if not confirm('

'+_('The selected books will be permanently deleted and the files removed from your computer. Are you sure?')+'

', 'library_delete_books', self): + if not confirm('

'+_('The selected books will be ' + 'permanently deleted and the files ' + 'removed from your computer. Are you sure?') + +'

', 'library_delete_books', self): return view.model().delete_books(rows) else: @@ -1410,8 +1378,15 @@ class Main(MainWindow, Ui_MainWindow): def initialize_database(self): self.library_path = prefs['library_path'] if self.library_path is None: # Need to migrate to new database layout + base = os.path.expanduser('~') + if iswindows: + from calibre import plugins + from PyQt4.Qt import QDir + base = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_PERSONAL) + if not base or not os.path.exists(base): + base = unicode(QDir.homePath()).replace('/', os.sep) dir = unicode(QFileDialog.getExistingDirectory(self, - _('Choose a location for your ebook library.'), os.getcwd())) + _('Choose a location for your ebook library.'), base)) if not dir: dir = os.path.expanduser('~/Library') self.library_path = os.path.abspath(dir) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index 37fdeb4ce4..e4fa71feed 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -4,13 +4,10 @@ __copyright__ = '2008, Kovid Goyal ' Backend that implements storage of ebooks in an sqlite database. ''' import sqlite3 as sqlite -import datetime, re, os, cPickle, sre_constants +import datetime, re, cPickle, sre_constants from zlib import compress, decompress -from calibre import sanitize_file_name -from calibre.ebooks.metadata.meta import set_metadata, metadata_from_formats from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks import BOOK_EXTENSIONS from calibre.web.feeds.recipes import migrate_automatic_profile_to_automatic_recipe class Concatenate(object): @@ -1391,117 +1388,12 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; - def import_book(self, mi, formats): - series_index = 1 if mi.series_index is None else mi.series_index - if not mi.authors: - mi.authors = [_('Unknown')] - aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors) - obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', - (mi.title, None, series_index, aus)) - id = obj.lastrowid - self.conn.commit() - self.set_metadata(id, mi) - for path in formats: - ext = os.path.splitext(path)[1][1:].lower() - stream = open(path, 'rb') - stream.seek(0, 2) - usize = stream.tell() - stream.seek(0) - data = sqlite.Binary(compress(stream.read())) - try: - self.conn.execute('INSERT INTO data(book, format, uncompressed_size, data) VALUES (?,?,?,?)', - (id, ext, usize, data)) - except sqlite.IntegrityError: - self.conn.execute('UPDATE data SET uncompressed_size=?, data=? WHERE book=? AND format=?', - (usize, data, id, ext)) - self.conn.commit() - - def import_book_directory_multiple(self, dirpath, callback=None): - dirpath = os.path.abspath(dirpath) - duplicates = [] - books = {} - for path in os.listdir(dirpath): - if callable(callback): - callback('.') - path = os.path.abspath(os.path.join(dirpath, path)) - if os.path.isdir(path) or not os.access(path, os.R_OK): - continue - ext = os.path.splitext(path)[1] - if not ext: - continue - ext = ext[1:].lower() - if ext not in BOOK_EXTENSIONS: - continue - - key = os.path.splitext(path)[0] - if not books.has_key(key): - books[key] = [] - - books[key].append(path) - - for formats in books.values(): - mi = metadata_from_formats(formats) - if mi.title is None: - continue - if self.has_book(mi): - duplicates.append((mi, formats)) - continue - self.import_book(mi, formats) - if callable(callback): - if callback(mi.title): - break - return duplicates - - - def import_book_directory(self, dirpath, callback=None): - dirpath = os.path.abspath(dirpath) - formats = [] - for path in os.listdir(dirpath): - if callable(callback): - callback('.') - path = os.path.abspath(os.path.join(dirpath, path)) - if os.path.isdir(path) or not os.access(path, os.R_OK): - continue - ext = os.path.splitext(path)[1] - if not ext: - continue - ext = ext[1:].lower() - if ext not in BOOK_EXTENSIONS: - continue - formats.append(path) - - if not formats: - return - - mi = metadata_from_formats(formats) - if mi.title is None: - return - if self.has_book(mi): - return [(mi, formats)] - self.import_book(mi, formats) - if callable(callback): - callback(mi.title) - - + def has_id(self, id): return self.conn.get('SELECT id FROM books where id=?', (id,), all=False) is not None - def recursive_import(self, root, single_book_per_directory=True, callback=None): - root = os.path.abspath(root) - duplicates = [] - for dirpath in os.walk(root): - res = self.import_book_directory(dirpath[0], callback=callback) if \ - single_book_per_directory else \ - self.import_book_directory_multiple(dirpath[0], callback=callback) - if res is not None: - duplicates.extend(res) - if callable(callback): - if callback(''): - break - - return duplicates - + class SearchToken(object): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 8c762f8680..a7f522cad0 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -12,7 +12,7 @@ from itertools import repeat from datetime import datetime from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock -from PyQt4.QtGui import QApplication, QPixmap, QImage +from PyQt4.QtGui import QApplication, QImage __app = None from calibre.library import title_sort @@ -20,12 +20,15 @@ from calibre.library.database import LibraryDatabase from calibre.library.sqlite import connect, IntegrityError from calibre.utils.search_query_parser import SearchQueryParser from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation -from calibre.ebooks.metadata.meta import get_metadata, set_metadata +from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \ + metadata_from_formats from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import run_plugins_on_import + from calibre import sanitize_file_name +from calibre.ebooks import BOOK_EXTENSIONS copyfile = os.link if hasattr(os, 'link') else shutil.copyfile @@ -627,7 +630,7 @@ class LibraryDatabase2(LibraryDatabase): if not QCoreApplication.instance(): global __app __app = QApplication([]) - p = QPixmap() + p = QImage() if callable(getattr(data, 'read', None)): data = data.read() p.loadFromData(data) @@ -1142,7 +1145,7 @@ class LibraryDatabase2(LibraryDatabase): def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True): ''' Add a book to the database. The result cache is not updated. - @param paths: List of paths to book files or file-like objects + :param:`paths` List of paths to book files or file-like objects ''' formats, metadata, uris = iter(formats), iter(metadata), iter(uris) duplicates = [] @@ -1180,17 +1183,17 @@ class LibraryDatabase2(LibraryDatabase): self.conn.commit() self.data.refresh_ids(self.conn, ids) # Needed to update format list and size if duplicates: - paths = tuple(duplicate[0] for duplicate in duplicates) - formats = tuple(duplicate[1] for duplicate in duplicates) - metadata = tuple(duplicate[2] for duplicate in duplicates) - uris = tuple(duplicate[3] for duplicate in duplicates) + paths = list(duplicate[0] for duplicate in duplicates) + formats = list(duplicate[1] for duplicate in duplicates) + metadata = list(duplicate[2] for duplicate in duplicates) + uris = list(duplicate[3] for duplicate in duplicates) return (paths, formats, metadata, uris), len(ids) return None, len(ids) - def import_book(self, mi, formats): + def import_book(self, mi, formats, notify=True): series_index = 1 if mi.series_index is None else mi.series_index if not mi.authors: - mi.authors = ['Unknown'] + mi.authors = [_('Unknown')] aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors) obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', (mi.title, None, series_index, aus)) @@ -1204,7 +1207,8 @@ class LibraryDatabase2(LibraryDatabase): self.add_format(id, ext, stream, index_is_id=True) self.conn.commit() self.data.refresh_ids(self.conn, [id]) # Needed to update format list and size - self.notify('add', [id]) + if notify: + self.notify('add', [id]) def move_library_to(self, newloc, progress=None): header = _(u'

Copying books to %s

')%newloc @@ -1388,6 +1392,10 @@ books_series_link feeds f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb') if not mi.authors: mi.authors = [_('Unknown')] + cdata = self.cover(id, index_is_id=True) + cname = sanitize_file_name(name)+'.jpg' + open(os.path.join(base, cname), 'wb').write(cdata) + mi.cover = cname opf = OPFCreator(base, mi) opf.render(f) f.close() @@ -1451,6 +1459,88 @@ books_series_link feeds if not callback(count, title): break return failures + + def find_books_in_directory(self, dirpath, single_book_per_directory): + dirpath = os.path.abspath(dirpath) + if single_book_per_directory: + formats = [] + for path in os.listdir(dirpath): + path = os.path.abspath(os.path.join(dirpath, path)) + if os.path.isdir(path) or not os.access(path, os.R_OK): + continue + ext = os.path.splitext(path)[1] + if not ext: + continue + ext = ext[1:].lower() + if ext not in BOOK_EXTENSIONS: + continue + formats.append(path) + yield formats + else: + books = {} + for path in os.listdir(dirpath): + path = os.path.abspath(os.path.join(dirpath, path)) + if os.path.isdir(path) or not os.access(path, os.R_OK): + continue + ext = os.path.splitext(path)[1] + if not ext: + continue + ext = ext[1:].lower() + if ext not in BOOK_EXTENSIONS: + continue + + key = os.path.splitext(path)[0] + if not books.has_key(key): + books[key] = [] + books[key].append(path) + + for formats in books.values(): + yield formats + + def import_book_directory_multiple(self, dirpath, callback=None): + duplicates = [] + for formats in self.find_books_in_directory(dirpath, False): + mi = metadata_from_formats(formats) + if mi.title is None: + continue + if self.has_book(mi): + duplicates.append((mi, formats)) + continue + self.import_book(mi, formats) + if callable(callback): + if callback(mi.title): + break + return duplicates + + def import_book_directory(self, dirpath, callback=None): + dirpath = os.path.abspath(dirpath) + formats = self.find_books_in_directory(dirpath, True) + if not formats: + return + + mi = metadata_from_formats(formats) + if mi.title is None: + return + if self.has_book(mi): + return [(mi, formats)] + self.import_book(mi, formats) + if callable(callback): + callback(mi.title) + + def recursive_import(self, root, single_book_per_directory=True, callback=None): + root = os.path.abspath(root) + duplicates = [] + for dirpath in os.walk(root): + res = self.import_book_directory(dirpath[0], callback=callback) if \ + single_book_per_directory else \ + self.import_book_directory_multiple(dirpath[0], callback=callback) + if res is not None: + duplicates.extend(res) + if callable(callback): + if callback(''): + break + + return duplicates diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index 60ae0761cf..dcbec14687 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -28,6 +28,7 @@ recipe_modules = ['recipe_' + r for r in ( 'la_tercera', 'el_mercurio_chile', 'la_cuarta', 'lanacion_chile', 'la_segunda', 'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz', 'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna', + 'pobjeda', )] import re, imp, inspect, time, os diff --git a/src/calibre/web/feeds/recipes/recipe_pobjeda.py b/src/calibre/web/feeds/recipes/recipe_pobjeda.py new file mode 100644 index 0000000000..9a4dbb0eee --- /dev/null +++ b/src/calibre/web/feeds/recipes/recipe_pobjeda.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2009, Darko Miletic ' + +''' +pobjeda.co.me +''' + +import re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +class Pobjeda(BasicNewsRecipe): + title = 'Pobjeda Online' + __author__ = 'Darko Miletic' + description = 'News from Montenegro' + publisher = 'Pobjeda a.d.' + category = 'news, politics, Montenegro' + language = _('Serbian') + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + remove_javascript = True + encoding = 'utf8' + remove_javascript = True + use_embedded_content = False + INDEX = u'http://www.pobjeda.co.me' + extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}' + + html2lrf_options = [ + '--comment', description + , '--category', category + , '--publisher', publisher + ] + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' + + preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] + + keep_only_tags = [dict(name='div', attrs={'class':'vijest'})] + + remove_tags = [dict(name=['object','link'])] + + feeds = [ + (u'Politika' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=1' ) + ,(u'Ekonomija' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=2' ) + ,(u'Drustvo' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=3' ) + ,(u'Crna Hronika' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=4' ) + ,(u'Kultura' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=5' ) + ,(u'Hronika Podgorice' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=7' ) + ,(u'Feljton' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=8' ) + ,(u'Crna Gora' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=9' ) + ,(u'Svijet' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=202') + ,(u'Ekonomija i Biznis', u'http://www.pobjeda.co.me/dodatak.php?rubrika=11' ) + ,(u'Djeciji Svijet' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=12' ) + ,(u'Kultura i Drustvo' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=13' ) + ,(u'Agora' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=133') + ,(u'Ekologija' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=252') + ] + + def preprocess_html(self, soup): + soup.html['xml:lang'] = 'sr-Latn-ME' + soup.html['lang'] = 'sr-Latn-ME' + mtag = '' + soup.head.insert(0,mtag) + for item in soup.findAll(style=True): + del item['style'] + return soup + + def get_cover_url(self): + cover_url = None + soup = self.index_to_soup(self.INDEX) + cover_item = soup.find('img',attrs={'alt':'Naslovna strana'}) + if cover_item: + cover_url = self.INDEX + cover_item.parent['href'] + return cover_url + + def parse_index(self): + totalfeeds = [] + lfeeds = self.get_feeds() + for feedobj in lfeeds: + feedtitle, feedurl = feedobj + self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl)) + articles = [] + soup = self.index_to_soup(feedurl) + for item in soup.findAll('div', attrs={'class':'vijest'}): + description = self.tag_to_string(item.h2) + atag = item.h1.find('a') + if atag: + url = self.INDEX + '/' + atag['href'] + title = self.tag_to_string(atag) + date = strftime(self.timefmt) + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description':description + }) + totalfeeds.append((feedtitle, articles)) + return totalfeeds + From 83bc5c809690eaa45a2e6217dd554cf2f5ccdda7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 10:40:47 -0800 Subject: [PATCH 04/21] PRS505/700: Dont fail to detect the device if SD card is buggy --- src/calibre/devices/prs505/driver.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 9308af2c5a..fb8bbf5378 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -248,15 +248,20 @@ class PRS505(Device): time.sleep(3) self.open_osx() if self._card_prefix is not None: - cachep = os.path.join(self._card_prefix, self.CACHE_XML) - if not os.path.exists(cachep): - os.makedirs(os.path.dirname(cachep), mode=0777) - f = open(cachep, 'wb') - f.write(u''' + try: + cachep = os.path.join(self._card_prefix, self.CACHE_XML) + if not os.path.exists(cachep): + os.makedirs(os.path.dirname(cachep), mode=0777) + f = open(cachep, 'wb') + f.write(u''' '''.encode('utf8')) - f.close() + f.close() + except: + self._card_prefix = None + import traceback + traceback.print_exc() def set_progress_reporter(self, pr): self.report_progress = pr From 1746aeafe00750113dbce50682010457ad15041e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 13:01:17 -0800 Subject: [PATCH 05/21] When saving to disk, also save the date. When importing from disk (ebook per directory), if an OPF file is present, use that to read metadata, including date. Adds new dependency on python-dateutil. Rewrite the Add books code for greateer speed and also fix the bug causing the Add books progress dialog to sometimes not close on windows. --- installer/linux/freeze.py | 3 ++- installer/osx/freeze.py | 1 + installer/windows/freeze.py | 3 ++- src/calibre/ebooks/metadata/__init__.py | 14 +++++++++----- src/calibre/ebooks/metadata/meta.py | 10 ++++++++-- src/calibre/ebooks/metadata/opf.xml | 2 +- src/calibre/ebooks/metadata/opf2.py | 2 ++ src/calibre/library/database2.py | 22 +++++++++++++++------- src/calibre/trac/plugins/download.py | 1 + 9 files changed, 41 insertions(+), 17 deletions(-) diff --git a/installer/linux/freeze.py b/installer/linux/freeze.py index c381041675..a6151c4931 100644 --- a/installer/linux/freeze.py +++ b/installer/linux/freeze.py @@ -81,7 +81,8 @@ def freeze(): 'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt', 'glib', 'gobject'] - packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg'] + packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg', + 'dateutil'] includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules] diff --git a/installer/osx/freeze.py b/installer/osx/freeze.py index 3ec24d3aba..dbaad72748 100644 --- a/installer/osx/freeze.py +++ b/installer/osx/freeze.py @@ -342,6 +342,7 @@ def main(): 'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*', 'keyword', 'codeop', 'pydoc', 'readline', 'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*', + 'dateutil', ], 'packages' : ['PIL', 'Authorization', 'lxml'], 'excludes' : ['IPython'], diff --git a/installer/windows/freeze.py b/installer/windows/freeze.py index ab58fb669d..56486f6bd5 100644 --- a/installer/windows/freeze.py +++ b/installer/windows/freeze.py @@ -179,7 +179,8 @@ def main(args=sys.argv): 'calibre.ebooks.lrf.fonts.prs500.*', 'PyQt4.QtWebKit', 'PyQt4.QtNetwork', ], - 'packages' : ['PIL', 'lxml', 'cherrypy'], + 'packages' : ['PIL', 'lxml', 'cherrypy', + 'dateutil'], 'excludes' : ["Tkconstants", "Tkinter", "tcl", "_imagingtk", "ImageTk", "FixTk" ], diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index eabd082142..d9b0514362 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -192,7 +192,8 @@ class MetaInformation(object): for attr in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', 'series', 'series_index', 'rating', 'isbn', 'tags', 'cover_data', 'application_id', 'guide', - 'manifest', 'spine', 'toc', 'cover', 'language', 'book_producer'): + 'manifest', 'spine', 'toc', 'cover', 'language', + 'book_producer', 'timestamp'): if hasattr(mi, attr): setattr(ans, attr, getattr(mi, attr)) @@ -217,7 +218,7 @@ class MetaInformation(object): for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', 'series', 'series_index', 'rating', 'isbn', 'language', 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', - 'book_producer', + 'book_producer', 'timestamp' ): setattr(self, x, getattr(mi, x, None)) @@ -235,7 +236,8 @@ class MetaInformation(object): for attr in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', 'series', 'series_index', 'rating', 'isbn', 'application_id', 'manifest', 'spine', 'toc', - 'cover', 'language', 'guide', 'book_producer'): + 'cover', 'language', 'guide', 'book_producer', + 'timestamp'): if hasattr(mi, attr): val = getattr(mi, attr) if val is not None: @@ -276,6 +278,8 @@ class MetaInformation(object): ans += u'Series : '+unicode(self.series) + ' #%s\n'%self.format_series_index() if self.language: ans += u'Language : ' + unicode(self.language) + u'\n' + if self.timestamp is not None: + ans += u'Timestamp : ' + self.timestamp.isoformat(' ') return ans.strip() def to_html(self): @@ -289,12 +293,12 @@ class MetaInformation(object): if self.series: ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())] ans += [(_('Language'), unicode(self.language))] + if self.timestamp is not None: + ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))] for i, x in enumerate(ans): ans[i] = u'%s%s'%x return u'%s
'%u'\n'.join(ans) - - def __str__(self): return self.__unicode__().encode('utf-8') diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index d8116b33d3..1241238f26 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -31,8 +31,14 @@ def metadata_from_formats(formats): mi = MetaInformation(None, None) formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], METADATA_PRIORITIES[path_to_ext(y)])) - for path in formats: - ext = path_to_ext(path) + extensions = list(map(path_to_ext, formats)) + if 'opf' in extensions: + opf = formats[extensions.index('opf')] + mi2 = opf_metadata(opf) + if mi2 is not None and mi2.title: + return mi2 + + for path, ext in zip(formats, extensions): stream = open(path, 'rb') try: mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True)) diff --git a/src/calibre/ebooks/metadata/opf.xml b/src/calibre/ebooks/metadata/opf.xml index 94a8f63b3c..9dab4efbf4 100644 --- a/src/calibre/ebooks/metadata/opf.xml +++ b/src/calibre/ebooks/metadata/opf.xml @@ -10,7 +10,7 @@ ${author} ${'%s (%s)'%(__appname__, __version__)} [http://${__appname__}.kovidgoyal.net] ${mi.application_id} - + ${mi.timestamp.isoformat()} ${mi.language if mi.language else 'UND'} ${mi.category} ${mi.comments} diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 32ba2cb45a..2e3b5ff047 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -12,6 +12,7 @@ from urllib import unquote from urlparse import urlparse from lxml import etree +from dateutil import parser from calibre.ebooks.chardet import xml_to_unicode from calibre import relpath @@ -436,6 +437,7 @@ class OPF(object): series = MetadataField('series', is_dc=False) series_index = MetadataField('series_index', is_dc=False, formatter=int, none_is=1) rating = MetadataField('rating', is_dc=False, formatter=int) + timestamp = MetadataField('date', formatter=parser.parse) def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index a7f522cad0..00da8f3e37 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -380,8 +380,10 @@ class LibraryDatabase2(LibraryDatabase): return row[loc] for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', - 'publisher', 'rating', 'series', 'series_index', 'tags', 'title'): - setattr(self, prop, functools.partial(get_property, loc=FIELD_MAP['comments' if prop == 'comment' else prop])) + 'publisher', 'rating', 'series', 'series_index', 'tags', + 'title', 'timestamp'): + setattr(self, prop, functools.partial(get_property, + loc=FIELD_MAP['comments' if prop == 'comment' else prop])) def initialize_database(self): from calibre.resources import metadata_sqlite @@ -590,6 +592,7 @@ class LibraryDatabase2(LibraryDatabase): mi.author_sort = self.author_sort(idx, index_is_id=index_is_id) mi.comments = self.comments(idx, index_is_id=index_is_id) mi.publisher = self.publisher(idx, index_is_id=index_is_id) + mi.timestamp = self.timestamp(idx, index_is_id=index_is_id) tags = self.tags(idx, index_is_id=index_is_id) if tags: mi.tags = [i.strip() for i in tags.split(',')] @@ -884,6 +887,8 @@ class LibraryDatabase2(LibraryDatabase): self.set_isbn(id, mi.isbn, notify=False) if mi.series_index and mi.series_index > 0: self.set_series_index(id, mi.series_index, notify=False) + if getattr(mi, 'timestamp', None) is not None: + self.set_timestamp(id, mi.timestamp, notify=False) self.set_path(id, True) self.notify('metadata', [id]) @@ -1203,6 +1208,8 @@ class LibraryDatabase2(LibraryDatabase): self.set_metadata(id, mi) for path in formats: ext = os.path.splitext(path)[1][1:].lower() + if ext == 'opf': + continue stream = open(path, 'rb') self.add_format(id, ext, stream, index_is_id=True) self.conn.commit() @@ -1392,10 +1399,11 @@ books_series_link feeds f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb') if not mi.authors: mi.authors = [_('Unknown')] - cdata = self.cover(id, index_is_id=True) - cname = sanitize_file_name(name)+'.jpg' - open(os.path.join(base, cname), 'wb').write(cdata) - mi.cover = cname + cdata = self.cover(int(id), index_is_id=True) + if cdata is not None: + cname = sanitize_file_name(name)+'.jpg' + open(os.path.join(base, cname), 'wb').write(cdata) + mi.cover = cname opf = OPFCreator(base, mi) opf.render(f) f.close() @@ -1472,7 +1480,7 @@ books_series_link feeds if not ext: continue ext = ext[1:].lower() - if ext not in BOOK_EXTENSIONS: + if ext not in BOOK_EXTENSIONS and ext != 'opf': continue formats.append(path) yield formats diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index 63bb4006e1..e63a7d8d0d 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -35,6 +35,7 @@ class Distribution(object): ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'), ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'), ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'), + ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil') ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'), ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'), ] From 97a64f159fb993fe8f8c120400586819bd708640 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 13:38:10 -0800 Subject: [PATCH 06/21] IGN:Mark EPUB output as stable and commit temoprary fix for #1817 --- src/calibre/ebooks/mobi/reader.py | 4 ++-- src/calibre/gui2/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 6811f9ccda..8b81702bc1 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -308,8 +308,8 @@ class MobiReader(object): if 'filepos-id' in attrib: attrib['id'] = attrib.pop('filepos-id') if 'filepos' in attrib: - filepos = int(attrib.pop('filepos')) - attrib['href'] = "#filepos%d" % filepos + filepos = attrib.pop('filepos') + attrib['href'] = "#filepos%s" % filepos if tag.tag == 'img': recindex = None for attr in self.IMAGE_ATTRS: diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index c2e755a742..0e59e35e1e 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -391,7 +391,7 @@ class Main(MainWindow, Ui_MainWindow): def change_output_format(self, x): of = unicode(x).strip() if of != prefs['output_format']: - if of not in ('LRF',): + if of not in ('LRF', 'EPUB'): warning_dialog(self, 'Warning', '

%s support is still in beta. If you find bugs, please report them by opening a ticket.'%of).exec_() prefs.set('output_format', of) From 4fa00105b39a1b802deafaf3c34ca803cddeee16 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 13:45:27 -0800 Subject: [PATCH 07/21] IGN:... --- src/calibre/ebooks/mobi/reader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 8b81702bc1..1c434c9472 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -309,7 +309,10 @@ class MobiReader(object): attrib['id'] = attrib.pop('filepos-id') if 'filepos' in attrib: filepos = attrib.pop('filepos') - attrib['href'] = "#filepos%s" % filepos + try: + attrib['href'] = "#filepos%d" % int(filepos) + except ValueError: + attrib['href'] = filepos if tag.tag == 'img': recindex = None for attr in self.IMAGE_ATTRS: From de1f54d39f7420e147c6cdc27309cd4570f2d24d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 13:45:52 -0800 Subject: [PATCH 08/21] IGN:... --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 1c434c9472..bfbe8f5ae5 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -311,7 +311,7 @@ class MobiReader(object): filepos = attrib.pop('filepos') try: attrib['href'] = "#filepos%d" % int(filepos) - except ValueError: + except: attrib['href'] = filepos if tag.tag == 'img': recindex = None From 5c4294652ef219df1989df78d7089af09850d784 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 13:59:37 -0800 Subject: [PATCH 09/21] Fix system tray icon not being hidden when quitting on windows --- src/calibre/gui2/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 0e59e35e1e..e674ca4fd6 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -1572,6 +1572,11 @@ def main(args=sys.argv): print 'Restarting with:', e, sys.argv os.execvp(e, sys.argv) else: + if iswindows: + try: + main.system_tray_icon.hide() + except: + pass return ret return 0 From 005ca51c8441f729b09cc9373eaadb481db5dbff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 14:23:29 -0800 Subject: [PATCH 10/21] Fix bug in handling of author names with commas when sending books to device --- src/calibre/gui2/library.py | 38 ++++++++++++++----------------------- src/calibre/gui2/main.py | 1 - 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index 9f0877ca09..199c4ada67 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -1,3 +1,4 @@ +from calibre.ebooks.metadata import authors_to_string __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' import os, textwrap, traceback, time, re @@ -371,35 +372,24 @@ class BooksModel(QAbstractTableModel): if not rows_are_ids: rows = [self.db.id(row.row()) for row in rows] for id in rows: - au = self.db.authors(id, index_is_id=True) - tags = self.db.tags(id, index_is_id=True) - if not au: - au = _('Unknown') - au = au.split(',') - if len(au) > 1: - t = ', '.join(au[:-1]) - t += ' & ' + au[-1] - au = t - else: - au = ' & '.join(au) - if not tags: - tags = [] - else: - tags = tags.split(',') - series = self.db.series(id, index_is_id=True) - if series is not None: - tags.append(series) - mi = { - 'title' : self.db.title(id, index_is_id=True), + mi = self.db.get_metadata(id, index_is_id=True) + au = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) + tags = mi.tags if mi.tags else [] + if mi.series is not None: + tags.append(mi.series) + info = { + 'title' : mi.title, 'authors' : au, 'cover' : self.db.cover(id, index_is_id=True), 'tags' : tags, - 'comments': self.db.comments(id, index_is_id=True), + 'comments': mi.comments, } - if series is not None: - mi['tag order'] = {series:self.db.books_in_series_of(id, index_is_id=True)} + if mi.series is not None: + info['tag order'] = { + mi.series:self.db.books_in_series_of(id, index_is_id=True) + } - metadata.append(mi) + metadata.append(info) return metadata def get_preferred_formats_from_ids(self, ids, all_formats, mode='r+b'): diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index e674ca4fd6..83665ac8a7 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -48,7 +48,6 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.library.database2 import LibraryDatabase2, CoverCache from calibre.parallel import JobKilled from calibre.utils.filenames import ascii_filename -from calibre.gui2.widgets import WarningDialog from calibre.gui2.dialogs.confirm_delete import confirm class Main(MainWindow, Ui_MainWindow): From 9991549be6eb776766f47035a7e182cfb5b6b588 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 14:37:15 -0800 Subject: [PATCH 11/21] version 0.4.136 --- src/calibre/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index cbb7fba14e..2d77964693 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = 'calibre' -__version__ = '0.4.135' +__version__ = '0.4.136' __author__ = "Kovid Goyal " ''' Various run time constants. From dd675d62102466c1b72c9facd27ec8a7b21e0aa8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 14:41:54 -0800 Subject: [PATCH 12/21] IGN:Tag release From b18f95c02f013eed0bc6150b3dfe7a157547b1a8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 16:07:19 -0800 Subject: [PATCH 13/21] Fix #1805 (spurious ![endif]>![if> 's found in title and chapter) --- src/calibre/ebooks/html.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/html.py b/src/calibre/ebooks/html.py index e264fec7cb..ad4538b302 100644 --- a/src/calibre/ebooks/html.py +++ b/src/calibre/ebooks/html.py @@ -330,7 +330,8 @@ class PreProcessor(object): sanitize_head), # Convert all entities, since lxml doesn't handle them well (re.compile(r'&(\S+?);'), convert_entities), - + # Remove ]*>'), lambda match: ''), ] # Fix pdftohtml markup From 4a172b34adc586bbf709e7ed6716c3a702236e39 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 16:09:27 -0800 Subject: [PATCH 14/21] IGN:... --- src/calibre/ebooks/html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/html.py b/src/calibre/ebooks/html.py index ad4538b302..f69f26a1e6 100644 --- a/src/calibre/ebooks/html.py +++ b/src/calibre/ebooks/html.py @@ -330,7 +330,7 @@ class PreProcessor(object): sanitize_head), # Convert all entities, since lxml doesn't handle them well (re.compile(r'&(\S+?);'), convert_entities), - # Remove ]*>'), lambda match: ''), ] From 5aa219339dc0a8697cff65009318fc27523133eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 16:14:48 -0800 Subject: [PATCH 15/21] IGN:... --- src/calibre/trac/plugins/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index e63a7d8d0d..7abf7faed4 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -35,7 +35,7 @@ class Distribution(object): ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'), ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'), ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'), - ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil') + ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'), ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'), ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'), ] From 78e5a7d928e2324663dbc1110fc754797671dd96 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 10 Feb 2009 18:19:58 -0800 Subject: [PATCH 16/21] IGN:Mark politika recipe as Serbian --- src/calibre/web/feeds/recipes/recipe_politika.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/web/feeds/recipes/recipe_politika.py b/src/calibre/web/feeds/recipes/recipe_politika.py index 1575d8984f..f1d84915ce 100644 --- a/src/calibre/web/feeds/recipes/recipe_politika.py +++ b/src/calibre/web/feeds/recipes/recipe_politika.py @@ -16,6 +16,7 @@ class Politika(BasicNewsRecipe): publisher = 'Politika novine i Magazini d.o.o' category = 'news, politics, Serbia' oldest_article = 2 + language = _('Serbian') max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False From 7b87aaf55f777895043e678be806658d12701246 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 Feb 2009 01:48:52 -0800 Subject: [PATCH 17/21] IGN:Add donate button to User Manual --- src/calibre/manual/templates/layout.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/calibre/manual/templates/layout.html diff --git a/src/calibre/manual/templates/layout.html b/src/calibre/manual/templates/layout.html new file mode 100644 index 0000000000..3564357684 --- /dev/null +++ b/src/calibre/manual/templates/layout.html @@ -0,0 +1,12 @@ +{% extends "!layout.html" %} +{% block sidebarlogo %} +{{ super() }} +

+ + + + +
+
+{% endblock %} + From ded6c02dba002ae98d60c31c5dcbeb86f489cb0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 Feb 2009 11:24:15 -0800 Subject: [PATCH 18/21] Fix nasty bug in handling of dates in the database. This can prevent calibre from starting. An update is highly recommended --- src/calibre/library/sqlite.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 938e5e665c..cc30f6dd5c 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -12,11 +12,50 @@ from sqlite3 import IntegrityError from threading import Thread from Queue import Queue from threading import RLock +from datetime import tzinfo, datetime, timedelta from calibre.library import title_sort global_lock = RLock() +def convert_timestamp(val): + datepart, timepart = val.split(' ') + tz, mult = None, 1 + x = timepart.split('+') + if len(x) > 1: + timepart, tz = x + else: + x = timepart.split('-') + if len(x) > 1: + timepart, tz = x + mult = -1 + + year, month, day = map(int, datepart.split("-")) + timepart_full = timepart.split(".") + hours, minutes, seconds = map(int, timepart_full[0].split(":")) + if len(timepart_full) == 2: + microseconds = int(timepart_full[1]) + else: + microseconds = 0 + if tz is not None: + h, m = map(int, tz.split(':')) + delta = timedelta(minutes=mult*(60*h + m)) + tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta, + 'dst':lambda self,dt:timedelta(0)})() + + val = datetime(year, month, day, hours, minutes, seconds, microseconds, + tzinfo=tz) + if tz is not None: + val = datetime(*(val.utctimetuple()[:6])) + return val + +def adapt_datetime(dt): + dt = datetime(*(dt.utctimetuple()[:6])) + return dt.isoformat(' ') + +sqlite.register_adapter(datetime, adapt_datetime) +sqlite.register_converter('timestamp', convert_timestamp) + class Concatenate(object): '''String concatenation aggregator for sqlite''' def __init__(self, sep=','): From 759c17bec3971ad01bd7b38c18c393bf1037e77e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 Feb 2009 11:33:25 -0800 Subject: [PATCH 19/21] Fix #1826 (sending to SD card on reader) --- src/calibre/devices/prs505/books.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index b63b089fdd..06d205fb02 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -186,7 +186,10 @@ class BookList(_BookList): node = self.document.createElement(self.prefix + "text") mime = MIME_MAP[name.rpartition('.')[-1].lower()] cid = self.max_id()+1 - sourceid = str(self[0].sourceid) if len(self) else "1" + try: + sourceid = str(self[0].sourceid) if len(self) else '1' + except: + sourceid = '1' attrs = { "title" : info["title"], 'titleSorter' : sortable_title(info['title']), From 2b4a4a31e362e03d5f6e1b1faafa349eb1792857 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 Feb 2009 11:46:30 -0800 Subject: [PATCH 20/21] Fix #1823 (Calibre crashes on import of Mobipocket file) --- src/calibre/library/database2.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 00da8f3e37..388a2d4fdb 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -19,7 +19,8 @@ from calibre.library import title_sort from calibre.library.database import LibraryDatabase from calibre.library.sqlite import connect, IntegrityError from calibre.utils.search_query_parser import SearchQueryParser -from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation +from calibre.ebooks.metadata import string_to_authors, authors_to_string, \ + MetaInformation, authors_to_sort_string from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \ metadata_from_formats from calibre.ebooks.metadata.opf2 import OPFCreator @@ -1197,11 +1198,17 @@ class LibraryDatabase2(LibraryDatabase): def import_book(self, mi, formats, notify=True): series_index = 1 if mi.series_index is None else mi.series_index + if not mi.title: + mi.title = _('Unknown') if not mi.authors: mi.authors = [_('Unknown')] - aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors) + aus = mi.author_sort if mi.author_sort else authors_to_sort_string(mi.authors) + if isinstance(aus, str): + aus = aus.decode(preferred_encoding, 'replace') + title = mi.title if isinstance(mi.title, unicode) else \ + mi.title.decode(preferred_encoding, 'replace') obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', - (mi.title, None, series_index, aus)) + (title, None, series_index, aus)) id = obj.lastrowid self.data.books_added([id], self.conn) self.set_path(id, True) From fbb6a3917eb66ebce310edd06847158928dcc241 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 11 Feb 2009 13:26:38 -0800 Subject: [PATCH 21/21] IGN:Show meaningful error if user tries to set cover from format with no format selected --- src/calibre/gui2/dialogs/metadata_single.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 385e105c3a..78415f3a19 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -113,6 +113,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): def set_cover(self): row = self.formats.currentRow() fmt = self.formats.item(row) + if fmt is None: + error_dialog(self, _('No format selected'), + _('No format selected')).exec_() + return ext = fmt.ext.lower() if fmt.path is None: stream = self.db.format(self.row, ext, as_file=True) @@ -121,7 +125,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): try: mi = get_metadata(stream, ext) except: - error_dialog(self, _('Could not read metadata'), + error_dialog(self, _('Could not read metadata'), _('Could not read metadata from %s format')%ext).exec_() return cdata = None