From 196a4231bb3b5f04c1a3d3a0d4bccf1f37b34101 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 30 Jun 2010 18:04:31 +0100 Subject: [PATCH 01/15] Add 'size' as a search term. Permit 'K', 'M', 'G' as qualtifiers --- src/calibre/library/caches.py | 7 ++++++- src/calibre/library/field_metadata.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 06cf07bb67..d68c81931f 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -341,8 +341,13 @@ class ResultCache(SearchQueryParser): cast = lambda x : float (x) adjust = lambda x: x + if len(query) > 1: + mult = query[-1:].lower() + if mult in ['k', 'm', 'g']: + query = query[:-1] + mult = {'k':1024., 'm': 1024.*1024, 'g': 1024.*1024*1024}[mult] try: - q = cast(query) + q = cast(query) * mult except: return matches diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 5ccc17d1eb..626683fee5 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -253,7 +253,7 @@ class FieldMetadata(dict): 'is_multiple':None, 'kind':'field', 'name':None, - 'search_terms':[], + 'search_terms':['size'], 'is_custom':False, 'is_category':False}), ('timestamp', {'table':None, From 8314a05a1ac595397d42de55f45ba17b9745b3c5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 30 Jun 2010 18:10:59 +0100 Subject: [PATCH 02/15] Fix uninitialized variable (mult) problem --- src/calibre/library/caches.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index d68c81931f..6716c1c491 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -343,9 +343,11 @@ class ResultCache(SearchQueryParser): if len(query) > 1: mult = query[-1:].lower() - if mult in ['k', 'm', 'g']: + mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + if mult != 1.0: query = query[:-1] - mult = {'k':1024., 'm': 1024.*1024, 'g': 1024.*1024*1024}[mult] + else: + mult = 1.0 try: q = cast(query) * mult except: From f124ad0b47fa9e0e433442a16af0e385cd27616f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 13:11:48 -0600 Subject: [PATCH 03/15] PDF metadata: Fix last character corrupted when setting metadata in encrypted files. The fix required a workaround, which means that setting metadata in encrypted PDF files will only work for values of metadata that can be encoded in the cp-1252 encoding. --- src/calibre/utils/podofo/podofo.cpp | 30 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/calibre/utils/podofo/podofo.cpp b/src/calibre/utils/podofo/podofo.cpp index d88e71628d..ace8c58c70 100644 --- a/src/calibre/utils/podofo/podofo.cpp +++ b/src/calibre/utils/podofo/podofo.cpp @@ -6,15 +6,10 @@ #include using namespace PoDoFo; -class podofo_pdfmem_wrapper : public PdfMemDocument { - public: - inline void set_info(PdfInfo *i) { this->SetInfo(i); } -}; - typedef struct { PyObject_HEAD /* Type-specific fields go here. */ - podofo_pdfmem_wrapper *doc; + PdfMemDocument *doc; } podofo_PDFDoc; @@ -33,7 +28,7 @@ podofo_PDFDoc_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self = (podofo_PDFDoc *)type->tp_alloc(type, 0); if (self != NULL) { - self->doc = new podofo_pdfmem_wrapper(); + self->doc = new PdfMemDocument(); if (self->doc == NULL) { Py_DECREF(self); return NULL; } } @@ -171,6 +166,19 @@ podofo_convert_pystring(PyObject *py) { return ans; } +static PdfString * +podofo_convert_pystring_single_byte(PyObject *py) { + Py_UNICODE* u = PyUnicode_AS_UNICODE(py); + PyObject *s = PyUnicode_Encode(u, PyUnicode_GET_SIZE(py), "cp1252", "replace"); + if (s == NULL) { PyErr_NoMemory(); return NULL; } + PdfString *ans = new PdfString(PyString_AS_STRING(s)); + Py_DECREF(s); + if (ans == NULL) PyErr_NoMemory(); + return ans; +} + + + static PyObject * podofo_PDFDoc_getter(podofo_PDFDoc *self, int field) { @@ -219,7 +227,10 @@ podofo_PDFDoc_setter(podofo_PDFDoc *self, PyObject *val, int field) { PyErr_SetString(PyExc_Exception, "You must first load a PDF Document"); return -1; } - PdfString *s = podofo_convert_pystring(val); + PdfString *s = NULL; + + if (self->doc->GetEncrypted()) s = podofo_convert_pystring_single_byte(val); + else s = podofo_convert_pystring(val); if (s == NULL) return -1; @@ -241,9 +252,6 @@ podofo_PDFDoc_setter(podofo_PDFDoc *self, PyObject *val, int field) { return -1; } - - self->doc->set_info(info); - return 0; } From 551bc933f9a09bfc6fe5a7666ed8be71dc72d867 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 13:17:22 -0600 Subject: [PATCH 04/15] ... --- src/calibre/manual/gui.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index dd0451c087..d27f09705f 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -213,14 +213,14 @@ isbn, date, pubdate, search``. The syntax for searching for dates and publication dates is:: pubdate:>2000-1 Will find all books published after Jan, 2000 - date:<=2000-1-3 Will find all books added to calibre beforre 3 Jan, 2000 + date:<=2000-1-3 Will find all books added to calibre before 3 Jan, 2000 pubdate:=2009 Will find all books published in 2009 The special field ``search`` is used for saved searches. So if you save a search with the name "My spouse's books" you can enter ``search:"My spouses' books"`` in the search bar to reuse the saved search. More about saving searches, below. -You can search for the absence or presnce of a filed using the specia "true" and "false" values. For example:: +You can search for the absence or presence of a field using the special "true" and "false" values. For example:: cover:false Will give you all books without a cover series:true Will give you all books that belong to a series From 1a5f73d86d03684d42f60cbab4a2b026795a0478 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 13:21:18 -0600 Subject: [PATCH 05/15] Add size searching to user manual --- src/calibre/manual/gui.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index d27f09705f..02018c0b02 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -208,7 +208,7 @@ You can build advanced search queries easily using the :guilabel:`Advanced Searc clicking the button |sbi|. Available fields for searching are: ``tag, title, author, publisher, series, rating cover, comments, format, -isbn, date, pubdate, search``. +isbn, date, pubdate, search, size``. To find the search name for a custom column, hover your mouse over the column header. The syntax for searching for dates and publication dates is:: @@ -216,6 +216,11 @@ The syntax for searching for dates and publication dates is:: date:<=2000-1-3 Will find all books added to calibre before 3 Jan, 2000 pubdate:=2009 Will find all books published in 2009 +You can search for books that have a format of a certain size like this:: + + size:>1.1M Will find books with a format larger than 1.1MB + size:<=1K Will find books with a format smaller than 1KB + The special field ``search`` is used for saved searches. So if you save a search with the name "My spouse's books" you can enter ``search:"My spouses' books"`` in the search bar to reuse the saved search. More about saving searches, below. From 71c8f750e7c3e44357be06bb704385a7c1a6f175 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 13:39:18 -0600 Subject: [PATCH 06/15] Tester for PoDoFo --- src/calibre/utils/podofo/test.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/calibre/utils/podofo/test.cpp diff --git a/src/calibre/utils/podofo/test.cpp b/src/calibre/utils/podofo/test.cpp new file mode 100644 index 0000000000..fb719fd0cc --- /dev/null +++ b/src/calibre/utils/podofo/test.cpp @@ -0,0 +1,26 @@ +#define USING_SHARED_PODOFO +#include +#include + +using namespace PoDoFo; +using namespace std; + + +int main(int argc, char **argv) { + if (argc < 2) return 1; + char *fname = argv[1]; + + PdfMemDocument doc(fname); + PdfInfo* info = doc.GetInfo(); + cout << endl; + cout << "is encrypted: " << doc.GetEncrypted() << endl; + PdfString old_title = info->GetTitle(); + cout << "is hex: " << old_title.IsHex() << endl; + PdfString new_title(reinterpret_cast("\0z\0z\0z"), 3); + cout << "is new unicode: " << new_title.IsUnicode() << endl; + info->SetTitle(new_title); + + doc.Write("/t/x.pdf"); + cout << "Output written to: " << "/t/x.pdf" << endl; + return 0; +} From d000fbc7b242ba5d2f95e3973db7fb0c331573aa Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 30 Jun 2010 21:05:09 +0100 Subject: [PATCH 07/15] Changes to search section of GUI manual. --- src/calibre/manual/gui.rst | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/calibre/manual/gui.rst b/src/calibre/manual/gui.rst index 02018c0b02..613eb82559 100644 --- a/src/calibre/manual/gui.rst +++ b/src/calibre/manual/gui.rst @@ -199,36 +199,59 @@ Searches are by default 'contains'. An item matches if the search string appears Two other kinds of searches are available: equality search and search using regular expressions. Equality searches are indicated by prefixing the search string with an equals sign (=). For example, the query -``tag:"=science"`` will match "science", but not "science fiction". Regular expression searches are +``tag:"=science"`` will match "science", but not "science fiction" or "hard science". Regular expression searches are indicated by prefixing the search string with a tilde (~). Any python-compatible regular expression can be used. Regular expression searches are contains searches unless the expression contains anchors. Should you need to search for a string with a leading equals or tilde, prefix the string with a backslash. +Enclose search strings with quotes (") if the string contains parenthesis or spaces. For example, to search +for the tag ``Science Fiction``, you would need to search for ``tag:"=science fiction"``. If you search for +``tag:=science fiction``, you will find all books with the tag 'science' and containing the word 'fiction' in any +metadata. + You can build advanced search queries easily using the :guilabel:`Advanced Search Dialog`, accessed by clicking the button |sbi|. -Available fields for searching are: ``tag, title, author, publisher, series, rating cover, comments, format, -isbn, date, pubdate, search, size``. To find the search name for a custom column, hover your mouse over the column header. +Available fields for searching are: ``tag, title, author, publisher, series, rating, cover, comments, format, +isbn, date, pubdate, search, size`` and custom columns. If a device is plugged in, the ``ondevice`` field +becomes available. To find the search name for a custom column, hover your mouse over the column header. -The syntax for searching for dates and publication dates is:: +The syntax for searching for dates is:: pubdate:>2000-1 Will find all books published after Jan, 2000 date:<=2000-1-3 Will find all books added to calibre before 3 Jan, 2000 pubdate:=2009 Will find all books published in 2009 +If the date is ambiguous, the current locale is used for date comparison. For example, in an mm/dd/yyyy +locale, 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale, it is interpreted as 2 Jan 2009. + +Some special date strings are available. The string ``today`` translates to today's date, whatever it is. The +strings `yesterday`` and ``thismonth`` also work. In addition, the string ``daysago`` can be used to compare +to a date some number of days ago, for example: date:>10daysago, date:<=45daysago. + You can search for books that have a format of a certain size like this:: size:>1.1M Will find books with a format larger than 1.1MB size:<=1K Will find books with a format smaller than 1KB +Dates and numeric fields support the operators ``=`` (equals), ``>`` (greater than), ``>=`` (greater than or +equal to), ``<`` (less than), ``<=`` (less than or equal to), and ``!=`` (not equal to). Rating fields are +considered to be numeric. For example, the search ``rating:>=3`` will find all books rated 3 or higher. + The special field ``search`` is used for saved searches. So if you save a search with the name -"My spouse's books" you can enter ``search:"My spouses' books"`` in the search bar to reuse the saved +"My spouse's books" you can enter ``search:"My spouse's books"`` in the search bar to reuse the saved search. More about saving searches, below. You can search for the absence or presence of a field using the special "true" and "false" values. For example:: - cover:false Will give you all books without a cover - series:true Will give you all books that belong to a series + cover:false will give you all books without a cover + series:true will give you all books that belong to a series + comments:false will give you all books with an empty comment + +Yes/no custom columns are searchable. Searching for ``false``, ``empty``, or ``blank`` will find all books +with undefined values in the column. Searching for ``true`` will find all books that do not have undefined +values in the column. Searching for ``yes`` or ``checked`` will find all books with ``Yes`` in the column. +Searching for ``no`` or ``unchecked`` will find all books with ``No`` in the column. .. |sbi| image:: images/search_button.png :align: middle From 8e00c3bc9c20476d3a51018774fb6131e843e0eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 14:25:16 -0600 Subject: [PATCH 08/15] Cleanup --- src/calibre/ebooks/metadata/pdf.py | 62 ++---------------------------- 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/src/calibre/ebooks/metadata/pdf.py b/src/calibre/ebooks/metadata/pdf.py index b4bc6f962f..2d1935539e 100644 --- a/src/calibre/ebooks/metadata/pdf.py +++ b/src/calibre/ebooks/metadata/pdf.py @@ -8,7 +8,7 @@ from functools import partial from calibre import prints from calibre.constants import plugins -from calibre.ebooks.metadata import MetaInformation, string_to_authors, authors_to_string +from calibre.ebooks.metadata import MetaInformation, string_to_authors pdfreflow, pdfreflow_error = plugins['pdfreflow'] @@ -56,66 +56,10 @@ def get_metadata(stream, cover=True): get_quick_metadata = partial(get_metadata, cover=False) -import cStringIO -from threading import Thread - -from calibre.utils.pdftk import set_metadata as pdftk_set_metadata -from calibre.utils.podofo import set_metadata as podofo_set_metadata, Unavailable +from calibre.utils.podofo import set_metadata as podofo_set_metadata def set_metadata(stream, mi): stream.seek(0) - try: - return podofo_set_metadata(stream, mi) - except Unavailable: - pass - try: - return pdftk_set_metadata(stream, mi) - except: - pass - set_metadata_pypdf(stream, mi) - - -class MetadataWriter(Thread): - - def __init__(self, out_pdf, buf): - self.out_pdf = out_pdf - self.buf = buf - Thread.__init__(self) - self.daemon = True - - def run(self): - try: - self.out_pdf.write(self.buf) - except RuntimeError: - pass - -def set_metadata_pypdf(stream, mi): - # Use a StringIO object for the pdf because we will want to over - # write it later and if we are working on the stream directly it - # could cause some issues. - - from pyPdf import PdfFileReader, PdfFileWriter - raw = cStringIO.StringIO(stream.read()) - orig_pdf = PdfFileReader(raw) - title = mi.title if mi.title else orig_pdf.documentInfo.title - author = authors_to_string(mi.authors) if mi.authors else orig_pdf.documentInfo.author - out_pdf = PdfFileWriter(title=title, author=author) - out_str = cStringIO.StringIO() - writer = MetadataWriter(out_pdf, out_str) - for page in orig_pdf.pages: - out_pdf.addPage(page) - writer.start() - writer.join(10) # Wait 10 secs for writing to complete - out_pdf.killed = True - writer.join() - if out_pdf.killed: - print 'Failed to set metadata: took too long' - return - - stream.seek(0) - stream.truncate() - out_str.seek(0) - stream.write(out_str.read()) - stream.seek(0) + return podofo_set_metadata(stream, mi) From e280121bec00e20397f2d111a37cf511c6884d88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 14:39:19 -0600 Subject: [PATCH 09/15] ... --- src/calibre/gui2/add.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 64743e914b..74f5a2148e 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -15,7 +15,7 @@ from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding, filesystem_encoding from calibre.utils.config import prefs -class DuplicatesAdder(QThread): +class DuplicatesAdder(QThread): # {{{ # Add duplicate books def __init__(self, parent, db, duplicates, db_adder): QThread.__init__(self, parent) @@ -34,9 +34,9 @@ class DuplicatesAdder(QThread): self.emit(SIGNAL('added(PyQt_PyObject)'), count) count += 1 self.emit(SIGNAL('adding_done()')) +# }}} - -class RecursiveFind(QThread): +class RecursiveFind(QThread): # {{{ def __init__(self, parent, db, root, single): QThread.__init__(self, parent) @@ -79,7 +79,9 @@ class RecursiveFind(QThread): if not self.canceled: self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) -class DBAdder(Thread): +# }}} + +class DBAdder(Thread): # {{{ def __init__(self, db, ids, nmap): self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap) @@ -219,8 +221,9 @@ class DBAdder(Thread): self.db.add_format(id, fmt, f, index_is_id=True, notify=False, replace=replace) +# }}} -class Adder(QObject): +class Adder(QObject): # {{{ ADD_TIMEOUT = 600 # seconds @@ -410,6 +413,7 @@ class Adder(QObject): return getattr(getattr(self, 'db_adder', None), 'infos', []) +# }}} ############################################################################### ############################## END ADDER ###################################### From c43277eae08ddbc9815792c3f3a8c38172fa6303 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 16:34:08 -0600 Subject: [PATCH 10/15] Simplify implementation of cover caching and ensure cover browser is updated when covers are changed --- src/calibre/gui2/actions.py | 5 +- src/calibre/gui2/library/models.py | 26 ++--- src/calibre/gui2/ui.py | 10 +- src/calibre/gui2/widgets.py | 6 +- src/calibre/library/caches.py | 167 +++++++++++------------------ src/calibre/library/database2.py | 34 ++++-- 6 files changed, 114 insertions(+), 134 deletions(-) diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py index 43a657ae67..5dde2f745b 100644 --- a/src/calibre/gui2/actions.py +++ b/src/calibre/gui2/actions.py @@ -645,6 +645,8 @@ class EditMetadataAction(object): # {{{ if x.exception is None: self.library_view.model().refresh_ids( x.updated, cr) + if self.cover_flow: + self.cover_flow.dataChanged() if x.failures: details = ['%s: %s'%(title, reason) for title, reason in x.failures.values()] @@ -689,7 +691,6 @@ class EditMetadataAction(object): # {{{ if rows: current = self.library_view.currentIndex() m = self.library_view.model() - m.refresh_cover_cache(map(m.id, rows)) if self.cover_flow: self.cover_flow.dataChanged() m.current_changed(current, previous) @@ -711,6 +712,8 @@ class EditMetadataAction(object): # {{{ self.library_view.model().resort(reset=False) self.library_view.model().research() self.tags_view.recount() + if self.cover_flow: + self.cover_flow.dataChanged() # Merge books {{{ def merge_books(self, safe_merge=False): diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 508b3e591c..c487a8a252 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -20,7 +20,8 @@ from calibre.utils.config import tweaks, prefs from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser -from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ + REGEXP_MATCH, CoverCache from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml from calibre.constants import filesystem_encoding @@ -149,21 +150,22 @@ class BooksModel(QAbstractTableModel): # {{{ self.build_data_convertors() self.reset() self.database_changed.emit(db) + if self.cover_cache is not None: + self.cover_cache.stop() + self.cover_cache = CoverCache(db) + self.cover_cache.start() + def refresh_cover(event, ids): + if event == 'cover' and self.cover_cache is not None: + self.cover_cache.refresh(ids) + db.add_listener(refresh_cover) def refresh_ids(self, ids, current_row=-1): rows = self.db.refresh_ids(ids) if rows: self.refresh_rows(rows, current_row=current_row) - def refresh_cover_cache(self, ids): - if self.cover_cache: - self.cover_cache.refresh(ids) - def refresh_rows(self, rows, current_row=-1): for row in rows: - if self.cover_cache: - id = self.db.id(row) - self.cover_cache.refresh([id]) if row == current_row: self.new_bookdisplay_data.emit( self.get_book_display_info(row)) @@ -326,7 +328,7 @@ class BooksModel(QAbstractTableModel): # {{{ def set_cache(self, idx): l, r = 0, self.count()-1 - if self.cover_cache: + if self.cover_cache is not None: l = max(l, idx-self.buffer_size) r = min(r, idx+self.buffer_size) k = min(r-idx, idx-l) @@ -494,11 +496,9 @@ class BooksModel(QAbstractTableModel): # {{{ data = None try: id = self.db.id(row_number) - if self.cover_cache: + if self.cover_cache is not None: img = self.cover_cache.cover(id) - if img: - if img.isNull(): - img = self.default_image + if not img.isNull(): return img if not data: data = self.db.cover(row_number) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index c2ff99932d..16a754fab7 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -38,7 +38,6 @@ from calibre.gui2.dialogs.config import ConfigDialog from calibre.gui2.dialogs.book_info import BookInfo from calibre.library.database2 import LibraryDatabase2 -from calibre.library.caches import CoverCache from calibre.gui2.init import ToolbarMixin, LibraryViewMixin, LayoutMixin from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin @@ -138,6 +137,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ self.restriction_in_effect = False self.progress_indicator = ProgressIndicator(self) + self.progress_indicator.pos = (0, 20) self.verbose = opts.verbose self.get_metadata = GetMetadata() self.upload_memory = {} @@ -230,9 +230,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ if self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() - self.cover_cache = CoverCache(self.library_path) - self.cover_cache.start() - self.library_view.model().cover_cache = self.cover_cache self.library_view.model().count_changed_signal.connect \ (self.location_view.count_changed) if not gprefs.get('quick_start_guide_added', False): @@ -606,9 +603,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False - self.cover_cache.stop() + cc = self.library_view.model().cover_cache + if cc is not None: + cc.stop() self.hide_windows() - self.cover_cache.terminate() self.emailer.stop() try: try: diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index 8857945027..97538d9a63 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -38,12 +38,16 @@ class ProgressIndicator(QWidget): self.status.setWordWrap(True) self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop) self.setVisible(False) + self.pos = None def start(self, msg=''): view = self.parent() pwidth, pheight = view.size().width(), view.size().height() self.resize(pwidth, min(pheight, 250)) - self.move(0, (pheight-self.size().height())/2.) + if self.pos is None: + self.move(0, (pheight-self.size().height())/2.) + else: + self.move(self.pos[0], self.pos[1]) self.pi.resize(self.pi.sizeHint()) self.pi.move(int((self.size().width()-self.pi.size().width())/2.), 0) self.status.resize(self.size().width(), self.size().height()-self.pi.size().height()-10) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 6716c1c491..d46ae23d90 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,11 +6,13 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import collections, glob, os, re, itertools, functools +import re, itertools, functools from itertools import repeat from datetime import timedelta +from threading import Thread, RLock +from Queue import Queue, Empty -from PyQt4.Qt import QThread, QReadWriteLock, QImage, Qt +from PyQt4.Qt import QImage, Qt from calibre.utils.config import tweaks from calibre.utils.date import parse_date, now, UNDEFINED_DATE @@ -19,120 +21,73 @@ from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort from calibre import fit_image -class CoverCache(QThread): +class CoverCache(Thread): - def __init__(self, library_path, parent=None): - QThread.__init__(self, parent) - self.library_path = library_path - self.id_map = None - self.id_map_lock = QReadWriteLock() - self.load_queue = collections.deque() - self.load_queue_lock = QReadWriteLock(QReadWriteLock.Recursive) - self.cache = {} - self.cache_lock = QReadWriteLock() - self.id_map_stale = True + def __init__(self, db): + Thread.__init__(self) + self.daemon = True + self.db = db + self.load_queue = Queue() self.keep_running = True - - def build_id_map(self): - self.id_map_lock.lockForWrite() - self.id_map = {} - for f in glob.glob(os.path.join(self.library_path, '*', '* (*)', 'cover.jpg')): - c = os.path.basename(os.path.dirname(f)) - try: - id = int(re.search(r'\((\d+)\)', c[c.rindex('('):]).group(1)) - self.id_map[id] = f - except: - continue - self.id_map_lock.unlock() - self.id_map_stale = False - - - def set_cache(self, ids): - self.cache_lock.lockForWrite() - already_loaded = set([]) - for id in self.cache.keys(): - if id in ids: - already_loaded.add(id) - else: - self.cache.pop(id) - self.cache_lock.unlock() - ids = [i for i in ids if i not in already_loaded] - self.load_queue_lock.lockForWrite() - self.load_queue = collections.deque(ids) - self.load_queue_lock.unlock() - - - def run(self): - while self.keep_running: - if self.id_map is None or self.id_map_stale: - self.build_id_map() - while True: # Load images from the load queue - self.load_queue_lock.lockForWrite() - try: - id = self.load_queue.popleft() - except IndexError: - break - finally: - self.load_queue_lock.unlock() - - self.cache_lock.lockForRead() - need = True - if id in self.cache.keys(): - need = False - self.cache_lock.unlock() - if not need: - continue - path = None - self.id_map_lock.lockForRead() - if id in self.id_map.keys(): - path = self.id_map[id] - else: - self.id_map_stale = True - self.id_map_lock.unlock() - if path and os.access(path, os.R_OK): - try: - img = QImage() - data = open(path, 'rb').read() - img.loadFromData(data) - if img.isNull(): - continue - scaled, nwidth, nheight = fit_image(img.width(), - img.height(), 600, 800) - if scaled: - img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio, - Qt.SmoothTransformation) - except: - continue - self.cache_lock.lockForWrite() - self.cache[id] = img - self.cache_lock.unlock() - - self.sleep(1) + self.cache = {} + self.lock = RLock() + self.null_image = QImage() def stop(self): self.keep_running = False - def cover(self, id): - val = None - if self.cache_lock.tryLockForRead(50): - val = self.cache.get(id, None) - self.cache_lock.unlock() - return val + def _image_for_id(self, id_): + img = self.db.cover(id_, index_is_id=True, as_image=True) + if img is None: + img = QImage() + if not img.isNull(): + scaled, nwidth, nheight = fit_image(img.width(), + img.height(), 600, 800) + if scaled: + img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio, + Qt.SmoothTransformation) + + return img + + def run(self): + while self.keep_running: + try: + id_ = self.load_queue.get(True, 1) + except Empty: + continue + try: + img = self._image_for_id(id_) + except: + import traceback + traceback.print_exc() + continue + with self.lock: + self.cache[id_] = img + + def set_cache(self, ids): + with self.lock: + already_loaded = set([]) + for id in self.cache.keys(): + if id in ids: + already_loaded.add(id) + else: + self.cache.pop(id) + for id_ in set(ids) - already_loaded: + self.load_queue.put(id_) + + def cover(self, id_): + with self.lock: + return self.cache.get(id_, self.null_image) def clear_cache(self): - self.cache_lock.lockForWrite() - self.cache = {} - self.cache_lock.unlock() + with self.lock: + self.cache = {} def refresh(self, ids): - self.cache_lock.lockForWrite() - for id in ids: - self.cache.pop(id, None) - self.cache_lock.unlock() - self.load_queue_lock.lockForWrite() - for id in ids: - self.load_queue.appendleft(id) - self.load_queue_lock.unlock() + with self.lock: + for id_ in ids: + self.cache.pop(id_, None) + self.load_queue.put(id_) ### Global utility function for get_match here and in gui2/library.py CONTAINS_MATCH = 0 diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 9f9488a052..1534d3ffbf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en' ''' The database used to store ebook metadata ''' -import os, sys, shutil, cStringIO, glob,functools, traceback +import os, sys, shutil, cStringIO, glob, time, functools, traceback from itertools import repeat from math import floor @@ -440,12 +440,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if os.access(path, os.R_OK): if as_path: return path - f = open(path, 'rb') + try: + f = open(path, 'rb') + except (IOError, OSError): + time.sleep(0.2) + f = open(path, 'rb') if as_image: img = QImage() img.loadFromData(f.read()) + f.close() return img - return f if as_file else f.read() + ans = f if as_file else f.read() + if ans is not f: + f.close() + return ans def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' @@ -492,12 +500,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') return os.access(path, os.R_OK) - def remove_cover(self, id): + def remove_cover(self, id, notify=True): path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.path.exists(path): - os.remove(path) + try: + os.remove(path) + except (IOError, OSError): + time.sleep(0.2) + os.remove(path) + if notify: + self.notify('cover', [id]) - def set_cover(self, id, data): + def set_cover(self, id, data, notify=True): ''' Set the cover for this book. @@ -509,7 +523,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: if callable(getattr(data, 'read', None)): data = data.read() - save_cover_data_to(data, path) + try: + save_cover_data_to(data, path) + except (IOError, OSError): + time.sleep(0.2) + save_cover_data_to(data, path) + if notify: + self.notify('cover', [id]) def book_on_device(self, id): if callable(self.book_on_device_func): From 75e6bd1452ad3b80ec8e4d49fdfaaddfbaa91ffa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 21:35:48 -0600 Subject: [PATCH 11/15] ... --- src/calibre/ebooks/metadata/opf2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 0bb0c570ed..36588471f2 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1069,8 +1069,10 @@ class OPFCreator(MetaInformation): dc_attrs={'id':__appname__+'_id'})) if getattr(self, 'pubdate', None) is not None: a(DC_ELEM('date', self.pubdate.isoformat())) - a(DC_ELEM('language', self.language if self.language else - get_lang().replace('_', '-'))) + lang = self.language + if not lang or lang.lower() == 'und': + lang = get_lang().replace('_', '-') + a(DC_ELEM('language', lang)) if self.comments: a(DC_ELEM('description', self.comments)) if self.publisher: From 536bd9a31bcb97f667c83baf17cf09d74738be50 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 22:18:40 -0600 Subject: [PATCH 12/15] Fix #5949 (Writing PDF metadata should spin Jobs spinner prior to upload) --- src/calibre/gui2/device.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index c5e042d3e3..4acde6089b 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -31,6 +31,8 @@ from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.devices.apple.driver import ITUNES_ASYNC from calibre.devices.folder_device.driver import FOLDER_DEVICE +from calibre.ebooks.metadata.meta import set_metadata +from calibre.constants import DEBUG # }}} @@ -304,6 +306,21 @@ class DeviceManager(Thread): # {{{ def _upload_books(self, files, names, on_card=None, metadata=None): '''Upload books to device: ''' + if metadata and files and len(metadata) == len(files): + for f, mi in zip(files, metadata): + if isinstance(f, unicode): + ext = f.rpartition('.')[-1].lower() + if ext: + try: + if DEBUG: + prints('Setting metadata in:', mi.title, 'at:', + f, file=sys.__stdout__) + with open(f, 'r+b') as stream: + set_metadata(stream, mi, stream_type=ext) + except: + if DEBUG: + prints(traceback.format_exc(), file=sys.__stdout__) + return self.device.upload_books(files, names, on_card, metadata=metadata, end_session=False) @@ -1145,7 +1162,6 @@ class DeviceMixin(object): # {{{ _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids, settings.format_map, - set_metadata=True, specific_format=specific_format, exclude_auto=do_auto_convert) if do_auto_convert: From 5a1731a38f92f775546da458aa90a8c6cce2cfcc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 22:43:40 -0600 Subject: [PATCH 13/15] PoDoFo cleanups --- setup/installer/linux/freeze.py | 1 + src/calibre/utils/podofo/__init__.py | 9 ++++++++- src/calibre/utils/podofo/podofo.cpp | 6 ++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup/installer/linux/freeze.py b/setup/installer/linux/freeze.py index 08237b83fa..8c56ed4fb7 100644 --- a/setup/installer/linux/freeze.py +++ b/setup/installer/linux/freeze.py @@ -40,6 +40,7 @@ class LinuxFreeze(Command): '/usr/bin/pdftohtml', '/usr/lib/libwmflite-0.2.so.7', '/usr/lib/liblcms.so.1', + '/usr/lib/libstlport.so.5.1', '/tmp/calibre-mount-helper', '/usr/lib/libunrar.so', '/usr/lib/libchm.so.0', diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 9fc0c981e6..284deb7c43 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -14,6 +14,7 @@ from calibre.ebooks.metadata import MetaInformation, string_to_authors, \ from calibre.utils.ipc.job import ParallelJob from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryFile +from calibre import prints podofo, podofo_err = plugins['podofo'] @@ -117,12 +118,18 @@ def set_metadata(stream, mi): job.update() server.close() - if job.result is not None: + if job.failed: + prints(job.details) + elif job.result is not None: stream.seek(0) stream.truncate() stream.write(job.result) stream.flush() stream.seek(0) + try: + os.remove(pt.name) + except: + pass diff --git a/src/calibre/utils/podofo/podofo.cpp b/src/calibre/utils/podofo/podofo.cpp index ace8c58c70..ea982167d3 100644 --- a/src/calibre/utils/podofo/podofo.cpp +++ b/src/calibre/utils/podofo/podofo.cpp @@ -55,8 +55,7 @@ podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { } else return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject * @@ -73,8 +72,7 @@ podofo_PDFDoc_open(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { } else return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject * From 1483703aa02e5ca1e25254da8c89ea514a414374 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 23:21:59 -0600 Subject: [PATCH 14/15] Set correct LD_LIBRARY_PATH onlinux before launching urls --- src/calibre/__init__.py | 7 ------ src/calibre/gui2/__init__.py | 28 ++++++++++++++++----- src/calibre/gui2/actions.py | 25 +++++++----------- src/calibre/gui2/book_details.py | 6 ++--- src/calibre/gui2/dialogs/book_info.py | 13 +++++----- src/calibre/gui2/dialogs/config/__init__.py | 11 ++++---- src/calibre/gui2/dialogs/user_profiles.py | 6 ++--- src/calibre/gui2/ui.py | 8 +++--- src/calibre/gui2/update.py | 6 ++--- src/calibre/gui2/viewer/main.py | 6 ++--- 10 files changed, 60 insertions(+), 56 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 285b2d35b4..92ee2ca6d2 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -342,13 +342,6 @@ def detect_ncpus(): return ans -def launch(path_or_url): - from PyQt4.QtCore import QUrl - from PyQt4.QtGui import QDesktopServices - if os.path.exists(path_or_url): - path_or_url = 'file:'+path_or_url - QDesktopServices.openUrl(QUrl(path_or_url)) - relpath = os.path.relpath _spat = re.compile(r'^the\s+|^a\s+|^an\s+', re.IGNORECASE) def english_sort(x, y): diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index f286402a37..9face1d9e9 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -1,18 +1,18 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' """ The GUI """ -import os +import os, sys from threading import RLock -from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ +from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \ QByteArray, QTranslator, QCoreApplication, QThread, \ - QEvent, QTimer, pyqtSignal, QDate -from PyQt4.QtGui import QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ - QIcon, QApplication, QDialog, QPushButton + QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ + QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ + QIcon, QApplication, QDialog, QPushButton, QUrl ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' -from calibre import islinux, iswindows, isosx, isfreebsd +from calibre.constants import islinux, iswindows, isosx, isfreebsd, isfrozen from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats @@ -579,6 +579,22 @@ class Application(QApplication): _store_app = None +def open_url(qurl): + paths = os.environ.get('LD_LIBRARY_PATH', + '').split(os.pathsep) + paths = [x for x in paths if x] + if isfrozen and islinux and paths: + npaths = [x for x in paths if x != sys.frozen_path] + os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(npaths) + QDesktopServices.openUrl(qurl) + if isfrozen and islinux and paths: + os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths) + + +def open_local_file(path): + url = QUrl.fromLocalFile(path) + open_url(url) + def is_ok_to_use_qt(): global gui_thread, _store_app if (islinux or isfreebsd) and ':' not in os.environ.get('DISPLAY', ''): diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py index 5dde2f745b..f36df397f9 100644 --- a/src/calibre/gui2/actions.py +++ b/src/calibre/gui2/actions.py @@ -5,17 +5,18 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, os, datetime, sys, time +import shutil, os, datetime, time from functools import partial from PyQt4.Qt import QInputDialog, pyqtSignal, QModelIndex, QThread, Qt, \ - SIGNAL, QPixmap, QTimer, QDesktopServices, QUrl, QDialog + SIGNAL, QPixmap, QTimer, QDialog from calibre import strftime from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.gui2 import error_dialog, Dispatcher, gprefs, choose_files, \ - choose_dir, warning_dialog, info_dialog, question_dialog, config + choose_dir, warning_dialog, info_dialog, question_dialog, config, \ + open_local_file from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString from calibre.utils.filenames import ascii_filename from calibre.gui2.widgets import IMAGE_EXTENSIONS @@ -25,7 +26,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \ fetch_scheduled_recipe, generate_catalog from calibre.constants import preferred_encoding, filesystem_encoding, \ - isosx, isfrozen, islinux + isosx from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2.dialogs.confirm_delete import confirm @@ -920,7 +921,7 @@ class SaveToDiskAction(object): # {{{ _('Could not save some books') + ', ' + _('Click the show details button to see which ones.'), u'\n\n'.join(failures), show=True) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def books_saved(self, job): if job.failed: @@ -1186,15 +1187,7 @@ class ViewAction(object): # {{{ self.job_manager.launch_gui_app(viewer, kwargs=dict(args=args)) else: - paths = os.environ.get('LD_LIBRARY_PATH', - '').split(os.pathsep) - paths = [x for x in paths if x] - if isfrozen and islinux and paths: - npaths = [x for x in paths if x != sys.frozen_path] - os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(npaths) - QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name) - if isfrozen and islinux and paths: - os.environ['LD_LIBRARY_PATH'] = os.pathsep.join(paths) + open_local_file(name) time.sleep(2) # User feedback finally: self.unsetCursor() @@ -1240,11 +1233,11 @@ class ViewAction(object): # {{{ return for row in rows: path = self.library_view.model().db.abspath(row.row()) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def view_folder_for_id(self, id_): path = self.library_view.model().db.abspath(id_, index_is_id=True) - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def view_book(self, triggered): rows = self.current_view().selectionModel().selectedRows() diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 4deadbc857..f08dd09429 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -9,14 +9,14 @@ import os, collections from PyQt4.Qt import QLabel, QPixmap, QSize, QWidget, Qt, pyqtSignal, \ QVBoxLayout, QScrollArea, QPropertyAnimation, QEasingCurve, \ - QSizePolicy, QPainter, QRect, pyqtProperty, QDesktopServices, QUrl + QSizePolicy, QPainter, QRect, pyqtProperty from calibre import fit_image, prepare_string_for_xml from calibre.gui2.widgets import IMAGE_EXTENSIONS from calibre.ebooks import BOOK_EXTENSIONS from calibre.constants import preferred_encoding from calibre.library.comments import comments_to_html -from calibre.gui2 import config +from calibre.gui2 import config, open_local_file # render_rows(data) {{{ WEIGHTS = collections.defaultdict(lambda : 100) @@ -294,7 +294,7 @@ class BookDetails(QWidget): # {{{ id_, fmt = val.split(':') self.view_specific_format.emit(int(id_), fmt) elif typ == 'devpath': - QDesktopServices.openUrl(QUrl.fromLocalFile(val)) + open_local_file(val) def mouseReleaseEvent(self, ev): diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index 20ddfae0b4..9770ef864f 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -5,11 +5,11 @@ __docformat__ = 'restructuredtext en' import textwrap, os, re -from PyQt4.QtCore import QCoreApplication, SIGNAL, QModelIndex, QUrl, QTimer, Qt -from PyQt4.QtGui import QDialog, QPixmap, QGraphicsScene, QIcon, QDesktopServices +from PyQt4.QtCore import QCoreApplication, SIGNAL, QModelIndex, QTimer, Qt +from PyQt4.QtGui import QDialog, QPixmap, QGraphicsScene, QIcon from calibre.gui2.dialogs.book_info_ui import Ui_BookInfo -from calibre.gui2 import dynamic +from calibre.gui2 import dynamic, open_local_file from calibre import fit_image from calibre.library.comments import comments_to_html @@ -49,12 +49,12 @@ class BookInfo(QDialog, Ui_BookInfo): def open_book_path(self, path): if os.sep in unicode(path): - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) else: format = unicode(path) path = self.view.model().db.format_abspath(self.current_row, format) if path is not None: - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) + open_local_file(path) def next(self): @@ -123,6 +123,7 @@ class BookInfo(QDialog, Ui_BookInfo): for key in info.keys(): if key == 'id': continue txt = info[key] - txt = u'
\n'.join(textwrap.wrap(txt, 120)) + if key != _('Path'): + txt = u'
\n'.join(textwrap.wrap(txt, 120)) rows += u'%s:%s'%(key, txt) self.text.setText(u''+rows+'
') diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index f17c0083ec..144d8f8586 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import os, re, time, textwrap, copy, sys from PyQt4.Qt import QDialog, QListWidgetItem, QIcon, \ - QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \ + QVBoxLayout, QLabel, QPlainTextEdit, \ QStringListModel, QAbstractItemModel, QFont, \ SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \ QModelIndex, QAbstractTableModel, \ @@ -15,8 +15,9 @@ from calibre.constants import iswindows, isosx from calibre.gui2.dialogs.config.config_ui import Ui_Dialog from calibre.gui2.dialogs.config.create_custom_column import CreateCustomColumn from calibre.gui2 import choose_dir, error_dialog, config, gprefs, \ - ALL_COLUMNS, NONE, info_dialog, choose_files, \ - warning_dialog, ResizableDialog, question_dialog + open_url, open_local_file, \ + ALL_COLUMNS, NONE, info_dialog, choose_files, \ + warning_dialog, ResizableDialog, question_dialog from calibre.utils.config import prefs from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.oeb.iterator import is_supported @@ -512,7 +513,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): def open_config_dir(self): from calibre.utils.config import config_dir - QDesktopServices.openUrl(QUrl.fromLocalFile(config_dir)) + open_local_file(config_dir) def create_symlinks(self): from calibre.utils.osx_symlinks import create_symlinks @@ -805,7 +806,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): self.stop.setEnabled(False) def test_server(self): - QDesktopServices.openUrl(QUrl('http://127.0.0.1:'+str(self.port.value()))) + open_url(QUrl('http://127.0.0.1:'+str(self.port.value()))) def compact(self, toggled): d = CheckIntegrity(self.db, self) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 7b26fea0ae..16f5d383ed 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal ' import time, os -from PyQt4.Qt import SIGNAL, QUrl, QDesktopServices, QAbstractListModel, Qt, \ +from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ QVariant, QInputDialog from calibre.web.feeds.recipes import compile_recipe from calibre.web.feeds.news import AutomaticNewsRecipe from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog -from calibre.gui2 import error_dialog, question_dialog, \ +from calibre.gui2 import error_dialog, question_dialog, open_url, \ choose_files, ResizableDialog, NONE from calibre.gui2.widgets import PythonHighlighter from calibre.ptempfile import PersistentTemporaryFile @@ -135,7 +135,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): url.addQueryItem('subject', subject) url.addQueryItem('body', body) url.addQueryItem('attachment', pt.name) - QDesktopServices.openUrl(url) + open_url(url) def current_changed(self, current, previous): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 16a754fab7..756e375e23 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -12,9 +12,9 @@ __docformat__ = 'restructuredtext en' import collections, os, sys, textwrap, time from Queue import Queue, Empty from threading import Thread -from PyQt4.Qt import Qt, SIGNAL, QObject, QUrl, QTimer, \ +from PyQt4.Qt import Qt, SIGNAL, QObject, QTimer, \ QPixmap, QMenu, QIcon, pyqtSignal, \ - QDialog, QDesktopServices, \ + QDialog, \ QSystemTrayIcon, QApplication, QKeySequence, QAction, \ QMessageBox, QHelpEvent @@ -23,7 +23,7 @@ from calibre.constants import __version__, __appname__, isosx from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server -from calibre.gui2 import error_dialog, GetMetadata, \ +from calibre.gui2 import error_dialog, GetMetadata, open_local_file, \ gprefs, max_available_height, config, info_dialog from calibre.gui2.cover_flow import CoverFlowMixin from calibre.gui2.widgets import ProgressIndicator @@ -572,7 +572,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{ pt = PersistentTemporaryFile('_donate.htm') pt.write(HTML.encode('utf-8')) pt.close() - QDesktopServices.openUrl(QUrl.fromLocalFile(pt.name)) + open_local_file(pt.name) def confirm_quit(self): diff --git a/src/calibre/gui2/update.py b/src/calibre/gui2/update.py index 9dcd4d9084..84168d17b5 100644 --- a/src/calibre/gui2/update.py +++ b/src/calibre/gui2/update.py @@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal ' import traceback -from PyQt4.Qt import QThread, pyqtSignal, QDesktopServices, QUrl, Qt +from PyQt4.Qt import QThread, pyqtSignal, Qt, QUrl import mechanize from calibre.constants import __appname__, __version__, iswindows, isosx from calibre import browser from calibre.utils.config import prefs -from calibre.gui2 import config, dynamic, question_dialog +from calibre.gui2 import config, dynamic, question_dialog, open_url URL = 'http://status.calibre-ebook.com/latest' @@ -64,7 +64,7 @@ class UpdateMixin(object): 'ge?')%(__appname__, version)): url = 'http://calibre-ebook.com/download_'+\ ('windows' if iswindows else 'osx' if isosx else 'linux') - QDesktopServices.openUrl(QUrl(url)) + open_url(QUrl(url)) dynamic.set('update to version %s'%version, False) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index fca3586f9d..ec88c3f886 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -6,7 +6,7 @@ from functools import partial from threading import Thread from PyQt4.Qt import QApplication, Qt, QIcon, QTimer, SIGNAL, QByteArray, \ - QDesktopServices, QDoubleSpinBox, QLabel, QTextBrowser, \ + QDoubleSpinBox, QLabel, QTextBrowser, \ QPainter, QBrush, QColor, QStandardItemModel, QPalette, \ QStandardItem, QUrl, QRegExpValidator, QRegExp, QLineEdit, \ QToolButton, QMenu, QInputDialog, QAction, QKeySequence @@ -17,7 +17,7 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager from calibre.gui2.widgets import ProgressIndicator from calibre.gui2.main_window import MainWindow from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \ - info_dialog, error_dialog + info_dialog, error_dialog, open_url from calibre.ebooks.oeb.iterator import EbookIterator from calibre.ebooks import DRMError from calibre.constants import islinux, isfreebsd @@ -472,7 +472,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): elif frag: self.view.scroll_to(frag) else: - QDesktopServices.openUrl(url) + open_url(url) def load_started(self): self.open_progress_indicator(_('Loading flow...')) From 9235d98428783a9ef773d692290ec71182963104 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 30 Jun 2010 23:24:31 -0600 Subject: [PATCH 15/15] Make the comments to HTML transform faster and more robust --- src/calibre/library/comments.py | 73 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/src/calibre/library/comments.py b/src/calibre/library/comments.py index 018c39bcf7..d4ed1908f5 100644 --- a/src/calibre/library/comments.py +++ b/src/calibre/library/comments.py @@ -8,9 +8,14 @@ __docformat__ = 'restructuredtext en' import re from calibre.constants import preferred_encoding -from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString, \ + CData, Comment, Declaration, ProcessingInstruction from calibre import prepare_string_for_xml +# Hackish - ignoring sentences ending or beginning in numbers to avoid +# confusion with decimal points. +lost_cr_pat = re.compile('([a-z])([\.\?!])([A-Z])') + def comments_to_html(comments): ''' Convert random comment text to normalized, xml-legal block of

s @@ -41,36 +46,25 @@ def comments_to_html(comments): if '<' not in comments: comments = prepare_string_for_xml(comments) - comments = comments.replace(u'\n', u'
') - return u'

%s

'%comments - - # Hackish - ignoring sentences ending or beginning in numbers to avoid - # confusion with decimal points. + parts = [u'

%s

'%x.replace(u'\n', u'
') + for x in comments.split('\n\n')] + return '\n'.join(parts) # Explode lost CRs to \n\n - for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])', comments): + for lost_cr in lost_cr_pat.finditer(comments): comments = comments.replace(lost_cr.group(), '%s%s\n\n%s' % (lost_cr.group(1), lost_cr.group(2), lost_cr.group(3))) + comments = comments.replace(u'\r', u'') # Convert \n\n to

s - if re.search('\n\n', comments): - soup = BeautifulSoup() - split_ps = comments.split(u'\n\n') - tsc = 0 - for p in split_ps: - pTag = Tag(soup,'p') - pTag.insert(0,p) - soup.insert(tsc,pTag) - tsc += 1 - comments = soup.renderContents(None) - + comments = comments.replace(u'\n\n', u'

') # Convert solo returns to
- comments = re.sub('[\r\n]','
', comments) - + comments = comments.replace(u'\n', '
') # Convert two hyphens to emdash - comments = re.sub('--', '—', comments) + comments = comments.replace('--', '—') + soup = BeautifulSoup(comments) result = BeautifulSoup() rtc = 0 @@ -85,35 +79,52 @@ def comments_to_html(comments): ptc = 0 pTag.insert(ptc,prepare_string_for_xml(token)) ptc += 1 - - elif token.name in ['br','b','i','em']: + elif type(token) in (CData, Comment, Declaration, + ProcessingInstruction): + continue + elif token.name in ['br', 'b', 'i', 'em', 'strong', 'span', 'font', 'a', + 'hr']: if not open_pTag: pTag = Tag(result,'p') open_pTag = True ptc = 0 pTag.insert(ptc, token) ptc += 1 - else: if open_pTag: result.insert(rtc, pTag) rtc += 1 open_pTag = False ptc = 0 - # Clean up NavigableStrings for xml - sub_tokens = list(token.contents) - for sub_token in sub_tokens: - if type(sub_token) is NavigableString: - sub_token.replaceWith(prepare_string_for_xml(sub_token)) result.insert(rtc, token) rtc += 1 if open_pTag: result.insert(rtc, pTag) - paras = result.findAll('p') - for p in paras: + for p in result.findAll('p'): p['class'] = 'description' + for t in result.findAll(text=True): + t.replaceWith(prepare_string_for_xml(unicode(t))) + return result.renderContents(encoding=None) +def test(): + for pat, val in [ + ('lineone\n\nlinetwo', + '

lineone

\n

linetwo

'), + ('a b&c\nf', '

a b&c;
f

'), + ('a b\n\ncd', '

a b

cd

'), + ]: + print + print 'Testing: %r'%pat + cval = comments_to_html(pat) + print 'Value: %r'%cval + if comments_to_html(pat) != val: + print 'FAILED' + break + +if __name__ == '__main__': + test() +