From d280eb8c29ebbb4e0c8991c9b892e1ecf9969f79 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 17 Jun 2011 17:48:15 +0100 Subject: [PATCH 01/21] Implement job step sequence priority --- src/calibre/gui2/device.py | 98 ++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index c9c79095f3..ad4ba82b45 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -119,6 +119,7 @@ class DeviceManager(Thread): # {{{ self.sleep_time = sleep_time self.connected_slot = connected_slot self.jobs = Queue.Queue(0) + self.job_steps = Queue.Queue(0) self.keep_going = True self.job_manager = job_manager self.reported_errors = set([]) @@ -235,6 +236,12 @@ class DeviceManager(Thread): # {{{ self.connected_device.unmount_device() def next(self): + if not self.job_steps.empty(): + try: + return self.job_steps.get_nowait() + except Queue.Empty: + pass + if not self.jobs.empty(): try: return self.jobs.get_nowait() @@ -271,13 +278,23 @@ class DeviceManager(Thread): # {{{ break time.sleep(self.sleep_time) - def create_job(self, func, done, description, args=[], kwargs={}): + def create_job_step(self, func, done, description, to_job, args=[], kwargs={}): job = DeviceJob(func, done, self.job_manager, args=args, kwargs=kwargs, description=description) self.job_manager.add_job(job) - self.jobs.put(job) + if (done is None or isinstance(done, FunctionDispatcher)) and \ + (to_job is not None and to_job == self.current_job): + print "adding as step", description + self.job_steps.put(job) + else: + print "adding as job not step", description + traceback.print_stack() + self.jobs.put(job) return job + def create_job(self, func, done, description, args=[], kwargs={}): + return self.create_job_step(func, done, description, None, args, kwargs) + def has_card(self): try: return bool(self.device.card_prefix()) @@ -295,10 +312,10 @@ class DeviceManager(Thread): # {{{ self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs} return info, cp, fs - def get_device_information(self, done): + def get_device_information(self, done, add_as_step_to_job=None): '''Get device information and free space on device''' - return self.create_job(self._get_device_information, done, - description=_('Get device information')) + return self.create_job_step(self._get_device_information, done, + description=_('Get device information'), to_job=add_as_step_to_job) def get_current_device_information(self): return self._device_information @@ -310,36 +327,38 @@ class DeviceManager(Thread): # {{{ cardblist = self.device.books(oncard='cardb') return (mainlist, cardalist, cardblist) - def books(self, done): + def books(self, done, add_as_step_to_job=None): '''Return callable that returns the list of books on device as two booklists''' - return self.create_job(self._books, done, description=_('Get list of books on device')) + return self.create_job_step(self._books, done, + description=_('Get list of books on device'), to_job=add_as_step_to_job) def _annotations(self, path_map): return self.device.get_annotations(path_map) - def annotations(self, done, path_map): + def annotations(self, done, path_map, add_as_step_to_job=None): '''Return mapping of ids to annotations. Each annotation is of the form (type, location_info, content). path_map is a mapping of ids to paths on the device.''' - return self.create_job(self._annotations, done, args=[path_map], - description=_('Get annotations from device')) + return self.create_job_step(self._annotations, done, args=[path_map], + description=_('Get annotations from device'), to_job=add_as_step_to_job) def _sync_booklists(self, booklists): '''Sync metadata to device''' self.device.sync_booklists(booklists, end_session=False) return self.device.card_prefix(end_session=False), self.device.free_space() - def sync_booklists(self, done, booklists, plugboards): + def sync_booklists(self, done, booklists, plugboards, add_as_step_to_job=None): if hasattr(self.connected_device, 'set_plugboards') and \ callable(self.connected_device.set_plugboards): self.connected_device.set_plugboards(plugboards, find_plugboard) - return self.create_job(self._sync_booklists, done, args=[booklists], - description=_('Send metadata to device')) + return self.create_job_step(self._sync_booklists, done, args=[booklists], + description=_('Send metadata to device'), to_job=add_as_step_to_job) - def upload_collections(self, done, booklist, on_card): - return self.create_job(booklist.rebuild_collections, done, + def upload_collections(self, done, booklist, on_card, add_as_step_to_job=None): + return self.create_job_step(booklist.rebuild_collections, done, args=[booklist, on_card], - description=_('Send collections to device')) + description=_('Send collections to device'), + to_job=add_as_step_to_job) def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None): '''Upload books to device: ''' @@ -374,11 +393,12 @@ class DeviceManager(Thread): # {{{ metadata=metadata, end_session=False) def upload_books(self, done, files, names, on_card=None, titles=None, - metadata=None, plugboards=None): + metadata=None, plugboards=None, add_as_step_to_job=None): desc = _('Upload %d books to device')%len(names) if titles: desc += u':' + u', '.join(titles) - return self.create_job(self._upload_books, done, args=[files, names], + return self.create_job_step(self._upload_books, done, to_job=add_as_step_to_job, + args=[files, names], kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc) def add_books_to_metadata(self, locations, metadata, booklists): @@ -388,9 +408,10 @@ class DeviceManager(Thread): # {{{ '''Remove books from device''' self.device.delete_books(paths, end_session=True) - def delete_books(self, done, paths): - return self.create_job(self._delete_books, done, args=[paths], - description=_('Delete books from device')) + def delete_books(self, done, paths, add_as_step_to_job=None): + return self.create_job_step(self._delete_books, done, args=[paths], + description=_('Delete books from device'), + to_job=add_as_step_to_job) def remove_books_from_metadata(self, paths, booklists): self.device.remove_books_from_metadata(paths, booklists) @@ -405,9 +426,10 @@ class DeviceManager(Thread): # {{{ self.device.get_file(path, f) f.close() - def save_books(self, done, paths, target): - return self.create_job(self._save_books, done, args=[paths, target], - description=_('Download books from device')) + def save_books(self, done, paths, target, add_as_step_to_job=None): + return self.create_job_step(self._save_books, done, args=[paths, target], + description=_('Download books from device'), + to_job=add_as_step_to_job) def _view_book(self, path, target): f = open(target, 'wb') @@ -415,9 +437,9 @@ class DeviceManager(Thread): # {{{ f.close() return target - def view_book(self, done, path, target): - return self.create_job(self._view_book, done, args=[path, target], - description=_('View book on device')) + def view_book(self, done, path, target, add_as_step_to_job=None): + return self.create_job_step(self._view_book, done, args=[path, target], + description=_('View book on device'), to_job=add_as_step_to_job) def set_current_library_uuid(self, uuid): self.current_library_uuid = uuid @@ -778,7 +800,8 @@ class DeviceMixin(object): # {{{ self.device_manager.device.icon) self.bars_manager.update_bars() self.status_bar.device_connected(info[0]) - self.device_manager.books(FunctionDispatcher(self.metadata_downloaded)) + self.device_manager.books(FunctionDispatcher(self.metadata_downloaded), + add_as_step_to_job=job) def metadata_downloaded(self, job): ''' @@ -788,7 +811,7 @@ class DeviceMixin(object): # {{{ self.device_job_exception(job) return # set_books_in_library might schedule a sync_booklists job - self.set_books_in_library(job.result, reset=True) + self.set_books_in_library(job.result, reset=True, add_as_step_to_job=job) mainlist, cardalist, cardblist = job.result self.memory_view.set_database(mainlist) self.memory_view.set_editable(self.device_manager.device.CAN_SET_METADATA, @@ -843,8 +866,8 @@ class DeviceMixin(object): # {{{ # set_books_in_library even though books were not added because # the deleted book might have been an exact match. Upload the booklists # if set_books_in_library did not. - if not self.set_books_in_library(self.booklists(), reset=True): - self.upload_booklists() + if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job): + self.upload_booklists(job) self.book_on_device(None, reset=True) # We need to reset the ondevice flags in the library. Use a big hammer, # so we don't need to worry about whether some succeeded or not. @@ -1193,13 +1216,14 @@ class DeviceMixin(object): # {{{ self.device_manager.sync_booklists(Dispatcher(lambda x: x), self.booklists(), plugboards) - def upload_booklists(self): + def upload_booklists(self, add_as_step_to_job=None): ''' Upload metadata to device. ''' plugboards = self.library_view.model().db.prefs.get('plugboards', {}) self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced), - self.booklists(), plugboards) + self.booklists(), plugboards, + add_as_step_to_job=add_as_step_to_job) def metadata_synced(self, job): ''' @@ -1274,8 +1298,8 @@ class DeviceMixin(object): # {{{ # because the UUID changed. Force both the device and the library view # to refresh the flags. Set_books_in_library could upload the booklists. # If it does not, then do it here. - if not self.set_books_in_library(self.booklists(), reset=True): - self.upload_booklists() + if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job): + self.upload_booklists(job) with self.library_view.preserve_selected_books: self.book_on_device(None, reset=True) self.refresh_ondevice() @@ -1335,7 +1359,7 @@ class DeviceMixin(object): # {{{ loc[4] |= self.book_db_uuid_path_map[id] return loc - def set_books_in_library(self, booklists, reset=False): + def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None): ''' Set the ondevice indications in the device database. This method should be called before book_on_device is called, because @@ -1487,7 +1511,7 @@ class DeviceMixin(object): # {{{ plugboards = self.library_view.model().db.prefs.get('plugboards', {}) self.device_manager.sync_booklists( FunctionDispatcher(self.metadata_synced), booklists, - plugboards) + plugboards, add_as_step_to_job) return update_metadata # }}} From 9772c12b4e37a1581a37ce24c5da5b2efe2208ff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Jun 2011 13:13:53 -0600 Subject: [PATCH 02/21] ... --- src/calibre/manual/develop.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/manual/develop.rst b/src/calibre/manual/develop.rst index c49176ceb2..7068bc2389 100644 --- a/src/calibre/manual/develop.rst +++ b/src/calibre/manual/develop.rst @@ -36,10 +36,13 @@ Code layout All the |app| python code is in the ``calibre`` package. This package contains the following main sub-packages * devices - All the device drivers. Just look through some of the built-in drivers to get an idea for how they work. + * For details, see: devices.interface which defines the interface supported by device drivers and devices.usbms which defines a generic driver that connects to a USBMS device. All USBMS based drivers in calibre inherit from it. * ebooks - All the ebook conversion code. A good starting point is ``calibre.ebooks.conversion.cli`` which is the - module powering the :command:`ebook-convert` command. - * library - The database backed and the content server. - * gui2 - The Graphical User Interface. + module powering the :command:`ebook-convert` command. The conversion process is controlled via conversion.plumber. + The format independent code is all in ebooks.oeb and the format dependent stuff is in ebooks.format_name. + * Metadata reading writing and downloading is all in ebooks.metadata + * library - The database backed and the content server. See library.database2 for the interface to the calibre library. library.server is the calibre Content Server. + * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer. Getting the code ------------------ From fb2d1a4ea49307918528e34fbc564c6d96d28e00 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Jun 2011 13:36:33 -0600 Subject: [PATCH 03/21] Doc updates to setting up a calibre development environment --- src/calibre/manual/develop.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/manual/develop.rst b/src/calibre/manual/develop.rst index 7068bc2389..cc037ee6dd 100644 --- a/src/calibre/manual/develop.rst +++ b/src/calibre/manual/develop.rst @@ -28,7 +28,7 @@ For example, adding support for a new device to |app| typically involves writing a device driver plugin. You can browse the `built-in drivers `_. Similarly, adding support for new conversion formats involves writing input/output format plugins. Another example of the modular design is the :ref:`recipe system ` for -fetching news. +fetching news. For more examples of plugins designed to add features to |app|, see the `plugin index `_. Code layout ^^^^^^^^^^^^^^ @@ -36,14 +36,18 @@ Code layout All the |app| python code is in the ``calibre`` package. This package contains the following main sub-packages * devices - All the device drivers. Just look through some of the built-in drivers to get an idea for how they work. - * For details, see: devices.interface which defines the interface supported by device drivers and devices.usbms which defines a generic driver that connects to a USBMS device. All USBMS based drivers in calibre inherit from it. - * ebooks - All the ebook conversion code. A good starting point is ``calibre.ebooks.conversion.cli`` which is the + * For details, see: devices.interface which defines the interface supported by device drivers and devices.usbms which + defines a generic driver that connects to a USBMS device. All USBMS based drivers in calibre inherit from it. + * ebooks - All the ebook conversion/metadata code. A good starting point is ``calibre.ebooks.conversion.cli`` which is the module powering the :command:`ebook-convert` command. The conversion process is controlled via conversion.plumber. The format independent code is all in ebooks.oeb and the format dependent stuff is in ebooks.format_name. * Metadata reading writing and downloading is all in ebooks.metadata * library - The database backed and the content server. See library.database2 for the interface to the calibre library. library.server is the calibre Content Server. * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer. +If you need help understanding the code, post in the `development forum `_ +and you will most likely get help from one of |app|'s many developers. + Getting the code ------------------ From f7a93ad92e7ec1f7e6dfe5e70a588f9f1070930a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Jun 2011 13:40:48 -0600 Subject: [PATCH 04/21] ... --- src/calibre/manual/develop.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/manual/develop.rst b/src/calibre/manual/develop.rst index cc037ee6dd..307df085e6 100644 --- a/src/calibre/manual/develop.rst +++ b/src/calibre/manual/develop.rst @@ -89,9 +89,9 @@ Now whenever you commit changes to your branch with the command:: bzr commit -m "Comment describing your change" -I can merge it directly from you branch into the main |app| source tree. You should also subscribe to the |app| -developers mailing list `calibre-devs `_. Before making major changes, you should -discuss them on the mailing list or the #calibre IRC channel on Freenode to ensure that the changes will be accepted once you're done. +I can merge it directly from you branch into the main |app| source tree. You should also keep an eye on the |app| +`development forum `. Before making major changes, you should +discuss them in the forum or contact Kovid directly (his email address is all over the source code). Windows development environment --------------------------------- From 950dd7fe921e930390af0f101b2e229514713937 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Jun 2011 13:48:00 -0600 Subject: [PATCH 05/21] ... --- src/calibre/manual/develop.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/manual/develop.rst b/src/calibre/manual/develop.rst index 307df085e6..fecdf28a47 100644 --- a/src/calibre/manual/develop.rst +++ b/src/calibre/manual/develop.rst @@ -36,12 +36,16 @@ Code layout All the |app| python code is in the ``calibre`` package. This package contains the following main sub-packages * devices - All the device drivers. Just look through some of the built-in drivers to get an idea for how they work. + * For details, see: devices.interface which defines the interface supported by device drivers and devices.usbms which defines a generic driver that connects to a USBMS device. All USBMS based drivers in calibre inherit from it. + * ebooks - All the ebook conversion/metadata code. A good starting point is ``calibre.ebooks.conversion.cli`` which is the module powering the :command:`ebook-convert` command. The conversion process is controlled via conversion.plumber. The format independent code is all in ebooks.oeb and the format dependent stuff is in ebooks.format_name. + * Metadata reading writing and downloading is all in ebooks.metadata + * library - The database backed and the content server. See library.database2 for the interface to the calibre library. library.server is the calibre Content Server. * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer. From 51546075f475883facb532ebdeae0510d1310a99 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Jun 2011 18:52:14 -0600 Subject: [PATCH 06/21] ... --- src/calibre/manual/conversion.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index fea828f706..9244109ba7 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -657,6 +657,7 @@ Some limitations of PDF input are: * Some PDFs store their images upside down with a rotation instruction, |app| currently doesn't support that instruction, so the images will be rotated in the output as well. * Links and Tables of Contents are not supported * PDFs that use embedded non-unicode fonts to represent non-English characters will result in garbled output for those characters + * Some PDFs are made up of photographs of the page with OCRed text behind them. In such cases |app| uses the OCRed text, which can be very different from what you see when you view the PDF file To re-iterate **PDF is a really, really bad** format to use as input. If you absolutely must use PDF, then be prepared for an output ranging anywhere from decent to unusable, depending on the input PDF. From f132c76e02cef783d48f31af94053e9f1c1a18b0 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Jun 2011 07:07:41 +0100 Subject: [PATCH 07/21] Remove print statements --- src/calibre/gui2/device.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index ad4ba82b45..99f795cdda 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -284,11 +284,8 @@ class DeviceManager(Thread): # {{{ self.job_manager.add_job(job) if (done is None or isinstance(done, FunctionDispatcher)) and \ (to_job is not None and to_job == self.current_job): - print "adding as step", description self.job_steps.put(job) else: - print "adding as job not step", description - traceback.print_stack() self.jobs.put(job) return job From 6d2208759ccc1ddac1ad6570c1bed406ed116e3f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 00:11:15 -0600 Subject: [PATCH 08/21] Conversion: Convert
tags with text in them into as ADE cannot handle them. Fixes #794427 (Private bug) --- src/calibre/ebooks/oeb/base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index db83fca496..a9c7664953 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1055,6 +1055,12 @@ class Manifest(object): and len(a) == 0 and not a.text: remove_elem(a) + # Convert
s with content into paragraphs as ADE can't handle + # them + for br in xpath(data, '//h:br'): + if len(br) > 0 or br.text: + br.tag = XHTML('div') + return data def _parse_txt(self, data): From 16560874daf9e8a5e83231928d4a647718e05766 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 00:19:49 -0600 Subject: [PATCH 09/21] Fix restore database not working on some windows installs --- src/calibre/library/restore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index 20065309aa..8bd7174849 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -24,6 +24,7 @@ NON_EBOOK_EXTENSIONS = frozenset([ class RestoreDatabase(LibraryDatabase2): PATH_LIMIT = 10 + WINDOWS_LIBRARY_PATH_LIMIT = 180 def set_path(self, *args, **kwargs): pass From dbf20716ed8e8c0aa83e31ad390e56497693c2ba Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Jun 2011 15:37:02 +0100 Subject: [PATCH 10/21] Make first-letter nodes in the tag browser click-searchable. Make regexp searching work in unicode --- src/calibre/gui2/tag_view.py | 18 +++++++++++++++++- src/calibre/library/caches.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index a6a852fbdd..61c67f347c 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -727,6 +727,15 @@ class TagTreeItem(object): # {{{ else: self.tag.state = set_to + def all_children(self): + res = [] + def recurse(nodes, res): + for t in nodes: + res.append(t) + recurse(t.children, res) + recurse(self.children, res) + return res + def child_tags(self): res = [] def recurse(nodes, res): @@ -1651,7 +1660,14 @@ class TagsModel(QAbstractItemModel): # {{{ ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state])) key = node.category_key - for tag_item in node.child_tags(): + for tag_item in node.all_children(): + if tag_item.type == TagTreeItem.CATEGORY: + if tag_item.temporary and not key.startswith('@') and tag_item.tag.state: + if node_searches[tag_item.tag.state] == 'true': + ans.append('%s:~^%s'%(key, tag_item.py_name)) + else: + ans.append('(not %s:~^%s )'%(key, tag_item.py_name)) + continue tag = tag_item.tag if tag.state != TAG_SEARCH_STATES['clear']: if tag.state == TAG_SEARCH_STATES['mark_minus'] or \ diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 601071a2ce..b9dd2f3ed7 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -145,7 +145,7 @@ def _match(query, value, matchkind): return True elif query == t: return True - elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored + elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I|re.UNICODE)) or ### search unanchored (matchkind == CONTAINS_MATCH and query in t)): return True except re.error: From 8a3ab0a104f111d52c2c200526cd84d97a6dc0da Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 18 Jun 2011 15:58:52 +0100 Subject: [PATCH 11/21] Only allow searching in first-letter mode. --- src/calibre/gui2/tag_view.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 61c67f347c..3b83e6d189 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -1278,6 +1278,7 @@ class TagsModel(QAbstractItemModel): # {{{ category_icon = category_node.icon, category_key=category_node.category_key, icon_map=self.icon_state_map) + sub_cat.tag.is_searchable = False self.endInsertRows() else: # by 'first letter' cl = cl_list[idx] @@ -1662,7 +1663,9 @@ class TagsModel(QAbstractItemModel): # {{{ key = node.category_key for tag_item in node.all_children(): if tag_item.type == TagTreeItem.CATEGORY: - if tag_item.temporary and not key.startswith('@') and tag_item.tag.state: + if self.collapse_model == 'first letter' and \ + tag_item.temporary and not key.startswith('@') \ + and tag_item.tag.state: if node_searches[tag_item.tag.state] == 'true': ans.append('%s:~^%s'%(key, tag_item.py_name)) else: From 5b207996292961d2adae3cd3883f6b571acf8825 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 09:55:13 -0600 Subject: [PATCH 12/21] Make first-letter parition nodes in the tag browser click-searchable. Make regexp searching use unicode character classes. --- src/calibre/gui2/tag_view.py | 21 ++++++++++++++++++++- src/calibre/library/caches.py | 2 +- src/calibre/manual/faq.rst | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index a6a852fbdd..3b83e6d189 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -727,6 +727,15 @@ class TagTreeItem(object): # {{{ else: self.tag.state = set_to + def all_children(self): + res = [] + def recurse(nodes, res): + for t in nodes: + res.append(t) + recurse(t.children, res) + recurse(self.children, res) + return res + def child_tags(self): res = [] def recurse(nodes, res): @@ -1269,6 +1278,7 @@ class TagsModel(QAbstractItemModel): # {{{ category_icon = category_node.icon, category_key=category_node.category_key, icon_map=self.icon_state_map) + sub_cat.tag.is_searchable = False self.endInsertRows() else: # by 'first letter' cl = cl_list[idx] @@ -1651,7 +1661,16 @@ class TagsModel(QAbstractItemModel): # {{{ ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state])) key = node.category_key - for tag_item in node.child_tags(): + for tag_item in node.all_children(): + if tag_item.type == TagTreeItem.CATEGORY: + if self.collapse_model == 'first letter' and \ + tag_item.temporary and not key.startswith('@') \ + and tag_item.tag.state: + if node_searches[tag_item.tag.state] == 'true': + ans.append('%s:~^%s'%(key, tag_item.py_name)) + else: + ans.append('(not %s:~^%s )'%(key, tag_item.py_name)) + continue tag = tag_item.tag if tag.state != TAG_SEARCH_STATES['clear']: if tag.state == TAG_SEARCH_STATES['mark_minus'] or \ diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 601071a2ce..b9dd2f3ed7 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -145,7 +145,7 @@ def _match(query, value, matchkind): return True elif query == t: return True - elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I)) or ### search unanchored + elif ((matchkind == REGEXP_MATCH and re.search(query, t, re.I|re.UNICODE)) or ### search unanchored (matchkind == CONTAINS_MATCH and query in t)): return True except re.error: diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 4b527e169c..733adb65ee 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -131,7 +131,7 @@ Follow these steps to find the problem: * Make sure that you are connecting only a single device to your computer at a time. Do not have another |app| supported device like an iPhone/iPad etc. at the same time. * If you are connecting an Apple iDevice (iPad, iPod Touch, iPhone), use the 'Connect to iTunes' method in the 'Getting started' instructions in `Calibre + Apple iDevices: Start here `_. * Make sure you are running the latest version of |app|. The latest version can always be downloaded from `the calibre website `_. - * Ensure your operating system is seeing the device. That is, the device should be mounted as a disk that you can access using Windows explorer or whatever the file management program on your computer is. + * Ensure your operating system is seeing the device. That is, the device should be mounted as a disk, that you can access using Windows explorer or whatever the file management program on your computer is. On Windows your device **must have been assigned a drive letter**, like K:. * In calibre, go to Preferences->Plugins->Device Interface plugin and make sure the plugin for your device is enabled, the plugin icon next to it should be green when it is enabled. * If all the above steps fail, go to Preferences->Miscellaneous and click debug device detection with your device attached and post the output as a ticket on `the calibre bug tracker `_. From 954e04a52d98d5922abfeea24db813ae4a26914e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 10:00:57 -0600 Subject: [PATCH 13/21] Fix #799171 (In 0.8.6 on Mac: Loading HTML pkg as ZIP file no longer picks up external style sheet from folder) --- src/calibre/ebooks/html/input.py | 2 +- src/calibre/ebooks/oeb/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index ce6c46c6cf..69eb493c7d 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -457,7 +457,7 @@ class HTMLInput(InputFormatPlugin): href=bhref) guessed = self.guess_type(href)[0] media_type = guessed or self.BINARY_MIME - if 'text' in media_type: + if media_type == 'text/plain': self.log.warn('Ignoring link to text file %r'%link_) return None diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index a9c7664953..d75620adbd 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1162,7 +1162,7 @@ class Manifest(object): data = self._parse_xml(data) elif self.media_type.lower() in OEB_STYLES: data = self._parse_css(data) - elif 'text' in self.media_type.lower(): + elif self.media_type.lower() == 'text/plain': self.oeb.log.warn('%s contains data in TXT format'%self.href, 'converting to HTML') data = self._parse_txt(data) From 64d60aedc4f48a2100b081d145bf6f9342d45ea3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 10:05:53 -0600 Subject: [PATCH 14/21] ... --- recipes/metro_uk.recipe | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/metro_uk.recipe b/recipes/metro_uk.recipe index 2d5155ef29..287af47f5c 100644 --- a/recipes/metro_uk.recipe +++ b/recipes/metro_uk.recipe @@ -1,3 +1,4 @@ +import re from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Metro UK' @@ -10,6 +11,7 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe): remove_empty_feeds = True remove_javascript = True + preprocess_regexps = [(re.compile(r'Tweet'), lambda a : '')] language = 'en_GB' From e54352dd55bc15e3691a8040fec1ad9670e729ab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Jun 2011 16:36:16 -0600 Subject: [PATCH 15/21] ... --- src/calibre/gui2/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/update.py b/src/calibre/gui2/update.py index aacf30fe10..c9a908bdf6 100644 --- a/src/calibre/gui2/update.py +++ b/src/calibre/gui2/update.py @@ -179,7 +179,7 @@ class UpdateMixin(object): def plugin_update_found(self, number_of_updates): # Change the plugin icon to indicate there are updates available - plugin = self.iactions.get('Plugin Updates', None) + plugin = self.iactions.get('Plugin Updater', None) if not plugin: return if number_of_updates: From 4e9f2253024b8e800114db0628b63e0feecb8a0b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 19 Jun 2011 12:00:23 -0600 Subject: [PATCH 16/21] Fix #799481 (Updated recipe for Perfil) --- recipes/perfil.recipe | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/recipes/perfil.recipe b/recipes/perfil.recipe index 1104202318..af7072c6f6 100644 --- a/recipes/perfil.recipe +++ b/recipes/perfil.recipe @@ -26,6 +26,7 @@ class Perfil(BasicNewsRecipe): .foto1 h1{font-size: x-small} h1{font-family: Georgia,"Times New Roman",serif} img{margin-bottom: 0.4em} + .hora{font-size: x-small; color: red} """ conversion_options = { @@ -60,7 +61,26 @@ class Perfil(BasicNewsRecipe): ,(u'Tecnologia' , u'http://www.perfil.com/rss/tecnologia.xml' ) ] + def get_article_url(self, article): + return article.get('guid', None) + def preprocess_html(self, soup): for item in soup.findAll(style=True): del item['style'] + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' return soup + \ No newline at end of file From c450f28524701d7d0dba022530029184cf911010 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 19 Jun 2011 19:20:48 +0100 Subject: [PATCH 17/21] Fix clicking on news category to respect the current language --- src/calibre/gui2/tag_view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 3b83e6d189..3b8c27866c 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -1654,9 +1654,9 @@ class TagsModel(QAbstractItemModel): # {{{ if node.tag.state: if node.category_key == "news": if node_searches[node.tag.state] == 'true': - ans.append('tags:=news') + ans.append('tags:"=' + _('News') + '"') else: - ans.append('( not tags:=news )') + ans.append('( not tags:"=' + _('News') + '")') else: ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state])) From 1994faa7b25dadaa5324452801b6885cc51b57eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 19 Jun 2011 14:04:16 -0600 Subject: [PATCH 18/21] Cleanup implementation of thumbnail caching in catalog geenrator --- src/calibre/library/catalog.py | 48 +++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 97454c90e2..006b381214 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -3,7 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Greg Riker' -import codecs, datetime, htmlentitydefs, os, re, shutil, time, zlib +import codecs, datetime, htmlentitydefs, os, re, shutil, zlib from collections import namedtuple from copy import deepcopy from xml.sax.saxutils import escape @@ -25,7 +25,7 @@ from calibre.utils.html2text import html2text from calibre.utils.icu import capitalize from calibre.utils.logging import default_log as log from calibre.utils.magick.draw import thumbnail -from calibre.utils.zipfile import ZipFile, ZipInfo +from calibre.utils.zipfile import ZipFile FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', @@ -4704,24 +4704,33 @@ Author '{0}': to be replaced. ''' + def open_archive(mode='r'): + try: + return ZipFile(self.__archive_path, mode=mode) + except: + # Happens on windows if the file is opened by another + # process + pass + # Generate crc for current cover #self.opts.log.info(" generateThumbnail():") - data = open(title['cover'], 'rb').read() + with open(title['cover'], 'rb') as f: + data = f.read() cover_crc = hex(zlib.crc32(data)) # Test cache for uuid - with ZipFile(self.__archive_path, mode='r') as zfr: - try: - t_info = zfr.getinfo(title['uuid']) - except: - pass - else: - if t_info.comment == cover_crc: + zf = open_archive() + if zf is not None: + with zf: + try: + zf.getinfo(title['uuid']+cover_crc) + except: + pass + else: # uuid found in cache with matching crc - thumb_data = zfr.read(title['uuid']) - zfr.extract(title['uuid'],image_dir) - os.rename(os.path.join(image_dir,title['uuid']), - os.path.join(image_dir,thumb_file)) + thumb_data = zf.read(title['uuid']) + with open(os.path.join(image_dir, thumb_file), 'wb') as f: + f.write(thumb_data) return @@ -4732,10 +4741,13 @@ Author '{0}': f.write(thumb_data) # Save thumb to archive - t_info = ZipInfo(title['uuid'],time.localtime()[0:6]) - t_info.comment = cover_crc - with ZipFile(self.__archive_path, mode='a') as zfw: - zfw.writestr(t_info, thumb_data) + if zf is not None: # Ensure that the read succeeded + # If we failed to open the zip file for reading, + # we dont know if it contained the thumb or not + zf = open_archive('a') + if zf is not None: + with zf: + zf.writestr(title['uuid']+cover_crc, thumb_data) def getFriendlyGenreTag(self, genre): # Find the first instance of friendly_tag matching genre From 56c5979b72536ccf64bcb6a87dc2319afebe28e4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 19 Jun 2011 17:12:39 -0600 Subject: [PATCH 19/21] Fix #799393 (Jobs window has no selection when first opened) --- src/calibre/gui2/jobs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 20154a298b..589b28d520 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -432,6 +432,10 @@ class JobsDialog(QDialog, Ui_JobsDialog): self.jobs_view.horizontalHeader().restoreState(QByteArray(state)) except: pass + idx = self.jobs_view.model().index(0, 0) + if idx.isValid(): + sm = self.jobs_view.selectionModel() + sm.select(idx, sm.ClearAndSelect|sm.Rows) def save_state(self): try: From 776bfaacedf72fa158ebb8c12d564596e88f0dc6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 19 Jun 2011 17:38:57 -0600 Subject: [PATCH 20/21] Fix #799310 (Private bug) --- src/calibre/devices/hanvon/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index 3798257c2d..3ce0fedac0 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -61,7 +61,7 @@ class LIBREAIR(N516): BCD = [0x399] VENDOR_NAME = 'ALURATEK' - WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGET' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET' EBOOK_DIR_MAIN = 'Books' class ALEX(N516): From 94583c3bfca98f10c92ffea51c7b18763ec12fe1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 19 Jun 2011 23:41:06 -0600 Subject: [PATCH 21/21] ... --- recipes/wsj.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index cf84722bac..a3bc041d25 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -51,7 +51,7 @@ class WallStreetJournal(BasicNewsRecipe): br['password'] = self.password res = br.submit() raw = res.read() - if 'Welcome,' not in raw: + if 'Welcome,' not in raw and '>Logout<' not in raw: raise ValueError('Failed to log in to wsj.com, check your ' 'username and password') return br