From 60c7c6634936f7c543ff25826630bc425d44e952 Mon Sep 17 00:00:00 2001 From: Li Fanxi Date: Wed, 2 Mar 2011 23:25:53 +0800 Subject: [PATCH 01/53] [Bug] Workaround a strange problem when extracting some SNB files with PDF contents. --- src/calibre/ebooks/snb/snbfile.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/snb/snbfile.py b/src/calibre/ebooks/snb/snbfile.py index e42533f241..10aa6a8715 100644 --- a/src/calibre/ebooks/snb/snbfile.py +++ b/src/calibre/ebooks/snb/snbfile.py @@ -75,15 +75,18 @@ class SNBFile: for i in range(self.plainBlock): bzdc = bz2.BZ2Decompressor() if (i < self.plainBlock - 1): - bSize = self.blocks[self.binBlock + i + 1].Offset - self.blocks[self.binBlock + i].Offset; + bSize = self.blocks[self.binBlock + i + 1].Offset - self.blocks[self.binBlock + i].Offset else: - bSize = self.tailOffset - self.blocks[self.binBlock + i].Offset; - snbFile.seek(self.blocks[self.binBlock + i].Offset); + bSize = self.tailOffset - self.blocks[self.binBlock + i].Offset + snbFile.seek(self.blocks[self.binBlock + i].Offset) try: data = snbFile.read(bSize) - uncompressedData += bzdc.decompress(data) + if len(data) < 32768: + uncompressedData += bzdc.decompress(data) except Exception, e: print e + if len(uncompressedData) != self.plainStreamSizeUncompressed: + raise Exception() f.fileBody = uncompressedData[plainPos:plainPos+f.fileSize] plainPos += f.fileSize elif f.attr & 0x01000000 == 0x01000000: From e373c822e2155f7595b1291764cb3030f418f282 Mon Sep 17 00:00:00 2001 From: Li Fanxi Date: Thu, 3 Mar 2011 00:06:34 +0800 Subject: [PATCH 02/53] [Bug] A better way to workaround a strange problem when extracting some SNB files with PDF contents. --- src/calibre/ebooks/snb/snbfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/snb/snbfile.py b/src/calibre/ebooks/snb/snbfile.py index 10aa6a8715..9a7d65e417 100644 --- a/src/calibre/ebooks/snb/snbfile.py +++ b/src/calibre/ebooks/snb/snbfile.py @@ -83,6 +83,8 @@ class SNBFile: data = snbFile.read(bSize) if len(data) < 32768: uncompressedData += bzdc.decompress(data) + else: + uncompressedData += data except Exception, e: print e if len(uncompressedData) != self.plainStreamSizeUncompressed: From 66e1fcc6827a89205bb44ff4b9b29c9c0ef91fab Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 6 Mar 2011 10:21:17 +0000 Subject: [PATCH 03/53] Fix #9293 - Category rename is not case sensitive, name can clash with itself --- src/calibre/gui2/tag_view.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 586715afd0..11b696d861 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -1340,7 +1340,8 @@ class TagsModel(QAbstractItemModel): # {{{ for c in sorted(user_cats.keys(), key=sort_key): if icu_lower(c).startswith(ckey_lower): if len(c) == len(ckey): - if nkey_lower in user_cat_keys_lower: + if strcmp(ckey, nkey) != 0 and \ + nkey_lower in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%nkey, show=True) return False @@ -1348,7 +1349,8 @@ class TagsModel(QAbstractItemModel): # {{{ del user_cats[ckey] elif c[len(ckey)] == '.': rest = c[len(ckey):] - if icu_lower(nkey + rest) in user_cat_keys_lower: + if strcmp(ckey, nkey) != 0 and \ + icu_lower(nkey + rest) in user_cat_keys_lower: error_dialog(self.tags_view, _('Rename user category'), _('The name %s is already used')%(nkey+rest), show=True) return False From 6fc235d5f9e577bad944ebde1a97abe9a447302e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 08:09:32 -0700 Subject: [PATCH 04/53] ... --- resources/images/news/kompiutierra.png | Bin 0 -> 654 bytes resources/images/news/rbc_ru.png | Bin 0 -> 371 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 resources/images/news/kompiutierra.png create mode 100644 resources/images/news/rbc_ru.png diff --git a/resources/images/news/kompiutierra.png b/resources/images/news/kompiutierra.png new file mode 100644 index 0000000000000000000000000000000000000000..272e3d905fb98bf2deef154486357ee490de0d22 GIT binary patch literal 654 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#O#Ex;TbdoL)NVymv^T%u)OFXR}nJbX8Wg zO2?{xYKwY4f5Db+!K_H#D2pcVzx;9C1%em*&z{k8DG|?5;K*fa?@+iUA~PAy(D^Lpd$N$+mX6MC@4@zA%zJKJ3L-Ok$^l&rTu z`ST-=)d%cYmaAO)FIb0vi~<=gKj^J|Ixwwki`CZs6x*S>GQ3vzBXhp*%@#GRrbeTcvepKP!CH z*>s*+_-dB=4u?>)If z?P`?flx>r%6s9qWO=MfS_iR Date: Sun, 6 Mar 2011 08:18:45 -0700 Subject: [PATCH 05/53] Search and replace preferences: Prevent very long strings from causing the wizard button to get pushed off the screen --- src/calibre/gui2/convert/xexp_edit.ui | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/convert/xexp_edit.ui b/src/calibre/gui2/convert/xexp_edit.ui index 18b7c39b52..68c0c8c98e 100644 --- a/src/calibre/gui2/convert/xexp_edit.ui +++ b/src/calibre/gui2/convert/xexp_edit.ui @@ -43,6 +43,9 @@ 0 + + QComboBox::AdjustToMinimumContentsLengthWithIcon + 30 From a94fe3adad7bfae3d94cdb9e9d11df85c0933756 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 08:31:58 -0700 Subject: [PATCH 06/53] FB2 Input: Support for tables. Fixes #9302 (fb2 --> epub, mobi Missing tables) --- resources/templates/fb2.xsl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/resources/templates/fb2.xsl b/resources/templates/fb2.xsl index 77c03cdc74..703d082467 100644 --- a/resources/templates/fb2.xsl +++ b/resources/templates/fb2.xsl @@ -293,6 +293,23 @@

Annotation

+ + + + +
+ + + + + + + + + + + +
From d68aaddbb4d97ca829c29acc8117314fe78363d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 08:39:53 -0700 Subject: [PATCH 07/53] ... --- resources/templates/fb2.xsl | 576 ++++++++++++++++++------------------ 1 file changed, 288 insertions(+), 288 deletions(-) diff --git a/resources/templates/fb2.xsl b/resources/templates/fb2.xsl index 703d082467..9d0abc7d45 100644 --- a/resources/templates/fb2.xsl +++ b/resources/templates/fb2.xsl @@ -19,21 +19,21 @@ ######################################################################### --> - - - - - - - - - - <xsl:value-of select="fb:description/fb:title-info/fb:book-title"/> - - + - - - -
- -
-
- - -
    - -
-
+ + + +
+ +
+
+ + +
    + +
+
- - - -
- - -

- -

- - - - - - - - - - - - + + + +
+ + +

+ +

+ + + + + + + + + + + + - -
- - - -
  • - - - , # - - - -
      - - - -
    -     - - - - - - - - - -
  • - - -
    -     -
    • - - -     - - - - - - -
    -     + +
    + + + +
  • + + + , # + + + +
      + + + +
    +     + + + + + + + + + +
  • + + +
    +     +
    • + + +     + + + + + + +
    +     - + @@ -164,15 +164,15 @@ - - - - - + + + + + - - - + + + @@ -181,79 +181,79 @@ TOC_ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + - -
    -     - - - - - - - -
    - -
    -     - - + +
    +     + + + + + + + +
    + +
    +     + + paragraph - - - - + + + + - - - - - - - - - - - - - - - -
    -     +     + + + + + + + + + + + + + + +
    +     @@ -261,140 +261,140 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    Annotation

    - -     - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    Annotation

    + +     + +
    -     - + + - - + + - + - - - -
    - - - - - - -
    -     - - -
    - -
    -     - - -
    - - - - - - -
    -     - - -
    -
    -     - - - - -     -
    -     - -     -
    -     -     -     - - -
    - - - - - - -
    -     +     + + +
    + + + + + + +
    +     + + +
    + +
    +     + + +
    + + + + + + +
    +     + + +
    +
    +     + + + + +     +
    +     + +     +
    +     +     +     + + +
    + + + + + + +
    +     - - - -
    -     - - - - - - - -
    -     - - -
    - - - - - - - - - - -
    -     + + + +
    +     + + + + + + + +
    +     + + +
    + + + + + + + + + + +
    +     From bcdd6d474ae2c64735bbda2ae643e1137f411ecf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 08:42:09 -0700 Subject: [PATCH 08/53] ... --- resources/templates/fb2.xsl | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/templates/fb2.xsl b/resources/templates/fb2.xsl index 9d0abc7d45..273edd71ae 100644 --- a/resources/templates/fb2.xsl +++ b/resources/templates/fb2.xsl @@ -4,6 +4,7 @@ # # # # # copyright 2002 Paul Henry Tremblay # +# Copyright 2011 Kovid Goyal # # # This program is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # From 43fcdf8ca958e699cd91e1082b78a6a293326665 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 09:24:55 -0700 Subject: [PATCH 09/53] Replace leading period in filenames with an underscore --- src/calibre/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 221f5911c6..716e3913fb 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -85,6 +85,8 @@ def sanitize_file_name(name, substitute='_', as_unicode=False): # Windows doesn't like path components that end with a period if one.endswith('.'): one = one[:-1]+'_' + if one.startswith('.'): + one = '_' + one[1:] return one From fe1aa601fe3db699d33df52d4b043230127ee663 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 6 Mar 2011 18:13:47 +0000 Subject: [PATCH 10/53] Fix exception in bulk edit when an enumeration value is missing from the valid set --- src/calibre/gui2/custom_column_widgets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index fa7ba3c56d..8641f9e712 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -795,6 +795,7 @@ class BulkEnumeration(BulkBase, Enumeration): return value def setup_ui(self, parent): + self.parent = parent self.make_widgets(parent, QComboBox) vals = self.col_metadata['display']['enum_values'] self.main_widget.blockSignals(True) From 56a89ca29f90552be35ac4142106b71ee73859bb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 11:42:22 -0700 Subject: [PATCH 11/53] Fix #7250 ("Save to disk" cannot save non-ASCII file name correctly if not convert to ASCII) --- src/calibre/__init__.py | 28 +++++++++++++++++++- src/calibre/library/save_to_disk.py | 14 ++++------ src/calibre/startup.py | 41 ----------------------------- 3 files changed, 32 insertions(+), 51 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 716e3913fb..5ba1aa42de 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -61,8 +61,9 @@ def osx_version(): if m: return int(m.group(1)), int(m.group(2)), int(m.group(3)) - _filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+/]') +_filename_sanitize_unicode = frozenset([u'\\', u'|', u'?', u'*', u'<', + u'"', u':', u'>', u'+', u'/'] + list(map(unichr, xrange(32)))) def sanitize_file_name(name, substitute='_', as_unicode=False): ''' @@ -85,6 +86,31 @@ def sanitize_file_name(name, substitute='_', as_unicode=False): # Windows doesn't like path components that end with a period if one.endswith('.'): one = one[:-1]+'_' + # Names starting with a period are hidden on Unix + if one.startswith('.'): + one = '_' + one[1:] + return one + +def sanitize_file_name_unicode(name, substitute='_'): + ''' + Sanitize the filename `name`. All invalid characters are replaced by `substitute`. + The set of invalid characters is the union of the invalid characters in Windows, + OS X and Linux. Also removes leading and trailing whitespace. + **WARNING:** This function also replaces path separators, so only pass file names + and not full paths to it. + ''' + if not isinstance(name, unicode): + return sanitize_file_name(name, substitute=substitute, as_unicode=True) + chars = [substitute if c in _filename_sanitize_unicode else c for c in + name] + one = u''.join(chars) + one = re.sub(r'\s', ' ', one).strip() + one = re.sub(r'^\.+$', '_', one) + one = one.replace('..', substitute) + # Windows doesn't like path components that end with a period or space + if one and one[-1] in ('.', ' '): + one = one[:-1]+'_' + # Names starting with a period are hidden on Unix if one.startswith('.'): one = '_' + one[1:] return one diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index de586048b7..96c42e6e0e 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -12,13 +12,13 @@ from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ - ascii_filename, sanitize_file_name + ascii_filename from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.meta import set_metadata -from calibre.constants import preferred_encoding, filesystem_encoding +from calibre.constants import preferred_encoding from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata import title_sort -from calibre import strftime, prints +from calibre import strftime, prints, sanitize_file_name_unicode plugboard_any_device_value = 'any device' plugboard_any_format_value = 'any format' @@ -197,12 +197,10 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, format_args[key] = '' components = SafeFormat().safe_format(template, format_args, 'G_C-EXCEPTION!', mi) - components = [x.strip() for x in components.split('/') if x.strip()] + components = [x.strip() for x in components.split('/')] components = [sanitize_func(x) for x in components if x] if not components: components = [str(id)] - components = [x.encode(filesystem_encoding, 'replace') if isinstance(x, - unicode) else x for x in components] if to_lowercase: components = [x.lower() for x in components] if replace_whitespace: @@ -247,7 +245,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, return True, id_, mi.title components = get_components(opts.template, mi, id_, opts.timefmt, length, - ascii_filename if opts.asciiize else sanitize_file_name, + ascii_filename if opts.asciiize else sanitize_file_name_unicode, to_lowercase=opts.to_lowercase, replace_whitespace=opts.replace_whitespace) base_path = os.path.join(root, *components) @@ -329,8 +327,6 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, def _sanitize_args(root, opts): if opts is None: opts = config().parse() - if isinstance(root, unicode): - root = root.encode(filesystem_encoding) root = os.path.abspath(root) opts.template = preprocess_template(opts.template) diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 41b20f3946..c883c43e8a 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -72,47 +72,6 @@ if not _run_once: pass ################################################################################ - # Improve builtin path functions to handle unicode sensibly - - _abspath = os.path.abspath - def my_abspath(path, encoding=sys.getfilesystemencoding()): - ''' - Work around for buggy os.path.abspath. This function accepts either byte strings, - in which it calls os.path.abspath, or unicode string, in which case it first converts - to byte strings using `encoding`, calls abspath and then decodes back to unicode. - ''' - to_unicode = False - if encoding is None: - encoding = preferred_encoding - if isinstance(path, unicode): - path = path.encode(encoding) - to_unicode = True - res = _abspath(path) - if to_unicode: - res = res.decode(encoding) - return res - - os.path.abspath = my_abspath - - _join = os.path.join - def my_join(a, *p): - encoding=sys.getfilesystemencoding() - if not encoding: - encoding = preferred_encoding - p = [a] + list(p) - _unicode = False - for i in p: - if isinstance(i, unicode): - _unicode = True - break - p = [i.encode(encoding) if isinstance(i, unicode) else i for i in p] - - res = _join(*p) - if _unicode: - res = res.decode(encoding) - return res - - os.path.join = my_join def local_open(name, mode='r', bufsize=-1): ''' From 0a22a234cd80511a6c0eb7419ca536c36c0291d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 11:46:04 -0700 Subject: [PATCH 12/53] ... --- src/calibre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5ba1aa42de..fa9a8f2404 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -84,7 +84,7 @@ def sanitize_file_name(name, substitute='_', as_unicode=False): one = one.decode(filesystem_encoding) one = one.replace('..', substitute) # Windows doesn't like path components that end with a period - if one.endswith('.'): + if one and one[-1] in ('.', ' '): one = one[:-1]+'_' # Names starting with a period are hidden on Unix if one.startswith('.'): From 1920f3e7bdb09006458571a4f62ddb834d090119 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 19:34:58 -0700 Subject: [PATCH 13/53] ... --- src/calibre/trac/bzr_commit_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/trac/bzr_commit_plugin.py b/src/calibre/trac/bzr_commit_plugin.py index 2f91804315..325bac7a79 100644 --- a/src/calibre/trac/bzr_commit_plugin.py +++ b/src/calibre/trac/bzr_commit_plugin.py @@ -19,7 +19,7 @@ in the working tree you want to use it with:: trac_reponame_password = ''' -import os, re, xmlrpclib +import os, re, xmlrpclib, subprocess from bzrlib.builtins import cmd_commit as _cmd_commit, tree_files from bzrlib import branch import bzrlib @@ -115,5 +115,7 @@ class cmd_commit(_cmd_commit): server.ticket.update(int(bug), msg, {'status':'closed', 'resolution':'fixed'}, True) + subprocess.Popen('/home/kovid/work/kde/mail.py -f --delay 10'.split()) + bzrlib.commands.register_command(cmd_commit) From 62105efcc9ac1bcca82f019ad12aa4868209ebdf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 22:04:56 -0700 Subject: [PATCH 14/53] Improved support for dragging and dropping cover images directly from web browsers into calibre. You can drop the images onto the cover in calibre and it will be replaced. Drag and drop implementations in various browsers/OSes are very flaky, so your mileage may vary, depending on the site you are dragigng from, the browser you are using and your operating system. If it doesn't work you can alway right click and Copy Image, then right click and paste it in calibre --- src/calibre/gui2/actions/add.py | 23 ++- src/calibre/gui2/book_details.py | 73 ++++--- src/calibre/gui2/dnd.py | 325 +++++++++++++++++++++++++++++++ src/calibre/gui2/init.py | 3 + src/calibre/gui2/widgets.py | 94 +++++---- 5 files changed, 438 insertions(+), 80 deletions(-) create mode 100644 src/calibre/gui2/dnd.py diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 0040acea28..cf67cd6cfa 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -204,15 +204,29 @@ class AddAction(InterfaceAction): to_device = self.gui.stack.currentIndex() != 0 self._add_books(paths, to_device) - def files_dropped_on_book(self, event, paths): + def remote_file_dropped_on_book(self, url, fname): + if self.gui.current_view() is not self.gui.library_view: + return + db = self.gui.library_view.model().db + current_idx = self.gui.library_view.currentIndex() + if not current_idx.isValid(): return + cid = db.id(current_idx.row()) + from calibre.gui2.dnd import DownloadDialog + d = DownloadDialog(url, fname, self.gui) + d.start_download() + if d.err is None: + self.files_dropped_on_book(None, [d.fpath], cid=cid) + + def files_dropped_on_book(self, event, paths, cid=None): accept = False if self.gui.current_view() is not self.gui.library_view: return db = self.gui.library_view.model().db cover_changed = False current_idx = self.gui.library_view.currentIndex() - if not current_idx.isValid(): return - cid = db.id(current_idx.row()) + if cid is None: + if not current_idx.isValid(): return + cid = db.id(current_idx.row()) if cid is None else cid for path in paths: ext = os.path.splitext(path)[1].lower() if ext: @@ -227,8 +241,9 @@ class AddAction(InterfaceAction): elif ext in BOOK_EXTENSIONS: db.add_format_with_hooks(cid, ext, path, index_is_id=True) accept = True - if accept: + if accept and event is not None: event.accept() + if current_idx.isValid(): self.gui.library_view.model().current_changed(current_idx, current_idx) if cover_changed: if self.gui.cover_flow: diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 63deccb2f0..a28759486e 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, collections, sys +import collections, sys from Queue import Queue from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \ @@ -14,7 +14,8 @@ from PyQt4.Qt import QPixmap, QSize, QWidget, Qt, pyqtSignal, QUrl, \ from PyQt4.QtWebKit import QWebView from calibre import fit_image, prepare_string_for_xml -from calibre.gui2.widgets import IMAGE_EXTENSIONS +from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ + IMAGE_EXTENSIONS, dnd_has_extension from calibre.ebooks import BOOK_EXTENSIONS from calibre.constants import preferred_encoding from calibre.library.comments import comments_to_html @@ -165,11 +166,12 @@ class CoverView(QWidget): # {{{ def copy_to_clipboard(self): QApplication.instance().clipboard().setPixmap(self.pixmap) - def paste_from_clipboard(self): - cb = QApplication.instance().clipboard() - pmap = cb.pixmap() - if pmap.isNull() and cb.supportsSelection(): - pmap = cb.pixmap(cb.Selection) + def paste_from_clipboard(self, pmap=None): + if not isinstance(pmap, QPixmap): + cb = QApplication.instance().clipboard() + pmap = cb.pixmap() + if pmap.isNull() and cb.supportsSelection(): + pmap = cb.pixmap(cb.Selection) if not pmap.isNull(): self.pixmap = pmap self.do_layout() @@ -226,6 +228,7 @@ class BookInfo(QWebView): self._link_clicked = False self.setAttribute(Qt.WA_OpaquePaintEvent, False) palette = self.palette() + self.setAcceptDrops(False) palette.setBrush(QPalette.Base, Qt.transparent) self.page().setPalette(palette) @@ -388,36 +391,50 @@ class BookDetails(QWidget): # {{{ show_book_info = pyqtSignal() open_containing_folder = pyqtSignal(int) view_specific_format = pyqtSignal(int, object) - - # Drag 'n drop {{{ - DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS + remote_file_dropped = pyqtSignal(object, object) files_dropped = pyqtSignal(object, object) cover_changed = pyqtSignal(object, object) - # application/x-moz-file-promise-url - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] + # Drag 'n drop {{{ + DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS) or \ + dnd_has_image(md): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - self.files_dropped.emit(event, paths) + md = event.mimeData() + + x, y = dnd_get_image(md) + if x is not None: + # We have an image, set cover + event.accept() + if y is None: + # Local image + self.cover_view.paste_from_clipboard(x) + else: + self.remote_file_dropped.emit(x, y) + # We do not support setting cover *and* adding formats for + # a remote drop, anyway, so return + return + + # Now look for ebook files + urls, filenames = dnd_get_files(md, BOOK_EXTENSIONS) + if not urls: + # Nothing found + return + + if not filenames: + # Local files + self.files_dropped.emit(event, urls) + else: + # Remote files, use the first file + self.remote_file_dropped.emit(urls[0], filenames[0]) + event.accept() + def dragMoveEvent(self, event): event.acceptProposedAction() diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py new file mode 100644 index 0000000000..928de72578 --- /dev/null +++ b/src/calibre/gui2/dnd.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import posixpath, os, urllib, re +from urlparse import urlparse, urlunparse +from threading import Thread +from Queue import Queue, Empty + +from PyQt4.Qt import QPixmap, Qt, QDialog, QLabel, QVBoxLayout, \ + QDialogButtonBox, QProgressBar, QTimer + +from calibre.constants import DEBUG, iswindows +from calibre.ptempfile import PersistentTemporaryFile +from calibre import browser, as_unicode, prints +from calibre.gui2 import error_dialog + +IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'bmp'] + +class Worker(Thread): # {{{ + + def __init__(self, url, fpath, rq): + Thread.__init__(self) + self.url, self.fpath = url, fpath + self.daemon = True + self.rq = rq + self.err = self.tb = None + + def run(self): + try: + br = browser() + br.retrieve(self.url, self.fpath, self.callback) + except Exception, e: + self.err = as_unicode(e) + import traceback + self.tb = traceback.format_exc() + + def callback(self, a, b, c): + self.rq.put((a, b, c)) +# }}} + +class DownloadDialog(QDialog): # {{{ + + def __init__(self, url, fname, parent): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Download %s')%fname) + self.l = QVBoxLayout(self) + self.purl = urlparse(url) + self.msg = QLabel(_('Downloading %s from %s')%(fname, + self.purl.netloc)) + self.msg.setWordWrap(True) + self.l.addWidget(self.msg) + self.pb = QProgressBar(self) + self.pb.setMinimum(0) + self.pb.setMaximum(0) + self.l.addWidget(self.pb) + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self) + self.l.addWidget(self.bb) + self.bb.rejected.connect(self.reject) + sz = self.sizeHint() + self.resize(max(sz.width(), 400), sz.height()) + + fpath = PersistentTemporaryFile(os.path.splitext(fname)[1]) + fpath.close() + self.fpath = fpath.name + + self.worker = Worker(url, self.fpath, Queue()) + self.rejected = False + + def reject(self): + self.rejected = True + QDialog.reject(self) + + def start_download(self): + self.worker.start() + QTimer.singleShot(50, self.update) + self.exec_() + if self.worker.err is not None: + error_dialog(self.parent(), _('Download failed'), + _('Failed to download from %r with error: %s')%( + self.worker.url, self.worker.err), + det_msg=self.worker.tb, show=True) + + def update(self): + if self.rejected: + return + + try: + progress = self.worker.rq.get_nowait() + except Empty: + pass + else: + self.update_pb(progress) + + if not self.worker.is_alive(): + return self.accept() + QTimer.singleShot(50, self.update) + + def update_pb(self, progress): + transferred, block_size, total = progress + if total == -1: + self.pb.setMaximum(0) + self.pb.setMinimum(0) + self.pb.setValue(0) + else: + so_far = transferred * block_size + self.pb.setMaximum(max(total, so_far)) + self.pb.setValue(so_far) + + @property + def err(self): + return self.worker.err + +# }}} + +def dnd_has_image(md): + return md.hasImage() + +def data_as_string(f, md): + raw = bytes(md.data(f)) + if '/x-moz' in f: + try: + raw = raw.decode('utf-16') + except: + pass + return raw + +def dnd_has_extension(md, extensions): + if DEBUG: + prints('Debugging DND event') + for f in md.formats(): + f = unicode(f) + prints(f, repr(data_as_string(f, md))[:300], '\n') + print () + if has_firefox_ext(md, extensions): + return True + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + if DEBUG: + prints('URLS:', urls) + prints('Paths:', [u2p(x) for x in purls]) + + exts = frozenset([posixpath.splitext(u.path)[1][1:].lower() for u in + purls]) + return bool(exts.intersection(frozenset(extensions))) + return False + +def u2p(url): + path = url.path + if iswindows: + if path.startswith('/'): + path = path[1:] + ans = path.replace('/', os.sep) + if os.path.exists(ans): + return ans + # Try unquoting the URL + return urllib.unquote(ans) + +def dnd_get_image(md, image_exts=IMAGE_EXTENSIONS): + ''' + Get the image in the QMimeData object md. + + :return: None, None if no image is found + QPixmap, None if an image is found, the pixmap is guaranteed not + null + url, filename if a URL that points to an image is found + ''' + if dnd_has_image(md): + for x in md.formats(): + x = unicode(x) + if x.startswith('image/'): + cdata = bytes(md.data(x)) + pmap = QPixmap() + pmap.loadFromData(cdata) + if not pmap.isNull(): + return pmap, None + break + + # No image, look for a URL pointing to an image + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + # First look for a local file + images = [u2p(x) for x in purls if x.scheme in ('', 'file') and + posixpath.splitext(urllib.unquote(x.path))[1][1:].lower() in + image_exts] + images = [x for x in images if os.path.exists(x)] + p = QPixmap() + for path in images: + try: + with open(path, 'rb') as f: + p.loadFromData(f.read()) + except: + continue + if not p.isNull(): + return p, None + + # No local images, look for remote ones + + # First, see if this is from Firefox + rurl, fname = get_firefox_rurl(md, image_exts) + + if rurl and fname: + return rurl, fname + # Look through all remaining URLs + remote_urls = [x for x in purls if x.scheme in ('http', 'https', + 'ftp') and posixpath.splitext(x.path)[1][1:].lower() in image_exts] + if remote_urls: + rurl = remote_urls[0] + fname = posixpath.basename(urllib.unquote(rurl.path)) + return urlunparse(rurl), fname + + return None, None + +def dnd_get_files(md, exts): + ''' + Get the file in the QMimeData object md with an extension that is one of + the extensions in exts. + + :return: None, None if no file is found + [paths], None if a local file is found + [urls], [filenames] if URLs that point to a files are found + ''' + # Look for a URL pointing to a file + if md.hasUrls(): + urls = [unicode(u.toString()) for u in + md.urls()] + purls = [urlparse(u) for u in urls] + # First look for a local file + local_files = [u2p(x) for x in purls if x.scheme in ('', 'file') and + posixpath.splitext(urllib.unquote(x.path))[1][1:].lower() in + exts] + local_files = [x for x in local_files if os.path.exists(x)] + if local_files: + return local_files, None + + # No local files, look for remote ones + + # First, see if this is from Firefox + rurl, fname = get_firefox_rurl(md, exts) + if rurl and fname: + return [rurl], [fname] + + # Look through all remaining URLs + remote_urls = [x for x in purls if x.scheme in ('http', 'https', + 'ftp') and posixpath.splitext(x.path)[1][1:].lower() in exts] + if remote_urls: + filenames = [posixpath.basename(urllib.unquote(rurl.path)) for rurl in + remote_urls] + return [urlunparse(x) for x in remote_urls], filenames + + return None, None + +def _get_firefox_pair(md, exts, url, fname): + url = bytes(md.data(url)).decode('utf-16') + fname = bytes(md.data(fname)).decode('utf-16') + while url.endswith('\x00'): + url = url[:-1] + while fname.endswith('\x00'): + fname = fname[:-1] + if not url or not fname: + return None, None + ext = posixpath.splitext(fname)[1][1:].lower() + # Weird firefox bug on linux + ext = {'jpe':'jpg', 'epu':'epub', 'mob':'mobi'}.get(ext, ext) + fname = os.path.splitext(fname)[0] + '.' + ext + if DEBUG: + prints('Firefox file promise:', url, fname) + if ext not in exts: + fname = url = None + return url, fname + + +def get_firefox_rurl(md, exts): + formats = frozenset([unicode(x) for x in md.formats()]) + url = fname = None + if 'application/x-moz-file-promise-url' in formats and \ + 'application/x-moz-file-promise-dest-filename' in formats: + try: + url, fname = _get_firefox_pair(md, exts, + 'application/x-moz-file-promise-url', + 'application/x-moz-file-promise-dest-filename') + except: + if DEBUG: + import traceback + traceback.print_exc() + if url is None and 'text/x-moz-url-data' in formats and \ + 'text/x-moz-url-desc' in formats: + try: + url, fname = _get_firefox_pair(md, exts, + 'text/x-moz-url-data', 'text/x-moz-url-desc') + except: + if DEBUG: + import traceback + traceback.print_exc() + + if url is None and '_NETSCAPE_URL' in formats: + try: + raw = bytes(md.data('_NETSCAPE_URL')) + raw = raw.decode('utf-8') + lines = raw.splitlines() + if len(lines) > 1 and re.match(r'[a-z]+://', lines[1]) is None: + url, fname = lines[:2] + ext = posixpath.splitext(fname)[1][1:].lower() + if ext not in exts: + fname = url = None + except: + if DEBUG: + import traceback + traceback.print_exc() + if DEBUG: + prints('Firefox rurl:', url, fname) + return url, fname + +def has_firefox_ext(md, exts): + return bool(get_firefox_rurl(md, exts)[0]) + diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index e8c2712c83..9119a8da77 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -264,6 +264,9 @@ class LayoutMixin(object): # {{{ self.book_details.files_dropped.connect(self.iactions['Add Books'].files_dropped_on_book) self.book_details.cover_changed.connect(self.bd_cover_changed, type=Qt.QueuedConnection) + self.book_details.remote_file_dropped.connect( + self.iactions['Add Books'].remote_file_dropped_on_book, + type=Qt.QueuedConnection) self.book_details.open_containing_folder.connect(self.iactions['View'].view_folder_for_id) self.book_details.view_specific_format.connect(self.iactions['View'].view_format_by_id) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index aa9d6c8b9f..8ebf9c2c21 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal ' ''' Miscellaneous widgets used in the GUI ''' -import re, os, traceback +import re, traceback from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \ QListWidgetItem, QTextCharFormat, QApplication, \ @@ -22,6 +22,8 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.metadata.meta import metadata_from_filename from calibre.utils.config import prefs, XMLConfig, tweaks from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator +from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ + IMAGE_EXTENSIONS, dnd_has_extension, DownloadDialog history = XMLConfig('history') @@ -141,36 +143,35 @@ class FilenamePattern(QWidget, Ui_Form): return pat -IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'gif', 'png', 'bmp'] - class FormatList(QListWidget): DROPABBLE_EXTENSIONS = BOOK_EXTENSIONS formats_dropped = pyqtSignal(object, object) delete_format = pyqtSignal() - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] - def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - self.formats_dropped.emit(event, paths) + md = event.mimeData() + # Now look for ebook files + urls, filenames = dnd_get_files(md, self.DROPABBLE_EXTENSIONS) + if not urls: + # Nothing found + return + + if not filenames: + # Local files + self.formats_dropped.emit(event, urls) + else: + # Remote files, use the first file + d = DownloadDialog(urls[0], filenames[0], self) + d.start_download() + if d.err is None: + self.formats_dropped.emit(event, [d.fpath]) + def dragMoveEvent(self, event): event.acceptProposedAction() @@ -183,7 +184,7 @@ class FormatList(QListWidget): class ImageDropMixin(object): # {{{ ''' - Adds support for dropping images onto widgets and a contect menu for + Adds support for dropping images onto widgets and a context menu for copy/pasting images. ''' DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS @@ -191,39 +192,36 @@ class ImageDropMixin(object): # {{{ def __init__(self): self.setAcceptDrops(True) - @classmethod - def paths_from_event(cls, event): - ''' - Accept a drop event and return a list of paths that can be read from - and represent files with extensions. - ''' - if event.mimeData().hasFormat('text/uri-list'): - urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()] - urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)] - return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS] - def dragEnterEvent(self, event): - if int(event.possibleActions() & Qt.CopyAction) + \ - int(event.possibleActions() & Qt.MoveAction) == 0: - return - paths = self.paths_from_event(event) - if paths: + md = event.mimeData() + if dnd_has_extension(md, self.DROPABBLE_EXTENSIONS) or \ + dnd_has_image(md): event.acceptProposedAction() def dropEvent(self, event): - paths = self.paths_from_event(event) event.setDropAction(Qt.CopyAction) - for path in paths: - pmap = QPixmap() - pmap.load(path) - if not pmap.isNull(): - self.handle_image_drop(path, pmap) - event.accept() - break + md = event.mimeData() - def handle_image_drop(self, path, pmap): + x, y = dnd_get_image(md) + if x is not None: + # We have an image, set cover + event.accept() + if y is None: + # Local image + self.handle_image_drop(x) + else: + # Remote files, use the first file + d = DownloadDialog(x, y, self) + d.start_download() + if d.err is None: + pmap = QPixmap() + pmap.loadFromData(open(d.fpath, 'rb').read()) + if not pmap.isNull(): + self.handle_image_drop(pmap) + + def handle_image_drop(self, pmap): self.set_pixmap(pmap) - self.cover_changed.emit(open(path, 'rb').read()) + self.cover_changed.emit(pixmap_to_data(pmap)) def dragMoveEvent(self, event): event.acceptProposedAction() From 39296a9b8d49f2544df6363806e1251080f623c6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 7 Mar 2011 11:56:27 +0000 Subject: [PATCH 15/53] Fix #9316: clicking saved search in tag browser does not work. Also fixes right-click searching not marking the node, incorrect permission to edit news and identifier values, and a few other oddities. This fix refactors tags_view to eliminate the overloading of the is_editable indicator. Nodes can now be independently set as searchable or editable. --- src/calibre/gui2/tag_view.py | 42 ++++++++++++++++++-------------- src/calibre/library/database2.py | 17 ++++++++----- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 11b696d861..c4871880a4 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -246,7 +246,7 @@ class TagsView(QTreeView): # {{{ self.add_subcategory.emit(key) return if action == 'search_category': - self.tags_marked.emit(key + ':' + search_state) + self._toggle(index, set_to=search_state) return if action == 'delete_user_category': self.delete_user_category.emit(key) @@ -320,6 +320,9 @@ class TagsView(QTreeView): # {{{ self.context_menu.addAction(_('Edit sort for %s')%tag.name, partial(self.context_menu_handler, action='edit_author_sort', index=tag.id)) + + # is_editable is also overloaded to mean 'can be added + # to a user category' m = self.context_menu.addMenu(self.user_category_icon, _('Add %s to user category')%tag.name) nt = self.model().category_node_tree @@ -345,7 +348,7 @@ class TagsView(QTreeView): # {{{ partial(self.context_menu_handler, action='delete_item_from_user_category', key = key, index = tag_item)) - # Add the search for value items + # Add the search for value items. All leaf nodes are searchable self.context_menu.addAction(self.search_icon, _('Search for %s')%tag.name, partial(self.context_menu_handler, action='search', @@ -373,7 +376,6 @@ class TagsView(QTreeView): # {{{ action='delete_user_category', key=key)) self.context_menu.addSeparator() # Hide/Show/Restore categories - #if not key.startswith('@') or key.find('.') < 0: self.context_menu.addAction(_('Hide category %s') % category, partial(self.context_menu_handler, action='hide', category=key)) @@ -384,16 +386,21 @@ class TagsView(QTreeView): # {{{ m.addAction(self.db.field_metadata[col]['name'], partial(self.context_menu_handler, action='show', category=col)) - # search by category - if key != 'search': + # search by category. Some categories are not searchable, such + # as search and news + if item.tag.is_searchable: self.context_menu.addAction(self.search_icon, _('Search for books in category %s')%category, - partial(self.context_menu_handler, action='search_category', - key=key, search_state='true')) + partial(self.context_menu_handler, + action='search_category', + index=self._model.createIndex(item.row(), 0, item), + search_state=TAG_SEARCH_STATES['mark_plus'])) self.context_menu.addAction(self.search_icon, _('Search for books not in category %s')%category, - partial(self.context_menu_handler, action='search_category', - key=key, search_state='false')) + partial(self.context_menu_handler, + action='search_category', + index=self._model.createIndex(item.row(), 0, item), + search_state=TAG_SEARCH_STATES['mark_minus'])) # Offer specific editors for tags/series/publishers/saved searches self.context_menu.addSeparator() if key in ['tags', 'publisher', 'series'] or \ @@ -559,8 +566,10 @@ class TagTreeItem(object): # {{{ self.bold_font = QVariant(self.bold_font) self.category_key = category_key self.temporary = temporary - self.tag = Tag(data) - self.tag.is_hierarchical = category_key.startswith('@') + self.tag = Tag(data, category=category_key, + is_editable=category_key not in ['news', 'search', 'identifiers'], + is_searchable=category_key not in ['news', 'search']) + elif self.type == self.TAG: self.icon_state_map[0] = QVariant(data.icon) self.tag = data @@ -660,14 +669,12 @@ class TagTreeItem(object): # {{{ ''' set_to: None => advance the state, otherwise a value from TAG_SEARCH_STATES ''' - basic_search_ok = self.tag.is_editable or \ - self.tag.category == 'formats' or self.tag.category == 'rating' if set_to is None: while True: self.tag.state = (self.tag.state + 1)%5 if self.tag.state == TAG_SEARCH_STATES['mark_plus'] or \ self.tag.state == TAG_SEARCH_STATES['mark_minus']: - if basic_search_ok: + if self.tag.is_searchable: break elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\ self.tag.state == TAG_SEARCH_STATES['mark_minusminus']: @@ -766,6 +773,7 @@ class TagsModel(QAbstractItemModel): # {{{ self.category_nodes.append(node) node.can_be_edited = (not is_gst) and (i == (len(path_parts)-1)) node.is_gst = is_gst + node.tag.is_hierarchical = not is_gst if not is_gst: tree_root[p] = {} tree_root = tree_root[p] @@ -1240,9 +1248,6 @@ class TagsModel(QAbstractItemModel): # {{{ n.id_set |= tag.id_set category_child_map[tag.name, tag.category] = n self.endInsertRows() - tag.is_editable = key != 'formats' and (key == 'news' or \ - self.db.field_metadata[tag.category]['datatype'] in \ - ['text', 'series', 'enumeration']) else: for i,comp in enumerate(components): if i == 0: @@ -1258,12 +1263,13 @@ class TagsModel(QAbstractItemModel): # {{{ if i < len(components)-1: t = copy.copy(tag) t.original_name = '.'.join(components[:i+1]) + # This 'manufactured' intermediate node can + # be searched, but cannot be edited. t.is_editable = False else: t = tag if not in_uc: t.original_name = t.name - t.is_editable = True t.is_hierarchical = True t.name = comp self.beginInsertRows(category_index, 999999, 1) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 38b70fc2bf..f1a5884e9f 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -47,13 +47,15 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile class Tag(object): def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, - tooltip=None, icon=None, category=None, id_set=None): + tooltip=None, icon=None, category=None, id_set=None, + is_editable = True, is_searchable=True): self.name = self.original_name = name self.id = id self.count = count self.state = state self.is_hierarchical = False - self.is_editable = True + self.is_editable = is_editable + self.is_searchable = is_searchable self.id_set = id_set self.avg_rating = avg/2.0 if avg is not None else 0 self.sort = sort @@ -1439,10 +1441,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): reverse=True items.sort(key=kf, reverse=reverse) + is_editable = category not in ['news', 'rating'] categories[category] = [tag_class(formatter(r.n), count=r.c, id=r.id, avg=avgr(r), sort=r.s, icon=icon, tooltip=tooltip, category=category, - id_set=r.id_set) + id_set=r.id_set, is_editable=is_editable) for r in items] #print 'end phase "tags list":', time.clock() - last, 'seconds' @@ -1479,7 +1482,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): all=False) if count > 0: categories['formats'].append(Tag(fmt, count=count, icon=icon, - category='formats')) + category='formats', is_editable=False)) if sort == 'popularity': categories['formats'].sort(key=lambda x: x.count, reverse=True) @@ -1507,7 +1510,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): all=False) if count > 0: categories['identifiers'].append(Tag(ident, count=count, icon=icon, - category='identifiers')) + category='identifiers', + is_editable=False)) if sort == 'popularity': categories['identifiers'].sort(key=lambda x: x.count, reverse=True) @@ -1566,7 +1570,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon = icon_map['search'] for srch in saved_searches().names(): items.append(Tag(srch, tooltip=saved_searches().lookup(srch), - sort=srch, icon=icon, category='search')) + sort=srch, icon=icon, category='search', + is_editable=False)) if len(items): if icon_map is not None: icon_map['search'] = icon_map['search'] From 5ade4834b9f73cc9520d148b8d434e2c44b0bfa7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 7 Mar 2011 14:12:56 +0000 Subject: [PATCH 16/53] Make search/replace work with the new identifiers field --- src/calibre/gui2/dialogs/metadata_bulk.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index c1627d7e12..af540ea4c3 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -495,6 +495,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): val = mi.get(field, None) if isinstance(val, (int, float, bool)): val = str(val) + elif fm['is_csp']: + # convert the csp dict into a list + val = [u'%s:%s'%(t[0], t[1]) for t in val.iteritems()] if val is None: val = [] if fm['is_multiple'] else [''] elif not fm['is_multiple']: @@ -635,6 +638,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if dest_mode != 0: dest_val = mi.get(dest, '') + if self.db.metadata_for_field(dest)['is_csp']: + # convert the csp dict into a list + dest_val = [u'%s:%s'%(t[0], t[1]) for t in dest_val.iteritems()] if dest_val is None: dest_val = [] elif not isinstance(dest_val, list): @@ -717,6 +723,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): 'Book title %s not processed')%mi.title, show=True) return + # convert the colon-separated pair strings back into a dict, which + # is what set_identifiers wants + if dfm['is_csp']: + val = dict([(t.split(':')) for t in val]) else: val = self.s_r_replace_mode_separator().join(val) if dest == 'title' and len(val) == 0: From 221420d44aaa12f82874d3e924d4151f688a0fd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 08:36:58 -0700 Subject: [PATCH 17/53] Komchadluek by ballsai. Fixes #9317 (Submit a recipe : KomChadLuek) --- resources/recipes/komchadluek.recipe | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 resources/recipes/komchadluek.recipe diff --git a/resources/recipes/komchadluek.recipe b/resources/recipes/komchadluek.recipe new file mode 100644 index 0000000000..5f0d2f58a2 --- /dev/null +++ b/resources/recipes/komchadluek.recipe @@ -0,0 +1,46 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class KomChadLuek(BasicNewsRecipe): + + title= 'KomChadLuek' + description = 'Komchadluek News' + __author__ = 'ballsaii and Chotechai' + __license__ = 'GPL v3' + publisher= 'Nation Media Group' + category = 'news, Thai' + language = 'th' + + oldest_article = 1 + max_articles_per_feed = 100 + no_stylesheets= True + remove_javascript=True + + cover_url = 'http://www.komchadluek.net/images_layout2/komchadluek_headerlogo.png' + + keep_only_tags = [] + keep_only_tags.append(dict(name = 'h2')) + keep_only_tags.append(dict(name = 'div', attrs={'id':'news_detail_news'})) + + remove_tags_after=[dict(name='hr')] + + feeds =( +(u'\u0e01\u0e32\u0e23\u0e40\u0e21\u0e37\u0e2d\u0e07','http://www.komchadluek.net/rss/politic.xml'), +(u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28','http://www.komchadluek.net/rss/sport.xml'), +(u'\u0e40\u0e01\u0e29\u0e15\u0e23','http://www.komchadluek.net/rss/agriculture.xml'), +(u'\u0e15\u0e48\u0e32\u0e07\u0e1b\u0e23\u0e30\u0e40\u0e17\u0e28','http://www.komchadluek.net/rss/foreign.xml'), +(u'\u0e1a\u0e31\u0e19\u0e40\u0e17\u0e34\u0e07','http://www.komchadluek.net/rss/entertainment.xml'), +(u'\u0e1c\u0e39\u0e49\u0e2b\u0e0d\u0e34\u0e07-\u0e41\u0e1f\u0e0a\u0e31\u0e48\u0e19','http://www.komchadluek.net/rss/fashion.xml'), +(u'\u0e1e\u0e23\u0e30\u0e40\u0e04\u0e23\u0e37\u0e48\u0e2d\u0e07','http://www.komchadluek.net/rss/amulet.xml'), +(u'\u0e20\u0e39\u0e21\u0e34\u0e20\u0e32\u0e04-\u0e1b\u0e23\u0e30\u0e0a\u0e32\u0e04\u0e21\u0e17\u0e49\u0e2d\u0e07\u0e16\u0e34\u0e48\u0e19','http://www.komchadluek.net/rss/local.xml'), +(u'\u0e25\u0e38\u0e07\u0e41\u0e08\u0e48\u0e21','http://www.komchadluek.net/rss/unclecham.xml'), +(u'\u0e44\u0e25\u0e1f\u0e4c\u0e2a\u0e44\u0e15\u0e25\u0e4c','http://www.komchadluek.net/rss/lifestyle.xml'), +(u'\u0e40\u0e28\u0e23\u0e29\u0e10\u0e01\u0e34\u0e08-\u0e01\u0e32\u0e23\u0e15\u0e25\u0e32\u0e14','http://www.komchadluek.net/rss/economic.xml'), +(u'\u0e2d\u0e32\u0e2b\u0e32\u0e23','http://www.komchadluek.net/rss/food.xml'), +(u'\u0e04\u0e19\u0e23\u0e31\u0e01\u0e1a\u0e49\u0e32\u0e19-\u0e22\u0e32\u0e19\u0e22\u0e19\u0e15\u0e4c','http://www.komchadluek.net/rss/homecar.xml'), +(u'\u0e14\u0e39\u0e14\u0e27\u0e07-\u0e42\u0e2b\u0e23\u0e32\u0e28\u0e32\u0e2a\u0e15\u0e23\u0e4c','http://www.komchadluek.net/rss/horoscope.xml'), +(u'\u0e27\u0e34\u0e17\u0e22\u0e4c\u0e28\u0e32\u0e2a\u0e15\u0e23\u0e4c-\u0e44\u0e2d\u0e17\u0e35','http://www.komchadluek.net/rss/scienceit.xml'), +(u'\u0e28\u0e32\u0e2a\u0e19\u0e32 \u0e28\u0e34\u0e25\u0e1b\u0e30-\u0e27\u0e31\u0e12\u0e19\u0e18\u0e23\u0e23\u0e21 \u0e2a\u0e32\u0e18\u0e32\u0e23\u0e13\u0e2a\u0e38\u0e02','http://www.komchadluek.net/rss/artculture.xml'), +(u'\u0e01\u0e32\u0e23\u0e28\u0e36\u0e01\u0e29\u0e32', 'http://www.komchadluek.net/rss/education.xml'), +(u'\u0e1a\u0e17\u0e04\u0e27\u0e32\u0e21','http://www.komchadluek.net/rss/article.xml'), +(u'\u0e2d\u0e32\u0e0a\u0e0d\u0e32\u0e01\u0e23\u0e23\u0e21', 'http://www.komchadluek.net/rss/crime.xml') +) From 9526d1b8b499c371415927924741186e2bdf20d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 08:44:28 -0700 Subject: [PATCH 18/53] Fix #9320 (Metadata class caching tags values) --- src/calibre/ebooks/metadata/book/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index a2599ab0b5..feb6ff4bb9 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -130,7 +130,7 @@ class Metadata(object): self.set_identifiers(val) elif field in STANDARD_METADATA_FIELDS: if val is None: - val = NULL_VALUES.get(field, None) + val = copy.copy(NULL_VALUES.get(field, None)) _data[field] = val elif field in _data['user_metadata'].iterkeys(): _data['user_metadata'][field]['#value#'] = val From db63283bf226d1672ba4dd8a47bc2a72431296ca Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 7 Mar 2011 15:48:14 +0000 Subject: [PATCH 19/53] Fix problem where Metadata NULL_VALUE can be corrupted. --- src/calibre/ebooks/metadata/book/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index a2599ab0b5..feb6ff4bb9 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -130,7 +130,7 @@ class Metadata(object): self.set_identifiers(val) elif field in STANDARD_METADATA_FIELDS: if val is None: - val = NULL_VALUES.get(field, None) + val = copy.copy(NULL_VALUES.get(field, None)) _data[field] = val elif field in _data['user_metadata'].iterkeys(): _data['user_metadata'][field]['#value#'] = val From bf68165605aa3465e4678ecc9abc07c2a020677a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 08:50:14 -0700 Subject: [PATCH 20/53] Update NRC Handelsblad Epub version --- resources/recipes/nrc-nl-epub.recipe | 50 +++++++++++++++++++--------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/resources/recipes/nrc-nl-epub.recipe b/resources/recipes/nrc-nl-epub.recipe index da9b9195ce..2d190e4d0a 100644 --- a/resources/recipes/nrc-nl-epub.recipe +++ b/resources/recipes/nrc-nl-epub.recipe @@ -1,14 +1,14 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- -#Based on Lars Jacob's Taz Digiabo recipe +#Based on veezh's original recipe and Kovid Goyal's New York Times recipe __license__ = 'GPL v3' -__copyright__ = '2010, veezh' +__copyright__ = '2011, Snaab' ''' www.nrc.nl ''' -import os, urllib2, zipfile +import os, zipfile import time from calibre.web.feeds.news import BasicNewsRecipe from calibre.ptempfile import PersistentTemporaryFile @@ -17,41 +17,59 @@ from calibre.ptempfile import PersistentTemporaryFile class NRCHandelsblad(BasicNewsRecipe): title = u'NRC Handelsblad' - description = u'De EPUB-versie van NRC' + description = u'De ePaper-versie van NRC' language = 'nl' lang = 'nl-NL' + needs_subscription = True - __author__ = 'veezh' + __author__ = 'Snaab' conversion_options = { 'no_default_epub_cover' : True } + def get_browser(self): + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open('http://login.nrc.nl/login') + br.select_form(nr=0) + br['username'] = self.username + br['password'] = self.password + br.submit() + return br + def build_index(self): + today = time.strftime("%Y%m%d") + domain = "http://digitaleeditie.nrc.nl" url = domain + "/digitaleeditie/helekrant/epub/nrc_" + today + ".epub" -# print url + #print url try: - f = urllib2.urlopen(url) - except urllib2.HTTPError: + br = self.get_browser() + f = br.open(url) + except: self.report_progress(0,_('Kan niet inloggen om editie te downloaden')) raise ValueError('Krant van vandaag nog niet beschikbaar') + tmp = PersistentTemporaryFile(suffix='.epub') self.report_progress(0,_('downloading epub')) tmp.write(f.read()) - tmp.close() - - zfile = zipfile.ZipFile(tmp.name, 'r') - self.report_progress(0,_('extracting epub')) - - zfile.extractall(self.output_dir) + f.close() + br.close() + if zipfile.is_zipfile(tmp): + try: + zfile = zipfile.ZipFile(tmp.name, 'r') + zfile.extractall(self.output_dir) + self.report_progress(0,_('extracting epub')) + except zipfile.BadZipfile: + self.report_progress(0,_('BadZip error, continuing')) tmp.close() - index = os.path.join(self.output_dir, 'content.opf') + index = os.path.join(self.output_dir, 'metadata.opf') self.report_progress(1,_('epub downloaded and extracted')) From 450b8f4341379d69e4f24de34971b591f9dce9f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 08:56:30 -0700 Subject: [PATCH 21/53] El Pais Babelia by oneillpt --- resources/recipes/el_pais_babelia.recipe | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 resources/recipes/el_pais_babelia.recipe diff --git a/resources/recipes/el_pais_babelia.recipe b/resources/recipes/el_pais_babelia.recipe new file mode 100644 index 0000000000..31b983ec0b --- /dev/null +++ b/resources/recipes/el_pais_babelia.recipe @@ -0,0 +1,49 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class ElPaisBabelia(BasicNewsRecipe): + + title = 'El Pais Babelia' + __author__ = 'oneillpt' + description = 'El Pais Babelia' + INDEX = 'http://www.elpais.com/suple/babelia/' + language = 'es' + + remove_tags_before = dict(name='div', attrs={'class':'estructura_2col'}) + keep_tags = [dict(name='div', attrs={'class':'estructura_2col'})] + remove_tags = [dict(name='div', attrs={'class':'votos estirar'}), + dict(name='div', attrs={'id':'utilidades'}), + dict(name='div', attrs={'class':'info_relacionada'}), + dict(name='div', attrs={'class':'mod_apoyo'}), + dict(name='div', attrs={'class':'contorno_f'}), + dict(name='div', attrs={'class':'pestanias'}), + dict(name='div', attrs={'class':'otros_webs'}), + dict(name='div', attrs={'id':'pie'}) + ] + #no_stylesheets = True + remove_javascript = True + + def parse_index(self): + articles = [] + soup = self.index_to_soup(self.INDEX) + feeds = [] + for section in soup.findAll('div', attrs={'class':'contenedor_nuevo'}): + section_title = self.tag_to_string(section.find('h1')) + articles = [] + for post in section.findAll('a', href=True): + url = post['href'] + if url.startswith('/'): + url = 'http://www.elpais.es'+url + title = self.tag_to_string(post) + if str(post).find('class=') > 0: + klass = post['class'] + if klass != "": + self.log() + self.log('--> post: ', post) + self.log('--> url: ', url) + self.log('--> title: ', title) + self.log('--> class: ', klass) + articles.append({'title':title, 'url':url}) + if articles: + feeds.append((section_title, articles)) + return feeds + From c384f93e4b0a2859344ff59f8a63b51e3e6f4d84 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 09:08:28 -0700 Subject: [PATCH 22/53] Comic Input: Fix conversion failing when output profile is set to Tablet Output --- src/calibre/ebooks/comic/input.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index 7710d41fb3..56fa123249 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -131,9 +131,12 @@ class PageProcessor(list): # {{{ newsizey = int(newsizex / aspect) deltax = 0 deltay = (SCRHEIGHT - newsizey) / 2 - wand.size = (newsizex, newsizey) - wand.set_border_color(pw) - wand.add_border(pw, deltax, deltay) + if newsizex < 20000 and newsizey < 20000: + # Too large and resizing fails, so better + # to leave it as original size + wand.size = (newsizex, newsizey) + wand.set_border_color(pw) + wand.add_border(pw, deltax, deltay) elif self.opts.wide: # Keep aspect and Use device height as scaled image width so landscape mode is clean aspect = float(sizex) / float(sizey) @@ -152,11 +155,15 @@ class PageProcessor(list): # {{{ newsizey = int(newsizex / aspect) deltax = 0 deltay = (wscreeny - newsizey) / 2 - wand.size = (newsizex, newsizey) - wand.set_border_color(pw) - wand.add_border(pw, deltax, deltay) + if newsizex < 20000 and newsizey < 20000: + # Too large and resizing fails, so better + # to leave it as original size + wand.size = (newsizex, newsizey) + wand.set_border_color(pw) + wand.add_border(pw, deltax, deltay) else: - wand.size = (SCRWIDTH, SCRHEIGHT) + if SCRWIDTH < 20000 and SCRHEIGHT < 20000: + wand.size = (SCRWIDTH, SCRHEIGHT) if not self.opts.dont_sharpen: wand.sharpen(0.0, 1.0) From 787ead16a2158af8d1dcee10151823ff2bc865eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 09:16:18 -0700 Subject: [PATCH 23/53] Updated Kompiuterra --- resources/recipes/kompiutierra.recipe | 73 ++++++++++++++------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/resources/recipes/kompiutierra.recipe b/resources/recipes/kompiutierra.recipe index 0d30afa3a7..528285b26c 100644 --- a/resources/recipes/kompiutierra.recipe +++ b/resources/recipes/kompiutierra.recipe @@ -1,36 +1,37 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2010, Vadim Dyadkin, dyadkin@gmail.com' -__author__ = 'Vadim Dyadkin' - -from calibre.web.feeds.news import BasicNewsRecipe - -class Computerra(BasicNewsRecipe): - title = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430' - recursion = 50 - oldest_article = 100 - __author__ = 'Vadim Dyadkin' - max_articles_per_feed = 100 - use_embedded_content = False - simultaneous_downloads = 5 - language = 'ru' - description = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u044b, \u043e\u043a\u043e\u043b\u043e\u043d\u0430\u0443\u0447\u043d\u044b\u0435 \u0438 \u043e\u043a\u043e\u043b\u043e\u0444\u0438\u043b\u043e\u0441\u043e\u0444\u0441\u043a\u0438\u0435 \u0441\u0442\u0430\u0442\u044c\u0438, \u0433\u0430\u0434\u0436\u0435\u0442\u044b.' - - keep_only_tags = [dict(name='div', attrs={'id': 'content'}),] - - - feeds = [(u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430', 'http://feeds.feedburner.com/ct_news/'),] - - remove_tags = [dict(name='div', attrs={'id': ['fin', 'idc-container', 'idc-noscript',]}), - dict(name='ul', attrs={'class': "related_post"}), - dict(name='p', attrs={'class': 'info'}), - dict(name='a', attrs={'rel': 'tag', 'class': 'twitter-share-button', 'type': 'button_count'}), - dict(name='h2', attrs={}),] - - extra_css = 'body { text-align: justify; }' - - def get_article_url(self, article): - return article.get('feedburner:origLink', article.get('guid')) - +#!/usr/bin/python +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2010, Vadim Dyadkin, dyadkin@gmail.com' +__author__ = 'Vadim Dyadkin' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Computerra(BasicNewsRecipe): + title = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430' + oldest_article = 100 + __author__ = 'Vadim Dyadkin (edited by A. Chewi)' + max_articles_per_feed = 50 + use_embedded_content = False + remove_javascript = True + no_stylesheets = True + conversion_options = {'linearize_tables' : True} + simultaneous_downloads = 5 + language = 'ru' + description = u'Компьютерра: все новости про компьютеры, железо, новые технологии, информационные технологии' + + keep_only_tags = [dict(name='div', attrs={'id': 'content'}),] + + feeds = [(u'Компьютерра-Онлайн', 'http://feeds.feedburner.com/ct_news/'),] + + remove_tags = [ + dict(name='div', attrs={'id': ['fin', 'idc-container', 'idc-noscript',]}), + dict(name='ul', attrs={'class': "related_post"}), + dict(name='p', attrs={'class': 'info'}), + dict(name='a', attrs={'class': 'twitter-share-button'}), + dict(name='a', attrs={'type': 'button_count'}), + dict(name='h2', attrs={}) + ] + + def print_version(self, url): + return url + '?print=true' From c9234596bd481e3f73a9e99d18bdbc787f7981bb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 11:22:26 -0700 Subject: [PATCH 24/53] Improve evz.ro --- resources/recipes/evz.ro.recipe | 74 +++++++++++++------------- resources/recipes/nationalgeoro.recipe | 2 +- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/resources/recipes/evz.ro.recipe b/resources/recipes/evz.ro.recipe index bce151d1fc..841dc80429 100644 --- a/resources/recipes/evz.ro.recipe +++ b/resources/recipes/evz.ro.recipe @@ -1,52 +1,54 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + __license__ = 'GPL v3' -__copyright__ = '2010, Darko Miletic ' +__copyright__ = u'2011, Silviu Cotoar\u0103' ''' evz.ro ''' -import re from calibre.web.feeds.news import BasicNewsRecipe -class EVZ_Ro(BasicNewsRecipe): - title = 'evz.ro' - __author__ = 'Darko Miletic' - description = 'News from Romania' - publisher = 'evz.ro' - category = 'news, politics, Romania' - oldest_article = 2 - max_articles_per_feed = 200 - no_stylesheets = True - encoding = 'utf8' - use_embedded_content = False +class EvenimentulZilei(BasicNewsRecipe): + title = u'Evenimentul Zilei' + __author__ = u'Silviu Cotoar\u0103' + description = '' + publisher = u'Evenimentul Zilei' + oldest_article = 5 language = 'ro' - masthead_url = 'http://www.evz.ro/fileadmin/images/logo.gif' - extra_css = ' body{font-family: Georgia,Arial,Helvetica,sans-serif } .firstP{font-size: 1.125em} .author,.articleInfo{font-size: small} ' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Stiri' + encoding = 'utf-8' + cover_url = 'http://www.evz.ro/fileadmin/images/evzLogo.png' conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language - } + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } - preprocess_regexps = [ - (re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '<head><title>') - ,(re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '') - ] + keep_only_tags = [ + dict(name='div', attrs={'class':'single'}) + , dict(name='img', attrs={'id':'placeholder'}) + , dict(name='a', attrs={'id':'holderlink'}) + ] - remove_tags = [ - dict(name=['form','embed','iframe','object','base','link','script','noscript']) - ,dict(attrs={'class':['section','statsInfo','email il']}) - ,dict(attrs={'id' :'gallery'}) - ] + remove_tags = [ + dict(name='p', attrs={'class':['articleInfo']}) + , dict(name='div', attrs={'id':['bannerAddoceansArticleJos']}) + , dict(name='div', attrs={'id':['bannerAddoceansArticle']}) + ] - remove_tags_after = dict(attrs={'class':'section'}) - keep_only_tags = [dict(attrs={'class':'single'})] - remove_attributes = ['height','width'] + remove_tags_after = [ + dict(name='div', attrs={'id':['bannerAddoceansArticleJos']}) + ] - feeds = [(u'Articles', u'http://www.evz.ro/rss.xml')] + feeds = [ + (u'Feeds', u'http://www.evz.ro/rss.xml') + ] def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - return soup + return self.adeify_images(soup) diff --git a/resources/recipes/nationalgeoro.recipe b/resources/recipes/nationalgeoro.recipe index a3c5727d38..8f989be74d 100644 --- a/resources/recipes/nationalgeoro.recipe +++ b/resources/recipes/nationalgeoro.recipe @@ -14,7 +14,7 @@ class NationalGeoRo(BasicNewsRecipe): __author__ = u'Silviu Cotoar\u0103' description = u'S\u0103 avem grij\u0103 de planet\u0103' publisher = 'National Geographic' - oldest_article = 5 + oldest_article = 35 language = 'ro' max_articles_per_feed = 100 no_stylesheets = True From c1c17aaf9d7149317aef66e3f3918fea58e71eab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 11:34:55 -0700 Subject: [PATCH 25/53] Various Romanian news sources by Silviu Cotoara --- resources/images/news/hitro.png | Bin 0 -> 521 bytes resources/images/news/kamikaze.png | Bin 0 -> 262 bytes resources/images/news/trombon.png | Bin 0 -> 375 bytes resources/images/news/wallstreetro.png | Bin 0 -> 768 bytes resources/recipes/hitro.recipe | 43 ++++++++++++++++++++ resources/recipes/kamikaze.recipe | 53 ++++++++++++++++++++++++ resources/recipes/kompiutierra.recipe | 2 +- resources/recipes/trombon.recipe | 51 +++++++++++++++++++++++ resources/recipes/wallstreetro.recipe | 54 +++++++++++++++++++++++++ 9 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 resources/images/news/hitro.png create mode 100644 resources/images/news/kamikaze.png create mode 100644 resources/images/news/trombon.png create mode 100644 resources/images/news/wallstreetro.png create mode 100644 resources/recipes/hitro.recipe create mode 100644 resources/recipes/kamikaze.recipe create mode 100644 resources/recipes/trombon.recipe create mode 100644 resources/recipes/wallstreetro.recipe diff --git a/resources/images/news/hitro.png b/resources/images/news/hitro.png new file mode 100644 index 0000000000000000000000000000000000000000..75c08a1c25776dbd9a2e5a12afdb09169a47e4a6 GIT binary patch literal 521 zcmV+k0`~ohP)War0EEZRi(|)rP7?WWx#V>|vy{PP|wdVy|OFw!jNo7s%OImcrpJ}UiB0LYimA3s6rUU1F{fWMZaC?b=| z(1e2#Xz4cB#FMgFfh ztGjU6p#-bgXqvj=oWpfp%w{tbMSmOIBp}^?r6G#me{Z4``ZQxKfK%2x98%997{gZFo-}4VUKL}Z-1d6Og zhT4o|qZycIZ=<{v*~Z_D?tOo>4YnuwJ{@j6cVwNZKLJ4a{{{R3Vt(Gmv>Kd<00000 LNkvXXu0mjfoS5EJ literal 0 HcmV?d00001 diff --git a/resources/images/news/kamikaze.png b/resources/images/news/kamikaze.png new file mode 100644 index 0000000000000000000000000000000000000000..49ef2f50a1601e51f4f29aefc4660b9cad727583 GIT binary patch literal 262 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbL!Yk*IPE0De;rE&#`xOj5-c+LpP zT#-}RGHFT(6U!AYnHDaQJ5n-NfQU^bg^%Tqn93PGpaPjIKn54j6d;;4$A*n1Ogv#0 z&>+T=AirP+hi5m^fSi0!7sn8Zsk#1d`I;Sgv|h~Ftn=>M*MI$5YOm(r{I%Fu)6pF1(mwzbIPgoLZ@qKP5`S4}GCPPJh$s7)%5kbG1w5INn|6y36z~S1)Ti&}s%xS3j3^P6}+amR8myT&djW; zq-~wZ?Tvk-n($u7)q)bLc ztgfv5`ut>GT(q{de0p@;+T1iUETpESM_Taj3wRWlR#BT`b4>to-hDRLrF`|=uP4oy V4|iC{(^voi002ovPDHLkV1n(1rmFw| literal 0 HcmV?d00001 diff --git a/resources/images/news/wallstreetro.png b/resources/images/news/wallstreetro.png new file mode 100644 index 0000000000000000000000000000000000000000..d72bc70ca0e89ed1277b6af8d3c2f8be513df034 GIT binary patch literal 768 zcmV+b1ONPqP))Hk`9gM=1A?$}bO$^3F|AxKu|MEXDCd*>XX3#Jtz|_Gu zn8R%R2uvAl-7cf;z2CDJmz~$=^U2A{^T>bAjtLGUfr$~1AQ(6SH9#TpCT;J_I;hvEP*LRzgh>LSz$h~S)I2#O&t!dqHd<;Ksy^CCE=UMcbQ+*SPA2Y|7= z|C}Fh{>pdPza^1OfkoWo~if`twpi>vdIX6<}&hmZcuFX)dl@J60?yj!$ z)BF-a5TK?cMELybH0AdT^yS~=YZw=kGXWg#~~y<CY=IvI1-3BFo-bT1Z{dUDV#|hP~k1UNgg~HXQCkh0000 Date: Mon, 7 Mar 2011 18:57:31 +0000 Subject: [PATCH 26/53] An attempt at S/R for identifiers. --- src/calibre/gui2/dialogs/metadata_bulk.py | 54 ++++++++++++++++++----- src/calibre/gui2/dialogs/metadata_bulk.ui | 52 +++++++++++++++++++++- src/calibre/library/database2.py | 4 ++ 3 files changed, 98 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index af540ea4c3..584b88a097 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -7,7 +7,7 @@ import re, os, inspect from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \ pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \ - QDate + QDate, QCompleter from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor @@ -363,7 +363,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if (f in ['author_sort'] or (fm[f]['datatype'] in ['text', 'series', 'enumeration'] and fm[f].get('search_terms', None) - and f not in ['formats', 'ondevice']) or + and f not in ['formats', 'ondevice', 'id']) or fm[f]['datatype'] in ['int', 'float', 'bool'] ): self.all_fields.append(f) self.writable_fields.append(f) @@ -393,6 +393,11 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.book_1_text.setObjectName(name) self.testgrid.addWidget(w, i+offset, 2, 1, 1) + ident_types = sorted(self.db.get_all_identifier_types(), key=sort_key) + self.s_r_dst_ident.setCompleter(QCompleter(ident_types)) + ident_types.insert(0, '') + self.s_r_src_ident.addItems(ident_types) + self.main_heading = _( 'You can destroy your library using this feature. ' 'Changes are permanent. There is no undo function. ' @@ -449,6 +454,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.test_text.editTextChanged[str].connect(self.s_r_paint_results) self.comma_separated.stateChanged.connect(self.s_r_paint_results) self.case_sensitive.stateChanged.connect(self.s_r_paint_results) + self.s_r_src_ident.currentIndexChanged[int].connect(self.s_r_paint_results) + self.s_r_dst_ident.textChanged.connect(self.s_r_paint_results) self.s_r_template.lost_focus.connect(self.s_r_template_changed) self.central_widget.setCurrentIndex(0) @@ -471,6 +478,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.query_field.addItems(sorted([q for q in self.queries], key=sort_key)) self.query_field.currentIndexChanged[str].connect(self.s_r_query_change) self.query_field.setCurrentIndex(0) + self.search_field.setCurrentIndex(0) + self.s_r_search_field_changed(0) def s_r_sf_itemdata(self, idx): if idx is None: @@ -497,7 +506,11 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): val = str(val) elif fm['is_csp']: # convert the csp dict into a list - val = [u'%s:%s'%(t[0], t[1]) for t in val.iteritems()] + id_type = unicode(self.s_r_src_ident.currentText()) + if id_type: + val = [val.get(id_type, '')] + else: + val = [u'%s:%s'%(t[0], t[1]) for t in val.iteritems()] if val is None: val = [] if fm['is_multiple'] else [''] elif not fm['is_multiple']: @@ -515,12 +528,17 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.s_r_search_field_changed(self.search_field.currentIndex()) def s_r_search_field_changed(self, idx): - if self.search_mode.currentIndex() != 0 and idx == 1: # Template + self.s_r_template.setVisible(False) + self.template_label.setVisible(False) + self.s_r_src_ident_label.setVisible(False) + self.s_r_src_ident.setVisible(False) + if idx == 1: # Template self.s_r_template.setVisible(True) self.template_label.setVisible(True) - else: - self.s_r_template.setVisible(False) - self.template_label.setVisible(False) + elif self.s_r_sf_itemdata(idx) == 'identifiers': + self.s_r_src_ident_label.setVisible(True) + self.s_r_src_ident.setVisible(True) + for i in range(0, self.s_r_number_of_books): w = getattr(self, 'book_%d_text'%(i+1)) mi = self.db.get_metadata(self.ids[i], index_is_id=True) @@ -538,10 +556,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.s_r_paint_results(None) def s_r_destination_field_changed(self, idx): + self.s_r_dst_ident_label.setVisible(False) + self.s_r_dst_ident.setVisible(False) txt = self.s_r_df_itemdata(idx) if not txt: txt = self.s_r_sf_itemdata(None) if txt and txt in self.writable_fields: + if txt == 'identifiers': + self.s_r_dst_ident_label.setVisible(True) + self.s_r_dst_ident.setVisible(True) self.destination_field_fm = self.db.metadata_for_field(txt) self.s_r_paint_results(None) @@ -639,8 +662,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if dest_mode != 0: dest_val = mi.get(dest, '') if self.db.metadata_for_field(dest)['is_csp']: - # convert the csp dict into a list - dest_val = [u'%s:%s'%(t[0], t[1]) for t in dest_val.iteritems()] + dst_id_type = unicode(self.s_r_dst_ident.text()) + if dst_id_type: + dest_val = [dest_val.get(dst_id_type, '')] + else: + # convert the csp dict into a list + dest_val = [u'%s:%s'%(t[0], t[1]) for t in dest_val.iteritems()] if dest_val is None: dest_val = [] elif not isinstance(dest_val, list): @@ -726,7 +753,14 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): # convert the colon-separated pair strings back into a dict, which # is what set_identifiers wants if dfm['is_csp']: - val = dict([(t.split(':')) for t in val]) + dst_id_type = unicode(self.s_r_dst_ident.text()) + if dst_id_type: + v = ''.join(val) + ids = mi.get(dest) + ids[dst_id_type] = v + val = ids + else: + val = dict([(t.split(':')) for t in val]) else: val = self.s_r_replace_mode_separator().join(val) if dest == 'title' and len(val) == 0: diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 1654ff8261..75ea1ce8bd 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -732,6 +732,29 @@ Future conversion of these books will use the default settings. + + + + Identifier: + + + s_r_src_ident + + + + + + + + 100 + 0 + + + + Choose which identifier to operate upon + + + @@ -910,7 +933,30 @@ not multiple and the destination field is multiple - + + + + Identifier: + + + s_r_dst_ident + + + + + + + + 100 + 0 + + + + Choose which identifier to operate upon + + + + @@ -996,7 +1042,7 @@ not multiple and the destination field is multiple - + QFrame::NoFrame @@ -1120,6 +1166,7 @@ not multiple and the destination field is multiple remove_button search_field search_mode + s_r_src_ident s_r_template search_for case_sensitive @@ -1128,6 +1175,7 @@ not multiple and the destination field is multiple destination_field replace_mode comma_separated + s_r_dst_ident results_count starting_from multiple_separator diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f1a5884e9f..3b712e1c10 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -2551,6 +2551,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return ans + def get_all_identifier_types(self): + idents = self.conn.get('SELECT DISTINCT type FROM identifiers') + return [ident[0] for ident in idents] + def _clean_identifier(self, typ, val): typ = icu_lower(typ).strip().replace(':', '').replace(',', '') val = val.strip().replace(',', '|').replace(':', '|') From 2c3ddf3b8891527a0050a30597e013a8d0403ef7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 12:03:22 -0700 Subject: [PATCH 27/53] ... --- src/calibre/ebooks/pdf/writer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index b0884417f6..516509fdd7 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -46,7 +46,8 @@ def get_pdf_printer(opts, for_comic=False): printer = QPrinter(QPrinter.HighResolution) custom_size = get_custom_size(opts) - if opts.output_profile.short_name == 'default': + if opts.output_profile.short_name == 'default' or \ + opts.output_profile.width > 10000: if custom_size is None: printer.setPaperSize(paper_size(opts.paper_size)) else: From 89282d6de6b35d90821f8c36df1dd6a7802531ed Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 7 Mar 2011 19:09:30 +0000 Subject: [PATCH 28/53] Remove 'id' as a writable field --- src/calibre/gui2/dialogs/metadata_bulk.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 584b88a097..a7d25c0cb4 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -363,8 +363,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if (f in ['author_sort'] or (fm[f]['datatype'] in ['text', 'series', 'enumeration'] and fm[f].get('search_terms', None) - and f not in ['formats', 'ondevice', 'id']) or - fm[f]['datatype'] in ['int', 'float', 'bool'] ): + and f not in ['formats', 'ondevice']) or + (fm[f]['datatype'] in ['int', 'float', 'bool'] and + f not in ['id'])): self.all_fields.append(f) self.writable_fields.append(f) if fm[f]['datatype'] == 'composite': From b7736da887165bf38cdc9085eba137c1c83fb08d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 19:13:11 -0700 Subject: [PATCH 29/53] Fix #9327 (Undetected Android Device) --- src/calibre/devices/android/driver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 3724f02ca2..a334112213 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -78,6 +78,9 @@ class ANDROID(USBMS): # Xperia 0x13d3 : { 0x3304 : [0x0001, 0x0002] }, + # CREEL?? Also Nextbook + 0x5e3 : { 0x726 : [0x222] }, + } EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books'] EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' From bfd77656a6fe12352d8f71119a6e0c7ad48d0203 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 19:42:10 -0700 Subject: [PATCH 30/53] Conversion pipeline: When converting the :first-letter pseudo CSS selector to a follow W3C rules for handling leading punctuation characters. Fixes #9319 (Problem with Mobi conversion and ePub conversion) --- src/calibre/ebooks/oeb/stylizer.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index efc8fe1463..0cd17387fe 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -8,11 +8,7 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Marshall T. Vandegrift ' -import os -import itertools -import re -import logging -import copy +import os, itertools, re, logging, copy, unicodedata from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils @@ -234,8 +230,18 @@ class Stylizer(object): for elem in matches: for x in elem.iter(): if x.text: - span = E.span(x.text[0]) - span.tail = x.text[1:] + punctuation_chars = [] + text = unicode(x.text) + while text: + if not unicodedata.category(text[0]).startswith('P'): + break + punctuation_chars.append(text[0]) + text = text[1:] + + special_text = u''.join(punctuation_chars) + \ + (text[0] if text else u'') + span = E.span(special_text) + span.tail = text[1:] x.text = None x.insert(0, span) self.style(span)._update_cssdict(cssdict) From 965ffa057dfa864ff8c7a9cc478320614536bac9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 20:00:29 -0700 Subject: [PATCH 31/53] News download: Fix regressiont hat caused the delay parameter in recipes to not actually delay downloads. Fixes #9332 (delay in recipe does not work as intended) --- src/calibre/web/fetch/simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index 67f19e40e4..f2e22c8f5e 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -193,8 +193,8 @@ class RecursiveFetcher(object): data = None self.log.debug('Fetching', url) delta = time.time() - self.last_fetch_at - if delta < self.delay: - time.sleep(delta) + if delta < self.delay: + time.sleep(self.delay - delta) if isinstance(url, unicode): url = url.encode('utf-8') # Not sure is this is really needed as I think mechanize From 5bdcea1b8cdffae4892179ffad13f8897faf2b28 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 20:34:10 -0700 Subject: [PATCH 32/53] Fix advanced recipe edit widget not using monospace font on non linux systems --- src/calibre/gui2/dialogs/user_profiles.py | 5 ++++- src/calibre/gui2/dialogs/user_profiles.ui | 5 ----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index f2388d2981..5453a90766 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' import time, os from PyQt4.Qt import SIGNAL, QUrl, QAbstractListModel, Qt, \ - QVariant + QVariant, QFont from calibre.web.feeds.recipes import compile_recipe, custom_recipes from calibre.web.feeds.news import AutomaticNewsRecipe @@ -83,6 +83,9 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self._model = self.model = CustomRecipeModel(recipe_model) self.available_profiles.setModel(self._model) self.available_profiles.currentChanged = self.current_changed + f = QFont() + f.setStyleHint(f.Monospace) + self.source_code.setFont(f) self.connect(self.remove_feed_button, SIGNAL('clicked(bool)'), self.added_feeds.remove_selected_items) diff --git a/src/calibre/gui2/dialogs/user_profiles.ui b/src/calibre/gui2/dialogs/user_profiles.ui index 97e3d37db2..7631c74768 100644 --- a/src/calibre/gui2/dialogs/user_profiles.ui +++ b/src/calibre/gui2/dialogs/user_profiles.ui @@ -410,11 +410,6 @@ p, li { white-space: pre-wrap; } 0 - - - DejaVu Sans Mono - - QTextEdit::NoWrap From 0285c22a84f6f40df7a9a1d855414fe52f134ef6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 21:31:59 -0700 Subject: [PATCH 33/53] Fix regression that caused the viewer to forget its window size and other attributes when launched from within calibre, after calibre is restarted. Fixes #9326 (Viewer Window Not Remembering Last Size) --- src/calibre/gui2/viewer/main.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index de0f83a5b2..964616ab48 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -17,17 +17,19 @@ from calibre.gui2.viewer.bookmarkmanager import BookmarkManager from calibre.gui2.widgets import ProgressIndicator from calibre.gui2.main_window import MainWindow from calibre.gui2 import Application, ORG_NAME, APP_UID, choose_files, \ - info_dialog, error_dialog, open_url, available_height, gprefs + info_dialog, error_dialog, open_url, available_height from calibre.ebooks.oeb.iterator import EbookIterator from calibre.ebooks import DRMError from calibre.constants import islinux, isfreebsd, isosx, filesystem_encoding -from calibre.utils.config import Config, StringConfig, dynamic +from calibre.utils.config import Config, StringConfig, JSONConfig from calibre.gui2.search_box import SearchBox2 from calibre.ebooks.metadata import MetaInformation from calibre.customize.ui import available_input_formats from calibre.gui2.viewer.dictionary import Lookup from calibre import as_unicode, force_unicode, isbytestring +vprefs = JSONConfig('viewer') + class TOCItem(QStandardItem): def __init__(self, toc): @@ -303,7 +305,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): m = self.open_history_menu m.clear() count = 0 - for path in gprefs.get('viewer_open_history', []): + for path in vprefs.get('viewer_open_history', []): if count > 9: break if os.path.exists(path): @@ -315,17 +317,17 @@ class EbookViewer(MainWindow, Ui_EbookViewer): return MainWindow.closeEvent(self, e) def save_state(self): - state = str(self.saveState(self.STATE_VERSION)) - dynamic['viewer_toolbar_state'] = state - dynamic.set('viewer_window_geometry', self.saveGeometry()) + state = bytearray(self.saveState(self.STATE_VERSION)) + vprefs['viewer_toolbar_state'] = state + vprefs.set('viewer_window_geometry', bytearray(self.saveGeometry())) if self.current_book_has_toc: - dynamic.set('viewer_toc_isvisible', bool(self.toc.isVisible())) + vprefs.set('viewer_toc_isvisible', bool(self.toc.isVisible())) if self.toc.isVisible(): - dynamic.set('viewer_splitter_state', + vprefs.set('viewer_splitter_state', bytearray(self.splitter.saveState())) def restore_state(self): - state = dynamic.get('viewer_toolbar_state', None) + state = vprefs.get('viewer_toolbar_state', None) if state is not None: try: state = QByteArray(state) @@ -676,13 +678,13 @@ class EbookViewer(MainWindow, Ui_EbookViewer): self.action_table_of_contents.setChecked(False) if isbytestring(pathtoebook): pathtoebook = force_unicode(pathtoebook, filesystem_encoding) - vh = gprefs.get('viewer_open_history', []) + vh = vprefs.get('viewer_open_history', []) try: vh.remove(pathtoebook) except: pass vh.insert(0, pathtoebook) - gprefs.set('viewer_open_history', vh[:50]) + vprefs.set('viewer_open_history', vh[:50]) self.build_recent_menu() self.action_table_of_contents.setDisabled(not self.iterator.toc) @@ -739,13 +741,13 @@ class EbookViewer(MainWindow, Ui_EbookViewer): c = config().parse() self.splitter.setSizes([1, 300]) if c.remember_window_size: - wg = dynamic.get('viewer_window_geometry', None) + wg = vprefs.get('viewer_window_geometry', None) if wg is not None: self.restoreGeometry(wg) - ss = dynamic.get('viewer_splitter_state', None) + ss = vprefs.get('viewer_splitter_state', None) if ss is not None: self.splitter.restoreState(ss) - self.show_toc_on_open = dynamic.get('viewer_toc_isvisible', False) + self.show_toc_on_open = vprefs.get('viewer_toc_isvisible', False) av = available_height() - 30 if self.height() > av: self.resize(self.width(), av) From 24d8f758e7fa94c17cb749853b3ae32a59c645ef Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 22:08:48 -0700 Subject: [PATCH 34/53] CHM Input: If an input encoding is specified, use it rather than trying to detect the encoding of the text in the CHM file. Fixes #9173 ("Input character encoding" is useless for chm file.) --- src/calibre/ebooks/chm/input.py | 5 +++-- src/calibre/ebooks/chm/reader.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/chm/input.py index 89efa2b4d1..f55a76d67e 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/chm/input.py @@ -22,7 +22,7 @@ class CHMInput(InputFormatPlugin): def _chmtohtml(self, output_dir, chm_path, no_images, log): from calibre.ebooks.chm.reader import CHMReader log.debug('Opening CHM file') - rdr = CHMReader(chm_path, log) + rdr = CHMReader(chm_path, log, self.opts) log.debug('Extracting CHM to %s' % output_dir) rdr.extract_content(output_dir) self._chm_reader = rdr @@ -32,13 +32,13 @@ class CHMInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): from calibre.ebooks.chm.metadata import get_metadata_from_reader from calibre.customize.ui import plugin_for_input_format + self.opts = options log.debug('Processing CHM...') with TemporaryDirectory('_chm2oeb') as tdir: html_input = plugin_for_input_format('html') for opt in html_input.options: setattr(options, opt.option.name, opt.recommended_value) - options.input_encoding = 'utf-8' no_images = False #options.no_images chm_name = stream.name #chm_data = stream.read() @@ -54,6 +54,7 @@ class CHMInput(InputFormatPlugin): odi = options.debug_pipeline options.debug_pipeline = None + options.input_encoding = 'utf-8' # try a custom conversion: #oeb = self._create_oebbook(mainpath, tdir, options, log, metadata) # try using html converter: diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index fe8b4fdbde..34d228ef3b 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -40,13 +40,14 @@ class CHMError(Exception): pass class CHMReader(CHMFile): - def __init__(self, input, log): + def __init__(self, input, log, opts): CHMFile.__init__(self) if isinstance(input, unicode): input = input.encode(filesystem_encoding) if not self.LoadCHM(input): raise CHMError("Unable to open CHM file '%s'"%(input,)) self.log = log + self.opts = opts self._sourcechm = input self._contents = None self._playorder = 0 @@ -151,6 +152,8 @@ class CHMReader(CHMFile): break def _reformat(self, data, htmlpath): + if self.opts.input_encoding: + data = data.decode(self.opts.input_encoding) try: data = xml_to_unicode(data, strip_encoding_pats=True)[0] soup = BeautifulSoup(data) From 7449f94fc8451f863f7a2375ee5030d5473f3c7e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 7 Mar 2011 23:14:05 -0700 Subject: [PATCH 35/53] Fix #9299 (Optimus V (Android phone) not detected) --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a334112213..e1a806af0f 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -57,7 +57,7 @@ class ANDROID(USBMS): 0x413c : { 0xb007 : [0x0100, 0x0224]}, # LG - 0x1004 : { 0x61cc : [0x100] }, + 0x1004 : { 0x61cc : [0x100], 0x61ce : [0x100] }, # Archos 0x0e79 : { From 2ff72a513feda237d9430b0cc463bc74d61a7071 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 8 Mar 2011 08:23:28 +0000 Subject: [PATCH 36/53] Fix #9323: selection lost when editing metadata on device --- src/calibre/gui2/device.py | 8 ++++++++ src/calibre/gui2/init.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 298e541730..2cbecc134c 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1160,6 +1160,14 @@ class DeviceMixin(object): # {{{ ), bad) d.exec_() + def upload_dirtied_booklists(self): + ''' + Upload metadata to device. + ''' + plugboards = self.library_view.model().db.prefs.get('plugboards', {}) + self.device_manager.sync_booklists(Dispatcher(lambda x: x), + self.booklists(), plugboards) + def upload_booklists(self): ''' Upload metadata to device. diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 9119a8da77..80f1f1c2cf 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -44,13 +44,13 @@ class LibraryViewMixin(object): # {{{ for view in (self.library_view, self.memory_view, self.card_a_view, self.card_b_view): getattr(view, func)(*args) - self.memory_view.connect_dirtied_signal(self.upload_booklists) + self.memory_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.memory_view.connect_upload_collections_signal( func=self.upload_collections, oncard=None) - self.card_a_view.connect_dirtied_signal(self.upload_booklists) + self.card_a_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.card_a_view.connect_upload_collections_signal( func=self.upload_collections, oncard='carda') - self.card_b_view.connect_dirtied_signal(self.upload_booklists) + self.card_b_view.connect_dirtied_signal(self.upload_dirtied_booklists) self.card_b_view.connect_upload_collections_signal( func=self.upload_collections, oncard='cardb') self.book_on_device(None, reset=True) From db47b8c067318dc1e49bcfeadeb191f718852a12 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 8 Mar 2011 08:53:54 +0000 Subject: [PATCH 37/53] Simplify identifier search/replace. --- src/calibre/gui2/dialogs/metadata_bulk.py | 9 ++++++++- src/calibre/gui2/dialogs/metadata_bulk.ui | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index a7d25c0cb4..e270cd0a55 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -396,7 +396,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): ident_types = sorted(self.db.get_all_identifier_types(), key=sort_key) self.s_r_dst_ident.setCompleter(QCompleter(ident_types)) - ident_types.insert(0, '') + try: + self.s_r_dst_ident.setPlaceholderText(_('Enter an identifier type')) + except: + pass self.s_r_src_ident.addItems(ident_types) self.main_heading = _( @@ -644,6 +647,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): dest = src dest_mode = self.replace_mode.currentIndex() + if self.destination_field_fm['is_csp']: + if not unicode(self.s_r_dst_ident.text()): + raise Exception(_('You must specify a destination identifier type')) + if self.destination_field_fm['is_multiple']: if self.comma_separated.isChecked(): if dest == 'authors': diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 75ea1ce8bd..59a68d6514 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -735,7 +735,7 @@ Future conversion of these books will use the default settings. - Identifier: + Identifier type: s_r_src_ident @@ -751,7 +751,7 @@ Future conversion of these books will use the default settings. - Choose which identifier to operate upon + Choose which identifier type to operate upon @@ -936,7 +936,7 @@ not multiple and the destination field is multiple - Identifier: + Identifier type: s_r_dst_ident @@ -952,7 +952,7 @@ not multiple and the destination field is multiple - Choose which identifier to operate upon + Choose which identifier type to operate upon From 7a0d79381864156f5cc3df2cb4311f541cc2d002 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 08:17:17 -0700 Subject: [PATCH 38/53] ... --- resources/recipes/economist.recipe | 2 +- resources/recipes/economist_free.recipe | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/recipes/economist.recipe b/resources/recipes/economist.recipe index 17bf4c8c20..9447fe2193 100644 --- a/resources/recipes/economist.recipe +++ b/resources/recipes/economist.recipe @@ -24,7 +24,7 @@ class Economist(BasicNewsRecipe): cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' remove_tags = [ dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + dict(attrs={'class':['dblClkTrk', 'ec-article-info', 'share_inline_header']}), {'class': lambda x: x and 'share-links-header' in x}, ] keep_only_tags = [dict(id='ec-article-body')] diff --git a/resources/recipes/economist_free.recipe b/resources/recipes/economist_free.recipe index f4a4efd932..d1766211d7 100644 --- a/resources/recipes/economist_free.recipe +++ b/resources/recipes/economist_free.recipe @@ -18,7 +18,8 @@ class Economist(BasicNewsRecipe): cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' remove_tags = [ dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), - dict(attrs={'class':['dblClkTrk', 'ec-article-info']}), + dict(attrs={'class':['dblClkTrk', 'ec-article-info', + 'share_inline_header']}), {'class': lambda x: x and 'share-links-header' in x}, ] keep_only_tags = [dict(id='ec-article-body')] From 4caa7653a747d120575a43344c7acb771f71b1b3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 8 Mar 2011 16:47:03 +0000 Subject: [PATCH 39/53] highlighting rows API --- src/calibre/gui2/library/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index b782cc7c72..33d12e8ab9 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -268,6 +268,15 @@ class BooksModel(QAbstractTableModel): # {{{ return None return self.get_current_highlighted_id() + def highlight_ids(self, ids_to_highlight): + self.ids_to_highlight = ids_to_highlight + self.ids_to_highlight_set = set(self.ids_to_highlight) + if self.ids_to_highlight: + self.current_highlighted_idx = 0 + else: + self.current_highlighted_idx = None + self.reset() + def search(self, text, reset=True): try: if self.highlight_only: From 585e5f398631e7d778f93853b00adc741c09d009 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 8 Mar 2011 18:30:47 +0000 Subject: [PATCH 40/53] API for setting and testing the virtual column 'marked' --- src/calibre/library/caches.py | 32 ++++++++++++++++++++++++++- src/calibre/library/database2.py | 3 +++ src/calibre/library/field_metadata.py | 10 +++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 823ef77bc5..9c157ef95a 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, itertools, time, traceback +import re, itertools, time, traceback, copy from itertools import repeat from datetime import timedelta from threading import Thread @@ -194,6 +194,7 @@ class ResultCache(SearchQueryParser): # {{{ self.first_sort = True self.search_restriction = '' self.search_restriction_book_count = 0 + self.marked_ids_dict = {} self.field_metadata = field_metadata self.all_search_locations = field_metadata.get_search_terms() SearchQueryParser.__init__(self, self.all_search_locations, optimize=True) @@ -775,6 +776,24 @@ class ResultCache(SearchQueryParser): # {{{ def get_search_restriction_book_count(self): return self.search_restriction_book_count + def set_marked_ids(self, id_dict): + if isinstance (id_dict, list): + # Simple list. Make it a dict of string 'true' + self.marked_ids_dict = dict([(id, u'true') for id in id_dict]) + else: + self.marked_ids_dict = copy.copy(id_dict) + # Ensure that all the items in the dict are text + for id_,val in self.marked_ids_dict.iteritems(): + self.marked_ids_dict[id_] = unicode(val) + + # Set the values in the cache + marked_col = self.FIELD_MAP['marked'] + for id_,val in self.marked_ids_dict.iteritems(): + try: + self._data[id_][marked_col] = val + except: + pass + # }}} def remove(self, id): @@ -824,6 +843,7 @@ class ResultCache(SearchQueryParser): # {{{ self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) + self._data[id].append(self.marked_ids_dict.get(id, None)) except IndexError: return None try: @@ -840,6 +860,7 @@ class ResultCache(SearchQueryParser): # {{{ self._data[id] = CacheRow(db, self.composites, db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) + self._data[id].append(self.marked_ids_dict.get(id, None)) self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -864,6 +885,15 @@ class ResultCache(SearchQueryParser): # {{{ for item in self._data: if item is not None: item.append(db.book_on_device_string(item[0])) + item.append(None) + + marked_col = self.FIELD_MAP['marked'] + for id_,val in self.marked_ids_dict.iteritems(): + try: + self._data[id_][marked_col] = val + except: + pass + self._map = [i[0] for i in self._data if i is not None] if field is not None: self.sort(field, ascending) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3b712e1c10..b506e7e82d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -374,6 +374,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.FIELD_MAP['ondevice'] = base = base+1 self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False) + self.FIELD_MAP['marked'] = base = base+1 + self.field_metadata.set_field_record_index('marked', base, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -421,6 +423,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.row = self.data.row self.has_id = self.data.has_id self.count = self.data.count + self.set_marked_ids = self.data.set_marked_ids for prop in ( 'author_sort', 'authors', 'comment', 'comments', diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index ff38af6890..b8180f9f39 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -273,6 +273,16 @@ class FieldMetadata(dict): 'is_custom':False, 'is_category':False, 'is_csp': False}), + ('marked', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name': None, + 'search_terms':['marked'], + 'is_custom':False, + 'is_category':False, + 'is_csp': False}), ('series_index',{'table':None, 'column':None, 'datatype':'float', From a83b144ea744e8bcde3d974d7487c05ef8b06ae8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 8 Mar 2011 18:40:54 +0000 Subject: [PATCH 41/53] Small search API change to let someone add, then set, a search restriction --- src/calibre/gui2/search_box.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 5a4c34a5cd..5b8501ebb5 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -436,17 +436,18 @@ class SavedSearchBoxMixin(object): # {{{ b = getattr(self, x+'_search_button') b.setStatusTip(b.toolTip()) - def saved_searches_changed(self): + def saved_searches_changed(self, set_restriction=None): p = sorted(saved_searches().names(), key=sort_key) - t = unicode(self.search_restriction.currentText()) + if set_restriction is None: + set_restriction = unicode(self.search_restriction.currentText()) # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') self.tags_view.recount() for s in p: self.search_restriction.addItem(s) - if t: # redo the search restriction if there was one - self.apply_named_search_restriction(t) + if set_restriction: # redo the search restriction if there was one + self.apply_named_search_restriction(set_restriction) def do_saved_search_edit(self, search): d = SavedSearchEditor(self, search) From f38000ac9df536dcaa9d9b35d0c347c72b5a6d93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 11:54:44 -0700 Subject: [PATCH 42/53] ... --- src/calibre/library/caches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 943e414c1a..d9ff43a67c 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, itertools, time, traceback, copy +import re, itertools, time, traceback from itertools import repeat, izip from datetime import timedelta from threading import Thread @@ -790,7 +790,7 @@ class ResultCache(SearchQueryParser): # {{{ # Simple list. Make it a dict of string 'true' self.marked_ids_dict = dict(izip(id_dict, repeat(u'true'))) else: - self.marked_ids_dict = copy.copy(id_dict) + self.marked_ids_dict = id_dict.copy() # Ensure that all the items in the dict are text for id_,val in self.marked_ids_dict.iteritems(): self.marked_ids_dict[id_] = unicode(val) From 1b5a67bd49bef0b14a50c867ffeea29fc2adf96a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 11:55:46 -0700 Subject: [PATCH 43/53] ... --- src/calibre/library/caches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index d9ff43a67c..acd3b78d68 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import re, itertools, time, traceback -from itertools import repeat, izip +from itertools import repeat from datetime import timedelta from threading import Thread @@ -788,7 +788,7 @@ class ResultCache(SearchQueryParser): # {{{ ''' if not hasattr(id_dict, 'items'): # Simple list. Make it a dict of string 'true' - self.marked_ids_dict = dict(izip(id_dict, repeat(u'true'))) + self.marked_ids_dict = dict.fromkeys(id_dict, u'true') else: self.marked_ids_dict = id_dict.copy() # Ensure that all the items in the dict are text From ca177e071bdba628c2a834178b7bf5f5a995e2e4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 11:58:44 -0700 Subject: [PATCH 44/53] ... --- src/calibre/library/caches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index acd3b78d68..76f5200f21 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -790,9 +790,9 @@ class ResultCache(SearchQueryParser): # {{{ # Simple list. Make it a dict of string 'true' self.marked_ids_dict = dict.fromkeys(id_dict, u'true') else: - self.marked_ids_dict = id_dict.copy() # Ensure that all the items in the dict are text - for id_,val in self.marked_ids_dict.iteritems(): + self.marked_ids_dict = {} + for id_, val in id_dict.iteritems(): self.marked_ids_dict[id_] = unicode(val) # Set the values in the cache From 9f138f31fc005d293c6ab1873e0d115c54e15d64 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 12:02:52 -0700 Subject: [PATCH 45/53] ... --- src/calibre/library/caches.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 76f5200f21..397f9bb403 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -779,9 +779,10 @@ class ResultCache(SearchQueryParser): # {{{ def set_marked_ids(self, id_dict): ''' ids in id_dict are "marked". They can be searched for by - using the search term ``marked:true`` + using the search term ``marked:true``. Pass in an empty dictionary or + set to clear marked ids. - :param id_dict: Either a dictionary mapping ids to values or a sequence + :param id_dict: Either a dictionary mapping ids to values or a set of ids. In the latter case, the value is set to 'true' for all ids. If a mapping is provided, then the search can be used to search for particular values: ``marked:value`` From 2c20826122a8692fd80f1eb3bc9d5ff09bec2d5a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 12:11:33 -0700 Subject: [PATCH 46/53] Fix set_marked not clearing previously marked ids --- src/calibre/library/caches.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 397f9bb403..63920c5dc9 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -798,7 +798,10 @@ class ResultCache(SearchQueryParser): # {{{ # Set the values in the cache marked_col = self.FIELD_MAP['marked'] - for id_,val in self.marked_ids_dict.iteritems(): + for id_ in self.iterallids(): + self._data[id_][marked_col] = None + + for id_, val in self.marked_ids_dict.iteritems(): try: self._data[id_][marked_col] = val except: From 25b9d4c118531df0bf49d376a5a00260d3f9a056 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 12:15:34 -0700 Subject: [PATCH 47/53] ... --- src/calibre/library/caches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 63920c5dc9..f27e15a74b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -798,8 +798,8 @@ class ResultCache(SearchQueryParser): # {{{ # Set the values in the cache marked_col = self.FIELD_MAP['marked'] - for id_ in self.iterallids(): - self._data[id_][marked_col] = None + for r in self.iterall(): + r[marked_col] = None for id_, val in self.marked_ids_dict.iteritems(): try: From 8580338d8c99f0167a30cdc81cf26df506bb9ba6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 12:25:02 -0700 Subject: [PATCH 48/53] ... --- src/calibre/library/caches.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index f27e15a74b..21a2622f33 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import re, itertools, time, traceback -from itertools import repeat +from itertools import repeat, izip, imap from datetime import timedelta from threading import Thread @@ -792,9 +792,8 @@ class ResultCache(SearchQueryParser): # {{{ self.marked_ids_dict = dict.fromkeys(id_dict, u'true') else: # Ensure that all the items in the dict are text - self.marked_ids_dict = {} - for id_, val in id_dict.iteritems(): - self.marked_ids_dict[id_] = unicode(val) + self.marked_ids_dict = dict(izip(id_dict.iterkeys(), imap(unicode, + id_dict.itervalues()))) # Set the values in the cache marked_col = self.FIELD_MAP['marked'] From 6046ed793e26c6d668fe9f389c44af00d6e6ed86 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 14:02:57 -0700 Subject: [PATCH 49/53] Fix dev.open in debug device detection not passing a library uuid --- src/calibre/devices/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 1918a36cc8..0d62a8f619 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -30,6 +30,7 @@ def strftime(epoch, zone=time.gmtime): def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner + import uuid dev = None scanner = DeviceScanner() scanner.scan() @@ -47,7 +48,7 @@ def get_connected_device(): for d in connected_devices: try: - d.open() + d.open(str(uuid.uuid4())) except: continue else: From 484caafc320b8bd408ed29086df5f5da694c6e9f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 14:03:46 -0700 Subject: [PATCH 50/53] Fix ebook-devide not passing a library uuid in dev.open() --- src/calibre/devices/prs500/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/prs500/cli/main.py b/src/calibre/devices/prs500/cli/main.py index cd8395467b..8a73f3fa23 100755 --- a/src/calibre/devices/prs500/cli/main.py +++ b/src/calibre/devices/prs500/cli/main.py @@ -6,7 +6,7 @@ Provides a command-line and optional graphical interface to the SONY Reader PRS- For usage information run the script. """ -import StringIO, sys, time, os +import StringIO, sys, time, os, uuid from optparse import OptionParser from calibre import __version__, __appname__ @@ -213,7 +213,7 @@ def main(): for d in connected_devices: try: - d.open() + d.open(str(uuid.uuid4())) except: continue else: From 14717a5e9287be8b189bffcccd9ec9e568b4cc36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 15:34:41 -0700 Subject: [PATCH 51/53] ImageMagick wrapper: Add API to get and set stroke and fill colors on a drawing wand --- src/calibre/utils/magick/__init__.py | 20 +++++++ src/calibre/utils/magick/magick.c | 82 ++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/src/calibre/utils/magick/__init__.py b/src/calibre/utils/magick/__init__.py index 834a798de5..6be5580d17 100644 --- a/src/calibre/utils/magick/__init__.py +++ b/src/calibre/utils/magick/__init__.py @@ -95,6 +95,26 @@ class DrawingWand(_magick.DrawingWand): # {{{ self.font_size_ = float(val) return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + @dynamic_property + def stroke_color(self): + def fget(self): + return self.stroke_color_.color + def fset(self, val): + col = PixelWand() + col.color = unicode(val) + self.stroke_color_ = col + return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + + @dynamic_property + def fill_color(self): + def fget(self): + return self.fill_color_.color + def fset(self, val): + col = PixelWand() + col.color = unicode(val) + self.fill_color_ = col + return property(fget=fget, fset=fset, doc=_magick.DrawingWand.font_size_.__doc__) + # }}} class Image(_magick.Image): # {{{ diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index 869b77c736..84c5f3a2ed 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -263,6 +263,78 @@ magick_DrawingWand_fontsize_setter(magick_DrawingWand *self, PyObject *val, void // }}} +// DrawingWand.stroke_color {{{ +static PyObject * +magick_DrawingWand_stroke_color_getter(magick_DrawingWand *self, void *closure) { + NULL_CHECK(NULL) + magick_PixelWand *pw; + PixelWand *wand = NewPixelWand(); + + if (wand == NULL) return PyErr_NoMemory(); + DrawGetStrokeColor(self->wand, wand); + + pw = (magick_PixelWand*) magick_PixelWandType.tp_alloc(&magick_PixelWandType, 0); + if (pw == NULL) return PyErr_NoMemory(); + pw->wand = wand; + return Py_BuildValue("O", (PyObject *)pw); +} + +static int +magick_DrawingWand_stroke_color_setter(magick_DrawingWand *self, PyObject *val, void *closure) { + NULL_CHECK(-1) + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete DrawingWand stroke color"); + return -1; + } + + magick_PixelWand *pw; + + pw = (magick_PixelWand*)val; + if (!IsPixelWand(pw->wand)) { PyErr_SetString(PyExc_TypeError, "Invalid PixelWand"); return -1; } + + DrawSetStrokeColor(self->wand, pw->wand); + + return 0; +} + +// }}} + +// DrawingWand.fill_color {{{ +static PyObject * +magick_DrawingWand_fill_color_getter(magick_DrawingWand *self, void *closure) { + NULL_CHECK(NULL) + magick_PixelWand *pw; + PixelWand *wand = NewPixelWand(); + + if (wand == NULL) return PyErr_NoMemory(); + DrawGetFillColor(self->wand, wand); + + pw = (magick_PixelWand*) magick_PixelWandType.tp_alloc(&magick_PixelWandType, 0); + if (pw == NULL) return PyErr_NoMemory(); + pw->wand = wand; + return Py_BuildValue("O", (PyObject *)pw); +} + +static int +magick_DrawingWand_fill_color_setter(magick_DrawingWand *self, PyObject *val, void *closure) { + NULL_CHECK(-1) + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete DrawingWand fill color"); + return -1; + } + + magick_PixelWand *pw; + + pw = (magick_PixelWand*)val; + if (!IsPixelWand(pw->wand)) { PyErr_SetString(PyExc_TypeError, "Invalid PixelWand"); return -1; } + + DrawSetFillColor(self->wand, pw->wand); + + return 0; +} + +// }}} + // DrawingWand.text_antialias {{{ static PyObject * magick_DrawingWand_textantialias_getter(magick_DrawingWand *self, void *closure) { @@ -336,6 +408,16 @@ static PyGetSetDef magick_DrawingWand_getsetters[] = { (char *)"DrawingWand fontsize", NULL}, + {(char *)"stroke_color_", + (getter)magick_DrawingWand_stroke_color_getter, (setter)magick_DrawingWand_stroke_color_setter, + (char *)"DrawingWand stroke color", + NULL}, + + {(char *)"fill_color_", + (getter)magick_DrawingWand_fill_color_getter, (setter)magick_DrawingWand_fill_color_setter, + (char *)"DrawingWand fill color", + NULL}, + {(char *)"text_antialias", (getter)magick_DrawingWand_textantialias_getter, (setter)magick_DrawingWand_textantialias_setter, (char *)"DrawingWand text antialias", From a4b50102ff69ef9dbc533ab514b9098b65ed5df0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 18:54:37 -0700 Subject: [PATCH 52/53] Updated Ming Pao --- resources/recipes/ming_pao.recipe | 585 +++++++++++++++++------------- 1 file changed, 331 insertions(+), 254 deletions(-) diff --git a/resources/recipes/ming_pao.recipe b/resources/recipes/ming_pao.recipe index bbdbbf7ace..4a405a59dd 100644 --- a/resources/recipes/ming_pao.recipe +++ b/resources/recipes/ming_pao.recipe @@ -1,7 +1,20 @@ __license__ = 'GPL v3' __copyright__ = '2010-2011, Eddie Lau' + +# Users of Kindle 3 (with limited system-level CJK support) +# please replace the following "True" with "False". +__MakePeriodical__ = True +# Turn it to True if your device supports display of CJK titles +__UseChineseTitle__ = False + + ''' Change Log: +2011/03/06: add new articles for finance section, also a new section "Columns" +2011/02/28: rearrange the sections + [Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles + View] make it the same title if generating a periodical, so past issue will be automatically put into "Past Issues" + folder in Kindle 3 2011/02/20: skip duplicated links in finance section, put photos which may extend a whole page to the back of the articles clean up the indentation 2010/12/07: add entertainment section, use newspaper front page as ebook cover, suppress date display in section list @@ -19,55 +32,58 @@ import os, datetime, re from calibre.web.feeds.recipes import BasicNewsRecipe from contextlib import nested - from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.toc import TOC from calibre.ebooks.metadata import MetaInformation class MPHKRecipe(BasicNewsRecipe): - IsCJKWellSupported = True # Set to False to avoid generating periodical in which CJK characters can't be displayed in section/article view - title = 'Ming Pao - Hong Kong' - oldest_article = 1 - max_articles_per_feed = 100 - __author__ = 'Eddie Lau' - description = ('Hong Kong Chinese Newspaper (http://news.mingpao.com). If' - 'you are using a Kindle with firmware < 3.1, customize the' - 'recipe') - publisher = 'MingPao' - category = 'Chinese, News, Hong Kong' - remove_javascript = True - use_embedded_content = False - no_stylesheets = True - language = 'zh' - encoding = 'Big5-HKSCS' - recursions = 0 - conversion_options = {'linearize_tables':True} - timefmt = '' - extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' - masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' - keep_only_tags = [dict(name='h1'), + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)' + publisher = 'MingPao' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + timefmt = '' + extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}' + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + keep_only_tags = [dict(name='h1'), dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title - dict(attrs={'id':['newscontent']}), # entertainment page content + dict(name='font', attrs={'color':['AA0000']}), # for column articles title + dict(attrs={'id':['newscontent']}), # entertainment and column page content dict(attrs={'id':['newscontent01','newscontent02']}), dict(attrs={'class':['photo']}) ] - remove_tags = [dict(name='style'), - dict(attrs={'id':['newscontent135']})] # for the finance page - remove_attributes = ['width'] - preprocess_regexps = [ + remove_tags = [dict(name='style'), + dict(attrs={'id':['newscontent135']}), # for the finance page + dict(name='table')] # for content fetched from life.mingpao.com + remove_attributes = ['width'] + preprocess_regexps = [ (re.compile(r'
    ', re.DOTALL|re.IGNORECASE), lambda match: '

    '), (re.compile(r'

    ', re.DOTALL|re.IGNORECASE), lambda match: ''), (re.compile(r'

    ', re.DOTALL|re.IGNORECASE), # for entertainment page - lambda match: '') + lambda match: ''), + # skip
    after title in life.mingpao.com fetched article + (re.compile(r"

    ", re.DOTALL|re.IGNORECASE), + lambda match: "
    "), + (re.compile(r"

    ", re.DOTALL|re.IGNORECASE), + lambda match: "") ] - def image_url_processor(cls, baseurl, url): - # trick: break the url at the first occurance of digit, add an additional - # '_' at the front - # not working, may need to move this to preprocess_html() method + def image_url_processor(cls, baseurl, url): + # trick: break the url at the first occurance of digit, add an additional + # '_' at the front + # not working, may need to move this to preprocess_html() method # minIdx = 10000 # i0 = url.find('0') # if i0 >= 0 and i0 < minIdx: @@ -99,253 +115,314 @@ class MPHKRecipe(BasicNewsRecipe): # i9 = url.find('9') # if i9 >= 0 and i9 < minIdx: # minIdx = i9 - return url + return url - def get_dtlocal(self): - dt_utc = datetime.datetime.utcnow() - # convert UTC to local hk time - at around HKT 6.00am, all news are available - dt_local = dt_utc - datetime.timedelta(-2.0/24) - return dt_local + def get_dtlocal(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time - at around HKT 6.00am, all news are available + dt_local = dt_utc - datetime.timedelta(-2.0/24) + return dt_local - def get_fetchdate(self): - return self.get_dtlocal().strftime("%Y%m%d") + def get_fetchdate(self): + return self.get_dtlocal().strftime("%Y%m%d") - def get_fetchformatteddate(self): - return self.get_dtlocal().strftime("%Y-%m-%d") + def get_fetchformatteddate(self): + return self.get_dtlocal().strftime("%Y-%m-%d") - def get_fetchday(self): - # convert UTC to local hk time - at around HKT 6.00am, all news are available - return self.get_dtlocal().strftime("%d") + def get_fetchday(self): + # convert UTC to local hk time - at around HKT 6.00am, all news are available + return self.get_dtlocal().strftime("%d") - def get_cover_url(self): - cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' - br = BasicNewsRecipe.get_browser() - try: - br.open(cover) - except: - cover = None - return cover + def get_cover_url(self): + cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg' + br = BasicNewsRecipe.get_browser() + try: + br.open(cover) + except: + cover = None + return cover - def parse_index(self): - feeds = [] - dateStr = self.get_fetchdate() - for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), - (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), - (u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://news.mingpao.com/' + dateStr + '/mrindex.htm'), - (u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() + + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), + (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), + (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special- editorial + ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr') + if ed_articles: + feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles)) + + for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), - (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm'), - ('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), - (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm'), - (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm'), - (u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), + (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special - finance + #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') + fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea') + if fin_articles: + feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) + + for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), + (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + + # special - entertainment + ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') + if ent_articles: + feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) + + for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'), (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]: - articles = self.parse_section(url) - if articles: - feeds.append((title, articles)) - # special - finance - fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm') - if fin_articles: - feeds.append((u'\u7d93\u6fdf Finance', fin_articles)) - # special - entertainment - ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm') - if ent_articles: - feeds.append((u'\u5f71\u8996 Film/TV', ent_articles)) - return feeds + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) - def parse_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) - current_articles = [] - included_urls = [] - divs.reverse() - for i in divs: - a = i.find('a', href = True) - title = self.tag_to_string(a) - url = a.get('href', False) - url = 'http://news.mingpao.com/' + dateStr + '/' +url - if url not in included_urls and url.rfind('Redirect') == -1: - current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) - included_urls.append(url) - current_articles.reverse() - return current_articles - def parse_fin_section(self, url): - dateStr = self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href= True) - current_articles = [] - included_urls = [] - for i in a: - url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) - if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: - title = self.tag_to_string(i) - current_articles.append({'title': title, 'url': url, 'description':''}) - included_urls.append(url) - return current_articles + # special- columns + col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn') + if col_articles: + feeds.append((u'\u5c08\u6b04 Columns', col_articles)) - def parse_ent_section(self, url): - self.get_fetchdate() - soup = self.index_to_soup(url) - a = soup.findAll('a', href=True) - a.reverse() - current_articles = [] - included_urls = [] - for i in a: - title = self.tag_to_string(i) - url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) - if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): - current_articles.append({'title': title, 'url': url, 'description': ''}) - included_urls.append(url) - current_articles.reverse() - return current_articles + return feeds - def preprocess_html(self, soup): - for item in soup.findAll(style=True): - del item['style'] - for item in soup.findAll(style=True): - del item['width'] - for item in soup.findAll(stype=True): - del item['absmiddle'] - return soup + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']}) + current_articles = [] + included_urls = [] + divs.reverse() + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + if url not in included_urls and url.rfind('Redirect') == -1: + current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - def create_opf(self, feeds, dir=None): - if dir is None: - dir = self.output_dir - if self.IsCJKWellSupported == True: - # use Chinese title - title = u'\u660e\u5831 (\u9999\u6e2f) ' + self.get_fetchformatteddate() - else: - # use English title - title = self.short_title() + ' ' + self.get_fetchformatteddate() - if True: # force date in title - # title += strftime(self.timefmt) - mi = MetaInformation(title, [self.publisher]) - mi.publisher = self.publisher - mi.author_sort = self.publisher - if self.IsCJKWellSupported == True: - mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() - else: - mi.publication_type = self.publication_type+':'+self.short_title() - #mi.timestamp = nowf() - mi.timestamp = self.get_dtlocal() - mi.comments = self.description - if not isinstance(mi.comments, unicode): - mi.comments = mi.comments.decode('utf-8', 'replace') - #mi.pubdate = nowf() - mi.pubdate = self.get_dtlocal() - opf_path = os.path.join(dir, 'index.opf') - ncx_path = os.path.join(dir, 'index.ncx') - opf = OPFCreator(dir, mi) - # Add mastheadImage entry to section - mp = getattr(self, 'masthead_path', None) - if mp is not None and os.access(mp, os.R_OK): - from calibre.ebooks.metadata.opf2 import Guide - ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) - ref.type = 'masthead' - ref.title = 'Masthead Image' - opf.guide.append(ref) + def parse_ed_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] - manifest.append(os.path.join(dir, 'index.html')) - manifest.append(os.path.join(dir, 'index.ncx')) + def parse_fin_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href= True) + current_articles = [] + included_urls = [] + for i in a: + #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1: + if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1): + title = self.tag_to_string(i) + current_articles.append({'title': title, 'url': url, 'description':''}) + included_urls.append(url) + return current_articles - # Get cover - cpath = getattr(self, 'cover_path', None) - if cpath is None: - pf = open(os.path.join(dir, 'cover.jpg'), 'wb') - if self.default_cover(pf): - cpath = pf.name - if cpath is not None and os.access(cpath, os.R_OK): - opf.cover = cpath - manifest.append(cpath) + def parse_ent_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://ol.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - # Get masthead - mpath = getattr(self, 'masthead_path', None) - if mpath is not None and os.access(mpath, os.R_OK): - manifest.append(mpath) + def parse_col_section(self, url): + self.get_fetchdate() + soup = self.index_to_soup(url) + a = soup.findAll('a', href=True) + a.reverse() + current_articles = [] + included_urls = [] + for i in a: + title = self.tag_to_string(i) + url = 'http://life.mingpao.com/cfm/' + i.get('href', False) + if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1): + current_articles.append({'title': title, 'url': url, 'description': ''}) + included_urls.append(url) + current_articles.reverse() + return current_articles - opf.create_manifest_from_files_in(manifest) - for mani in opf.manifest: - if mani.path.endswith('.ncx'): - mani.id = 'ncx' - if mani.path.endswith('mastheadImage.jpg'): - mani.id = 'masthead-image' - entries = ['index.html'] - toc = TOC(base_path=dir) - self.play_order_counter = 0 - self.play_order_map = {} + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(style=True): + del item['width'] + for item in soup.findAll(stype=True): + del item['absmiddle'] + return soup - def feed_index(num, parent): - f = feeds[num] - for j, a in enumerate(f): - if getattr(a, 'downloaded', False): - adir = 'feed_%d/article_%d/'%(num, j) - auth = a.author - if not auth: - auth = None - desc = a.text_summary - if not desc: - desc = None - else: - desc = self.description_limiter(desc) - entries.append('%sindex.html'%adir) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), + def create_opf(self, feeds, dir=None): + if dir is None: + dir = self.output_dir + if __UseChineseTitle__ == True: + title = u'\u660e\u5831 (\u9999\u6e2f)' + else: + title = self.short_title() + # if not generating a periodical, force date to apply in title + if __MakePeriodical__ == False: + title = title + ' ' + self.get_fetchformatteddate() + if True: + mi = MetaInformation(title, [self.publisher]) + mi.publisher = self.publisher + mi.author_sort = self.publisher + if __MakePeriodical__ == True: + mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title() + else: + mi.publication_type = self.publication_type+':'+self.short_title() + #mi.timestamp = nowf() + mi.timestamp = self.get_dtlocal() + mi.comments = self.description + if not isinstance(mi.comments, unicode): + mi.comments = mi.comments.decode('utf-8', 'replace') + #mi.pubdate = nowf() + mi.pubdate = self.get_dtlocal() + opf_path = os.path.join(dir, 'index.opf') + ncx_path = os.path.join(dir, 'index.ncx') + opf = OPFCreator(dir, mi) + # Add mastheadImage entry to section + mp = getattr(self, 'masthead_path', None) + if mp is not None and os.access(mp, os.R_OK): + from calibre.ebooks.metadata.opf2 import Guide + ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu()) + ref.type = 'masthead' + ref.title = 'Masthead Image' + opf.guide.append(ref) + + manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))] + manifest.append(os.path.join(dir, 'index.html')) + manifest.append(os.path.join(dir, 'index.ncx')) + + # Get cover + cpath = getattr(self, 'cover_path', None) + if cpath is None: + pf = open(os.path.join(dir, 'cover.jpg'), 'wb') + if self.default_cover(pf): + cpath = pf.name + if cpath is not None and os.access(cpath, os.R_OK): + opf.cover = cpath + manifest.append(cpath) + + # Get masthead + mpath = getattr(self, 'masthead_path', None) + if mpath is not None and os.access(mpath, os.R_OK): + manifest.append(mpath) + + opf.create_manifest_from_files_in(manifest) + for mani in opf.manifest: + if mani.path.endswith('.ncx'): + mani.id = 'ncx' + if mani.path.endswith('mastheadImage.jpg'): + mani.id = 'masthead-image' + entries = ['index.html'] + toc = TOC(base_path=dir) + self.play_order_counter = 0 + self.play_order_map = {} + + def feed_index(num, parent): + f = feeds[num] + for j, a in enumerate(f): + if getattr(a, 'downloaded', False): + adir = 'feed_%d/article_%d/'%(num, j) + auth = a.author + if not auth: + auth = None + desc = a.text_summary + if not desc: + desc = None + else: + desc = self.description_limiter(desc) + entries.append('%sindex.html'%adir) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'), play_order=po, author=auth, description=desc) - last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) - for sp in a.sub_pages: - prefix = os.path.commonprefix([opf_path, sp]) - relp = sp[len(prefix):] - entries.append(relp.replace(os.sep, '/')) - last = sp + last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep)) + for sp in a.sub_pages: + prefix = os.path.commonprefix([opf_path, sp]) + relp = sp[len(prefix):] + entries.append(relp.replace(os.sep, '/')) + last = sp - if os.path.exists(last): - with open(last, 'rb') as fi: - src = fi.read().decode('utf-8') - soup = BeautifulSoup(src) - body = soup.find('body') - if body is not None: - prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - templ = self.navbar.generate(True, num, j, len(f), + if os.path.exists(last): + with open(last, 'rb') as fi: + src = fi.read().decode('utf-8') + soup = BeautifulSoup(src) + body = soup.find('body') + if body is not None: + prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) + templ = self.navbar.generate(True, num, j, len(f), not self.has_single_feed, a.orig_url, self.publisher, prefix=prefix, center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) - with open(last, 'wb') as fi: - fi.write(unicode(soup).encode('utf-8')) - if len(feeds) == 0: - raise Exception('All feeds are empty, aborting.') + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) + with open(last, 'wb') as fi: + fi.write(unicode(soup).encode('utf-8')) + if len(feeds) == 0: + raise Exception('All feeds are empty, aborting.') - if len(feeds) > 1: - for i, f in enumerate(feeds): - entries.append('feed_%d/index.html'%i) - po = self.play_order_map.get(entries[-1], None) - if po is None: - self.play_order_counter += 1 - po = self.play_order_counter - auth = getattr(f, 'author', None) - if not auth: - auth = None - desc = getattr(f, 'description', None) - if not desc: - desc = None - feed_index(i, toc.add_item('feed_%d/index.html'%i, None, + if len(feeds) > 1: + for i, f in enumerate(feeds): + entries.append('feed_%d/index.html'%i) + po = self.play_order_map.get(entries[-1], None) + if po is None: + self.play_order_counter += 1 + po = self.play_order_counter + auth = getattr(f, 'author', None) + if not auth: + auth = None + desc = getattr(f, 'description', None) + if not desc: + desc = None + feed_index(i, toc.add_item('feed_%d/index.html'%i, None, f.title, play_order=po, description=desc, author=auth)) - else: - entries.append('feed_%d/index.html'%0) - feed_index(0, toc) + else: + entries.append('feed_%d/index.html'%0) + feed_index(0, toc) - for i, p in enumerate(entries): - entries[i] = os.path.join(dir, p.replace('/', os.sep)) - opf.create_spine(entries) - opf.set_toc(toc) + for i, p in enumerate(entries): + entries[i] = os.path.join(dir, p.replace('/', os.sep)) + opf.create_spine(entries) + opf.set_toc(toc) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): - opf.render(opf_file, ncx_file) + with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + opf.render(opf_file, ncx_file) From 4c180ba4cf23085a0f6583b585af4f566ed9bca8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 8 Mar 2011 19:07:16 -0700 Subject: [PATCH 53/53] Fix #9283 (Custom column with integers accepts only negative values in 0.7.48) --- src/calibre/gui2/custom_column_widgets.py | 11 +++++------ src/calibre/gui2/library/delegates.py | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 8641f9e712..beaca77a38 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -5,7 +5,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys from functools import partial from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \ @@ -85,7 +84,7 @@ class Int(Base): self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), QSpinBox(parent)] w = self.widgets[1] - w.setRange(-100, sys.maxint) + w.setRange(-100, 100000000) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -108,7 +107,7 @@ class Float(Int): self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), QDoubleSpinBox(parent)] w = self.widgets[1] - w.setRange(-100., float(sys.maxint)) + w.setRange(-100., float(100000000)) w.setDecimals(2) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -289,7 +288,7 @@ class Series(Base): self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent)) w = QDoubleSpinBox(parent) - w.setRange(-100., float(sys.maxint)) + w.setRange(-100., float(100000000)) w.setDecimals(2) w.setSpecialValueText(_('Undefined')) w.setSingleStep(1) @@ -595,7 +594,7 @@ class BulkInt(BulkBase): def setup_ui(self, parent): self.make_widgets(parent, QSpinBox) - self.main_widget.setRange(-100, sys.maxint) + self.main_widget.setRange(-100, 100000000) self.main_widget.setSpecialValueText(_('Undefined')) self.main_widget.setSingleStep(1) @@ -617,7 +616,7 @@ class BulkFloat(BulkInt): def setup_ui(self, parent): self.make_widgets(parent, QDoubleSpinBox) - self.main_widget.setRange(-100., float(sys.maxint)) + self.main_widget.setRange(-100., float(100000000)) self.main_widget.setDecimals(2) self.main_widget.setSpecialValueText(_('Undefined')) self.main_widget.setSingleStep(1) diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index 87da6818eb..3a090f8102 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -5,7 +5,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys from math import cos, sin, pi from PyQt4.Qt import QColor, Qt, QModelIndex, QSize, \ @@ -245,13 +244,13 @@ class CcTextDelegate(QStyledItemDelegate): # {{{ typ = m.custom_columns[col]['datatype'] if typ == 'int': editor = QSpinBox(parent) - editor.setRange(-100, sys.maxint) + editor.setRange(-100, 100000000) editor.setSpecialValueText(_('Undefined')) editor.setSingleStep(1) elif typ == 'float': editor = QDoubleSpinBox(parent) editor.setSpecialValueText(_('Undefined')) - editor.setRange(-100., float(sys.maxint)) + editor.setRange(-100., 100000000) editor.setDecimals(2) else: editor = MultiCompleteLineEdit(parent)