From f22b9453ae5248670079646e48493c7b893f93ff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 12:32:55 -0600 Subject: [PATCH 01/13] New recipe for Variety by Darko Miletic --- resources/images/news/variety.png | Bin 0 -> 332 bytes resources/recipes/variety.recipe | 46 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 resources/images/news/variety.png create mode 100644 resources/recipes/variety.recipe diff --git a/resources/images/news/variety.png b/resources/images/news/variety.png new file mode 100644 index 0000000000000000000000000000000000000000..5bb127d4b42851491ac8d77b0e5b4de02998f3d8 GIT binary patch literal 332 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*VRoKjC0#}JFtXD2!G9Z=w4VZYq|ySQqDRESE2 zcBkW%#;F|tw{`^ue!p`=IJWKRZ1I~1e(APNU(q;YZ{LoS_J@}>t`Y6ES=Ka9=|WLO zG8@~3O%1P(+3_4$w?JL}!OjCVOGIQeJr4d}u%r30q>Aqx{z+}soAf?T-COWmbF$`~ z-yo-|mbgZgq$HN4S|t~y0x1R~0|QH4LjzqS^AJNLDFo9^0`;w~;)WG2B>gTe~DWM4fG%#;N literal 0 HcmV?d00001 diff --git a/resources/recipes/variety.recipe b/resources/recipes/variety.recipe new file mode 100644 index 0000000000..7321e0ad33 --- /dev/null +++ b/resources/recipes/variety.recipe @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2009, Darko Miletic ' +''' +www.variety.com +''' + +from calibre.web.feeds.recipes import BasicNewsRecipe + +class Variety(BasicNewsRecipe): + title = 'Variety' + __author__ = 'Darko Miletic' + description = 'Breaking entertainment movie news, movie reviews, entertainment industry events, news and reviews from Cannes, Oscars, and Hollywood awards. Featuring box office charts, archives and more.' + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + encoding = 'cp1252' + publisher = 'Red Business Information' + category = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood' + language = 'en' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + remove_tags = [dict(name=['object','link','map'])] + + keep_only_tags = [dict(name='div', attrs={'id':'article'})] + + feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )] + + def print_version(self, url): + rpt = url.rpartition('?')[0] + artid = rpt.rpartition('/')[2] + catidr = url.rpartition('categoryid=')[2] + catid = catidr.partition('&')[0] + return 'http://www.variety.com/index.asp?layout=print_story&articleid=' + artid + '&categoryid=' + catid + + def get_article_url(self, article): + return article.get('feedburner_origlink', None) + From 138a7c53ad97fdcfb1fbb1216cba4d081f014c3b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 12:41:58 -0600 Subject: [PATCH 02/13] IGN:Move building of linux 64 bit binary to a VM --- setup/installer/linux/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/setup/installer/linux/__init__.py b/setup/installer/linux/__init__.py index 46014dfac6..f80efbd125 100644 --- a/setup/installer/linux/__init__.py +++ b/setup/installer/linux/__init__.py @@ -6,10 +6,9 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os from setup.installer import VMInstaller -from setup import Command, installer_name +from setup import Command class Linux32(VMInstaller): @@ -21,17 +20,12 @@ class Linux32(VMInstaller): FREEZE_COMMAND = 'linux_freeze' -class Linux64(Command): +class Linux64(Linux32): description = 'Build 64bit linux binary installer' - - sub_commands = ['linux_freeze'] - - def run(self, opts): - installer = installer_name('tar.bz2', True) - if not os.path.exists(installer): - raise Exception('Failed to build installer '+installer) - return os.path.basename(installer) + VM_NAME = 'gentoo64_build' + VM = '/vmware/bin/gentoo64_build' + IS_64_BIT = True class Linux(Command): From 10c9f4032a1b956fe87ed836169e28c587385829 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 12:46:08 -0600 Subject: [PATCH 03/13] Improve header and footer regular expression matching --- src/calibre/ebooks/conversion/preprocess.py | 11 +++++------ src/calibre/ebooks/oeb/iterator.py | 4 ++-- src/calibre/gui2/convert/regex_builder.py | 6 +++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index 39ca28e87f..b63c7ca861 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -228,17 +228,16 @@ class HTMLPreProcessor(object): else: rules = [] - pre_rules = [] + end_rules = [] if getattr(self.extra_opts, 'remove_header', None): - pre_rules.append( + end_rules.append( (re.compile(getattr(self.extra_opts, 'header_regex')), lambda match : '') ) if getattr(self.extra_opts, 'remove_footer', None): - pre_rules.append( + end_rules.append( (re.compile(getattr(self.extra_opts, 'footer_regex')), lambda match : '') ) - - end_rules = [] + if getattr(self.extra_opts, 'unwrap_factor', 0.0) > 0.01: length = line_length(html, getattr(self.extra_opts, 'unwrap_factor')) if length: @@ -247,7 +246,7 @@ class HTMLPreProcessor(object): (re.compile(r'(?<=.{%i}[a-z\.,;:)-IA])\s*(?P)?\s*()\s*(?=(<(i|b|u)>)?\s*[\w\d(])' % length, re.UNICODE), wrap_lines), ) - for rule in self.PREPROCESS + pre_rules + rules + end_rules: + for rule in self.PREPROCESS + rules + end_rules: html = rule[0].sub(rule[1], html) # Handle broken XHTML w/ SVG (ugh) diff --git a/src/calibre/ebooks/oeb/iterator.py b/src/calibre/ebooks/oeb/iterator.py index 762b14c3e5..565ceed519 100644 --- a/src/calibre/ebooks/oeb/iterator.py +++ b/src/calibre/ebooks/oeb/iterator.py @@ -123,7 +123,7 @@ class EbookIterator(object): else: print 'Loaded embedded font:', repr(family) - def __enter__(self, raw_only=False): + def __enter__(self, processed=False): self.delete_on_exit = [] self._tdir = TemporaryDirectory('_ebook_iter') self.base = self._tdir.__enter__() @@ -140,7 +140,7 @@ class EbookIterator(object): plumber.opts, plumber.input_fmt, self.log, {}, self.base) - if not raw_only and plumber.input_fmt.lower() in ('pdf', 'rb'): + if processed or plumber.input_fmt.lower() in ('pdf', 'rb'): self.pathtoopf = create_oebbook(self.log, self.pathtoopf, plumber.opts, plumber.input_plugin) if hasattr(self.pathtoopf, 'manifest'): diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index 20da8d7aaf..b1d8fbcbd5 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -87,12 +87,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): def open_book(self, pathtoebook): self.iterator = EbookIterator(pathtoebook) - self.iterator.__enter__(raw_only=True) + self.iterator.__enter__(processed=True) text = [u''] for path in self.iterator.spine: - html = open(path, 'rb').read().decode(path.encoding, 'replace') + html = open(path, 'rb').read().decode('utf-8', 'replace') text.append(html) - self.preview.setPlainText('\n\n'.join(text)) + self.preview.setPlainText('\n---\n'.join(text)) def button_clicked(self, button): if button == self.button_box.button(QDialogButtonBox.Open): From 53f7cdec0b3ffbb7db784bcb6c2eca3caa8f7061 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 13:06:24 -0600 Subject: [PATCH 04/13] IGN:More conversion documentation --- src/calibre/manual/conversion.rst | 64 +++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index 1f23d43419..6124d24aa2 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -301,11 +301,17 @@ Removing headers and footers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These options are useful primarily for conversion of PDF documents. Often, the conversion leaves -behing page headers and footers in the text. These options use regular expressions to try and detect +behind page headers and footers in the text. These options use regular expressions to try and detect the headers and footers and remove them. Remember that they operate on the intermediate XHTML produced by the conversion pipeline. There is also a wizard to help you customize the regular expressions for your document. +The header and footer regular expressions are used in conjunction with the remove header and footer options. +If the remove option is not enabled the regular expression will not be applied to remove the matched text. +The removal works by using a python regular expression. All matched text is simply removed from +the document. You can learn more about regular expressions and their syntax at +http://docs.python.org/library/re.html. + Miscellaneous ~~~~~~~~~~~~~~ @@ -403,7 +409,9 @@ This will result in an automatically generated two level Table of Contents that Format specific tips ---------------------- -Here you will find tips specific to the conversion of particular formats. +Here you will find tips specific to the conversion of particular formats. Options specific to particular +format, whether input or output are available in the conversion dialog under their own section, for example +`TXT Input` or `EPUB Output`. Convert Microsoft Word documents ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -419,7 +427,57 @@ generating the Table of Contents much simpler. It is called BookCreator and is a Convert TXT documents ~~~~~~~~~~~~~~~~~~~~~~ +TXT documents have no well defined way to specify formatting like bold, italics, etc, or document structure like paragraphs, headings, sections and so on. +Since TXT documents provide no way to explicitly mark parts of +the text, by default |app| only groups lines in the input document into paragraphs. The default is to assume one or +more blank lines are a paragraph boundary:: + + This is the first. + + This is the + second paragraph. + +TXT input supports a number of options to differentiate how paragraphs are detected. + + :guilabel:`Treat each line as a paragraph` + Assumes that every line is a paragraph:: + + This is the first. + This is the second. + This is the third. + + :guilabel:`Assume print formatting` + Assumes that every paragraph starts with an indent (either a tab or 2+ spaces). Paragraphs end when + the next line that starts with an indent is reached:: + + This is the + first. + This is the second. + + This is the + third. + + :guilabel:`Process using markdown` + |app| also supports running TXT input though a transformation preprocessor known as markdown. Markdown + allows for basic formatting to be added to TXT documents, such as bold, italics, section headings, tables, + loists, a Table of Contents, etc. Marking chapter headings with a leading # and setting the chapter XPath detection + expression to "//h:h1" is the easiest way to have a proper table of contents generated from a TXT document. + You can learn more about the markdown syntax at http://daringfireball.net/projects/markdown/syntax. + + Convert PDF documents -~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +PDF documents are one of the worst formats to convert from. They are a fixed page size and text placement format. +Meaning, it is very difficult to determine where one paragraph ends and another begins. |app| will try to unwrap +paragraphs using a configurable, :guilabel:`Line Un-Wrapping Factor`. This is a scale used to determine the length at which a line should be unwrapped. Valid values are a decimal +between 0 and 1. The default is 0.5, this is the median line length. Lower this value to include more +text in the unwrapping. Increase to include less. + +Also, they often have headers and footers as part of the document that will become included with the text. +Use the options to remove headers and footers to mitigate this issue. If the headers and footers are not +removed from the text it can throw off the paragraph unwrapping. + +Some limitations of PDF input is complex, multi-column, and image based documents are not supported. +Extraction of vector images and tables from within the document is also not supported. From 1d7c3277ec760064a4760acc4da3a09652cf8ccf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 13:07:37 -0600 Subject: [PATCH 05/13] IGN:PML Output: Replace non breaking spaces with normal spaces --- src/calibre/ebooks/pml/pmlml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/pml/pmlml.py b/src/calibre/ebooks/pml/pmlml.py index 27e88eb48b..aa608496c7 100644 --- a/src/calibre/ebooks/pml/pmlml.py +++ b/src/calibre/ebooks/pml/pmlml.py @@ -153,6 +153,10 @@ class PMLMLizer(object): for unused in anchors.difference(links): text = text.replace('\\Q="%s"' % unused, '') + # Replace bad characters. + text = text.replace(u'\xc2', '') + text = text.replace(u'\xa0', ' ') + # Turn all html entities into unicode. This should not be necessary as # lxml should have already done this but we want to be sure it happens. for entity in set(re.findall('&.+?;', text)): From ad156151b21dc0ea64bbf731eb0af0b13cdfce72 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 16:31:41 -0600 Subject: [PATCH 06/13] IGN:... --- src/calibre/manual/conversion.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index 6124d24aa2..54fbc7362e 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -470,7 +470,8 @@ Convert PDF documents PDF documents are one of the worst formats to convert from. They are a fixed page size and text placement format. Meaning, it is very difficult to determine where one paragraph ends and another begins. |app| will try to unwrap -paragraphs using a configurable, :guilabel:`Line Un-Wrapping Factor`. This is a scale used to determine the length at which a line should be unwrapped. Valid values are a decimal +paragraphs using a configurable, :guilabel:`Line Un-Wrapping Factor`. This is a scale used to determine the length +at which a line should be unwrapped. Valid values are a decimal between 0 and 1. The default is 0.5, this is the median line length. Lower this value to include more text in the unwrapping. Increase to include less. From 919f675039069bfec688175e9ede2835070e7c18 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 21:31:42 -0600 Subject: [PATCH 07/13] IGN:... --- resources/recipes/outlook_india.recipe | 1 - src/calibre/manual/conversion.rst | 4 ++-- src/calibre/utils/complete.py | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/recipes/outlook_india.recipe b/resources/recipes/outlook_india.recipe index 6bec491c75..1dccd468e5 100644 --- a/resources/recipes/outlook_india.recipe +++ b/resources/recipes/outlook_india.recipe @@ -14,7 +14,6 @@ class OutlookIndia(BasicNewsRecipe): encoding = 'utf-8' language = 'en_IN' - recursions = 1 extra_css = ''' body{font-family:Arial,Helvetica,sans-serif; font-size:xx-small;} .fspheading{color:#AF0E25 ; font-family:"Times New Roman",Times,serif; font-weight:bold ; font-size:large; } diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst index 54fbc7362e..949bd9527b 100644 --- a/src/calibre/manual/conversion.rst +++ b/src/calibre/manual/conversion.rst @@ -418,11 +418,11 @@ Convert Microsoft Word documents |app| does not directly convert .doc files from Microsoft Word. However, in Word, you can save the document as HTML and then convert the resulting HTML file with |app|. When saving as HTML, be sure to use the -"Save as filtered HTML" option as this will produce clean HTML that will convert well. +"Save as Web Page, Filtered" option as this will produce clean HTML that will convert well. There is a Word macro package that can automate the conversion of Word documents using |app|. It also makes generating the Table of Contents much simpler. It is called BookCreator and is available for free -`here `_. +`here `_. Convert TXT documents ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/utils/complete.py b/src/calibre/utils/complete.py index 26b3108c0b..d196a9521b 100644 --- a/src/calibre/utils/complete.py +++ b/src/calibre/utils/complete.py @@ -65,6 +65,7 @@ def split(src): def files_and_dirs(prefix, allowed_exts=[]): + prefix = os.path.expanduser(prefix) for i in glob.iglob(prefix+'*'): _, ext = os.path.splitext(i) ext = ext.lower().replace('.', '') From 8a0aad39f90399dd78d3b444a9805483d92815d1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 21:47:39 -0600 Subject: [PATCH 08/13] Content Server: Add a mobile friendly interface. Now if you access the content server using a mobile browser, a list of books that is easy to browse on small/e-ink screens is returned. Fixes #3870 (Add a mobile user interface for calibre) --- src/calibre/library/server.py | 217 +++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/server.py b/src/calibre/library/server.py index f615827cde..015df2b017 100644 --- a/src/calibre/library/server.py +++ b/src/calibre/library/server.py @@ -24,7 +24,7 @@ except ImportError: from calibre.constants import __version__, __appname__ from calibre.utils.genshi.template import MarkupTemplate from calibre import fit_image, guess_type, prepare_string_for_xml, \ - strftime as _strftime + strftime as _strftime, prints from calibre.library import server_config as config from calibre.library.database2 import LibraryDatabase2, FIELD_MAP from calibre.utils.config import config_dir @@ -77,6 +77,159 @@ class LibraryServer(object): ''') + MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian)') + + MOBILE_BOOK = textwrap.dedent('''\ + + + + + + + ${format.lower()}  + + ${r[1]} by ${authors} - ${r[6]/1024}k - ${r[3] if r[3] else ''} ${pubdate} ${'['+r[7]+']' if r[7] else ''} + + + ''') + + MOBILE = MarkupTemplate(textwrap.dedent('''\ + + + + + + + + +
+ Books ${start} to ${ min((start+num-1) , total) } of ${total} + + + + + +
+ First + Previous + + Next + Last +
+
+
+ + + ${Markup(book)} + +
+ + + ''')) + LIBRARY = MarkupTemplate(textwrap.dedent('''\ @@ -534,6 +687,52 @@ class LibraryServer(object): next_link=next_link, updated=updated, id='urn:calibre:main').render('xml') + @expose + def mobile(self, start='1', num='25', sort='date', search='', + _=None, order='descending'): + ''' + Serves metadata from the calibre database as XML. + + :param sort: Sort results by ``sort``. Can be one of `title,author,rating`. + :param search: Filter results by ``search`` query. See :class:`SearchQueryParser` for query syntax + :param start,num: Return the slice `[start:start+num]` of the sorted and filtered results + :param _: Firefox seems to sometimes send this when using XMLHttpRequest with no caching + ''' + try: + start = int(start) + except ValueError: + raise cherrypy.HTTPError(400, 'start: %s is not an integer'%start) + try: + num = int(num) + except ValueError: + raise cherrypy.HTTPError(400, 'num: %s is not an integer'%num) + ids = self.db.data.parse(search) if search and search.strip() else self.db.data.universal_set() + ids = sorted(ids) + items = [r for r in iter(self.db) if r[0] in ids] + if sort is not None: + self.sort(items, sort, (order.lower().strip() == 'ascending')) + + book, books = MarkupTemplate(self.MOBILE_BOOK), [] + for record in items[(start-1):(start-1)+num]: + aus = record[2] if record[2] else __builtin__._('Unknown') + authors = '|'.join([i.replace('|', ',') for i in aus.split(',')]) + record[10] = fmt_sidx(float(record[10])) + ts, pd = strftime('%Y/%m/%d %H:%M:%S', record[5]), \ + strftime('%Y/%m/%d %H:%M:%S', record[FIELD_MAP['pubdate']]) + books.append(book.generate(r=record, authors=authors, timestamp=ts, + pubdate=pd).render('xml').decode('utf-8')) + updated = self.db.last_modified() + + cherrypy.response.headers['Content-Type'] = 'text/html; charset=utf-8' + cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + + + url_base = "/mobile?search=" + search+";order="+order+";sort="+sort+";num="+str(num) + + return self.MOBILE.generate(books=books, start=start, updated=updated, search=search, sort=sort, order=order, num=num, + total=len(ids), url_base=url_base).render('html') + + @expose def library(self, start='0', num='50', sort=None, search=None, _=None, order='ascending'): @@ -584,10 +783,22 @@ class LibraryServer(object): cherrypy.request.headers.get('Stanza-Device-Name', 919) != 919 or \ cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \ ua.startswith('Stanza') - return self.stanza(search=kwargs.get('search', None), sortby=kwargs.get('sortby',None), authorid=kwargs.get('authorid',None), + + # A better search would be great + want_mobile = self.MOBILE_UA.search(ua) is not None + if self.opts.develop and not want_mobile: + prints('User agent:', ua) + + if want_opds: + return self.stanza(search=kwargs.get('search', None), sortby=kwargs.get('sortby',None), authorid=kwargs.get('authorid',None), tagid=kwargs.get('tagid',None), seriesid=kwargs.get('seriesid',None), - offset=kwargs.get('offset', 0)) if want_opds else self.static('index.html') + offset=kwargs.get('offset', 0)) + + if want_mobile: + return self.mobile() + + return self.static('index.html') @expose From 619a4d6e777039a76981eb55f4c351839085713b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 27 Oct 2009 22:05:18 -0600 Subject: [PATCH 09/13] Fix #3872 (EBook Viewer Bug Prevents Any Actions) --- src/calibre/gui2/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 426f4da50b..14789332e4 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -130,7 +130,7 @@ class CopyButton(QPushButton): return except: pass - return QPushButton.event(self, ev) + QPushButton.keyPressEvent(self, ev) def keyReleaseEvent(self, ev): @@ -139,7 +139,7 @@ class CopyButton(QPushButton): return except: pass - return QPushButton.event(self, ev) + QPushButton.keyReleaseEvent(self, ev) def mouseReleaseEvent(self, ev): ev.accept() From d3b61d909402f50e493d34ec24387fbd56119049 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 28 Oct 2009 10:25:53 -0600 Subject: [PATCH 10/13] Fix #3828 (Error 13, 'Permission denied')) --- src/calibre/ebooks/html/input.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index cf9b087295..36e21a0dc8 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -409,6 +409,9 @@ class HTMLInput(InputFormatPlugin): link = os.path.abspath(link) if not os.access(link, os.R_OK): return link_ + if os.path.isdir(link): + self.log.warn(link_, 'is a link to a directory. Ignoring.') + return link_ if not islinux: link = link.lower() if link not in self.added_resources: From b5fcc2466bb9ab1fb46539d4ba8bff85f53ff41c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 28 Oct 2009 14:18:17 -0600 Subject: [PATCH 11/13] Implement #3834 (Feature Request - Active Job Count in Tray Tooltip) --- src/calibre/gui2/jobs.py | 24 +++++++++++++++++++++++- src/calibre/gui2/main.py | 19 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index 1d86910e15..6bb9f52b36 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -11,7 +11,7 @@ from Queue import Empty, Queue from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ QTimer, SIGNAL, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ - QSize, QStyleOptionProgressBarV2, QString, QStyle + QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob @@ -57,6 +57,28 @@ class JobManager(QAbstractTableModel): else: return QVariant(section+1) + def show_tooltip(self, arg): + widget, pos = arg + QToolTip.showText(pos, self.get_tooltip()) + + def get_tooltip(self): + running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING] + waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING] + lines = [_('There are %d running jobs:')%len(running_jobs)] + for job in running_jobs: + desc = job.description + if not desc: + desc = _('Unknown job') + p = 100. if job.is_finished else job.percent + lines.append('%s: %.0f%% done'%(desc, p)) + lines.extend(['', _('There are %d waiting jobs:')%len(waiting_jobs)]) + for job in waiting_jobs: + desc = job.description + if not desc: + desc = _('Unknown job') + lines.append(desc) + return '\n'.join(['calibre', '']+ lines) + def data(self, index, role): try: if role not in (Qt.DisplayRole, Qt.DecorationRole): diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 26bd764ad4..66b880f0b2 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -11,7 +11,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \ QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \ QToolButton, QDialog, QDesktopServices, QFileDialog, \ QSystemTrayIcon, QApplication, QKeySequence, QAction, \ - QMessageBox, QStackedLayout + QMessageBox, QStackedLayout, QHelpEvent from PyQt4.QtSvg import QSvgRenderer from calibre import prints, patheq @@ -89,6 +89,18 @@ class Listener(Thread): except: pass +class SystemTrayIcon(QSystemTrayIcon): + + def __init__(self, icon, parent): + QSystemTrayIcon.__init__(self, icon, parent) + + def event(self, ev): + if ev.type() == ev.ToolTip: + evh = QHelpEvent(ev) + self.emit(SIGNAL('tooltip_requested(PyQt_PyObject)'), + (self, evh.globalPos())) + return True + return QSystemTrayIcon.event(self, ev) class Main(MainWindow, Ui_MainWindow, DeviceGUI): 'The main GUI' @@ -144,8 +156,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_connected = False self.viewers = collections.deque() self.content_server = None - self.system_tray_icon = QSystemTrayIcon(QIcon(I('library.png')), self) + self.system_tray_icon = SystemTrayIcon(QIcon(I('library.png')), self) self.system_tray_icon.setToolTip('calibre') + self.connect(self.system_tray_icon, + SIGNAL('tooltip_requested(PyQt_PyObject)'), + self.job_manager.show_tooltip) if not config['systray_icon']: self.system_tray_icon.hide() else: From 4c415a5ce0db52c834ae377b4757ffcfbc1bc249 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 28 Oct 2009 23:21:36 -0600 Subject: [PATCH 12/13] Stanza integration: Use a UUID instead of the database rowid as a unique identifier for each book. This means that, only after this upgrade, Stanza will forget which books it has already downloaded. Fixes #3137 (Provide optional user-customizable "Stanza Unique Identifier"). Also use a UUID when converting to EPUB as the book identifier. --- resources/recipes/daily_telegraph.recipe | 4 +- src/calibre/ebooks/html/input.py | 4 +- src/calibre/ebooks/metadata/__init__.py | 6 +-- src/calibre/ebooks/metadata/opf2.py | 30 ++++++++++- src/calibre/ebooks/oeb/reader.py | 7 ++- src/calibre/ebooks/oeb/transforms/metadata.py | 9 +++- src/calibre/library/cli.py | 9 ++-- src/calibre/library/database2.py | 53 +++++++++++++++++-- src/calibre/library/server.py | 3 +- src/calibre/library/sqlite.py | 3 +- 10 files changed, 106 insertions(+), 22 deletions(-) diff --git a/resources/recipes/daily_telegraph.recipe b/resources/recipes/daily_telegraph.recipe index 9935face07..61054e1db0 100644 --- a/resources/recipes/daily_telegraph.recipe +++ b/resources/recipes/daily_telegraph.recipe @@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class DailyTelegraph(BasicNewsRecipe): title = u'Daily Telegraph' __author__ = u'AprilHare' - language = 'en' + language = 'en_AU' description = u'News from down under' oldest_article = 2 @@ -45,4 +45,4 @@ class DailyTelegraph(BasicNewsRecipe): (u'NRL', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_nrl_345.xml'), (u'Rugby Union', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_rugby_union_342.xml'), (u'Soccer', u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_soccer_344.xml') - ] \ No newline at end of file + ] diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 36e21a0dc8..202475d7c9 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -320,8 +320,8 @@ class HTMLInput(InputFormatPlugin): oeb.logger.warn('Title not specified') metadata.add('title', self.oeb.translate(__('Unknown'))) - bookid = "urn:uuid:%s" % str(uuid.uuid4()) - metadata.add('identifier', bookid, id='calibre-uuid') + bookid = str(uuid.uuid4()) + metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') for ident in metadata.identifier: if 'id' in ident.attrib: self.oeb.uid = metadata.identifier[0] diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index fb56074079..ab9858b5ff 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -218,7 +218,7 @@ class MetaInformation(object): 'isbn', 'tags', 'cover_data', 'application_id', 'guide', 'manifest', 'spine', 'toc', 'cover', 'language', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', - 'pubdate', 'rights', 'publication_type'): + 'pubdate', 'rights', 'publication_type', 'uuid'): if hasattr(mi, attr): setattr(ans, attr, getattr(mi, attr)) @@ -244,7 +244,7 @@ class MetaInformation(object): 'series', 'series_index', 'rating', 'isbn', 'language', 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', - 'rights', 'publication_type', + 'rights', 'publication_type', 'uuid', ): setattr(self, x, getattr(mi, x, None)) @@ -264,7 +264,7 @@ class MetaInformation(object): 'isbn', 'application_id', 'manifest', 'spine', 'toc', 'cover', 'language', 'guide', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights', - 'publication_type'): + 'publication_type', 'uuid',): if hasattr(mi, attr): val = getattr(mi, attr) if val is not None: diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 4a19a6492d..c2244fd892 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -432,6 +432,9 @@ class OPF(object): identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]') application_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+ '(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]') + uuid_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+ + '(re:match(@opf:scheme, "uuid", "i") or re:match(@scheme, "uuid", "i"))]') + manifest_path = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]') manifest_ppath = XPath('descendant::*[re:match(name(), "manifest", "i")]') spine_path = XPath('descendant::*[re:match(name(), "spine", "i")]/*[re:match(name(), "itemref", "i")]') @@ -747,6 +750,25 @@ class OPF(object): return property(fget=fget, fset=fset) + @dynamic_property + def uuid(self): + + def fget(self): + for match in self.uuid_id_path(self.metadata): + return self.get_text(match) or None + + def fset(self, val): + matches = self.uuid_id_path(self.metadata) + if not matches: + attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'uuid'} + matches = [self.create_metadata_element('identifier', + attrib=attrib)] + self.set_text(matches[0], unicode(val)) + + return property(fget=fget, fset=fset) + + + @dynamic_property def book_producer(self): @@ -977,6 +999,9 @@ def metadata_to_opf(mi, as_string=True): if not mi.application_id: mi.application_id = str(uuid.uuid4()) + if not mi.uuid: + mi.uuid = str(uuid.uuid4()) + if not mi.book_producer: mi.book_producer = __appname__ + ' (%s) '%__version__ + \ '[http://calibre-ebook.com]' @@ -986,13 +1011,14 @@ def metadata_to_opf(mi, as_string=True): root = etree.fromstring(textwrap.dedent( ''' - + %(id)s + %(uuid)s - '''%dict(a=__appname__, id=mi.application_id))) + '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid))) metadata = root[0] guide = root[1] metadata[0].tail = '\n'+(' '*8) diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 87587e3ef5..339df7be60 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -139,10 +139,9 @@ class OEBReader(object): mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\ dict(a=__appname__, v=__version__) meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger) - bookid = "urn:uuid:%s" % str(uuid.uuid4()) if mi.application_id is None \ - else mi.application_id - self.oeb.metadata.add('identifier', bookid, id='calibre-uuid') - self.oeb.uid = self.oeb.metadata.identifier[0] + self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id', + scheme='uuid') + self.oeb.uid = self.oeb.metadata.identifier[-1] def _manifest_prune_invalid(self): ''' diff --git a/src/calibre/ebooks/oeb/transforms/metadata.py b/src/calibre/ebooks/oeb/transforms/metadata.py index 11509a1edd..bb621c9412 100644 --- a/src/calibre/ebooks/oeb/transforms/metadata.py +++ b/src/calibre/ebooks/oeb/transforms/metadata.py @@ -80,12 +80,19 @@ class MergeMetadata(object): def __call__(self, oeb, mi, opts): self.oeb, self.log = oeb, oeb.log m = self.oeb.metadata - meta_info_to_oeb_metadata(mi, m, oeb.log) self.log('Merging user specified metadata...') + meta_info_to_oeb_metadata(mi, m, oeb.log) cover_id = self.set_cover(mi, opts.prefer_metadata_cover) m.clear('cover') if cover_id is not None: m.add('cover', cover_id) + if mi.uuid is not None: + m.filter('identifier', lambda x:x.id=='uuid_id') + self.oeb.metadata.add('identifier', mi.uuid, id='uuid_id', + scheme='uuid') + self.oeb.uid = self.oeb.metadata.identifier[-1] + + def set_cover(self, mi, prefer_metadata_cover): diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 1890c54223..f3fddcd637 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -18,7 +18,9 @@ from calibre.library.database2 import LibraryDatabase2 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF from calibre.utils.genshi.template import MarkupTemplate -FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover']) +FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', + 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', + 'formats', 'isbn', 'uuid', 'cover']) XML_TEMPLATE = '''\ @@ -26,6 +28,7 @@ XML_TEMPLATE = '''\ ${record['id']} + ${record['uuid']} ${record['title']} @@ -71,7 +74,7 @@ STANZA_TEMPLATE='''\ ${record['title']} - urn:calibre:${record['id']} + urn:calibre:${record['uuid']} ${record['author_sort']} ${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')} @@ -227,7 +230,7 @@ def command_list(args, dbpath): if not set(fields).issubset(FIELDS): parser.print_help() print - print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(FIELDS) + print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(sorted(FIELDS)) return 1 db = get_db(dbpath, opts) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 6cdcd631bc..cec88b6d5c 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -59,7 +59,7 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5, 'size':6, 'tags':7, 'comments':8, 'series':9, 'series_index':10, 'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15, - 'lccn':16, 'pubdate':17, 'flags':18, 'cover':19} + 'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20} INDEX_MAP = dict(zip(FIELD_MAP.values(), FIELD_MAP.keys())) @@ -447,7 +447,7 @@ class LibraryDatabase2(LibraryDatabase): for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', 'publisher', 'rating', 'series', 'series_index', 'tags', - 'title', 'timestamp'): + 'title', 'timestamp', 'uuid'): setattr(self, prop, functools.partial(get_property, loc=FIELD_MAP['comments' if prop == 'comment' else prop])) @@ -622,6 +622,50 @@ class LibraryDatabase2(LibraryDatabase): END TRANSACTION; ''') + def upgrade_version_7(self): + 'Add uuid column' + self.conn.executescript(''' + BEGIN TRANSACTION; + ALTER TABLE books ADD COLUMN uuid TEXT; + DROP TRIGGER IF EXISTS books_insert_trg; + DROP TRIGGER IF EXISTS books_update_trg; + UPDATE books SET uuid=uuid4(); + + CREATE TRIGGER books_insert_trg AFTER INSERT ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id; + END; + + CREATE TRIGGER books_update_trg AFTER UPDATE ON books + BEGIN + UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id; + END; + + DROP VIEW meta; + CREATE VIEW meta AS + SELECT id, title, + (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors, + (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher, + (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating, + timestamp, + (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size, + (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags, + (SELECT text FROM comments WHERE book=books.id) comments, + (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series, + series_index, + sort, + author_sort, + (SELECT concat(format) FROM data WHERE data.book=books.id) formats, + isbn, + path, + lccn, + pubdate, + flags, + uuid + FROM books; + + END TRANSACTION; + ''') def last_modified(self): @@ -785,6 +829,7 @@ class LibraryDatabase2(LibraryDatabase): mi.publisher = self.publisher(idx, index_is_id=index_is_id) mi.timestamp = self.timestamp(idx, index_is_id=index_is_id) mi.pubdate = self.pubdate(idx, index_is_id=index_is_id) + mi.uuid = self.uuid(idx, index_is_id=index_is_id) tags = self.tags(idx, index_is_id=index_is_id) if tags: mi.tags = [i.strip() for i in tags.split(',')] @@ -1530,7 +1575,9 @@ class LibraryDatabase2(LibraryDatabase): ''' if prefix is None: prefix = self.library_path - FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn']) + FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', + 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', + 'isbn', 'uuid']) data = [] for record in self.data: if record is None: continue diff --git a/src/calibre/library/server.py b/src/calibre/library/server.py index 015df2b017..fc855f35ce 100644 --- a/src/calibre/library/server.py +++ b/src/calibre/library/server.py @@ -242,7 +242,7 @@ class LibraryServer(object): STANZA_ENTRY=MarkupTemplate(textwrap.dedent('''\ ${record[FM['title']]} - urn:calibre:${record[FM['id']]} + urn:calibre:${urn} ${authors} ${timestamp} @@ -678,6 +678,7 @@ class LibraryServer(object): extra='\n'.join(extra), mimetype=mimetype, fmt=fmt, + urn=record[FIELD_MAP['uuid']], timestamp=strftime('%Y-%m-%dT%H:%M:%S+00:00', record[5]) ) books.append(self.STANZA_ENTRY.generate(**data)\ diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 3c23dd67cc..b498ae9f3a 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' Wrapper for multi-threaded access to a single sqlite database connection. Serializes all calls. ''' -import sqlite3 as sqlite, traceback, time +import sqlite3 as sqlite, traceback, time, uuid from sqlite3 import IntegrityError from threading import Thread from Queue import Queue @@ -121,6 +121,7 @@ class DBThread(Thread): self.conn.create_aggregate('concat', 1, Concatenate) self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) self.conn.create_function('title_sort', 1, title_sort) + self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) def run(self): try: From ef6e9a9abc6434bd970c956b46aee9db998c88eb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 29 Oct 2009 11:52:06 -0600 Subject: [PATCH 13/13] calibre-debug: -e switch now allows you to pass arbitrary command line options to your script --- src/calibre/debug.py | 10 ++++++++++ src/calibre/ebooks/__init__.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 224186878d..39c323825f 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -157,6 +157,16 @@ def add_simple_plugin(path_to_plugin): def main(args=sys.argv): from calibre.constants import debug debug() + if len(args) > 2 and args[1] in ('-e', '--exec-file'): + sys.argv = [args[2]] + args[3:] + ef = os.path.abspath(args[2]) + base = os.path.dirname(ef) + sys.path.insert(0, base) + g = globals() + g['__name__'] = '__main__' + execfile(ef, g) + return + opts, args = option_parser().parse_args(args) if opts.gui: from calibre.gui2.main import main diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index 15f172815c..219eac1dca 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -102,7 +102,6 @@ def render_html(path_to_html, width=590, height=750): page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff) loop = QEventLoop() renderer = HTMLRenderer(page, loop) - page.connect(page, SIGNAL('loadFinished(bool)'), renderer, Qt.QueuedConnection) page.mainFrame().load(QUrl.fromLocalFile(path_to_html))