sync to trunk.

Changelog.yaml

@@ -5,7 +5,7 @@
 # Also, each release can have new and improved recipes.
 
 # - version: ?.?.?
-#   date: 2011-??-??
+#   date: 2012-??-??
 #
 #   new features:
 #     - title: 
@@ -19,8 +19,68 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.36
+  date: 2012-01-20
+
+  new features:
+    - title: "Decrease startup time for large libraries with at least one composite custom column by reading format info on demand"
+
+    - title: "When automatically deleting news older than x days, from the calibre library, only delete the book if it both has the tag News and the author calibre. This prevents accidental deletion of books tagged with News by the user."
+
+    - title: "Driver for Infibeam Pi 2"
+
+    - title: "Add a Tag Editor for tags like custom columns to the edit metadata dialog"
+ 
+  bug fixes:
+    - title: "E-book viewer: Fix regression in 0.8.35 that caused viewer to raise an error on books that did not define a language"
+
+    - title: "Content server: Fix grouping for categories based on custom columns."
+      tickets: [919011]
+
+    - title: "Edit metadata dialog: When setting the series from a format or via metadata download, ensure that the series index is not automatically changed, when closing the dialog."
+      tickets: [918751]
+
+    - title: "When reading metadata from Topaz (azw1) files, handle non ascii metadata correctly."
+      tickets: [917419]
+
+    - title: "CHM Input: Do not choke on CHM files with non ascii internal filenames on windows."
+      tickets: [917696]
+
+    - title: "Fix reading metadata from CHM files with non-ascii titles"
+
+    - title: "Fix HTML 5 parser choking on comments"
+
+    - title: "If calibre is started from a directory that does not exist, automatically use the home directory as the working directory, instead of crashing"
+
+    - title: "Fix iriver story HD Wi-Fi device and external SD card swapped"
+      tickets: [916364] 
+
+    - title: "Content server: Fix ugly URLs for specific format download in the book details and permalink panels"
+
+    - title: "When adding FB2 files do not set the date field from the metadata in the file"
+
+  improved recipes:
+    - OReilly Premuim
+    - Variety
+    - Blic
+    - New Journal of Physics
+    - Der Tagesspiegel
+
+  new recipes:
+    - title: Tweakers.net 
+      author: Roedi06
+
+    - title: Village Voice 
+      author: Barty
+
+    - title: Edge.org Conversations 
+      author: levien
+
+    - title: Novi list - printed edition
+      author: Darko Miletic
+
 - version: 0.8.35
-  date: 2011-01-13
+  date: 2012-01-13
 
   new features:
     - title: "Metadata plugboards: Allow creation of plugboards for email delivery."

recipes/al_masry_al_youm.recipe

@@ -0,0 +1,50 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Pat Stapleton <pat.stapleton at gmail.com>'
+'''
+abc.net.au/news
+'''
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class TheDailyNewsEG(BasicNewsRecipe):
+    title          	   = u'al-masry al-youm'
+    __author__             = 'Omm Mishmishah'
+    description            = 'Independent News from Egypt'
+    masthead_url           = 'http://www.almasryalyoum.com/sites/default/files/img/english_logo.png'
+    cover_url              = 'http://www.almasryalyoum.com/sites/default/files/img/english_logo.png'
+
+    auto_cleanup           = True
+    oldest_article         = 7
+    max_articles_per_feed  = 100
+    no_stylesheets         = False
+    #delay                  = 1
+    use_embedded_content   = False
+    encoding               = 'utf8'
+    publisher              = 'Independent News Egypt'
+    category               = 'News, Egypt, World'
+    language               = 'en_EG'
+    publication_type       = 'newsportal'
+#    preprocess_regexps     = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
+#Remove annoying map links (inline-caption class is also used for some image captions! hence regex to match maps.google)
+    preprocess_regexps     = [(re.compile(r'<a class="inline-caption" href="http://maps\.google\.com.*?/a>', re.DOTALL), lambda m: '')]
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                            ,'linearize_tables': False
+                         }
+
+    keep_only_tags = [dict(attrs={'class':['article section']})]
+
+    remove_tags = [dict(attrs={'class':['related', 'tags', 'tools', 'attached-content ready',
+        'inline-content story left', 'inline-content map left contracted', 'published',
+        'story-map', 'statepromo', 'topics', ]})]
+
+    remove_attributes = ['width','height']
+
+    feeds          = [(u'English News', u'http://www.almasryalyoum.com/en/rss_feed_term/113/rss.xml'),
+                      (u'News Features', u'http://www.almasryalyoum.com/en/rss_feed_term/115/rss.xml'),
+                      (u'Culture', u'http://www.almasryalyoum.com/en/rss_feed_term/133/rss.xml'),
+                      (u'Cinema', u'http://www.almasryalyoum.com/en/rss_feed_term/134/rss.xml')
+                     ]

recipes/klip_me.recipe

@@ -0,0 +1,72 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1299694372(BasicNewsRecipe):
+    title                             = u'Klipme'
+    __author__                  = 'Ken Sun'
+    publisher                     = 'Klip.me'
+    category                      = 'info, custom, Klip.me'
+    oldest_article               = 365
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    remove_tags              = [
+    dict(name='div', attrs={'id':'text_controls_toggle'})
+    ,dict(name='script')
+    ,dict(name='div', attrs={'id':'text_controls'})
+    ,dict(name='div', attrs={'id':'editing_controls'})
+    ,dict(name='div', attrs={'class':'bar bottom'})
+     ]
+    use_embedded_content  = False
+    needs_subscription    = True
+    INDEX                 = u'http://www.klip.me'
+    LOGIN                 = INDEX + u'/fav/signin?callback=/fav'
+
+
+    feeds          = [
+            (u'Klip.me unread', u'http://www.klip.me/fav'),
+            (u'Klip.me started', u'http://www.klip.me/fav?s=starred')
+            ]
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None:
+            br.open(self.LOGIN)
+            br.select_form(nr=0)
+            br['Email'] = self.username
+            if self.password is not None:
+               br['Passwd'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, 'Fetching feed'+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for item in soup.findAll('table',attrs={'class':['item','item new']}):
+                atag = item.a
+                if atag and atag.has_key('href'):
+                    url         = atag['href']
+                    articles.append({
+                                     'url'        :url
+                                    })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds
+
+    def print_version(self, url):
+        return 'http://www.klip.me' + url
+
+    def populate_article_metadata(self, article, soup, first):
+        article.title  = soup.find('title').contents[0].strip()
+
+    def postprocess_html(self, soup, first_fetch):
+        for link_tag in soup.findAll(attrs={"id" : "story"}):
+            link_tag.insert(0,'<h1>'+soup.find('title').contents[0].strip()+'</h1>')
+            print link_tag
+
+        return soup
+

recipes/new_scientist.recipe

@@ -1,16 +1,35 @@
-__license__   = 'GPL v3'
+##
-__copyright__ = '2008-2010, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
+## Title:        Microwave Journal RSS recipe
+## Contact:      AprilHare, Darko Miletic <darko.miletic at gmail.com>
+##
+## License:      GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
+## Copyright:    2008-2010, AprilHare, Darko Miletic <darko.miletic at gmail.com>
+##
+## Written:      2008
+## Last Edited:  Jan 2012
+##
+
+'''
+01-19-2012: Added GrayScale Image conversion and Duplicant article removals
+'''
+
+__license__   = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
+__copyright__ = '2008-2012, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
+__version__     = 'v0.5.0'
+__date__        = '2012-01-19'
+__author__      = 'Darko Miletic'
+
 '''
 newscientist.com
 '''
 
 import re
 import urllib
+from calibre.utils.magick import Image
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
     title                 = 'New Scientist - Online News w. subscription'
-    __author__            = 'Darko Miletic'
     description           = 'Science news and science articles from New Scientist.'
     language              = 'en'
     publisher             = 'Reed Business Information Ltd.'
@@ -39,10 +58,19 @@ class NewScientist(BasicNewsRecipe):
 
     keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
+    # Whether to omit duplicates of articles (typically arsing when articles are indexed in
+    # more than one section). If True, only the first occurance will be downloaded.
+    filterDuplicates = True
+
+    # Whether to convert images to grayscale for eInk readers.
+    Convert_Grayscale = False
+
+    url_list = []   # This list is used to check if an article had already been included.
+
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
         br.open('http://www.newscientist.com/')
-        if self.username is not None and self.password is not None:        
+        if self.username is not None and self.password is not None:
             br.open('https://www.newscientist.com/user/login')
             data = urllib.urlencode({ 'source':'form'
                                      ,'redirectURL':''
@@ -80,6 +108,10 @@ class NewScientist(BasicNewsRecipe):
         return article.get('guid',  None)
 
     def print_version(self, url):
+        if self.filterDuplicates:
+            if url in self.url_list:
+                return
+        self.url_list.append(url)
         return url + '?full=true&print=true'
 
     def preprocess_html(self, soup):
@@ -91,7 +123,7 @@ class NewScientist(BasicNewsRecipe):
             item.name='p'
         for item in soup.findAll(['xref','figref']):
             tstr = item.string
-            item.replaceWith(tstr)            
+            item.replaceWith(tstr)
         for tg in soup.findAll('a'):
             if tg.string == 'Home':
                 tg.parent.extract()
@@ -101,3 +133,16 @@ class NewScientist(BasicNewsRecipe):
                    tg.replaceWith(tstr)
         return soup
 
+    # Converts images to Gray Scale
+    def postprocess_html(self, soup, first):
+        if self.Convert_Grayscale:
+            #process all the images
+            for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+                iurl = tag['src']
+                img = Image()
+                img.open(iurl)
+                if img < 0:
+                    raise RuntimeError('Out of memory')
+                img.type = "GrayscaleType"
+                img.save(iurl)
+        return soup

recipes/the_daily_news_egypt.recipe

@@ -0,0 +1,46 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Pat Stapleton <pat.stapleton at gmail.com>'
+'''
+abc.net.au/news
+'''
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class TheDailyNewsEG(BasicNewsRecipe):
+    title          	   = u'The Daily News Egypt'
+    __author__             = 'Omm Mishmishah'
+    description            = 'News from Egypt'
+    masthead_url           = 'http://www.thedailynewsegypt.com/images/DailyNews-03_05.gif'
+    cover_url              = 'http://www.thedailynewsegypt.com/images/DailyNews-03_05.gif'
+
+    auto_cleanup           = True
+    oldest_article         = 7
+    max_articles_per_feed  = 100
+    no_stylesheets         = False
+    #delay                  = 1
+    use_embedded_content   = False
+    encoding               = 'utf8'
+    publisher              = 'The Daily News Egypt'
+    category               = 'News, Egypt, World'
+    language               = 'en_EG'
+    publication_type       = 'newsportal'
+#    preprocess_regexps     = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
+#Remove annoying map links (inline-caption class is also used for some image captions! hence regex to match maps.google)
+    preprocess_regexps     = [(re.compile(r'<a class="inline-caption" href="http://maps\.google\.com.*?/a>', re.DOTALL), lambda m: '')]
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                            ,'linearize_tables': False
+                         }
+
+    keep_only_tags = [dict(attrs={'class':['article section']})]
+
+    remove_tags = [dict(attrs={'class':['related', 'tags', 'tools', 'attached-content ready',
+        'inline-content story left', 'inline-content map left contracted', 'published',
+        'story-map', 'statepromo', 'topics', ]})]
+
+    remove_attributes = ['width','height']
+
+    feeds          = [(u'The Daily News Egypt', u'http://www.thedailynewsegypt.com/rss.php?sectionid=all')]

resources/content_server/browse/browse.html

@@ -3,7 +3,7 @@
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
 <head>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
     <title>..:: calibre {library} ::.. {title}</title>
     <meta http-equiv="X-UA-Compatible" content="IE=100" />
     <link rel="icon" type="image/x-icon" href="http://calibre-ebook.com/favicon.ico" />
@@ -58,7 +58,7 @@
                 method="post" title="Donate to support the development of calibre">
                 <div>
                     <input type="hidden" name="cmd" value="_s-xclick"></input>
-                    <input type="hidden" name="hosted_button_id" value="3028915"></input>
+                    <input type="hidden" name="hosted_button_id" value="MZQCP8EESW4H4"></input>
                     <input type="image"
                     src="{prefix}/static/button-donate.png"
                         name="submit"></input>

setup/hosting.py

@@ -26,7 +26,11 @@ def login_to_google(username, password):
     br.form['Email'] = username
     br.form['Passwd'] = password
     raw = br.submit().read()
-    if b'<title>Account overview - Account Settings</title>' not in raw:
+    if re.search(br'<title>.*?Account Settings</title>', raw) is None:
+        x = re.search(br'(?is)<title>.*?</title>', raw)
+        if x is not None:
+            print ('Title of post login page: %s'%x.group())
+        #open('/tmp/goog.html', 'wb').write(raw)
         raise ValueError(('Failed to login to google with credentials: %s %s'
             '\nGoogle sometimes requires verification when logging in from a '
             'new IP address. Use lynx to login and supply the verification, '

setup/iso_639/de.po

@@ -18,14 +18,14 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2012-01-08 20:03+0000\n"
+"PO-Revision-Date: 2012-01-14 02:30+0000\n"
-"Last-Translator: Simeon <Unknown>\n"
+"Last-Translator: Wolfgang Rohdewald <wolfgang@rohdewald.de>\n"
 "Language-Team: German <debian-l10n-german@lists.debian.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2012-01-09 04:49+0000\n"
+"X-Launchpad-Export-Date: 2012-01-15 05:18+0000\n"
-"X-Generator: Launchpad (build 14640)\n"
+"X-Generator: Launchpad (build 14664)\n"
 "Language: de\n"
 
 #. name for aaa

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 35)
+numeric_version = (0, 8, 36)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/devices/android/driver.py

@@ -162,7 +162,7 @@ class ANDROID(USBMS):
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
             'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
-            'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO']
+            'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -175,7 +175,7 @@ class ANDROID(USBMS):
             'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A',
             'ALPANDIGITAL', 'ANDROID_MID', 'VTAB1008', 'EMX51_BBG_ANDROI',
             'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107',
-            'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET']
+            'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

src/calibre/ebooks/metadata/topaz.py

@@ -8,6 +8,7 @@ import StringIO, sys
 from struct import pack
 
 from calibre.ebooks.metadata import MetaInformation
+from calibre import force_unicode
 
 class StreamSlicer(object):
 
@@ -245,7 +246,9 @@ class MetadataUpdater(object):
     def get_metadata(self):
         ''' Return MetaInformation with title, author'''
         self.get_original_metadata()
-        return MetaInformation(self.metadata['Title'], [self.metadata['Authors']])
+        title = force_unicode(self.metadata['Title'], 'utf-8')
+        authors = force_unicode(self.metadata['Authors'], 'utf-8').split(';')
+        return MetaInformation(title, authors)
 
     def get_original_metadata(self):
         offset = self.base + self.topaz_headers['metadata']['blocks'][0]['offset']

src/calibre/ebooks/oeb/display/cfi.coffee

@@ -102,7 +102,7 @@ viewport_to_document = (x, y, doc=window?.document) -> # {{{
     return [x, y]
 # }}}
 
-# Equivalent for caretRangeFromPoint for non WebKit browsers {{{
+# Convert point to character offset {{{
 range_has_point = (range, x, y) ->
     for rect in range.getClientRects()
         if (rect.left <= x <= rect.right) and (rect.top <= y <= rect.bottom)

src/calibre/gui2/metadata/basic_widgets.py

@@ -573,6 +573,9 @@ class SeriesIndexEdit(QDoubleSpinBox):
                 import traceback
                 traceback.print_exc()
 
+    def reset_original(self):
+        self.original_series_name = self.series_edit.current_val
+
     def break_cycles(self):
         try:
             self.series_edit.currentIndexChanged.disconnect()

src/calibre/gui2/metadata/single.py

@@ -376,6 +376,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         if not mi.is_null('series') and mi.series.strip():
             self.series.current_val = mi.series
             if mi.series_index is not None:
+                self.series_index.reset_original()
                 self.series_index.current_val = float(mi.series_index)
         if not mi.is_null('languages'):
             langs = [canonicalize_lang(x) for x in mi.languages]

src/calibre/gui2/preferences/main.py

@@ -325,6 +325,7 @@ class Preferences(QMainWindow):
             return
         rc = self.showing_widget.restart_critical
         self.committed = True
+        do_restart = False
         if must_restart:
             self.must_restart = True
             msg = _('Some of the changes you made require a restart.'
@@ -335,12 +336,24 @@ class Preferences(QMainWindow):
                         'set any more preferences, until you restart.')
 
 
-            warning_dialog(self, _('Restart needed'), msg, show=True,
+            d = warning_dialog(self, _('Restart needed'), msg,
                     show_copy_button=False)
+            b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
+            b.setIcon(QIcon(I('lt.png')))
+            d.do_restart = False
+            def rf():
+                d.do_restart = True
+            b.clicked.connect(rf)
+            d.set_details('')
+            d.exec_()
+            b.clicked.disconnect()
+            do_restart = d.do_restart
         self.showing_widget.refresh_gui(self.gui)
         self.hide_plugin()
-        if self.close_after_initial or (must_restart and rc):
+        if self.close_after_initial or (must_restart and rc) or do_restart:
             self.close()
+        if do_restart:
+            self.gui.quit(restart=True)
 
 
     def cancel(self, *args):

src/calibre/library/server/browse.py

@@ -497,7 +497,8 @@ class BrowseServer(object):
                         xml(s, True),
                         xml(_('Loading, please wait'))+'…',
                         unicode(c),
-                        xml(u'/browse/category_group/%s/%s'%(category,
+                        xml(u'/browse/category_group/%s/%s'%(
+                            hexlify(category.encode('utf-8')),
                             hexlify(s.encode('utf-8'))), True),
                         self.opts.url_prefix)
                     for s, c in category_groups.items()]
@@ -531,6 +532,13 @@ class BrowseServer(object):
             sort = None
         if sort not in ('rating', 'name', 'popularity'):
             sort = 'name'
+        try:
+            category = unhexlify(category)
+            if isbytestring(category):
+                category = category.decode('utf-8')
+        except:
+            raise cherrypy.HTTPError(404, 'invalid category')
+
         categories = self.categories_cache()
         if category not in categories:
             raise cherrypy.HTTPError(404, 'category not found')

src/calibre/utils/localization.py

@@ -121,6 +121,7 @@ _extra_lang_codes = {
         'en_JP' : _('English (Japan)'),
         'en_DE' : _('English (Germany)'),
         'en_BG' : _('English (Bulgaria)'),
+        'en_EG' : _('English (Egypt)'),
         'en_NZ' : _('English (New Zealand)'),
         'en_CA' : _('English (Canada)'),
         'en_GR' : _('English (Greece)'),