Sync to trunk.

resources/recipes/heise.recipe

@@ -13,8 +13,6 @@ class heiseDe(BasicNewsRecipe):
     title = 'heise'
     description = 'Computernews from Germany'
     __author__ = 'Oliver Niesner'
-    language = 'de'
-
     use_embedded_content   = False
     timefmt = ' [%d %b %Y]'
     max_articles_per_feed = 40
@@ -35,12 +33,10 @@ class heiseDe(BasicNewsRecipe):
 		   dict(name='div', attrs={'class':'bcadv ISI_IGNORE'}),
 		   dict(name='p', attrs={'class':'news_option'}),
 		   dict(name='p', attrs={'class':'news_navi'}),
-		   dict(name='p', attrs={'class':'news_foren'})]
+		   dict(name='div', attrs={'class':'news_foren'})]
-    remove_tags_after = [dict(name='p', attrs={'class':'news_foren'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'news_foren'})]
     
     feeds =  [ ('heise', 'http://www.heise.de/newsticker/heise.rdf') ] 
     
 
 
-
-

resources/recipes/new_scientist.recipe

@@ -1,11 +1,11 @@
-#!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2009, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newscientist.com
 '''
 
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
@@ -15,12 +15,14 @@ class NewScientist(BasicNewsRecipe):
     language              = 'en'
     publisher             = 'New Scientist'
     category              = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
-    delay                 = 3
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    cover_url             = 'http://www.newscientist.com/currentcover.jpg'
+    masthead_url          = 'http://www.newscientist.com/img/misc/ns_logo.jpg'
     encoding              = 'utf-8'
+    extra_css             = ' body{font-family: Arial,sans-serif} img{margin-bottom: 0.8em} '
 
     conversion_options = {
                           'comment'          : description
@@ -28,14 +30,18 @@ class NewScientist(BasicNewsRecipe):
                         , 'publisher'        : publisher
                         , 'language'         : language
                         }
+    preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol']})]
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','nsblgposts','hldgalcols']})]
 
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
                     ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
+                    ,dict(name='meta' , attrs={'name' :'description'                       })
                   ]
+    remove_tags_after = dict(attrs={'class':'nbpcopy'})
+    remove_attributes = ['height','width']
 
     feeds          = [
                         (u'Latest Headlines'        , u'http://feeds.newscientist.com/science-news'              )
@@ -50,9 +56,15 @@ class NewScientist(BasicNewsRecipe):
                      ]
 
     def get_article_url(self, article):
-        url = article.get('guid',  None)
+        return article.get('guid',  None)
-        return url
 
     def print_version(self, url):
         return url + '?full=true&print=true'
 
+    def preprocess_html(self, soup):
+        for tg in soup.findAll('a'):
+            if tg.string == 'Home':
+                tg.parent.extract()
+                return self.adeify_images(soup)
+        return self.adeify_images(soup)
+

src/calibre/customize/builtins.py

@@ -407,7 +407,7 @@ from calibre.devices.blackberry.driver import BLACKBERRY
 from calibre.devices.cybook.driver import CYBOOK
 from calibre.devices.eb600.driver import EB600, COOL_ER, SHINEBOOK, \
                 POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, \
-                BOOQ
+                BOOQ, ELONEX
 from calibre.devices.iliad.driver import ILIAD
 from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800
 from calibre.devices.jetbook.driver import JETBOOK
@@ -496,6 +496,7 @@ plugins += [
     README,
     N516,
     EB511,
+    ELONEX
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/customize/profiles.py

@@ -235,7 +235,7 @@ class SonyReaderOutput(OutputProfile):
     description = _('This profile is intended for the SONY PRS line. '
                     'The 500/505/600/700 etc.')
 
-    screen_size               = (600, 775)
+    screen_size               = (590, 775)
     dpi                       = 168.451
     fbase                     = 12
     fsizes                    = [7.5, 9, 10, 12, 15.5, 20, 22, 24]

src/calibre/devices/eb600/driver.py

@@ -195,3 +195,15 @@ class BOOQ(EB600):
     WINDOWS_MAIN_MEM = 'EB600'
     WINDOWS_CARD_A_MEM = 'EB600'
 
+class ELONEX(EB600):
+
+    name = 'Elonex 600EB'
+    gui_name = 'Elonex'
+
+    FORMATS = ['epub', 'pdf', 'txt', 'html']
+
+    VENDOR_NAME = 'ELONEX'
+    WINDOWS_MAIN_MEM = 'EBOOK'
+    WINDOWS_CARD_A_MEM = 'EBOOK'
+
+

src/calibre/ebooks/chm/input.py

@@ -4,11 +4,11 @@ __license__ = 'GPL v3'
 __copyright__  = '2008, Kovid Goyal <kovid at kovidgoyal.net>,' \
                  ' and Alex Bramley <a.bramley at gmail.com>.'
 
-import os, shutil, uuid
+import os, shutil, uuid, re
 from tempfile import mkdtemp
 from mimetypes import guess_type as guess_mimetype
 
-from BeautifulSoup import BeautifulSoup
+from BeautifulSoup import BeautifulSoup, NavigableString
 from lxml import html
 from pychm.chm import CHMFile
 from pychm.chmlib import (
@@ -29,6 +29,17 @@ def match_string(s1, s2_already_lowered):
             return True
     return False
 
+def check_all_prev_empty(tag):
+    if tag is None:
+        return True
+    if tag.__class__ == NavigableString and not check_empty(tag):
+        return False
+    return check_all_prev_empty(tag.previousSibling)
+
+def check_empty(s, rex = re.compile(r'\S')):
+    return rex.search(s) is None
+
+
 def option_parser():
     parser = OptionParser(usage=_('%prog [options] mybook.chm'))
     parser.add_option('--output-dir', '-d', default='.', help=_('Output directory. Defaults to current directory'), dest='output')
@@ -155,6 +166,12 @@ class CHMReader(CHMFile):
         # for some very odd reason each page's content appears to be in a table
         # too. and this table has sub-tables for random asides... grr.
 
+        # remove br at top of page if present after nav bars removed
+        br = soup('br')
+        if br:
+            if check_all_prev_empty(br[0].previousSibling):
+                br[0].extract()
+
         # some images seem to be broken in some chm's :/
         for img in soup('img'):
             try:

src/calibre/ebooks/metadata/mobi.py

@@ -311,9 +311,10 @@ class MetadataUpdater(object):
         return StreamSlicer(self.stream, start, stop)
 
     def update(self, mi):
-        def pop_exth_record(exth_id):
+        def update_exth_record(rec):
-            if exth_id in self.original_exth_records:
+            recs.append(rec)
-                self.original_exth_records.pop(exth_id)
+            if rec[0] in self.original_exth_records:
+                self.original_exth_records.pop(rec[0])
 
         if self.type != "BOOKMOBI":
                 raise MobiError("Setting metadata only supported for MOBI files of type 'BOOK'.\n"
@@ -328,47 +329,36 @@ class MetadataUpdater(object):
             pas = False
         if mi.author_sort and pas:
             authors = mi.author_sort
-            recs.append((100, authors.encode(self.codec, 'replace')))
+            update_exth_record((100, authors.encode(self.codec, 'replace')))
-            pop_exth_record(100)
         elif mi.authors:
             authors = '; '.join(mi.authors)
-            recs.append((100, authors.encode(self.codec, 'replace')))
+            update_exth_record((100, authors.encode(self.codec, 'replace')))
-            pop_exth_record(100)
         if mi.publisher:
-            recs.append((101, mi.publisher.encode(self.codec, 'replace')))
+            update_exth_record((101, mi.publisher.encode(self.codec, 'replace')))
-            pop_exth_record(101)
         if mi.comments:
-            recs.append((103, mi.comments.encode(self.codec, 'replace')))
+            update_exth_record((103, mi.comments.encode(self.codec, 'replace')))
-            pop_exth_record(103)
         if mi.isbn:
-            recs.append((104, mi.isbn.encode(self.codec, 'replace')))
+            update_exth_record((104, mi.isbn.encode(self.codec, 'replace')))
-            pop_exth_record(104)
         if mi.tags:
             subjects = '; '.join(mi.tags)
-            recs.append((105, subjects.encode(self.codec, 'replace')))
+            update_exth_record((105, subjects.encode(self.codec, 'replace')))
-            pop_exth_record(105)
         if mi.pubdate:
-            recs.append((106, str(mi.pubdate).encode(self.codec, 'replace')))
+            update_exth_record((106, str(mi.pubdate).encode(self.codec, 'replace')))
-            pop_exth_record(106)
         elif mi.timestamp:
-            recs.append((106, str(mi.timestamp).encode(self.codec, 'replace')))
+            update_exth_record((106, str(mi.timestamp).encode(self.codec, 'replace')))
-            pop_exth_record(106)
         elif self.timestamp:
-            recs.append((106, self.timestamp))
+            update_exth_record((106, self.timestamp))
-            pop_exth_record(106)
         else:
-            recs.append((106, nowf().isoformat().encode(self.codec, 'replace')))
+            update_exth_record((106, nowf().isoformat().encode(self.codec, 'replace')))
-            pop_exth_record(106)
         if self.cover_record is not None:
-            recs.append((201, pack('>I', self.cover_rindex)))
+            update_exth_record((201, pack('>I', self.cover_rindex)))
-            recs.append((203, pack('>I', 0)))
+            update_exth_record((203, pack('>I', 0)))
-            pop_exth_record(201)
-            pop_exth_record(203)
         if self.thumbnail_record is not None:
-            recs.append((202, pack('>I', self.thumbnail_rindex)))
+            update_exth_record((202, pack('>I', self.thumbnail_rindex)))
-            pop_exth_record(202)
+        if 503 in self.original_exth_records:
+            update_exth_record((503, mi.title.encode(self.codec, 'replace')))
 
-        # Restore any original EXTH fields that weren't updated
+        # Include remaining original EXTH fields
         for id in sorted(self.original_exth_records):
             recs.append((id, self.original_exth_records[id]))
         recs = sorted(recs, key=lambda x:(x[0],x[0]))

src/calibre/ebooks/mobi/reader.py

@@ -10,6 +10,7 @@ import re
 import struct
 import textwrap
 import cStringIO
+import sys
 
 try:
     from PIL import Image as PILImage
@@ -806,6 +807,13 @@ def get_metadata(stream):
             if mh.exth.mi is not None:
                 mi = mh.exth.mi
         else:
+            size = sys.maxint
+            if hasattr(stream, 'seek') and hasattr(stream, 'tell'):
+                pos = stream.tell()
+                stream.seek(0, 2)
+                size = stream.tell()
+                stream.seek(pos)
+            if size < 4*1024*1024:
                 with TemporaryDirectory('_mobi_meta_reader') as tdir:
                     with CurrentDir(tdir):
                         mr = MobiReader(stream, log)

src/calibre/gui2/catalog/catalog_epub_mobi.py

@@ -64,5 +64,4 @@ class PluginWidget(QWidget,Ui_Form):
                 opts_dict[opt[0]] = opt_value.split(',')
         opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']]
 
-
         return opts_dict

src/calibre/gui2/dialogs/catalog.py

@@ -8,7 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 import os, sys
 
-from PyQt4.Qt import QDialog
+from PyQt4 import QtGui
+from PyQt4.Qt import QDialog, SIGNAL
 
 from calibre.customize.ui import config
 from calibre.gui2.dialogs.catalog_ui import Ui_Dialog
@@ -119,9 +120,11 @@ class Catalog(QDialog, Ui_Dialog):
             self.sync.setChecked(dynamic.get('catalog_sync_to_device', True))
 
         self.format.currentIndexChanged.connect(self.show_plugin_tab)
+        self.connect(self.buttonBox.button(QtGui.QDialogButtonBox.Apply),
+                     SIGNAL("clicked()"),
+                     self.apply)
         self.show_plugin_tab(None)
 
-
     def show_plugin_tab(self, idx):
         cf = unicode(self.format.currentText()).lower()
         while self.tabs.count() > 1:
@@ -147,11 +150,20 @@ class Catalog(QDialog, Ui_Dialog):
             ans = w.options()
         return ans
 
-    def accept(self):
+    def save_catalog_settings(self):
         self.catalog_format = unicode(self.format.currentText())
         dynamic.set('catalog_preferred_format', self.catalog_format)
         self.catalog_title = unicode(self.title.text())
         dynamic.set('catalog_last_used_title', self.catalog_title)
         self.catalog_sync = bool(self.sync.isChecked())
         dynamic.set('catalog_sync_to_device', self.catalog_sync)
-        QDialog.accept(self)
+
+    def apply(self):
+        # Store current values without building catalog
+        self.save_catalog_settings()
+        if self.tabs.count() > 1:
+            self.tabs.widget(1).options()
+
+    def accept(self):
+        self.save_catalog_settings()
+        return QDialog.accept(self)

src/calibre/gui2/dialogs/catalog.ui

@@ -17,31 +17,22 @@
    <iconset>
     <normaloff>:/images/library.png</normaloff>:/images/library.png</iconset>
   </property>
-  <widget class="QDialogButtonBox" name="buttonBox">
+  <layout class="QGridLayout" name="gridLayout">
-   <property name="geometry">
+   <item row="0" column="0">
-    <rect>
+    <widget class="QLabel" name="count">
-     <x>430</x>
+     <property name="font">
-     <y>470</y>
+      <font>
-     <width>164</width>
+       <weight>75</weight>
-     <height>32</height>
+       <bold>true</bold>
-    </rect>
+      </font>
      </property>
-   <property name="orientation">
+     <property name="text">
-    <enum>Qt::Horizontal</enum>
+      <string>Generate catalog for {0} books</string>
-   </property>
-   <property name="standardButtons">
-    <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
      </property>
     </widget>
+   </item>
+   <item row="1" column="0" colspan="2">
     <widget class="QTabWidget" name="tabs">
-   <property name="geometry">
-    <rect>
-     <x>12</x>
-     <y>39</y>
-     <width>579</width>
-     <height>411</height>
-    </rect>
-   </property>
      <property name="currentIndex">
       <number>0</number>
      </property>
@@ -102,25 +93,18 @@
       </layout>
      </widget>
     </widget>
-  <widget class="QLabel" name="count">
+   </item>
-   <property name="geometry">
+   <item row="2" column="1">
-    <rect>
+    <widget class="QDialogButtonBox" name="buttonBox">
-     <x>12</x>
+     <property name="orientation">
-     <y>12</y>
+      <enum>Qt::Horizontal</enum>
-     <width>301</width>
-     <height>17</height>
-    </rect>
      </property>
-   <property name="font">
+     <property name="standardButtons">
-    <font>
+      <set>QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     <weight>75</weight>
-     <bold>true</bold>
-    </font>
-   </property>
-   <property name="text">
-    <string>Generate catalog for {0} books</string>
      </property>
     </widget>
+   </item>
+  </layout>
  </widget>
  <resources>
   <include location="../../../work/calibre/resources/images.qrc"/>

src/calibre/gui2/tools.py

@@ -250,7 +250,8 @@ def generate_catalog(parent, dbspec, ids, device):
 
     # Profile the connected device
     # Parallel initialization in calibre.library.cli:command_catalog()
-    connected_device = { 'storage':None,'serial':None,'name':None}
+    connected_device = { 'storage':None,'serial':None,'save_template':None,'name':None}
+
     if device:
         try:
             storage = []
@@ -263,6 +264,7 @@ def generate_catalog(parent, dbspec, ids, device):
             connected_device = {      'storage': storage,
                                        'serial': device.detected_device.serial if \
                                                  hasattr(device.detected_device,'serial') else None,
+                                'save_template': device.save_template(),
                                          'name': device.gui_name}
         except:
             pass

src/calibre/library/catalog.py

@@ -1226,7 +1226,7 @@ class EPUB_MOBI(CatalogPlugin):
                         emTag = Tag(soup, "em")
                         emTag.insert(0, aTag)
                         if ttc < len(title['tags']):
-                            emTag.insert(1, NavigableString(', '))
+                            emTag.insert(1, NavigableString(' &middot; '))
                         tagsTag.insert(ttc, emTag)
                         ttc += 1
 
@@ -3448,6 +3448,10 @@ class EPUB_MOBI(CatalogPlugin):
 
         build_log = []
 
+        build_log.append(u"%s(): Generating %s %sin %s environment" %
+            (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '',
+             'CLI' if opts.cli_environment else 'GUI'))
+
         # If exclude_genre is blank, assume user wants all genre tags included
         if opts.exclude_genre.strip() == '':
             opts.exclude_genre = '\[^.\]'
@@ -3455,22 +3459,21 @@ class EPUB_MOBI(CatalogPlugin):
 
         if opts.connected_device['name']:
             if opts.connected_device['serial']:
-                build_log.append(" connected_device: '%s' #%s%s " % \
+                build_log.append(u" connected_device: '%s' #%s%s " % \
                     (opts.connected_device['name'],
                      opts.connected_device['serial'][0:4],
                      'x' * (len(opts.connected_device['serial']) - 4)))
+                build_log.append(u" save_template: '%s'" % opts.connected_device['save_template'])
             else:
-                build_log.append(" connected_device: '%s'" % opts.connected_device['name'])
+                build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
                 for storage in opts.connected_device['storage']:
                     if storage:
-                        build_log.append("  mount point: %s" % storage)
+                        build_log.append(u"  mount point: %s" % storage)
+                build_log.append(u"  save_template: '%s'" % opts.connected_device['save_template'])
 
         opts_dict = vars(opts)
-        build_log.append(u"%s(): Generating %s %sin %s environment" %
-            (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '',
-             'CLI' if opts.cli_environment else 'GUI'))
         if opts_dict['ids']:
-            build_log.append(" Book count: %d" % len(opts_dict['ids']))
+            build_log.append(" book count: %d" % len(opts_dict['ids']))
 
         sections_list = ['Descriptions','Authors']
         if opts.generate_titles:
@@ -3479,7 +3482,7 @@ class EPUB_MOBI(CatalogPlugin):
             sections_list.append('Recently Added')
         if not opts.exclude_genre.strip() == '.':
             sections_list.append('Genres')
-        build_log.append(u"Creating Sections for %s" % ', '.join(sections_list))
+        build_log.append(u" Sections: %s" % ', '.join(sections_list))
 
         # Display opts
         keys = opts_dict.keys()

src/calibre/library/cli.py

@@ -683,7 +683,7 @@ def command_catalog(args, dbpath):
 
     # No support for connected device in CLI environment
     # Parallel initialization in calibre.gui2.tools:generate_catalog()
-    opts.connected_device = {'storage':None,'serial':None,'name':None}
+    opts.connected_device = { 'storage':None,'serial':None,'save_template':None,'name':None}
 
     with plugin:
         plugin.run(args[1], opts, get_db(dbpath, opts))

src/calibre/manual/faq.rst

@@ -180,7 +180,7 @@ Why is my device not detected in linux?
     
     grep SYSFS_DEPRECATED /boot/config-`uname -r`
 
-You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``. 
+You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``. If you don't you have to either recompile your kernel with the correct setting, or upgrade your linux distro to a more modern version, where this will not be set.
 
 Library Management
 ------------------
@@ -305,7 +305,7 @@ Your antivirus program is wrong. |app| is a completely open source product. You
 
 How do I use purchased EPUB books with |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. This prevents |app| from opening them. You can still use |app| to store and transfer them to your SONY Reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with |app| will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to |app|.
+Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. This prevents |app| from opening them. You can still use |app| to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with |app| will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to |app|.
 
 
 I want some feature added to |app|. What can I do?

src/calibre/manual/gui.rst

@@ -134,7 +134,7 @@ The :guilabel:`Convert E-books` action has three variations, accessed by the arr
                 
     2. **Bulk convert**: This allows you to specify options only once to convert a number of ebooks in bulk.
     
-    3. **Create catalog**: This action allow yous to generate a complete listing with all metadata of the books in your library, in several formats, like XML, CSV, EPUB and MOBI. The catalog will contain all the books showing in the library view currently, so you can use the search features to limit the books to be catalogued. In addition, if you select multiple books using the mouse, only those books will be added to the catalog. If you generate the catalog in an e-book format such as EPUB or MOBI, the next time you connect your e-book reader, the catalog will be automatically sent to the device.
+    3. **Create catalog**: This action allows you to generate a complete listing with all metadata of the books in your library, in several formats, like XML, CSV, EPUB and MOBI. The catalog will contain all the books showing in the library view currently, so you can use the search features to limit the books to be catalogued. In addition, if you select multiple books using the mouse, only those books will be added to the catalog. If you generate the catalog in an e-book format such as EPUB or MOBI, the next time you connect your e-book reader, the catalog will be automatically sent to the device. For details on how catalogs work, see `here <http://www.mobileread.com/forums/showthread.php?p=755468#post755468>`_.
                                             
 .. _view:
 
@@ -156,7 +156,7 @@ A *catalog* is a collection of books. |app| can manage three different catalogs:
                 
     2. **Reader**:  This is a collection of books stored in the main memory of your ebook reader. It will be available when you connect the reader to your computer.
                 
-    3. **Card**: This is a collection of books stored on the storage card in your reader. Note that ${app} supports only a single storage card at a time.
+    3. **Card**: This is a collection of books stored on the storage card in your reader.
 
 .. _search_sort:
 

todo

@@ -1,11 +0,0 @@
-
-* Refactor web.fetch.simple to use per connection timeouts via the timeout kwarg for mechanize.open
-
-* Testing framework
-
-
-* Add a languages column to books. Best implementation is comma separated IANA codes
-* Add a hash column to the formats table (used for stanza identifier)
-
-* Fix blockquote handling in sphinx templates
-* Fix ebook-viewer going to links