Merge from trunk

Changelog.yaml

@@ -4,6 +4,72 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.9
+  date: 2010-07-17
+
+  new features:
+    - title: "New unified toolbar"
+      type: major
+      description: >
+        "A new unified toolbar combines the old toolbar and device display, to save space. Now when a device is connected, buttons 
+        are created in the unified toolbar for the device and its storage cards. Click the arrow next to the button to eject the device."
+
+    - title: "Device drivers: Add option to allow calibre to automatically manage metadata on the device in Preferences->Add/Save->Sending to device"
+
+    - title: "BibTeX output for catalogs. The list of books in calibre can now also be output as a .bib file"
+
+    - title: "A new toolbar button to choose/create different calibre libraries. Be careful using it if you also use custom columns."
+
+    - title: "Support for the MiBuk"
+
+  bug fixes:
+    - title: "MOBI metadata: Replace HTML entities in the title read from the MOBI file"
+
+    - title: "Conversion pipeline: Handle elements with percentage sizes that are children of zero size parents correctly."
+      tickets: [6155]
+
+    - title: "Fix regression that made LRF conversion less robust"
+      tickets: [6180]
+
+    - title: "FB2 Input: Handle embedded images correctly, so that EPUB generated from FB2 works with Adobe Digital Editions."
+      tickets: [6183]
+
+    - title: "Fix regression that prevented old news from being deleted in the calibre library if calibre is never kept running for more than an hour"
+
+    - title: "RTF Input: Fix handling of text align and superscript/subscripts"
+      tickets: [3644,5060]
+
+    - title: "Fix long series or publisher names causing convert dialog to become too wide"
+
+    - title: "SONY driver: Fix handling of invalid XML databases with null bytes"
+      tickets: [6165]
+
+    - title: "iTunes driver: Better series_index sorting"
+
+    - title: "Improved editing of dates for custom columns"
+
+    - title: "Linux USB scanner: Don't fail to start calibre if SYFS is not present. Instead simply fail to detect devices"
+      tickets: [6156]
+
+    - title: "Android driver: Show books on device if Aldiko is being used"
+      tickets: [6100]
+
+    - title: "Upgrade to Qt 4.6.3 in all binary builds to ensure proper rendering of the new toolbar icons"
+
+    - title: "Fix handling of entities in epub files by the epub-fix command"
+      tickets: [6136]
+
+  new recipes:
+    - title: "EL Pain Impresso"
+      author: Darko Miletic
+
+    - title: "MIT Technology Review, Alternet, Waco Tribune Herald and Orlando Sentinel"
+      author: rty
+
+  improved recipes:
+    - Google Reader
+
+
 - version: 0.7.8
   date: 2010-07-09
 

resources/recipes/alternet.recipe

@@ -15,7 +15,7 @@ class Alternet(BasicNewsRecipe):
 	(u'Top Ten Campaigns', u'http://feeds.feedblitz.com/alternet_top_10_campaigns'),
 	(u'Special Coverage Areas', u'http://feeds.feedblitz.com/alternet_coverage')
 	]
-
+    remove_attributes = ['width', 'align','cellspacing']
     remove_javascript = True
     use_embedded_content   = False
     no_stylesheets = True

resources/recipes/technology_review.recipe

@@ -0,0 +1,44 @@
+import string
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TechnologyReview(BasicNewsRecipe):
+    title          = u'Technology Review'
+    __author__ = 'rty'
+    description = 'MIT Technology Magazine'
+    publisher = 'Technology Review Inc.'
+    category = 'Technology, Innovation, R&D'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    No_stylesheets = True
+    extra_css = """
+    .ArticleBody {font: normal; text-align: justify}
+    .headline {font: bold x-large}
+    .subheadline {font: italic large}
+    """
+    feeds = [
+	(u'Computing', u'http://feeds.technologyreview.com/technology_review_Computing'),
+	(u'Web', u'http://feeds.technologyreview.com/technology_review_Web'),
+	(u'Communications', u'http://feeds.technologyreview.com/technology_review_Communications'),
+	(u'Energy', u'http://feeds.technologyreview.com/technology_review_Energy'),
+	(u'Materials', u'http://feeds.technologyreview.com/technology_review_Materials'),
+	(u'Biomedicine', u'http://feeds.technologyreview.com/technology_review_Biotech'),
+	(u'Business', u'http://feeds.technologyreview.com/technology_review_Biztech')
+	]
+    remove_attributes = ['width', 'align','cellspacing']
+
+    remove_tags = [
+                    dict(name='div', attrs={'id':['CloseLink','footerAdDiv','copyright']}),
+                         ]
+    remove_tags_after = [dict(name='div', attrs={'id':'copyright'})]
+
+    def get_article_url(self, article):
+        return article.get('guid', article.get('id', None))
+
+
+    def print_version(self, url):
+        baseurl='http://www.technologyreview.com/printer_friendly_article.aspx?id='
+        split1 = string.split(url,"/")
+        xxx=split1 [4]
+        split2= string.split(xxx,"/")
+        s =  baseurl + split2[0]
+        return s

src/calibre/__init__.py

@@ -440,6 +440,9 @@ xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions = {
     '>' : '>',
     '&' : '&'})
 
+def replace_entities(raw):
+    return _ent_pat.sub(entity_to_unicode, raw)
+
 def prepare_string_for_xml(raw, attribute=False):
     raw = _ent_pat.sub(entity_to_unicode, raw)
     raw = raw.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.8'
+__version__   = '0.7.9'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/ebooks/fb2/input.py

@@ -60,6 +60,9 @@ class FB2Input(InputFormatPlugin):
 
         transform = etree.XSLT(styledoc)
         result = transform(doc)
+        for img in result.xpath('//img[@src]'):
+            src = img.get('src')
+            img.set('src', self.binary_map.get(src, src))
         open('index.xhtml', 'wb').write(transform.tostring(result))
         stream.seek(0)
         mi = get_metadata(stream, 'fb2')
@@ -83,9 +86,15 @@ class FB2Input(InputFormatPlugin):
         return os.path.join(os.getcwd(), 'metadata.opf')
 
     def extract_embedded_content(self, doc):
+        self.binary_map = {}
         for elem in doc.xpath('./*'):
             if 'binary' in elem.tag and elem.attrib.has_key('id'):
+                ct = elem.get('content-type', '')
                 fname = elem.attrib['id']
+                ext = ct.rpartition('/')[-1].lower()
+                if ext in ('png', 'jpeg', 'jpg'):
+                    fname += '.' + ext
+                    self.binary_map[elem.get('id')] = fname
                 data = b64decode(elem.text.strip())
                 open(fname, 'wb').write(data)
 

src/calibre/ebooks/lrf/input.py

@@ -368,7 +368,15 @@ class LRFInput(InputFormatPlugin):
         if options.verbose > 2:
             open('lrs.xml', 'wb').write(xml.encode('utf-8'))
         parser = etree.XMLParser(no_network=True, huge_tree=True)
-        doc = etree.fromstring(xml, parser=parser)
+        try:
+            doc = etree.fromstring(xml, parser=parser)
+        except:
+            self.log.warn('Failed to parse XML. Trying to recover')
+            parser = etree.XMLParser(no_network=True, huge_tree=True,
+                    recover=True)
+            doc = etree.fromstring(xml, parser=parser)
+
+
         char_button_map = {}
         for x in doc.xpath('//CharButton[@refobj]'):
             ro = x.get('refobj')

src/calibre/ebooks/mobi/reader.py

@@ -14,7 +14,8 @@ except ImportError:
 
 from lxml import html, etree
 
-from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode
+from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode, \
+    replace_entities
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import parse_date
 from calibre.ptempfile import TemporaryDirectory
@@ -70,7 +71,7 @@ class EXTHHeader(object):
             #else:
             #    print 'unknown record', id, repr(content)
         if title:
-            self.mi.title = title
+            self.mi.title = replace_entities(title)
 
     def process_metadata(self, id, content, codec):
         if id == 100:

src/calibre/ebooks/oeb/stylizer.py

@@ -475,7 +475,8 @@ class Style(object):
             value = float(m.group(1))
             unit = m.group(2)
             if unit == '%':
-                base = base or self.width
+                if base is None:
+                    base = self.width
                 result = (value / 100.0) * base
             elif unit == 'px':
                 result = value * 72.0 / self._profile.dpi

src/calibre/gui2/convert/structure_detection.ui

@@ -58,7 +58,7 @@
    <item row="9" column="0" colspan="2">
     <widget class="XPathEdit" name="opt_page_breaks_before" native="true"/>
    </item>
-   <item row="10" column="0">
+   <item row="10" column="0" colspan="2">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>

src/calibre/gui2/convert/xexp_edit.ui

@@ -43,12 +43,6 @@
          <height>0</height>
         </size>
        </property>
-       <property name="maximumSize">
-        <size>
-         <width>500</width>
-         <height>16777215</height>
-        </size>
-       </property>
        <property name="minimumContentsLength">
         <number>30</number>
        </property>

src/calibre/gui2/dialogs/choose_library.py

@@ -10,7 +10,7 @@ import os
 from PyQt4.Qt import QDialog
 
 from calibre.gui2.dialogs.choose_library_ui import Ui_Dialog
-from calibre.gui2 import error_dialog, choose_dir
+from calibre.gui2 import error_dialog, choose_dir, warning_dialog
 from calibre.constants import filesystem_encoding
 from calibre import isbytestring, patheq
 from calibre.utils.config import prefs
@@ -24,6 +24,7 @@ class ChooseLibrary(QDialog, Ui_Dialog):
         self.db = db
         self.new_db = None
         self.callback = callback
+        self.location.initialize('choose_library_dialog')
 
         lp = db.library_path
         if isbytestring(lp):
@@ -61,6 +62,12 @@ class ChooseLibrary(QDialog, Ui_Dialog):
         return True
 
     def perform_action(self, ac, loc):
+        if ac in ('new', 'existing'):
+            warning_dialog(self.parent(), _('Custom columns'),
+                    _('If you use custom columns and they differ between '
+                        'libraries, you will have various problems. Best '
+                        'to ensure you have the same custom columns in each '
+                        'library.'), show=True)
         if ac in ('new', 'existing'):
             prefs['library_path'] = loc
             self.callback(loc)
@@ -79,4 +86,5 @@ class ChooseLibrary(QDialog, Ui_Dialog):
                 loc):
             return
         QDialog.accept(self)
+        self.location.save_history()
         self.perform_action(action, loc)

src/calibre/gui2/dialogs/choose_library.ui

@@ -18,7 +18,7 @@
     <normaloff>:/images/lt.png</normaloff>:/images/lt.png</iconset>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0" colspan="3">
+   <item row="0" column="0" colspan="4">
     <widget class="QLabel" name="old_location">
      <property name="text">
       <string>Your calibre library is currently located at {0}</string>
@@ -38,14 +38,7 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="1">
-    <widget class="QLineEdit" name="location">
-     <property name="readOnly">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item row="4" column="0" colspan="3">
+   <item row="4" column="0" colspan="4">
     <widget class="QRadioButton" name="existing_library">
      <property name="text">
       <string>Use &amp;existing library at the new location</string>
@@ -55,21 +48,21 @@
      </property>
     </widget>
    </item>
-   <item row="5" column="0" colspan="2">
+   <item row="5" column="0" colspan="3">
     <widget class="QRadioButton" name="empty_library">
      <property name="text">
       <string>&amp;Create an empty library at the new location</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="0" colspan="2">
+   <item row="6" column="0" colspan="3">
     <widget class="QRadioButton" name="move_library">
      <property name="text">
       <string>&amp;Move current library to new location</string>
      </property>
     </widget>
    </item>
-   <item row="8" column="1">
+   <item row="8" column="2">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -79,7 +72,7 @@
      </property>
     </widget>
    </item>
-   <item row="7" column="1">
+   <item row="7" column="2">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -118,7 +111,7 @@
      </property>
     </spacer>
    </item>
-   <item row="2" column="2">
+   <item row="2" column="3">
     <widget class="QToolButton" name="browse_button">
      <property name="text">
       <string>...</string>
@@ -129,8 +122,18 @@
      </property>
     </widget>
    </item>
+   <item row="2" column="1" colspan="2">
+    <widget class="HistoryLineEdit" name="location"/>
+   </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>HistoryLineEdit</class>
+   <extends>QComboBox</extends>
+   <header>calibre/gui2/widgets.h</header>
+  </customwidget>
+ </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>
  </resources>

src/calibre/gui2/dialogs/jobs.ui

@@ -60,7 +60,7 @@
    <item>
     <widget class="QPushButton" name="stop_all_jobs_button">
      <property name="text">
-      <string>Stop &amp;all jobs</string>
+      <string>Stop &amp;all non device jobs</string>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/scheduler.py

@@ -10,7 +10,7 @@ Scheduler for automated recipe downloads
 from datetime import timedelta
 
 from PyQt4.Qt import QDialog, SIGNAL, Qt, QTime, QObject, QMenu, \
-        QAction, QIcon, QMutex, QTimer
+        QAction, QIcon, QMutex, QTimer, pyqtSignal
 
 from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
 from calibre.gui2.search_box import SearchBox2
@@ -203,6 +203,9 @@ class Scheduler(QObject):
 
     INTERVAL = 1 # minutes
 
+    delete_old_news = pyqtSignal(object)
+    start_recipe_fetch = pyqtSignal(object)
+
     def __init__(self, parent, db):
         QObject.__init__(self, parent)
         self.internet_connection_failed = False
@@ -225,20 +228,23 @@ class Scheduler(QObject):
                 self.download_all_scheduled)
 
         self.timer = QTimer(self)
-        self.timer.start(int(self.INTERVAL * 60000))
+        self.timer.start(int(self.INTERVAL * 60 * 1000))
         self.oldest_timer = QTimer()
         self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check)
         self.connect(self.timer, SIGNAL('timeout()'), self.check)
         self.oldest = gconf['oldest_news']
-        self.oldest_timer.start(int(60 * 60000))
-        self.oldest_check()
+        self.oldest_timer.start(int(60 * 60 * 1000))
+        QTimer.singleShot(5 * 1000, self.oldest_check)
+        self.database_changed = self.recipe_model.database_changed
 
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
             ids = self.recipe_model.db.tags_older_than(_('News'), delta)
             if ids:
-                self.emit(SIGNAL('delete_old_news(PyQt_PyObject)'), ids)
+                ids = list(ids)
+                if ids:
+                    self.delete_old_news.emit(ids)
 
     def show_dialog(self, *args):
         self.lock.lock()
@@ -282,7 +288,7 @@ class Scheduler(QObject):
                     'urn':urn,
                    }
             self.download_queue.add(urn)
-            self.emit(SIGNAL('start_recipe_fetch(PyQt_PyObject)'), arg)
+            self.start_recipe_fetch.emit(arg)
         finally:
             self.lock.unlock()
 

src/calibre/gui2/layout.py

@@ -21,6 +21,7 @@ from calibre.gui2.widgets import ComboBoxWithHelp
 from calibre import human_readable
 from calibre.utils.config import prefs
 from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.gui2.dialogs.scheduler import Scheduler
 
 ICON_SIZE = 48
 
@@ -83,7 +84,7 @@ class LocationManager(QObject): # {{{
 
         ac('library', _('Library'), 'lt.png',
                 _('Show books in calibre library'))
-        ac('main', _('Main'), 'reader.svg',
+        ac('main', _('Reader'), 'reader.svg',
                 _('Show books in the main memory of the device'))
         ac('carda', _('Card A'), 'sd.svg',
                 _('Show books in storage card A'))
@@ -307,7 +308,7 @@ class Action(QAction):
 
 class MainWindowMixin(object):
 
-    def __init__(self):
+    def __init__(self, db):
         self.device_connected = None
         self.setObjectName('MainWindow')
         self.setWindowIcon(QIcon(I('library.png')))
@@ -323,6 +324,7 @@ class MainWindowMixin(object):
         self.donate_button.set_normal_icon_size(ICON_SIZE, ICON_SIZE)
         self.location_manager = LocationManager(self)
 
+        self.init_scheduler(db)
         all_actions = self.setup_actions()
 
         self.search_bar = SearchBar(self)
@@ -334,6 +336,11 @@ class MainWindowMixin(object):
         l = self.centralwidget.layout()
         l.addWidget(self.search_bar)
 
+    def init_scheduler(self, db):
+        self.scheduler = Scheduler(self, db)
+        self.scheduler.start_recipe_fetch.connect(
+                self.download_scheduled_recipe, type=Qt.QueuedConnection)
+
 
     def read_toolbar_settings(self):
         pass
@@ -392,6 +399,10 @@ class MainWindowMixin(object):
         ac(-1, -1, 0, 'books_with_the_same_tags', _('Books with the same tags'),
                 'tags.svg')
 
+        self.action_news.setMenu(self.scheduler.news_menu)
+        self.action_news.triggered.connect(
+                self.scheduler.show_dialog)
+
         self.action_help.triggered.connect(self.show_help)
         md = QMenu()
         md.addAction(_('Edit metadata individually'),

src/calibre/gui2/ui.py

@@ -27,7 +27,6 @@ from calibre.gui2 import error_dialog, GetMetadata, open_local_file, \
         gprefs, max_available_height, config, info_dialog
 from calibre.gui2.cover_flow import CoverFlowMixin
 from calibre.gui2.widgets import ProgressIndicator
-from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import UpdateMixin
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2.layout import MainWindowMixin
@@ -119,7 +118,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
                 self.another_instance_wants_to_talk)
         self.check_messages_timer.start(1000)
 
-        MainWindowMixin.__init__(self)
+        MainWindowMixin.__init__(self, db)
 
         # Jobs Button {{{
         self.job_manager = JobManager()
@@ -253,18 +252,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
                     db, server_config().parse())
             self.test_server_timer = QTimer.singleShot(10000, self.test_server)
 
-
-        self.scheduler = Scheduler(self, self.library_view.model().db)
-        self.action_news.setMenu(self.scheduler.news_menu)
-        self.connect(self.action_news, SIGNAL('triggered(bool)'),
-                self.scheduler.show_dialog)
-        self.connect(self.scheduler, SIGNAL('delete_old_news(PyQt_PyObject)'),
-                self.library_view.model().delete_books_by_id,
-                Qt.QueuedConnection)
-        self.connect(self.scheduler,
-                SIGNAL('start_recipe_fetch(PyQt_PyObject)'),
-                self.download_scheduled_recipe, Qt.QueuedConnection)
-
         self.keyboard_interrupt.connect(self.quit, type=Qt.QueuedConnection)
         AddAction.__init__(self)
 
@@ -272,6 +259,11 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         self.finalize_layout()
         self.donate_button.start_animation()
 
+        self.scheduler.delete_old_news.connect(
+                self.library_view.model().delete_books_by_id,
+                type=Qt.QueuedConnection)
+
+
     def resizeEvent(self, ev):
         MainWindow.resizeEvent(self, ev)
         self.search.setMaximumWidth(self.width()-150)
@@ -402,6 +394,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         self.search.clear_to_help()
         self.book_details.reset_info()
         self.library_view.model().count_changed()
+        self.scheduler.database_changed(db)
         prefs['library_path'] = self.library_path
 
     def show_book_info(self, *args):

src/calibre/web/feeds/recipes/model.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 import os, copy
 
 from PyQt4.Qt import QAbstractItemModel, QVariant, Qt, QColor, QFont, QIcon, \
-        QModelIndex, SIGNAL
+        QModelIndex, SIGNAL, QMetaObject, pyqtSlot
 
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.gui2 import NONE
@@ -131,6 +131,16 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
         self.scheduler_config = SchedulerConfig()
         self.do_refresh()
 
+    @pyqtSlot()
+    def do_database_change(self):
+        self.db = self.newdb
+        self.newdb = None
+        self.do_refresh()
+
+    def database_changed(self, db):
+        self.newdb = db
+        QMetaObject.invokeMethod(self, 'do_database_change', Qt.QueuedConnection)
+
     def get_builtin_recipe(self, urn, download=True):
         if download:
             try: