Sync to trunk.

resources/recipes/ecotrend.recipe

@@ -0,0 +1,42 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+globaleconomicanalysis.blogspot.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class GlobalEconomicAnalysis(BasicNewsRecipe):
+    title                 = "Mish's Global Economic Trend Analysis"
+    __author__            = 'Darko Miletic'
+    description           = 'Thoughts on the global economy, housing, gold, silver, interest rates, oil, energy, China, commodities, the  dollar, Euro, Renminbi, Yen, inflation, deflation, stagflation, precious metals, emerging markets, and policy decisions that affect the global markets.'
+    publisher             = 'Mike Shedlock'
+    category              = 'news, politics, economy, banking'
+    oldest_article        = 7
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = True
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'blog'
+    masthead_url          = 'http://www.pagina12.com.ar/commons/imgs/logo-home.gif'
+    extra_css             = """
+                               body{font-family: Arial,Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','iframe','object','embed'])
+                    ,dict(attrs={'class':'blogger-post-footer'})
+                  ]
+    remove_attributes=['border']
+
+    feeds = [(u'Articles', u'http://feeds2.feedburner.com/MishsGlobalEconomicTrendAnalysis')]

resources/recipes/gva_be.recipe

@@ -40,13 +40,12 @@ class GazetvanAntwerpen(BasicNewsRecipe):
     remove_tags_after  = dict(name='span', attrs={'class':'author'})
 
     feeds = [
-              (u'Overzicht & Blikvanger', u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/overview/overzicht'       )
+              (u'Binnenland'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/binnenland'    )
+             ,(u'Buitenland'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/buitenland'    )
              ,(u'Stad & Regio'          , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/stadenregio'   )
              ,(u'Economie'              , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/economie'      )
-             ,(u'Binnenland'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/binnenland'    )
-             ,(u'Buitenland'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/buitenland'    )
              ,(u'Media & Cultur'        , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/mediaencultuur')
-             ,(u'Wetenschap'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/mediaencultuur')
+             ,(u'Wetenschap'            , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/wetenschap'    )
              ,(u'Sport'                 , u'http://www.gva.be/syndicationservices/artfeedservice.svc/rss/mostrecent/sport'         )
             ]
 

setup/installer/linux/freeze2.py

@@ -318,7 +318,11 @@ class LinuxFreeze(Command):
             import codecs
 
             def set_default_encoding():
-                locale.setlocale(locale.LC_ALL, '')
+                try:
+                    locale.setlocale(locale.LC_ALL, '')
+                except:
+                    print 'WARNING: Failed to set default libc locale, using en_US.UTF-8'
+                    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
                 enc = locale.getdefaultlocale()[1]
                 if not enc:
                     enc = locale.nl_langinfo(locale.CODESET)

src/calibre/devices/usbms/device.py

@@ -605,8 +605,9 @@ class Device(DeviceConfig, DevicePlugin):
 
         main, carda, cardb = self.find_device_nodes()
         if main is None:
-            raise DeviceError(_('Unable to detect the %s disk drive. Your '
+            raise DeviceError(_('Unable to detect the %s disk drive. Either '
-            ' kernel is probably exporting a deprecated version of SYSFS.')
+            'the device has already been ejected, or your '
+            'kernel is exporting a deprecated version of SYSFS.')
                     %self.__class__.__name__)
 
         self._linux_mount_map = {}

src/calibre/ebooks/metadata/__init__.py

@@ -57,6 +57,8 @@ _ignore_starts = u'\'"'+u''.join(unichr(x) for x in range(0x2018, 0x201e)+[0x203
 
 def title_sort(title):
     title = title.strip()
+    if tweaks['title_series_sorting'] == 'strictly_alphabetic':
+        return title
     if title and title[0] in _ignore_starts:
         title = title[1:]
     match = _title_pat.search(title)

src/calibre/gui2/actions/delete.py

@@ -97,10 +97,15 @@ class DeleteAction(InterfaceAction):
         for action in list(self.delete_menu.actions())[1:]:
             action.setEnabled(enabled)
 
-    def _get_selected_formats(self, msg):
+    def _get_selected_formats(self, msg, ids):
         from calibre.gui2.dialogs.select_formats import SelectFormats
-        fmts = self.gui.library_view.model().db.all_formats()
+        fmts = set([])
-        d = SelectFormats([x.lower() for x in fmts], msg, parent=self.gui)
+        db = self.gui.library_view.model().db
+        for x in ids:
+            fmts_ = db.formats(x, index_is_id=True, verify_formats=False)
+            if fmts_:
+                fmts.update(frozenset([x.lower() for x in fmts_.split(',')]))
+        d = SelectFormats(list(sorted(fmts)), msg, parent=self.gui)
         if d.exec_() != d.Accepted:
             return None
         return d.selected_formats
@@ -118,7 +123,7 @@ class DeleteAction(InterfaceAction):
         if not ids:
             return
         fmts = self._get_selected_formats(
-            _('Choose formats to be deleted'))
+            _('Choose formats to be deleted'), ids)
         if not fmts:
             return
         for id in ids:
@@ -136,7 +141,7 @@ class DeleteAction(InterfaceAction):
         if not ids:
             return
         fmts = self._get_selected_formats(
-            '<p>'+_('Choose formats <b>not</b> to be deleted'))
+            '<p>'+_('Choose formats <b>not</b> to be deleted'), ids)
         if fmts is None:
             return
         for id in ids:

src/calibre/gui2/convert/mobi_output.py

@@ -11,7 +11,6 @@ from PyQt4.Qt import Qt
 from calibre.gui2.convert.mobi_output_ui import Ui_Form
 from calibre.gui2.convert import Widget
 from calibre.gui2.widgets import FontFamilyModel
-from calibre.utils.fonts import fontconfig
 
 font_family_model = None
 
@@ -28,6 +27,7 @@ class PluginWidget(Widget, Ui_Form):
                     'mobi_ignore_margins',
                 'dont_compress', 'no_inline_toc', 'masthead_font','personal_doc']
                 )
+        from calibre.utils.fonts import fontconfig
         self.db, self.book_id = db, book_id
 
         global font_family_model

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to edit metadata in bulk'''
 
-import re
+import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
                      pyqtSignal, QDialogButtonBox
@@ -12,12 +12,41 @@ from PyQt4 import QtGui
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, authors_to_string
+from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
+from calibre.utils.config import prefs
+from calibre.utils.magick.draw import identify_data
+
+def get_cover_data(path):
+    old = prefs['read_file_metadata']
+    if not old:
+        prefs['read_file_metadata'] = True
+    cdata = area = None
+
+    try:
+        mi = get_metadata(open(path, 'rb'),
+                os.path.splitext(path)[1][1:].lower())
+        if mi.cover and os.access(mi.cover, os.R_OK):
+            cdata = open(mi.cover).read()
+        elif mi.cover_data[1] is not None:
+            cdata = mi.cover_data[1]
+        if cdata:
+            width, height, fmt = identify_data(cdata)
+            area = width*height
+    except:
+        cdata = area = None
+
+    if old != prefs['read_file_metadata']:
+        prefs['read_file_metadata'] = old
+
+    return cdata, area
+
+
 
 class MyBlockingBusy(QDialog):
 
@@ -146,6 +175,20 @@ class MyBlockingBusy(QDialog):
                 cdata = calibre_cover(mi.title, mi.format_field('authors')[-1],
                         series_string=series_string)
                 self.db.set_cover(id, cdata)
+            elif cover_action == 'fromfmt':
+                fmts = self.db.formats(id, index_is_id=True, verify_formats=False)
+                if fmts:
+                    covers = []
+                    for fmt in fmts.split(','):
+                        fmt = self.db.format_abspath(id, fmt, index_is_id=True)
+                        if not fmt: continue
+                        cdata, area = get_cover_data(fmt)
+                        if cdata:
+                            covers.append((cdata, area))
+                    covers.sort(key=lambda x: x[1])
+                    if covers:
+                        self.db.set_cover(id, covers[-1][0])
+                    covers = []
         elif self.current_phase == 2:
             # All of these just affect the DB, so we can tolerate a total rollback
             if do_auto_author:
@@ -700,6 +743,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             cover_action = 'remove'
         elif self.cover_generate.isChecked():
             cover_action = 'generate'
+        elif self.cover_from_fmt.isChecked():
+            cover_action = 'fromfmt'
 
         args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series,
                 do_autonumber, do_remove_format, remove_format, do_swap_ta,

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -414,6 +414,13 @@ Future conversion of these books will use the default settings.</string>
               </property>
              </widget>
             </item>
+            <item>
+             <widget class="QRadioButton" name="cover_from_fmt">
+              <property name="text">
+               <string>Set from &amp;ebook file(s)</string>
+              </property>
+             </widget>
+            </item>
            </layout>
           </widget>
          </item>
@@ -686,8 +693,8 @@ nothing should be put between the original text and the inserted text</string>
              <rect>
               <x>0</x>
               <y>0</y>
-              <width>122</width>
+              <width>726</width>
-              <height>38</height>
+              <height>334</height>
              </rect>
             </property>
             <layout class="QGridLayout" name="testgrid">

src/calibre/gui2/viewer/main.py

@@ -650,7 +650,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
             self.current_book_has_toc = bool(self.iterator.toc)
             self.current_title = title
-            self.setWindowTitle(self.base_window_title+' - '+title)
+            self.setWindowTitle(self.base_window_title+' - '+title +
+                    ' [%s]'%os.path.splitext(pathtoebook)[1][1:].upper())
             self.pos.setMaximum(sum(self.iterator.pages))
             self.pos.setSuffix(' / %d'%sum(self.iterator.pages))
             self.vertical_scrollbar.setMinimum(100)

src/calibre/gui2/widgets.py

@@ -19,7 +19,6 @@ from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
 from calibre.constants import isosx
 from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
-from calibre.utils.fonts import fontconfig
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata.meta import metadata_from_filename
 from calibre.utils.config import prefs, XMLConfig
@@ -283,6 +282,7 @@ class FontFamilyModel(QAbstractListModel):
 
     def __init__(self, *args):
         QAbstractListModel.__init__(self, *args)
+        from calibre.utils.fonts import fontconfig
         try:
             self.families = fontconfig.find_font_families()
         except:

src/calibre/library/database2.py

@@ -1128,6 +1128,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 for l in list:
                     (id, val, sort_val) = (l[0], l[1], l[2])
                     tids[category][val] = (id, sort_val)
+            elif cat['datatype'] == 'series':
+                for l in list:
+                    (id, val) = (l[0], l[1])
+                    tids[category][val] = (id, title_sort(val))
             elif cat['datatype'] == 'rating':
                 for l in list:
                     (id, val) = (l[0], l[1])

src/calibre/library/server/content.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, os
+import re, os, posixpath
 
 import cherrypy
 
@@ -88,17 +88,24 @@ class ContentServer(object):
     def static(self, name):
         'Serves static content'
         name = name.lower()
-        cherrypy.response.headers['Content-Type'] = {
+        fname = posixpath.basename(name)
+        try:
+            cherrypy.response.headers['Content-Type'] = {
                      'js'   : 'text/javascript',
                      'css'  : 'text/css',
                      'png'  : 'image/png',
                      'gif'  : 'image/gif',
                      'html' : 'text/html',
-                     ''      : 'application/octet-stream',
+                     }[fname.rpartition('.')[-1].lower()]
-                     }[name.rpartition('.')[-1].lower()]
+        except KeyError:
+            raise cherrypy.HTTPError(404, '%r not a valid resource type'%name)
         cherrypy.response.headers['Last-Modified'] = self.last_modified(self.build_time)
-        path = P('content_server/'+name)
+        basedir = os.path.abspath(P('content_server'))
-        if not os.path.exists(path):
+        path = os.path.join(basedir, name.replace('/', os.sep))
+        path = os.path.abspath(path)
+        if not path.startswith(basedir):
+            raise cherrypy.HTTPError(403, 'Access to %s is forbidden'%name)
+        if not os.path.exists(path) or not os.path.isfile(path):
             raise cherrypy.HTTPError(404, '%s not found'%name)
         if self.opts.develop:
             lm = fromtimestamp(os.stat(path).st_mtime)

src/calibre/library/sqlite.py

@@ -161,10 +161,7 @@ class DBThread(Thread):
             self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
         self.conn.create_collation('PYNOCASE', partial(pynocase,
             encoding=encoding))
-        if tweaks['title_series_sorting'] == 'strictly_alphabetic':
+        self.conn.create_function('title_sort', 1, title_sort)
-            self.conn.create_function('title_sort', 1, lambda x:x)
-        else:
-            self.conn.create_function('title_sort', 1, title_sort)
         self.conn.create_function('author_to_author_sort', 1,
                 _author_to_author_sort)
         self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))

src/calibre/utils/fonts/__init__.py

@@ -7,15 +7,19 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, sys
-from threading import Thread
 
-from calibre.constants import plugins, iswindows
+from calibre.constants import plugins, iswindows, islinux, isfreebsd
 
 _fc, _fc_err = plugins['fontconfig']
 
 if _fc is None:
     raise RuntimeError('Failed to load fontconfig with error:'+_fc_err)
 
+if islinux or isfreebsd:
+    Thread = object
+else:
+    from threading import Thread
+
 class FontConfig(Thread):
 
     def __init__(self):
@@ -45,7 +49,8 @@ class FontConfig(Thread):
             self.failed = True
 
     def wait(self):
-        self.join()
+        if not (islinux or isfreebsd):
+            self.join()
         if self.failed:
             raise RuntimeError('Failed to initialize fontconfig')
 
@@ -144,7 +149,13 @@ class FontConfig(Thread):
         return fonts if all else (fonts[0] if fonts else None)
 
 fontconfig = FontConfig()
-fontconfig.start()
+if islinux or isfreebsd:
+    # On X11 Qt also uses fontconfig, so initialization must happen in the
+    # main thread. In any case on X11 initializing fontconfig should be very
+    # fast
+    fontconfig.run()
+else:
+    fontconfig.start()
 
 def test():
     from pprint import pprint;