Merge from trunk

resources/recipes/theeconomictimes_india.recipe

@@ -4,6 +4,7 @@ __copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 economictimes.indiatimes.com
 '''
 
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class TheEconomicTimes(BasicNewsRecipe):
@@ -32,18 +33,17 @@ class TheEconomicTimes(BasicNewsRecipe):
                         , 'language'         : language
                         }
 
-    keep_only_tags = [dict(attrs={'class':'printdiv'})]
-    remove_tags    = [dict(name=['object','link','embed','iframe','base','table','meta'])]
-    remove_attributes = ['name']
 
+    remove_tags_before = dict(name='h1')
     feeds          = [(u'All articles', u'http://economictimes.indiatimes.com/rssfeedsdefault.cms')]
 
     def print_version(self, url):
         rest, sep, art = url.rpartition('/articleshow/')
+        return 'http://m.economictimes.com/PDAET/articleshow/' + art
         return 'http://economictimes.indiatimes.com/articleshow/' + art + '?prtpage=1'
 
     def get_article_url(self, article):
-        rurl = article.get('link',  None)
+        rurl = article.get('guid',  None)
         if (rurl.find('/quickieslist/') > 0) or (rurl.find('/quickiearticleshow/') > 0):
            return None
         return rurl

resources/recipes/toi.recipe

@@ -8,9 +8,10 @@ class TimesOfIndia(BasicNewsRecipe):
     max_articles_per_feed = 25
 
     no_stylesheets = True
-    keep_only_tags = [dict(attrs={'class':'prttabl'})]
+    keep_only_tags = [dict(attrs={'class':'maintable12'})]
     remove_tags = [
-            dict(style=lambda x: x and 'float' in x)
+            dict(style=lambda x: x and 'float' in x),
+            dict(attrs={'class':'prvnxtbg'}),
     ]
 
     feeds          = [

src/calibre/ebooks/epub/output.py

@@ -101,6 +101,13 @@ class EPUBOutput(OutputFormatPlugin):
             )
         ),
 
+        OptionRecommendation(name='epub_flatten', recommended_value=False,
+            help=_('This option is needed only if you intend to use the EPUB'
+                ' with FBReaderJ. It will flatten the file system inside the'
+                ' EPUB, putting all files into the top level.')
+        ),
+
+
         ])
 
     recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)])
@@ -142,8 +149,12 @@ class EPUBOutput(OutputFormatPlugin):
     def convert(self, oeb, output_path, input_plugin, opts, log):
         self.log, self.opts, self.oeb = log, opts, oeb
 
-        #from calibre.ebooks.oeb.transforms.filenames import UniqueFilenames
-        #UniqueFilenames()(oeb, opts)
+        if self.opts.epub_flatten:
+            from calibre.ebooks.oeb.transforms.filenames import FlatFilenames
+            FlatFilenames()(oeb, opts)
+        else:
+            from calibre.ebooks.oeb.transforms.filenames import UniqueFilenames
+            UniqueFilenames()(oeb, opts)
 
         self.workaround_ade_quirks()
         self.workaround_webkit_quirks()

src/calibre/ebooks/metadata/fetch.py

@@ -145,7 +145,7 @@ class MetadataSource(Plugin): # {{{
             setattr(w, '_'+x, cb)
             cb.setChecked(c.get(x, True))
             w._layout.addWidget(cb)
-        
+
         if self.has_html_comments:
             cb = QCheckBox(_('Convert comments downloaded from %s to plain text')%(self.name))
             setattr(w, '_textcomments', cb)
@@ -238,12 +238,13 @@ def result_index(source, result):
     return -1
 
 def merge_results(one, two):
-    for x in two:
-        idx = result_index(one, x)
-        if idx < 0:
-            one.append(x)
-        else:
-            one[idx].smart_update(x)
+    if two is not None and one is not None:
+        for x in two:
+            idx = result_index(one, x)
+            if idx < 0:
+                one.append(x)
+            else:
+                one[idx].smart_update(x)
 
 class MetadataSources(object):
 
@@ -299,7 +300,7 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
         manager(title, author, publisher, isbn, verbose)
         manager.join()
 
-    results = list(fetchers[0].results)
+    results = list(fetchers[0].results) if fetchers else []
     for fetcher in fetchers[1:]:
         merge_results(results, fetcher.results)
 

src/calibre/ebooks/metadata/isbndb.py

@@ -240,8 +240,8 @@ def option_parser():
         %prog [options] key
 
         Fetch metadata for books from isndb.com. You can specify either the
-        books ISBN ID or its title and author. If you specify the title and author,
-        then more than one book may be returned.
+        books ISBN ID or keywords or its title and author.
+        If you specify the title and author or keywords, then more than one book may be returned.
 
         key is the account key you generate after signing up for a free account from isbndb.com.
 
@@ -287,4 +287,4 @@ def main(args=sys.argv):
 if __name__ == '__main__':
     sys.exit(main())
 
-# calibre-debug -e "H:\Mes eBooks\Developpement\calibre\src\calibre\ebooks\metadata\isbndb-bis.py" -m 5 -a gore -v PWEK5WY4>data.html
+# calibre-debug -e "H:\Mes eBooks\Developpement\calibre\src\calibre\ebooks\metadata\isbndb.py" -m 5 -a gore -v HLLXQX2A>data.html

src/calibre/ebooks/mobi/reader.py

@@ -29,6 +29,9 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.ebooks.metadata.toc import TOC
 
+class TopazError(ValueError):
+    pass
+
 class EXTHHeader(object):
 
     def __init__(self, raw, codec, title):
@@ -239,7 +242,7 @@ class MobiReader(object):
         self.base_css_rules = textwrap.dedent('''
                 blockquote { margin: 0em 0em 0em 2em; text-align: justify }
 
-                p { margin: 0em; text-align: justify }
+                p { margin: 0em; text-align: justify; text-indent: 1.5em }
 
                 .bold { font-weight: bold }
 
@@ -259,7 +262,7 @@ class MobiReader(object):
 
         raw = stream.read()
         if raw.startswith('TPZ'):
-            raise ValueError(_('This is an Amazon Topaz book. It cannot be processed.'))
+            raise TopazError(_('This is an Amazon Topaz book. It cannot be processed.'))
 
         self.header   = raw[0:72]
         self.name     = self.header[:32].replace('\x00', '')
@@ -832,6 +835,15 @@ class MobiReader(object):
             im.save(open(path, 'wb'), format='JPEG')
 
 def get_metadata(stream):
+    stream.seek(0)
+    try:
+        raw = stream.read(3)
+    except:
+        raw = ''
+    stream.seek(0)
+    if raw == 'TPZ':
+        from calibre.ebooks.metadata.topaz import get_metadata
+        return get_metadata(stream)
     from calibre.utils.logging import Log
     log = Log()
     mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')])
@@ -861,7 +873,10 @@ def get_metadata(stream):
         cover_index = mh.first_image_index + mh.exth.cover_offset
         data  = mh.section_data(int(cover_index))
     else:
-        data  = mh.section_data(mh.first_image_index)
+        try:
+            data  = mh.section_data(mh.first_image_index)
+        except:
+            data = ''
     buf = cStringIO.StringIO(data)
     try:
         im = PILImage.open(buf)

src/calibre/ebooks/oeb/transforms/filenames.py

@@ -13,15 +13,16 @@ import cssutils
 
 from calibre.ebooks.oeb.base import rewrite_links, urlnormalize
 
-class RenameFiles(object):
+class RenameFiles(object): # {{{
 
     '''
     Rename files and adjust all links pointing to them. Note that the spine
     and manifest are not touched by this transform.
     '''
 
-    def __init__(self, rename_map):
+    def __init__(self, rename_map, renamed_items_map = None):
         self.rename_map = rename_map
+        self.renamed_items_map = renamed_items_map
 
     def __call__(self, oeb, opts):
         self.log = oeb.logger
@@ -49,7 +50,6 @@ class RenameFiles(object):
         if self.oeb.toc:
             self.fix_toc_entry(self.oeb.toc)
 
-
     def fix_toc_entry(self, toc):
         if toc.href:
             href = urlnormalize(toc.href)
@@ -66,18 +66,22 @@ class RenameFiles(object):
             self.fix_toc_entry(x)
 
     def url_replacer(self, orig_url):
-         url = urlnormalize(orig_url)
-         path, frag = urldefrag(url)
-         href = self.current_item.abshref(path)
-         replacement = self.rename_map.get(href, None)
-         if replacement is None:
-             return orig_url
-         replacement = self.current_item.relhref(replacement)
-         if frag:
-             replacement += '#' + frag
-         return replacement
+        url = urlnormalize(orig_url)
+        path, frag = urldefrag(url)
+        if self.renamed_items_map:
+            orig_item = self.renamed_items_map.get(self.current_item.href, self.current_item)
+        else:
+            orig_item = self.current_item
 
-class UniqueFilenames(object):
+        href = orig_item.abshref(path)
+        replacement = self.current_item.relhref(self.rename_map.get(href, href))
+        if frag:
+            replacement += '#' + frag
+        return replacement
+
+# }}}
+
+class UniqueFilenames(object): # {{{
 
     'Ensure that every item in the manifest has a unique filename'
 
@@ -127,4 +131,48 @@ class UniqueFilenames(object):
             candidate = base + suffix + ext
             if candidate not in self.seen_filenames:
                 return suffix
+# }}}
+
+class FlatFilenames(object): # {{{
+
+    'Ensure that every item in the manifest has a unique filename without subdirectories.'
+
+    def __call__(self, oeb, opts):
+        self.log = oeb.logger
+        self.opts = opts
+        self.oeb = oeb
+
+        self.rename_map = {}
+        self.renamed_items_map = {}
+
+        for item in list(oeb.manifest.items):
+            # Flatten URL by removing directories.
+            # Example: a/b/c/index.html -> a_b_c_index.html
+            nhref = item.href.replace("/", "_")
+
+            if item.href == nhref:
+                # URL hasn't changed, skip item.
+                continue
+
+            data = item.data
+            nhref = oeb.manifest.generate(href=nhref)[1]
+            nitem = oeb.manifest.add(item.id, nhref, item.media_type, data=data,
+                                     fallback=item.fallback)
+            self.rename_map[item.href] = nhref
+            self.renamed_items_map[nhref] = item
+            if item.spine_position is not None:
+                oeb.spine.insert(item.spine_position, nitem, item.linear)
+                oeb.spine.remove(item)
+            oeb.manifest.remove(item)
+
+        if self.rename_map:
+            self.log('Found non-flat filenames, renaming to support broken'
+                    ' EPUB readers like FBReader...')
+            from pprint import pformat
+            self.log.debug(pformat(self.rename_map))
+            self.log.debug(pformat(self.renamed_items_map))
+
+            renamer = RenameFiles(self.rename_map, self.renamed_items_map)
+            renamer(oeb, opts)
+# }}}
 

src/calibre/gui2/actions/add.py

@@ -120,6 +120,7 @@ class AddAction(InterfaceAction):
         if self.gui.current_view() is not self.gui.library_view:
             return
         db = self.gui.library_view.model().db
+        cover_changed = False
         current_idx = self.gui.library_view.currentIndex()
         if not current_idx.isValid(): return
         cid = db.id(current_idx.row())
@@ -133,12 +134,16 @@ class AddAction(InterfaceAction):
                 if not pmap.isNull():
                     accept = True
                     db.set_cover(cid, pmap)
+                    cover_changed = True
             elif ext in BOOK_EXTENSIONS:
                 db.add_format_with_hooks(cid, ext, path, index_is_id=True)
                 accept = True
         if accept:
             event.accept()
             self.gui.library_view.model().current_changed(current_idx, current_idx)
+        if cover_changed:
+            if self.gui.cover_flow:
+                self.gui.cover_flow.dataChanged()
 
     def __add_filesystem_book(self, paths, allow_device=True):
         if isinstance(paths, basestring):

src/calibre/gui2/book_details.py

@@ -253,7 +253,12 @@ class BookInfo(QWebView):
                     % (left_pane, right_pane)))
 
     def mouseDoubleClickEvent(self, ev):
-        ev.ignore()
+        if self.width() - ev.x() < 25 or \
+            self.height() - ev.y() < 25:
+            # Filter out double clicks on the scroll bar
+            ev.accept()
+        else:
+            ev.ignore()
 
 # }}}
 

src/calibre/gui2/convert/epub_output.py

@@ -21,7 +21,7 @@ class PluginWidget(Widget, Ui_Form):
         Widget.__init__(self, parent,
                 ['dont_split_on_page_breaks', 'flow_size',
                     'no_default_epub_cover', 'no_svg_cover',
-                    'preserve_cover_aspect_ratio',]
+                    'preserve_cover_aspect_ratio', 'epub_flatten']
                 )
         for i in range(2):
             self.opt_no_svg_cover.toggle()

src/calibre/gui2/convert/epub_output.ui

@@ -81,6 +81,13 @@
      </property>
     </spacer>
    </item>
+   <item row="1" column="1">
+    <widget class="QCheckBox" name="opt_epub_flatten">
+     <property name="text">
+      <string>&amp;Flatten EPUB file structure</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/dialogs/book_info.py

@@ -23,10 +23,6 @@ class BookInfo(QDialog, Ui_BookInfo):
         self.comments.sizeHint = self.comments_size_hint
         self.view_func = view_func
 
-        desktop = QCoreApplication.instance().desktop()
-        screen_height = desktop.availableGeometry().height() - 100
-        self.resize(self.size().width(), screen_height)
-
 
         self.view = view
         self.current_row = None
@@ -40,8 +36,13 @@ class BookInfo(QDialog, Ui_BookInfo):
         self.fit_cover.stateChanged.connect(self.toggle_cover_fit)
         self.cover.resizeEvent = self.cover_view_resized
 
+        desktop = QCoreApplication.instance().desktop()
+        screen_height = desktop.availableGeometry().height() - 100
+        self.resize(self.size().width(), screen_height)
+
+
     def comments_size_hint(self):
-        return QSize(350, 350)
+        return QSize(350, 250)
 
     def toggle_cover_fit(self, state):
         dynamic.set('book_info_dialog_fit_cover', self.fit_cover.isChecked())

src/calibre/gui2/dialogs/book_info.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>917</width>
-    <height>783</height>
+    <height>480</height>
    </rect>
   </property>
   <property name="windowTitle">

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -399,14 +399,11 @@ Future conversion of these books will use the default settings.</string>
            <property name="title">
             <string>Change &amp;cover</string>
            </property>
-           <layout class="QVBoxLayout" name="verticalLayout_2">
+           <layout class="QHBoxLayout" name="horizontalLayout">
             <item>
-             <widget class="QRadioButton" name="cover_no_change">
+             <widget class="QRadioButton" name="cover_generate">
               <property name="text">
-               <string>&amp;No change</string>
-              </property>
-              <property name="checked">
-               <bool>true</bool>
+               <string>&amp;Generate default cover</string>
               </property>
              </widget>
             </item>
@@ -417,13 +414,6 @@ Future conversion of these books will use the default settings.</string>
               </property>
              </widget>
             </item>
-            <item>
-             <widget class="QRadioButton" name="cover_generate">
-              <property name="text">
-               <string>&amp;Generate default cover</string>
-              </property>
-             </widget>
-            </item>
            </layout>
           </widget>
          </item>

src/calibre/gui2/library/views.py

@@ -105,7 +105,8 @@ class BooksView(QTableView): # {{{
         hv.setCursor(Qt.PointingHandCursor)
         self.selected_ids = []
         self._model.about_to_be_sorted.connect(self.about_to_be_sorted)
-        self._model.sorting_done.connect(self.sorting_done)
+        self._model.sorting_done.connect(self.sorting_done,
+                type=Qt.QueuedConnection)
 
     # Column Header Context Menu {{{
     def column_header_context_handler(self, action=None, column=None):
@@ -227,6 +228,7 @@ class BooksView(QTableView): # {{{
             sm = self.selectionModel()
             for idx in indices:
                 sm.select(idx, sm.Select|sm.Rows)
+            self.scroll_to_row(indices[0].row())
         self.selected_ids = []
     # }}}
 

src/calibre/gui2/preferences/look_feel.py

@@ -88,7 +88,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         name = unicode(fi.family())
 
         self.font_display.setFont(font)
-        self.font_display.setText(_('Current font:') + ' ' + name +
+        self.font_display.setText(name +
                 ' [%dpt]'%fi.pointSize())
 
     def change_font(self, *args):

src/calibre/gui2/preferences/look_feel.ui

@@ -183,6 +183,34 @@
      </layout>
     </widget>
    </item>
+   <item row="8" column="0">
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Interface font:</string>
+       </property>
+       <property name="buddy">
+        <cstring>font_display</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLineEdit" name="font_display">
+       <property name="readOnly">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="8" column="1">
+    <widget class="QPushButton" name="change_font_button">
+     <property name="text">
+      <string>Change &amp;font (needs restart)</string>
+     </property>
+    </widget>
+   </item>
    <item row="9" column="0" colspan="2">
     <spacer name="verticalSpacer">
      <property name="orientation">
@@ -196,20 +224,6 @@
      </property>
     </spacer>
    </item>
-   <item row="8" column="0">
-    <widget class="QLineEdit" name="font_display">
-     <property name="readOnly">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item row="8" column="1">
-    <widget class="QPushButton" name="change_font_button">
-     <property name="text">
-      <string>Change &amp;font (needs restart)</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/tools.py

@@ -236,6 +236,10 @@ def fetch_scheduled_recipe(arg):
         recs.append(('header', True, OptionRecommendation.HIGH))
         recs.append(('header_format', '%t', OptionRecommendation.HIGH))
 
+    epub = load_defaults('epub_output')
+    if epub.get('epub_flatten', False):
+        recs.append(('epub_flatten', True, OptionRecommendation.HIGH))
+
     args = [arg['recipe'], pt.name, recs]
     if arg['username'] is not None:
         recs.append(('username', arg['username'], OptionRecommendation.HIGH))

src/calibre/library/database2.py

@@ -1153,6 +1153,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 else:
                     vals = book[dex].split(mult)
                     for val in vals:
+                        if not val: continue
                         try:
                             (item_id, sort_val) = tids[cat][val] # let exceptions fly
                             item = tcategories[cat].get(val, None)

src/calibre/library/server/browse.py

@@ -552,16 +552,18 @@ class BrowseServer(object):
                 ids = self.search_cache('search:"%s"'%which)
             except:
                 raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
-        elif category == 'newest':
-            ids = self.search_cache('')
+        all_ids = self.search_cache('')
+        if category == 'newest':
+            ids = all_ids
             hide_sort = 'true'
         elif category == 'allbooks':
-            ids = self.search_cache('')
+            ids = all_ids
         else:
             q = category
             if q == 'news':
                 q = 'tags'
             ids = self.db.get_books_for_category(q, cid)
+            ids = [x for x in ids if x in all_ids]
 
         items = [self.db.data._data[x] for x in ids]
         if category == 'newest':