From db5f63bcac8a0112022e4bafb54939804b1e1f1e Mon Sep 17 00:00:00 2001
From: bulislaw <bulislaw@gmail.com>
Date: Sat, 22 Jan 2011 17:44:09 +0000
Subject: [PATCH 01/52] Persistent Search & Replace query UI proposal

---
 src/calibre/gui2/dialogs/metadata_bulk.ui | 135 ++++++++++++++++++----
 1 file changed, 113 insertions(+), 22 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index f8ae926be6..a1e1d8c550 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -44,8 +44,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>842</width>
-        <height>589</height>
+        <width>832</width>
+        <height>574</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -55,7 +55,7 @@
        <item>
         <widget class="QTabWidget" name="central_widget">
          <property name="currentIndex">
-          <number>0</number>
+          <number>2</number>
          </property>
          <widget class="QWidget" name="tabWidgetPage1">
           <attribute name="title">
@@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeConstraint">
             <enum>QLayout::SetMinimumSize</enum>
            </property>
-           <item row="1" column="0" colspan="3">
+           <item row="1" column="0" colspan="4">
             <widget class="QLabel" name="s_r_heading">
              <property name="wordWrap">
               <bool>true</bool>
@@ -591,7 +591,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="0">
+           <item row="11" column="0">
             <widget class="QLabel" name="xlabel_21">
              <property name="text">
               <string>Search &amp;field:</string>
@@ -601,14 +601,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="1">
+           <item row="11" column="2">
             <widget class="QComboBox" name="search_field">
              <property name="toolTip">
               <string>The name of the field that you want to search</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="2">
+           <item row="11" column="3">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QLabel" name="xlabel_24">
@@ -642,7 +642,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="4" column="0">
+           <item row="12" column="0">
             <widget class="QLabel" name="template_label">
              <property name="text">
               <string>Te&amp;mplate:</string>
@@ -652,7 +652,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="4" column="1">
+           <item row="12" column="2">
             <widget class="HistoryLineEdit" name="s_r_template">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -665,7 +665,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="0">
+           <item row="13" column="0">
             <widget class="QLabel" name="xlabel_2">
              <property name="text">
               <string>&amp;Search for:</string>
@@ -675,7 +675,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="1">
+           <item row="13" column="2">
             <widget class="HistoryLineEdit" name="search_for">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -688,7 +688,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="5" column="2">
+           <item row="13" column="3">
             <widget class="QCheckBox" name="case_sensitive">
              <property name="toolTip">
               <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -701,7 +701,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="0">
+           <item row="14" column="0">
             <widget class="QLabel" name="xlabel_4">
              <property name="text">
               <string>&amp;Replace with:</string>
@@ -711,14 +711,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="1">
+           <item row="14" column="2">
             <widget class="HistoryLineEdit" name="replace_with">
              <property name="toolTip">
               <string>The replacement text. The matched search text will be replaced with this string</string>
              </property>
             </widget>
            </item>
-           <item row="6" column="2">
+           <item row="14" column="3">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="label_41">
@@ -753,7 +753,7 @@ field is processed. In regular expression mode, only the matched text is process
              </item>
             </layout>
            </item>
-           <item row="7" column="0">
+           <item row="15" column="0">
             <widget class="QLabel" name="destination_field_label">
              <property name="text">
               <string>&amp;Destination field:</string>
@@ -763,7 +763,7 @@ field is processed. In regular expression mode, only the matched text is process
              </property>
             </widget>
            </item>
-           <item row="7" column="1">
+           <item row="15" column="2">
             <widget class="QComboBox" name="destination_field">
              <property name="toolTip">
               <string>The field that the text will be put into after all replacements.
@@ -771,7 +771,7 @@ If blank, the source field is used if the field is modifiable</string>
              </property>
             </widget>
            </item>
-           <item row="7" column="2">
+           <item row="15" column="3">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="replace_mode_label">
@@ -820,7 +820,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="8" column="1" colspan="2">
+           <item row="16" column="2" colspan="2">
             <layout class="QHBoxLayout" name="horizontalLayout_21">
              <item>
               <spacer name="HSpacer_347">
@@ -906,7 +906,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="9" column="0" colspan="4">
+           <item row="17" column="0" colspan="5">
             <widget class="QScrollArea" name="scrollArea11">
              <property name="frameShape">
               <enum>QFrame::NoFrame</enum>
@@ -919,8 +919,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>197</width>
-                <height>60</height>
+                <width>810</width>
+                <height>264</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">
@@ -968,6 +968,77 @@ not multiple and the destination field is multiple</string>
              </widget>
             </widget>
            </item>
+           <item row="3" column="0">
+            <widget class="QLabel" name="xlabel_22">
+             <property name="text">
+              <string>Load query:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_field</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <widget class="QComboBox" name="query_field">
+             <property name="toolTip">
+              <string>The name of the field that you want to search</string>
+             </property>
+            </widget>
+           </item>
+           <item row="4" column="0" colspan="4">
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="3">
+            <layout class="QHBoxLayout" name="horizontalLayout_6">
+             <item>
+              <spacer name="horizontalSpacer_3">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeType">
+                <enum>QSizePolicy::Fixed</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>20</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
+              <widget class="QPushButton" name="save_button">
+               <property name="text">
+                <string>Save</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QPushButton" name="remove_button">
+               <property name="text">
+                <string>Remove</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer_2">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
+           </item>
           </layout>
          </widget>
         </widget>
@@ -1030,6 +1101,9 @@ not multiple and the destination field is multiple</string>
   <tabstop>series_numbering_restarts</tabstop>
   <tabstop>series_start_number</tabstop>
   <tabstop>button_box</tabstop>
+  <tabstop>query_field</tabstop>
+  <tabstop>save_button</tabstop>
+  <tabstop>remove_button</tabstop>
   <tabstop>search_field</tabstop>
   <tabstop>search_mode</tabstop>
   <tabstop>s_r_template</tabstop>
@@ -1045,6 +1119,23 @@ not multiple and the destination field is multiple</string>
   <tabstop>multiple_separator</tabstop>
   <tabstop>test_text</tabstop>
   <tabstop>test_result</tabstop>
+  <tabstop>scrollArea</tabstop>
+  <tabstop>central_widget</tabstop>
+  <tabstop>swap_title_and_author</tabstop>
+  <tabstop>clear_series</tabstop>
+  <tabstop>adddate</tabstop>
+  <tabstop>clear_adddate_button</tabstop>
+  <tabstop>apply_adddate</tabstop>
+  <tabstop>pubdate</tabstop>
+  <tabstop>clear_pubdate_button</tabstop>
+  <tabstop>apply_pubdate</tabstop>
+  <tabstop>remove_format</tabstop>
+  <tabstop>change_title_to_title_case</tabstop>
+  <tabstop>remove_conversion_settings</tabstop>
+  <tabstop>cover_generate</tabstop>
+  <tabstop>cover_remove</tabstop>
+  <tabstop>cover_from_fmt</tabstop>
+  <tabstop>scrollArea11</tabstop>
  </tabstops>
  <resources>
   <include location="../../../../resources/images.qrc"/>

From e42664da72abf03af7c86d6b8ded585389e61b65 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 22 Jan 2011 11:08:13 -0700
Subject: [PATCH 02/52] Roger Ebert by Shane Erstad

---
 resources/recipes/roger_ebert.recipe | 120 +++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 resources/recipes/roger_ebert.recipe

diff --git a/resources/recipes/roger_ebert.recipe b/resources/recipes/roger_ebert.recipe
new file mode 100644
index 0000000000..2ea5b52a45
--- /dev/null
+++ b/resources/recipes/roger_ebert.recipe
@@ -0,0 +1,120 @@
+import re
+import urllib2
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, SoupStrainer
+
+class Ebert(BasicNewsRecipe):
+    title                 = 'Roger Ebert'
+    __author__            = 'Shane Erstad'
+    description           = 'Roger Ebert Movie Reviews'
+    publisher             = 'Chicago Sun Times'
+    category              = 'movies'
+    oldest_article        = 8
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    masthead_url          = 'http://rogerebert.suntimes.com/graphics/global/roger.jpg'
+    language              = 'en'
+    remove_empty_feeds    = False
+    PREFIX                  = 'http://rogerebert.suntimes.com'
+    patternReviews                = r'<span class="*?movietitle"*?>(.*?)</span>.*?<div class="*?headline"*?>(.*?)</div>(.*?)</div>'
+    patternCommentary       = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?COMMENTARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternPeople           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?PEOPLE.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternGlossary           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?GLOSSARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+
+    feeds          = [
+                        (u'Reviews'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=reviews' )
+                        ,(u'Commentary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=COMMENTARY')
+                        ,(u'Great Movies'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=REVIEWS08')
+                        ,(u'People'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=PEOPLE')
+                        ,(u'Glossary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=GLOSSARY')
+
+                     ]
+
+    preprocess_regexps = [
+        (re.compile(r'<font.*?>.*?This is a printer friendly.*?</font>.*?<hr>', re.DOTALL|re.IGNORECASE),
+            lambda m: '')
+    ]
+
+
+
+    def print_version(self, url):
+        return url + '&template=printart'
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.log('\tFeedurl: ', feedurl)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            page = urllib2.urlopen(feedurl).read()
+
+            if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    pattern = self.patternReviews
+            elif feedtitle == 'Commentary':
+                    pattern = self.patternCommentary
+            elif feedtitle == 'People':
+                    pattern = self.patternPeople
+            elif feedtitle == 'Glossary':
+                    pattern = self.patternGlossary
+
+
+            regex = re.compile(pattern, re.IGNORECASE|re.DOTALL)
+
+            for match in regex.finditer(page):
+                if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    movietitle = match.group(1)
+                    thislink = match.group(2)
+                    description = match.group(3)
+                elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                    thislink = match.group(1)
+                    description = match.group(2)
+
+                self.log(thislink)
+
+                for link in BeautifulSoup(thislink, parseOnlyThese=SoupStrainer('a')):
+                    thisurl = self.PREFIX + link['href']
+                    thislinktext = self.tag_to_string(link)
+
+                    if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                        thistitle = movietitle
+                    elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary':
+                        thistitle = thislinktext
+
+                    if thistitle == '':
+                        thistitle = 'Ebert Journal Post'
+
+                    """
+                    pattern2 = r'AID=\/(.*?)\/'
+                    reg2 = re.compile(pattern2, re.IGNORECASE|re.DOTALL)
+                    match2 = reg2.search(thisurl)
+                    date = match2.group(1)
+                    c = time.strptime(match2.group(1),"%Y%m%d")
+                    date=time.strftime("%a, %b %d, %Y", c)
+                    self.log(date)
+                    """
+
+                    articles.append({
+                                      'title'      :thistitle
+                                     ,'date'       :''
+                                     ,'url'        :thisurl
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+
+        return totalfeeds
+

From 57883c120e95c72c7b47e82524ec1d468cdff76b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 22 Jan 2011 11:28:47 -0700
Subject: [PATCH 03/52] Fix regression in converting HTML files that have non
 ASCII characters inside their <style> tags. Apparently Word generates these.
 Fixes #8494 (trouble converting htm files (for john))

---
 src/calibre/ebooks/oeb/base.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py
index e11f6b45be..9389964962 100644
--- a/src/calibre/ebooks/oeb/base.py
+++ b/src/calibre/ebooks/oeb/base.py
@@ -221,7 +221,10 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                         el.text):
             stylesheet = parseString(el.text)
             replaceUrls(stylesheet, link_repl_func)
-            el.text = '\n'+stylesheet.cssText + '\n'
+            repl = stylesheet.cssText
+            if isbytestring(repl):
+                repl = repl.decode('utf-8')
+            el.text = '\n'+ repl + '\n'
 
         if 'style' in el.attrib:
             text = el.attrib['style']
@@ -234,8 +237,11 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                             set_property(item)
                     elif v.CSS_PRIMITIVE_VALUE == v.cssValueType:
                         set_property(v)
-                el.attrib['style'] = stext.cssText.replace('\n', ' ').replace('\r',
+                repl = stext.cssText.replace('\n', ' ').replace('\r',
                         ' ')
+                if isbytestring(repl):
+                    repl = repl.decode('utf-8')
+                el.attrib['style'] = repl
 
 
 

From 25a62ee96c68232a01c9c50bf8c42fb3d61872bb Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 22 Jan 2011 13:54:45 -0700
Subject: [PATCH 04/52] Proper fix for switch library mem leak (I hope)

---
 src/calibre/gui2/actions/choose_library.py | 28 ++++++++++++++++------
 src/calibre/gui2/main.py                   |  7 +++---
 src/calibre/gui2/tag_view.py               | 12 ++++++++--
 src/calibre/library/caches.py              |  8 +++++++
 src/calibre/library/database2.py           |  4 +++-
 5 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index 930e5e29aa..8455703bad 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -384,14 +384,28 @@ class ChooseLibraryAction(InterfaceAction):
             return
 
         prefs['library_path'] = loc
-        #from calibre.utils.mem import memory
-        #import weakref, gc
-        #ref = weakref.ref(self.gui.library_view.model().db)
-        #before = memory()/1024**2
+        from calibre.utils.mem import memory
+        import weakref
+        from PyQt4.Qt import QTimer
+        self.dbref = weakref.ref(self.gui.library_view.model().db)
+        self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        #print gc.get_referrers(ref)[0]
-        #for i in xrange(3): gc.collect()
-        #print 'leaked:', memory()/1024**2 - before
+        QTimer.singleShot(1000, self.debug_leak)
+
+    def debug_leak(self):
+        import gc
+        from calibre.utils.mem import memory
+        ref = self.dbref
+        for i in xrange(3): gc.collect()
+        if ref() is not None:
+            print 11111, ref()
+            for r in gc.get_referrers(ref())[:10]:
+                print r
+                print
+        print 'before:', self.before_mem
+        print 'after:', memory()/1024**2
+        self.dbref = self.before_mem = None
+
 
     def qs_requested(self, idx, *args):
         self.switch_requested(self.qs_locations[idx])
diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py
index aaca398e44..b88b1d680d 100644
--- a/src/calibre/gui2/main.py
+++ b/src/calibre/gui2/main.py
@@ -150,13 +150,13 @@ class GuiRunner(QObject):
         if DEBUG:
             prints('Starting up...')
 
-    def start_gui(self):
+    def start_gui(self, db):
         from calibre.gui2.ui import Main
         main = Main(self.opts, gui_debug=self.gui_debug)
         if self.splash_screen is not None:
             self.splash_screen.showMessage(_('Initializing user interface...'))
             self.splash_screen.finish(main)
-        main.initialize(self.library_path, self.db, self.listener, self.actions)
+        main.initialize(self.library_path, db, self.listener, self.actions)
         if DEBUG:
             prints('Started up in', time.time() - self.startup_time)
         add_filesystem_book = partial(main.iactions['Add Books'].add_filesystem_book, allow_device=False)
@@ -200,8 +200,7 @@ class GuiRunner(QObject):
                     det_msg=traceback.format_exc(), show=True)
                 self.initialization_failed()
 
-        self.db = db
-        self.start_gui()
+        self.start_gui(db)
 
     def initialize_db(self):
         db = None
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index f6eac49426..837776cb9a 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -114,6 +114,9 @@ class TagsView(QTreeView): # {{{
 
     def set_database(self, db, tag_match, sort_by):
         self.hidden_categories = config['tag_browser_hidden_categories']
+        old = getattr(self, '_model', None)
+        if old is not None:
+            old.break_cycles()
         self._model = TagsModel(db, parent=self,
                                 hidden_categories=self.hidden_categories,
                                 search_restriction=None,
@@ -371,6 +374,9 @@ class TagsView(QTreeView): # {{{
     # model. Reason: it is much easier than reconstructing the browser tree.
     def set_new_model(self, filter_categories_by=None):
         try:
+            old = getattr(self, '_model', None)
+            if old is not None:
+                old.break_cycles()
             self._model = TagsModel(self.db, parent=self,
                                     hidden_categories=self.hidden_categories,
                                     search_restriction=self.search_restriction,
@@ -544,6 +550,9 @@ class TagsModel(QAbstractItemModel): # {{{
                     tooltip=tt, category_key=r)
         self.refresh(data=data)
 
+    def break_cycles(self):
+        self.db = self.root_item = None
+
     def mimeTypes(self):
         return ["application/calibre+from_library"]
 
@@ -1109,8 +1118,7 @@ class TagBrowserMixin(object): # {{{
 
     def __init__(self, db):
         self.library_view.model().count_changed_signal.connect(self.tags_view.recount)
-        self.tags_view.set_database(self.library_view.model().db,
-                self.tag_match, self.sort_by)
+        self.tags_view.set_database(db, self.tag_match, self.sort_by)
         self.tags_view.tags_marked.connect(self.search.set_search_string)
         self.tags_view.tag_list_edit.connect(self.do_tags_list_edit)
         self.tags_view.user_category_edit.connect(self.do_user_categories_edit)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 4168360d3a..291d71f572 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -42,6 +42,9 @@ class MetadataBackup(Thread): # {{{
 
     def stop(self):
         self.keep_running = False
+        # Break cycles so that this object doesn't hold references to db
+        self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \
+            self.set_dirtied = self.db = None
 
     def run(self):
         while self.keep_running:
@@ -185,6 +188,11 @@ class ResultCache(SearchQueryParser): # {{{
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
+    def break_cycles(self):
+        self._data = self.field_metadata = self.FIELD_MAP = \
+            self.numeric_search_relops = self.date_search_relops = \
+            self.all_search_locations = None
+
 
     def __getitem__(self, row):
         return self._data[self._map_filtered[row]]
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 3dc110c1c8..4b66b6620f 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -362,7 +362,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.last_update_check = self.last_modified()
 
     def break_cycles(self):
-        self.data = self.field_metadata = self.prefs = self.listeners = None
+        self.data.break_cycles()
+        self.data = self.field_metadata = self.prefs = self.listeners = \
+            self.refresh_ondevice = None
 
     def initialize_database(self):
         metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read()

From beb919607963bfe9d7ac823394e9478a4e7a6c4a Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 22 Jan 2011 13:57:02 -0700
Subject: [PATCH 05/52] ...

---
 src/calibre/gui2/actions/choose_library.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index 8455703bad..d726241432 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -384,13 +384,13 @@ class ChooseLibraryAction(InterfaceAction):
             return
 
         prefs['library_path'] = loc
-        from calibre.utils.mem import memory
-        import weakref
-        from PyQt4.Qt import QTimer
-        self.dbref = weakref.ref(self.gui.library_view.model().db)
-        self.before_mem = memory()/1024**2
+        #from calibre.utils.mem import memory
+        #import weakref
+        #from PyQt4.Qt import QTimer
+        #self.dbref = weakref.ref(self.gui.library_view.model().db)
+        #self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        QTimer.singleShot(1000, self.debug_leak)
+        #QTimer.singleShot(1000, self.debug_leak)
 
     def debug_leak(self):
         import gc

From 83840957be81e8ce10e9555a42533c2ba5f76cd4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 23 Jan 2011 11:52:28 +0000
Subject: [PATCH 06/52] tag browser changes: 1) make user categories a search
 target in the find box (move colon to front) 2) show empty categories 3) add
 an api to find the path to a category

---
 src/calibre/gui2/tag_view.py          | 32 ++++++++++++++++++++-------
 src/calibre/library/database2.py      | 31 +++++++++++++-------------
 src/calibre/library/field_metadata.py |  6 ++---
 3 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index f6eac49426..36b5eea940 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -509,8 +509,8 @@ class TagsModel(QAbstractItemModel): # {{{
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
-        # before a QPaintDevice'. The ':' in front avoids polluting either the
-        # user-defined categories (':' at end) or columns namespaces (no ':').
+        # before a QPaintDevice'. The ':' at the end avoids polluting either the
+        # user-defined categories (':' at front) or columns namespaces (no ':').
         iconmap = {}
         for key in category_icon_map:
             iconmap[key] = QIcon(I(category_icon_map[key]))
@@ -681,7 +681,7 @@ class TagsModel(QAbstractItemModel): # {{{
         tb_cats = self.db.field_metadata
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
-            cat_name = user_cat+':' # add the ':' to avoid name collision
+            cat_name = ':' + user_cat # add the ':' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
@@ -988,7 +988,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
-            if key.endswith(':'):
+            if key.startswith(':'):
                 # User category, so skip it. The tag will be marked in its real category
                 continue
             category_item = self.root_item.children[row_index]
@@ -1007,7 +1007,7 @@ class TagsModel(QAbstractItemModel): # {{{
                         ans.append('%s%s:"=%s"'%(prefix, category, tag.name))
         return ans
 
-    def find_node(self, key, txt, start_path):
+    def find_item_node(self, key, txt, start_path):
         '''
         Search for an item (a node) in the tags browser list that matches both
         the key (exact case-insensitive match) and txt (contains case-
@@ -1061,6 +1061,22 @@ class TagsModel(QAbstractItemModel): # {{{
                 break
         return self.path_found
 
+    def find_category_node(self, key):
+        '''
+        Search for an category node (a top-level node) in the tags browser list
+        that matches the key (exact case-insensitive match). Returns the path to
+        the node. Paths are as in find_item_node.
+        '''
+        if not key:
+            return None
+
+        for i in xrange(self.rowCount(QModelIndex())):
+            idx = self.index(i, 0, QModelIndex())
+            ckey = idx.internalPointer().category_key
+            if strcmp(ckey, key) == 0:
+                return self.path_for_index(idx)
+        return None
+
     def show_item_at_path(self, path, box=False):
         '''
         Scroll the browser and open categories to show the item referenced by
@@ -1347,15 +1363,15 @@ class TagBrowserWidget(QWidget): # {{{
         self.search_button.setFocus(True)
         self.item_search.lineEdit().blockSignals(False)
 
-        colon = txt.find(':')
         key = None
+        colon = txt.rfind(':') if len(txt) > 2 else 0
         if colon > 0:
             key = self.parent.library_view.model().db.\
                         field_metadata.search_term_to_field_key(txt[:colon])
             txt = txt[colon+1:]
 
-        self.current_find_position = model.find_node(key, txt,
-                                                     self.current_find_position)
+        self.current_find_position = \
+            model.find_item_node(key, txt, self.current_find_position)
         if self.current_find_position:
             model.show_item_at_path(self.current_find_position, box=True)
         elif self.item_search.text():
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 3dc110c1c8..4b6ab247c5 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -319,7 +319,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.field_metadata.remove_dynamic_categories()
         tb_cats = self.field_metadata
         for user_cat in sorted(self.prefs.get('user_categories', {}).keys(), key=sort_key):
-            cat_name = user_cat+':' # add the ':' to avoid name collision
+            cat_name = ':' + user_cat # add the ':' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
@@ -1241,7 +1241,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     if category in icon_map:
                         icon = icon_map[label]
                 else:
-                    icon = icon_map[':custom']
+                    icon = icon_map['custom:']
                     icon_map[category] = icon
 
             datatype = cat['datatype']
@@ -1337,20 +1337,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 if label in taglist and name in taglist[label]:
                     items.append(taglist[label][name])
                 # else: do nothing, to not include nodes w zero counts
-            if len(items):
-                cat_name = user_cat+':' # add the ':' to avoid name collision
-                # Not a problem if we accumulate entries in the icon map
-                if icon_map is not None:
-                    icon_map[cat_name] = icon_map[':user']
-                if sort == 'popularity':
-                    categories[cat_name] = \
-                        sorted(items, key=lambda x: x.count, reverse=True)
-                elif sort == 'name':
-                    categories[cat_name] = \
-                        sorted(items, key=lambda x: sort_key(x.sort))
-                else:
-                    categories[cat_name] = \
-                        sorted(items, key=lambda x:x.avg_rating, reverse=True)
+            cat_name = ':' + user_cat # add the ':' to avoid name collision
+            # Not a problem if we accumulate entries in the icon map
+            if icon_map is not None:
+                icon_map[cat_name] = icon_map['user:']
+            if sort == 'popularity':
+                categories[cat_name] = \
+                    sorted(items, key=lambda x: x.count, reverse=True)
+            elif sort == 'name':
+                categories[cat_name] = \
+                    sorted(items, key=lambda x: sort_key(x.sort))
+            else:
+                categories[cat_name] = \
+                    sorted(items, key=lambda x:x.avg_rating, reverse=True)
 
         #### Finally, the saved searches category ####
         items = []
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 2a9b7e7003..e9402d1227 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -16,7 +16,7 @@ class TagsIcons(dict):
     '''
 
     category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
-                      'news',    'tags',   ':custom', ':user',     'search',]
+                      'news',    'tags',   'custom:', 'user:',     'search',]
     def __init__(self, icon_dict):
         for a in self.category_icons:
             if a not in icon_dict:
@@ -31,8 +31,8 @@ category_icon_map = {
                     'rating'    : 'rating.png',
                     'news'      : 'news.png',
                     'tags'      : 'tags.png',
-                    ':custom'   : 'column.png',
-                    ':user'     : 'drawer.png',
+                    'custom:'   : 'column.png',
+                    'user:'     : 'drawer.png',
                     'search'    : 'search.png'
             }
 

From 6eef57563cfe8717c5524d3f38f8e77007cdeccd Mon Sep 17 00:00:00 2001
From: GRiker <griker@hotmail.com>
Date: Sun, 23 Jan 2011 06:46:39 -0700
Subject: [PATCH 07/52] GwR patch mapping ESC to clear search box

---
 src/calibre/gui2/search_box.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index e4073a01c9..99d435c913 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -76,7 +76,10 @@ class SearchBox2(QComboBox): # {{{
         c.setCompletionMode(c.PopupCompletion)
         c.highlighted[QString].connect(self.completer_used)
         c.activated[QString].connect(self.history_selected)
-
+        self.esc_action = QAction(self)
+        self.addAction(self.esc_action)
+        self.esc_action.setShortcut(QKeySequence(Qt.Key_Escape))
+        self.esc_action.triggered.connect(self.esc)
         self.line_edit.key_pressed.connect(self.key_pressed, type=Qt.DirectConnection)
         self.activated.connect(self.history_selected)
         self.setEditable(True)
@@ -91,6 +94,9 @@ class SearchBox2(QComboBox): # {{{
         self._in_a_search = False
         self.tool_tip_text = self.toolTip()
 
+    def esc(self, *args):
+        self.clear()
+
     def initialize(self, opt_name, colorize=False, help_text=_('Search')):
         self.as_you_type = config['search_as_you_type']
         self.opt_name = opt_name

From 6896e2530a8a6b5d864d29df5f82ef7002733d4d Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 23 Jan 2011 09:10:04 -0700
Subject: [PATCH 08/52] Everett Herald by 77jag5. Fixes #8534 (new newsfeed for
 Everett Herald)

---
 resources/recipes/everett_herald.recipe | 36 +++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 resources/recipes/everett_herald.recipe

diff --git a/resources/recipes/everett_herald.recipe b/resources/recipes/everett_herald.recipe
new file mode 100644
index 0000000000..3d91836b48
--- /dev/null
+++ b/resources/recipes/everett_herald.recipe
@@ -0,0 +1,36 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295088390(BasicNewsRecipe):
+     title          = u'Everett Herald'
+     language = 'en'
+     __author__ = '77ja65'
+     oldest_article = 4
+     max_articles_per_feed = 50
+     no_stylesheets = True
+     masthead_url          = 'http://heraldnet.com/images/hnet/jQueryComponents/jQueryNavigation/heraldnet_logo.png'
+     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
+
+     feeds          = [(u'Local News',
+ u'http://heraldnet.com/section/RSS02&mime=xml'),
+         (u'Sports', u'http://heraldnet.com/section/RSS04&mime=xml'),
+         (u'Entertainment',
+ u'http://heraldnet.com/section/RSS07&mime=xml'),
+         (u'Life', u'http://heraldnet.com/section/RSS03&mime=xml'),
+         (u'Breaking News',
+ u'http://heraldnet.com/section/RSS34&mime=xml'),
+         (u'Seahawks', u'http://heraldnet.com/section/RSS22&mime=xml'),
+         (u'HeraldNet', u'http://heraldnet.com/section/RSS01&mime=xml'),
+         (u'Inside Everett',
+ u'http://heraldnet.com/section/RSS26&mime=xml')
+         ]
+
+     def print_version(self, url):
+       return url + "&template=PrinterFriendly"
+
+     extra_css = '''
+                     h1{font-family:Arial,Helvetica,sans-serif; font-
+ weight:bold;font-size:large;}
+                     h2{font-family:Arial,Helvetica,sans-serif; font-
+ weight:normal;font-size:small;}
+                 '''
+

From 1c80e5e4d1618acbf92ec83b572ebcbf79b135fb Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 23 Jan 2011 10:01:08 -0700
Subject: [PATCH 09/52] Bulk metadata edit: Set initial date for the added date
 control to today

---
 src/calibre/gui2/dialogs/metadata_bulk.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 6e6b553dba..2c88556d7b 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -6,8 +6,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
-                     pyqtSignal, QDialogButtonBox
-from PyQt4 import QtGui
+                     pyqtSignal, QDialogButtonBox, QDate, QLineEdit
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -302,6 +301,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.pubdate.setSpecialValueText(_('Undefined'))
         self.clear_pubdate_button.clicked.connect(self.clear_pubdate)
         self.pubdate.dateChanged.connect(self.do_apply_pubdate)
+        self.adddate.setDate(QDate.currentDate())
         self.adddate.setMinimumDate(UNDEFINED_QDATE)
         self.adddate.setSpecialValueText(_('Undefined'))
         self.clear_adddate_button.clicked.connect(self.clear_adddate)
@@ -365,16 +365,16 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         offset = 10
         self.s_r_number_of_books = min(10, len(self.ids))
         for i in range(1,self.s_r_number_of_books+1):
-            w = QtGui.QLabel(self.tabWidgetPage3)
+            w = QLabel(self.tabWidgetPage3)
             w.setText(_('Book %d:')%i)
             self.testgrid.addWidget(w, i+offset, 0, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_text'%i
             setattr(self, name, w)
             self.book_1_text.setObjectName(name)
             self.testgrid.addWidget(w, i+offset, 1, 1, 1)
-            w = QtGui.QLineEdit(self.tabWidgetPage3)
+            w = QLineEdit(self.tabWidgetPage3)
             w.setReadOnly(True)
             name = 'book_%d_result'%i
             setattr(self, name, w)

From 26cd80fe1adea36d9b11a8b6500e99a1bae4eb67 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 23 Jan 2011 18:14:15 +0000
Subject: [PATCH 10/52] Change user category indicator from ':' to '@'

---
 src/calibre/gui2/tag_view.py     | 8 ++++----
 src/calibre/library/database2.py | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 00f34aa171..d68be3b7d6 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -515,8 +515,8 @@ class TagsModel(QAbstractItemModel): # {{{
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
-        # before a QPaintDevice'. The ':' at the end avoids polluting either the
-        # user-defined categories (':' at front) or columns namespaces (no ':').
+        # before a QPaintDevice'. The ':' at the end avoids polluting either of
+        # the other namespaces (alpha, '#', or '@')
         iconmap = {}
         for key in category_icon_map:
             iconmap[key] = QIcon(I(category_icon_map[key]))
@@ -690,7 +690,7 @@ class TagsModel(QAbstractItemModel): # {{{
         tb_cats = self.db.field_metadata
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
-            cat_name = ':' + user_cat # add the ':' to avoid name collision
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
@@ -997,7 +997,7 @@ class TagsModel(QAbstractItemModel): # {{{
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
                 continue
             row_index += 1
-            if key.startswith(':'):
+            if key.startswith('@'):
                 # User category, so skip it. The tag will be marked in its real category
                 continue
             category_item = self.root_item.children[row_index]
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index e0ddcfd7c8..5638bad1ee 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -319,7 +319,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.field_metadata.remove_dynamic_categories()
         tb_cats = self.field_metadata
         for user_cat in sorted(self.prefs.get('user_categories', {}).keys(), key=sort_key):
-            cat_name = ':' + user_cat # add the ':' to avoid name collision
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
             tb_cats.add_user_category(label=cat_name, name=user_cat)
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
@@ -1339,7 +1339,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 if label in taglist and name in taglist[label]:
                     items.append(taglist[label][name])
                 # else: do nothing, to not include nodes w zero counts
-            cat_name = ':' + user_cat # add the ':' to avoid name collision
+            cat_name = '@' + user_cat # add the '@' to avoid name collision
             # Not a problem if we accumulate entries in the icon map
             if icon_map is not None:
                 icon_map[cat_name] = icon_map['user:']

From 2be9281390cb693b4c9a5506801f216c82117938 Mon Sep 17 00:00:00 2001
From: GRiker <griker@hotmail.com>
Date: Sun, 23 Jan 2011 11:39:26 -0700
Subject: [PATCH 11/52] GwR 2nd patch mapping ESC to SearchBox2.clear()

---
 src/calibre/gui2/search_box.py | 8 +-------
 src/calibre/gui2/ui.py         | 9 +++++++--
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 99d435c913..e4073a01c9 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -76,10 +76,7 @@ class SearchBox2(QComboBox): # {{{
         c.setCompletionMode(c.PopupCompletion)
         c.highlighted[QString].connect(self.completer_used)
         c.activated[QString].connect(self.history_selected)
-        self.esc_action = QAction(self)
-        self.addAction(self.esc_action)
-        self.esc_action.setShortcut(QKeySequence(Qt.Key_Escape))
-        self.esc_action.triggered.connect(self.esc)
+
         self.line_edit.key_pressed.connect(self.key_pressed, type=Qt.DirectConnection)
         self.activated.connect(self.history_selected)
         self.setEditable(True)
@@ -94,9 +91,6 @@ class SearchBox2(QComboBox): # {{{
         self._in_a_search = False
         self.tool_tip_text = self.toolTip()
 
-    def esc(self, *args):
-        self.clear()
-
     def initialize(self, opt_name, colorize=False, help_text=_('Search')):
         self.as_you_type = config['search_as_you_type']
         self.opt_name = opt_name
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index b33c059c9b..6a74ccd6ea 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -16,7 +16,7 @@ from PyQt4.Qt import Qt, SIGNAL, QTimer, \
                      QPixmap, QMenu, QIcon, pyqtSignal, \
                      QDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, \
-                     QMessageBox, QHelpEvent
+                     QMessageBox, QHelpEvent, QAction
 
 from calibre import  prints
 from calibre.constants import __appname__, isosx
@@ -198,6 +198,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.system_tray_icon.activated.connect(
             self.system_tray_icon_activated)
 
+        self.esc_action = QAction(self)
+        self.addAction(self.esc_action)
+        self.esc_action.setShortcut(QKeySequence(Qt.Key_Escape))
+        self.esc_action.triggered.connect(self.esc)
 
         ####################### Start spare job server ########################
         QTimer.singleShot(1000, self.add_spare_server)
@@ -294,6 +298,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                         'the file: %s<p>The '
                         'log will be displayed automatically.')%self.gui_debug, show=True)
 
+    def esc(self, *args):
+        self.search.clear()
 
     def start_content_server(self):
         from calibre.library.server.main import start_threaded_server
@@ -305,7 +311,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.content_server.state_callback(True)
         self.test_server_timer = QTimer.singleShot(10000, self.test_server)
 
-
     def resizeEvent(self, ev):
         MainWindow.resizeEvent(self, ev)
         self.search.setMaximumWidth(self.width()-150)

From f202d4b99d8a617442fee7d41ad2c5bd2f29c0c5 Mon Sep 17 00:00:00 2001
From: bulislaw <bulislaw@gmail.com>
Date: Sun, 23 Jan 2011 21:57:48 +0000
Subject: [PATCH 12/52] Add saving/loading search and replace queries

---
 src/calibre/gui2/dialogs/metadata_bulk.py | 119 +++++++++++++++++++++-
 src/calibre/gui2/dialogs/metadata_bulk.ui |   8 +-
 2 files changed, 124 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 6e6b553dba..1f7a8ac16c 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -6,7 +6,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import re, os
 
 from PyQt4.Qt import Qt, QDialog, QGridLayout, QVBoxLayout, QFont, QLabel, \
-                     pyqtSignal, QDialogButtonBox
+                     pyqtSignal, QDialogButtonBox, QInputDialog, QLineEdit, \
+                     QMessageBox
 from PyQt4 import QtGui
 
 from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
@@ -17,7 +18,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
 from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.utils.config import dynamic
+from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import sort_key, capitalize
 from calibre.utils.config import prefs, tweaks
@@ -451,6 +452,18 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
         self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
+        self.save_button.clicked.connect(self.save_query)
+        self.remove_button.clicked.connect(self.remove_query)
+        self.query_field.currentIndexChanged[str].connect(self.query_change)
+
+        self.queries = JSONConfig("queries")
+
+        self.query_field.addItem("")
+        for item in self.queries:
+            self.query_field.addItem(item)
+
+        self.query_field.setCurrentIndex(0)
+
     def s_r_get_field(self, mi, field):
         if field:
             if field == '{template}':
@@ -862,3 +875,105 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
     def series_changed(self, *args):
         self.write_series = True
 
+    def remove_query(self, *args):
+        if not self.query_field.currentText():
+            return
+
+        ret = QMessageBox.question(self, _("Delete query"), 
+                _("Selected query will be deleted. Are You sure?"),
+                QMessageBox.Ok, QMessageBox.Cancel)
+
+        if ret == QMessageBox.Cancel:
+            return
+
+        item_id = self.query_field.currentIndex()
+        item_name = self.query_field.currentText()
+        
+        self.query_field.removeItem(item_id)
+
+        if item_name in self.queries.keys():
+            del(self.queries[str(item_name)])
+            self.queries.commit()
+
+    def save_query(self, *args):
+        query = {}
+        query['name'], ok =  QInputDialog.getText(self, _('Save query'), 
+                _('Query name:'))
+        if not ok:
+            return
+        query['name'] = str(query['name'])
+
+        new = True
+        if query['name'] in self.queries.keys():
+            ret = QMessageBox.question(self, _("Save query"), 
+                    _("Query already exists, old one would be overwritten." \
+                            " Are You sure?"),
+                    QMessageBox.Ok, QMessageBox.Cancel)
+            if ret == QMessageBox.Cancel:
+                return
+            new = False
+
+        query['search_field'] = str(self.search_field.currentText())
+        query['search_mode'] = str(self.search_mode.currentText())
+        query['s_r_template'] = str(self.s_r_template.text())
+        query['search_for'] = str(self.search_for.text())
+        query['case_sensitive'] = self.case_sensitive.isChecked()
+        query['replace_with'] = str(self.replace_with.text())
+        query['replace_func'] = str(self.replace_func.currentText())
+        query['destination_field'] = str(self.destination_field. \
+                currentText())
+        query['replace_mode'] = str(self.replace_mode.currentText())
+        query['comma_separated'] = self.comma_separated.isChecked()
+        query['results_count'] = self.results_count.value()
+        query['starting_from'] = self.starting_from.value()
+        query['multiple_separator'] = str(self.multiple_separator.text())
+
+        self.queries[query['name']] = query
+        self.queries.commit()
+
+        if new:
+            self.query_field.addItem(query['name'])
+        self.query_field.setCurrentIndex(self.query_field.findText(query['name']))
+
+    def query_change(self, item_name):
+        item = self.queries.get(str(item_name), None)
+        if item is None:
+            self.reset_query_fields()
+            return
+
+        self.search_field.setCurrentIndex(
+                self.search_field.findText(item['search_field']))
+        self.search_mode.setEditText(item['search_mode'])
+        self.search_mode.setCurrentIndex(
+                self.search_mode.findText(item['search_mode']))
+        self.s_r_template.setText(item['search_mode'])
+        self.search_for.setText(item['search_for'])
+        self.case_sensitive.setChecked(item['case_sensitive'])
+        self.replace_with.setText(item['replace_with'])
+        self.replace_func.setCurrentIndex(
+                self.replace_func.findText(item['replace_func']))
+        self.destination_field.setCurrentIndex(
+                self.destination_field.findText(item['destination_field']))
+        self.replace_mode.setCurrentIndex(
+                self.replace_mode.findText(item['replace_mode']))
+        self.comma_separated.setChecked(item['comma_separated'])
+        self.results_count.setValue(int(item['results_count']))
+        self.starting_from.setValue(int(item['starting_from']))
+        self.multiple_separator.setText(item['multiple_separator'])
+
+    def reset_query_fields(self):
+        self.search_field.setCurrentIndex(0)
+        self.search_mode.setEditText("")
+        self.search_mode.setCurrentIndex(0)
+        self.s_r_template.setText("")
+        self.search_for.setText("")
+        self.case_sensitive.setChecked(False)
+        self.replace_with.setText("")
+        self.replace_func.setCurrentIndex(0)
+        self.destination_field.setCurrentIndex(0)
+        self.replace_mode.setCurrentIndex(0)
+        self.comma_separated.setChecked(True)
+        self.results_count.setValue(999)
+        self.starting_from.setValue(1)
+        self.multiple_separator.setText(" ::: ")
+
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index a1e1d8c550..04bf9e364c 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -981,7 +981,7 @@ not multiple and the destination field is multiple</string>
            <item row="3" column="2">
             <widget class="QComboBox" name="query_field">
              <property name="toolTip">
-              <string>The name of the field that you want to search</string>
+              <string>Select query to load.</string>
              </property>
             </widget>
            </item>
@@ -1012,6 +1012,9 @@ not multiple and the destination field is multiple</string>
              </item>
              <item>
               <widget class="QPushButton" name="save_button">
+               <property name="toolTip">
+                <string>Save current query.</string>
+               </property>
                <property name="text">
                 <string>Save</string>
                </property>
@@ -1019,6 +1022,9 @@ not multiple and the destination field is multiple</string>
              </item>
              <item>
               <widget class="QPushButton" name="remove_button">
+               <property name="toolTip">
+                <string>Remove loadded query.</string>
+               </property>
                <property name="text">
                 <string>Remove</string>
                </property>

From f95cc7eddc02153cc280e7f93dab695b388e7a86 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 23 Jan 2011 16:58:57 -0700
Subject: [PATCH 13/52] Heuristics, italicize common cases: Enhance pattern
 matching to match punctuation after pattern.

---
 src/calibre/ebooks/conversion/utils.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index aabb1b8bc4..ad7f5f117d 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -137,17 +137,17 @@ class HeuristicProcessor(object):
         ]
 
         ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=\s)',
-            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=\s)',
-            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=\s)',
-            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=\s)',
-            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=\s)',
-            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=\s)',
-            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=\s)',
-            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=\s)',
-            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=\s)',
-            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=\s)',
-            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=\s)',
+            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])',
+            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])',
         ]
 
         for word in ITALICIZE_WORDS:

From 6e64f5ec4e0cb66cc0bf722ecf050f72fb9120dc Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 12:20:50 +0800
Subject: [PATCH 14/52] doc tweaks, delete empty paragraphs during Heuristics

---
 src/calibre/ebooks/conversion/utils.py | 2 ++
 src/calibre/manual/conversion.rst      | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index aabb1b8bc4..14eca46b07 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -367,6 +367,8 @@ class HeuristicProcessor(object):
         html = re.sub(ur'\s*<o:p>\s*</o:p>', ' ', html)
         # Delete microsoft 'smart' tags
         html = re.sub('(?i)</?st1:\w+>', '', html)
+        # Delete self closing paragraph tags
+        html = re.sub('<p\s?/>', '', html)
         # Get rid of empty span, bold, font, em, & italics tags
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*</(font|[ibu]|em)>\s*){0,2}\s*</(font|[ibu]|em)>", " ", html)
diff --git a/src/calibre/manual/conversion.rst b/src/calibre/manual/conversion.rst
index 6ec986f26a..7f3ff21fe0 100644
--- a/src/calibre/manual/conversion.rst
+++ b/src/calibre/manual/conversion.rst
@@ -587,11 +587,11 @@ TXT input supports a number of options to differentiate how paragraphs are detec
         Assumes that every paragraph starts with an indent (either a tab or 2+ spaces). Paragraphs end when
         the next line that starts with an indent is reached::
 
-            This is the
+              This is the
             first.
-            This is the second.
+              This is the second.
             
-            This is the
+              This is the
             third.
 
     :guilabel:`Paragraph Style: Unformatted`
@@ -603,7 +603,7 @@ TXT input supports a number of options to differentiate how paragraphs are detec
         formatting will be applied.
 
     :guilabel:`Formatting Style: Heuristic`
-        Analyses the document for common chapter headings, scene breaks, and italicized words and applies the
+        Analyzes the document for common chapter headings, scene breaks, and italicized words and applies the
         appropriate html markup during conversion.
 
     :guilabel:`Formatting Style: Markdown`

From 1e675af91aa665334261dafdc50f89724d2bf4f4 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 12:22:42 +0800
Subject: [PATCH 15/52] Always apply title case to downloaded metadata - tired
 of crap metadata sources with 99% lowercase characters

---
 src/calibre/ebooks/metadata/fetch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py
index 390f288d8e..8018f42b13 100644
--- a/src/calibre/ebooks/metadata/fetch.py
+++ b/src/calibre/ebooks/metadata/fetch.py
@@ -411,7 +411,7 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
                     r.pubdate = pubdate
 
     def fix_case(x):
-        if x and x.isupper():
+        if x:
             x = titlecase(x)
         return x
 

From a20cd3336dc5a309543dd5134a4f85bf563b9b09 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 13:47:53 +0800
Subject: [PATCH 16/52] improved negative lookaheads in heuristics

---
 src/calibre/ebooks/conversion/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index e8c2acb9d6..22639801ff 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -203,8 +203,8 @@ class HeuristicProcessor(object):
             blank_lines = ""
         opt_title_open = "("
         opt_title_close = ")?"
-        n_lookahead_open = "\s+(?!"
-        n_lookahead_close = ")"
+        n_lookahead_open = "(?!\s*"
+        n_lookahead_close = ")\s*"
 
         default_title = r"(<[ibu][^>]*>)?\s{0,3}(?!Chapter)([\w\:\'’\"-]+\s{0,3}){1,5}?(</[ibu][^>]*>)?(?=<)"
         simple_title = r"(<[ibu][^>]*>)?\s{0,3}(?!(Chapter|\s+<)).{0,65}?(</[ibu][^>]*>)?(?=<)"

From d03ae9e001165d6edc82fbdbf884b89cd0b35ed9 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 14:20:48 +0800
Subject: [PATCH 17/52] fixed a regression in chapter markup logic which was
 causing false negatives

---
 src/calibre/ebooks/conversion/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 22639801ff..1eb063cdd8 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -215,7 +215,7 @@ class HeuristicProcessor(object):
             [r"[^'\"]?(Introduction|Synopsis|Acknowledgements|Epilogue|CHAPTER|Kapitel|Volume\b|Prologue|Book\b|Part\b|Dedication|Preface)\s*([\d\w-]+\:?\'?\s*){0,5}", True, True, True, False, "Searching for common section headings", 'common'],
             [r"[^'\"]?(CHAPTER|Kapitel)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for most common chapter headings", 'chapter'],  # Highest frequency headings which include titles
             [r"<b[^>]*>\s*(<span[^>]*>)?\s*(?!([*#•=]+\s*)+)(\s*(?=[\d.\w#\-*\s]+<)([\d.\w#-*]+\s*){1,5}\s*)(?!\.)(</span>)?\s*</b>", True, True, True, False, "Searching for emphasized lines", 'emphasized'], # Emphasized lines
-            [r"[^'\"]?(\d+(\.|:))\s*([\dA-Z\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
+            [r"[^'\"]?(\d+(\.|:))\s*([\w\-\'\"#,]+\s*){0,7}\s*", True, True, True, False, "Searching for numeric chapter headings", 'numeric'],  # Numeric Chapters
             [r"([A-Z]\s+){3,}\s*([\d\w-]+\s*){0,3}\s*", True, True, True, False, "Searching for letter spaced headings", 'letter_spaced'],  # Spaced Lettering
             [r"[^'\"]?(\d+\.?\s+([\d\w-]+\:?\'?-?\s?){0,5})\s*", True, True, True, False, "Searching for numeric chapters with titles", 'numeric_title'], # Numeric Titles
             [r"[^'\"]?(\d+)\s*([\dA-Z\-\'\"\?!#,]+\s*){0,7}\s*", True, True, True, False, "Searching for simple numeric headings", 'plain_number'],  # Numeric Chapters, no dot or colon
@@ -275,7 +275,7 @@ class HeuristicProcessor(object):
                         self.log.debug(unicode(type_name)+" had "+unicode(hits)+" hits - "+unicode(self.chapters_no_title)+" chapters with no title, "+unicode(self.chapters_with_title)+" chapters with titles, "+unicode(float(self.chapters_with_title) / float(hits))+" percent. ")
                         if type_name == 'common':
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
-                        elif self.min_chapters <= hits < max_chapters:
+                        elif self.min_chapters <= hits < max_chapters or self.min_chapters < 3 > hits:
                             analysis_result.append([chapter_type, n_lookahead_req, strict_title, ignorecase, title_req, log_message, type_name])
                             break
                 else:

From 670fc644edb235b4a1fed74ded7924a2b8c13ec4 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 16:00:05 +0800
Subject: [PATCH 18/52] made chapter markup routine more Sigil friendly

---
 src/calibre/ebooks/conversion/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 1eb063cdd8..d9350e6adb 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -43,7 +43,7 @@ class HeuristicProcessor(object):
             self.html_preprocess_sections = self.html_preprocess_sections + 1
             self.log.debug("marked " + unicode(self.html_preprocess_sections) +
                     " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
-            return '<h2>'+chap+'</h2>\n<h3>'+title+'</h3>\n'
+            return '<h2 title="'+chap+', '+title+'">'+chap+'</h2>\n<h3 class="sigilNotInTOC">'+title+'</h3>\n'
 
     def chapter_break(self, match):
         chap = match.group('section')

From 7485c9a5e200c20285ead3795025781f4d9ef31e Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Mon, 24 Jan 2011 16:30:29 +0800
Subject: [PATCH 19/52] fixed sigil integration to strip html from chapter
 titles, fixed softbreak handling and enabled integration with extra_css

---
 src/calibre/ebooks/conversion/utils.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index d9350e6adb..f6e259b6f9 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -25,13 +25,15 @@ class HeuristicProcessor(object):
         self.chapters_with_title = 0
         self.blanks_deleted = False
         self.linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
-        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sid=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.blankreg = re.compile(r'\s*(?P<openline><p(?!\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        self.softbreak = re.compile(r'\s*(?P<openline><p(?=\sclass=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         self.multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
 
     def is_pdftohtml(self, src):
         return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
 
     def chapter_head(self, match):
+        from calibre.utils.html2text import html2text
         chap = match.group('chap')
         title = match.group('title')
         if not title:
@@ -40,10 +42,12 @@ class HeuristicProcessor(object):
                     " chapters. - " + unicode(chap))
             return '<h2>'+chap+'</h2>\n'
         else:
+            txt_chap = html2text(chap)
+            txt_title = html2text(title)
             self.html_preprocess_sections = self.html_preprocess_sections + 1
             self.log.debug("marked " + unicode(self.html_preprocess_sections) +
                     " chapters & titles. - " + unicode(chap) + ", " + unicode(title))
-            return '<h2 title="'+chap+', '+title+'">'+chap+'</h2>\n<h3 class="sigilNotInTOC">'+title+'</h3>\n'
+            return '<h2 title="'+txt_chap+', '+txt_title+'">'+chap+'</h2>\n<h3 class="sigilNotInTOC">'+title+'</h3>\n'
 
     def chapter_break(self, match):
         chap = match.group('section')
@@ -469,7 +473,7 @@ class HeuristicProcessor(object):
         if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
             self.log.debug("deleting blank lines")
             self.blanks_deleted = True
-            html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
             html = self.blankreg.sub('', html)
 
         # Determine line ending type
@@ -524,11 +528,11 @@ class HeuristicProcessor(object):
             # Center separator lines
             html = re.sub(u'<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*(?P<break>([*#•=✦]+\s*)+)\s*(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>', '<p style="text-align:center; margin-top:1.25em; margin-bottom:1.25em">' + '\g<break>' + '</p>', html)
             if not self.blanks_deleted:
-                html = self.multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
-            html = re.sub('<p\s+id="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
+                html = self.multi_blank.sub('\n<p class="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = re.sub('<p\s+class="softbreak"[^>]*>\s*</p>', '<div id="softbreak" style="margin-left: 45%; margin-right: 45%; margin-top:1.5em; margin-bottom:1.5em"><hr style="height: 3px; background:#505050" /></div>', html)
 
         if self.deleted_nbsps:
             # put back non-breaking spaces in empty paragraphs to preserve original formatting
             html = self.blankreg.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
-
+            html = self.softbreak.sub('\n'+r'\g<openline>'+u'\u00a0'+r'\g<closeline>', html)
         return html

From 65e6102e3317d3e20d118837eff41efbd3e28b2e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 24 Jan 2011 09:15:05 +0000
Subject: [PATCH 20/52] Add right-click category search support to the tags
 browser

---
 src/calibre/gui2/tag_view.py          | 14 +++++++++++++-
 src/calibre/library/caches.py         | 25 ++++++++++++++++++++++++-
 src/calibre/library/database2.py      |  2 +-
 src/calibre/library/field_metadata.py | 11 +++++++----
 4 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index d68be3b7d6..8b574948ff 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -186,7 +186,7 @@ class TagsView(QTreeView): # {{{
             self.clear()
 
     def context_menu_handler(self, action=None, category=None,
-                             key=None, index=None):
+                             key=None, index=None, negate=None):
         if not action:
             return
         try:
@@ -199,6 +199,9 @@ class TagsView(QTreeView): # {{{
             if action == 'manage_categories':
                 self.user_category_edit.emit(category)
                 return
+            if action == 'search_category':
+                self.tags_marked.emit(category + ':' + str(not negate))
+                return
             if action == 'manage_searches':
                 self.saved_search_edit.emit(category)
                 return
@@ -268,6 +271,15 @@ class TagsView(QTreeView): # {{{
                         m.addAction(col,
                             partial(self.context_menu_handler, action='show', category=col))
 
+                # search by category
+                self.context_menu.addAction(
+                        _('Search for books in category %s')%category,
+                        partial(self.context_menu_handler, action='search_category',
+                                category=key, negate=False))
+                self.context_menu.addAction(
+                        _('Search for books not in category %s')%category,
+                        partial(self.context_menu_handler, action='search_category',
+                                category=key, negate=True))
                 # Offer specific editors for tags/series/publishers/saved searches
                 self.context_menu.addSeparator()
                 if key in ['tags', 'publisher', 'series'] or \
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 291d71f572..9f10d9f890 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -172,8 +172,9 @@ class ResultCache(SearchQueryParser): # {{{
     '''
     Stores sorted and filtered metadata in memory.
     '''
-    def __init__(self, FIELD_MAP, field_metadata):
+    def __init__(self, FIELD_MAP, field_metadata, db_prefs = None):
         self.FIELD_MAP = FIELD_MAP
+        self.db_prefs = db_prefs
         self.composites = {}
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
@@ -405,6 +406,22 @@ class ResultCache(SearchQueryParser): # {{{
                 matches.add(item[0])
         return matches
 
+    def get_user_category_matches(self, location, query, candidates):
+        res = set([])
+        if self.db_prefs is None:
+            return  res
+        user_cats = self.db_prefs['user_categories']
+        if location not in user_cats:
+            return res
+        c = set(candidates)
+        for (item, category, ign) in user_cats[location]:
+            s = self.get_matches(category, '=' + item, candidates=c)
+            c -= s
+            res |= s
+        if query == 'false':
+            return candidates - res
+        return res
+
     def get_matches(self, location, query, allow_recursion=True, candidates=None):
         matches = set([])
         if candidates is None:
@@ -443,6 +460,10 @@ class ResultCache(SearchQueryParser): # {{{
                     return self.get_numeric_matches(location, query[1:],
                                                     candidates, val_func=vf)
 
+            # check for user categories
+            if len(location) >= 2 and location.startswith('@'):
+                return self.get_user_category_matches(location[1:], query.lower(),
+                                                      candidates)
             # everything else, or 'all' matches
             matchkind = CONTAINS_MATCH
             if (len(query) > 1):
@@ -468,6 +489,8 @@ class ResultCache(SearchQueryParser): # {{{
             for x in range(len(self.FIELD_MAP)):
                 col_datatype.append('')
             for x in self.field_metadata:
+                if x.startswith('@'):
+                    continue
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
                     if self.field_metadata[x]['datatype'] not in \
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 5638bad1ee..f2b2c94e31 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -332,7 +332,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 traceback.print_exc()
 
         self.book_on_device_func = None
-        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata)
+        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata, db_prefs=self.prefs)
         self.search  = self.data.search
         self.search_getting_ids  = self.data.search_getting_ids
         self.refresh = functools.partial(self.data.refresh, self)
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index e9402d1227..78fe899fa8 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -475,6 +475,8 @@ class FieldMetadata(dict):
             val = self._tb_cats[key]
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
+                if key in self._search_term_map:
+                    del self._search_term_map[key]
 
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1
@@ -482,11 +484,12 @@ class FieldMetadata(dict):
     def add_user_category(self, label, name):
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
-        self._tb_cats[label] = {'table':None,        'column':None,
-                                'datatype':None,     'is_multiple':None,
-                                'kind':'user',       'name':name,
-                                'search_terms':[],    'is_custom':False,
+        self._tb_cats[label] = {'table':None,          'column':None,
+                                'datatype':None,       'is_multiple':None,
+                                'kind':'user',         'name':name,
+                                'search_terms':[label],'is_custom':False,
                                 'is_category':True}
+        self._add_search_terms_to_map(label, [label])
 
     def add_search_category(self, label, name):
         if label in self._tb_cats:

From 2012f5cc0a28a322b41ee43bfe6bea579f0a8773 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 24 Jan 2011 11:15:57 +0000
Subject: [PATCH 21/52] Saved queries after review

---
 src/calibre/gui2/dialogs/metadata_bulk.py | 119 +++++++-------
 src/calibre/gui2/dialogs/metadata_bulk.ui | 190 +++++++++++-----------
 2 files changed, 159 insertions(+), 150 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index dc801a048e..3944bdbe18 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -452,16 +452,13 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.results_count.valueChanged[int].connect(self.s_r_display_bounds_changed)
         self.starting_from.valueChanged[int].connect(self.s_r_display_bounds_changed)
 
-        self.save_button.clicked.connect(self.save_query)
-        self.remove_button.clicked.connect(self.remove_query)
-        self.query_field.currentIndexChanged[str].connect(self.query_change)
-
-        self.queries = JSONConfig("queries")
+        self.save_button.clicked.connect(self.s_r_save_query)
+        self.remove_button.clicked.connect(self.s_r_remove_query)
 
+        self.queries = JSONConfig("search_replace_queries")
         self.query_field.addItem("")
-        for item in self.queries:
-            self.query_field.addItem(item)
-
+        self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+        self.query_field.currentIndexChanged[str].connect(self.s_r_query_change)
         self.query_field.setCurrentIndex(0)
 
     def s_r_get_field(self, mi, field):
@@ -875,96 +872,108 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
     def series_changed(self, *args):
         self.write_series = True
 
-    def remove_query(self, *args):
-        if not self.query_field.currentText():
+    def s_r_remove_query(self, *args):
+        if self.query_field.currentIndex() == 0:
             return
 
-        ret = QMessageBox.question(self, _("Delete query"), 
-                _("Selected query will be deleted. Are You sure?"),
+        ret = QMessageBox.question(self, _("Delete saved search/replace"),
+                _("The selected saved search/replace will be deleted. "
+                  "Are you sure?"),
                 QMessageBox.Ok, QMessageBox.Cancel)
 
         if ret == QMessageBox.Cancel:
             return
 
         item_id = self.query_field.currentIndex()
-        item_name = self.query_field.currentText()
-        
+        item_name = unicode(self.query_field.currentText())
+
+        self.query_field.blockSignals(True)
         self.query_field.removeItem(item_id)
+        self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(0)
 
         if item_name in self.queries.keys():
-            del(self.queries[str(item_name)])
+            del(self.queries[item_name])
             self.queries.commit()
 
-    def save_query(self, *args):
-        query = {}
-        query['name'], ok =  QInputDialog.getText(self, _('Save query'), 
-                _('Query name:'))
+    def s_r_save_query(self, *args):
+        name, ok =  QInputDialog.getText(self, _('Save search/replace'),
+                _('Search/replace name:'))
         if not ok:
             return
-        query['name'] = str(query['name'])
 
         new = True
-        if query['name'] in self.queries.keys():
-            ret = QMessageBox.question(self, _("Save query"), 
-                    _("Query already exists, old one would be overwritten." \
-                            " Are You sure?"),
+        name = unicode(name)
+        if name in self.queries.keys():
+            ret = QMessageBox.question(self, _("Save search/replace"),
+                    _("That saved search/replace already exists and will be overwritten. "
+                      "Are you sure?"),
                     QMessageBox.Ok, QMessageBox.Cancel)
             if ret == QMessageBox.Cancel:
                 return
             new = False
 
-        query['search_field'] = str(self.search_field.currentText())
-        query['search_mode'] = str(self.search_mode.currentText())
-        query['s_r_template'] = str(self.s_r_template.text())
-        query['search_for'] = str(self.search_for.text())
+        query = {}
+        query['name'] = name
+        query['search_field'] = unicode(self.search_field.currentText())
+        query['search_mode'] = unicode(self.search_mode.currentText())
+        query['s_r_template'] = unicode(self.s_r_template.text())
+        query['search_for'] = unicode(self.search_for.text())
         query['case_sensitive'] = self.case_sensitive.isChecked()
-        query['replace_with'] = str(self.replace_with.text())
-        query['replace_func'] = str(self.replace_func.currentText())
-        query['destination_field'] = str(self.destination_field. \
-                currentText())
-        query['replace_mode'] = str(self.replace_mode.currentText())
+        query['replace_with'] = unicode(self.replace_with.text())
+        query['replace_func'] = unicode(self.replace_func.currentText())
+        query['destination_field'] = unicode(self.destination_field.currentText())
+        query['replace_mode'] = unicode(self.replace_mode.currentText())
         query['comma_separated'] = self.comma_separated.isChecked()
         query['results_count'] = self.results_count.value()
         query['starting_from'] = self.starting_from.value()
-        query['multiple_separator'] = str(self.multiple_separator.text())
+        query['multiple_separator'] = unicode(self.multiple_separator.text())
 
-        self.queries[query['name']] = query
+        self.queries[name] = query
         self.queries.commit()
 
         if new:
-            self.query_field.addItem(query['name'])
-        self.query_field.setCurrentIndex(self.query_field.findText(query['name']))
+            self.query_field.blockSignals(True)
+            self.query_field.clear()
+            self.query_field.addItem('')
+            self.query_field.addItems(sorted([q for q in self.queries], key=sort_key))
+            self.query_field.blockSignals(False)
+        self.query_field.setCurrentIndex(self.query_field.findText(name))
 
-    def query_change(self, item_name):
-        item = self.queries.get(str(item_name), None)
+    def s_r_query_change(self, item_name):
+        if not item_name:
+            self.s_r_reset_query_fields()
+            return
+        item = self.queries.get(unicode(item_name), None)
         if item is None:
-            self.reset_query_fields()
+            self.s_r_reset_query_fields()
             return
 
-        self.search_field.setCurrentIndex(
-                self.search_field.findText(item['search_field']))
-        self.search_mode.setEditText(item['search_mode'])
-        self.search_mode.setCurrentIndex(
-                self.search_mode.findText(item['search_mode']))
-        self.s_r_template.setText(item['search_mode'])
+        def set_index(attr, txt):
+            try:
+                attr.setCurrentIndex(attr.findText(txt))
+            except:
+                attr.setCurrentIndex(0)
+
+        set_index(self.search_mode, item['search_mode'])
+        set_index(self.search_field, item['search_field'])
+        self.s_r_template.setText(item['s_r_template'])
+        self.s_r_template_changed() #simulate gain/loss of focus
         self.search_for.setText(item['search_for'])
         self.case_sensitive.setChecked(item['case_sensitive'])
         self.replace_with.setText(item['replace_with'])
-        self.replace_func.setCurrentIndex(
-                self.replace_func.findText(item['replace_func']))
-        self.destination_field.setCurrentIndex(
-                self.destination_field.findText(item['destination_field']))
-        self.replace_mode.setCurrentIndex(
-                self.replace_mode.findText(item['replace_mode']))
+        set_index(self.replace_func, item['replace_func'])
+        set_index(self.destination_field, item['destination_field'])
+        set_index(self.replace_mode, item['replace_mode'])
         self.comma_separated.setChecked(item['comma_separated'])
         self.results_count.setValue(int(item['results_count']))
         self.starting_from.setValue(int(item['starting_from']))
         self.multiple_separator.setText(item['multiple_separator'])
 
-    def reset_query_fields(self):
+    def s_r_reset_query_fields(self):
+        # Don't reset the search mode. The user will probably want to use it
+        # as it was
         self.search_field.setCurrentIndex(0)
-        self.search_mode.setEditText("")
-        self.search_mode.setCurrentIndex(0)
         self.s_r_template.setText("")
         self.search_for.setText("")
         self.case_sensitive.setChecked(False)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index 04bf9e364c..3d64a89bb8 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -574,7 +574,7 @@ Future conversion of these books will use the default settings.</string>
            <property name="sizeConstraint">
             <enum>QLayout::SetMinimumSize</enum>
            </property>
-           <item row="1" column="0" colspan="4">
+           <item row="0" column="0" colspan="4">
             <widget class="QLabel" name="s_r_heading">
              <property name="wordWrap">
               <bool>true</bool>
@@ -584,14 +584,91 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="2" column="0">
+           <item row="1" column="0">
             <widget class="QLabel" name="filler">
              <property name="text">
               <string/>
              </property>
             </widget>
            </item>
-           <item row="11" column="0">
+           <item row="2" column="0" colspan="3">
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="0">
+            <widget class="QLabel" name="xlabel_22">
+             <property name="text">
+              <string>Load searc&amp;h/replace:</string>
+             </property>
+             <property name="buddy">
+              <cstring>search_field</cstring>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="1">
+            <widget class="QComboBox" name="query_field">
+             <property name="toolTip">
+              <string>Select saved search/replace to load.</string>
+             </property>
+            </widget>
+           </item>
+           <item row="3" column="2">
+            <layout class="QHBoxLayout" name="horizontalLayout_6">
+             <item>
+              <spacer name="horizontalSpacer_3">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeType">
+                <enum>QSizePolicy::Fixed</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>20</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+             <item>
+              <widget class="QPushButton" name="save_button">
+               <property name="toolTip">
+                <string>Save current search/replace</string>
+               </property>
+               <property name="text">
+                <string>Sa&amp;ve</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QPushButton" name="remove_button">
+               <property name="toolTip">
+                <string>Delete saved search/replace</string>
+               </property>
+               <property name="text">
+                <string>Delete</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer_2">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>20</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
+           </item>
+           <item row="4" column="0">
             <widget class="QLabel" name="xlabel_21">
              <property name="text">
               <string>Search &amp;field:</string>
@@ -601,14 +678,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="11" column="2">
+           <item row="4" column="1">
             <widget class="QComboBox" name="search_field">
              <property name="toolTip">
               <string>The name of the field that you want to search</string>
              </property>
             </widget>
            </item>
-           <item row="11" column="3">
+           <item row="4" column="2">
             <layout class="QHBoxLayout" name="HLayout_3">
              <item>
               <widget class="QLabel" name="xlabel_24">
@@ -642,7 +719,7 @@ Future conversion of these books will use the default settings.</string>
              </item>
             </layout>
            </item>
-           <item row="12" column="0">
+           <item row="5" column="0">
             <widget class="QLabel" name="template_label">
              <property name="text">
               <string>Te&amp;mplate:</string>
@@ -652,7 +729,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="12" column="2">
+           <item row="5" column="1">
             <widget class="HistoryLineEdit" name="s_r_template">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -665,7 +742,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="13" column="0">
+           <item row="6" column="0">
             <widget class="QLabel" name="xlabel_2">
              <property name="text">
               <string>&amp;Search for:</string>
@@ -675,7 +752,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="13" column="2">
+           <item row="6" column="1">
             <widget class="HistoryLineEdit" name="search_for">
              <property name="sizePolicy">
               <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
@@ -688,7 +765,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="13" column="3">
+           <item row="6" column="2">
             <widget class="QCheckBox" name="case_sensitive">
              <property name="toolTip">
               <string>Check this box if the search string must match exactly upper and lower case. Uncheck it if case is to be ignored</string>
@@ -701,7 +778,7 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="14" column="0">
+           <item row="7" column="0">
             <widget class="QLabel" name="xlabel_4">
              <property name="text">
               <string>&amp;Replace with:</string>
@@ -711,14 +788,14 @@ Future conversion of these books will use the default settings.</string>
              </property>
             </widget>
            </item>
-           <item row="14" column="2">
+           <item row="7" column="1">
             <widget class="HistoryLineEdit" name="replace_with">
              <property name="toolTip">
               <string>The replacement text. The matched search text will be replaced with this string</string>
              </property>
             </widget>
            </item>
-           <item row="14" column="3">
+           <item row="7" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="label_41">
@@ -753,7 +830,7 @@ field is processed. In regular expression mode, only the matched text is process
              </item>
             </layout>
            </item>
-           <item row="15" column="0">
+           <item row="8" column="0">
             <widget class="QLabel" name="destination_field_label">
              <property name="text">
               <string>&amp;Destination field:</string>
@@ -763,7 +840,7 @@ field is processed. In regular expression mode, only the matched text is process
              </property>
             </widget>
            </item>
-           <item row="15" column="2">
+           <item row="8" column="1">
             <widget class="QComboBox" name="destination_field">
              <property name="toolTip">
               <string>The field that the text will be put into after all replacements.
@@ -771,7 +848,7 @@ If blank, the source field is used if the field is modifiable</string>
              </property>
             </widget>
            </item>
-           <item row="15" column="3">
+           <item row="8" column="2">
             <layout class="QHBoxLayout" name="verticalLayout">
              <item>
               <widget class="QLabel" name="replace_mode_label">
@@ -820,7 +897,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="16" column="2" colspan="2">
+           <item row="9" column="1" colspan="2">
             <layout class="QHBoxLayout" name="horizontalLayout_21">
              <item>
               <spacer name="HSpacer_347">
@@ -906,7 +983,7 @@ not multiple and the destination field is multiple</string>
              </item>
             </layout>
            </item>
-           <item row="17" column="0" colspan="5">
+           <item row="10" column="0" colspan="4">
             <widget class="QScrollArea" name="scrollArea11">
              <property name="frameShape">
               <enum>QFrame::NoFrame</enum>
@@ -968,83 +1045,6 @@ not multiple and the destination field is multiple</string>
              </widget>
             </widget>
            </item>
-           <item row="3" column="0">
-            <widget class="QLabel" name="xlabel_22">
-             <property name="text">
-              <string>Load query:</string>
-             </property>
-             <property name="buddy">
-              <cstring>search_field</cstring>
-             </property>
-            </widget>
-           </item>
-           <item row="3" column="2">
-            <widget class="QComboBox" name="query_field">
-             <property name="toolTip">
-              <string>Select query to load.</string>
-             </property>
-            </widget>
-           </item>
-           <item row="4" column="0" colspan="4">
-            <widget class="Line" name="line">
-             <property name="orientation">
-              <enum>Qt::Horizontal</enum>
-             </property>
-            </widget>
-           </item>
-           <item row="3" column="3">
-            <layout class="QHBoxLayout" name="horizontalLayout_6">
-             <item>
-              <spacer name="horizontalSpacer_3">
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
-               </property>
-               <property name="sizeType">
-                <enum>QSizePolicy::Fixed</enum>
-               </property>
-               <property name="sizeHint" stdset="0">
-                <size>
-                 <width>20</width>
-                 <height>20</height>
-                </size>
-               </property>
-              </spacer>
-             </item>
-             <item>
-              <widget class="QPushButton" name="save_button">
-               <property name="toolTip">
-                <string>Save current query.</string>
-               </property>
-               <property name="text">
-                <string>Save</string>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <widget class="QPushButton" name="remove_button">
-               <property name="toolTip">
-                <string>Remove loadded query.</string>
-               </property>
-               <property name="text">
-                <string>Remove</string>
-               </property>
-              </widget>
-             </item>
-             <item>
-              <spacer name="horizontalSpacer_2">
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
-               </property>
-               <property name="sizeHint" stdset="0">
-                <size>
-                 <width>40</width>
-                 <height>20</height>
-                </size>
-               </property>
-              </spacer>
-             </item>
-            </layout>
-           </item>
           </layout>
          </widget>
         </widget>

From 45ee4769896363190af6a2ab00dcdc9597de140c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 09:22:43 -0700
Subject: [PATCH 22/52] Updated recipes for Heise, HNA and ZDNET

---
 resources/recipes/heise.recipe |  2 ++
 resources/recipes/hna.recipe   |  3 ++-
 resources/recipes/zdnet.recipe | 25 ++++++++++++++++++++++++-
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/resources/recipes/heise.recipe b/resources/recipes/heise.recipe
index 9edf3774fc..56d5516656 100644
--- a/resources/recipes/heise.recipe
+++ b/resources/recipes/heise.recipe
@@ -52,6 +52,7 @@ class heiseDe(BasicNewsRecipe):
 		   dict(id='navi_login'),
 		   dict(id='navigation'),
 		   dict(id='breadcrumb'),
+		   dict(id='adblockerwarnung'),
 		   dict(id=''),
 		   dict(id='sitemap'),
 		   dict(id='bannerzone'),
@@ -67,3 +68,4 @@ class heiseDe(BasicNewsRecipe):
 
 
 
+
diff --git a/resources/recipes/hna.recipe b/resources/recipes/hna.recipe
index 6e843800ee..e3349f0c7b 100644
--- a/resources/recipes/hna.recipe
+++ b/resources/recipes/hna.recipe
@@ -21,7 +21,7 @@ class hnaDe(BasicNewsRecipe):
     max_articles_per_feed = 40
     no_stylesheets = True
     remove_javascript = True
-    encoding = 'iso-8859-1'
+    encoding = 'utf-8'
 
     remove_tags = [dict(id='topnav'),
 		   dict(id='nav_main'),
@@ -60,3 +60,4 @@ class hnaDe(BasicNewsRecipe):
     feeds =  [ ('hna_soehre', 'http://feeds2.feedburner.com/hna/soehre'),
 	       ('hna_kassel', 'http://feeds2.feedburner.com/hna/kassel') ]
 
+
diff --git a/resources/recipes/zdnet.recipe b/resources/recipes/zdnet.recipe
index 9673eb1fcf..1a0f1562b5 100644
--- a/resources/recipes/zdnet.recipe
+++ b/resources/recipes/zdnet.recipe
@@ -27,12 +27,34 @@ class cdnet(BasicNewsRecipe):
 		   dict(id='header'),
 		   dict(id='search'),
 		   dict(id='nav'),
+		   dict(id='blog-author-info'),
+		   dict(id='post-tags'),
+		   dict(id='bio-naraine'),
+		   dict(id='bio-kennedy'),
+		   dict(id='author-short-disclosure-kennedy'),
 		   dict(id=''),
 		   dict(name='div', attrs={'class':'banner'}),
+		   dict(name='div', attrs={'class':'int'}),
+		   dict(name='div', attrs={'class':'talkback clear space-2'}),
+		   dict(name='div', attrs={'class':'content-1 clear'}),
+		   dict(name='div', attrs={'class':'space-2'}),
+		   dict(name='div', attrs={'class':'space-3'}),
+		   dict(name='div', attrs={'class':'thumb-2 left'}),
+		   dict(name='div', attrs={'class':'hotspot'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'view-1 clear content-3 space-2'}),
+		   dict(name='div', attrs={'class':'hed hed-1 space-1'}),
+		   dict(name='div', attrs={'class':'hed hed-1'}),
+		   dict(name='div', attrs={'class':'post-header'}),
+		   dict(name='div', attrs={'class':'lvl-nav clear'}),
+		   dict(name='div', attrs={'class':'t-share-overlay overlay-pop contain-overlay-4'}),
 		   dict(name='p', attrs={'class':'tags'}),
+		   dict(name='span', attrs={'class':'follow'}),
+		   dict(name='span', attrs={'class':'int'}),
+		   dict(name='h4', attrs={'class':'h s-4'}),
 		   dict(name='a', attrs={'href':'http://www.twitter.com/ryanaraine'}),
 		   dict(name='div', attrs={'class':'special1'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'bloggerDesc clear'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'clear'})]
 
     feeds =  [ ('zdnet', 'http://feeds.feedburner.com/zdnet/security') ]
 
@@ -43,3 +65,4 @@ class cdnet(BasicNewsRecipe):
         return soup
 
 
+

From ffb62ece0e975a526ad60b14a5e7ab46f413b8d3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 09:32:23 -0700
Subject: [PATCH 23/52] Fix #8031 (Output Options)

---
 src/calibre/gui2/widgets.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py
index 9e117822e4..8077fe46f5 100644
--- a/src/calibre/gui2/widgets.py
+++ b/src/calibre/gui2/widgets.py
@@ -16,7 +16,6 @@ from PyQt4.Qt import QIcon, QFont, QLabel, QListWidget, QAction, \
                         QTimer, QRect
 
 from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs
-from calibre.constants import isosx
 from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
 from calibre.ebooks import BOOK_EXTENSIONS
@@ -304,8 +303,9 @@ class FontFamilyModel(QAbstractListModel):
             return NONE
         if role == Qt.DisplayRole:
             return QVariant(family)
-        if not isosx and role == Qt.FontRole:
-            # Causes a Qt crash with some fonts on OS X
+        if False and role == Qt.FontRole:
+            # Causes a Qt crash with some fonts
+            # so disabled.
             return QVariant(QFont(family))
         return NONE
 

From 5474aa42682d79c9967544f93e38103d70c0b841 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 09:47:20 -0700
Subject: [PATCH 24/52] tportal.h by DM. Fixes #8552 (New recipe for daily
 tportal)

---
 resources/images/news/dailytportal.png | Bin 0 -> 635 bytes
 resources/recipes/dailytportal.recipe  |  66 +++++++++++++++++++++++++
 src/calibre/utils/localization.py      |   1 +
 3 files changed, 67 insertions(+)
 create mode 100644 resources/images/news/dailytportal.png
 create mode 100644 resources/recipes/dailytportal.recipe

diff --git a/resources/images/news/dailytportal.png b/resources/images/news/dailytportal.png
new file mode 100644
index 0000000000000000000000000000000000000000..38b06e675a24eccbca5297987fe696e531bf9dfd
GIT binary patch
literal 635
zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b
zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#Np$x;TbdoQ|EmuusfUpk@E7nf4V;Oe!{v
z86sU1J}{_g2sIsCXvk77*68$+!G^`sLhpxlrTT++tQ@;<x6YXP*SvoA+`I4At(*1g
z>e=7?YkgEE3T@Loz|_Lqpu8ee&5^NbX6LPQi#1fIoIJNiA%1_&qhEj2S!X!+%vf=u
zJ><3v%jRCyG%Z)|ly$tt>^_|=Uh-T5eG0}ag;;kSj6alDWaSpw{r%H7hPemr#dALY
zej}cLf6tOE|DUhkw7mYD+_dxJw1_qq&tAq(<saW3Pw75w@0)DW@c8|53zwjcu2qbk
za~EDSQ3y(SaPxDx%XHH%cC{aODSEy!Vr-0#pS$?39b4(!%x0I0^|G_9i&?f!Y*O94
zTVdh4?CQglPgU8S(KieAHZI9r^1<WVvQ>Mhw(q?3UD2yP@O$kAz1hnyT4c3zPwEJq
zAY`3Z^!^^#wyB0UdKeRDv)+&uVfLx5oo`cj<=W2ceK#bXgdTod>~m#mmV}<(dWIi=
zK6b3$EPQHNC1X&4_<}E{_ZBUGaQ8QVh}ye*JNMS<&t9He^7<aLdHm5to&yJqHr(d>
z#eYQS#k?EWwH^W^PqoA~q9i4;B-JXpC>2OC7#SEE>KYj88kvL`SX!BwS{WJW8kk!d
s7$g;NK0wiso1c=IR*9^^(7?*r2#6pWW~RFK12r&sy85}Sb4q9e0DvF?i~s-t

literal 0
HcmV?d00001

diff --git a/resources/recipes/dailytportal.recipe b/resources/recipes/dailytportal.recipe
new file mode 100644
index 0000000000..6e2646bfca
--- /dev/null
+++ b/resources/recipes/dailytportal.recipe
@@ -0,0 +1,66 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+daily.tportal.hr
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pagina12(BasicNewsRecipe):
+    title                 = 'Daily tportal.h'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Croatia'
+    publisher             = 'tportal.hr'
+    category              = 'news, politics, Croatia'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    language              = 'en_HR'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    extra_css             = """
+                               body{font-family: Verdana,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h1,h2{color: #2D648A; font-family: Georgia,serif}
+                               .artAbstract{font-size: 1.2em; font-family: Georgia,serif}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(name=['meta','link','embed','object','iframe','base'])
+                    ,dict(name='div', attrs={'class':'artInfo'})
+                  ]
+    remove_attributes=['lang']
+
+    keep_only_tags=dict(attrs={'class':'articleDetails'})
+
+    feeds = [(u'News', u'http://daily.tportal.hr/rss/dailynaslovnicarss.xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+
diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py
index d452721113..b9995db2bf 100644
--- a/src/calibre/utils/localization.py
+++ b/src/calibre/utils/localization.py
@@ -105,6 +105,7 @@ _extra_lang_codes = {
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
         'en_PK' : _('English (Pakistan)'),
+        'en_HR' : _('English (Croatia)'),
         'en_IL' : _('English (Israel)'),
         'en_SG' : _('English (Singapore)'),
         'en_YE' : _('English (Yemen)'),

From e1ebc4946f2f0fa744682440e7b363202036cf56 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 12:46:12 -0700
Subject: [PATCH 25/52] Fix #8558 (Tag editor issues with version 0.7.42)

---
 src/calibre/gui2/dialogs/tag_editor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/dialogs/tag_editor.py b/src/calibre/gui2/dialogs/tag_editor.py
index d4325354a1..6bd8eb7dbe 100644
--- a/src/calibre/gui2/dialogs/tag_editor.py
+++ b/src/calibre/gui2/dialogs/tag_editor.py
@@ -16,7 +16,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         self.setupUi(self)
 
         self.db = db
-        self.index = db.row(id_)
+        self.index = db.row(id_) if id_ is not None else None
         if self.index is not None:
             tags = self.db.tags(self.index)
         else:

From 135edd0a7c2a20847cc2b7f9a49c6803460984d0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 12:54:50 -0700
Subject: [PATCH 26/52] Bulk metadata edit dialog: Remember last used size.
 Fixes #8525 (Edit Multiple Books Window Size)

---
 src/calibre/gui2/dialogs/metadata_bulk.py | 14 +++++++++++++-
 src/calibre/gui2/dialogs/metadata_bulk.ui | 12 ++++++------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 36a035cd94..cf4252e9ed 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -15,7 +15,7 @@ from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.book.base import composite_formatter
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
-from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE
+from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, gprefs
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic, JSONConfig
 from calibre.utils.titlecase import titlecase
@@ -321,8 +321,15 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             'This operation cannot be canceled or undone'))
         self.do_again = False
         self.central_widget.setCurrentIndex(tab)
+        geom = gprefs.get('bulk_metadata_window_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(bytes(geom))
         self.exec_()
 
+    def save_state(self, *args):
+        gprefs['bulk_metadata_window_geometry'] = \
+            bytearray(self.saveGeometry())
+
     def do_apply_pubdate(self, *args):
         self.apply_pubdate.setChecked(True)
 
@@ -790,7 +797,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             self.series_start_number.setEnabled(False)
             self.series_start_number.setValue(1)
 
+    def reject(self):
+        self.save_state()
+        ResizableDialog.reject(self)
+
     def accept(self):
+        self.save_state()
         if len(self.ids) < 1:
             return QDialog.accept(self)
 
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui
index 481a485bc2..163d49b328 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>850</width>
-    <height>700</height>
+    <width>962</width>
+    <height>727</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -44,8 +44,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>842</width>
-        <height>639</height>
+        <width>954</width>
+        <height>666</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -996,8 +996,8 @@ not multiple and the destination field is multiple</string>
                <rect>
                 <x>0</x>
                 <y>0</y>
-                <width>826</width>
-                <height>323</height>
+                <width>197</width>
+                <height>60</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">

From e526bcc3361ba6bdf634220a2223a96ff7cec969 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 13:11:38 -0700
Subject: [PATCH 27/52] Fix #8563 (Calibre not recognizing archos 70 250GB)

---
 src/calibre/devices/android/driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index a95e3c46fa..16022fc752 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -54,7 +54,7 @@ class ANDROID(USBMS):
             0x1004 : { 0x61cc : [0x100] },
 
             # Archos
-            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216]},
+            0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
@@ -70,7 +70,7 @@ class ANDROID(USBMS):
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

From 0906058fb58e24ae7c0f077600de720fa692cf1b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 14:06:23 -0700
Subject: [PATCH 28/52] Various Czech news source by FunThomas. Fixes #8562
 (New Czech recipes)

---
 resources/recipes/abc.recipe   | 43 +++++++++++++++++++++++++++
 resources/recipes/idnes.recipe | 54 ++++++++++++++++++++++++++++++++++
 resources/recipes/root.recipe  | 39 ++++++++++++++++++++++++
 3 files changed, 136 insertions(+)
 create mode 100644 resources/recipes/abc.recipe
 create mode 100644 resources/recipes/idnes.recipe
 create mode 100644 resources/recipes/root.recipe

diff --git a/resources/recipes/abc.recipe b/resources/recipes/abc.recipe
new file mode 100644
index 0000000000..c4ae0aa308
--- /dev/null
+++ b/resources/recipes/abc.recipe
@@ -0,0 +1,43 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ABCRecipe(BasicNewsRecipe):
+  title          = u'ABC Linuxu'
+  oldest_article = 5
+  max_articles_per_feed = 3#5
+  __author__ = 'Funthomas'
+  language = 'cs'
+
+  feeds = [
+    #(u'Blogy', u'http://www.abclinuxu.cz/auto/blogDigest.rss'),
+    (u'Články', u'http://www.abclinuxu.cz/auto/abc.rss'),
+    (u'Zprávičky','http://www.abclinuxu.cz/auto/zpravicky.rss')
+  ]
+
+  remove_javascript = True
+  no_stylesheets    = True
+  remove_attributes = ['width','height']
+
+  remove_tags_before = dict(name='h1')
+  remove_tags = [
+    dict(attrs={'class':['meta-vypis','page_tools','cl_perex']}),
+    dict(attrs={'class':['cl_nadpis-link','komix-nav']})
+  ]
+
+  remove_tags_after = [
+    dict(name='div',attrs={'class':['cl_perex','komix-nav']}),
+    dict(attrs={'class':['meta-vypis','page_tools']}),
+    dict(name='',attrs={'':''}),
+  ]
+
+
+  preprocess_regexps = [
+    (re.compile(r'</div>.*<p class="perex">', re.DOTALL),lambda match: '</div><p class="perex">')
+  ]
+  def print_version(self, url):
+    return url + '?varianta=print&noDiz'
+
+  extra_css = '''
+            h1 {font-size:130%; font-weight:bold}
+            h3 {font-size:111%; font-weight:bold}
+        '''
diff --git a/resources/recipes/idnes.recipe b/resources/recipes/idnes.recipe
new file mode 100644
index 0000000000..0bd4de2327
--- /dev/null
+++ b/resources/recipes/idnes.recipe
@@ -0,0 +1,54 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class iHeuteRecipe(BasicNewsRecipe):
+    __author__  = 'FunThomas'
+    title = u'iDnes.cz'
+    publisher = u'MAFRA a.s.'
+    description = 'iDNES.cz Zprávy, Technet, Komiksy a další'
+    oldest_article = 3
+    max_articles_per_feed = 2
+
+    feeds = [
+            (u'Zprávy', u'http://servis.idnes.cz/rss.asp?c=zpravodaj'),
+            (u'Sport', u'http://servis.idnes.cz/rss.asp?c=sport'),
+            (u'Technet', u'http://servis.idnes.cz/rss.asp?c=technet'),
+            (u'Mobil', u'http://servis.idnes.cz/rss.asp?c=mobil'),
+            (u'Ekonomika', u'http://servis.idnes.cz/rss.asp?c=ekonomikah'),
+            #(u'Kultura', u'http://servis.idnes.cz/rss.asp?c=kultura'),
+            (u'Cestování', u'http://servis.idnes.cz/rss.asp?c=iglobe'),
+            #(u'Kavárna', u'http://servis.idnes.cz/rss.asp?r=kavarna'),
+            (u'Komixy', u'http://servis.idnes.cz/rss.asp?c=komiksy')
+            ]
+
+
+    encoding = 'cp1250'
+    language = 'cs'
+    cover_url = 'http://g.idnes.cz/u/loga-n4/idnes.gif'
+    remove_javascript = True
+    no_stylesheets = True
+
+    remove_attributes = ['width','height']
+    remove_tags = [dict(name='div',   attrs={'id':['zooming']}),
+                   dict(name='div',   attrs={'class':['related','mapa-wrapper']}),
+                   dict(name='table', attrs={'id':['opener-img','portal']}),
+                   dict(name='table', attrs={'class':['video-16ku9']})]
+    remove_tags_after  = [dict(name='div',attrs={'id':['related','related2']})]
+
+    keep_only_tags = [dict(name='div', attrs={'class':['art-full adwords-text','dil-day']})
+                      ,dict(name='table',attrs={'class':['kemel-box']})]
+
+    def print_version(self, url):
+        print_url = url
+        split_url = url.split("?")
+        if (split_url[0].rfind('dilbert.asp')  != -1):      #dilbert komix
+            print_url = print_url.replace('.htm','.gif&tisk=1')
+            print_url = print_url.replace('.asp','.aspx')
+        elif (split_url[0].rfind('kemel.asp')  == -1):      #not Kemel komix
+            print_url = 'http://zpravy.idnes.cz/tiskni.asp?' +  split_url[1]
+        #kemel                  kemel print page doesn't work
+        return print_url
+
+    extra_css = '''
+                  h1 {font-size:125%; font-weight:bold}
+                  h3 {font-size:110%; font-weight:bold}
+                '''
diff --git a/resources/recipes/root.recipe b/resources/recipes/root.recipe
new file mode 100644
index 0000000000..da065829a7
--- /dev/null
+++ b/resources/recipes/root.recipe
@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1289939440(BasicNewsRecipe):
+  __author__  = 'FunThomas'
+  title       = u'Root.cz'
+  description = u'Zprávičky a články z Root.cz'
+  publisher   = u'Internet Info, s.r.o'
+  oldest_article = 2               #max stari clanku ve dnech
+  max_articles_per_feed = 50       #max pocet clanku na feed
+
+  feeds = [
+    (u'Články', u'http://www.root.cz/rss/clanky/'),
+    (u'Zprávičky', u'http://www.root.cz/rss/zpravicky/')
+  ]
+
+  publication_type = u'magazine'
+  language = u'cs'
+  no_stylesheets = True
+  remove_javascript = True
+  cover_url = u'http://i.iinfo.cz/urs/logo-root-bila-oranzova-cerna-111089527143118.gif'
+
+  remove_attributes = ['width','height','href'] #,'href'
+  keep_only_tags = [
+    dict(name='h1'),
+    dict(name='a',attrs={'class':'author'}),
+    dict(name='p', attrs={'class':'intro'}),
+    dict(name='div',attrs={'class':'urs'})
+  ]
+
+  preprocess_regexps = [
+    (re.compile(u'<p class="perex[^"]*">[^<]*<img[^>]*>', re.DOTALL),lambda match: '<p class="intro">'),
+    (re.compile(u'<h3><a name="tucnak">Tričko tučňák.*</body>', re.DOTALL),lambda match: '<!--deleted-->')
+  ]
+
+  extra_css = '''
+                h1 {font-size:130%; font-weight:bold}
+                h3 {font-size:111%; font-weight:bold}
+              '''

From fdedd9803d2f077e111f9777b601ac1f11a60ad2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 14:32:56 -0700
Subject: [PATCH 29/52] 20 Minutos and La Tribuna de Talavera by Luis Hernandez

---
 resources/recipes/20_minutos.recipe | 17 +++++++++++++++++
 resources/recipes/la_tribuna.recipe | 29 +++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 resources/recipes/20_minutos.recipe
 create mode 100644 resources/recipes/la_tribuna.recipe

diff --git a/resources/recipes/20_minutos.recipe b/resources/recipes/20_minutos.recipe
new file mode 100644
index 0000000000..8205c918f5
--- /dev/null
+++ b/resources/recipes/20_minutos.recipe
@@ -0,0 +1,17 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1295310874(BasicNewsRecipe):
+    title          = u'20 Minutos (Boletin)'
+    __author__            = 'Luis Hernandez'
+    description           = 'Periódico gratuito en español'
+    cover_url     = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif'
+    language      = 'es'
+
+    oldest_article = 2
+    max_articles_per_feed = 50
+
+    feeds          = [(u'VESPERTINO', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss')
+                        , (u'DEPORTES', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss')
+                        , (u'CULTURA', u'http://www.20minutos.es/rss/ocio/')
+                        , (u'TV', u'http://20minutos.feedsportal.com/c/32489/f/490877/index.rss')
+]
diff --git a/resources/recipes/la_tribuna.recipe b/resources/recipes/la_tribuna.recipe
new file mode 100644
index 0000000000..11bdda8f3e
--- /dev/null
+++ b/resources/recipes/la_tribuna.recipe
@@ -0,0 +1,29 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+    title             = u'La Tribuna de Talavera'
+    __author__  = 'Luis Hernández'
+    description   = 'Diario de Talavera de la Reina'
+    cover_url     = 'http://www.latribunadetalavera.es/entorno/mancheta.gif'
+
+    oldest_article = 5
+    max_articles_per_feed = 50
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'utf-8'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['articulo']})
+                                  ,dict(name='div', attrs={'class':['foto']})
+                                  ,dict(name='p', attrs={'id':['texto']})
+                                ]
+
+    remove_tags_before = dict(name='div' , attrs={'class':['comparte']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['relacionadas']})
+
+
+    feeds          = [(u'Portada', u'http://www.latribunadetalavera.es/rss.html')]

From 896133c7d40f39b05b1306f147f814a681908124 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 15:38:27 -0700
Subject: [PATCH 30/52] Sinfest by nadid

---
 resources/recipes/sinfest.recipe | 33 ++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 resources/recipes/sinfest.recipe

diff --git a/resources/recipes/sinfest.recipe b/resources/recipes/sinfest.recipe
new file mode 100644
index 0000000000..bb0ef2e22e
--- /dev/null
+++ b/resources/recipes/sinfest.recipe
@@ -0,0 +1,33 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Nadid <nadid.skywalker at gmail.com>'
+'''
+http://www.sinfest.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SinfestBig(BasicNewsRecipe):
+    title                  = 'Sinfest'
+    __author__             = 'nadid'
+    description            = 'Sinfest'
+    reverse_article_order = False
+    oldest_article         = 5
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'utf-8'
+    publisher              = 'Tatsuya Ishida/Museworks'
+    category               = 'comic'
+    language               = 'en'
+
+    conversion_options = {
+                             'comments'        : description
+                            ,'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                         }
+
+    feeds = [(u'SinFest', u'http://henrik.nyh.se/scrapers/sinfest.rss' )]
+    def get_article_url(self, article):
+        return article.get('link')
+

From b408af075d5dfd9a0e5edd47cae9250b59466970 Mon Sep 17 00:00:00 2001
From: Ben Collier <bcollier@cmu.edu>
Date: Mon, 24 Jan 2011 18:14:44 -0500
Subject: [PATCH 31/52] Updates to New York Times recipe for downloading high
 resolution images rather than thumbnails and cleaning up readability

---
 resources/recipes/nytimes_sub.recipe | 120 +++++++++++++++++++++++----
 1 file changed, 103 insertions(+), 17 deletions(-)

diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe
index 2424113e31..863e4b22ba 100644
--- a/resources/recipes/nytimes_sub.recipe
+++ b/resources/recipes/nytimes_sub.recipe
@@ -1,6 +1,5 @@
 #!/usr/bin/env  python
 # -*- coding: utf-8 -*-
-
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
@@ -28,6 +27,10 @@ class NYTimes(BasicNewsRecipe):
     # previous paid versions of the new york times to best sent to the back issues folder on the kindle
     replaceKindleVersion = False
 
+    # download higher resolution images than the small thumbnails typically included in the article
+    # the down side of having large beautiful images is the file size is much larger, on the order of 7MB per paper
+    useHighResImages = True
+
     # includeSections: List of sections to include. If empty, all sections found will be included.
     # Otherwise, only the sections named will be included. For example,
     #
@@ -90,7 +93,6 @@ class NYTimes(BasicNewsRecipe):
                     (u'Sunday Magazine',u'magazine'),
                     (u'Week in Review',u'weekinreview')]
 
-
     if headlinesOnly:
         title='New York Times Headlines'
         description = 'Headlines from the New York Times'
@@ -127,7 +129,7 @@ class NYTimes(BasicNewsRecipe):
 
     earliest_date = date.today() - timedelta(days=oldest_article)
 
-    __author__  = 'GRiker/Kovid Goyal/Nick Redding'
+    __author__  = 'GRiker/Kovid Goyal/Nick Redding/Ben Collier'
     language = 'en'
     requires_version = (0, 7, 5)
 
@@ -149,7 +151,7 @@ class NYTimes(BasicNewsRecipe):
                             'dottedLine',
                             'entry-meta',
                             'entry-response module',
-                            'icon enlargeThis',
+                            #'icon enlargeThis', #removed to provide option for high res images
                             'leftNavTabs',
                             'metaFootnote',
                             'module box nav',
@@ -163,7 +165,23 @@ class NYTimes(BasicNewsRecipe):
                             'entry-tags', #added for DealBook
                             'footer promos clearfix', #added for DealBook
                             'footer links clearfix', #added for DealBook
-                            'inlineImage module', #added for DealBook
+                            'tabsContainer', #added for other blog downloads
+                            'column lastColumn', #added for other blog downloads
+                            'pageHeaderWithLabel', #added for other gadgetwise downloads
+                            'column two', #added for other blog downloads
+                            'column two last', #added for other blog downloads
+                            'column three', #added for other blog downloads
+                            'column three last', #added for other blog downloads
+                            'column four',#added for other blog downloads
+                            'column four last',#added for other blog downloads
+                            'column last', #added for other blog downloads
+                            'timestamp published', #added for other blog downloads
+                            'entry entry-related',
+                            'subNavigation tabContent active', #caucus blog navigation
+                            'columnGroup doubleRule',
+                            'mediaOverlay slideshow',
+                            'headlinesOnly multiline flush',
+                            'wideThumb',
                             re.compile('^subNavigation'),
                             re.compile('^leaderboard'),
                             re.compile('^module'),
@@ -254,7 +272,7 @@ class NYTimes(BasicNewsRecipe):
     def exclude_url(self,url):
         if not url.startswith("http"):
             return True
-        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url: #added for DealBook
+        if not url.endswith(".html") and 'dealbook.nytimes.com' not in url and 'blogs.nytimes.com' not in url: #added for DealBook
             return True
         if 'nytimes.com' not in url:
             return True
@@ -480,7 +498,7 @@ class NYTimes(BasicNewsRecipe):
                 for lidiv in div.findAll('li'):
                     if not skipping:
                         self.handle_article(lidiv)
-
+            
         self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)]
         return self.filter_ans(self.ans)
 
@@ -591,20 +609,85 @@ class NYTimes(BasicNewsRecipe):
                 if article_date < self.earliest_date:
                     self.log("Skipping article dated %s" % date_str)
                     return None
+                    
+        #all articles are from today, no need to print the date on every page
+        try:
+            if not self.webEdition:
+                date_tag = soup.find(True,attrs={'class': ['dateline','date']})
+                if date_tag:
+                    date_tag.extract()
+        except:
+            self.log("Error removing the published date")
 
-        kicker_tag = soup.find(attrs={'class':'kicker'})
-        if kicker_tag: # remove Op_Ed author head shots
-            tagline = self.tag_to_string(kicker_tag)
-            if tagline=='Op-Ed Columnist':
-                img_div = soup.find('div','inlineImage module')
-                if img_div:
-                    img_div.extract()
-
+        if self.useHighResImages:
+            try:
+                #open up all the "Enlarge this Image" pop-ups and download the full resolution jpegs
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupreflink = popupref.find('a')
+                        if popupreflink:
+                            reflinkstring = str(popupreflink['href'])
+                            refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('")
+                            refend = reflinkstring.find(".html", refstart) + len(".html")
+                            reflinkstring = reflinkstring[refstart:refend]
+                            
+                            popuppage = self.browser.open(reflinkstring)
+                            popuphtml = popuppage.read()
+                            popuppage.close()
+                            if popuphtml:
+                                st = time.localtime()
+                                year = str(st.tm_year)
+                                month = "%.2d" % st.tm_mon
+                                day = "%.2d" % st.tm_mday
+                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')                                
+                                highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
+                                popupSoup = BeautifulSoup(popuphtml)
+                                highResTag = popupSoup.find('img', {'src':highResImageLink})
+                                if highResTag:
+                                    try:
+                                        newWidth = highResTag['width']
+                                        newHeight = highResTag['height']
+                                        imageTag = popupref.parent.find("img")
+                                    except:
+                                        self.log("Error: finding width and height of img")
+                                    popupref.extract()
+                                    if imageTag:
+                                        try:
+                                            imageTag['src'] = highResImageLink
+                                            imageTag['width'] = newWidth
+                                            imageTag['height'] = newHeight
+                                        except:
+                                            self.log("Error setting the src width and height parameters")
+            except Exception as e:
+                self.log("Error pulling high resolution images")
+                
+            try:
+                #remove "Related content" bar
+                runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']})
+                if runAroundsFound:
+                    for runAround in runAroundsFound:
+                        #find all section headers
+                        hlines = runAround.findAll(True ,{'class':['sectionHeader','sectionHeader flushBottom']})
+                        if hlines:
+                            for hline in hlines:
+                                hline.extract()
+            except:
+                self.log("Error removing related content bar")
+     
+                
+            try:
+                #in case pulling images failed, delete the enlarge this text
+                enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
+                if enlargeThisList:
+                    for popupref in enlargeThisList:
+                        popupref.extract()
+            except:
+                self.log("Error removing Enlarge this text")
 
         return self.strip_anchors(soup)
 
     def postprocess_html(self,soup, True):
-
         try:
                 if self.one_picture_per_article:
                         # Remove all images after first
@@ -766,6 +849,8 @@ class NYTimes(BasicNewsRecipe):
         try:
             if len(article.text_summary.strip()) == 0:
                 articlebodies = soup.findAll('div',attrs={'class':'articleBody'})
+                if not articlebodies: #added to account for blog formats
+                    articlebodies = soup.findAll('div', attrs={'class':'entry-content'}) #added to account for blog formats
                 if articlebodies:
                     for articlebody in articlebodies:
                         if articlebody:
@@ -774,13 +859,14 @@ class NYTimes(BasicNewsRecipe):
                                 refparagraph = self.massageNCXText(self.tag_to_string(p,use_alt=False)).strip()
                                 #account for blank paragraphs and short paragraphs by appending them to longer ones
                                 if len(refparagraph) > 0:
-                                    if len(refparagraph) > 70: #approximately one line of text
+                                    if len(refparagraph) > 140: #approximately two lines of text
                                         article.summary = article.text_summary = shortparagraph + refparagraph
                                         return
                                     else:
                                         shortparagraph = refparagraph + " "
                                         if shortparagraph.strip().find(" ") == -1 and not shortparagraph.strip().endswith(":"):
                                             shortparagraph = shortparagraph + "- "
+
         except:
             self.log("Error creating article descriptions")
             return

From 64ad0d7a00a69d1912876a66d13a1fb4bb810e0b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 18:11:03 -0700
Subject: [PATCH 32/52] ...

---
 src/calibre/ebooks/metadata/sources/__init__.py | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 src/calibre/ebooks/metadata/sources/__init__.py

diff --git a/src/calibre/ebooks/metadata/sources/__init__.py b/src/calibre/ebooks/metadata/sources/__init__.py
new file mode 100644
index 0000000000..68dfb8d2b5
--- /dev/null
+++ b/src/calibre/ebooks/metadata/sources/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

From a843fae9d7a189463a0c8227d48c5952b8c9c99c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 18:23:45 -0700
Subject: [PATCH 33/52] ...

---
 src/calibre/gui2/convert/search_and_replace.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/convert/search_and_replace.py b/src/calibre/gui2/convert/search_and_replace.py
index ec59268ec8..88446344ec 100644
--- a/src/calibre/gui2/convert/search_and_replace.py
+++ b/src/calibre/gui2/convert/search_and_replace.py
@@ -42,9 +42,15 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
     def break_cycles(self):
         Widget.break_cycles(self)
 
-        self.opt_sr1_search.doc_update.disconnect()
-        self.opt_sr2_search.doc_update.disconnect()
-        self.opt_sr3_search.doc_update.disconnect()
+        def d(x):
+            try:
+                x.disconnect()
+            except:
+                pass
+
+        d(self.opt_sr1_search)
+        d(self.opt_sr2_search)
+        d(self.opt_sr3_search)
 
         self.opt_sr1_search.break_cycles()
         self.opt_sr2_search.break_cycles()

From 559ff8c59f26a25731350fe16791ead51233ac50 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 24 Jan 2011 19:57:26 -0700
Subject: [PATCH 34/52] Manual updates

---
 src/calibre/manual/faq.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 5ebe91bc76..7a04e0f642 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -310,7 +310,9 @@ What formats does |app| read metadata from?
 
 Where are the book files stored?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Metadata about the books is stored in the file ``metadata.db`` (which is a sqlite database).
+When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Note that the contents of this folder are automatically managed by |app|, **do not** add any files/folders manually to this folder, as they may be automatically deleted. If you want to add a file associated to a particular book, use the top right area of :guilabel:`Edit metadata` dialog to do so. Then, |app| will automatically put that file into the correct folder and move it around when the title/author changes.
+
+Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders. 
 
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From d3cfeb56a241bc727e682a37fcb6282e0087cd6b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 09:52:00 +0000
Subject: [PATCH 35/52] #8554: Folder Device Plugin: to have option to disable
 subfolder

---
 src/calibre/devices/folder_device/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py
index b852715b97..d75697a6cb 100644
--- a/src/calibre/devices/folder_device/driver.py
+++ b/src/calibre/devices/folder_device/driver.py
@@ -22,7 +22,7 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS):
     PRODUCT_ID  = [0xffff]
     BCD         = [0xffff]
     DEVICE_PLUGBOARD_NAME = 'FOLDER_DEVICE'
-
+    SUPPORTS_SUB_DIRS = True
 
 class FOLDER_DEVICE(USBMS):
     type = _('Device Interface')

From 8b62f6be458e796ccca6922a549d8bdca33f8665 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 13:33:27 +0000
Subject: [PATCH 36/52] 1) Fix bug where search was shown as a searchable
 category 2) Add search for item

---
 src/calibre/gui2/tag_view.py | 66 ++++++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 21 deletions(-)

diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 8b574948ff..5a57395530 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -98,6 +98,7 @@ class TagsView(QTreeView): # {{{
             self.collapse_model = 'disable'
         else:
             self.collapse_model = gprefs['tags_browser_partition_method']
+        self.search_icon = QIcon(I('search.png'))
 
     def set_pane_is_visible(self, to_what):
         pv = self.pane_is_visible
@@ -199,6 +200,10 @@ class TagsView(QTreeView): # {{{
             if action == 'manage_categories':
                 self.user_category_edit.emit(category)
                 return
+            if action == 'search':
+                self.tags_marked.emit(('not ' if negate else '') +
+                                      category + ':"=' + key + '"')
+                return
             if action == 'search_category':
                 self.tags_marked.emit(category + ':' + str(not negate))
                 return
@@ -208,6 +213,7 @@ class TagsView(QTreeView): # {{{
             if action == 'edit_author_sort':
                 self.author_sort_edit.emit(self, index)
                 return
+
             if action == 'hide':
                 self.hidden_categories.add(category)
             elif action == 'show':
@@ -248,19 +254,36 @@ class TagsView(QTreeView): # {{{
                 if key not in self.db.field_metadata:
                     return True
 
-                # If the user right-clicked on an editable item, then offer
-                # the possibility of renaming that item
-                if tag_name and \
-                        (key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
-                        self.db.field_metadata[key]['is_custom'] and \
-                        self.db.field_metadata[key]['datatype'] != 'rating'):
-                    self.context_menu.addAction(_('Rename \'%s\'')%tag_name,
-                            partial(self.context_menu_handler, action='edit_item',
-                                    category=tag_item, index=index))
-                    if key == 'authors':
-                        self.context_menu.addAction(_('Edit sort for \'%s\'')%tag_name,
-                                partial(self.context_menu_handler,
-                                        action='edit_author_sort', index=tag_id))
+                # Did the user click on a leaf node?
+                if tag_name:
+                    # If the user right-clicked on an editable item, then offer
+                    # the possibility of renaming that item.
+                    if key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
+                            (self.db.field_metadata[key]['is_custom'] and \
+                             self.db.field_metadata[key]['datatype'] != 'rating'):
+                        # Add the 'rename' items
+                        self.context_menu.addAction(_('Rename %s')%tag_name,
+                                partial(self.context_menu_handler, action='edit_item',
+                                        category=tag_item, index=index))
+                        if key == 'authors':
+                            self.context_menu.addAction(_('Edit sort for %s')%tag_name,
+                                    partial(self.context_menu_handler,
+                                            action='edit_author_sort', index=tag_id))
+                    # Add the search for value items
+                    n = tag_name
+                    c = category
+                    if self.db.field_metadata[key]['datatype'] == 'rating':
+                        n = str(len(tag_name))
+                    elif self.db.field_metadata[key]['kind'] in ['user', 'search']:
+                        c = tag_item.tag.category
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for everything but %s')%tag_name,
+                            partial(self.context_menu_handler, action='search',
+                                    category=c, key=n, negate=True))
                     self.context_menu.addSeparator()
                 # Hide/Show/Restore categories
                 self.context_menu.addAction(_('Hide category %s') % category,
@@ -272,14 +295,15 @@ class TagsView(QTreeView): # {{{
                             partial(self.context_menu_handler, action='show', category=col))
 
                 # search by category
-                self.context_menu.addAction(
-                        _('Search for books in category %s')%category,
-                        partial(self.context_menu_handler, action='search_category',
-                                category=key, negate=False))
-                self.context_menu.addAction(
-                        _('Search for books not in category %s')%category,
-                        partial(self.context_menu_handler, action='search_category',
-                                category=key, negate=True))
+                if key != 'search':
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=False))
+                    self.context_menu.addAction(self.search_icon,
+                            _('Search for books not in category %s')%category,
+                            partial(self.context_menu_handler, action='search_category',
+                                    category=key, negate=True))
                 # Offer specific editors for tags/series/publishers/saved searches
                 self.context_menu.addSeparator()
                 if key in ['tags', 'publisher', 'series'] or \

From 9287eb4445d803cbef826bb82f377d277c30517e Mon Sep 17 00:00:00 2001
From: GRiker <griker@hotmail.com>
Date: Tue, 25 Jan 2011 07:16:14 -0700
Subject: [PATCH 37/52] various GwR revisions

---
 resources/catalog/stylesheet.css              | 12 ++++++++++++
 src/calibre/gui2/__init__.py                  |  8 ++++++++
 src/calibre/gui2/actions/annotate.py          |  4 ++++
 src/calibre/gui2/catalog/catalog_epub_mobi.py |  3 ++-
 src/calibre/library/catalog.py                | 11 ++++++++---
 5 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/resources/catalog/stylesheet.css b/resources/catalog/stylesheet.css
index bf83a4c60b..336d015e44 100644
--- a/resources/catalog/stylesheet.css
+++ b/resources/catalog/stylesheet.css
@@ -62,6 +62,18 @@ div.description {
 	text-indent: 1em;
 	}
 
+/*
+* 	Attempt to minimize widows and orphans by logically grouping chunks
+* 	Recommend enabling for iPad
+*   Some reports of problems with Sony ereaders, presumably ADE engines
+*/
+/*
+div.logical_group {
+	display:inline-block;
+	width:100%;
+	}
+*/
+
 p.date_index {
 	font-size:x-large;
 	text-align:center;
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index c94b99f141..84a26cea18 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -550,6 +550,14 @@ def choose_dir(window, name, title, default_dir='~'):
     if dir:
         return dir[0]
 
+def choose_osx_app(window, name, title, default_dir='/Applications'):
+    fd = FileDialog(title=title, parent=window, name=name, mode=QFileDialog.ExistingFile,
+            default_dir=default_dir)
+    app = fd.get_files()
+    fd.setParent(None)
+    if app:
+        return app
+
 def choose_files(window, name, title,
                  filters=[], all_files=True, select_only_single_file=False):
     '''
diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py
index dfafcd1a39..8714654d4b 100644
--- a/src/calibre/gui2/actions/annotate.py
+++ b/src/calibre/gui2/actions/annotate.py
@@ -165,10 +165,12 @@ class FetchAnnotationsAction(InterfaceAction):
                 ka_soup.insert(0,divTag)
                 return ka_soup
 
+            '''
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
                 if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
+            '''
 
             def canceled(self):
                 self.pd.hide()
@@ -201,10 +203,12 @@ class FetchAnnotationsAction(InterfaceAction):
                         # Update library comments
                         self.db.set_comment(id, mi.comments)
 
+                        '''
                         # Update 'read' tag except for Catalogs/Clippings
                         if bm.value.percent_read >= self.FINISHED_READING_PCT_THRESHOLD:
                             if not set(mi.tags).intersection(ignore_tags):
                                 self.mark_book_as_read(id)
+                        '''
 
                         # Add bookmark file to id
                         self.db.add_format_with_hooks(id, bm.value.bookmark_extension,
diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py
index 94760306c3..d5149569be 100644
--- a/src/calibre/gui2/catalog/catalog_epub_mobi.py
+++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py
@@ -335,7 +335,7 @@ class PluginWidget(QWidget,Ui_Form):
         '''
 
         return
-
+        '''
         if new_state == 0:
             # unchecked
             self.merge_source_field.setEnabled(False)
@@ -348,6 +348,7 @@ class PluginWidget(QWidget,Ui_Form):
             self.merge_before.setEnabled(True)
             self.merge_after.setEnabled(True)
             self.include_hr.setEnabled(True)
+        '''
 
     def header_note_source_field_changed(self,new_index):
         '''
diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py
index 95e738dd58..f0e4778de4 100644
--- a/src/calibre/library/catalog.py
+++ b/src/calibre/library/catalog.py
@@ -1820,6 +1820,9 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                 self.booksByTitle_noSeriesPrefix = nspt
 
             # Loop through the books by title
+            # Generate one divRunningTag per initial letter for the purposes of
+            # minimizing widows and orphans on readers that can handle large
+            # <divs> styled as inline-block
             title_list = self.booksByTitle
             if not self.useSeriesPrefixInTitlesSection:
                 title_list = self.booksByTitle_noSeriesPrefix
@@ -1832,7 +1835,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                         divTag.insert(dtc, divRunningTag)
                         dtc += 1
                     divRunningTag = Tag(soup, 'div')
-                    divRunningTag['style'] = 'display:inline-block;width:100%'
+                    divRunningTag['class'] = "logical_group"
                     drtc = 0
                     current_letter = self.letter_or_symbol(book['title_sort'][0])
                     pIndexTag = Tag(soup, "p")
@@ -1954,6 +1957,8 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
             drtc = 0
 
             # Loop through booksByAuthor
+            # Each author/books group goes in an openingTag div (first) or
+            # a runningTag div (subsequent)
             book_count = 0
             current_author = ''
             current_letter = ''
@@ -1977,7 +1982,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
                     current_letter = self.letter_or_symbol(book['author_sort'][0].upper())
                     author_count = 0
                     divOpeningTag = Tag(soup, 'div')
-                    divOpeningTag['style'] = 'display:inline-block;width:100%'
+                    divOpeningTag['class'] = "logical_group"
                     dotc = 0
                     pIndexTag = Tag(soup, "p")
                     pIndexTag['class'] = "letter_index"
@@ -2001,7 +2006,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1])
 
                         # Create a divRunningTag for the rest of the authors in this letter
                         divRunningTag = Tag(soup, 'div')
-                        divRunningTag['style'] = 'display:inline-block;width:100%'
+                        divRunningTag['class'] = "logical_group"
                         drtc = 0
 
                     non_series_books = 0

From c9bb59a5d5a443ae277dbf16d0b261e86c8521a2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 08:35:29 -0700
Subject: [PATCH 38/52] Fix extra spaces being inserted into TOC title when
 reading TOC from OPD guide element. Fixes #8569 (Chapters headers with
 internal tags not added to TOC correctly.)

---
 src/calibre/ebooks/oeb/reader.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py
index 8e11ac6498..d08a68c0bc 100644
--- a/src/calibre/ebooks/oeb/reader.py
+++ b/src/calibre/ebooks/oeb/reader.py
@@ -17,7 +17,7 @@ from lxml import etree
 import cssutils
 
 from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \
-    DC_NSES, OPF
+    DC_NSES, OPF, xml2text
 from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \
     PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME
 from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, \
@@ -423,7 +423,7 @@ class OEBReader(object):
             path, frag = urldefrag(href)
             if path not in self.oeb.manifest.hrefs:
                 continue
-            title = ' '.join(xpath(anchor, './/text()'))
+            title = xml2text(anchor)
             title = COLLAPSE_RE.sub(' ', title.strip())
             if href not in titles:
                 order.append(href)

From 336874d87f7520616d06310460d143653d5c5001 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 09:38:38 -0700
Subject: [PATCH 39/52] ...

---
 resources/recipes/20_minutos.recipe  | 74 +++++++++++++++++++++++-----
 resources/recipes/nytimes_sub.recipe | 16 +++---
 src/calibre/gui2/actions/annotate.py |  2 +-
 3 files changed, 71 insertions(+), 21 deletions(-)

diff --git a/resources/recipes/20_minutos.recipe b/resources/recipes/20_minutos.recipe
index 8205c918f5..1f862847dc 100644
--- a/resources/recipes/20_minutos.recipe
+++ b/resources/recipes/20_minutos.recipe
@@ -1,17 +1,67 @@
+# -*- coding: utf-8
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+description   = 'Periódico gratuito en español - v0.5 - 25 Jan 2011'
+
+'''
+www.20minutos.es
+'''
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1295310874(BasicNewsRecipe):
-    title          = u'20 Minutos (Boletin)'
-    __author__            = 'Luis Hernandez'
-    description           = 'Periódico gratuito en español'
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'20 Minutos'
+    publisher      = u'Grupo 20 Minutos'
+
+    __author__            = u'Luis Hernández'
+    description           = u'Periódico gratuito en español'
     cover_url     = 'http://estaticos.20minutos.es/mmedia/especiales/corporativo/css/img/logotipos_grupo20minutos.gif'
-    language      = 'es'
 
-    oldest_article = 2
-    max_articles_per_feed = 50
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    encoding              = 'ISO-8859-1'
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    keep_only_tags     = [dict(name='div', attrs={'id':['content']})
+                                  ,dict(name='div', attrs={'class':['boxed','description','lead','article-content']})
+                                  ,dict(name='span', attrs={'class':['photo-bar']})
+                                  ,dict(name='ul', attrs={'class':['article-author']})
+                                ]
+
+    remove_tags_before = dict(name='ul' , attrs={'class':['servicios-sub']})
+    remove_tags_after  = dict(name='div' , attrs={'class':['related-news','col']})
+
+    remove_tags = [
+                     dict(name='ol', attrs={'class':['navigation',]})
+                    ,dict(name='span', attrs={'class':['action']})
+                    ,dict(name='div', attrs={'class':['twitter comments-list hidden','related-news','col']})
+                    ,dict(name='div', attrs={'id':['twitter-destacados']})
+                    ,dict(name='ul', attrs={'class':['article-user-actions','stripped-list']})
+                                          ]
+
+    feeds = [
+              (u'Portada'              , u'http://www.20minutos.es/rss/')
+             ,(u'Nacional'             , u'http://www.20minutos.es/rss/nacional/')
+             ,(u'Internacional'       , u'http://www.20minutos.es/rss/internacional/')
+             ,(u'Economia'           , u'http://www.20minutos.es/rss/economia/')
+             ,(u'Deportes'            , u'http://www.20minutos.es/rss/deportes/')
+             ,(u'Tecnologia'          , u'http://www.20minutos.es/rss/tecnologia/')
+             ,(u'Gente - TV'         , u'http://www.20minutos.es/rss/gente-television/')
+             ,(u'Motor'                 , u'http://www.20minutos.es/rss/motor/')
+             ,(u'Salud'                 , u'http://www.20minutos.es/rss/belleza-y-salud/')
+             ,(u'Viajes'                , u'http://www.20minutos.es/rss/viajes/')
+             ,(u'Vivienda'             , u'http://www.20minutos.es/rss/vivienda/')
+             ,(u'Empleo'              , u'http://www.20minutos.es/rss/empleo/')
+             ,(u'Cine'                  , u'http://www.20minutos.es/rss/cine/')
+             ,(u'Musica'               , u'http://www.20minutos.es/rss/musica/')
+             ,(u'Comunidad20'     , u'http://www.20minutos.es/rss/zona20/')
+            ]
 
-    feeds          = [(u'VESPERTINO', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss')
-                        , (u'DEPORTES', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss')
-                        , (u'CULTURA', u'http://www.20minutos.es/rss/ocio/')
-                        , (u'TV', u'http://20minutos.feedsportal.com/c/32489/f/490877/index.rss')
-]
diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe
index 863e4b22ba..81b8bd5cb7 100644
--- a/resources/recipes/nytimes_sub.recipe
+++ b/resources/recipes/nytimes_sub.recipe
@@ -498,7 +498,7 @@ class NYTimes(BasicNewsRecipe):
                 for lidiv in div.findAll('li'):
                     if not skipping:
                         self.handle_article(lidiv)
-            
+
         self.ans = [(k, self.articles[k]) for k in self.ans if self.articles.has_key(k)]
         return self.filter_ans(self.ans)
 
@@ -609,7 +609,7 @@ class NYTimes(BasicNewsRecipe):
                 if article_date < self.earliest_date:
                     self.log("Skipping article dated %s" % date_str)
                     return None
-                    
+
         #all articles are from today, no need to print the date on every page
         try:
             if not self.webEdition:
@@ -631,7 +631,7 @@ class NYTimes(BasicNewsRecipe):
                             refstart = reflinkstring.find("javascript:pop_me_up2('") + len("javascript:pop_me_up2('")
                             refend = reflinkstring.find(".html", refstart) + len(".html")
                             reflinkstring = reflinkstring[refstart:refend]
-                            
+
                             popuppage = self.browser.open(reflinkstring)
                             popuphtml = popuppage.read()
                             popuppage.close()
@@ -640,7 +640,7 @@ class NYTimes(BasicNewsRecipe):
                                 year = str(st.tm_year)
                                 month = "%.2d" % st.tm_mon
                                 day = "%.2d" % st.tm_mday
-                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')                                
+                                imgstartpos = popuphtml.find('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/') + len('http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/')
                                 highResImageLink = 'http://graphics8.nytimes.com/images/' + year + '/' +  month +'/' + day +'/' + popuphtml[imgstartpos:popuphtml.find('.jpg',imgstartpos)+4]
                                 popupSoup = BeautifulSoup(popuphtml)
                                 highResTag = popupSoup.find('img', {'src':highResImageLink})
@@ -659,9 +659,9 @@ class NYTimes(BasicNewsRecipe):
                                             imageTag['height'] = newHeight
                                         except:
                                             self.log("Error setting the src width and height parameters")
-            except Exception as e:
+            except Exception:
                 self.log("Error pulling high resolution images")
-                
+
             try:
                 #remove "Related content" bar
                 runAroundsFound = soup.findAll('div',{'class':['articleInline runaroundLeft','articleInline doubleRule runaroundLeft','articleInline runaroundLeft firstArticleInline']})
@@ -674,8 +674,8 @@ class NYTimes(BasicNewsRecipe):
                                 hline.extract()
             except:
                 self.log("Error removing related content bar")
-     
-                
+
+
             try:
                 #in case pulling images failed, delete the enlarge this text
                 enlargeThisList = soup.findAll('div',{'class':'icon enlargeThis'})
diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py
index 8714654d4b..a702ba045e 100644
--- a/src/calibre/gui2/actions/annotate.py
+++ b/src/calibre/gui2/actions/annotate.py
@@ -9,7 +9,7 @@ import os, datetime
 
 from PyQt4.Qt import pyqtSignal, QModelIndex, QThread, Qt
 
-from calibre.gui2 import error_dialog, gprefs
+from calibre.gui2 import error_dialog
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
 from calibre import strftime
 from calibre.gui2.actions import InterfaceAction

From 503038f39febac1075a1a809db557db05a133a5b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 10:38:32 -0700
Subject: [PATCH 40/52] Fix #8576 (Issue with new yorker download)

---
 resources/recipes/new_yorker.recipe | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/resources/recipes/new_yorker.recipe b/resources/recipes/new_yorker.recipe
index 0c95aa358d..d69a4df24f 100644
--- a/resources/recipes/new_yorker.recipe
+++ b/resources/recipes/new_yorker.recipe
@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newyorker.com
 '''
@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://feeds.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return url + '?printable=true'
+        return 'http://www.newyorker.com' + url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

From 05be08a9ee9a645fe63c520d98989d33c6f58209 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 11:09:49 -0700
Subject: [PATCH 41/52] Fix the metadata backup thread to more reliably flush
 an in_limbo id

---
 src/calibre/gui2/ui.py        |  2 --
 src/calibre/library/caches.py | 15 ++++++++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 6a74ccd6ea..c0658536bb 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -638,8 +638,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         except KeyboardInterrupt:
             pass
         time.sleep(2)
-        if mb is not None:
-            mb.flush()
         self.hide_windows()
         return True
 
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 77e75736cf..7d6511e8a5 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -42,6 +42,7 @@ class MetadataBackup(Thread): # {{{
 
     def stop(self):
         self.keep_running = False
+        self.flush()
         # Break cycles so that this object doesn't hold references to db
         self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \
             self.set_dirtied = self.db = None
@@ -57,7 +58,10 @@ class MetadataBackup(Thread): # {{{
             except:
                 # Happens during interpreter shutdown
                 break
+            if not self.keep_running:
+                break
 
+            self.in_limbo = id_
             try:
                 path, mi = self.get_metadata_for_dump(id_)
             except:
@@ -72,10 +76,10 @@ class MetadataBackup(Thread): # {{{
                     continue
 
             # at this point the dirty indication is off
-
             if mi is None:
                 continue
-            self.in_limbo = id_
+            if not self.keep_running:
+                break
 
             # Give the GUI thread a chance to do something. Python threads don't
             # have priorities, so this thread would naturally keep the processor
@@ -89,6 +93,9 @@ class MetadataBackup(Thread): # {{{
                 traceback.print_exc()
                 continue
 
+            if not self.keep_running:
+                break
+
             time.sleep(0.1) # Give the GUI thread a chance to do something
             try:
                 self.do_write(path, raw)
@@ -102,7 +109,8 @@ class MetadataBackup(Thread): # {{{
                     prints('Failed to write backup metadata for id:', id_,
                             'again, giving up')
                     continue
-        self.in_limbo = None
+
+            self.in_limbo = None
 
     def flush(self):
         'Used during shutdown to ensure that a dirtied book is not missed'
@@ -111,6 +119,7 @@ class MetadataBackup(Thread): # {{{
                 self.db.dirtied([self.in_limbo])
             except:
                 traceback.print_exc()
+            self.in_limbo = None
 
     def write(self, path, raw):
         with lopen(path, 'wb') as f:

From 80bb00057570f8b6af3d617f5f55d491e1b86073 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 12:50:57 -0700
Subject: [PATCH 42/52] ...

---
 src/calibre/gui2/actions/choose_library.py | 4 ++--
 src/calibre/gui2/tag_view.py               | 6 +++++-
 src/calibre/library/caches.py              | 5 ++++-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index d726241432..001970f9db 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -390,7 +390,7 @@ class ChooseLibraryAction(InterfaceAction):
         #self.dbref = weakref.ref(self.gui.library_view.model().db)
         #self.before_mem = memory()/1024**2
         self.gui.library_moved(loc)
-        #QTimer.singleShot(1000, self.debug_leak)
+        #QTimer.singleShot(5000, self.debug_leak)
 
     def debug_leak(self):
         import gc
@@ -398,7 +398,7 @@ class ChooseLibraryAction(InterfaceAction):
         ref = self.dbref
         for i in xrange(3): gc.collect()
         if ref() is not None:
-            print 11111, ref()
+            print 'DB object alive:', ref()
             for r in gc.get_referrers(ref())[:10]:
                 print r
                 print
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 5a57395530..2160e13b65 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -727,7 +727,11 @@ class TagsModel(QAbstractItemModel): # {{{
         for user_cat in sorted(self.db.prefs.get('user_categories', {}).keys(),
                                key=sort_key):
             cat_name = '@' + user_cat # add the '@' to avoid name collision
-            tb_cats.add_user_category(label=cat_name, name=user_cat)
+            try:
+                tb_cats.add_user_category(label=cat_name, name=user_cat)
+            except ValueError:
+                import traceback
+                traceback.print_exc()
         if len(saved_searches().names()):
             tb_cats.add_search_category(label='search', name=_('Searches'))
 
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 7d6511e8a5..af47a79e49 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -42,7 +42,8 @@ class MetadataBackup(Thread): # {{{
 
     def stop(self):
         self.keep_running = False
-        self.flush()
+
+    def break_cycles(self):
         # Break cycles so that this object doesn't hold references to db
         self.do_write = self.get_metadata_for_dump = self.clear_dirtied = \
             self.set_dirtied = self.db = None
@@ -111,6 +112,8 @@ class MetadataBackup(Thread): # {{{
                     continue
 
             self.in_limbo = None
+        self.flush()
+        self.break_cycles()
 
     def flush(self):
         'Used during shutdown to ensure that a dirtied book is not missed'

From 14bffa2fdd3abe1312f3375d998b4e88d7e0e0b3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 13:14:23 -0700
Subject: [PATCH 43/52] Ensure the db call in check for expired news happens in
 the GUI thread

---
 src/calibre/gui2/dialogs/scheduler.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py
index 572bbcf1c4..8aa624cacc 100644
--- a/src/calibre/gui2/dialogs/scheduler.py
+++ b/src/calibre/gui2/dialogs/scheduler.py
@@ -250,22 +250,27 @@ class Scheduler(QObject):
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60 * 1000))
-        self.oldest_timer = QTimer()
-        self.connect(self.oldest_timer, SIGNAL('timeout()'), self.oldest_check)
         self.connect(self.timer, SIGNAL('timeout()'), self.check)
         self.oldest = gconf['oldest_news']
-        self.oldest_timer.start(int(60 * 60 * 1000))
         QTimer.singleShot(5 * 1000, self.oldest_check)
         self.database_changed = self.recipe_model.database_changed
 
     def oldest_check(self):
         if self.oldest > 0:
             delta = timedelta(days=self.oldest)
-            ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            try:
+                ids = self.recipe_model.db.tags_older_than(_('News'), delta)
+            except:
+                # Should never happen
+                ids = []
+                import traceback
+                traceback.print_exc()
             if ids:
                 ids = list(ids)
                 if ids:
                     self.delete_old_news.emit(ids)
+        QTimer.singleShot(60 * 60 * 1000, self.oldest_check)
+
 
     def show_dialog(self, *args):
         self.lock.lock()

From 8276d40b8b5e0554cce3c11bebb09d29cd1b6715 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 13:19:27 -0700
Subject: [PATCH 44/52] ...

---
 src/calibre/library/database2.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index f2b2c94e31..ed47abbdb3 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1376,10 +1376,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def tags_older_than(self, tag, delta):
         tag = tag.lower().strip()
         now = nowf()
+        tindex = self.FIELD_MAP['timestamp']
+        gindex = self.FIELD_MAP['tags']
         for r in self.data._data:
             if r is not None:
-                if (now - r[self.FIELD_MAP['timestamp']]) > delta:
-                    tags = r[self.FIELD_MAP['tags']]
+                if (now - r[tindex]) > delta:
+                    tags = r[gindex]
                     if tags and tag in [x.strip() for x in
                             tags.lower().split(',')]:
                         yield r[self.FIELD_MAP['id']]

From 929f91a203cbf7e95056dacfe317538dfcfc424b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 13:47:42 -0700
Subject: [PATCH 45/52] Fix remove_dynamic_categories case handling

---
 src/calibre/library/field_metadata.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 78fe899fa8..93ac607fcf 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -477,6 +477,8 @@ class FieldMetadata(dict):
                 del self._tb_cats[key]
                 if key in self._search_term_map:
                     del self._search_term_map[key]
+                if key.lower() in self._search_term_map:
+                    del self._search_term_map[key.lower()]
 
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1

From 65796db14ca18a22523e3aff8afc23ee885e9437 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 14:40:18 -0700
Subject: [PATCH 46/52] Fix #8579 (Font Size)

---
 src/calibre/gui2/convert/look_and_feel.ui | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui
index 0edc324dc5..3dea1f66d7 100644
--- a/src/calibre/gui2/convert/look_and_feel.ui
+++ b/src/calibre/gui2/convert/look_and_feel.ui
@@ -43,7 +43,7 @@
       <double>0.000000000000000</double>
      </property>
      <property name="maximum">
-      <double>30.000000000000000</double>
+      <double>50.000000000000000</double>
      </property>
      <property name="singleStep">
       <double>1.000000000000000</double>

From ab4ad1be85e4a89ec41e5f4dbd2f4dcd143927e4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 22:17:28 +0000
Subject: [PATCH 47/52] Fix adding user category search terms

---
 src/calibre/library/field_metadata.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 78fe899fa8..cfd95b2e3c 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -7,6 +7,7 @@ import copy
 
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.config import tweaks
+from calibre.utils.icu import lower
 
 class TagsIcons(dict):
     '''
@@ -471,7 +472,8 @@ class FieldMetadata(dict):
             self.custom_label_to_key_map[label+'_index'] = key
 
     def remove_dynamic_categories(self):
-        for key in list(self._tb_cats.keys()):
+        keys = list(self._tb_cats.keys())[:]
+        for key in keys:
             val = self._tb_cats[key]
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
@@ -482,6 +484,7 @@ class FieldMetadata(dict):
         return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
+        label = lower(label)
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
         self._tb_cats[label] = {'table':None,          'column':None,

From 0b0662c63232a11157eda63497e59c2655674669 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 22:20:11 +0000
Subject: [PATCH 48/52] Integrate fix for multiple search terms

---
 src/calibre/library/field_metadata.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 902e43891c..c8706f2137 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -472,15 +472,14 @@ class FieldMetadata(dict):
             self.custom_label_to_key_map[label+'_index'] = key
 
     def remove_dynamic_categories(self):
-        keys = list(self._tb_cats.keys())[:]
-        for key in keys:
+        for key in list(self._tb_cats.keys()):
             val = self._tb_cats[key]
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
                 if key in self._search_term_map:
                     del self._search_term_map[key]
-                if key.lower() in self._search_term_map:
-                    del self._search_term_map[key.lower()]
+                if key in self._search_term_map:
+                    del self._search_term_map[key]
 
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1

From ede45bbc7865b903eb491a4246f4fdc0acd79e5c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 22:40:38 +0000
Subject: [PATCH 49/52] Another 'right' fix for the multiple user category
 problem

---
 src/calibre/library/field_metadata.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index c8706f2137..474f70d500 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -485,7 +485,6 @@ class FieldMetadata(dict):
         return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
-        label = lower(label)
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
         self._tb_cats[label] = {'table':None,          'column':None,
@@ -525,7 +524,6 @@ class FieldMetadata(dict):
     def _add_search_terms_to_map(self, key, terms):
         if terms is not None:
             for t in terms:
-                t = t.lower()
                 if t in self._search_term_map:
                     raise ValueError('Attempt to add duplicate search term "%s"'%t)
                 self._search_term_map[t] = key

From bb063d655b0eadcceca2aa6e4fc18d3ce86b4073 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 15:55:12 -0700
Subject: [PATCH 50/52] ...

---
 resources/calibre-portable.bat | 101 ++++++++++++++++++++-------------
 1 file changed, 63 insertions(+), 38 deletions(-)

diff --git a/resources/calibre-portable.bat b/resources/calibre-portable.bat
index 473cdc4236..f22c72cd8c 100644
--- a/resources/calibre-portable.bat
+++ b/resources/calibre-portable.bat
@@ -6,25 +6,37 @@ REM  - Calibre Library Files
 REM  - Calibre Config Files
 REM  - Calibre Metadata database
 REM  - Calibre Source files
+REM  - Calibre Temp Files
 REM By setting the paths correctly it can be used to run:
 REM  - A "portable calibre" off a USB stick.
 REM  - A network installation with local metadata database
 REM    (for performance) and books stored on a network share 
+REM  - A local installation using customised settings
 REM
-REM If trying to run off a USB stick then the following 
-REM folder structure is recommended:
+REM If trying to run off a USB stick then the folder structure
+REM shown below is recommended (relative to the location of 
+REM this batch file).  This can structure can also be used
+REM when running of a local hard disk if you want to get the
+REM level of control this batch file provides.
 REM  - Calibre2			Location of program files
 REM  - CalibreConfig		Location of Configuration files
 REM  - CalibreLibrary		Location of Books and metadata
+REM  - CalibreSource 		Location of Calibre Source files (Optional)
+REM
+REM This batch file is designed so that if you create the recommended
+REM folder structure then it can be used 'as is' without modification.
 
 
 REM -------------------------------------
 REM Set up Calibre Config folder
+REM
+REM This is where user specific settings
+REM are stored.
 REM -------------------------------------
 
 IF EXIST CalibreConfig (
 	SET CALIBRE_CONFIG_DIRECTORY=%cd%\CalibreConfig
-	ECHO CONFIG=%cd%\CalibreConfig
+	ECHO CONFIG FILES:    %cd%\CalibreConfig
 )
 
 
@@ -35,21 +47,18 @@ REM Location where Book files are located
 REM Either set explicit path, or if running from a USB stick
 REM a relative path can be used to avoid need to know the
 REM drive letter of the USB stick.
-
+REM
 REM Comment out any of the following that are not to be used
+REM (although leaving them in does not really matter)
 REM --------------------------------------------------------------
 
 IF EXIST U:\eBooks\CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=U:\eBOOKS\CalibreLibrary
-	ECHO LIBRARY=U:\eBOOKS\CalibreLibrary
+	ECHO LIBRARY FILES:   U:\eBOOKS\CalibreLibrary
 )
 IF EXIST CalibreLibrary (
 	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreLibrary
-	ECHO LIBRARY=%cd%\CalibreLibrary
-)
-IF EXIST CalibreBooks (
-	SET CALIBRE_LIBRARY_DIRECTORY=%cd%\CalibreBooks
-	ECHO LIBRARY=%cd%\CalibreBooks
+	ECHO LIBRARY FILES:   %cd%\CalibreLibrary
 )
 
 
@@ -60,7 +69,7 @@ REM Location where the metadata.db file is located.  If not set
 REM the same location as Books files will be assumed.  This.
 REM options is used to get better performance when the Library is
 REM on a (slow) network drive.  Putting the metadata.db file 
-REM locally makes gives a big performance improvement.
+REM locally then makes gives a big performance improvement.
 REM
 REM NOTE.  If you use this option, then the ability to switch
 REM        libraries within Calibre will be disabled.  Therefore
@@ -68,19 +77,10 @@ REM        you do not want to set it if the metadata.db file
 REM        is at the same location as the book files.
 REM --------------------------------------------------------------
 
-IF EXIST CalibreBooks (
-	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreBooks" (
-		SET SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreBooks\metadata.db
-		ECHO DATABASE=%cd%\CalibreBooks\metadata.db
-		ECHO '
-		ECHO ***CAUTION*** Library Switching will be disabled 
-		ECHO '
-	)
-)
-IF EXIST CalibreMetadata (
+IF EXIST %cd%\CalibreMetadata\metadata.db (
 	IF NOT "%CALIBRE_LIBRARY_DIRECTORY%" == "%cd%\CalibreMetadata" (
 		SET CALIBRE_OVERRIDE_DATABASE_PATH=%cd%\CalibreMetadata\metadata.db
-		ECHO DATABASE=%cd%\CalibreMetadata\metadata.db
+		ECHO DATABASE:        %cd%\CalibreMetadata\metadata.db
 		ECHO '
 		ECHO ***CAUTION*** Library Switching will be disabled 
 		ECHO '
@@ -96,37 +96,60 @@ REM When running from source the GUI will have a '*' after the version.
 REM number that is displayed at the bottom of the Calibre main screen.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre\src (
-	SET CALIBRE_DEVELOP_FROM=%cd%\Calibre\src
-	ECHO SOURCE=%cd%\Calibre\src
-)
-IF EXIST D:\Calibre\Calibre\src (
-	SET CALIBRE_DEVELOP_FROM=D:\Calibre\Calibre\src
-	ECHO SOURCE=D:\Calibre\Calibre\src
+IF EXIST CalibreSource\src (
+	SET CALIBRE_DEVELOP_FROM=%cd%\CalibreSource\src
+	ECHO SOURCE FILES:    %cd%\CalibreSource\src
 )
 
+
 REM --------------------------------------------------------------
 REM Specify Location of calibre binaries (optional)
 REM
 REM To avoid needing Calibre to be set in the search path, ensure
 REM that Calibre Program Files is current directory when starting.
 REM The following test falls back to using search path .
-REM This folder can be populated by cpying the Calibre2 folder from
-REM an existing isntallation or by isntalling direct to here.
+REM This folder can be populated by copying the Calibre2 folder from
+REM an existing installation or by installing direct to here.
 REM --------------------------------------------------------------
 
-IF EXIST Calibre2 (
-	Calibre2 CD Calibre2
-	ECHO PROGRAMS=%cd%
+IF EXIST %cd%\Calibre2 (
+	CD %cd%\Calibre2
+	ECHO PROGRAM FILES:   %cd%
 )
 
+
+REM --------------------------------------------------------------
+REM Location of Calibre Temporary files  (optional)
+REM
+REM Calibre creates a lot of temproary files while running
+REM In theory these are removed when Calibre finishes, but
+REM in practise files can be left behind (particularily if
+REM any errors occur.  Using this option allows some
+REM explicit clean-up of these files.
+REM If not set Calibre uses the normal system TEMP location
+REM --------------------------------------------------------------
+
+SET CALIBRE_TEMP_DIR=%TEMP%\CALIBRE_TEMP
+ECHO TEMPORARY FILES: %CALIBRE_TEMP_DIR%
+
+IF NOT "%CALIBRE_TEMP_DIR%" == "" (
+	IF EXIST "%CALIBRE_TEMP_DIR%" RMDIR /s /q "%CALIBRE_TEMP_DIR%"
+	MKDIR "%CALIBRE_TEMP_DIR%"
+	REM set the following for any components that do
+	REM not obey the CALIBRE_TEMP_DIR setting
+	SET TMP=%CALIBRE_TEMP_DIR%
+	SET TEMP=%CALIBRE_TEMP_DIR%
+)
+
+
 REM ----------------------------------------------------------
 REM  The following gives a chance to check the settings before
 REM  starting Calibre.  It can be commented out if not wanted.
 REM ----------------------------------------------------------
 
-echo "Press CTRL-C if you do not want to continue"
-pause
+ECHO '
+ECHO "Press CTRL-C if you do not want to continue"
+PAUSE
 
 
 REM --------------------------------------------------------
@@ -141,5 +164,7 @@ REM If used without /WAIT opotion launches Calibre and contines batch file.
 REM Use with /WAIT to wait until Calibre completes to run a task on exit
 REM --------------------------------------------------------
 
-echo "Starting up Calibre"
-START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"
+ECHO "Starting up Calibre"
+ECHO OFF
+ECHO %cd%
+START /belownormal Calibre --with-library "%CALIBRE_LIBRARY_DIRECTORY%"
\ No newline at end of file

From 9006bf624db25b44d9003b74d462564d39608977 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 25 Jan 2011 16:00:12 -0700
Subject: [PATCH 51/52] ...

---
 src/calibre/library/field_metadata.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 474f70d500..a7d05d396a 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -7,7 +7,6 @@ import copy
 
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.config import tweaks
-from calibre.utils.icu import lower
 
 class TagsIcons(dict):
     '''
@@ -478,8 +477,6 @@ class FieldMetadata(dict):
                 del self._tb_cats[key]
                 if key in self._search_term_map:
                     del self._search_term_map[key]
-                if key in self._search_term_map:
-                    del self._search_term_map[key]
 
     def cc_series_index_column_for(self, key):
         return self._tb_cats[key]['rec_index'] + 1

From df5766f02aae244028590b8274737992932d49e5 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 25 Jan 2011 23:29:00 +0000
Subject: [PATCH 52/52] Another 'right' fix for the multiple user category
 problem

---
 src/calibre/library/caches.py         | 7 ++++++-
 src/calibre/library/field_metadata.py | 3 ---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index af47a79e49..7c935a4320 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -424,6 +424,11 @@ class ResultCache(SearchQueryParser): # {{{
         if self.db_prefs is None:
             return  res
         user_cats = self.db_prefs.get('user_categories', [])
+        # translate the case of the location
+        for loc in user_cats:
+            if location == icu_lower(loc):
+                location = loc
+                break
         if location not in user_cats:
             return res
         c = set(candidates)
@@ -445,7 +450,7 @@ class ResultCache(SearchQueryParser): # {{{
         if query and query.strip():
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
-            location = self.field_metadata.search_term_to_field_key(location.lower().strip())
+            location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index c8706f2137..224b6aa79f 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -7,7 +7,6 @@ import copy
 
 from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.config import tweaks
-from calibre.utils.icu import lower
 
 class TagsIcons(dict):
     '''
@@ -485,7 +484,6 @@ class FieldMetadata(dict):
         return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
-        label = lower(label)
         if label in self._tb_cats:
             raise ValueError('Duplicate user field [%s]'%(label))
         self._tb_cats[label] = {'table':None,          'column':None,
@@ -525,7 +523,6 @@ class FieldMetadata(dict):
     def _add_search_terms_to_map(self, key, terms):
         if terms is not None:
             for t in terms:
-                t = t.lower()
                 if t in self._search_term_map:
                     raise ValueError('Attempt to add duplicate search term "%s"'%t)
                 self._search_term_map[t] = key