merge from trunk

resources/recipes/DrawAndCook.recipe

@@ -1,8 +1,11 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+import re
 
 class DrawAndCook(BasicNewsRecipe):
     title               = 'DrawAndCook'
     __author__          = 'Starson17'
+    __version__         = 'v1.10'
+    __date__            = '13 March 2011'
     description         = 'Drawings of recipes!'
     language            = 'en'
     publisher           = 'Starson17'
@@ -13,6 +16,7 @@ class DrawAndCook(BasicNewsRecipe):
     remove_javascript   = True
     remove_empty_feeds    = True
     cover_url           = 'http://farm5.static.flickr.com/4043/4471139063_4dafced67f_o.jpg'
+    INDEX = 'http://www.theydrawandcook.com'
     max_articles_per_feed = 30
 
     remove_attributes = ['style', 'font']
@@ -34,20 +38,21 @@ class DrawAndCook(BasicNewsRecipe):
         date = ''
         current_articles = []
         soup = self.index_to_soup(url)
-        recipes = soup.findAll('div', attrs={'class': 'date-outer'})
+        featured_major_slider = soup.find(name='div', attrs={'id':'featured_major_slider'})
+        recipes = featured_major_slider.findAll('li', attrs={'data-id': re.compile(r'artwork_entry_\d+', re.DOTALL)})
         for recipe in recipes:
-            title = recipe.h3.a.string
+            page_url = self.INDEX + recipe.a['href']
-            page_url = recipe.h3.a['href']
+            print 'page_url is: ', page_url
+            title = recipe.find('strong').string
+            print 'title is: ', title
             current_articles.append({'title': title, 'url': page_url, 'description':'', 'date':date})
         return current_articles
 
-
+    keep_only_tags     = [dict(name='h1', attrs={'id':'page_title'})
-    keep_only_tags     = [dict(name='h3', attrs={'class':'post-title entry-title'})
+                         ,dict(name='section', attrs={'id':'artwork'})
-                         ,dict(name='div', attrs={'class':'post-body entry-content'})
                          ]
 
-    remove_tags = [dict(name='div', attrs={'class':['separator']})
+    remove_tags = [dict(name='article', attrs={'id':['recipe_actions', 'metadata']})
-                  ,dict(name='div', attrs={'class':['post-share-buttons']})
                   ]
 
     extra_css = '''

resources/recipes/rbc_ru.recipe

@@ -1,24 +1,25 @@
+# -*- coding: utf-8 -*-
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class AdvancedUserRecipe1286819935(BasicNewsRecipe):
+class RBC_ru(BasicNewsRecipe):
     title = u'RBC.ru'
     __author__ = 'A. Chewi'
-    oldest_article = 7
+    description = u'Российское информационное агентство «РосБизнесКонсалтинг» (РБК) - ленты новостей политики, экономики и финансов, аналитические материалы, комментарии и прогнозы, тематические статьи'
-    max_articles_per_feed = 100
+    needs_subscription = False
+    cover_url = 'http://pics.rbc.ru/img/fp_v4/skin/img/logo.gif'
+    cover_margins = (80, 160, '#ffffff')
+    oldest_article = 10
+    max_articles_per_feed = 50
+    summary_length = 200
+    remove_empty_feeds = True
     no_stylesheets = True
+    remove_javascript = True
     use_embedded_content  = False
     conversion_options = {'linearize_tables' : True}
-    remove_attributes = ['style']
     language = 'ru'
     timefmt = ' [%a, %d %b, %Y]'
 
-    keep_only_tags = [dict(name='h2', attrs={}),
-	dict(name='div', attrs={'class': 'box _ga1_on_'}),
-	dict(name='h1', attrs={'class': 'news_section'}),
-	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
-	dict(name='table', attrs={'class': 'newsBody'}),
-	dict(name='h2', attrs={'class': 'black'})]
-
     feeds = [(u'Главные новости', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/mainnews.rss'),
 	(u'Политика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/politics.rss'),
 	(u'Экономика', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/economics.rss'),
@@ -26,6 +27,12 @@ class AdvancedUserRecipe1286819935(BasicNewsRecipe):
 	(u'Происшествия', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/rbc.ru/incidents.rss'),
 	(u'Финансовые новости Quote.rbc.ru', u'http://static.feed.rbc.ru/rbc/internal/rss.rbc.ru/quote.ru/mainnews.rss')]
 
+    keep_only_tags = [dict(name='h2', attrs={}),
+	dict(name='div', attrs={'class': 'box _ga1_on_'}),
+	dict(name='h1', attrs={'class': 'news_section'}),
+	dict(name='div', attrs={'class': 'news_body dotted_border_bottom'}),
+	dict(name='table', attrs={'class': 'newsBody'}),
+	dict(name='h2', attrs={'class': 'black'})]
 
     remove_tags = [dict(name='div', attrs={'class': "video-frame"}),
 	dict(name='div', attrs={'class': "photo-container videoContainer videoSWFLinks videoPreviewSlideContainer notes"}),

src/calibre/devices/__init__.py

@@ -47,7 +47,7 @@ def get_connected_device():
 
     for d in connected_devices:
         try:
-            d.open()
+            d.open(None)
         except:
             continue
         else:
@@ -121,7 +121,7 @@ def debug(ioreg_to_tmp=False, buf=None):
             out('Trying to open', dev.name, '...', end=' ')
             try:
                 dev.reset(detected_device=det)
-                dev.open()
+                dev.open(None)
                 out('OK')
             except:
                 import traceback

src/calibre/devices/android/driver.py

@@ -48,6 +48,7 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x6877 : [0x0400],
                      },
 
             # Acer

src/calibre/devices/prs505/driver.py

@@ -221,7 +221,8 @@ class PRS505(USBMS):
                                           os.path.splitext(os.path.basename(p))[0],
                                           book, p)
                     except:
-                        debug_print('FAILED to upload cover', p)
+                        debug_print('FAILED to upload cover',
+                                prefix, book.lpath)
         else:
             debug_print('PRS505: NOT uploading covers in sync_booklists')
 

src/calibre/devices/usbms/driver.py

@@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented
 for a particular device.
 '''
 
-import os, re, time, json, uuid
+import os, re, time, json, uuid, functools
 from itertools import cycle
 
 from calibre.constants import numeric_version
@@ -372,15 +372,21 @@ class USBMS(CLI, Device):
 
     @classmethod
     def build_template_regexp(cls):
-        def replfunc(match):
+        def replfunc(match, seen=None):
-            if match.group(1) in ['title', 'series', 'series_index', 'isbn']:
+            v = match.group(1)
-                return '(?P<' + match.group(1) + '>.+?)'
+            if v in ['title', 'series', 'series_index', 'isbn']:
-            elif match.group(1) in ['authors', 'author_sort']:
+                if v not in seen:
-                return '(?P<author>.+?)'
+                    seen |= set([v])
-            else:
+                    return '(?P<' + v + '>.+?)'
-                return '(.+?)'
+            elif v in ['authors', 'author_sort']:
+                if v not in seen:
+                    seen |= set([v])
+                    return '(?P<author>.+?)'
+            return '(.+?)'
+        s = set()
+        f = functools.partial(replfunc, seen=s)
         template = cls.save_template().rpartition('/')[2]
-        return re.compile(re.sub('{([^}]*)}', replfunc, template) + '([_\d]*$)')
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
 
     @classmethod
     def path_to_unicode(cls, path):

src/calibre/ebooks/metadata/book/base.py

@@ -92,8 +92,6 @@ class Metadata(object):
     def is_null(self, field):
         null_val = NULL_VALUES.get(field, None)
         val = getattr(self, field, None)
-        if val is False or val in (0, 0.0):
-            return True
         return not val or val == null_val
 
     def __getattribute__(self, field):
@@ -129,6 +127,8 @@ class Metadata(object):
             field, val = self._clean_identifier(field, val)
             _data['identifiers'].update({field: val})
         elif field == 'identifiers':
+            if not val:
+                val = copy.copy(NULL_VALUES.get('identifiers', None))
             self.set_identifiers(val)
         elif field in STANDARD_METADATA_FIELDS:
             if val is None:

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -783,6 +783,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
             books_to_refresh = self.db.set_custom(id, val, label=dfm['label'],
                                                   extra=extra, commit=False,
                                                   allow_case_change=True)
+        elif dest.startswith('#') and dest.endswith('_index'):
+            label = self.db.field_metadata[dest[:-6]]['label']
+            series = self.db.get_custom(id, label=label, index_is_id=True)
+            books_to_refresh = self.db.set_custom(id, series, label=label,
+                                                  extra=val, commit=False,
+                                                  allow_case_change=True)
         else:
             if dest == 'comments':
                 setter = self.db.set_comment

src/calibre/gui2/dialogs/saved_search_editor.py

@@ -9,12 +9,13 @@ from PyQt4.QtGui import QDialog
 from calibre.gui2.dialogs.saved_search_editor_ui import Ui_SavedSearchEditor
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
 
-    def __init__(self, window, initial_search=None):
+    def __init__(self, parent, initial_search=None):
-        QDialog.__init__(self, window)
+        QDialog.__init__(self, parent)
         Ui_SavedSearchEditor.__init__(self)
         self.setupUi(self)
 
@@ -22,12 +23,13 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
         self.connect(self.search_name_box, SIGNAL('currentIndexChanged(int)'),
                                     self.current_index_changed)
         self.connect(self.delete_search_button, SIGNAL('clicked()'), self.del_search)
+        self.rename_button.clicked.connect(self.rename_search)
 
         self.current_search_name = None
         self.searches = {}
-        self.searches_to_delete = []
         for name in saved_searches().names():
             self.searches[name] = saved_searches().lookup(name)
+        self.search_names = set([icu_lower(n) for n in saved_searches().names()])
 
         self.populate_search_list()
         if initial_search is not None and initial_search in self.searches:
@@ -42,6 +44,11 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
         search_name = unicode(self.input_box.text()).strip()
         if search_name == '':
             return False
+        if icu_lower(search_name) in self.search_names:
+            error_dialog(self, _('Saved search already exists'),
+                     _('The saved search %s already exists, perhaps with '
+                       'different case')%search_name).exec_()
+            return False
         if search_name not in self.searches:
             self.searches[search_name] = ''
             self.populate_search_list()
@@ -57,10 +64,25 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
                         +'</p>', 'saved_search_editor_delete', self):
                 return
             del self.searches[self.current_search_name]
-            self.searches_to_delete.append(self.current_search_name)
             self.current_search_name = None
             self.search_name_box.removeItem(self.search_name_box.currentIndex())
 
+    def rename_search(self):
+        new_search_name = unicode(self.input_box.text()).strip()
+        if new_search_name == '':
+            return False
+        if icu_lower(new_search_name) in self.search_names:
+            error_dialog(self, _('Saved search already exists'),
+                    _('The saved search %s already exists, perhaps with '
+                      'different case')%new_search_name).exec_()
+            return False
+        if self.current_search_name in self.searches:
+            self.searches[new_search_name] = self.searches[self.current_search_name]
+            del self.searches[self.current_search_name]
+            self.populate_search_list()
+            self.select_search(new_search_name)
+        return True
+
     def select_search(self, name):
         self.search_name_box.setCurrentIndex(self.search_name_box.findText(name))
 
@@ -78,7 +100,7 @@ class SavedSearchEditor(QDialog, Ui_SavedSearchEditor):
     def accept(self):
         if self.current_search_name:
             self.searches[self.current_search_name] = unicode(self.search_text.toPlainText())
-        for name in self.searches_to_delete:
+        for name in saved_searches().names():
             saved_searches().delete(name)
         for name in self.searches:
             saved_searches().add(name, self.searches[name])

src/calibre/gui2/dialogs/saved_search_editor.ui

@@ -134,6 +134,20 @@
        </property>
       </widget>
      </item>
+     <item row="0" column="6">
+      <widget class="QToolButton" name="rename_button">
+       <property name="toolTip">
+        <string>Rename the current search to what is in the box</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset>
+         <normaloff>:/images/edit-undo.png</normaloff>:/images/edit-undo.png</iconset>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
    <item row="1" column="0">

src/calibre/gui2/preferences/look_feel.py

@@ -67,6 +67,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     if db.field_metadata[k]['is_category'] and
                        db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']])
         choices -= set(['authors', 'publisher', 'formats', 'news', 'identifiers'])
+        choices |= set(['search'])
         self.opt_categories_using_hierarchy.update_items_cache(choices)
         r('categories_using_hierarchy', db.prefs, setting=CommaSeparatedList,
           choices=sorted(list(choices), key=sort_key))

src/calibre/gui2/tag_view.py

@@ -533,7 +533,9 @@ class TagsView(QTreeView): # {{{
             self.setModel(self._model)
         except:
             # The DB must be gone. Set the model to None and hope that someone
-            # will call set_database later. I don't know if this in fact works
+            # will call set_database later. I don't know if this in fact works.
+            # But perhaps a Bad Thing Happened, so print the exception
+            traceback.print_exc()
             self._model = None
             self.setModel(None)
     # }}}
@@ -678,7 +680,8 @@ class TagTreeItem(object): # {{{
                         break
                 elif self.tag.state == TAG_SEARCH_STATES['mark_plusplus'] or\
                         self.tag.state == TAG_SEARCH_STATES['mark_minusminus']:
-                    if self.tag.is_hierarchical and len(self.children):
+                    if self.tag.is_searchable and self.tag.is_hierarchical \
+                                and len(self.children):
                         break
                 else:
                     break
@@ -1258,19 +1261,22 @@ class TagsModel(QAbstractItemModel): # {{{
                                             if t.type != TagTreeItem.CATEGORY])
                         if (comp,tag.category) in child_map:
                             node_parent = child_map[(comp,tag.category)]
-                            node_parent.tag.is_hierarchical = True
+                            node_parent.tag.is_hierarchical = key != 'search'
                         else:
                             if i < len(components)-1:
                                 t = copy.copy(tag)
                                 t.original_name = '.'.join(components[:i+1])
-                                # This 'manufactured' intermediate node can
+                                if key != 'search':
-                                # be searched, but cannot be edited.
+                                    # This 'manufactured' intermediate node can
-                                t.is_editable = False
+                                    # be searched, but cannot be edited.
+                                    t.is_editable = False
+                                else:
+                                    t.is_searchable = t.is_editable = False
                             else:
                                 t = tag
                                 if not in_uc:
                                     t.original_name = t.name
-                            t.is_hierarchical = True
+                            t.is_hierarchical = key != 'search'
                             t.name = comp
                             self.beginInsertRows(category_index, 999999, 1)
                             node_parent = TagTreeItem(parent=node_parent, data=t,

src/calibre/library/caches.py

@@ -123,14 +123,22 @@ REGEXP_MATCH   = 2
 def _match(query, value, matchkind):
     if query.startswith('..'):
         query = query[1:]
-        prefix_match_ok = False
+        sq = query[1:]
+        internal_match_ok = True
     else:
-        prefix_match_ok = True
+        internal_match_ok = False
     for t in value:
         t = icu_lower(t)
         try:     ### ignore regexp exceptions, required because search-ahead tries before typing is finished
             if (matchkind == EQUALS_MATCH):
-                if prefix_match_ok and query[0] == '.':
+                if internal_match_ok:
+                    if query == t:
+                        return True
+                    comps = [c.strip() for c in t.split('.') if c.strip()]
+                    for comp in comps:
+                        if sq == comp:
+                            return True
+                elif query[0] == '.':
                     if t.startswith(query[1:]):
                         ql = len(query) - 1
                         if (len(t) == ql) or (t[ql:ql+1] == '.'):

src/calibre/library/database2.py

@@ -56,7 +56,7 @@ class Tag(object):
         self.is_hierarchical = False
         self.is_editable = is_editable
         self.is_searchable = is_searchable
-        self.id_set = id_set
+        self.id_set = id_set if id_set is not None else set([])
         self.avg_rating = avg/2.0 if avg is not None else 0
         self.sort = sort
         if self.avg_rating > 0:
@@ -1691,10 +1691,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return books_to_refresh
 
     def set_metadata(self, id, mi, ignore_errors=False, set_title=True,
-                     set_authors=True, commit=True, force_cover=False,
+                     set_authors=True, commit=True, force_changes=False):
-                     force_tags=False):
         '''
         Set metadata for the book `id` from the `Metadata` object `mi`
+
+        Setting force_changes=True will force set_metadata to update fields even
+        if mi contains empty values. In this case, 'None' is distinguished from
+        'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is.
+        The tags, identifiers, and cover attributes are special cases. Tags and
+        identifiers cannot be set to None so then will always be replaced if
+        force_changes is true. You must ensure that mi contains the values you
+        want the book to have. Covers are always changed if a new cover is
+        provided, but are never deleted. Also note that force_changes has no
+        effect on setting title or authors.
         '''
         if callable(getattr(mi, 'to_book_metadata', None)):
             # Handle code passing in a OPF object instead of a Metadata object
@@ -1708,12 +1717,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     traceback.print_exc()
                 else:
                     raise
-        # force_changes has no role to play in setting title or author
+
+        def should_replace_field(attr):
+            return (force_changes and (mi.get(attr, None) is not None)) or \
+                    not mi.is_null(attr)
+
         path_changed = False
-        if set_title and not mi.is_null('title'):
+        if set_title and mi.title:
             self._set_title(id, mi.title)
             path_changed = True
-        if set_authors and not mi.is_null('authors'):
+        if set_authors:
+            if not mi.authors:
+                    mi.authors = [_('Unknown')]
             authors = []
             for a in mi.authors:
                 authors += string_to_authors(a)
@@ -1722,17 +1737,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if path_changed:
             self.set_path(id, index_is_id=True)
 
-        if not mi.is_null('author_sort'):
+        if should_replace_field('author_sort'):
             doit(self.set_author_sort, id, mi.author_sort, notify=False,
                     commit=False)
-        if not mi.is_null('publisher'):
+        if should_replace_field('publisher'):
             doit(self.set_publisher, id, mi.publisher, notify=False,
                     commit=False)
-        if not mi.is_null('rating'):
+
+        # Setting rating to zero is acceptable.
+        if mi.rating is not None:
             doit(self.set_rating, id, mi.rating, notify=False, commit=False)
-        if not mi.is_null('series'):
+        if should_replace_field('series'):
             doit(self.set_series, id, mi.series, notify=False, commit=False)
 
+        # force_changes has no effect on cover manipulation
         if mi.cover_data[1] is not None:
             doit(self.set_cover, id, mi.cover_data[1], commit=False)
         elif mi.cover is not None:
@@ -1741,36 +1759,45 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     raw = f.read()
                 if raw:
                     doit(self.set_cover, id, raw, commit=False)
-        elif force_cover:
-            doit(self.remove_cover, id, notify=False, commit=False)
 
-        if force_tags or not mi.is_null('tags'):
+        # if force_changes is true, tags are always replaced because the
+        # attribute cannot be set to None.
+        if should_replace_field('tags'):
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
-        if not mi.is_null('comments'):
+
+        if should_replace_field('comments'):
             doit(self.set_comment, id, mi.comments, notify=False, commit=False)
-        if not mi.is_null('series_index'):
+
+        # Setting series_index to zero is acceptable
+        if mi.series_index is not None:
             doit(self.set_series_index, id, mi.series_index, notify=False,
                     commit=False)
-        if not mi.is_null('pubdate'):
+        if should_replace_field('pubdate'):
             doit(self.set_pubdate, id, mi.pubdate, notify=False, commit=False)
         if getattr(mi, 'timestamp', None) is not None:
             doit(self.set_timestamp, id, mi.timestamp, notify=False,
                     commit=False)
 
+        # identifiers will always be replaced if force_changes is True
         mi_idents = mi.get_identifiers()
-        if mi_idents:
+        if force_changes:
+            self.set_identifiers(id, mi_idents, notify=False, commit=False)
+        elif mi_idents:
             identifiers = self.get_identifiers(id, index_is_id=True)
             for key, val in mi_idents.iteritems():
-                if val and val.strip():
+                if val and val.strip(): # Don't delete an existing identifier
                     identifiers[icu_lower(key)] = val
             self.set_identifiers(id, identifiers, notify=False, commit=False)
 
+
         user_mi = mi.get_all_user_metadata(make_copy=False)
         for key in user_mi.iterkeys():
             if key in self.field_metadata and \
                     user_mi[key]['datatype'] == self.field_metadata[key]['datatype']:
-                doit(self.set_custom, id, val=mi.get(key), commit=False,
+                val = mi.get(key, None)
-                     extra=mi.get_extra(key), label=user_mi[key]['label'])
+                if force_changes or val is not None:
+                    doit(self.set_custom, id, val=val, extra=mi.get_extra(key),
+                         label=user_mi[key]['label'], commit=False)
         if commit:
             self.conn.commit()
         self.notify('metadata', [id])