Merge from trunk

recipes/cnn.recipe

@@ -3,71 +3,39 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Profile to download CNN
 '''
+
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 
 class CNN(BasicNewsRecipe):
 
     title = 'CNN'
     description = 'Global news'
     timefmt  = ' [%d %b %Y]'
-    __author__ = 'Krittika Goyal and Sujata Raman'
+    __author__ = 'Kovid Goyal'
     language = 'en'
 
     no_stylesheets = True
     use_embedded_content   = False
     oldest_article        = 15
-    recursions = 1
+    #recursions = 1
-    match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html']
+    #match_regexps = [r'http://sportsillustrated.cnn.com/.*/[1-9].html']
     max_articles_per_feed = 25
 
-    extra_css = '''
+    preprocess_regexps = [
-                .cnn_strycntntlft{font-family :Arial,Helvetica,sans-serif;}
+        (re.compile(r'<!--\[if.*if\]-->', re.DOTALL), lambda m: ''),
-                h2{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
+        (re.compile(r'<script.*?</script>', re.DOTALL), lambda m: ''),
-                .cnnTxtCmpnt{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
+        (re.compile(r'<style.*?</style>', re.DOTALL), lambda m: ''),
-                .cnnTMcontent{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757}
-                .storytext{font-family :Arial,Helvetica,sans-serif; font-size:small}
-                .storybyline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .credit{font-family :Arial,Helvetica,sans-serif; font-size:xx-small; color:#575757}
-                .storyBrandingBanner{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .storytimestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .timestamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#575757}
-                .cnn_strytmstmp{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_stryimg640caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_strylccimg300cntr{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_stryichgfcpt{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnnByline{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .cnn_bulletbin cnnStryHghLght{ font-size:xx-small;}
-                .subhead p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
-                .cnnStoryContent{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .cnnContentContainer{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .col1{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#666666;}
-                .col3{color:#333333; font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;}
-                .cnnInlineT1Caption{font-family :Arial,Helvetica,sans-serif; font-size:x-small;font-weight:bold;}
-                .cnnInlineT1Credit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;}
-                .col10{color:#5A637E;}
-                .cnnInlineRailBulletList{color:black;}
-                .cnnLine0{font-family :Arial,Helvetica,sans-serif; color:#666666;font-weight:bold;}
-                .cnnTimeStamp{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#333333;}
-                .galleryhedDek{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;}
-                .galleryWidgetHeader{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#004276;}
-                .article-content{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                .cnnRecapStory{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
-                h1{font-family :Arial,Helvetica,sans-serif; font-size:x-large}
-                .captionname{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#575757;}
-                inStoryIE{{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
-                '''
-
-    #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
-    remove_tags = [
-       dict(name='iframe'),
-       dict(name='div', attrs={'class':['cnnEndOfStory', 'cnnShareThisItem', 'cnn_strylctcntr cnn_strylctcqrelt', 'cnnShareBoxContent', 'cnn_strybtmcntnt', 'cnn_strycntntrgt']}),
-       dict(name='div', attrs={'id':['IEContainer', 'clickIncludeBox']}),
-       #dict(name='ul', attrs={'class':'article-tools'}),
-       #dict(name='ul', attrs={'class':'articleTools'}),
     ]
 
+    keep_only_tags = [dict(id='cnnContentContainer')]
+    remove_tags = [
+            {'class':['cnn_strybtntools', 'cnn_strylftcntnt',
+                'cnn_strybtntools', 'cnn_strybtntoolsbttm', 'cnn_strybtmcntnt',
+                'cnn_strycntntrgt']},
+    ]
+
+
     feeds =  [
              ('Top News', 'http://rss.cnn.com/rss/cnn_topstories.rss'),
              ('World', 'http://rss.cnn.com/rss/cnn_world.rss'),
@@ -84,15 +52,8 @@ class CNN(BasicNewsRecipe):
              ('Offbeat', 'http://rss.cnn.com/rss/cnn_offbeat.rss'),
              ('Most Popular', 'http://rss.cnn.com/rss/cnn_mostpopular.rss')
              ]
-    def preprocess_html(self, soup):
+
-        story = soup.find(name='div', attrs={'class':'cnnBody_Left'})
+    def get_article_url(self, article):
-        if story is None:
+        ans = BasicNewsRecipe.get_article_url(self, article)
-           story = soup.find(name='div', attrs={'id':'cnnContentContainer'})
+        return ans.partition('?')[0]
-           soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
+
-           body = soup.find(name='body')
-           body.insert(0, story)
-        else:
-           soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
-           body = soup.find(name='body')
-           body.insert(0, story)
-        return soup

recipes/runa.recipe

@@ -1,52 +0,0 @@
-#!/usr/bin/env  python
-
-__license__ = 'GPL v3'
-__author__ = 'Mori'
-__version__ = 'v. 0.1'
-'''
-www.runa.pl/blog
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
-import re
-
-class FantazmatyRecipe(BasicNewsRecipe):
-	__author__ = 'Mori'
-	language = 'pl'
-
-	title = u'Fantazmaty'
-	publisher = u'Agencja Wydawnicza Runa'
-	description = u'Blog Agencji Wydawniczej Runa'
-	
-	no_stylesheets = True
-	remove_javascript = True
-	encoding = 'utf-8'
-	
-	oldest_article = 100
-	max_articles_per_feed = 100
-	
-	extra_css = '''
-		img{float: left; padding-right: 10px; padding-bottom: 5px;}
-	'''
-	
-	feeds = [
-		(u'Fantazmaty', u'http://www.runa.pl/blog/rss.xml')
-	]
-	
-	remove_tags = [
-		dict(name = 'div', attrs = {'class' : 'path'}),
-		dict(name = 'div', attrs = {'class' : 'drdot'}),
-		dict(name = 'div', attrs = {'class' : 'picture'})
-	]
-	
-	remove_tags_after = [
-		dict(name = 'div', attrs = {'class' : 'content'})
-	]
-	
-	preprocess_regexps = [
-		(re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in 
-		[
-			(r'<body>.*?<div id="primary"', lambda match: '<body><div id="primary"'),
-			(r'<!--.*?-->', lambda match: '')
-		]
-	]

src/calibre/devices/android/driver.py

@@ -52,6 +52,7 @@ class ANDROID(USBMS):
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
                        0x681c : [0x0222, 0x0224, 0x0400],
                        0x6640 : [0x0100],
+                       0x685e : [0x0400],
                        0x6877 : [0x0400],
                      },
 
@@ -113,7 +114,8 @@ class ANDROID(USBMS):
             'MB525']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
-            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD']
+            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
+            '__UMS_COMPOSITE']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -2743,7 +2743,6 @@ class ITUNES(DriverBase):
         # Update metadata from plugboard
         # If self.plugboard is None (no transforms), original metadata is returned intact
         metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
-        self.log("metadata.title_sort: %s  metadata_x.title_sort: %s" % (metadata.title_sort, metadata_x.title_sort))
         if isosx:
             if lb_added:
                 lb_added.name.set(metadata_x.title)
@@ -3024,6 +3023,8 @@ class ITUNES_ASYNC(ITUNES):
                 pythoncom.CoInitialize()
                 self._launch_iTunes()
             except:
+                import traceback
+                traceback.print_exc()
                 raise UserFeedback('unable to launch iTunes', details=None, level=UserFeedback.WARN)
             finally:
                 pythoncom.CoUninitialize()

src/calibre/devices/eb600/driver.py

@@ -35,8 +35,8 @@ class EB600(USBMS):
     PRODUCT_ID  = [0x1688]
     BCD         = [0x110]
 
-    VENDOR_NAME      = 'NETRONIX'
+    VENDOR_NAME      = ['NETRONIX', 'WOLDER']
-    WINDOWS_MAIN_MEM = 'EBOOK'
+    WINDOWS_MAIN_MEM = ['EBOOK', 'MIBUK_GAMMA_6.2']
     WINDOWS_CARD_A_MEM = 'EBOOK'
 
     OSX_MAIN_MEM = 'EB600 Internal Storage Media'

src/calibre/devices/nook/driver.py

@@ -115,5 +115,6 @@ class NOOK_TSR(NOOK):
     BCD         = [0x216]
 
     EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'My Files/Books'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOK_DISK'
 
 

src/calibre/ebooks/oeb/iterator.py

@@ -18,7 +18,7 @@ from calibre.ebooks.chardet import xml_to_unicode
 from calibre.utils.zipfile import safe_replace
 from calibre.utils.config import DynamicConfig
 from calibre.utils.logging import Log
-from calibre import guess_type, prints
+from calibre import guess_type, prints, prepare_string_for_xml
 from calibre.ebooks.oeb.transforms.cover import CoverManager
 
 TITLEPAGE = CoverManager.SVG_TEMPLATE.decode('utf-8').replace(\
@@ -229,8 +229,8 @@ class EbookIterator(object):
         cover = self.opf.cover
         if self.ebook_ext in ('lit', 'mobi', 'prc', 'opf', 'fb2') and cover:
             cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
-            chtml = (TITLEPAGE%os.path.relpath(cover, self.base).replace(os.sep,
+            rcpath = os.path.relpath(cover, self.base).replace(os.sep, '/')
-                '/')).encode('utf-8')
+            chtml = (TITLEPAGE%prepare_string_for_xml(rcpath, True)).encode('utf-8')
             open(cfile, 'wb').write(chtml)
             self.spine[0:0] = [SpineItem(cfile,
                 mime_type='application/xhtml+xml')]

src/calibre/ebooks/rtf/rtfml.py

@@ -68,8 +68,13 @@ TODO:
 '''
 
 def txt2rtf(text):
+    # Escape { and } in the text.
+    text = text.replace('{', r'\'7b')
+    text = text.replace('}', r'\'7d')
+
     if not isinstance(text, unicode):
         return text
+
     buf = cStringIO.StringIO()
     for x in text:
         val = ord(x)

src/calibre/gui2/preferences/coloring.py

@@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en'
 from PyQt4.Qt import (QWidget, QDialog, QLabel, QGridLayout, QComboBox, QSize,
         QLineEdit, QIntValidator, QDoubleValidator, QFrame, QColor, Qt, QIcon,
         QScrollArea, QPushButton, QVBoxLayout, QDialogButtonBox, QToolButton,
-        QListView, QAbstractListModel, pyqtSignal)
+        QListView, QAbstractListModel, pyqtSignal, QSizePolicy, QSpacerItem)
 
 from calibre.utils.icu import sort_key
 from calibre.gui2 import error_dialog
@@ -31,6 +31,14 @@ class ConditionEditor(QWidget): # {{{
                     (_('is false'), 'is false'),
                     (_('is undefined'), 'is undefined')
             ),
+            'ondevice' : (
+                    (_('is true'), 'is set',),
+                    (_('is false'), 'is not set'),
+            ),
+            'identifiers' : (
+                (_('has id'), 'has id'),
+                (_('does not have id'), 'does not have id'),
+            ),
             'int' : (
                 (_('is equal to'), 'eq'),
                 (_('is less than'), 'lt'),
@@ -72,7 +80,7 @@ class ConditionEditor(QWidget): # {{{
         self.action_box = QComboBox(self)
         l.addWidget(self.action_box, 0, 3)
 
-        self.l3 = l3 = QLabel(_(' the value '))
+        self.l3 = l3 = QLabel(_(' value '))
         l.addWidget(l3, 0, 4)
 
         self.value_box = QLineEdit(self)
@@ -81,7 +89,7 @@ class ConditionEditor(QWidget): # {{{
         self.column_box.addItem('', '')
         for key in sorted(
                 conditionable_columns(fm),
-                key=lambda x:sort_key(fm[x]['name'])):
+                key=sort_key):
             self.column_box.addItem(key, key)
         self.column_box.setCurrentIndex(0)
 
@@ -155,7 +163,12 @@ class ConditionEditor(QWidget): # {{{
         if dt in self.action_map:
             actions = self.action_map[dt]
         else:
-            k = 'multiple' if m['is_multiple'] else 'single'
+            if col == 'ondevice':
+                k = 'ondevice'
+            elif col == 'identifiers':
+                k = 'identifiers'
+            else:
+                k = 'multiple' if m['is_multiple'] else 'single'
             actions = self.action_map[k]
 
         for text, key in actions:
@@ -176,7 +189,10 @@ class ConditionEditor(QWidget): # {{{
         if not col or not action:
             return
         tt = ''
-        if dt in ('int', 'float', 'rating'):
+        if col == 'identifiers':
+            tt = _('Enter either an identifier type or an '
+                    'identifier type and value of the form identifier:value')
+        elif dt in ('int', 'float', 'rating'):
             tt = _('Enter a number')
             v = QIntValidator if dt == 'int' else QDoubleValidator
             self.value_box.setValidator(v(self.value_box))
@@ -184,9 +200,12 @@ class ConditionEditor(QWidget): # {{{
             self.value_box.setInputMask('9999-99-99')
             tt = _('Enter a date in the format YYYY-MM-DD')
         else:
-            tt = _('Enter a string')
+            tt = _('Enter a string.')
             if 'pattern' in action:
                 tt = _('Enter a regular expression')
+            elif m.get('is_multiple', False):
+                tt += '\n' + _('You can match multiple values by separating'
+                        ' them with %s')%m['is_multiple']
         self.value_box.setToolTip(tt)
         if action in ('is set', 'is not set', 'is true', 'is false',
                 'is undefined'):
@@ -207,11 +226,11 @@ class RuleEditor(QDialog): # {{{
 
         self.l1 = l1 = QLabel(_('Create a coloring rule by'
             ' filling in the boxes below'))
-        l.addWidget(l1, 0, 0, 1, 4)
+        l.addWidget(l1, 0, 0, 1, 5)
 
         self.f1 = QFrame(self)
         self.f1.setFrameShape(QFrame.HLine)
-        l.addWidget(self.f1, 1, 0, 1, 4)
+        l.addWidget(self.f1, 1, 0, 1, 5)
 
         self.l2 = l2 = QLabel(_('Set the color of the column:'))
         l.addWidget(l2, 2, 0)
@@ -220,37 +239,36 @@ class RuleEditor(QDialog): # {{{
         l.addWidget(self.column_box, 2, 1)
 
         self.l3 = l3 = QLabel(_('to'))
-        l3.setAlignment(Qt.AlignHCenter)
         l.addWidget(l3, 2, 2)
 
         self.color_box = QComboBox(self)
         l.addWidget(self.color_box, 2, 3)
+        l.addItem(QSpacerItem(10, 10, QSizePolicy.Expanding), 2, 4)
 
         self.l4 = l4 = QLabel(
             _('Only if the following conditions are all satisfied:'))
-        l4.setAlignment(Qt.AlignHCenter)
+        l.addWidget(l4, 3, 0, 1, 5)
-        l.addWidget(l4, 3, 0, 1, 4)
 
         self.scroll_area = sa = QScrollArea(self)
         sa.setMinimumHeight(300)
         sa.setMinimumWidth(950)
         sa.setWidgetResizable(True)
-        l.addWidget(sa, 4, 0, 1, 4)
+        l.addWidget(sa, 4, 0, 1, 5)
 
         self.add_button = b = QPushButton(QIcon(I('plus.png')),
                 _('Add another condition'))
-        l.addWidget(b, 5, 0, 1, 4)
+        l.addWidget(b, 5, 0, 1, 5)
         b.clicked.connect(self.add_blank_condition)
 
         self.l5 = l5 = QLabel(_('You can disable a condition by'
             ' blanking all of its boxes'))
-        l.addWidget(l5, 6, 0, 1, 4)
+        l.addWidget(l5, 6, 0, 1, 5)
 
         self.bb = bb = QDialogButtonBox(
                 QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
         bb.accepted.connect(self.accept)
         bb.rejected.connect(self.reject)
-        l.addWidget(bb, 7, 0, 1, 4)
+        l.addWidget(bb, 7, 0, 1, 5)
 
         self.conditions_widget = QWidget(self)
         sa.setWidget(self.conditions_widget)
@@ -264,7 +282,7 @@ class RuleEditor(QDialog): # {{{
 
         for key in sorted(
                 displayable_columns(fm),
-                key=lambda x:sort_key(fm[x]['name'])):
+                key=sort_key):
             name = fm[key]['name']
             if name:
                 self.column_box.addItem(key, key)
@@ -408,7 +426,7 @@ class RulesModel(QAbstractListModel): # {{{
         self.reset()
 
     def rule_to_html(self, col, rule):
-        if isinstance(rule, basestring):
+        if not isinstance(rule, Rule):
             return _('''
             <p>Advanced Rule for column <b>%s</b>:
             <pre>%s</pre>
@@ -422,7 +440,7 @@ class RulesModel(QAbstractListModel): # {{{
 
     def condition_to_html(self, condition):
         return (
-            _('<li>If the <b>%s</b> column <b>%s</b> the value: <b>%s</b>') %
+            _('<li>If the <b>%s</b> column <b>%s</b> value: <b>%s</b>') %
                 tuple(condition))
 
 # }}}
@@ -575,7 +593,7 @@ if __name__ == '__main__':
 
     db = db()
 
-    if True:
+    if False:
         d = RuleEditor(db.field_metadata)
         d.add_blank_condition()
         d.exec_()

src/calibre/gui2/preferences/metadata_sources.py

@@ -283,7 +283,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.fields_model.dataChanged.connect(self.changed_signal)
 
         self.select_all_button.clicked.connect(self.fields_model.select_all)
+        self.select_all_button.clicked.connect(self.changed_signal)
         self.clear_all_button.clicked.connect(self.fields_model.clear_all)
+        self.clear_all_button.clicked.connect(self.changed_signal)
 
     def configure_plugin(self):
         for index in self.sources_view.selectionModel().selectedRows():

src/calibre/gui2/shortcuts.py

@@ -44,9 +44,9 @@ class Customize(QFrame, Ui_Frame):
             clear.clicked.connect(partial(self.clear_clicked, which=x))
 
     def clear_clicked(self, which=0):
-         button = getattr(self, 'button%d'%which)
+        button = getattr(self, 'button%d'%which)
-         button.setText(_('None'))
+        button.setText(_('None'))
-         setattr(self, 'shortcut%d'%which, None)
+        setattr(self, 'shortcut%d'%which, None)
 
     def custom_toggled(self, checked):
         for w in ('1', '2'):

src/calibre/gui2/store/gandalf_plugin.py

@@ -37,7 +37,7 @@ class GandalfStore(BasicStoreConfig, StorePlugin):
     def search(self, query, max_results=10, timeout=60):
         url = 'http://www.gandalf.com.pl/s/'
         values={
-            'search': query.encode('iso8859_2'),
+            'search': query.decode('utf-8').encode('iso8859_2'),
             'dzialx':'11'
             }
 

src/calibre/gui2/store/gutenberg_plugin.py

@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import urllib2
+import urllib
 from contextlib import closing
 
 from lxml import html
@@ -42,7 +42,7 @@ class GutenbergStore(BasicStoreConfig, StorePlugin):
     def search(self, query, max_results=10, timeout=60):
         # Gutenberg's website does not allow searching both author and title.
         # Using a google search so we can search on both fields at once.
-        url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query)
+        url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib.quote_plus(query)
         
         br = browser()
         

src/calibre/gui2/store/legimi_plugin.py

@@ -40,7 +40,7 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.legimi.com/pl/ebooks/?price=any&lang=pl&search=' + urllib.quote_plus(query.encode('utf-8')) + '&sort=relevance'
+        url = 'http://www.legimi.com/pl/ebooks/?price=any&lang=pl&search=' + urllib.quote_plus(query) + '&sort=relevance'
         
         br = browser()
         

src/calibre/gui2/store/manybooks_plugin.py

@@ -7,7 +7,7 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import re
-import urllib2
+import urllib
 from contextlib import closing
 
 from lxml import html
@@ -43,7 +43,7 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin):
         # It also doesn't do a clear job of references authors and
         # secondary titles. Google is also faster.
         # Using a google search so we can search on both fields at once.
-        url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query)
+        url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib.quote_plus(query)
         
         br = browser()
         

src/calibre/gui2/store/nexto_plugin.py

@@ -44,7 +44,7 @@ class NextoStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.nexto.pl/szukaj.xml?search-clause=' + urllib.quote_plus(query.encode('utf-8')) + '&scid=1015'
+        url = 'http://www.nexto.pl/szukaj.xml?search-clause=' + urllib.quote_plus(query) + '&scid=1015'
 
         br = browser()
 

src/calibre/gui2/store/search/search.py

@@ -186,7 +186,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         # Remove excess whitespace.
         query = re.sub(r'\s{2,}', ' ', query)
         query = query.strip()
-        return query
+        return query.encode('utf-8')
 
     def save_state(self):
         self.config['geometry'] = bytearray(self.saveGeometry())

src/calibre/gui2/store/virtualo_plugin.py

@@ -35,7 +35,7 @@ class VirtualoStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://virtualo.pl/c2/?q=' + urllib.quote(query.encode('utf-8'))
+        url = 'http://virtualo.pl/c2/?q=' + urllib.quote(query)
 
         br = browser()
 

src/calibre/library/coloring.py

@@ -70,6 +70,12 @@ class Rule(object): # {{{
         m = self.fm[col]
         dt = m['datatype']
 
+        if col == 'ondevice':
+            return self.ondevice_condition(col, action, val)
+
+        if col == 'identifiers':
+            return self.identifiers_condition(col, action, val)
+
         if dt == 'bool':
             return self.bool_condition(col, action, val)
 
@@ -85,6 +91,17 @@ class Rule(object): # {{{
                 return self.multiple_condition(col, action, val, ism)
             return self.text_condition(col, action, val)
 
+    def identifiers_condition(self, col, action, val):
+        if action == 'has id':
+            return "identifier_in_list(field('identifiers'), '%s', '1', '')"%val
+        return "identifier_in_list(field('identifiers'), '%s', '', '1')"%val
+
+    def ondevice_condition(self, col, action, val):
+        if action == 'is set':
+            return "test(ondevice(), '1', '')"
+        if action == 'is not set':
+            return "test(ondevice(), '', '1')"
+
     def bool_condition(self, col, action, val):
         test = {'is true': 'True',
                 'is false': 'False',
@@ -98,7 +115,7 @@ class Rule(object): # {{{
                 'gt': ('', '', '1')
         }[action]
         lt, eq, gt = '', '1', ''
-        return "cmp(field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
+        return "cmp(raw_field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt)
 
     def date_condition(self, col, action, val):
         lt, eq, gt = {

src/calibre/library/database2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid
+        json, uuid, tempfile
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -591,11 +591,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     f.write(cdata)
             for format in formats:
                 # Get data as string (can't use file as source and target files may be the same)
-                f = self.format(id, format, index_is_id=True, as_file=False)
+                f = self.format(id, format, index_is_id=True, as_file=True)
-                if not f:
+                if f is None:
                     continue
-                stream = cStringIO.StringIO(f)
+                with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
-                self.add_format(id, format, stream, index_is_id=True,
+                    shutil.copyfileobj(f, stream)
+                    stream.seek(0)
+                    self.add_format(id, format, stream, index_is_id=True,
                         path=tpath, notify=False)
         self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
         self.dirtied([id], commit=False)

src/calibre/manual/portable.rst

@@ -11,6 +11,7 @@ You can "install" calibre onto a USB stick that you can take with you and use on
     * Run a Mobile Calibre installation with both the Calibre binaries and your ebook library resident on a USB disk or other portable media. In particular it is not necessary to have Calibre installed on the Windows PC that is to run Calibre. This batch file also does not care what drive letter is assigned when you plug in the USB device. It also will not affect any settings on the host machine being a completely self-contained Calibre installation.
     * Run a networked Calibre installation optimised for performance when the ebook files are located on a networked share.
 
+If you find setting up the bat file too challenging, there is a third party portable calibre build available at `portableapps.com http://portableapps.com`_.
 
 This calibre-portable.bat file is intended for use on Windows based systems, but the principles are easily adapted for use on Linux or OS X based systems. Note that calibre requires the Microsoft Visual C++ 2008 runtimes to run. Most windows computers have them installed already, but it may be a good idea to have the installer for installing them on your USB stick. The installer is available from `Microsoft <http://www.microsoft.com/downloads/details.aspx?FamilyID=9b2da534-3e03-4391-8a4d-074b9f2bc1bf&displaylang=en>`_.