KG updates

resources/recipes/wash_post.recipe

@@ -21,16 +21,20 @@ class WashingtonPost(BasicNewsRecipe):
                         body{font-family:arial,helvetica,sans-serif}
                             '''
 
-    feeds = [    ('Today\'s Highlights', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/03/24/LI2005032400102.xml'),
+    feeds = [   ('Today\'s Highlights', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/03/24/LI2005032400102.xml'),
-     	         ('Politics', 'http://www.washingtonpost.com/wp-dyn/rss/politics/index.xml'),
+                ('Politics', 'http://www.washingtonpost.com/wp-dyn/rss/politics/index.xml'),
-     	         ('Nation', 'http://www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
+                ('Nation', 'http://www.washingtonpost.com/wp-dyn/rss/nation/index.xml'),
-     	         ('World', 'http://www.washingtonpost.com/wp-dyn/rss/world/index.xml'),
+                ('World', 'http://www.washingtonpost.com/wp-dyn/rss/world/index.xml'),
-     	         ('Business', 'http://www.washingtonpost.com/wp-dyn/rss/business/index.xml'),
+                ('Business', 'http://www.washingtonpost.com/wp-dyn/rss/business/index.xml'),
-     	         ('Technology', 'http://www.washingtonpost.com/wp-dyn/rss/technology/index.xml'),
+                ('Technology', 'http://www.washingtonpost.com/wp-dyn/rss/technology/index.xml'),
-     	         ('Health', 'http://www.washingtonpost.com/wp-dyn/rss/health/index.xml'),
+                ('Health', 'http://www.washingtonpost.com/wp-dyn/rss/health/index.xml'),
-     	         ('Education', 'http://www.washingtonpost.com/wp-dyn/rss/education/index.xml'),
+                ('Education', 'http://www.washingtonpost.com/wp-dyn/rss/education/index.xml'),
-     	         ('Editorials', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/05/30/LI2005053000331.xml'),
+                ('Style',
-     	]
+                     'http://www.washingtonpost.com/wp-dyn/rss/print/style/index.xml'),
+                ('Sports',
+                     'http://feeds.washingtonpost.com/wp-dyn/rss/linkset/2010/08/19/LI2010081904067_xml'),
+                ('Editorials', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/05/30/LI2005053000331.xml'),
+    ]
 
     remove_tags = [{'id':['pfmnav', 'ArticleCommentsWrapper']}]
 

src/calibre/__init__.py

@@ -455,6 +455,24 @@ def prepare_string_for_xml(raw, attribute=False):
 def isbytestring(obj):
     return isinstance(obj, (str, bytes))
 
+def force_unicode(obj, enc=preferred_encoding):
+    if isbytestring(obj):
+        try:
+            obj = obj.decode(enc)
+        except:
+            try:
+                obj = obj.decode(filesystem_encoding if enc ==
+                        preferred_encoding else preferred_encoding)
+            except:
+                try:
+                    obj = obj.decode('utf-8')
+                except:
+                    obj = repr(obj)
+                    if isbytestring(obj):
+                        obj = obj.decode('utf-8')
+    return obj
+
+
 def human_readable(size):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"

src/calibre/devices/folder_device/driver.py

@@ -31,7 +31,7 @@ class FOLDER_DEVICE(USBMS):
     description    = _('Use an arbitrary folder as a device.')
     author         = 'John Schember/Charles Haley'
     supported_platforms = ['windows', 'osx', 'linux']
-    FORMATS     = ['epub', 'fb2', 'mobi', 'azw', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb']
+    FORMATS     = FOLDER_DEVICE_FOR_CONFIG.FORMATS
 
     VENDOR_ID   = 0xffff
     PRODUCT_ID  = 0xffff

src/calibre/ebooks/oeb/base.py

@@ -654,8 +654,6 @@ class Metadata(object):
             if predicate(x):
                 l.remove(x)
 
-
-
     def __getitem__(self, key):
         return self.items[key]
 

src/calibre/ebooks/oeb/reader.py

@@ -132,17 +132,23 @@ class OEBReader(object):
         if not mi.language:
             mi.language = get_lang().replace('_', '-')
         self.oeb.metadata.add('language', mi.language)
-        if not mi.title:
-            mi.title = self.oeb.translate(__('Unknown'))
-        if not mi.authors:
-            mi.authors = [self.oeb.translate(__('Unknown'))]
         if not mi.book_producer:
-            mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\
+            mi.book_producer = '%(a)s (%(v)s) [http://%(a)s-ebook.com]'%\
                 dict(a=__appname__, v=__version__)
         meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger)
-        self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id',
+        m = self.oeb.metadata
-                                scheme='uuid')
+        m.add('identifier', str(uuid.uuid4()), id='uuid_id', scheme='uuid')
         self.oeb.uid = self.oeb.metadata.identifier[-1]
+        if not m.title:
+            m.add('title', self.oeb.translate(__('Unknown')))
+        has_aut = False
+        for x in m.creator:
+            if getattr(x, 'role', '').lower() in ('', 'aut'):
+                has_aut = True
+                break
+        if not has_aut:
+            m.add('creator', self.oeb.translate(__('Unknown')), role='aut')
+
 
     def _manifest_prune_invalid(self):
         '''

src/calibre/gui2/__init__.py

@@ -3,6 +3,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
 import os, sys, Queue, threading
 from threading import RLock
+from urllib import unquote
 
 from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
                          QByteArray, QTranslator, QCoreApplication, QThread, \
@@ -505,6 +506,11 @@ class FileDialog(QObject):
             fs = QFileDialog.getOpenFileNames(parent, title, initial_dir, ftext, "")
             for f in fs:
                 f = unicode(f)
+                if not f: continue
+                if not os.path.exists(f):
+                    # QFileDialog for some reason quotes spaces
+                    # on linux if there is more than one space in a row
+                    f = unquote(f)
                 if f and os.path.exists(f):
                     self.selected_files.append(f)
         else:

src/calibre/gui2/device.py

@@ -23,7 +23,7 @@ from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
                         warning_dialog, \
                         question_dialog, info_dialog, choose_dir
 from calibre.ebooks.metadata import authors_to_string
-from calibre import preferred_encoding, prints
+from calibre import preferred_encoding, prints, force_unicode
 from calibre.utils.filenames import ascii_filename
 from calibre.devices.errors import FreeSpaceError
 from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
@@ -964,12 +964,12 @@ class DeviceMixin(object): # {{{
         for jobname, exception, tb in results:
             title = jobname.partition(':')[-1]
             if exception is not None:
-                errors.append([title, exception, tb])
+                errors.append(list(map(force_unicode, [title, exception, tb])))
             else:
                 good.append(title)
         if errors:
-            errors = '\n'.join([
+            errors = u'\n'.join([
-                    '%s\n\n%s\n%s\n' %
+                    u'%s\n\n%s\n%s\n' %
                     (title, e, tb) for \
                             title, e, tb in errors
                     ])

src/calibre/gui2/dialogs/check_library.py

@@ -73,7 +73,7 @@ class CheckLibraryDialog(QDialog):
         QDialog.accept(self)
 
     def box_to_list(self, txt):
-        return [f.strip().lower() for f in txt.split(',') if f.strip()]
+        return [f.strip() for f in txt.split(',') if f.strip()]
 
     def run_the_check(self):
         checker = CheckLibrary(self.db.library_path, self.db)

src/calibre/gui2/dialogs/user_profiles.ui

@@ -375,7 +375,7 @@ p, li { white-space: pre-wrap; }
                  <item>
                   <widget class="QLabel" name="label_8">
                    <property name="text">
-                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://__appname__.kovidgoyal.net/user_manual/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
+                    <string>For help with writing advanced news recipes, please visit &lt;a href=&quot;http://__appname__-ebook.com/user_manual/news.html&quot;&gt;User Recipes&lt;/a&gt;</string>
                    </property>
                    <property name="wordWrap">
                     <bool>true</bool>

src/calibre/library/cli.py

@@ -302,7 +302,7 @@ def do_add_empty(db, title, authors, isbn):
     if isbn:
         mi.isbn = isbn
     db.import_book(mi, [])
-    write_dirtied()
+    write_dirtied(db)
     send_message()
 
 def command_add(args, dbpath):
@@ -456,7 +456,7 @@ def do_set_metadata(db, id, stream):
     db.set_metadata(id, mi)
     db.clean()
     do_show_metadata(db, id, False)
-    write_dirtied()
+    write_dirtied(db)
     send_message()
 
 def set_metadata_option_parser():
@@ -930,11 +930,11 @@ def command_check_library(args, dbpath):
     if opts.names is None:
         names = []
     else:
-        names = [f.strip().lower() for f in opts.names.split(',') if f.strip()]
+        names = [f.strip() for f in opts.names.split(',') if f.strip()]
     if opts.exts is None:
         exts = []
     else:
-        exts = [f.strip().lower() for f in opts.exts.split(',') if f.strip()]
+        exts = [f.strip() for f in opts.exts.split(',') if f.strip()]
 
     def print_one(checker, check):
         attr = check[0]
@@ -971,7 +971,7 @@ def restore_database_option_parser():
     files in each directory of the calibre library. This is
     useful if your metadata.db file has been corrupted.
 
-    WARNING: This completely regenrates your datbase. You will
+    WARNING: This completely regenerates your datbase. You will
     lose stored per-book conversion settings and custom recipes.
     '''))
     return parser

src/calibre/library/save_to_disk.py

@@ -17,7 +17,7 @@ from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import preferred_encoding, filesystem_encoding
 from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata import title_sort
-from calibre import strftime
+from calibre import strftime, prints
 
 plugboard_any_device_value = 'any device'
 plugboard_any_format_value = 'any format'
@@ -114,8 +114,14 @@ class SafeFormat(TemplateFormatter):
 
     def get_value(self, key, args, kwargs):
         try:
-            b = self.book.get_user_metadata(key, False)
             key = key.lower()
+            try:
+                b = self.book.get_user_metadata(key, False)
+            except:
+                prints('save_to_disk get value exception')
+                traceback.print_exc()
+                b = None
+
             if b is not None and b['datatype'] == 'composite':
                 if key in self.composite_values:
                     return self.composite_values[key]
@@ -124,9 +130,11 @@ class SafeFormat(TemplateFormatter):
                     self.vformat(b['display']['composite_template'], [], kwargs)
                 return self.composite_values[key]
             if kwargs[key]:
-                return self.sanitize(kwargs[key.lower()])
+                return self.sanitize(kwargs[key])
             return ''
         except:
+            print('save_to_disk general exception')
+            traceback.print_exc()
             return ''
 
 safe_formatter = SafeFormat()

src/calibre/manual/metadata.rst

@@ -9,7 +9,7 @@ Editing E-book Metadata
   :depth: 2
   :local:
 
-E-books come in all shapes and sizes and more often than not, their metadata (things like title/author/series/publisher) is incomplete or incorrect. 
+E-books come in all shapes and sizes and more often than not, their metadata (things like title/author/series/publisher) is incomplete or incorrect.
 The simplest way to change metadata in |app| is to simply double click on an entry and type in the correct replacement.
 For more sophisticated, "power editing" use the edit metadata tools discussed below.
 
@@ -18,7 +18,7 @@ Editing the metadata of one book at a time
 
 Click the book you want to edit and then click the :guilabel:`Edit metadata` button or press the ``E`` key. A dialog opens that allows you to edit all aspects of the metadata. It has various features to make editing faster and more efficient. A list of the commonly used tips:
 
-    * You can click the button in between title and authors to swap them automatically. Or 
+    * You can click the button in between title and authors to swap them automatically. 
     * You can click the button next to author sort to automatically to have |app| automatically fill it from the author name.
     * You can click the button next to tags to use the Tag Editor to manage the tags associated with the book.
     * The ISBN box will have a red background if you enter an invalid ISBN. It will be green for valid ISBNs
@@ -27,9 +27,9 @@ Click the book you want to edit and then click the :guilabel:`Edit metadata` but
 Downloading metadata
 ^^^^^^^^^^^^^^^^^^^^^
 
-The nicest feature of the edit metadata dialog is its ability to automatically fill in many metadata fields by getting metadata from various websites. Currently, |app| uses isbndb.com, Google Books, Amazon and Library Thing. The metadata download can fill in Title, author, series, tags, rating, description and ISBN for you. 
+The nicest feature of the edit metadata dialog is its ability to automatically fill in many metadata fields by getting metadata from various websites. Currently, |app| uses isbndb.com, Google Books, Amazon and Library Thing. The metadata download can fill in Title, author, series, tags, rating, description and ISBN for you.
 
-To use the download, fill in the title and author fields and click the :guilabel:`Fetch metadata` button. |app| will present you with a list of books that most closely match the title and author. If you fill in the ISBN field first, it will be used in preference to the title and author. If no matches are found, try making your search a little less specific by including only some key words in the title and only the author last name. 
+To use the download, fill in the title and author fields and click the :guilabel:`Fetch metadata` button. |app| will present you with a list of books that most closely match the title and author. If you fill in the ISBN field first, it will be used in preference to the title and author. If no matches are found, try making your search a little less specific by including only some key words in the title and only the author last name.
 
 Managing book formats
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -54,9 +54,33 @@ Search and replace
 
 The Bulk metadata edit dialog allows you to perform arbitrarily powerful search and replace operations on the selected books. By default it uses a simple text search and replace, but it also support *regular expressions*. For more on regular expressions, see :ref:`regexptutorial`.
 
+As noted above, there are two search and replace modes: character match and regular expression. Character match will look in the `Search field` you choose for the characters you type in the `search for` box and replace those characters with what you type in the `replace with` box. Each occurance of the search characters in the field will be replaced. For example, assume the field being searched contains `a bad cat`. if you search for `a` to be replaced with `HELLO`, then the result will be `HELLO bHELLOd cHELLOt`.
+
+If the field you are searching on is a `multiple` field like tags, then each tag is treated separately. For example, if your tags contain `Horror, Scary`, the search expression `r,` will not match anything because the expression will first be applied to `Horror` and then to `Scary`.
+
+If you want the search to ignore upper/lowercase differences, uncheck the `Case sensitive` box.
+
+You can have |app| change the case of the result (information after the replace has happened) by choosing one of the functions from the `Apply function after replace` box. The operations available are:
+
+    *`Lower case` -- change all the characters in the field to lower case
+    *`Upper case` -- change all the characters in the field to upper case
+    *`Title case` -- capitalize each word in the result.
+
+The `Your test` box is provided for you to enter text to check that search/replace is doing what you want. In the majority of cases the book test boxes will be sufficient, but it is possible that there is a case you want to check that isn't shown in these boxes. Enter that case into `Your test`.
+
+Regular expression mode has some differences from character mode, beyond (of course) using regular expressions. The first is that functions are applied to the parts of the string matched by the search string, not the entire field. The second is that functions apply to the replacement string, not to the entire field.
+
+The third and most important is that the replace string can make reference to parts of the search string by using backreferences. A backreference is ``\\n`` where n is an integer that refers to the n'th parenthesized group in the search expression. For example, given the same example as above, `a bad cat`, a search expression `a (...) (...)`, and a replace expression `a \\2 \\1`, the result will be `a cat bad`. Please see the :ref:`regexptutorial` for more information on backreferences.
+
+One useful pattern: assume you want to change the case of an entire field. The easiest way to do this is to use character mode, but lets further assume you want to use regular expression mode. The search expression should be `(.*)` the replace expression should be `\1`, and the desired case change function should be selected.
+
+Finally, in regular expression mode you can copy values from one field to another. Simply make the source and destination field different. The copy can replace the destination field, prepend to the field (add to the front), or append to the field (add at the end). The 'use comma' checkbox tells |app| to (or not to) add a comma between the text and the destination field in prepend and append modes. If the destination is multiple (e.g., tags), then you cannot uncheck this box.
+
+Search and replace is done after all the other metadata changes in the other tabs are applied. This can lead to some confusion, because the test boxes will show the information before the other changes, but the operation will be applied after the other changes. If you have any doubts about what is going to happen, do not mix search/replace with other changes.
+
 Bulk downloading of metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you want to download the metadata for multiple books at once, click the arrow next to the :guilabel:`Edit metadata` button and select :guilabel:`Download metadata and covers`. You can choose to download only metadata, only covers, both or only social metadata (tags/rating/series). 
+If you want to download the metadata for multiple books at once, click the arrow next to the :guilabel:`Edit metadata` button and select :guilabel:`Download metadata and covers`. You can choose to download only metadata, only covers, both or only social metadata (tags/rating/series).
 
 

src/calibre/manual/template_lang.rst

@@ -108,7 +108,7 @@ Function references appear in the format part, going after the ``:`` and before
 
 Functions are always applied before format specifications. See further down for an example of using both a format and a function, where this order is demonstrated.
 
-The syntax for using functions is ``{field:function(arguments)}``, or ``{field:function(arguments)|prefix|suffix}``. Argument values cannot contain a comma, because it is used to separate arguments. The last (or only) argument cannot contain a closing parenthesis ( ')' ). Functions return the value of the field used in the template, suitably modified.
+The syntax for using functions is ``{field:function(arguments)}``, or ``{field:function(arguments)|prefix|suffix}``. Arguments are separated by commas. Commas inside arguments must be preceeded by a backslash ( '\\' ). The last (or only) argument cannot contain a closing parenthesis ( ')' ). Functions return the value of the field used in the template, suitably modified.
 
 The functions available are:
 
@@ -119,6 +119,7 @@ The functions available are:
     * ``ifempty(text)``	-- if the field is not empty, return the value of the field. Otherwise return `text`.
     * ``test(text if not empty, text if empty)`` -- return `text if not empty` if the field is not empty, otherwise return `text if empty`.
     * ``contains(pattern, text if match, text if not match`` -- checks if field contains matches for the regular expression `pattern`. Returns `text if match` if matches are found, otherwise it returns `text if no match`.
+    * ``switch(pattern, value, pattern, value, ..., else_value)`` -- for each ``pattern, value`` pair, checks if the field matches the ``pattern`` and if so, returns that ``value``. If no ``pattern`` matches, then ``else_value`` is returned. You can have as many ``pattern, value`` pairs as you want.
     * ``shorten(left chars, middle text, right chars)`` -- Return a shortened version of the field, consisting of `left chars` characters from the beginning of the field, followed by `middle text`, followed by `right chars` characters from the end of the string. `Left chars` and `right chars` must be integers. For example, assume the title of the book is `Ancient English Laws in the Times of Ivanhoe`, and you want it to fit in a space of at most 15 characters. If you use ``{title:shorten(9,-,5)}``, the result will be `Ancient E-nhoe`. If the field's length is less than ``left chars`` + ``right chars`` + the length of ``middle text``, then the field will be used intact. For example, the title `The Dome` would not be changed.
     * ``lookup(field if not empty, field if empty)`` -- like test, except the arguments are field (metadata) names, not text. The value of the appropriate field will be fetched and used. Note that because composite columns are fields, you can use this function in one composite field to use the value of some other composite field. This is extremely useful when constructing variable save paths (more later).
     * ``re(pattern, replacement)`` -- return the field after applying the regular expression. All instances of `pattern` are replaced with `replacement`. As in all of |app|, these are python-compatible regular expressions.

src/calibre/startup.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 Perform various initialization tasks.
 '''
 
-import locale, sys, os
+import locale, sys, os, re
 
 # Default translation is NOOP
 import __builtin__
@@ -129,6 +129,18 @@ if not _run_once:
                     fobject = object.__getattribute__(self, 'fobject')
                     return setattr(fobject, attr, val)
 
+                def __repr__(self):
+                    fobject = object.__getattribute__(self, 'fobject')
+                    name = object.__getattribute__(self, 'name')
+                    return re.sub(r'''['"]<fdopen>['"]''', repr(name),
+                            repr(fobject))
+
+                def __str__(self):
+                    return repr(self)
+
+                def __unicode__(self):
+                    return repr(self).decode('utf-8')
+
 
             m = mode[0]
             random = len(mode) > 1 and mode[1] == '+'

src/calibre/translations/calibre.pot

@@ -5,8 +5,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: calibre 0.7.21\n"
-"POT-Creation-Date: 2010-10-01 14:42+MDT\n"
+"POT-Creation-Date: 2010-10-02 11:26+MDT\n"
-"PO-Revision-Date: 2010-10-01 14:42+MDT\n"
+"PO-Revision-Date: 2010-10-02 11:26+MDT\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@@ -10057,7 +10057,7 @@ msgid ""
 "    files in each directory of the calibre library. This is\n"
 "    useful if your metadata.db file has been corrupted.\n"
 "\n"
-"    WARNING: This completely regenrates your datbase. You will\n"
+"    WARNING: This completely regenerates your datbase. You will\n"
 "    lose stored per-book conversion settings and custom recipes.\n"
 "    "
 msgstr ""

src/calibre/utils/date.py

@@ -120,6 +120,9 @@ def utcfromtimestamp(stamp):
 
 def format_date(dt, format, assume_utc=False, as_utc=False):
     ''' Return a date formatted as a string using a subset of Qt's formatting codes '''
+    if not format:
+        format = 'dd MMM yyyy'
+
     if hasattr(dt, 'tzinfo'):
         if dt.tzinfo is None:
             dt = dt.replace(tzinfo=_utc_tz if assume_utc else

src/calibre/utils/formatter.py

@@ -39,6 +39,15 @@ class TemplateFormatter(string.Formatter):
         else:
             return value_if_not
 
+    def _switch(self, val, *args):
+        i = 0
+        while i < len(args):
+            if i + 1 >= len(args):
+                return args[i]
+            if re.search(args[i], val):
+                return args[i+1]
+            i += 2
+
     def _re(self, val, pattern, replacement):
         return re.sub(pattern, replacement, val)
 
@@ -66,12 +75,19 @@ class TemplateFormatter(string.Formatter):
                     'lookup'        : (2, _lookup),
                     're'            : (2, _re),
                     'shorten'       : (3, _shorten),
+                    'switch'        : (-1, _switch),
                     'test'          : (2, _test),
         }
 
     format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
     compress_spaces = re.compile(r'\s+')
 
+    arg_parser = re.Scanner([
+                (r',', lambda x,t: ''),
+                (r'.*?((?<!\\),)', lambda x,t: t[:-1]),
+                (r'.*?\)', lambda x,t: t[:-1]),
+        ])
+
     def get_value(self, key, args, kwargs):
         raise Exception('get_value must be implemented in the subclass')
 
@@ -105,7 +121,7 @@ class TemplateFormatter(string.Formatter):
             if fmt[colon:p] in self.functions:
                 field = fmt[colon:p]
                 func = self.functions[field]
-                args = fmt[p+1:-1].split(',')
+                args = self.arg_parser.scan(fmt[p+1:])[0]
                 if (func[0] == 0 and (len(args) != 1 or args[0])) or \
                         (func[0] > 0 and func[0] != len(args)):
                     raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])