KG updates

Changelog.yaml

@@ -55,7 +55,7 @@
     - title: "Add search to the plugin preferences dialog"
   
   bug fixes:
-    - title: "Fix a bug that could cause fiels to be lost when changing metadata on east asian windows installs if the title and/or author is very long."
+    - title: "Fix a bug that could cause files to be lost when changing metadata on east asian windows installs if the title and/or author is very long."
       tickets: [8620]
 
     - title: "Tag browser: Fix searching with items in a user category not owrking if the main category is hidden"
@@ -88,7 +88,7 @@
     - title: "Do not discard the result of a conversion if the user opens the edit metadata dialog while the conversion is running"
       tickets: [8672]
 
-    - title: "CHM Input: When the chm file lacks a hhc, lookf for index.html instead"
+    - title: "CHM Input: When the chm file lacks a hhc, look for index.html instead"
       tickets: [8688]
 
     - title: "EPUB Input: Filter some invalid media types from the spine"

resources/default_tweaks.py

@@ -10,7 +10,7 @@ you know what you are doing. If you delete this file, it will be recreated from
 defaults.
 '''
 
-
+#: Auto increment series index
 # The algorithm used to assign a new book in an existing series a series number.
 # New series numbers assigned using this tweak are always integer values, except
 # if a constant non-integer is specified.
@@ -29,10 +29,19 @@ defaults.
 # series_index_auto_increment = 16.5
 series_index_auto_increment = 'next'
 
+#: Add separator after completing an author name
+# Should the completion separator be append
+# to the end of the completed text to
+# automatically begin a new completion operation
+# for authors.
+# Can be either True or False
+authors_completer_append_separator = False
 
+
+#: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
-#  invert: use "fn ln" -> "ln, fn" (the original algorithm)
+#  invert: use "fn ln" -> "ln, fn" (the default algorithm)
 #  copy  : copy author to author_sort without modification
 #  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
 #  nocomma : "fn ln" -> "ln fn" (without the comma)
@@ -41,6 +50,7 @@ series_index_auto_increment = 'next'
 # selecting 'manage authors', and pressing 'Recalculate all author sort values'.
 author_sort_copy_method = 'invert'
 
+#: Use author sort in Tag Browser
 # Set which author field to display in the tags pane (the list of authors,
 # series, publishers etc on the left hand side). The choices are author and
 # author_sort. This tweak affects only what is displayed under the authors
@@ -55,6 +65,7 @@ author_sort_copy_method = 'invert'
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
+#: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
 # name, categories_collapsed_rating_template if sorting by average rating, and
@@ -66,24 +77,25 @@ categories_use_field_for_author_name = 'author'
 # author category will be the name of the author. The sub-values available are:
 #  name: the printable name of the item
 #  count: the number of books that references this item
-#  avg_rating: the averate rating of all the books referencing this item
+#  avg_rating: the average rating of all the books referencing this item
 #  sort: the sort value. For authors, this is the author_sort for that author
 #  category: the category (e.g., authors, series) that the item is in.
 # Note that the "r'" in front of the { is necessary if there are backslashes
 # (\ characters) in the template. It doesn't hurt anything to leave it there
 # even if there aren't any backslashes.
-categories_collapsed_name_template = r'{first.sort:shorten(4,'',0)} - {last.sort:shorten(4,'',0)}'
+categories_collapsed_name_template = r'{first.sort:shorten(4,"",0)} - {last.sort:shorten(4,"",0)}'
 categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
 categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
 
+#: Set boolean custom columns to be tristate
 # Set whether boolean custom columns are two- or three-valued.
 #  Two-values for true booleans
 #  three-values for yes/no/unknown
 # Set to 'yes' for three-values, 'no' for two-values
 bool_custom_columns_are_tristate = 'yes'
 
-
+#: Specify columns to sort the booklist by on startup
 # Provide a set of columns to be sorted on when calibre starts
 #  The argument is None if saved sort history is to be used
 #  otherwise it is a list of column,order pairs. Column is the
@@ -93,6 +105,7 @@ bool_custom_columns_are_tristate = 'yes'
 # title within authors.
 sort_columns_at_startup = None
 
+#; Control how dates are displayed
 # Format to be used for publication date and the timestamp (date).
 #  A string controlling how the publication date is displayed in the GUI
 #  d    the day as number without a leading zero (1 to 31)
@@ -113,6 +126,7 @@ sort_columns_at_startup = None
 gui_pubdate_display_format = 'MMM yyyy'
 gui_timestamp_display_format = 'dd MMM yyyy'
 
+#: Control sorting of titles and series in the display
 # Control title and series sorting in the library view.
 # If set to 'library_order', Leading articles such as The and A will be ignored.
 # If set to 'strictly_alphabetic', the titles will be sorted without processing
@@ -124,6 +138,7 @@ gui_timestamp_display_format = 'dd MMM yyyy'
 # without changing anything is sufficient to change the sort.
 title_series_sorting = 'library_order'
 
+#: Control formatting of title and series when used in templates
 # Control how title and series names are formatted when saving to disk/sending
 # to device. If set to library_order, leading articles such as The and A will
 # be put at the end
@@ -132,6 +147,7 @@ title_series_sorting = 'library_order'
 # strictly_alphabetic, it would remain "The Client".
 save_template_title_series_sorting = 'library_order'
 
+#: Set the list of words considered to be "articles" for sort strings
 # Set the list of words that are to be considered 'articles' when computing the
 # title sort strings. The list is a regular expression, with the articles
 # separated by 'or' bars. Comparisons are case insensitive, and that cannot be
@@ -141,7 +157,7 @@ save_template_title_series_sorting = 'library_order'
 # Default: '^(A|The|An)\s+'
 title_sort_articles=r'^(A|The|An)\s+'
 
-
+#: Specify a folder calibre should connect to at startup
 # Specify a folder that calibre should connect to at startup using
 # connect_to_folder. This must be a full path to the folder. If the folder does
 # not exist when calibre starts, it is ignored. If there are '\' characters in
@@ -151,7 +167,7 @@ title_sort_articles=r'^(A|The|An)\s+'
 #     auto_connect_to_folder = '/home/dropbox/My Dropbox/someone/library'
 auto_connect_to_folder = ''
 
-
+#: Specify renaming rules for SONY collections
 # Specify renaming rules for sony collections. This tweak is only applicable if
 # metadata management is set to automatic. Collections on Sonys are named
 # depending upon whether the field is standard or custom. A collection derived
@@ -204,7 +220,7 @@ auto_connect_to_folder = ''
 sony_collection_renaming_rules={}
 sony_collection_name_template='{value}{category:| (|)}'
 
-
+#: Specify how SONY collections are sorted
 # Specify how sony collections are sorted. This tweak is only applicable if
 # metadata management is set to automatic. You can indicate which metadata is to
 # be used to sort on a collection-by-collection basis. The format of the tweak
@@ -223,7 +239,7 @@ sony_collection_name_template='{value}{category:| (|)}'
 sony_collection_sorting_rules = []
 
 
-# Create search terms to apply a query across several built-in search terms.
+#: Create search terms to apply a query across several built-in search terms.
 # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
 # Example: create the term 'myseries' that when used as myseries:foo would
 # search all of the search categories 'series', '#myseries', and '#myseries2':
@@ -236,15 +252,17 @@ sony_collection_sorting_rules = []
 grouped_search_terms = {}
 
 
-# Set this to True (not 'True') to ensure that tags in 'Tags to add when adding
+#: Control how tags are applied when copying books to another library
+# Set this to True to ensure that tags in 'Tags to add when adding
 # a book' are added when copying books to another library
 add_new_book_tags_when_importing_books = False
 
 
-# Set the maximum number of tags to show per book in the content server
+#: Set the maximum number of tags to show per book in the content server
 max_content_server_tags_shown=5
 
-# Set custom metadata fields that the content server will or will not display.
+
+#: Set custom metadata fields that the content server will or will not display.
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
 # wont_display has priority over will_display.
@@ -262,13 +280,27 @@ max_content_server_tags_shown=5
 content_server_will_display = ['*']
 content_server_wont_display = []
 
-# Same as above (content server) but for the book details pane. Same syntax.
+#: Set custom metadata fields that the book details panel will or will not display.
+# book_details_will_display is a list of custom fields to be displayed.
+# book_details_wont_display is a list of custom fields not to be displayed.
+# wont_display has priority over will_display.
+# The special value '*' means all custom fields. The value [] means no entries.
+# Defaults:
+#    book_details_will_display = ['*']
+#    book_details_wont_display = []
+# Examples:
+# To display only the custom fields #mytags and #genre:
+#   book_details_will_display = ['#mytags', '#genre']
+#   book_details_wont_display = []
+# To display all fields except #mycomments:
+#   book_details_will_display = ['*']
+#   book_details_wont_display['#mycomments']
 # As above, this tweak affects only display of custom fields. The standard
 # fields are not affected
 book_details_will_display = ['*']
 book_details_wont_display = []
 
-
+#: Set the maximum number of sort 'levels'
 # Set the maximum number of sort 'levels' that calibre will use to resort the
 # library after certain operations such as searches or device insertion. Each
 # sort level adds a performance penalty. If the database is large (thousands of
@@ -276,16 +308,14 @@ book_details_wont_display = []
 # level sorts, and if you are seeing a slowdown, reduce the value of this tweak.
 maximum_resort_levels = 5
 
-# Absolute path to a TTF font file to use as the font for the title and author
+#: Specify which font to use when generating a default cover
-# when generating a default cover. Useful if the default font (Liberation
+# Absolute path to .ttf font files to use as the fonts for the title, author
+# and footer when generating a default cover. Useful if the default font (Liberation
 # Serif) does not contain glyphs for the language of the books in your library.
 generate_cover_title_font = None
-
-# Absolute path to a TTF font file to use as the font for the footer in the
-# default cover
 generate_cover_foot_font = None
 
-
+#: Control behavior of double clicks on the book list
 # Behavior of doubleclick on the books list. Choices: open_viewer, do_nothing,
 # edit_cell, edit_metadata. Selecting edit_metadata has the side effect of
 # disabling editing a field using a single click.
@@ -294,7 +324,8 @@ generate_cover_foot_font = None
 doubleclick_on_library_view = 'open_viewer'
 
 
-# Language to use when sorting. Setting this tweak will force sorting to use the
+#: Language to use when sorting.
+# Setting this tweak will force sorting to use the
 # collating order for the specified language. This might be useful if you run
 # calibre in English but want sorting to work in the language where you live.
 # Set the tweak to the desired ISO 639-1 language code, in lower case.
@@ -305,12 +336,13 @@ doubleclick_on_library_view = 'open_viewer'
 # Example: locale_for_sorting = 'nb' -- sort using Norwegian rules.
 locale_for_sorting =  ''
 
-
+#: Number of columns for custom metadata in the edit metadata dialog
 # Set whether to use one or two columns for custom metadata when editing
 # metadata  one book at a time. If True, then the fields are laid out using two
 # columns. If False, one column is used.
 metadata_single_use_2_cols_for_custom_fields = True
 
+#: The number of seconds to wait before sending emails
 # The number of seconds to wait before sending emails when using a
 # public email server like gmail or hotmail. Default is: 5 minutes
 # Setting it to lower may cause the server's SPAM controls to kick in,
@@ -318,3 +350,9 @@ metadata_single_use_2_cols_for_custom_fields = True
 # calibre.
 public_smtp_relay_delay = 301
 
+#: Remove the bright yellow lines at the edges of the book list
+# Control whether the bright yellow lines at the edges of book list are drawn
+# when a section of the user interface is hidden. Changes will take effect
+# after a restart of calibre.
+draw_hidden_section_indicators = True
+

resources/recipes/b92.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
@@ -10,7 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class B92(BasicNewsRecipe):
     title                 = 'B92'
     __author__            = 'Darko Miletic'
-    description           = 'B92 info, najnovije vesti iz Srbije, regiona i sveta'    
+    description           = 'Najnovije vesti iz Srbije, regiona i sveta, aktuelne teme iz sveta politike, ekonomije, drustva, foto galerija, kolumne'    
     publisher             = 'B92'
     category              = 'news, politics, Serbia'
     oldest_article        = 2
@@ -20,34 +20,44 @@ class B92(BasicNewsRecipe):
     encoding              = 'cp1250'
     language              = 'sr'
     publication_type      = 'newsportal'
-    extra_css             = ' @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif} '
+    masthead_url          = 'http://www.b92.net/images/fp/logo.gif'
+    extra_css             = """ 
+                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
+                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
+                                body{font-family: Arial,Helvetica,sans1,sans-serif} 
+                                .articledescription{font-family: serif1, serif}
+                                .article-info2,.article-info1{text-transform: uppercase; font-size: small}
+                            """
     
     conversion_options = {
-                          'comment'          : description
+                          'comment'  : description
-                        , 'tags'             : category
+                        , 'tags'     : category
-                        , 'publisher'        : publisher
+                        , 'publisher': publisher
-                        , 'language'         : language
+                        , 'language' : language
                         , 'linearize_tables' : True
                         }
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
-    keep_only_tags     = [dict(name='table', attrs={'class':'maindocument'})]
+    keep_only_tags    = [dict(attrs={'class':['article-info1','article-text']})]
-
+    remove_attributes = ['width','height','align','hspace','vspace','border']
-    remove_tags = [
+    remove_tags       = [dict(name=['embed','link','base','meta'])]
-                     dict(name='ul', attrs={'class':'comment-nav'})
-                    ,dict(name=['embed','link','base']            )
-                    ,dict(name='div', attrs={'class':'udokum'}    )
-                  ]
 
     feeds          = [
-                        (u'Vesti', u'http://www.b92.net/info/rss/vesti.xml')
+                        (u'Vesti'      , u'http://www.b92.net/info/rss/vesti.xml'     )
-                       ,(u'Biz'  , u'http://www.b92.net/info/rss/biz.xml'  )
+                       ,(u'Biz'        , u'http://www.b92.net/info/rss/biz.xml'       )
+                       ,(u'Sport'      , u'http://www.b92.net/info/rss/sport.xml'     )
+                       ,(u'Zivot'      , u'http://www.b92.net/info/rss/zivot.xml'     )
+                       ,(u'Kultura'    , u'http://www.b92.net/info/rss/kultura.xml'   )
+                       ,(u'Automobili' , u'http://www.b92.net/info/rss/automobili.xml')
+                       ,(u'Tehnopolis' , u'http://www.b92.net/info/rss/tehnopolis.xml')
                      ]
 
-    def print_version(self, url):
-        return url + '&version=print'
-
     def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+        for item in soup.findAll(style=True):
-
+            del item['style']
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)            
+        return soup

resources/recipes/cinebel_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 cinebel.be
 '''
@@ -14,14 +14,14 @@ class Cinebel(BasicNewsRecipe):
     description           = u'Cinema news from Belgium in French'
     publisher             = u'cinebel.be'
     category              = 'news, cinema, movie, Belgium'
-    oldest_article        = 3
+    oldest_article        = 15
-    encoding              = 'utf8'
+    language              = 'fr'
-    language              = 'fr_BE'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
+    filterDuplicates      = True
 
     keep_only_tags = [
 	 dict(name = 'span', attrs = {'class': 'movieMainTitle'})
@@ -35,6 +35,13 @@ class Cinebel(BasicNewsRecipe):
         ,(u'Top 10'                    , u'http://www.cinebel.be/Servlets/RssServlet?languageCode=fr&rssType=2' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.has_key('href'):
+               tstr = "Site officiel: " + alink['href']
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.cinebel.be/portal/resources/common/logo_index.gif'
         return cover_url

resources/recipes/dhnet_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 dhnet.be
 '''
@@ -16,7 +16,8 @@ class DHNetBe(BasicNewsRecipe):
     publisher             = u'dhnet.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.dhnet.be/images/homepage_logo_dh.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -34,6 +35,13 @@ class DHNetBe(BasicNewsRecipe):
         ,(u'La Une Info'    , u'http://www.dhnet.be/rss/dhinfos/' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.dhnet.be/pdfonline/image/%Y%m%d/dh_%Y%m%d_nam_infoge_001.pdf.L.jpg')
         return cover_url

resources/recipes/europa_press.recipe

@@ -0,0 +1,55 @@
+__license__   = 'GPL v3'
+__author__    = 'Luis Hernandez'
+__copyright__ = 'Luis Hernandez<tolyluis@gmail.com>'
+__version__     = 'v1.0'
+__date__        = '30 January 2011'
+
+'''
+www.europapress.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1294946868(BasicNewsRecipe):
+
+    title          = u'Europa Press'
+    author            = 'Luis Hernandez'
+    description           = 'spanish news agency'
+
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    remove_javascript = True
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    language              = 'es'
+    timefmt        = '[%a, %d %b, %Y]'
+
+    remove_tags_before = dict(name='div' , attrs={'class':['nivel1 bg_3col']})
+    remove_tags_after  = dict(name='div' , attrs={'id':['ImprimirEnviarNoticia']})
+
+    remove_tags = [
+                             dict(name='ul', attrs={'id':['entidadesNoticia','MenuSecciones']})
+                            ,dict(name='div', attrs={'id':['ImprimirEnviarNoticia','PublicidadSuperior','CabeceraDerecha','Comentarios','comentarios full fbConnectAPI','ComentarEstaNoticia','ctl00_Superior_Main_MasEnChance_cajamasnoticias','gl_chn','videos_portada_derecha','galeria_portada_central','galeria_portada_central_boxes']})
+                            ,dict(name='div', attrs={'class':['infoRelacionada','col_1','buscador','caja doblecolumna strong','CHANCE_EP_Encuesta_frontal text','seccionportada col_0','seccion header','text','pie caption_over']})
+                            ,dict(name='a', attrs={'class':['buscadorLabel']})
+                            ,dict(name='span', attrs={'class':['editado']})
+                            ,dict(name='table')
+                            ,dict(name='li')
+                        ]
+
+
+    feeds = [
+                 (u'Portada'              , u'http://www.europapress.es/rss/rss.aspx')
+                ,(u'Nacional'             , u'http://www.europapress.es/rss/rss.aspx?ch=66')
+                ,(u'Internacional'       , u'http://www.europapress.es/rss/rss.aspx?ch=69')
+                ,(u'Economia'           , u'http://www.europapress.es/rss/rss.aspx?ch=136')
+                ,(u'Deportes'            , u'http://www.europapress.es/rss/rss.aspx?ch=67')
+                ,(u'Cultura'               , u'http://www.europapress.es/rss/rss.aspx?ch=126')
+                ,(u'Sociedad'            , u'http://www.europapress.es/rss/rss.aspx?ch=73')
+                ,(u'Motor'                 , u'http://www.europapress.es/rss/rss.aspx?ch=435')
+                ,(u'CHANCE'             , u'http://www.europapress.es/rss/rss.aspx?ch=549')
+                ,(u'Comunicados'      , u'http://www.europapress.es/rss/rss.aspx?ch=137')
+             ]
+

resources/recipes/irish_times.recipe

@@ -35,7 +35,7 @@ class IrishTimes(BasicNewsRecipe):
     def print_version(self, url):
          if url.count('rss.feedsportal.com'):
             u = 'http://www.irishtimes.com' + \
-                     (((url[69:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html')
+                     (((url[70:].replace('0C','/')).replace('0A','0'))).replace('0Bhtml/story01.htm','_pf.html')
          else:
              u = url.replace('.html','_pf.html')
          return u

resources/recipes/jakarta_post.recipe

@@ -0,0 +1,67 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2011, Adrian Gunawan <agunawan at adrnalin.com>'
+__author__    = 'Adrian Gunawan'
+__version__   = 'v1.0'
+__date__      = '02 February 2011'
+
+'''
+http://www.thejakartapost.com/
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class JakartaPost(BasicNewsRecipe):
+    title          = u'Jakarta Post'
+    masthead_url   = 'http://www.thejakartapost.com/images/jakartapost_logo.jpg'
+    cover_url      = 'http://www.thejakartapost.com/images/jakartapost_logo.jpg'
+
+    __author__     = u'Adrian Gunawan'
+    description    = u'Indonesian Newspaper in English from Jakarta Post Online Edition'
+    category       = 'breaking news, national, business, international, Indonesia'
+    language       = 'en_ID'
+    oldest_article = 2
+    max_articles_per_feed = 100
+
+    no_stylesheets        = True
+    use_embedded_content  = False
+    no_javascript         = True
+    remove_empty_feeds    = True
+
+    timefmt               = ' [%A, %d %B, %Y]'
+    encoding              = 'utf-8'
+
+    keep_only_tags = [dict(name='div', attrs ={'id':'news-main'})]
+
+    extra_css = '''
+                  h1{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:large;}
+                  .cT-storyDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .articleBody{font-family:Arial,Helvetica,sans-serif; color:black;font-size:small;}
+                  .cT-imageLandscape{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:x-small;}
+                  .source{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:xx-small;}
+                  #content{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  .pageprint{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                  #bylineDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .featurePic-wide{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  #idfeaturepic{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  h3{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h2{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h4{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h5{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  body{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                '''
+
+    remove_tags     = [
+                        dict(name='div', attrs ={'class':['text-size']}),
+                       ]
+
+    feeds          = [
+
+                      (u'Breaking News', u'http://www.thejakartapost.com/breaking/feed'),
+                      (u'National', u'http://www.thejakartapost.com/channel/national/feed'),
+                      (u'Archipelago', u'http://www.thejakartapost.com/channel/archipelago/feed'),
+                      (u'Business', u'http://www.thejakartapost.com/channel/business/feed'),
+                      (u'Jakarta', u'http://www.thejakartapost.com/channel/jakarta/feed'),
+                      (u'World', u'http://www.thejakartapost.com/channel/world/feed'),
+                      (u'Sports', u'http://www.thejakartapost.com/channel/sports/feed'),
+                      ]

resources/recipes/kompas.recipe

@@ -0,0 +1,77 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2011, Adrian Gunawan <agunawan at adrnalin.com>'
+__author__    = 'Adrian Gunawan'
+__version__   = 'v1.0'
+__date__      = '02 February 2011'
+
+'''
+http://www.kompas.com/
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Kompas(BasicNewsRecipe):
+    title          = u'Kompas'
+    masthead_url   = 'http://stat.k.kidsklik.com/data/2k10/kompascom2011/images/logo_kompas.png'
+    cover_url   = 'http://stat.k.kidsklik.com/data/2k10/kompascom2011/images/logo_kompas.png'
+
+    __author__     = u'Adrian Gunawan'
+    description    = u'Indonesian News from Kompas Online Edition'
+    category       = 'local news, international, business, Indonesia'
+    language       = 'id'
+    oldest_article = 5
+    max_articles_per_feed = 100
+
+    no_stylesheets        = True
+    use_embedded_content  = False
+    no_javascript         = True
+    remove_empty_feeds    = True
+
+    timefmt               = ' [%A, %d %B, %Y]'
+    encoding              = 'utf-8'
+
+    keep_only_tags = [dict(name='div', attrs ={'class':'content_kiri_detail'})]
+
+    extra_css = '''
+                  h1{font-family:Georgia,"Times New Roman",Times,serif; font-weight:bold; font-size:large;}
+                  .cT-storyDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .articleBody{font-family:Arial,Helvetica,sans-serif; color:black;font-size:small;}
+                  .cT-imageLandscape{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:x-small;}
+                  .source{font-family:Arial,Helvetica,sans-serif; color:#333333 ;font-size:xx-small;}
+                  #content{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  .pageprint{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                  #bylineDetails{font-family:Arial,Helvetica,sans-serif; color:#666666;font-size:x-small;}
+                  .featurePic-wide{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  #idfeaturepic{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
+                  h3{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h2{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h4{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  h5{font-family:Georgia,"Times New Roman",Times,serif; font-size:small;}
+                  body{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                '''
+
+    remove_tags     = [
+                        dict(name='div', attrs ={'class':['c_biru_kompas2011', 'c_abu01_kompas2011', 'c_abu_01_kompas2011', 'right', 'clearit']}),
+                        dict(name='div', attrs ={'id':['comment_list', 'comment_paging', 'share']}),
+                        dict(name='form'),
+                        dict(name='ul'),
+                       ]
+
+    preprocess_regexps = [
+                          (re.compile(r'<!--TERKAIT -->.*<!--TERKAIT END -->', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r'<strong>Sent Using.*</body>', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r'<strong>Kirim Komentar Anda</strong>', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                          (re.compile(r'<a[^>]*>Kembali ke Index Topik Pilihan</a>', re.DOTALL|re.IGNORECASE),lambda match: ''),
+                         ]
+
+    feeds          = [
+                      (u'Nasional', u'http://www.kompas.com/getrss/nasional'),
+                      (u'Regional', u'http://www.kompas.com/getrss/regional'),
+                      (u'Internasional', u'http://www.kompas.com/getrss/internasional'),
+                      (u'Megapolitan', u'http://www.kompas.com/getrss/megapolitan'),
+                      (u'Bisnis Keuangan', u'http://www.kompas.com/getrss/bisniskeuangan'),
+                      (u'Kesehatan', u'http://www.kompas.com/getrss/kesehatan'),
+                      (u'Olahraga', u'http://www.kompas.com/getrss/olahraga'),
+                      ]

resources/recipes/kopalniawiedzy.recipe

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2011, Attis <attis@attis.one.pl>'
+__version__ = 'v. 0.1'
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class KopalniaWiedzy(BasicNewsRecipe):
+		title          = u'Kopalnia Wiedzy'
+		publisher      = u'Kopalnia Wiedzy'
+		description    = u'Ciekawostki ze świata nauki i techniki'
+		encoding       = 'utf-8'
+		__author__     = 'Attis'
+		language       = 'pl'
+		oldest_article = 7
+		max_articles_per_feed = 100
+		INDEX          = u'http://kopalniawiedzy.pl/'
+		remove_javascript     = True
+		no_stylesheets        = True
+
+		remove_tags    = [{'name':'p', 'attrs': {'class': 'keywords'} }]
+		remove_tags_after = dict(attrs={'class':'ad-square'})
+		keep_only_tags    = [dict(name="div", attrs={'id':'articleContent'})]
+		extra_css      = '.topimage {margin-top: 30px}'
+
+		preprocess_regexps = [
+				(re.compile(u'<a .* rel="lightboxText" .*><img (.*)></a>'),
+				lambda match: '<img class="topimage" ' + match.group(1) + '>' ),
+				(re.compile(u'<br  /><br  />'),
+				lambda match: '<br\/>')
+			]
+
+		feeds = [
+			(u'Biologia', u'http://kopalniawiedzy.pl/wiadomosci_biologia.rss'),
+			(u'Medycyna', u'http://kopalniawiedzy.pl/wiadomosci_medycyna.rss'),
+			(u'Psychologia', u'http://kopalniawiedzy.pl/wiadomosci_psychologia.rss'),
+			(u'Technologie', u'http://kopalniawiedzy.pl/wiadomosci_technologie.rss'),
+			(u'Ciekawostki', u'http://kopalniawiedzy.pl/wiadomosci_ciekawostki.rss'),
+			(u'Artykuły', u'http://kopalniawiedzy.pl/artykuly.rss')
+		]
+
+		def is_link_wanted(self, url, tag):
+			return tag['class'] == 'next'
+
+		def remove_beyond(self, tag, next):
+				while tag is not None and getattr(tag, 'name', None) != 'body':
+						after = getattr(tag, next)
+						while after is not None:
+								ns = getattr(tag, next)
+								after.extract()
+								after = ns
+						tag = tag.parent
+
+		def append_page(self, soup, appendtag, position):
+				pager = soup.find('a',attrs={'class':'next'})
+				if pager:
+					nexturl = self.INDEX + pager['href']
+					soup2 = self.index_to_soup(nexturl)
+					texttag = soup2.find('div', attrs={'id':'articleContent'})
+
+					tag = texttag.find(attrs={'class':'pages'})
+					self.remove_beyond(tag, 'nextSibling')
+
+					newpos = len(texttag.contents)
+					self.append_page(soup2,texttag,newpos)
+
+					appendtag.insert(position,texttag)
+
+
+		def preprocess_html(self, soup):
+				self.append_page(soup, soup.body, 3)
+
+				for item in soup.findAll('div',attrs={'class':'pages'}):
+					item.extract()
+
+				for item in soup.findAll('p', attrs={'class':'wykop'}):
+					item.extract()
+
+				return soup

resources/recipes/korespondent.recipe

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Attis <attis@attis.one.pl>'
+__version__ = 'v. 0.1'
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class KorespondentPL(BasicNewsRecipe):
+    title          = u'Korespondent.pl'
+    publisher      = u'Korespondent.pl'
+    description    = u'Centrum wolnorynkowe - serwis ludzi wolnych'
+    encoding       = 'utf-8'
+    __author__     = 'Attis'
+    language       = 'pl'
+    oldest_article = 15
+    max_articles_per_feed = 100
+    remove_javascript     = True
+    no_stylesheets        = True
+
+    keep_only_tags = [dict(name='div', attrs={'class':'publicystyka'})]
+    remove_tags    = [{'name': 'meta'}, {'name':'div', 'attrs': {'class': 'zdjecie'} }]
+    extra_css      = '.naglowek {font-size: small}\n .tytul {font-size: x-large; padding-bottom: 10px; padding-top: 30px} \n .external {font-size: small}'
+
+    preprocess_regexps = [
+			(re.compile(u'<a href="index\.php.*>(.*)</a>'),
+			lambda match: match.group(1) ),
+			(re.compile(u'<i>'),
+			lambda match:'<i class="external">' ),
+			(re.compile(u'<p></p>Więcej'),
+			lambda match:'Więcej' ),
+			(re.compile(u'target="_blank"'),
+			lambda match:'target="_blank" class="external"' ),
+			(re.compile(u'<p align="center">\nPoczytaj inne teksty w <a href="http://www.korespondent.pl">Serwisie wolnorynkowym Korespondent.pl</a>.*</body>', re.DOTALL|re.IGNORECASE),
+			lambda match: '</div></body>'),
+			]
+
+    feeds = [(u'Serwis informacyjny', u'http://korespondent.pl/rss.xml')]
+

resources/recipes/lalibre_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lalibre.be
 '''
@@ -16,18 +16,18 @@ class LaLibre(BasicNewsRecipe):
     publisher             = u'lalibre.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://www.lalibre.be/img/logoLaLibre.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%d %b %Y]'
 
-    keep_only_tags = [
+    remove_tags_before = dict(name = 'div', attrs = {'class': 'extraMainContent'})
-	 dict(name = 'div', attrs = {'id': 'articleHat'})
+    remove_tags_after  = dict(name = 'div', attrs = {'id': 'articleText'})
-	,dict(name = 'p', attrs = {'id': 'publicationDate'})
+
-        ,dict(name = 'div', attrs = {'id': 'articleText'})
+    remove_tags = [dict(name = 'div', attrs = {'id': 'strongArticleLinks'})]
-    ]
 
     feeds = [
          (u'L\'actu'           , u'http://www.lalibre.be/rss/?section=10' )
@@ -38,6 +38,13 @@ class LaLibre(BasicNewsRecipe):
         ,(u'Societe'           , u'http://www.lalibre.be/rss/?section=12' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = strftime('http://pdf-online.lalibre.be/pdfonline/image/%Y%m%d/llb_%Y%m%d_nam_libre_001.pdf.L.jpg')
         return cover_url

resources/recipes/lameuse_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lameuse.be
 '''
@@ -16,8 +16,8 @@ class LaMeuse(BasicNewsRecipe):
     publisher             = u'lameuse.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
+    language              = 'fr'
-    language              = 'fr_BE'
+    masthead_url          = 'http://www.lameuse.be/images/SPV3/logo_header_LM.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -32,6 +32,11 @@ class LaMeuse(BasicNewsRecipe):
          dict(name = 'div', attrs = {'class': 'sb-group'})
         ,dict(name = 'div', attrs = {'id': 'share'})
         ,dict(name = 'div', attrs = {'id': 'commentaires'})
+        ,dict(name = 'ul', attrs = {'class': 'right liensutiles'})
+        ,dict(name = 'ul', attrs = {'class': 'bas liensutiles'})
+        ,dict(name = 'p', attrs = {'class': 'ariane'})
+        ,dict(name = 'div', attrs = {'class': 'inner-bloc'})
+        ,dict(name = 'div', attrs = {'class': 'block-01'})
     ]
 
     feeds = [

resources/recipes/lavenir_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lavenir.net
 '''
@@ -15,8 +15,7 @@ class LAvenir(BasicNewsRecipe):
     publisher             = u'lavenir.net'
     category              = 'news, Belgium'
     oldest_article        = 3
-    encoding              = 'utf8'
+    language              = 'fr'
-    language              = 'fr_BE'
 
     max_articles_per_feed = 20
     no_stylesheets        = True
@@ -35,6 +34,13 @@ class LAvenir(BasicNewsRecipe):
         ,(u'Societe'        , u'http://www.lavenir.net/rss.aspx?foto=1&intro=1&section=info&info=12e1a2f4-7e03-4cf1-afec-016869072317' )
     ]
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
     def get_cover_url(self):
         cover_url = 'http://www.lavenir.net/extra/Static/journal/Pdf/1/UNE_Nationale.PDF'
         return cover_url

resources/recipes/lesoir_be.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Lionel Bergeret <lbergeret at gmail.com>'
+__copyright__ = '2008-2011, Lionel Bergeret <lbergeret at gmail.com>'
 '''
 lesoir.be
 '''
@@ -16,7 +16,8 @@ class LeSoirBe(BasicNewsRecipe):
     publisher             = u'lesoir.be'
     category              = 'news, Belgium'
     oldest_article        = 3
-    language              = 'fr_BE'
+    language              = 'fr'
+    masthead_url          = 'http://pdf.lesoir.be/pdf/images/SOIR//logo.gif'
 
     max_articles_per_feed = 20
     no_stylesheets        = True

resources/recipes/new_yorker.recipe

@@ -54,10 +54,10 @@ class NewYorker(BasicNewsRecipe):
                         ,dict(attrs={'id':['show-header','show-footer'] })
                      ]
     remove_attributes = ['lang']
-    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/rss/feeds/everything.xml')]
+    feeds             = [(u'The New Yorker', u'http://www.newyorker.com/services/mrss/feeds/everything.xml')]
 
     def print_version(self, url):
-        return 'http://www.newyorker.com' + url + '?printable=true'
+        return url + '?printable=true'
 
     def image_url_processor(self, baseurl, url):
         return url.strip()

resources/recipes/njuz_net.recipe

@@ -0,0 +1,61 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+njuz.net
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class NjuzNet(BasicNewsRecipe):
+    title                 = 'Njuz.net'
+    __author__            = 'Darko Miletic'
+    description           = 'Iscasene vesti iz Srbije, regiona i sveta'
+    publisher             = 'njuz.net'
+    category              = 'news, politics, humor, Serbia'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf8'
+    language              = 'sr'
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://www.njuz.net/njuznet.jpg'
+    extra_css             = """
+                               @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
+                                body{font-family: serif1, serif}
+                               .articledescription{font-family: serif1, serif}
+                               .wp-caption-text{font-size: x-small}
+                            """
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        }
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags     = [
+                           dict(attrs={'id':'entryMeta'})
+                          ,dict(attrs={'class':'post'})
+                         ]
+
+    remove_tags = [
+                     dict(name=['embed','link','base','iframe','object','meta','fb:like'])
+                    ,dict(name='div', attrs={'id':'tagsandcats'})
+                  ]
+    remove_tags_after= dict(name='div', attrs={'id':'tagsandcats'})
+    remove_attributes= ['lang']
+    feeds            = [(u'Clanci', u'http://www.njuz.net/feed/')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+

resources/recipes/radio_prague.recipe

@@ -0,0 +1,43 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = 'News and information from and about The Czech republic. '
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = 'Radio Prague'
+    category               = 'News'
+    language               = 'en_CZ'
+    publication_type       = 'newsportal'
+
+    extra_css = 'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+       dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Current Affairs', 'http://www.radio.cz/feeds/rss/en/themes/curraffrs.xml'),
+                      (u'Society', 'http://www.radio.cz/feeds/rss/en/themes/society.xml'),
+                      (u'European Union', 'http:http://www.radio.cz/feeds/rss/en/themes/eu.xml'),
+                      (u'Foreign policy', 'http://www.radio.cz/feeds/rss/en/themes/foreignpolicy.xml'),
+                      (u'Business', 'http://www.radio.cz/feeds/rss/en/themes/business.xml'),
+                      (u'Culture', 'http://www.radio.cz/feeds/rss/en/themes/culture.xml'),
+                      (u'Czechs abroad', 'http://www.radio.cz/feeds/rss/en/themes/czechabroad.xml'),
+                      (u'History', 'http://www.radio.cz/feeds/rss/en/themes/history.xml'),
+                      (u'Nature', 'http://www.radio.cz/feeds/rss/en/themes/nature.xml'),
+                      (u'Science', 'http://www.radio.cz/feeds/rss/en/themes/science.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/en/themes/sport.xml'),
+                      (u'Travel', 'http://www.radio.cz/feeds/rss/en/themes/travel.xml'),
+                    ]

resources/recipes/radio_praha.recipe

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1291540961(BasicNewsRecipe):
+
+    title          = u'Radio Praha'
+    __author__             = 'Francois Pellicaan'
+    description            = u'Česká oficiální mezinárodní vysílací stanice.'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_empty_feeds     = True
+    encoding               = 'utf8'
+    publisher              = u'Český rozhlas'
+    category               = 'News'
+    language               = 'cs'
+    publication_type       = 'newsportal'
+
+    extra_css = u'h1 .section { display: block; text-transform: uppercase; font-size: 10px; margin-top: 4em; } \n .title { font-size: 14px; margin-top: 4em; } \n a.photo { display: block; clear:both; }  \n .caption { font-size: 9px; display: block; clear:both; padding:0px 0px 20px 0px; } \n a { font-type: normal; }'
+
+
+    keep_only_tags = [
+                       dict(name='div', attrs={'class':['main']})
+                        ]
+    remove_tags = [
+                       dict(name='div', attrs={'class':['cleaner', 'options', 'toolsXXL']}),
+	   dict(name='ul', attrs={'class':['tools']})
+                        ]
+    feeds = [
+                      (u'Domácí politika', 'http://www.radio.cz/feeds/rss/cs/oblast/dompol.xml'),
+                      (u'Společnost', 'http://www.radio.cz/feeds/rss/cs/oblast/spolecnost.xml'),
+                      (u'Evropská unie', 'http://www.radio.cz/feeds/rss/cs/oblast/eu.xml'),
+                      (u'Zahraniční politika', 'http://www.radio.cz/feeds/rss/cs/oblast/zahrpol.xml'),
+                      (u'Ekonomika', 'http://www.radio.cz/feeds/rss/cs/oblast/ekonomika.xml'),
+                      (u'Kultura', 'http://www.radio.cz/feeds/rss/cs/oblast/kultura.xml'),
+                      (u'Krajané', 'http://www.radio.cz/feeds/rss/cs/oblast/krajane.xml'),
+                      (u'Historie', 'http://www.radio.cz/feeds/rss/cs/oblast/historie.xml'),
+                      (u'Příroda', 'http://www.radio.cz/feeds/rss/cs/oblast/priroda.xml'),
+                      (u'Věda', 'http://www.radio.cz/feeds/rss/cs/oblast/veda.xml'),
+                      (u'Sport', 'http://www.radio.cz/feeds/rss/cs/oblast/sport.xml'),
+                      (u'Cestování', 'http://www.radio.cz/feeds/rss/cs/oblast/cestovani.xml'),
+                    ]

resources/recipes/taz.recipe

@@ -27,7 +27,7 @@ class TazDigiabo(BasicNewsRecipe):
     }
 
     def build_index(self):
-        domain = "http://www.taz.de"
+        domain = "http://dl.taz.de"
 
         url = domain + "/epub/"
 

setup/publish.py

@@ -43,7 +43,7 @@ class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
    sub_commands = ['upload_user_manual', 'upload_demo', 'sdist',
-            'upload_to_google_code', 'upload_to_sourceforge',
+            'upload_to_sourceforge', 'upload_to_google_code',
             'tag_release', 'upload_to_server',
             'upload_to_mobileread',
    ]

setup/upload.py

@@ -324,7 +324,7 @@ class UploadToServer(Command):
 
     def run(self, opts):
         check_call('ssh divok rm -f %s/calibre-\*.tar.gz'%DOWNLOADS, shell=True)
-        check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True)
+        #check_call('scp dist/calibre-*.tar.gz divok:%s/'%DOWNLOADS, shell=True)
         check_call('gpg --armor --detach-sign dist/calibre-*.tar.gz',
                 shell=True)
         check_call('scp dist/calibre-*.tar.gz.asc divok:%s/signatures/'%DOWNLOADS,

src/calibre/customize/builtins.py

@@ -325,6 +325,17 @@ class TXTMetadataReader(MetadataReaderPlugin):
         from calibre.ebooks.metadata.txt import get_metadata
         return get_metadata(stream)
 
+class TXTZMetadataReader(MetadataReaderPlugin):
+
+    name        = 'Read TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Read metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.txtz import get_metadata
+        return get_metadata(stream)
+
 class ZipMetadataReader(MetadataReaderPlugin):
 
     name = 'Read ZIP metadata'
@@ -412,6 +423,17 @@ class TOPAZMetadataWriter(MetadataWriterPlugin):
         from calibre.ebooks.metadata.topaz import set_metadata
         set_metadata(stream, mi)
 
+class TXTZMetadataWriter(MetadataWriterPlugin):
+
+    name        = 'Set TXTZ metadata'
+    file_types  = set(['txtz'])
+    description = _('Set metadata from %s files') % 'TXTZ'
+    author      = 'John Schember'
+
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.txtz import set_metadata
+        set_metadata(stream, mi)
+
 # }}}
 
 from calibre.ebooks.comic.input import ComicInput
@@ -446,6 +468,7 @@ from calibre.ebooks.rb.output import RBOutput
 from calibre.ebooks.rtf.output import RTFOutput
 from calibre.ebooks.tcr.output import TCROutput
 from calibre.ebooks.txt.output import TXTOutput
+from calibre.ebooks.txt.output import TXTZOutput
 from calibre.ebooks.html.output import HTMLOutput
 from calibre.ebooks.snb.output import SNBOutput
 
@@ -474,7 +497,7 @@ from calibre.devices.binatone.driver import README
 from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
-        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, KOGAN, \
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, Q600, LUMIREAD, ALURATEK_COLOR, \
@@ -531,6 +554,7 @@ plugins += [
     RTFOutput,
     TCROutput,
     TXTOutput,
+    TXTZOutput,
     HTMLOutput,
     SNBOutput,
 ]
@@ -581,9 +605,8 @@ plugins += [
     ELONEX,
     TECLAST_K3,
     NEWSMY,
-    PICO, SUNSTECH_EB700, ARCHOS7O,
+    PICO, SUNSTECH_EB700, ARCHOS7O, SOVOS, STASH,
     IPAPYRUS,
-    SOVOS,
     EDGE,
     SNE,
     ALEX,
@@ -767,6 +790,17 @@ class Toolbar(PreferencesPlugin):
     description = _('Customize the toolbars and context menus, changing which'
             ' actions are available in each')
 
+class Search(PreferencesPlugin):
+    name = 'Search'
+    icon = I('search.png')
+    gui_name = _('Customize searching')
+    category = 'Interface'
+    gui_category = _('Interface')
+    category_order = 1
+    name_order = 5
+    config_widget = 'calibre.gui2.preferences.search'
+    description = _('Customize the way searching for books works in calibre')
+
 class InputOptions(PreferencesPlugin):
     name = 'Input Options'
     icon = I('arrow-down.png')
@@ -917,7 +951,7 @@ class Misc(PreferencesPlugin):
     config_widget = 'calibre.gui2.preferences.misc'
     description = _('Miscellaneous advanced configuration')
 
-plugins += [LookAndFeel, Behavior, Columns, Toolbar, InputOptions,
+plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
         CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
         Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
 

src/calibre/devices/android/driver.py

@@ -19,10 +19,15 @@ class ANDROID(USBMS):
 
     VENDOR_ID   = {
             # HTC
-            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226], 0x0c01 : [0x100,
+            0x0bb4 : { 0x0c02 : [0x100, 0x0227, 0x0226],
-                0x0227, 0x0226], 0x0ff9
+                       0x0c01 : [0x100, 0x0227, 0x0226],
-                : [0x0100, 0x0227, 0x0226], 0x0c87: [0x0100, 0x0227, 0x0226],
+                       0x0ff9 : [0x0100, 0x0227, 0x0226],
-                0xc92 : [0x100], 0xc97: [0x226], 0xc99 : [0x0100]},
+                       0x0c87 : [0x0100, 0x0227, 0x0226],
+                       0xc92  : [0x100],
+                       0xc97  : [0x226],
+                       0xc99  : [0x0100],
+                       0xca3  : [0x100],
+            },
 
             # Eken
             0x040d : { 0x8510 : [0x0001], 0x0851 : [0x1] },
@@ -57,6 +62,12 @@ class ANDROID(USBMS):
             # Archos
             0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]},
 
+            # Huawei
+            0x45e : { 0x00e1 : [0x007], },
+
+            # T-Mobile
+            0x0408 : { 0x03ba : [0x0109], },
+
             }
     EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books']
     EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
@@ -66,12 +77,13 @@ class ANDROID(USBMS):
 
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
-            'TELECHIP']
+            'TELECHIP', 'HUAWEI', 'T-MOBILE', ]
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
             'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE',
-            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H']
+            'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
+            'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT']

src/calibre/devices/eb600/driver.py

@@ -172,10 +172,10 @@ class INVESBOOK(EB600):
     gui_name = 'Inves Book 600'
 
     FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'html', 'pdf', 'rtf', 'txt']
+    BCD         = [0x110, 0x323]
 
-    VENDOR_NAME = 'INVES_E6'
+    VENDOR_NAME = ['INVES_E6', 'INVES-WI']
-    WINDOWS_MAIN_MEM = '00INVES_E600'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['00INVES_E600', 'INVES-WIBOOK']
-    WINDOWS_CARD_A_MEM = '00INVES_E600'
 
 class BOOQ(EB600):
     name = 'Booq Device Interface'

src/calibre/devices/teclast/driver.py

@@ -92,3 +92,15 @@ class SUNSTECH_EB700(TECLAST_K3):
     VENDOR_NAME = 'SUNEB700'
     WINDOWS_MAIN_MEM = 'USB-MSC'
 
+class STASH(TECLAST_K3):
+
+    name = 'Stash device interface'
+    gui_name = 'Stash'
+    description    = _('Communicate with the Stash W950 reader.')
+
+    FORMATS = ['epub', 'fb2', 'lrc', 'pdb', 'html', 'fb2', 'wtxt',
+            'txt', 'pdf']
+
+    VENDOR_NAME = 'STASH'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'W950'
+

src/calibre/ebooks/__init__.py

@@ -25,7 +25,7 @@ class DRMError(ValueError):
 class ParserError(ValueError):
     pass
 
-BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
+BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'htm', 'xhtm',
                    'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'mbp', 'tan', 'snb']

src/calibre/ebooks/conversion/preprocess.py

@@ -245,17 +245,17 @@ class Dehyphenator(object):
         self.html = html
         self.format = format
         if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)
         elif format == 'pdf':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
         elif format == 'txt':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)
         elif format == 'individual_words':
-            intextmatch = re.compile(u'(?!<)(?P<firstpart>\w+)(-|‐)\s*(?P<secondpart>\w+)(?![^<]*?>)')
+            intextmatch = re.compile(u'(?!<)(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<secondpart>\w+)(?![^<]*?>)')
         elif format == 'html_cleanup':
-            intextmatch = re.compile(u'(?P<firstpart>[^\[\]\\\^\$\.\|\?\*\+\(\)“"\s>]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
+            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
         elif format == 'txt_cleanup':
-            intextmatch = re.compile(u'(?P<firstpart>\w+)(-|‐)(?P<wraptags>\s+)(?P<secondpart>[\w\d]+)')
+            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)(?P<wraptags>\s+)(?P<secondpart>[\w\d]+)')
 
 
         html = intextmatch.sub(self.dehyphenate, html)

src/calibre/ebooks/conversion/utils.py

@@ -11,6 +11,7 @@ from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.utils.logging import default_log
 from calibre.utils.wordcount import get_wordcount_obj
 
+
 class HeuristicProcessor(object):
 
     def __init__(self, extra_opts=None, log=None):
@@ -34,10 +35,15 @@ class HeuristicProcessor(object):
         self.line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>"
         self.single_blank = re.compile(r'(\s*<p[^>]*>\s*</p>)', re.IGNORECASE)
         self.scene_break_open = '<p class="scenebreak" style="text-align:center; text-indent:0%; margin-top:1em; margin-bottom:1em; page-break-before:avoid">'
+        self.common_in_text_endings = u'[\"\'—’”,\.!\?\…\)„\w]'
+        self.common_in_text_beginnings = u'[\w\'\"“‘‛]'
 
     def is_pdftohtml(self, src):
         return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
 
+    def is_abbyy(self, src):
+        return '<meta name="generator" content="ABBYY FineReader' in src[:1000]
+
     def chapter_head(self, match):
         from calibre.utils.html2text import html2text
         chap = match.group('chap')
@@ -75,22 +81,23 @@ class HeuristicProcessor(object):
 
     def insert_indent(self, match):
         pstyle = match.group('formatting')
+        tag = match.group('tagtype')
         span = match.group('span')
         self.found_indents = self.found_indents + 1
         if pstyle:
-            if pstyle.lower().find('style'):
+            if pstyle.lower().find('style') != -1:
                 pstyle = re.sub(r'"$', '; text-indent:3%"', pstyle)
             else:
                 pstyle = pstyle+' style="text-indent:3%"'
             if not span:
-                return '<p '+pstyle+'>'
+                return '<'+tag+' '+pstyle+'>'
             else:
-                return '<p '+pstyle+'>'+span
+                return '<'+tag+' '+pstyle+'>'+span
         else:
             if not span:
-                return '<p style="text-indent:3%">'
+                return '<'+tag+' style="text-indent:3%">'
             else:
-                return '<p style="text-indent:3%">'+span
+                return '<'+tag+' style="text-indent:3%">'+span
 
     def no_markup(self, raw, percent):
         '''
@@ -149,17 +156,17 @@ class HeuristicProcessor(object):
         ]
 
         ITALICIZE_STYLE_PATS = [
-            r'(?msu)(?<=\s)_(?P<words>\S[^_]{0,40}?\S)?_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])_(?P<words>[^_]+)?_',
-            r'(?msu)(?<=\s)/(?P<words>\S[^/]{0,40}?\S)?/(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])/(?P<words>[^/]+)?/',
-            r'(?msu)(?<=\s)~~(?P<words>\S[^~]{0,40}?\S)?~~(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])~~(?P<words>[^~]+)?~~',
-            r'(?msu)(?<=\s)\*(?P<words>\S[^\*]{0,40}?\S)?\*(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])\*(?P<words>[^\*]+)?\*',
-            r'(?msu)(?<=\s)~(?P<words>\S[^~]{0,40}?\S)?~(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])~(?P<words>[^~]+)?~',
-            r'(?msu)(?<=\s)_/(?P<words>\S[^/_]{0,40}?\S)?/_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])_/(?P<words>[^/_]+)?/_',
-            r'(?msu)(?<=\s)_\*(?P<words>\S[^\*_]{0,40}?\S)?\*_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])_\*(?P<words>[^\*_]+)?\*_',
-            r'(?msu)(?<=\s)\*/(?P<words>\S[^/\*]{0,40}?\S)?/\*(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])\*/(?P<words>[^/\*]+)?/\*',
-            r'(?msu)(?<=\s)_\*/(?P<words>\S[^\*_]{0,40}?\S)?/\*_(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])_\*/(?P<words>[^\*_]+)?/\*_',
-            r'(?msu)(?<=\s)/:(?P<words>\S[^:/]{0,40}?\S)?:/(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])/:(?P<words>[^:/]+)?:/',
-            r'(?msu)(?<=\s)\|:(?P<words>\S[^:\|]{0,40}?\S)?:\|(?=[\s\.,\!\?])',
+            r'(?msu)(?<=[\s>])\|:(?P<words>[^:\|]+)?:\|',
         ]
 
         for word in ITALICIZE_WORDS:
@@ -335,11 +342,9 @@ class HeuristicProcessor(object):
         return content
 
     def txt_process(self, match):
-        from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
+        from calibre.ebooks.txt.processor import convert_basic, separate_paragraphs_single_line
-        separate_paragraphs_single_line
         content = match.group('text')
         content = separate_paragraphs_single_line(content)
-        content = preserve_spaces(content)
         content = convert_basic(content, epub_split_size_kb=0)
         return content
 
@@ -349,6 +354,8 @@ class HeuristicProcessor(object):
             self.log.debug("Running Text Processing")
             outerhtml = re.compile(r'.*?(?<=<pre>)(?P<text>.*?)</pre>', re.IGNORECASE|re.DOTALL)
             html = outerhtml.sub(self.txt_process, html)
+            from calibre.ebooks.conversion.preprocess import convert_entities
+            html = re.sub(r'&(\S+?);', convert_entities, html)
         else:
             # Add markup naively
             # TODO - find out if there are cases where there are more than one <pre> tag or
@@ -363,7 +370,7 @@ class HeuristicProcessor(object):
         return html
 
     def fix_nbsp_indents(self, html):
-        txtindent = re.compile(ur'<p(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
+        txtindent = re.compile(ur'<(?P<tagtype>p|div)(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
         html = txtindent.sub(self.insert_indent, html)
         if self.found_indents > 1:
             self.log.debug("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")
@@ -516,6 +523,111 @@ class HeuristicProcessor(object):
 
         return scene_break
 
+    def abbyy_processor(self, html):
+        abbyy_line = re.compile('((?P<linestart><p\sstyle="(?P<styles>[^\"]*?);?">)(?P<content>.*?)(?P<lineend></p>)|(?P<image><img[^>]*>))', re.IGNORECASE)
+        empty_paragraph = '\n<p> </p>\n'
+        self.in_blockquote = False
+        self.previous_was_paragraph = False
+        html = re.sub('</?a[^>]*>', '', html)
+
+        def check_paragraph(content):
+            content = re.sub('\s*</?span[^>]*>\s*', '', content)
+            if re.match('.*[\"\'.!?:]$', content):
+                #print "detected this as a paragraph"
+                return True
+            else:
+                return False
+
+        def convert_styles(match):
+            #print "raw styles are: "+match.group('styles')
+            content = match.group('content')
+            #print "raw content is: "+match.group('content')
+            image = match.group('image')
+
+            is_paragraph = False
+            text_align = ''
+            text_indent = ''
+            paragraph_before = ''
+            paragraph_after = ''
+            blockquote_open = '\n<blockquote>\n'
+            blockquote_close = '</blockquote>\n'
+            indented_text = 'text-indent:3%;'
+            blockquote_open_loop = ''
+            blockquote_close_loop = ''
+            debugabby = False
+
+            if image:
+                debugabby = True
+                if self.in_blockquote:
+                    self.in_blockquote = False
+                    blockquote_close_loop = blockquote_close
+                self.previous_was_paragraph = False
+                return blockquote_close_loop+'\n'+image+'\n'
+            else:
+                styles = match.group('styles').split(';')
+                is_paragraph = check_paragraph(content)
+                #print "styles for this line are: "+str(styles)
+                split_styles = []
+                for style in styles:
+                   #print "style is: "+str(style)
+                   newstyle = style.split(':')
+                   #print "newstyle is: "+str(newstyle)
+                   split_styles.append(newstyle)
+                styles = split_styles
+                for style, setting in styles:
+                    if style == 'text-align' and setting != 'left':
+                        text_align = style+':'+setting+';'
+                    if style == 'text-indent':
+                        setting = int(re.sub('\s*pt\s*', '', setting))
+                        if 9 < setting < 14:
+                            text_indent = indented_text
+                        else:
+                            text_indent = style+':'+str(setting)+'pt;'
+                    if style == 'padding':
+                        setting = re.sub('pt', '', setting).split(' ')
+                        if int(setting[1]) < 16 and int(setting[3]) < 16:
+                            if self.in_blockquote:
+                                debugabby = True
+                                if is_paragraph:
+                                    self.in_blockquote = False
+                                    blockquote_close_loop = blockquote_close
+                            if int(setting[3]) > 8 and text_indent == '':
+                                text_indent = indented_text
+                            if int(setting[0]) > 5:
+                                paragraph_before = empty_paragraph
+                            if int(setting[2]) > 5:
+                                paragraph_after = empty_paragraph
+                        elif not self.in_blockquote and self.previous_was_paragraph:
+                            debugabby = True
+                            self.in_blockquote = True
+                            blockquote_open_loop = blockquote_open
+                        if debugabby:
+                            self.log.debug('\n\n******\n')
+                            self.log.debug('padding top is: '+str(setting[0]))
+                            self.log.debug('padding right is:'
+                                    +str(setting[1]))
+                            self.log.debug('padding bottom is: ' +
+                                    str(setting[2]))
+                            self.log.debug('padding left is: '
+                                    +str(setting[3]))
+
+                #print "text-align is: "+str(text_align)
+                #print "\n***\nline is:\n     "+str(match.group(0))+'\n'
+                if debugabby:
+                    #print "this line is a paragraph = "+str(is_paragraph)+", previous line was "+str(self.previous_was_paragraph)
+                    self.log.debug("styles for this line were:", styles)
+                    self.log.debug('newline is:')
+                    self.log.debug(blockquote_open_loop+blockquote_close_loop+
+                            paragraph_before+'<p style="'+text_indent+text_align+
+                            '">'+content+'</p>'+paragraph_after+'\n\n\n\n\n')
+                #print "is_paragraph is "+str(is_paragraph)+", previous_was_paragraph is "+str(self.previous_was_paragraph)
+                self.previous_was_paragraph = is_paragraph
+                #print "previous_was_paragraph is now set to "+str(self.previous_was_paragraph)+"\n\n\n"
+                return blockquote_open_loop+blockquote_close_loop+paragraph_before+'<p style="'+text_indent+text_align+'">'+content+'</p>'+paragraph_after
+
+        html = abbyy_line.sub(convert_styles, html)
+        return html
+
 
     def __call__(self, html):
         self.log.debug("*********  Heuristic processing HTML  *********")
@@ -530,6 +642,10 @@ class HeuristicProcessor(object):
             self.log.warn("flow is too short, not running heuristics")
             return html
 
+        is_abbyy = self.is_abbyy(html)
+        if is_abbyy:
+            html = self.abbyy_processor(html)
+
         # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
         html = self.arrange_htm_line_endings(html)
         #self.dump(html, 'after_arrange_line_endings')
@@ -638,7 +754,7 @@ class HeuristicProcessor(object):
             blanks_count = len(self.any_multi_blank.findall(html))
             if blanks_count >= 1:
                 html = self.merge_blanks(html, blanks_count)
-            scene_break_regex = self.line_open+'(?![\w\'\"])(?P<break>((?P<break_char>((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
+            scene_break_regex = self.line_open+'(?!('+self.common_in_text_beginnings+'|.*?'+self.common_in_text_endings+'<))(?P<break>((?P<break_char>((?!\s)\W))\s*(?P=break_char)?)+)\s*'+self.line_close
             scene_break = re.compile(r'%s' % scene_break_regex, re.IGNORECASE|re.UNICODE)
             # If the user has enabled scene break replacement, then either softbreaks
             # or 'hard' scene breaks are replaced, depending on which is in use

src/calibre/ebooks/lit/input.py

@@ -37,13 +37,12 @@ class LITInput(InputFormatPlugin):
                 body = body[0]
                 if len(body) == 1 and body[0].tag == XHTML('pre'):
                     pre = body[0]
-                    from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
+                    from calibre.ebooks.txt.processor import convert_basic, \
                         separate_paragraphs_single_line
                     from calibre.ebooks.chardet import xml_to_unicode
                     from lxml import etree
                     import copy
                     html = separate_paragraphs_single_line(pre.text)
-                    html = preserve_spaces(html)
                     html = convert_basic(html).replace('<html>',
                             '<html xmlns="%s">'%XHTML_NS)
                     html = xml_to_unicode(html, strip_encoding_pats=True,

src/calibre/ebooks/metadata/book/base.py

@@ -129,10 +129,7 @@ class Metadata(object):
                 val = NULL_VALUES.get(field, None)
             _data[field] = val
         elif field in _data['user_metadata'].iterkeys():
-            if _data['user_metadata'][field]['datatype'] == 'composite':
+            _data['user_metadata'][field]['#value#'] = val
-                _data['user_metadata'][field]['#value#'] = None
-            else:
-                _data['user_metadata'][field]['#value#'] = val
             _data['user_metadata'][field]['#extra#'] = extra
         else:
             # You are allowed to stick arbitrary attributes onto this object as

src/calibre/ebooks/metadata/txt.py

@@ -1,16 +1,20 @@
-'''Read meta information from TXT files'''
+# -*- coding: utf-8 -*-
-
-from __future__ import with_statement
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 
+'''
+Read meta information from TXT files
+'''
+
 import re
 
 from calibre.ebooks.metadata import MetaInformation
 
 def get_metadata(stream, extract_cover=True):
-    """ Return metadata as a L{MetaInfo} object """
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
     mi = MetaInformation(_('Unknown'), [_('Unknown')])
     stream.seek(0)
 

src/calibre/ebooks/metadata/txtz.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+
+'''
+Read meta information from TXT files
+'''
+
+import os
+
+from cStringIO import StringIO
+
+from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile, safe_replace
+
+def get_metadata(stream, extract_cover=True):
+    '''
+    Return metadata as a L{MetaInfo} object
+    '''
+    mi = MetaInformation(_('Unknown'), [_('Unknown')])
+    stream.seek(0)
+
+    with TemporaryDirectory('_untxtz_mdata') as tdir:
+        try:
+            zf = ZipFile(stream)
+            zf.extract('metadata.opf', tdir)
+            with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff:
+                mi = OPF(opff).to_book_metadata()
+        except:
+            return mi
+    return mi
+
+def set_metadata(stream, mi):
+    opf = StringIO(metadata_to_opf(mi))
+    safe_replace(stream, 'metadata.opf', opf)

src/calibre/ebooks/oeb/reader.py

@@ -103,6 +103,8 @@ class OEBReader(object):
         data = self.oeb.container.read(None)
         data = self.oeb.decode(data)
         data = XMLDECL_RE.sub('', data)
+        data = data.replace('http://openebook.org/namespaces/oeb-package/1.0',
+                OPF1_NS)
         try:
             opf = etree.fromstring(data)
         except etree.XMLSyntaxError:

src/calibre/ebooks/oeb/transforms/jacket.py

@@ -15,6 +15,7 @@ from calibre import guess_type, strftime
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.oeb.base import XPath, XHTML_NS, XHTML
 from calibre.library.comments import comments_to_html
+from calibre.utils.date import is_date_undefined
 
 JACKET_XPATH = '//h:meta[@name="calibre-content" and @content="jacket"]'
 
@@ -130,7 +131,10 @@ def render_jacket(mi, output_profile,
         publisher = ''
 
     try:
-        pubdate = strftime(u'%Y', mi.pubdate.timetuple())
+        if is_date_undefined(mi.pubdate):
+            pubdate = ''
+        else:
+            pubdate = strftime(u'%Y', mi.pubdate.timetuple())
     except:
         pubdate = ''
 
@@ -175,19 +179,24 @@ def render_jacket(mi, output_profile,
         soup = BeautifulSoup(generated_html)
         if not series:
             series_tag = soup.find(attrs={'class':'cbj_series'})
-            series_tag.extract()
+            if series_tag is not None:
+                series_tag.extract()
         if not rating:
             rating_tag = soup.find(attrs={'class':'cbj_rating'})
-            rating_tag.extract()
+            if rating_tag is not None:
+                rating_tag.extract()
         if not tags:
             tags_tag = soup.find(attrs={'class':'cbj_tags'})
-            tags_tag.extract()
+            if tags_tag is not None:
+                tags_tag.extract()
         if not pubdate:
-            pubdate_tag = soup.find(attrs={'class':'cbj_pubdate'})
+            pubdate_tag = soup.find(attrs={'class':'cbj_pubdata'})
-            pubdate_tag.extract()
+            if pubdate_tag is not None:
+                pubdate_tag.extract()
         if output_profile.short_name != 'kindle':
             hr_tag = soup.find('hr', attrs={'class':'cbj_kindle_banner_hr'})
-            hr_tag.extract()
+            if hr_tag is not None:
+                hr_tag.extract()
 
         return soup.renderContents(None)
 

src/calibre/ebooks/rtf2xml/ParseRtf.py

@@ -226,7 +226,7 @@ class ParseRtf:
         try:
             return_value = process_tokens_obj.process_tokens()
         except InvalidRtfException, msg:
-            #Check to see if the file is correctly encoded
+            # Check to see if the file is correctly encoded
             encode_obj = default_encoding.DefaultEncoding(
             in_file = self.__temp_file,
             run_level = self.__run_level,
@@ -237,14 +237,14 @@ class ParseRtf:
             check_encoding_obj = check_encoding.CheckEncoding(
                     bug_handler = RtfInvalidCodeException,
                         )
-            enc = 'cp' + encode_obj.get_codepage()
+            enc = encode_obj.get_codepage()
-            if enc == 'cp10000':
+            if enc != 'mac_roman':
-                enc = 'mac_roman'
+                enc = 'cp' + enc
-            msg = 'Exception in token processing'
+            msg = '%s\nException in token processing' % str(msg)
             if check_encoding_obj.check_encoding(self.__file, enc):
                 file_name = self.__file if isinstance(self.__file, str) \
                                     else self.__file.encode('utf-8')
-                msg = 'File %s does not appear to be correctly encoded.\n' % file_name
+                msg +='\nFile %s does not appear to be correctly encoded.\n' % file_name
             try:
                 os.remove(self.__temp_file)
             except OSError:

src/calibre/ebooks/rtf2xml/colors.py

@@ -210,7 +210,7 @@ class Colors:
             hex_num = self.__color_dict.get(num)
         if hex_num is None:
             hex_num = '0'
-            if self.__run_level > 5:
+            if self.__run_level > 3:
                 msg = 'no value in self.__color_dict' \
                 'for key %s at line %d\n' % (num, self.__line)
                 raise self.__bug_handler, msg

src/calibre/ebooks/rtf2xml/process_tokens.py

@@ -786,21 +786,23 @@ class ProcessTokens:
                     token = line.replace("\n","")
                     line_count += 1
                     if line_count == 1 and token != '\\{':
-                            msg = 'Invalid RTF: document doesn\'t start with {\n'
+                            msg = '\nInvalid RTF: document doesn\'t start with {\n'
                             raise self.__exception_handler, msg
                     elif line_count == 2 and token[0:4] != '\\rtf':
-                            msg = 'Invalid RTF: document doesn\'t start with \\rtf \n'
+                            msg = '\nInvalid RTF: document doesn\'t start with \\rtf \n'
                             raise self.__exception_handler, msg
 
                     the_index = token.find('\\ ')
                     if token is not None and  the_index > -1:
-                        msg = 'Invalid RTF: token "\\ " not valid.\n'
+                        msg = '\nInvalid RTF: token "\\ " not valid.\nError at line %d'\
+                            % line_count
                         raise self.__exception_handler, msg
                     elif token[:1] == "\\":
                         try:
                             token.decode('us-ascii')
                         except UnicodeError, msg:
-                            msg = 'Invalid RTF: Tokens not ascii encoded.\n%s' % str(msg)
+                            msg = '\nInvalid RTF: Tokens not ascii encoded.\n%s\nError at line %d'\
+                                % (str(msg), line_count)
                             raise self.__exception_handler, msg
                         line = self.process_cw(token)
                         if line is not None:
@@ -816,7 +818,7 @@ class ProcessTokens:
                                 write_obj.write('tx<nu<__________<%s\n' % field)
 
         if not line_count:
-            msg = 'Invalid RTF: file appears to be empty.\n'
+            msg = '\nInvalid RTF: file appears to be empty.\n'
             raise self.__exception_handler, msg
 
         copy_obj = copy.Copy(bug_handler = self.__bug_handler)
@@ -827,7 +829,7 @@ class ProcessTokens:
 
         bad_brackets = self.__check_brackets(self.__file)
         if bad_brackets:
-            msg = 'Invalid RTF: document does not have matching brackets.\n'
+            msg = '\nInvalid RTF: document does not have matching brackets.\n'
             raise self.__exception_handler, msg
         else:
             return self.__return_code

src/calibre/ebooks/rtf2xml/tokenize.py

@@ -117,6 +117,7 @@ class Tokenize:
         input_file = self.__replace_spchar.mreplace(input_file)
         # this is for older RTF
         input_file = self.__par_exp.sub('\n\\par \n', input_file)
+        input_file = self.__cwdigit_exp.sub("\g<1>\n\g<2>", input_file)
         input_file = self.__ms_hex_exp.sub("\\mshex0\g<1> ", input_file)
         input_file = self.__utf_ud.sub("\\{\\uc0 \g<1>\\}", input_file)
         #remove \n in bin data
@@ -139,17 +140,17 @@ class Tokenize:
             "\\_": "\\_ ",
             "\\:": "\\: ",
             "\\-": "\\- ",
-            # turn into a generic token to eliminate special
+            #turn into a generic token to eliminate special
-            # cases and make processing easier
+            #cases and make processing easier
             "\\{": "\\ob ",
-            # turn into a generic token to eliminate special
+            #turn into a generic token to eliminate special
-            # cases and make processing easier
+            #cases and make processing easier
             "\\}": "\\cb ",
-            # put a backslash in front of to eliminate special cases and
+            #put a backslash in front of to eliminate special cases and
-            # make processing easier
+            #make processing easier
             "{": "\\{",
-            # put a backslash in front of to eliminate special cases and
+            #put a backslash in front of to eliminate special cases and
-            # make processing easier
+            #make processing easier
             "}": "\\}",
             }
         self.__replace_spchar = MReplace(SIMPLE_RPL)
@@ -165,21 +166,9 @@ class Tokenize:
         #remove \n from endline char
         self.__splitexp = re.compile(r"(\\[{}]|\n|\\[^\s\\{}&]+(?:[ \t\r\f\v])?)")
         #this is for old RTF
-        self.__par_exp = re.compile(r'\\\n+')
+        self.__par_exp = re.compile(r'(\\\n+|\\ )')
         #handle cw using a digit as argument and without space as delimiter
         self.__cwdigit_exp = re.compile(r"(\\[a-zA-Z]+[\-0-9]+)([^0-9 \\]+)")
-        #self.__bin_exp = re.compile(r"\\bin(-?\d{1,8}) {0,1}")
-        #self.__utf_exp = re.compile(r"^\\u(-?\d{3,6})")
-        #self.__splitexp = re.compile(r"(\\[\\{}]|{|}|\n|\\[^\s\\{}&]+(?:\s)?)")
-        #self.__remove_line = re.compile(r'\n+')
-        ##self.num_exp = re.compile(r"(\*|:|[a-zA-Z]+)(.*)")
-
-    def __correct_spliting(self, token):
-        match_obj = re.search(self.__cwdigit_exp, token)
-        if match_obj is None:
-            return token
-        else:
-            return '%s\n%s' % (match_obj.group(1), match_obj.group(2))
 
     def tokenize(self):
         """Main class for handling other methods. Reads the file \
@@ -196,8 +185,6 @@ class Tokenize:
         tokens = map(self.__unicode_process, tokens)
         #remove empty items created by removing \uc
         tokens = filter(lambda x: len(x) > 0, tokens)
-        #handles bothersome cases
-        tokens = map(self.__correct_spliting, tokens)
 
         #write
         with open(self.__write_to, 'wb') as write_obj:

src/calibre/ebooks/txt/input.py

@@ -4,23 +4,27 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import glob
 import os
 
+from calibre import _ent_pat, xml_entity_to_unicode
 from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator
 from calibre.ebooks.chardet import detect
 from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \
     separate_paragraphs_single_line, separate_paragraphs_print_formatted, \
     preserve_spaces, detect_paragraph_type, detect_formatting_type, \
-    normalize_line_endings, convert_textile
+    normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \
-from calibre import _ent_pat, xml_entity_to_unicode
+    separate_hard_scene_breaks
+from calibre.ptempfile import TemporaryDirectory
+from calibre.utils.zipfile import ZipFile
 
 class TXTInput(InputFormatPlugin):
 
     name        = 'TXT Input'
     author      = 'John Schember'
     description = 'Convert TXT files to HTML'
-    file_types  = set(['txt'])
+    file_types  = set(['txt', 'txtz'])
 
     options = set([
         OptionRecommendation(name='paragraph_type', recommended_value='auto',
@@ -47,6 +51,9 @@ class TXTInput(InputFormatPlugin):
         OptionRecommendation(name='preserve_spaces', recommended_value=False,
             help=_('Normally extra spaces are condensed into a single space. '
                 'With this option all spaces will be displayed.')),
+        OptionRecommendation(name='txt_in_remove_indents', recommended_value=False,
+            help=_('Normally extra space at the beginning of lines is retained. '
+                   'With this option they will be removed.')),
         OptionRecommendation(name="markdown_disable_toc", recommended_value=False,
             help=_('Do not insert a Table of Contents into the output text.')),
     ])
@@ -54,9 +61,24 @@ class TXTInput(InputFormatPlugin):
     def convert(self, stream, options, file_ext, log,
                 accelerators):
         self.log = log
+        txt = ''
         log.debug('Reading text from file...')
+        length = 0
+
+        # Extract content from zip archive.
+        if file_ext == 'txtz':
+            log.debug('De-compressing content to temporary directory...')
+            with TemporaryDirectory('_untxtz') as tdir:
+                zf = ZipFile(stream)
+                zf.extractall(tdir)
+
+                txts = glob.glob(os.path.join(tdir, '*.txt'))
+                for t in txts:
+                    with open(t, 'rb') as tf:
+                        txt += tf.read()
+        else:
+            txt = stream.read()
 
-        txt = stream.read()
         # Get the encoding of the document.
         if options.input_encoding:
             ienc = options.input_encoding
@@ -70,23 +92,12 @@ class TXTInput(InputFormatPlugin):
             log.debug('No input encoding specified and could not auto detect using %s' % ienc)
         txt = txt.decode(ienc, 'replace')
 
+        # Replace entities
         txt = _ent_pat.sub(xml_entity_to_unicode, txt)
 
         # Normalize line endings
         txt = normalize_line_endings(txt)
 
-        if options.formatting_type == 'auto':
-            options.formatting_type = detect_formatting_type(txt)
-
-        if options.formatting_type == 'heuristic':
-            setattr(options, 'enable_heuristics', True)
-            setattr(options, 'markup_chapter_headings', True)
-            setattr(options, 'italicize_common_cases', True)
-            setattr(options, 'fix_indents', True)
-            setattr(options, 'delete_blank_paragraphs', True)
-            setattr(options, 'format_scene_breaks', True)
-            setattr(options, 'dehyphenate', True)
-
         # Determine the paragraph type of the document.
         if options.paragraph_type == 'auto':
             options.paragraph_type = detect_paragraph_type(txt)
@@ -96,50 +107,71 @@ class TXTInput(InputFormatPlugin):
             else:
                 log.debug('Auto detected paragraph type as %s' % options.paragraph_type)
 
+        # Detect formatting
+        if options.formatting_type == 'auto':
+            options.formatting_type = detect_formatting_type(txt)
+            log.debug('Auto detected formatting as %s' % options.formatting_type)
+
+        if options.formatting_type == 'heuristic':
+            setattr(options, 'enable_heuristics', True)
+            setattr(options, 'unwrap_lines', False)
+            setattr(options, 'smarten_punctuation', True)
+
+        # Reformat paragraphs to block formatting based on the detected type.
+        # We don't check for block because the processor assumes block.
+        # single and print at transformed to block for processing.
+        if options.paragraph_type == 'single':
+            txt = separate_paragraphs_single_line(txt)
+        elif options.paragraph_type == 'print':
+            txt = separate_hard_scene_breaks(txt)
+            txt = separate_paragraphs_print_formatted(txt)
+            txt = block_to_single_line(txt)
+        elif options.paragraph_type == 'unformatted':
+            from calibre.ebooks.conversion.utils import HeuristicProcessor
+            # unwrap lines based on punctuation
+            docanalysis = DocAnalysis('txt', txt)
+            length = docanalysis.line_length(.5)
+            preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None))
+            txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
+            txt = separate_paragraphs_single_line(txt)
+        else:
+            txt = separate_hard_scene_breaks(txt)
+            txt = block_to_single_line(txt)
+
+        if getattr(options, 'enable_heuristics', False) and getattr(options, 'dehyphenate', False):
+            docanalysis = DocAnalysis('txt', txt)
+            if not length:
+                length = docanalysis.line_length(.5)
+            dehyphenator = Dehyphenator(options.verbose, log=self.log)
+            txt = dehyphenator(txt,'txt', length)
+
+        # User requested transformation on the text.
+        if options.txt_in_remove_indents:
+            txt = remove_indents(txt)
+
         # Preserve spaces will replace multiple spaces to a space
         # followed by the &nbsp; entity.
         if options.preserve_spaces:
             txt = preserve_spaces(txt)
 
-        # Get length for hyphen removal and punctuation unwrap
+        # Process the text using the appropriate text processor.
-        docanalysis = DocAnalysis('txt', txt)
+        html = ''
-        length = docanalysis.line_length(.5)
-
         if options.formatting_type == 'markdown':
-            log.debug('Running text though markdown conversion...')
+            log.debug('Running text through markdown conversion...')
             try:
                 html = convert_markdown(txt, disable_toc=options.markdown_disable_toc)
             except RuntimeError:
                 raise ValueError('This txt file has malformed markup, it cannot be'
                     ' converted by calibre. See http://daringfireball.net/projects/markdown/syntax')
         elif options.formatting_type == 'textile':
-            log.debug('Running text though textile conversion...')
+            log.debug('Running text through textile conversion...')
             html = convert_textile(txt)
-
         else:
-            # Dehyphenate
+            log.debug('Running text through basic conversion...')
-            dehyphenator = Dehyphenator(options.verbose, log=self.log)
-            txt = dehyphenator(txt,'txt', length)
-
-            # We don't check for block because the processor assumes block.
-            # single and print at transformed to block for processing.
-
-            if options.paragraph_type == 'single' or options.paragraph_type == 'unformatted':
-                txt = separate_paragraphs_single_line(txt)
-            elif options.paragraph_type == 'print':
-                txt = separate_paragraphs_print_formatted(txt)
-
-            if options.paragraph_type == 'unformatted':
-                from calibre.ebooks.conversion.utils import HeuristicProcessor
-                # get length
-
-                # unwrap lines based on punctuation
-                preprocessor = HeuristicProcessor(options, log=getattr(self, 'log', None))
-                txt = preprocessor.punctuation_unwrap(length, txt, 'txt')
-
             flow_size = getattr(options, 'flow_size', 0)
             html = convert_basic(txt, epub_split_size_kb=flow_size)
 
+        # Run the HTMLized text through the html processing plugin.
         from calibre.customize.ui import plugin_for_input_format
         html_input = plugin_for_input_format('html')
         for opt in html_input.options:
@@ -158,8 +190,16 @@ class TXTInput(InputFormatPlugin):
             htmlfile.write(html.encode('utf-8'))
         odi = options.debug_pipeline
         options.debug_pipeline = None
+        # Generate oeb from htl conversion.
         oeb = html_input.convert(open(htmlfile.name, 'rb'), options, 'html', log,
                 {})
         options.debug_pipeline = odi
         os.remove(htmlfile.name)
+        
+        # Set metadata from file.
+        from calibre.customize.ui import get_file_type_metadata
+        from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
+        mi = get_file_type_metadata(stream, file_ext)
+        meta_info_to_oeb_metadata(mi, oeb.metadata, log)
+        
         return oeb

src/calibre/ebooks/txt/markdownml.py

@@ -35,11 +35,9 @@ class MarkdownMLizer(object):
             html = unicode(etree.tostring(item.data, encoding=unicode))
             
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
             if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
+                html = re.sub(r'<\s*img[^>]*>', '', html)\
-                html = re.sub(r'<\s*img\s*>', '', html)
             
             text = html2text(html)
         

src/calibre/ebooks/txt/output.py

@@ -5,11 +5,18 @@ __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
+import shutil
+
+from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
+from calibre.ebooks.oeb.base import OEB_IMAGES 
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
+from calibre.ptempfile import TemporaryDirectory, TemporaryFile
+from calibre.utils.cleantext import clean_ascii_chars
+from calibre.utils.zipfile import ZipFile
 
 class TXTOutput(OutputFormatPlugin):
 
@@ -73,6 +80,7 @@ class TXTOutput(OutputFormatPlugin):
             writer = TXTMLizer(log)
 
         txt = writer.extract_content(oeb_book, opts)
+        txt = clean_ascii_chars(txt)
 
         log.debug('\tReplacing newlines with selected type...')
         txt = specified_newlines(TxtNewlines(opts.newline).newline, txt)
@@ -93,3 +101,32 @@ class TXTOutput(OutputFormatPlugin):
         if close:
             out_stream.close()
 
+
+class TXTZOutput(TXTOutput):
+    
+    name = 'TXTZ Output'
+    author = 'John Schember'
+    file_type = 'txtz'
+
+    def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        with TemporaryDirectory('_txtz_output') as tdir:
+            # TXT
+            with TemporaryFile('index.txt') as tf:
+                TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
+                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+
+            # Images
+            for item in oeb_book.manifest:
+                if item.media_type in OEB_IMAGES:
+                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if not os.path.exists(path):
+                        os.makedirs(path)
+                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                        imgf.write(item.data)
+            
+            # Metadata
+            with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: 
+                mdataf.write(etree.tostring(oeb_book.metadata.to_opf1()))
+            
+            txtz = ZipFile(output_path, 'w')
+            txtz.add_dir(tdir)

src/calibre/ebooks/txt/processor.py

@@ -18,20 +18,23 @@ from calibre.utils.cleantext import clean_ascii_chars
 HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>%s</title></head><body>\n%s\n</body></html>'
 
 def clean_txt(txt):
+    '''
+    Run transformations on the text to put it into
+    consistent state.
+    '''
     if isbytestring(txt):
         txt = txt.decode('utf-8', 'replace')
     # Strip whitespace from the end of the line. Also replace
     # all line breaks with \n.
     txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
     
-    # Replace whitespace at the beginning of the list with &nbsp;
+    # Replace whitespace at the beginning of the line with &nbsp;
-    txt = re.sub('(?m)(?P<space>[ ]+)', lambda mo: '&nbsp;' * mo.groups('space').count(' '), txt)
+    txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', '&nbsp;' * 4, txt)
-    txt = re.sub('(?m)(?P<space>[\t]+)', lambda mo: '&nbsp;' * 4 * mo.groups('space').count('\t'), txt)
 
     # Condense redundant spaces
     txt = re.sub('[ ]{2,}', ' ', txt)
 
-    # Remove blank lines from the beginning and end of the document.
+    # Remove blank space from the beginning and end of the document.
     txt = re.sub('^\s+(?=.)', '', txt)
     txt = re.sub('(?<=.)\s+$', '', txt)
     # Remove excessive line breaks.
@@ -42,6 +45,15 @@ def clean_txt(txt):
     return txt
 
 def split_txt(txt, epub_split_size_kb=0):
+    '''
+    Ensure there are split points for converting
+    to EPUB. A misdetected paragraph type can
+    result in the entire document being one giant
+    paragraph. In this case the EPUB parser will not
+    be able to determine where to split the file
+    to accomidate the EPUB file size limitation
+    and will fail.
+    '''
     #Takes care if there is no point to split
     if epub_split_size_kb > 0:
         if isinstance(txt, unicode):
@@ -59,6 +71,12 @@ def split_txt(txt, epub_split_size_kb=0):
     return txt
 
 def convert_basic(txt, title='', epub_split_size_kb=0):
+    '''
+    Converts plain text to html by putting all paragraphs in
+    <p> tags. It condense and retains blank lines when necessary.
+    
+    Requires paragraphs to be in single line format.
+    '''
     txt = clean_txt(txt)
     txt = split_txt(txt, epub_split_size_kb)
 
@@ -99,14 +117,37 @@ def separate_paragraphs_single_line(txt):
     return txt
 
 def separate_paragraphs_print_formatted(txt):
-    txt = re.sub(u'(?miu)^(\t+|[ ]{2,})(?=.)', '\n\t', txt)
+    txt = re.sub(u'(?miu)^(?P<indent>\t+|[ ]{2,})(?=.)', lambda mo: '\n%s' % mo.group('indent'), txt)
+    return txt
+
+def separate_hard_scene_breaks(txt):
+    def sep_break(line):
+        if len(line.strip()) > 0:
+            return '\n%s\n' % line
+        else:
+            return line
+    txt = re.sub(u'(?miu)^[ \t-=~\/]+$', lambda mo: sep_break(mo.group()), txt)
+    return txt
+
+def block_to_single_line(txt):
+    txt = re.sub(r'(?<=.)\n(?=.)', ' ', txt)
     return txt
 
 def preserve_spaces(txt):
+    '''
+    Replaces spaces multiple spaces with &nbsp; entities.
+    '''
     txt = re.sub('(?P<space>[ ]{2,})', lambda mo: ' ' + ('&nbsp;' * (len(mo.group('space')) - 1)), txt)
     txt = txt.replace('\t', '&nbsp;&nbsp;&nbsp;&nbsp;')
     return txt
 
+def remove_indents(txt):
+    '''
+    Remove whitespace at the beginning of each line.
+    '''
+    txt = re.sub('(?miu)^\s+', '', txt)
+    return txt
+
 def opf_writer(path, opf_name, manifest, spine, mi):
     opf = OPFCreator(path, mi)
     opf.create_manifest(manifest)
@@ -114,7 +155,10 @@ def opf_writer(path, opf_name, manifest, spine, mi):
     with open(os.path.join(path, opf_name), 'wb') as opffile:
         opf.render(opffile)
 
-def split_string_separator(txt, size) :
+def split_string_separator(txt, size):
+    '''
+    Splits the text by putting \n\n at the point size.
+    '''
     if len(txt) > size:
         txt = ''.join([re.sub(u'\.(?P<ends>[^.]*)$', '.\n\n\g<ends>',
             txt[i:i+size], 1) for i in
@@ -123,7 +167,7 @@ def split_string_separator(txt, size) :
 
 def detect_paragraph_type(txt):
     '''
-    Tries to determine the formatting of the document.
+    Tries to determine the paragraph type of the document.
 
     block: Paragraphs are separated by a blank line.
     single: Each line is a paragraph.
@@ -166,6 +210,16 @@ def detect_paragraph_type(txt):
 
 
 def detect_formatting_type(txt):
+    '''
+    Tries to determine the formatting of the document.
+    
+    markdown: Markdown formatting is used.
+    textile: Textile formatting is used.
+    heuristic: When none of the above formatting types are
+               detected heuristic is returned.
+    '''
+    # Keep a count of the number of format specific object
+    # that are found in the text.
     markdown_count = 0
     textile_count = 0
 
@@ -175,9 +229,9 @@ def detect_formatting_type(txt):
     markdown_count += len(re.findall('(?mu)^=+$', txt))
     markdown_count += len(re.findall('(?mu)^-+$', txt))
     # Images
-    markdown_count += len(re.findall('(?u)!\[.*?\]\(.+?\)', txt))
+    markdown_count += len(re.findall('(?u)!\[.*?\](\[|\()', txt))
     # Links
-    markdown_count += len(re.findall('(?u)(^|(?P<pre>[^!]))\[.*?\]\([^)]+\)', txt))
+    markdown_count += len(re.findall('(?u)^|[^!]\[.*?\](\[|\()', txt))
 
     # Check for textile
     # Headings
@@ -185,10 +239,12 @@ def detect_formatting_type(txt):
     # Block quote.
     textile_count += len(re.findall(r'(?mu)^bq\.', txt))
     # Images
-    textile_count += len(re.findall(r'\![^\s]+(?=.*?/)(:[^\s]+)*', txt))
+    textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
-    textile_count += len(re.findall(r'"(?=".*?\()(\(.+?\))*[^\(]+?(\(.+?\))*":[^\s]+', txt))
+    textile_count += len(re.findall(r'"[^"]*":\S+', txt))
 
+    # Decide if either markdown or textile is used in the text
+    # based on the number of unique formatting elements found.
     if markdown_count > 5 or textile_count > 5:
         if markdown_count > textile_count:
             return 'markdown'

src/calibre/ebooks/txt/textileml.py

@@ -36,11 +36,9 @@ class TextileMLizer(object):
             html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
 
             if not self.opts.keep_links:
-                html = re.sub(r'<\s*a[^>]*>', '', html)
+                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
-                html = re.sub(r'<\s*/\s*a\s*>', '', html)
             if not self.opts.keep_image_references:
                 html = re.sub(r'<\s*img[^>]*>', '', html)
-                html = re.sub(r'<\s*img\s*>', '', html)
 
             text = html2textile(html)
 

src/calibre/ebooks/txt/txtml.py

@@ -55,6 +55,7 @@ class TXTMLizer(object):
         self.log.info('Converting XHTML to TXT...')
         self.oeb_book = oeb_book
         self.opts = opts
+        self.toc_titles = []
         self.toc_ids = []
         self.last_was_heading = False
         
@@ -94,8 +95,8 @@ class TXTMLizer(object):
         if getattr(self.opts, 'inline_toc', None):
             self.log.debug('Generating table of contents...')
             toc.append(u'%s\n\n' % _(u'Table of Contents:'))
-            for item in self.oeb_book.toc:
+            for item in self.toc_titles:
-                toc.append(u'* %s\n\n' % item.title)
+                toc.append(u'* %s\n\n' % item)
         return ''.join(toc)
 
     def create_flat_toc(self, nodes):
@@ -103,6 +104,7 @@ class TXTMLizer(object):
         Turns a hierarchical list of TOC href's into a flat list.
         '''
         for item in nodes:
+            self.toc_titles.append(item.title)
             self.toc_ids.append(item.href)
             self.create_flat_toc(item.nodes)
 

src/calibre/gui2/__init__.py

@@ -50,6 +50,7 @@ gprefs.defaults['action-layout-context-menu-device'] = (
 
 gprefs.defaults['show_splash_screen'] = True
 gprefs.defaults['toolbar_icon_size'] = 'medium'
+gprefs.defaults['automerge'] = 'ignore'
 gprefs.defaults['toolbar_text'] = 'auto'
 gprefs.defaults['show_child_bar'] = False
 gprefs.defaults['font'] = None
@@ -105,9 +106,13 @@ def _config():
                 'clicked'))
     c.add_opt('asked_library_thing_password', default=False,
             help='Asked library thing password at least once.')
-    c.add_opt('search_as_you_type', default=True,
+    c.add_opt('search_as_you_type', default=False,
-            help='Start searching as you type. If this is disabled then search will '
+            help=_('Start searching as you type. If this is disabled then search will '
-            'only take place when the Enter or Return key is pressed.')
+            'only take place when the Enter or Return key is pressed.'))
+    c.add_opt('highlight_search_matches', default=False,
+            help=_('When searching, show all books with search results '
+            'highlighted instead of showing only the matches. You can use the '
+            'N or F3 keys to go to the next match.'))
     c.add_opt('save_to_disk_template_history', default=[],
         help='Previously used Save to Disk templates')
     c.add_opt('send_to_device_template_history', default=[],

src/calibre/gui2/actions/add.py

@@ -244,8 +244,8 @@ class AddAction(InterfaceAction):
                     x.decode(preferred_encoding, 'replace') for x in
                     self._adder.merged_books])
             info_dialog(self.gui, _('Merged some books'),
-                    _('Some duplicates were found and merged into the '
+                    _('The following duplicate books were found and incoming book formats were '
-                        'following existing books:'), det_msg=books, show=True)
+                        'processed and merged into your Calibre database according to your automerge settings:'), det_msg=books, show=True)
         if getattr(self._adder, 'critical', None):
             det_msg = []
             for name, log in self._adder.critical.items():

src/calibre/gui2/actions/device.py

@@ -94,6 +94,7 @@ class ShareConnMenu(QMenu): # {{{
                             I('mail.png'), _('Email to') + ' ' +account)
                     self.addAction(ac)
                     self.email_actions.append(ac)
+                    ac.a_s.connect(sync_menu.action_triggered)
                 action1.a_s.connect(sync_menu.action_triggered)
                 action2.a_s.connect(sync_menu.action_triggered)
             ac = self.addMenu(self.email_to_and_delete_menu)

src/calibre/gui2/actions/next_match.py

@@ -28,21 +28,12 @@ class NextMatchAction(InterfaceAction):
         self.gui.addAction(self.p_action)
         self.p_action.triggered.connect(self.move_backward)
 
-    def gui_layout_complete(self):
-        self.gui.search_highlight_only.setVisible(True)
-
     def location_selected(self, loc):
         self.can_move = loc == 'library'
-        try:
-            self.gui.search_highlight_only.setVisible(self.can_move)
-        except:
-            import traceback
-            traceback.print_exc()
 
     def move_forward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=True)
@@ -50,7 +41,6 @@ class NextMatchAction(InterfaceAction):
     def move_backward(self):
         if self.can_move is None:
             self.can_move = self.gui.current_view() is self.gui.library_view
-            self.gui.search_highlight_only.setVisible(self.can_move)
 
         if self.can_move:
             self.gui.current_view().move_highlighted_row(forward=False)

src/calibre/gui2/actions/preferences.py

@@ -33,7 +33,8 @@ class PreferencesAction(InterfaceAction):
             x.triggered.connect(self.do_config)
 
 
-    def do_config(self, checked=False, initial_plugin=None):
+    def do_config(self, checked=False, initial_plugin=None,
+            close_after_initial=False):
         if self.gui.job_manager.has_jobs():
             d = error_dialog(self.gui, _('Cannot configure'),
                     _('Cannot configure while there are running jobs.'))
@@ -44,7 +45,8 @@ class PreferencesAction(InterfaceAction):
                     _('Cannot configure before calibre is restarted.'))
             d.exec_()
             return
-        d = Preferences(self.gui, initial_plugin=initial_plugin)
+        d = Preferences(self.gui, initial_plugin=initial_plugin,
+                close_after_initial=close_after_initial)
         d.show()
         d.run_wizard_requested.connect(self.gui.run_wizard,
                 type=Qt.QueuedConnection)

src/calibre/gui2/add.py

@@ -8,7 +8,7 @@ from functools import partial
 from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
 
 from calibre.gui2.dialogs.progress import ProgressDialog
-from calibre.gui2 import question_dialog, error_dialog, info_dialog
+from calibre.gui2 import question_dialog, error_dialog, info_dialog, gprefs
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
@@ -179,23 +179,47 @@ class DBAdder(QObject): # {{{
                     cover = f.read()
             orig_formats = formats
             formats = [f for f in formats if not f.lower().endswith('.opf')]
-            if prefs['add_formats_to_existing']:
+            if prefs['add_formats_to_existing']: #automerge is on
                 identical_book_list = self.db.find_identical_books(mi)
-
+                if identical_book_list:  # books with same author and nearly same title exist in db
-                if identical_book_list: # books with same author and nearly same title exist in db
                     self.merged_books.add(mi.title)
+                    seen_fmts = set([])
+
                     for identical_book in identical_book_list:
-                        self.add_formats(identical_book, formats, replace=False)
+                        ib_fmts = self.db.formats(identical_book, index_is_id=True)
+                        if ib_fmts:
+                            seen_fmts |= set(ib_fmts.split(','))
+                        replace = gprefs['automerge'] == 'overwrite'
+                        self.add_formats(identical_book, formats,
+                                replace=replace)
+                    if gprefs['automerge'] == 'new record':
+                        incoming_fmts = \
+                            set([os.path.splitext(path)[-1].replace('.',
+                                '').upper() for path in formats])
+                        if incoming_fmts.intersection(seen_fmts):
+                            # There was at least one duplicate format
+                            # so create a new record and put the
+                            # incoming formats into it
+                            # We should arguably put only the duplicate
+                            # formats, but no real harm is done by having
+                            # all formats
+                            id_ = self.db.create_book_entry(mi, cover=cover,
+                                    add_duplicates=True)
+                            self.number_of_books_added += 1
+                            self.add_formats(id_, formats)
+
                 else:
-                    id = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
+                    # books with same author and nearly same title do not exist in db
+                    id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=True)
                     self.number_of_books_added += 1
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
-            else:
+
-                id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+            else: #automerge is off
-                if id is None:
+                id_ = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
+                if id_ is None:
                     self.duplicates.append((mi, cover, orig_formats))
                 else:
-                    self.add_formats(id, formats)
+                    self.add_formats(id_, formats)
                     self.number_of_books_added += 1
         else:
             self.names.append(name)

src/calibre/gui2/complete.py

@@ -6,158 +6,39 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from PyQt4.Qt import QLineEdit, QListView, QAbstractListModel, Qt, QTimer, \
+from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
-        QApplication, QPoint, QItemDelegate, QStyleOptionViewItem, \
+        QApplication, QCompleter
-        QStyle, QEvent, pyqtSignal
 
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
-from calibre.gui2.widgets import EnComboBox
+from calibre.gui2.widgets import EnComboBox, LineEditECM
-
-class CompleterItemDelegate(QItemDelegate): # {{{
-
-    ''' Renders the current item as thought it were selected '''
-
-    def __init__(self, view):
-        self.view = view
-        QItemDelegate.__init__(self, view)
-
-    def paint(self, p, opt, idx):
-        opt = QStyleOptionViewItem(opt)
-        opt.showDecorationSelected = True
-        if self.view.currentIndex() == idx:
-            opt.state |= QStyle.State_HasFocus
-        QItemDelegate.paint(self, p, opt, idx)
-
-# }}}
-
-class CompleteWindow(QListView): # {{{
-
-    '''
-    The completion popup. For keyboard and mouse handling see
-    :meth:`eventFilter`.
-    '''
-
-    #: This signal is emitted when the user selects one of the listed
-    #: completions, by mouse or keyboard
-    completion_selected = pyqtSignal(object)
-
-    def __init__(self, widget, model):
-        self.widget = widget
-        QListView.__init__(self)
-        self.setVisible(False)
-        self.setParent(None, Qt.Popup)
-        self.setAlternatingRowColors(True)
-        self.setFocusPolicy(Qt.NoFocus)
-        self._d = CompleterItemDelegate(self)
-        self.setItemDelegate(self._d)
-        self.setModel(model)
-        self.setFocusProxy(widget)
-        self.installEventFilter(self)
-        self.clicked.connect(self.do_selected)
-        self.entered.connect(self.do_entered)
-        self.setMouseTracking(True)
-
-    def do_entered(self, idx):
-        if idx.isValid():
-            self.setCurrentIndex(idx)
-
-    def do_selected(self, idx=None):
-        idx = self.currentIndex() if idx is None else idx
-        if idx.isValid():
-            data = unicode(self.model().data(idx, Qt.DisplayRole))
-            self.completion_selected.emit(data)
-        self.hide()
-
-    def eventFilter(self, o, e):
-        if o is not self:
-            return False
-        if e.type() == e.KeyPress:
-            key = e.key()
-            if key in (Qt.Key_Escape, Qt.Key_Backtab) or \
-                    (key == Qt.Key_F4 and (e.modifiers() & Qt.AltModifier)):
-                self.hide()
-                return True
-            elif key in (Qt.Key_Enter, Qt.Key_Return, Qt.Key_Tab):
-                if key == Qt.Key_Tab and not self.currentIndex().isValid():
-                    if self.model().rowCount() > 0:
-                        self.setCurrentIndex(self.model().index(0))
-                self.do_selected()
-                return True
-            elif key in (Qt.Key_Up, Qt.Key_Down, Qt.Key_PageUp,
-                    Qt.Key_PageDown):
-                return False
-            # Send key event to associated line edit
-            self.widget.eat_focus_out = False
-            try:
-                self.widget.event(e)
-            finally:
-                self.widget.eat_focus_out = True
-            if not self.widget.hasFocus():
-                # Line edit lost focus
-                self.hide()
-            if e.isAccepted():
-                # Line edit consumed event
-                return True
-        elif e.type() == e.MouseButtonPress:
-            # Hide popup if user clicks outside it, otherwise
-            # pass event to popup
-            if not self.underMouse():
-                self.hide()
-                return True
-        elif e.type() in (e.InputMethod, e.ShortcutOverride):
-            QApplication.sendEvent(self.widget, e)
-
-        return False # Do not filter this event
-
-# }}}
 
 class CompleteModel(QAbstractListModel):
 
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
-        self.sep = ','
-        self.space_before_sep = False
         self.items = []
-        self.lowered_items = []
-        self.matches = []
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
         self.items = list(sorted(items, key=lambda x: sort_key(x)))
         self.lowered_items = [lower(x) for x in self.items]
-        self.matches = []
         self.reset()
 
     def rowCount(self, *args):
-        return len(self.matches)
+        return len(self.items)
 
     def data(self, index, role):
         if role == Qt.DisplayRole:
             r = index.row()
             try:
-                return self.matches[r]
+                return self.items[r]
             except IndexError:
                 pass
         return NONE
 
-    def get_matches(self, prefix):
-        '''
-        Return all matches that (case insensitively) start with prefix
-        '''
-        prefix = lower(prefix)
-        ans = []
-        if prefix:
-            for i, test in enumerate(self.lowered_items):
-                if test.startswith(prefix):
-                    ans.append(self.items[i])
-        return ans
 
-    def update_matches(self, matches):
+class MultiCompleteLineEdit(QLineEdit, LineEditECM):
-        self.matches = matches
-        self.reset()
-
-class MultiCompleteLineEdit(QLineEdit):
     '''
     A line edit that completes on multiple items separated by a
     separator. Use the :meth:`update_items_cache` to set the list of
@@ -169,16 +50,28 @@ class MultiCompleteLineEdit(QLineEdit):
     '''
 
     def __init__(self, parent=None):
-        self.eat_focus_out = True
-        self.max_visible_items = 7
-        self.current_prefix = None
         QLineEdit.__init__(self, parent)
 
+        self.sep = ','
+        self.space_before_sep = False
+        self.add_separator = True
+        self.original_cursor_pos = None
+
         self._model = CompleteModel(parent=self)
-        self.complete_window = CompleteWindow(self, self._model)
+        self._completer = c = QCompleter(self._model, self)
+        c.setWidget(self)
+        c.setCompletionMode(QCompleter.PopupCompletion)
+        c.setCaseSensitivity(Qt.CaseInsensitive)
+        c.setModelSorting(QCompleter.CaseInsensitivelySortedModel)
+        c.setCompletionRole(Qt.DisplayRole)
+        p = c.popup()
+        p.setMouseTracking(True)
+        p.entered.connect(self.item_entered)
+        c.popup().setAlternatingRowColors(True)
+
+        c.activated.connect(self.completion_selected,
+                type=Qt.QueuedConnection)
         self.textEdited.connect(self.text_edited)
-        self.complete_window.completion_selected.connect(self.completion_selected)
-        self.installEventFilter(self)
 
     # Interface {{{
     def update_items_cache(self, complete_items):
@@ -190,117 +83,62 @@ class MultiCompleteLineEdit(QLineEdit):
     def set_space_before_sep(self, space_before):
         self.space_before_sep = space_before
 
+    def set_add_separator(self, what):
+        self.add_separator = bool(what)
+
     # }}}
 
-    def eventFilter(self, o, e):
+    def item_entered(self, idx):
-        if self.eat_focus_out and o is self and e.type() == QEvent.FocusOut:
+        self._completer.popup().setCurrentIndex(idx)
-            if self.complete_window.isVisible():
-                return True # Filter this event since the cw is visible
-        return QLineEdit.eventFilter(self, o, e)
-
-    def hide_completion_window(self):
-        self.complete_window.hide()
-
 
     def text_edited(self, *args):
         self.update_completions()
+        self._completer.complete()
 
     def update_completions(self):
         ' Update the list of completions '
-        if not self.complete_window.isVisible() and not self.hasFocus():
+        self.original_cursor_pos = cpos = self.cursorPosition()
-            return
-        cpos = self.cursorPosition()
         text = unicode(self.text())
         prefix = text[:cpos]
         self.current_prefix = prefix
         complete_prefix = prefix.lstrip()
         if self.sep:
-            complete_prefix = prefix = prefix.split(self.sep)[-1].lstrip()
+            complete_prefix = prefix.split(self.sep)[-1].lstrip()
-
+        self._completer.setCompletionPrefix(complete_prefix)
-        matches = self._model.get_matches(complete_prefix)
-        self.update_complete_window(matches)
 
     def get_completed_text(self, text):
-        '''
+        'Get completed text in before and after parts'
-        Get completed text from current cursor position and the completion
-        text
-        '''
         if self.sep is None:
-            return -1, text
+            return text, ''
         else:
-            cursor_pos = self.cursorPosition()
+            cursor_pos = self.original_cursor_pos
-            before_text = unicode(self.text())[:cursor_pos]
+            if cursor_pos is None:
-            after_text = unicode(self.text())[cursor_pos:]
+                cursor_pos = self.cursorPosition()
-            after_parts = after_text.split(self.sep)
+            self.original_cursor_pos = None
-            if len(after_parts) < 3 and not after_parts[-1].strip():
+            # Split text
-                after_text = u''
+            curtext = unicode(self.text())
-            prefix_len = len(before_text.split(self.sep)[-1].lstrip())
+            before_text = curtext[:cursor_pos]
-            return prefix_len, \
+            after_text = curtext[cursor_pos:].rstrip()
-                before_text[:cursor_pos - prefix_len] + text + after_text
+            # Remove the completion prefix from the before text
+            before_text = self.sep.join(before_text.split(self.sep)[:-1]).rstrip()
+            if before_text:
+                # Add the separator to the end of before_text
+                if self.space_before_sep:
+                    before_text += ' '
+                before_text += self.sep + ' '
+            if self.add_separator or after_text:
+                # Add separator to the end of completed text
+                if self.space_before_sep:
+                    text = text.rstrip() + ' '
+                completed_text = text + self.sep + ' '
+            else:
+                completed_text = text
+            return before_text + completed_text, after_text
 
     def completion_selected(self, text):
-        prefix_len, ctext = self.get_completed_text(text)
+        before_text, after_text = self.get_completed_text(unicode(text))
-        if self.sep is None:
+        self.setText(before_text + after_text)
-            self.setText(ctext)
+        self.setCursorPosition(len(before_text))
-            self.setCursorPosition(len(ctext))
-        else:
-            cursor_pos = self.cursorPosition()
-            self.setText(ctext)
-            self.setCursorPosition(cursor_pos - prefix_len + len(text))
-
-    def update_complete_window(self, matches):
-        self._model.update_matches(matches)
-        if matches:
-            self.show_complete_window()
-        else:
-            self.complete_window.hide()
-
-
-    def position_complete_window(self):
-        popup = self.complete_window
-        screen = QApplication.desktop().availableGeometry(self)
-        h = (popup.sizeHintForRow(0) * min(self.max_visible_items,
-            popup.model().rowCount()) + 3) + 3
-        hsb = popup.horizontalScrollBar()
-        if hsb and hsb.isVisible():
-            h += hsb.sizeHint().height()
-
-        rh = self.height()
-        pos = self.mapToGlobal(QPoint(0, self.height() - 2))
-        w = self.width()
-
-        if w > screen.width():
-            w = screen.width()
-        if (pos.x() + w) > (screen.x() + screen.width()):
-            pos.setX(screen.x() + screen.width() - w)
-        if (pos.x() < screen.x()):
-            pos.setX(screen.x())
-
-        top = pos.y() - rh - screen.top() + 2
-        bottom = screen.bottom() - pos.y()
-        h = max(h, popup.minimumHeight())
-        if h > bottom:
-            h = min(max(top, bottom), h)
-            if top > bottom:
-                pos.setY(pos.y() - h - rh + 2)
-
-        popup.setGeometry(pos.x(), pos.y(), w, h)
-
-
-    def show_complete_window(self):
-        self.position_complete_window()
-        self.complete_window.show()
-
-    def moveEvent(self, ev):
-        ret = QLineEdit.moveEvent(self, ev)
-        QTimer.singleShot(0, self.position_complete_window)
-        return ret
-
-    def resizeEvent(self, ev):
-        ret = QLineEdit.resizeEvent(self, ev)
-        QTimer.singleShot(0, self.position_complete_window)
-        return ret
-
 
     @dynamic_property
     def all_items(self):
@@ -310,22 +148,6 @@ class MultiCompleteLineEdit(QLineEdit):
             self._model.set_items(items)
         return property(fget=fget, fset=fset)
 
-    @dynamic_property
-    def sep(self):
-        def fget(self):
-            return self._model.sep
-        def fset(self, val):
-            self._model.sep = val
-        return property(fget=fget, fset=fset)
-
-    @dynamic_property
-    def space_before_sep(self):
-        def fget(self):
-            return self._model.space_before_sep
-        def fset(self, val):
-            self._model.space_before_sep = val
-        return property(fget=fget, fset=fset)
-
 class MultiCompleteComboBox(EnComboBox):
 
     def __init__(self, *args):
@@ -346,6 +168,9 @@ class MultiCompleteComboBox(EnComboBox):
     def set_space_before_sep(self, space_before):
         self.lineEdit().set_space_before_sep(space_before)
 
+    def set_add_separator(self, what):
+        self.lineEdit().set_add_separator(what)
+
 
 
 if __name__ == '__main__':

src/calibre/gui2/convert/metadata.py

@@ -19,6 +19,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2.convert import Widget
 from calibre.utils.icu import sort_key
 from calibre.library.comments import comments_to_html
+from calibre.utils.config import tweaks
 
 def create_opf_file(db, book_id):
     mi = db.get_metadata(book_id, index_is_id=True)
@@ -108,6 +109,7 @@ class MetadataWidget(Widget, Ui_Form):
         all_authors.sort(key=lambda x : sort_key(x[1]))
         self.author.set_separator('&')
         self.author.set_space_before_sep(True)
+        self.author.set_add_separator(tweaks['authors_completer_append_separator'])
         self.author.update_items_cache(self.db.all_author_names())
 
         for i in all_authors:

src/calibre/gui2/convert/txt_input.py

@@ -16,7 +16,8 @@ class PluginWidget(Widget, Ui_Form):
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
-            ['paragraph_type', 'formatting_type', 'markdown_disable_toc', 'preserve_spaces'])
+            ['paragraph_type', 'formatting_type', 'markdown_disable_toc',
+             'preserve_spaces', 'txt_in_remove_indents'])
         self.db, self.book_id = db, book_id
         for x in get_option('paragraph_type').option.choices:
             self.opt_paragraph_type.addItem(x)

src/calibre/gui2/convert/txt_input.ui

@@ -7,57 +7,95 @@
     <x>0</x>
     <y>0</y>
     <width>518</width>
-    <height>300</height>
+    <height>353</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
-  <layout class="QGridLayout" name="gridLayout">
+  <layout class="QVBoxLayout" name="verticalLayout_3">
-   <item row="0" column="0">
+   <item>
-    <widget class="QLabel" name="label_2">
+    <widget class="QGroupBox" name="groupBox_3">
-     <property name="text">
+     <property name="title">
-      <string>Paragraph style:</string>
+      <string>Structure</string>
      </property>
+     <layout class="QGridLayout" name="gridLayout">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label_2">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+        <property name="text">
+         <string>Paragraph style:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QComboBox" name="opt_paragraph_type">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+        <property name="text">
+         <string>Formatting style:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QComboBox" name="opt_formatting_type">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+       </widget>
+      </item>
+     </layout>
     </widget>
    </item>
-   <item row="0" column="1">
+   <item>
-    <widget class="QComboBox" name="opt_paragraph_type"/>
+    <widget class="QGroupBox" name="groupBox_2">
-   </item>
+     <property name="title">
-   <item row="5" column="0" colspan="2">
+      <string>Common</string>
-    <widget class="QCheckBox" name="opt_preserve_spaces">
-     <property name="text">
-      <string>Preserve &amp;spaces</string>
      </property>
+     <layout class="QVBoxLayout" name="verticalLayout_2">
+      <item>
+       <widget class="QCheckBox" name="opt_preserve_spaces">
+        <property name="text">
+         <string>Preserve &amp;spaces</string>
+        </property>
+       </widget>
+      </item>
+      <item>
+       <widget class="QCheckBox" name="opt_txt_in_remove_indents">
+        <property name="text">
+         <string>Remove indents at the beginning of lines</string>
+        </property>
+       </widget>
+      </item>
+     </layout>
     </widget>
    </item>
-   <item row="6" column="0" colspan="2">
+   <item>
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>213</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="1" column="1">
-    <widget class="QComboBox" name="opt_formatting_type"/>
-   </item>
-   <item row="1" column="0">
-    <widget class="QLabel" name="label_3">
-     <property name="text">
-      <string>Formatting style:</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="0" rowspan="2" colspan="2">
     <widget class="QGroupBox" name="groupBox">
      <property name="title">
-      <string>Markdown Options</string>
+      <string>Markdown</string>
      </property>
      <layout class="QVBoxLayout" name="verticalLayout">
       <item>
@@ -83,6 +121,19 @@
      </layout>
     </widget>
    </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>213</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/convert/txt_output.py

@@ -8,8 +8,6 @@ __docformat__ = 'restructuredtext en'
 from calibre.gui2.convert.txt_output_ui import Ui_Form
 from calibre.gui2.convert import Widget
 
-newline_model = None
-
 class PluginWidget(Widget, Ui_Form):
 
     TITLE = _('TXT Output')

src/calibre/gui2/convert/txtz_output.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+
+from calibre.gui2.convert.txt_output import PluginWidget as TXTPluginWidget
+
+class PluginWidget(TXTPluginWidget):
+
+    TITLE = _('TXTZ Output')
+    HELP = _('Options specific to')+' TXTZ '+_('output')
+    COMMIT_NAME = 'txtz_output'

src/calibre/gui2/device.py

@@ -1026,6 +1026,20 @@ class DeviceMixin(object): # {{{
                     self.location_manager.free[1] : 'carda',
                     self.location_manager.free[2] : 'cardb' }
                 on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
+                try:
+                    total_size = sum([os.stat(f).st_size for f in files])
+                except:
+                    try:
+                        import traceback
+                        traceback.print_exc()
+                    except:
+                        pass
+                    total_size = self.location_manager.free[0]
+                if self.location_manager.free[0] > total_size + (1024**2):
+                    # Send news to main memory if enough space available
+                    # as some devices like the Nook Color cannot handle
+                    # periodicals on SD cards properly
+                    on_card = None
                 self.upload_books(files, names, metadata,
                         on_card=on_card,
                         memory=[files, remove])

src/calibre/gui2/dialogs/add_empty_book.py

@@ -9,6 +9,7 @@ from PyQt4.Qt import QDialog, QGridLayout, QLabel, QDialogButtonBox,  \
 from calibre.ebooks.metadata import authors_to_string, string_to_authors
 from calibre.utils.icu import sort_key
 from calibre.gui2.complete import MultiCompleteComboBox
+from calibre.utils.config import tweaks
 
 class AddEmptyBookDialog(QDialog):
 
@@ -69,6 +70,7 @@ class AddEmptyBookDialog(QDialog):
 
         self.authors_combo.set_separator('&')
         self.authors_combo.set_space_before_sep(True)
+        self.authors_combo.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors_combo.update_items_cache(db.all_author_names())
 
     @property

src/calibre/gui2/dialogs/message_box.py

@@ -89,7 +89,8 @@ class MessageBox(QDialog, Ui_Dialog):
                 (__version__, unicode(self.windowTitle()),
                     unicode(self.msg.text()),
                     unicode(self.det_msg.toPlainText())))
-        self.ctc_button.setText(_('Copied'))
+        if hasattr(self, 'ctc_button'):
+            self.ctc_button.setText(_('Copied'))
 
     def showEvent(self, ev):
         ret = QDialog.showEvent(self, ev)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -781,6 +781,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
 
         self.authors.set_separator('&')
         self.authors.set_space_before_sep(True)
+        self.authors.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors.update_items_cache(self.db.all_author_names())
 
     def initialize_series(self):

src/calibre/gui2/dialogs/metadata_single.py

@@ -616,6 +616,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)
+            self.central_widget.setTabEnabled(1, False)
         else:
             self.create_custom_column_editors()
         self.generate_cover_button.clicked.connect(self.generate_cover)
@@ -734,6 +735,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
         self.authors.set_separator('&')
         self.authors.set_space_before_sep(True)
+        self.authors.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors.update_items_cache(self.db.all_author_names())
 
     def initialize_series(self):
@@ -780,8 +782,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     _('You have changed the tags. In order to use the tags'
                        ' editor, you must either discard or apply these '
                        'changes. Apply changes?'), show_copy_button=False):
-                self.books_to_refresh |= self.apply_tags(commit=True, notify=True,
+                self.books_to_refresh |= self.apply_tags(commit=True,
-                                                         allow_case_change=True)
+                        notify=True)
                 self.original_tags = unicode(self.tags.text())
             else:
                 self.tags.setText(self.original_tags)
@@ -951,8 +953,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             for w in getattr(self, 'custom_column_widgets', []):
                 self.books_to_refresh |= w.commit(self.id)
             self.db.commit()
-        except IOError, err:
+        except (IOError, OSError) as err:
-            if err.errno == 13: # Permission denied
+            if getattr(err, 'errno', -1) == 13: # Permission denied
                 fname = err.filename if err.filename else 'file'
                 return error_dialog(self, _('Permission denied'),
                         _('Could not open %s. Is it being used by another'

src/calibre/gui2/dialogs/search.py

@@ -9,6 +9,7 @@ from calibre.gui2.dialogs.search_ui import Ui_Dialog
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
 from calibre.gui2 import gprefs
 from calibre.utils.icu import sort_key
+from calibre.utils.config import tweaks
 
 box_values = {}
 
@@ -31,6 +32,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.authors_box.setEditText('')
         self.authors_box.set_separator('&')
         self.authors_box.set_space_before_sep(True)
+        self.authors_box.set_add_separator(tweaks['authors_completer_append_separator'])
         self.authors_box.update_items_cache(db.all_author_names())
 
         all_series = db.all_series()

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from PyQt4.QtCore import SIGNAL, Qt
+from PyQt4.QtCore import Qt, QString
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
@@ -11,30 +11,38 @@ class ListWidgetItem(QListWidgetItem):
 
     def __init__(self, txt):
         QListWidgetItem.__init__(self, txt)
-        self.old_value = txt
+        self.initial_value = QString(txt)
-        self.cur_value = txt
+        self.current_value = QString(txt)
+        self.previous_value = QString(txt)
 
     def data(self, role):
         if role == Qt.DisplayRole:
-            if self.old_value != self.cur_value:
+            if self.initial_value != self.current_value:
-                return _('%s (was %s)')%(self.cur_value, self.old_value)
+                return _('%s (was %s)')%(self.current_value, self.initial_value)
             else:
-                return self.cur_value
+                return self.current_value
         elif role == Qt.EditRole:
-            return self.cur_value
+            return self.current_value
         else:
             return QListWidgetItem.data(self, role)
 
     def setData(self, role, data):
         if role == Qt.EditRole:
-            self.cur_value = data.toString()
+            self.previous_value = self.current_value
+            self.current_value = data.toString()
         QListWidgetItem.setData(self, role, data)
 
     def text(self):
-        return self.cur_value
+        return self.current_value
+
+    def initial_text(self):
+        return self.initial_value
+
+    def previous_text(self):
+        return self.previous_value
 
     def setText(self, txt):
-        self.cur_value = txt
+        self.current_value = txt
         QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
@@ -49,7 +57,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.setWindowIcon(icon)
 
         self.to_rename = {}
-        self.to_delete = []
+        self.to_delete = set([])
         self.all_tags = {}
 
         for k,v in data:
@@ -57,6 +65,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for tag in sorted(self.all_tags.keys(), key=key):
             item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
+            item.setFlags (item.flags() | Qt.ItemIsEditable)
             self.available_tags.addItem(item)
 
         if tag_to_match is not None:
@@ -64,23 +73,20 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
 
-        self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
+        self.delete_button.clicked.connect(self.delete_tags)
-        self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
+        self.rename_button.clicked.connect(self.rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemDoubleClicked(QListWidgetItem *)'), self._rename_tag)
+        self.available_tags.itemDoubleClicked.connect(self._rename_tag)
-        self.connect(self.available_tags, SIGNAL('itemChanged(QListWidgetItem *)'), self.finish_editing)
+        self.available_tags.itemChanged.connect(self.finish_editing)
 
     def finish_editing(self, item):
         if not item.text():
                 error_dialog(self, _('Item is blank'),
                              _('An item cannot be set to nothing. Delete it instead.')).exec_()
-                item.setText(self.item_before_editing.text())
+                item.setText(item.previous_text())
                 return
-        if item.text() != self.item_before_editing.text():
+        if item.text() != item.initial_text():
-            (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
+            id_ = item.data(Qt.UserRole).toInt()[0]
-            if item.text() not in self.to_rename:
+            self.to_rename[id_] = unicode(item.text())
-                self.to_rename[item.text()] = [id]
-            else:
-                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -91,8 +97,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             error_dialog(self, _('No item selected'),
                          _('You must select one item from the list of Available items.')).exec_()
             return
-        self.item_before_editing = item.clone()
-        item.setFlags (item.flags() | Qt.ItemIsEditable);
         self.available_tags.editItem(item)
 
     def delete_tags(self, item=None):
@@ -108,7 +112,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         row = self.available_tags.row(deletes[0])
         for item in deletes:
             (id,ign) = item.data(Qt.UserRole).toInt()
-            self.to_delete.append(id)
+            self.to_delete.add(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
         if row >= self.available_tags.count():

src/calibre/gui2/init.py

@@ -64,6 +64,7 @@ class LibraryViewMixin(object): # {{{
             view.verticalHeader().sectionDoubleClicked.connect(self.iactions['View'].view_specific_book)
 
         self.build_context_menus()
+        self.library_view.model().set_highlight_only(config['highlight_search_matches'])
 
     def build_context_menus(self):
         lm = QMenu(self)

src/calibre/gui2/layout.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 from functools import partial
 
 from PyQt4.Qt import QIcon, Qt, QWidget, QToolBar, QSize, \
-    pyqtSignal, QToolButton, QMenu, QCheckBox, \
+    pyqtSignal, QToolButton, QMenu, \
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup
 
 
@@ -156,7 +156,8 @@ class SearchBar(QWidget): # {{{
         x = ComboBoxWithHelp(self)
         x.setMaximumSize(QSize(150, 16777215))
         x.setObjectName("search_restriction")
-        x.setToolTip(_("Books display will be restricted to those matching the selected saved search"))
+        x.setToolTip(_('Books display will be restricted to those matching the '
+                       'selected saved search'))
         l.addWidget(x)
         parent.search_restriction = x
 
@@ -175,7 +176,8 @@ class SearchBar(QWidget): # {{{
         x = parent.search = SearchBox2(self)
         x.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum)
         x.setObjectName("search")
-        x.setToolTip(_("<p>Search the list of books by title, author, publisher, tags, comments, etc.<br><br>Words separated by spaces are ANDed"))
+        x.setToolTip(_("<p>Search the list of books by title, author, publisher, "
+                       "tags, comments, etc.<br><br>Words separated by spaces are ANDed"))
         l.addWidget(x)
 
         self.search_button = QToolButton()
@@ -194,13 +196,11 @@ class SearchBar(QWidget): # {{{
         l.addWidget(x)
         x.setToolTip(_("Reset Quick Search"))
 
-        x = parent.search_highlight_only = QCheckBox()
+        x = parent.search_options_button = QToolButton(self)
-        x.setText(_('&Highlight'))
+        x.setIcon(QIcon(I('config.png')))
-        x.setToolTip('<p>'+_('When searching, highlight matched books, instead '
+        x.setObjectName("search_option_button")
-            'of restricting the book list to the matches.<p> You can use the '
-            'N or F3 keys to go to the next match.'))
         l.addWidget(x)
-        x.setVisible(False)
+        x.setToolTip(_("Change the way searching for books works"))
 
         x = parent.saved_search = SavedSearchBox(self)
         x.setMaximumSize(QSize(150, 16777215))
@@ -227,7 +227,6 @@ class SearchBar(QWidget): # {{{
         x.setToolTip(_("Delete current saved search"))
 
 
-
 # }}}
 
 class Spacer(QWidget): # {{{

src/calibre/gui2/library/delegates.py

@@ -177,6 +177,8 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
             editor = MultiCompleteLineEdit(parent)
             editor.set_separator(self.sep)
             editor.set_space_before_sep(self.space_before_sep)
+            if self.sep == '&':
+                editor.set_add_separator(tweaks['authors_completer_append_separator'])
             if not index.model().is_custom_column(col):
                 all_items = getattr(self.db, self.items_func_name)()
             else:

src/calibre/gui2/library/models.py

@@ -238,8 +238,6 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_highlight_only(self, toWhat):
         self.highlight_only = toWhat
-        if self.last_search:
-            self.research()
 
     def get_current_highlighted_id(self):
         if len(self.ids_to_highlight) == 0 or self.current_highlighted_idx is None:
@@ -791,6 +789,16 @@ class BooksModel(QAbstractTableModel): # {{{
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
             val = unicode(value.toString()).strip()
+            if val:
+                pat = re.compile(r'\[([.0-9]+)\]')
+                match = pat.search(val)
+                if match is not None:
+                    s_index = float(match.group(1))
+                    val = pat.sub('', val).strip()
+                elif val:
+                    if tweaks['series_index_auto_increment'] != 'const':
+                        s_index = self.db.get_next_cc_series_num_for(val,
+                                                        label=label, num=None)
         elif typ == 'composite':
             tmpl = unicode(value.toString()).strip()
             disp = cc['display']

src/calibre/gui2/metadata/basic_widgets.py

@@ -177,6 +177,7 @@ class AuthorsEdit(MultiCompleteComboBox):
 
         self.set_separator('&')
         self.set_space_before_sep(True)
+        self.set_add_separator(tweaks['authors_completer_append_separator'])
         self.update_items_cache(db.all_author_names())
 
         au = db.authors(id_, index_is_id=True)

src/calibre/gui2/metadata/bulk_download.py

@@ -11,7 +11,7 @@ from threading import Thread
 from Queue import Queue, Empty
 from functools import partial
 
-from PyQt4.Qt import QObject, Qt, pyqtSignal, QTimer, QDialog, \
+from PyQt4.Qt import QObject, QTimer, QDialog, \
         QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
 
 from calibre.ebooks.metadata.fetch import search, get_social_metadata
@@ -163,27 +163,23 @@ class DownloadMetadata(Thread):
 
 class DoDownload(QObject):
 
-    idle_process = pyqtSignal()
-
     def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
             get_social_metadata=True):
         QObject.__init__(self, parent)
         self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
         self.pd.canceled_signal.connect(self.cancel)
-        self.idle_process.connect(self.do_one, type=Qt.QueuedConnection)
         self.downloader = None
         self.create = partial(DownloadMetadata, db, ids, get_covers,
                 set_metadata=set_metadata,
                 get_social_metadata=get_social_metadata)
-        self.timer = QTimer(self)
         self.get_covers = get_covers
-        self.timer.timeout.connect(self.do_one, type=Qt.QueuedConnection)
         self.db = db
         self.updated = set([])
         self.total = len(ids)
+        self.keep_going = True
 
     def exec_(self):
-        self.timer.start(50)
+        QTimer.singleShot(50, self.do_one)
         ret = self.pd.exec_()
         if getattr(self.downloader, 'exception', None) is not None and \
                 ret == self.pd.Accepted:
@@ -194,30 +190,37 @@ class DoDownload(QObject):
         return ret
 
     def cancel(self, *args):
-        self.timer.stop()
+        self.keep_going = False
         self.downloader.keep_going = False
         self.pd.reject()
 
     def do_one(self):
-        if self.downloader is None:
-            self.downloader = self.create()
-            self.downloader.start()
-            self.pd.set_min(0)
-            self.pd.set_max(self.downloader.total)
         try:
-            r = self.downloader.results.get_nowait()
+            if not self.keep_going:
-            self.handle_result(r)
+                return
-        except Empty:
+            if self.downloader is None:
-            pass
+                self.downloader = self.create()
-        if not self.downloader.is_alive():
+                self.downloader.start()
-            self.timer.stop()
+                self.pd.set_min(0)
-            while True:
+                self.pd.set_max(self.downloader.total)
-                try:
+            try:
-                    r = self.downloader.results.get_nowait()
+                r = self.downloader.results.get_nowait()
-                    self.handle_result(r)
+                self.handle_result(r)
-                except Empty:
+            except Empty:
-                    break
+                pass
-            self.pd.accept()
+            if not self.downloader.is_alive():
+                while True:
+                    try:
+                        r = self.downloader.results.get_nowait()
+                        self.handle_result(r)
+                    except Empty:
+                        break
+                self.pd.accept()
+                return
+        except:
+            self.cancel()
+            raise
+        QTimer.singleShot(50, self.do_one)
 
     def handle_result(self, r):
         id_, typ, ok, title = r

src/calibre/gui2/metadata/single.py

@@ -197,7 +197,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.books_to_refresh = set([])
         for widget in self.basic_metadata_widgets:
             widget.initialize(self.db, id_)
-        for widget in self.custom_metadata_widgets:
+        for widget in getattr(self, 'custom_metadata_widgets', []):
             widget.initialize(id_)
         # Commented out as it doesn't play nice with Next, Prev buttons
         #self.fetch_metadata_button.setFocus(Qt.OtherFocusReason)

src/calibre/gui2/preferences/adding.py

@@ -12,6 +12,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
 from calibre.gui2.preferences.adding_ui import Ui_Form
 from calibre.utils.config import prefs
 from calibre.gui2.widgets import FilenamePattern
+from calibre.gui2 import gprefs
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
@@ -23,18 +24,23 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('read_file_metadata', prefs)
         r('swap_author_names', prefs)
         r('add_formats_to_existing', prefs)
+        choices = [
+                (_('Ignore duplicate incoming formats'), 'ignore'),
+                (_('Overwrite existing duplicate formats'), 'overwrite'),
+                (_('Create new record for each duplicate format'), 'new record')]
+        r('automerge', gprefs, choices=choices)
         r('new_book_tags', prefs, setting=CommaSeparatedList)
 
         self.filename_pattern = FilenamePattern(self)
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.filename_pattern.changed_signal.connect(self.changed_signal.emit)
 
-
     def initialize(self):
         ConfigWidgetBase.initialize(self)
         self.filename_pattern.blockSignals(True)
         self.filename_pattern.initialize()
         self.filename_pattern.blockSignals(False)
+        self.opt_automerge.setEnabled(self.opt_add_formats_to_existing.isChecked())
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)

src/calibre/gui2/preferences/adding.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>750</width>
+    <width>753</width>
     <height>339</height>
    </rect>
   </property>
@@ -58,16 +58,33 @@
      </item>
     </layout>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_add_formats_to_existing">
      <property name="toolTip">
-      <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
-to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
+existing book records. The box to the right controls what happens when an existing record already has
+the incoming format. Note that this option also affects the Copy to library action.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
      </property>
      <property name="text">
-      <string>If books with similar titles and authors found, &amp;merge the new files automatically</string>
+      <string>&amp;Automerge added books if they already exist in the calibre library:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QComboBox" name="opt_automerge">
+     <property name="toolTip">
+      <string>Automerge: If books with similar titles and authors found, merge the incoming formats automatically into
+existing book records. This box controls what happens when an existing record already has
+the incoming format: 
+
+Ignore duplicate incoming files - means that existing files in your calibre library will not be replaced
+Overwrite existing duplicate files - means that existing files in your calibre library will be replaced
+Create new record for each duplicate file - means that a new book entry will be created for each duplicate file
+
+Title matching ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc.
+Author matching is exact.</string>
      </property>
     </widget>
    </item>
@@ -113,5 +130,22 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
   </layout>
  </widget>
  <resources/>
- <connections/>
+ <connections>
+  <connection>
+   <sender>opt_add_formats_to_existing</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>opt_automerge</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>406</x>
+     <y>83</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>457</x>
+     <y>83</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
 </ui>

src/calibre/gui2/preferences/look_feel.py

@@ -46,7 +46,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('disable_tray_notification', config)
         r('use_roman_numerals_for_series_number', config)
         r('separate_cover_flow', config, restart_required=True)
-        r('search_as_you_type', config)
         r('show_child_bar', gprefs)
 
         choices = [(_('Small'), 'small'), (_('Medium'), 'medium'),
@@ -116,7 +115,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
 
     def refresh_gui(self, gui):
-        gui.search.search_as_you_type(config['search_as_you_type'])
         self.update_font_display()
         gui.tags_view.reread_collapse_parameters()
 

src/calibre/gui2/preferences/look_feel.ui

@@ -124,23 +124,13 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="0">
+   <item row="6" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_separate_cover_flow">
      <property name="text">
       <string>Show cover &amp;browser in a separate window (needs restart)</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="1">
-    <widget class="QCheckBox" name="opt_search_as_you_type">
-     <property name="text">
-      <string>Search as you type</string>
-     </property>
-     <property name="checked">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
    <item row="7" column="0" colspan="2">
     <layout class="QHBoxLayout">
      <item>
@@ -177,7 +167,7 @@ if you never want subcategories</string>
      <item>
       <widget class="QSpinBox" name="opt_tags_browser_collapse_at">
        <property name="toolTip">
-           <string>If a Tag Browser category has more than this number of items, it is divided
+        <string>If a Tag Browser category has more than this number of items, it is divided
 up into sub-categories. If the partition method is set to disable, this value is ignored.</string>
        </property>
        <property name="maximum">

src/calibre/gui2/preferences/main.py

@@ -157,11 +157,12 @@ class Preferences(QMainWindow):
 
     run_wizard_requested = pyqtSignal()
 
-    def __init__(self, gui, initial_plugin=None):
+    def __init__(self, gui, initial_plugin=None, close_after_initial=False):
         QMainWindow.__init__(self, gui)
         self.gui = gui
         self.must_restart = False
         self.committed = False
+        self.close_after_initial = close_after_initial
 
         self.resize(900, 720)
         nh, nw = min_available_height()-25, available_width()-10
@@ -306,7 +307,7 @@ class Preferences(QMainWindow):
 
     def esc(self, *args):
         if self.stack.currentIndex() == 1:
-            self.hide_plugin()
+            self.cancel()
         elif self.stack.currentIndex() == 0:
             self.close()
 
@@ -331,12 +332,15 @@ class Preferences(QMainWindow):
                     show_copy_button=False)
         self.showing_widget.refresh_gui(self.gui)
         self.hide_plugin()
-        if must_restart and rc:
+        if self.close_after_initial or (must_restart and rc):
             self.close()
 
 
     def cancel(self, *args):
-        self.hide_plugin()
+        if self.close_after_initial:
+            self.close()
+        else:
+            self.hide_plugin()
 
     def restore_defaults(self, *args):
         self.showing_widget.restore_defaults()

src/calibre/gui2/preferences/plugins.py

@@ -109,7 +109,7 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{
         return self.index(ans[0], 0, QModelIndex()) if ans[1] < 0 else \
                 self.index(ans[1], 0, self.index(ans[0], 0, QModelIndex()))
 
-    def index(self, row, column, parent):
+    def index(self, row, column, parent=QModelIndex()):
         if not self.hasIndex(row, column, parent):
             return QModelIndex()
 
@@ -165,8 +165,6 @@ class PluginModel(QAbstractItemModel, SearchQueryParser): # {{{
     def flags(self, index):
         if not index.isValid():
             return 0
-        if index.internalId() == 0:
-            return Qt.ItemIsEnabled
         flags = Qt.ItemIsSelectable | Qt.ItemIsEnabled
         return flags
 
@@ -329,7 +327,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     return error_dialog(self, _('Must restart'),
                             _('You must restart calibre before you can'
                                 ' configure the <b>%s</b> plugin')%plugin.name, show=True)
-                if plugin.do_user_config():
+                if plugin.do_user_config(self.gui):
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 msg = _('Plugin <b>{0}</b> successfully removed').format(plugin.name)

src/calibre/gui2/preferences/search.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QApplication
+
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, \
+        CommaSeparatedList
+from calibre.gui2.preferences.search_ui import Ui_Form
+from calibre.gui2 import config
+from calibre.utils.config import prefs
+
+class ConfigWidget(ConfigWidgetBase, Ui_Form):
+
+    def genesis(self, gui):
+        self.gui = gui
+
+        r = self.register
+
+        r('search_as_you_type', config)
+        r('highlight_search_matches', config)
+        r('limit_search_columns', prefs)
+        r('limit_search_columns_to', prefs, setting=CommaSeparatedList)
+        fl = gui.library_view.model().db.field_metadata.get_search_terms()
+        self.opt_limit_search_columns_to.update_items_cache(fl)
+
+    def refresh_gui(self, gui):
+        gui.search.search_as_you_type(config['search_as_you_type'])
+        gui.library_view.model().set_highlight_only(config['highlight_search_matches'])
+        gui.search.do_search()
+
+if __name__ == '__main__':
+    app = QApplication([])
+    test_widget('Interface', 'Search')
+

src/calibre/gui2/preferences/search.ui

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>670</width>
+    <height>392</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0">
+    <widget class="QCheckBox" name="opt_search_as_you_type">
+     <property name="text">
+      <string>Search as you &amp;type</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="0">
+    <widget class="QCheckBox" name="opt_highlight_search_matches">
+     <property name="text">
+      <string>&amp;Highlight search results instead of restricting the book list to the results</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>What to search by default</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
+      <item row="0" column="0" colspan="2">
+       <widget class="QLabel" name="label">
+        <property name="text">
+         <string>When you enter a search term without a prefix, by default calibre will search all metadata for matches. For example, entering, &quot;asimov&quot; will search not just authors but title/tags/series/comments/etc. Use these options if you would like to change this behavior.</string>
+        </property>
+        <property name="wordWrap">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0" colspan="2">
+       <widget class="QCheckBox" name="opt_limit_search_columns">
+        <property name="text">
+         <string>&amp;Limit the searched metadata</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_2">
+        <property name="text">
+         <string>&amp;Columns that non-prefixed searches are limited to:</string>
+        </property>
+        <property name="buddy">
+         <cstring>opt_limit_search_columns_to</cstring>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <widget class="MultiCompleteLineEdit" name="opt_limit_search_columns_to"/>
+      </item>
+      <item row="5" column="0" colspan="2">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>Note that this option affects all searches, including saved searches and restrictions. Therefore, if you use this option, it is best to ensure that you always use prefixes in your saved searches. For example, use &quot;series:Foundation&quot; rather than just &quot;Foundation&quot; in  a saved search</string>
+        </property>
+        <property name="wordWrap">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="3" column="0">
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>0</width>
+       <height>0</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>MultiCompleteLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>calibre/gui2.complete.h</header>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/gui2/preferences/tweaks.py

@@ -5,37 +5,312 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import textwrap
+
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, AbortCommit
 from calibre.gui2.preferences.tweaks_ui import Ui_Form
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, NONE
 from calibre.utils.config import read_raw_tweaks, write_tweaks
+from calibre.gui2.widgets import PythonHighlighter
+from calibre import isbytestring
 
+from PyQt4.Qt import QAbstractListModel, Qt, QStyledItemDelegate, QStyle, \
+    QStyleOptionViewItem, QFont, QDialogButtonBox, QDialog, \
+    QVBoxLayout, QPlainTextEdit, QLabel
+
+class Delegate(QStyledItemDelegate): # {{{
+    def __init__(self, view):
+        QStyledItemDelegate.__init__(self, view)
+        self.view = view
+
+    def paint(self, p, opt, idx):
+        copy = QStyleOptionViewItem(opt)
+        copy.showDecorationSelected = True
+        if self.view.currentIndex() == idx:
+            copy.state |= QStyle.State_HasFocus
+        QStyledItemDelegate.paint(self, p, copy, idx)
+
+# }}}
+
+class Tweak(object): # {{{
+
+    def __init__(self, name, doc, var_names, defaults, custom):
+        self.name = name
+        self.doc = doc.strip()
+        self.var_names = var_names
+        self.default_values = {}
+        for x in var_names:
+            self.default_values[x] = defaults[x]
+        self.custom_values = {}
+        for x in var_names:
+            if x in custom:
+                self.custom_values[x] = custom[x]
+
+    def __str__(self):
+        ans = ['#: ' + self.name]
+        for line in self.doc.splitlines():
+            if line:
+                ans.append('# ' + line)
+        for key, val in self.default_values.iteritems():
+            val = self.custom_values.get(key, val)
+            ans.append('%s = %r'%(key, val))
+        ans = '\n'.join(ans)
+        if isinstance(ans, unicode):
+            ans = ans.encode('utf-8')
+        return ans
+
+    def __cmp__(self, other):
+        return cmp(self.is_customized, getattr(other, 'is_customized', False))
+
+    @property
+    def is_customized(self):
+        for x, val in self.default_values.iteritems():
+            if self.custom_values.get(x, val) != val:
+                return True
+        return False
+
+    @property
+    def edit_text(self):
+        ans = ['# %s'%self.name]
+        for x, val in self.default_values.iteritems():
+            val = self.custom_values.get(x, val)
+            ans.append('%s = %r'%(x, val))
+        return '\n\n'.join(ans)
+
+    def restore_to_default(self):
+        self.custom_values.clear()
+
+    def update(self, varmap):
+        self.custom_values.update(varmap)
+
+# }}}
+
+class Tweaks(QAbstractListModel): # {{{
+
+    def __init__(self, parent=None):
+        QAbstractListModel.__init__(self, parent)
+        raw_defaults, raw_custom = read_raw_tweaks()
+
+        self.parse_tweaks(raw_defaults, raw_custom)
+
+    def rowCount(self, *args):
+        return len(self.tweaks)
+
+    def data(self, index, role):
+        row = index.row()
+        try:
+            tweak = self.tweaks[row]
+        except:
+            return NONE
+        if role == Qt.DisplayRole:
+            return textwrap.fill(tweak.name, 40)
+        if role == Qt.FontRole and tweak.is_customized:
+            ans = QFont()
+            ans.setBold(True)
+            return ans
+        if role == Qt.ToolTipRole:
+            tt = _('This tweak has it default value')
+            if tweak.is_customized:
+                tt = _('This tweak has been customized')
+            return tt
+        if role == Qt.UserRole:
+            return tweak
+        return NONE
+
+    def parse_tweaks(self, defaults, custom):
+        l, g = {}, {}
+        try:
+            exec custom in g, l
+        except:
+            print 'Failed to load custom tweaks file'
+            import traceback
+            traceback.print_exc()
+        dl, dg = {}, {}
+        exec defaults in dg, dl
+        lines = defaults.splitlines()
+        pos = 0
+        self.tweaks = []
+        while pos < len(lines):
+            line = lines[pos]
+            if line.startswith('#:'):
+                pos = self.read_tweak(lines, pos, dl, l)
+            pos += 1
+
+        default_keys = set(dl.iterkeys())
+        custom_keys = set(l.iterkeys())
+
+        self.plugin_tweaks = {}
+        for key in custom_keys - default_keys:
+            self.plugin_tweaks[key] = l[key]
+
+    def read_tweak(self, lines, pos, defaults, custom):
+        name = lines[pos][2:].strip()
+        doc, var_names = [], []
+        while True:
+            pos += 1
+            line = lines[pos]
+            if not line.startswith('#'):
+                break
+            doc.append(line[1:].strip())
+        doc = '\n'.join(doc)
+        while True:
+            line = lines[pos]
+            if not line.strip():
+                break
+            spidx1 = line.find(' ')
+            spidx2 = line.find('=')
+            spidx = spidx1 if spidx1 > 0 and (spidx2 == 0 or spidx2 > spidx1) else spidx2
+            if spidx > 0:
+                var = line[:spidx]
+                if var not in defaults:
+                    raise ValueError('%r not in default tweaks dict'%var)
+                var_names.append(var)
+            pos += 1
+        if not var_names:
+            raise ValueError('Failed to find any variables for %r'%name)
+        self.tweaks.append(Tweak(name, doc, var_names, defaults, custom))
+        #print '\n\n', self.tweaks[-1]
+        return pos
+
+    def restore_to_default(self, idx):
+        tweak = self.data(idx, Qt.UserRole)
+        if tweak is not NONE:
+            tweak.restore_to_default()
+            self.dataChanged.emit(idx, idx)
+
+    def restore_to_defaults(self):
+        for r in range(self.rowCount()):
+            self.restore_to_default(self.index(r))
+
+    def update_tweak(self, idx, varmap):
+        tweak = self.data(idx, Qt.UserRole)
+        if tweak is not NONE:
+            tweak.update(varmap)
+            self.dataChanged.emit(idx, idx)
+
+    def to_string(self):
+        ans = ['#!/usr/bin/env python',
+               '# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai', '',
+               '# This file was automatically generated by calibre, do not'
+               ' edit it unless you know what you are doing.', '',
+            ]
+        for tweak in self.tweaks:
+            ans.extend(['', str(tweak), ''])
+
+        if self.plugin_tweaks:
+            ans.extend(['', '',
+                '# The following are tweaks for installed plugins', ''])
+            for key, val in self.plugin_tweaks.iteritems():
+                ans.extend(['%s = %r'%(key, val), '', ''])
+        return '\n'.join(ans)
+
+    @property
+    def plugin_tweaks_string(self):
+        ans = []
+        for key, val in self.plugin_tweaks.iteritems():
+            ans.extend(['%s = %r'%(key, val), '', ''])
+        ans = '\n'.join(ans)
+        if isbytestring(ans):
+            ans = ans.decode('utf-8')
+        return ans
+
+    def set_plugin_tweaks(self, d):
+        self.plugin_tweaks = d
+
+# }}}
+
+class PluginTweaks(QDialog): # {{{
+
+    def __init__(self, raw, parent=None):
+        QDialog.__init__(self, parent)
+        self.edit = QPlainTextEdit(self)
+        self.highlighter = PythonHighlighter(self.edit.document())
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.l.addWidget(QLabel(
+            _('Add/edit tweaks for any custom plugins you have installed.')))
+        self.l.addWidget(self.edit)
+        self.edit.setPlainText(raw)
+        self.bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel,
+                Qt.Horizontal, self)
+        self.bb.accepted.connect(self.accept)
+        self.bb.rejected.connect(self.reject)
+        self.l.addWidget(self.bb)
+        self.resize(550, 300)
+
+# }}}
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
-        self.current_tweaks.textChanged.connect(self.changed)
+        self.delegate = Delegate(self.tweaks_view)
+        self.tweaks_view.setItemDelegate(self.delegate)
+        self.tweaks_view.currentChanged = self.current_changed
+        self.highlighter = PythonHighlighter(self.edit_tweak.document())
+        self.restore_default_button.clicked.connect(self.restore_to_default)
+        self.apply_button.clicked.connect(self.apply_tweak)
+        self.plugin_tweaks_button.clicked.connect(self.plugin_tweaks)
+
+    def plugin_tweaks(self):
+        raw = self.tweaks.plugin_tweaks_string
+        d = PluginTweaks(raw, self)
+        if d.exec_() == d.Accepted:
+            g, l = {}, {}
+            try:
+                exec unicode(d.edit.toPlainText()) in g, l
+            except:
+                import traceback
+                return error_dialog(self, _('Failed'),
+                    _('There was a syntax error in your tweak. Click '
+                        'the show details button for details.'), show=True,
+                    det_msg=traceback.format_exc())
+            self.tweaks.set_plugin_tweaks(l)
+            self.changed()
+
+    def current_changed(self, current, previous):
+        tweak = self.tweaks.data(current, Qt.UserRole)
+        self.help.setPlainText(tweak.doc)
+        self.edit_tweak.setPlainText(tweak.edit_text)
 
     def changed(self, *args):
         self.changed_signal.emit()
 
     def initialize(self):
-        deft, curt = read_raw_tweaks()
+        self.tweaks = Tweaks()
-        self.current_tweaks.blockSignals(True)
+        self.tweaks_view.setModel(self.tweaks)
-        self.current_tweaks.setPlainText(curt.decode('utf-8'))
-        self.current_tweaks.blockSignals(False)
 
-        self.default_tweaks.setPlainText(deft.decode('utf-8'))
+    def restore_to_default(self, *args):
+        idx = self.tweaks_view.currentIndex()
+        if idx.isValid():
+            self.tweaks.restore_to_default(idx)
+            tweak = self.tweaks.data(idx, Qt.UserRole)
+            self.edit_tweak.setPlainText(tweak.edit_text)
+            self.changed()
 
     def restore_defaults(self):
         ConfigWidgetBase.restore_defaults(self)
-        deft, curt = read_raw_tweaks()
+        self.tweaks.restore_to_defaults()
-        self.current_tweaks.setPlainText(deft.decode('utf-8'))
+        self.changed()
 
+    def apply_tweak(self):
+        idx = self.tweaks_view.currentIndex()
+        if idx.isValid():
+            l, g = {}, {}
+            try:
+                exec unicode(self.edit_tweak.toPlainText()) in g, l
+            except:
+                import traceback
+                error_dialog(self.gui, _('Failed'),
+                        _('There was a syntax error in your tweak. Click '
+                            'the show details button for details.'),
+                        det_msg=traceback.format_exc(), show=True)
+                return
+            self.tweaks.update_tweak(idx, l)
+            self.changed()
 
     def commit(self):
-        raw = unicode(self.current_tweaks.toPlainText()).encode('utf-8')
+        raw = self.tweaks.to_string()
         try:
             exec raw
         except:
@@ -54,5 +329,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication
     app = QApplication([])
+    #Tweaks()
+    #test_widget
     test_widget('Advanced', 'Tweaks')
 

src/calibre/gui2/preferences/tweaks.ui

@@ -7,31 +7,73 @@
     <x>0</x>
     <y>0</y>
     <width>660</width>
-    <height>351</height>
+    <height>531</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
+  <layout class="QGridLayout" name="gridLayout_2">
-   <item>
+   <item row="0" column="0" rowspan="2">
-    <widget class="QLabel" name="label_18">
+    <layout class="QVBoxLayout" name="verticalLayout_2">
-     <property name="text">
+     <item>
-      <string>Values for the tweaks are shown below. Edit them to change the behavior of calibre. Your changes will only take effect after a restart of calibre.</string>
+      <widget class="QLabel" name="label_18">
-     </property>
+       <property name="text">
-     <property name="wordWrap">
+        <string>Values for the tweaks are shown below. Edit them to change the behavior of calibre. Your changes will only take effect &lt;b&gt;after a restart&lt;/b&gt; of calibre.</string>
-      <bool>true</bool>
+       </property>
-     </property>
+       <property name="wordWrap">
-    </widget>
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QListView" name="tweaks_view">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Minimum" vsizetype="Expanding">
+         <horstretch>0</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+       <property name="minimumSize">
+        <size>
+         <width>300</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="alternatingRowColors">
+        <bool>true</bool>
+       </property>
+       <property name="spacing">
+        <number>5</number>
+       </property>
+       <property name="uniformItemSizes">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="plugin_tweaks_button">
+       <property name="toolTip">
+        <string>Edit tweaks for any custom plugins you have installed</string>
+       </property>
+       <property name="text">
+        <string>&amp;Plugin tweaks</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
    </item>
-   <item>
+   <item row="0" column="1">
-    <widget class="QGroupBox" name="groupBox_6">
+    <widget class="QGroupBox" name="groupBox">
      <property name="title">
-      <string>All available tweaks</string>
+      <string>Help</string>
      </property>
-     <layout class="QGridLayout" name="gridLayout_11">
+     <layout class="QVBoxLayout" name="verticalLayout">
-      <item row="0" column="0">
+      <item>
-       <widget class="QPlainTextEdit" name="default_tweaks">
+       <widget class="QPlainTextEdit" name="help">
+        <property name="lineWrapMode">
+         <enum>QPlainTextEdit::NoWrap</enum>
+        </property>
         <property name="readOnly">
          <bool>true</bool>
         </property>
@@ -40,14 +82,38 @@
      </layout>
     </widget>
    </item>
-   <item>
+   <item row="1" column="1">
-    <widget class="QGroupBox" name="groupBox_7">
+    <widget class="QGroupBox" name="groupBox_2">
      <property name="title">
-      <string>&amp;Current tweaks</string>
+      <string>Edit tweak</string>
      </property>
-     <layout class="QGridLayout" name="gridLayout_10">
+     <layout class="QGridLayout" name="gridLayout">
-      <item row="0" column="0">
+      <item row="0" column="0" colspan="2">
-       <widget class="QPlainTextEdit" name="current_tweaks"/>
+       <widget class="QPlainTextEdit" name="edit_tweak">
+        <property name="lineWrapMode">
+         <enum>QPlainTextEdit::NoWrap</enum>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="0">
+       <widget class="QPushButton" name="restore_default_button">
+        <property name="toolTip">
+         <string>Restore this tweak to its default value</string>
+        </property>
+        <property name="text">
+         <string>Restore &amp;default</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QPushButton" name="apply_button">
+        <property name="toolTip">
+         <string>Apply any changes you made to this tweak</string>
+        </property>
+        <property name="text">
+         <string>&amp;Apply</string>
+        </property>
+       </widget>
       </item>
      </layout>
     </widget>

src/calibre/gui2/search_box.py

@@ -16,7 +16,6 @@ from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
-from calibre.utils.config import dynamic
 from calibre.utils.search_query_parser import saved_searches
 from calibre.utils.icu import sort_key
 
@@ -271,7 +270,7 @@ class SavedSearchBox(QComboBox): # {{{
     def initialize(self, _search_box, colorize=False, help_text=_('Search')):
         self.search_box = _search_box
         try:
-           self.line_edit.setPlaceholderText(help_text)
+            self.line_edit.setPlaceholderText(help_text)
         except:
             # Using Qt < 4.7
             pass
@@ -376,9 +375,7 @@ class SearchBoxMixin(object): # {{{
             unicode(self.search.toolTip())))
         self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip())
         self.clear_button.setStatusTip(self.clear_button.toolTip())
-        self.search_highlight_only.stateChanged.connect(self.highlight_only_changed)
+        self.search_options_button.clicked.connect(self.search_options_button_clicked)
-        self.search_highlight_only.setChecked(
-                            dynamic.get('search_highlight_only', False))
 
     def focus_search_box(self, *args):
         self.search.setFocus(Qt.OtherFocusReason)
@@ -402,14 +399,13 @@ class SearchBoxMixin(object): # {{{
         self.search.do_search()
         self.focus_to_library()
 
+    def search_options_button_clicked(self):
+        self.iactions['Preferences'].do_config(initial_plugin=('Interface',
+            'Search'), close_after_initial=True)
+
     def focus_to_library(self):
         self.current_view().setFocus(Qt.OtherFocusReason)
 
-    def highlight_only_changed(self, toWhat):
-        dynamic.set('search_highlight_only', toWhat)
-        self.current_view().model().set_highlight_only(toWhat)
-        self.focus_to_library()
-
     # }}}
 
 class SavedSearchBoxMixin(object): # {{{

src/calibre/gui2/tag_view.py

@@ -1214,7 +1214,7 @@ class TagBrowserMixin(object): # {{{
             db.field_metadata.remove_user_categories()
             for k in d.categories:
                 db.field_metadata.add_user_category('@' + k, k)
-            db.data.sqp_change_locations(db.field_metadata.get_search_terms())
+            db.data.change_search_locations(db.field_metadata.get_search_terms())
             self.tags_view.set_new_model()
             self.tags_view.recount()
 
@@ -1259,9 +1259,8 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for item in to_delete:
                     delete_func(item)
-                for text in to_rename:
+                for old_id in to_rename:
-                        for old_id in to_rename[text]:
+                    rename_func(old_id, new_name=unicode(to_rename[old_id]))
-                            rename_func(old_id, new_name=unicode(text))
 
             # Clean up the library view
             self.do_tag_item_renamed()

src/calibre/gui2/tools.py

@@ -9,7 +9,7 @@ Logic for setting up conversion jobs
 
 import cPickle, os
 
-from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer, SIGNAL
+from PyQt4.Qt import QDialog, QProgressDialog, QString, QTimer
 
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2 import warning_dialog, question_dialog
@@ -24,7 +24,8 @@ from calibre.ebooks.conversion.config import GuiRecommendations, \
     load_defaults, load_specifics, save_specifics
 from calibre.gui2.convert import bulk_defaults_for_input_format
 
-def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None):
+def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
+        out_format=None):
     changed = False
     jobs = []
     bad = []
@@ -95,7 +96,9 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format
             msg).exec_()
 
     return jobs, changed, bad
+# }}}
 
+# Bulk convert {{{
 def convert_bulk_ebook(parent, queue, db, book_ids, out_format=None, args=[]):
     total = len(book_ids)
     if total == 0:
@@ -125,14 +128,11 @@ class QueueBulk(QProgressDialog):
         self.parent = parent
         self.use_saved_single_settings = use_saved_single_settings
         self.i, self.bad, self.jobs, self.changed = 0, [], [], False
-        self.timer = QTimer(self)
+        QTimer.singleShot(0, self.do_book)
-        self.connect(self.timer, SIGNAL('timeout()'), self.do_book)
-        self.timer.start()
         self.exec_()
 
     def do_book(self):
         if self.i >= len(self.book_ids):
-            self.timer.stop()
             return self.do_queue()
         book_id = self.book_ids[self.i]
         self.i += 1
@@ -191,6 +191,7 @@ class QueueBulk(QProgressDialog):
             self.setValue(self.i)
         except NoSupportedInputFormats:
             self.bad.append(book_id)
+        QTimer.singleShot(0, self.do_book)
 
     def do_queue(self):
         self.hide()
@@ -209,7 +210,9 @@ class QueueBulk(QProgressDialog):
         self.jobs.reverse()
         self.queue(self.jobs, self.changed, self.bad, *self.args)
 
-def fetch_scheduled_recipe(arg):
+# }}}
+
+def fetch_scheduled_recipe(arg): # {{{
     fmt = prefs['output_format'].lower()
     pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower())
     pt.close()
@@ -250,7 +253,9 @@ def fetch_scheduled_recipe(arg):
 
     return 'gui_convert', args, _('Fetch news from ')+arg['title'], fmt.upper(), [pt]
 
-def generate_catalog(parent, dbspec, ids, device_manager, db):
+# }}}
+
+def generate_catalog(parent, dbspec, ids, device_manager, db): # {{{
     from calibre.gui2.dialogs.catalog import Catalog
 
     # Build the Catalog dialog in gui2.dialogs.catalog
@@ -308,8 +313,9 @@ def generate_catalog(parent, dbspec, ids, device_manager, db):
     # Which then calls gui2.convert.gui_conversion:gui_catalog() with the args inline
     return 'gui_catalog', args, _('Generate catalog'), out.name, d.catalog_sync, \
             d.catalog_title
+# }}}
 
-def convert_existing(parent, db, book_ids, output_format):
+def convert_existing(parent, db, book_ids, output_format): # {{{
     already_converted_ids = []
     already_converted_titles = []
     for book_id in book_ids:
@@ -325,3 +331,5 @@ def convert_existing(parent, db, book_ids, output_format):
             book_ids = [x for x in book_ids if x not in already_converted_ids]
 
     return book_ids
+# }}}
+

src/calibre/gui2/ui.py

@@ -483,8 +483,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             action.location_selected(location)
         if location == 'library':
             self.search_restriction.setEnabled(True)
+            self.search_options_button.setEnabled(True)
         else:
             self.search_restriction.setEnabled(False)
+            self.search_options_button.setEnabled(False)
             # Reset the view in case something changed while it was invisible
             self.current_view().reset()
         self.set_number_of_books_shown()

src/calibre/gui2/widgets.py

@@ -20,7 +20,7 @@ from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata.meta import metadata_from_filename
-from calibre.utils.config import prefs, XMLConfig
+from calibre.utils.config import prefs, XMLConfig, tweaks
 from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator
 
 history = XMLConfig('history')
@@ -932,7 +932,7 @@ class SplitterHandle(QSplitterHandle):
 
     def paintEvent(self, ev):
         QSplitterHandle.paintEvent(self, ev)
-        if self.highlight:
+        if self.highlight and tweaks['draw_hidden_section_indicators']:
             painter = QPainter(self)
             painter.setClipRect(ev.rect())
             painter.fillRect(self.rect(), Qt.yellow)

src/calibre/library/caches.py

@@ -11,7 +11,7 @@ from itertools import repeat
 from datetime import timedelta
 from threading import Thread
 
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
@@ -182,15 +182,16 @@ class ResultCache(SearchQueryParser): # {{{
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
-        all_search_locations = field_metadata.get_search_terms()
+        self.all_search_locations = field_metadata.get_search_terms()
-        SearchQueryParser.__init__(self, all_search_locations, optimize=True)
+        SearchQueryParser.__init__(self, self.all_search_locations, optimize=True)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
     def break_cycles(self):
         self._data = self.field_metadata = self.FIELD_MAP = \
             self.numeric_search_relops = self.date_search_relops = \
-            self.db_prefs = None
+            self.db_prefs = self.all_search_locations = None
+        self.sqp_change_locations([])
 
 
     def __getitem__(self, row):
@@ -218,6 +219,10 @@ class ResultCache(SearchQueryParser): # {{{
     def universal_set(self):
         return set([i[0] for i in self._data if i is not None])
 
+    def change_search_locations(self, locations):
+        self.sqp_change_locations(locations)
+        self.all_search_locations = locations
+
     def build_date_relop_dict(self):
         '''
         Because the database dates have time in them, we can't use direct
@@ -432,6 +437,7 @@ class ResultCache(SearchQueryParser): # {{{
             # get metadata key associated with the search term. Eliminates
             # dealing with plurals and other aliases
             location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
+            # grouped search terms
             if isinstance(location, list):
                 if allow_recursion:
                     for loc in location:
@@ -440,6 +446,20 @@ class ResultCache(SearchQueryParser): # {{{
                     return matches
                 raise ParseException(query, len(query), 'Recursive query group detected', self)
 
+            # apply the limit if appropriate
+            if location == 'all' and prefs['limit_search_columns'] and \
+                            prefs['limit_search_columns_to']:
+                terms = set([])
+                for l in prefs['limit_search_columns_to']:
+                    l = icu_lower(l.strip())
+                    if l and l != 'all' and l in self.all_search_locations:
+                        terms.add(l)
+                if terms:
+                    for l in terms:
+                        matches |= self.get_matches(l, query,
+                            candidates=candidates, allow_recursion=allow_recursion)
+                    return matches
+
             if location in self.field_metadata:
                 fm = self.field_metadata[location]
                 # take care of dates special case

src/calibre/library/custom_columns.py

@@ -445,7 +445,7 @@ class CustomColumns(object):
         rv = self._set_custom(id, val, label=label, num=num, append=append,
                          notify=notify, extra=extra,
                          allow_case_change=allow_case_change)
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|rv, commit=False)
         if commit:
             self.conn.commit()
         return rv
@@ -484,7 +484,9 @@ class CustomColumns(object):
             if not existing:
                 existing = []
             for x in set(set_val) - set(existing):
-                if x is None:
+                # normalized types are text and ratings, so we can do this check
+                # to see if we need to re-add the value
+                if not x:
                     continue
                 case_change = False
                 existing = list(self.all_custom(num=data['num']))

src/calibre/library/database2.py

@@ -414,7 +414,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         row = self.data._data[index] if index_is_id else self.data[index]
         return row[self.FIELD_MAP['path']].replace('/', os.sep)
 
-
     def abspath(self, index, index_is_id=False, create_dirs=True):
         'Return the absolute path to the directory containing this books files as a unicode string.'
         path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id))
@@ -422,7 +421,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             os.makedirs(path)
         return path
 
-
     def construct_path_name(self, id):
         '''
         Construct the directory name for this book based on its metadata.
@@ -432,7 +430,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             authors = _('Unknown')
         author = ascii_filename(authors.split(',')[0])[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
         title  = ascii_filename(self.title(id, index_is_id=True))[:self.PATH_LIMIT].decode(filesystem_encoding, 'replace')
-        path   = author + '/' + title + ' (%d)'%id
+        while author[-1] in (' ', '.'):
+            author = author[:-1]
+        if not author:
+            author = ascii_filename(_('Unknown')).decode(filesystem_encoding, 'replace')
+        path = author + '/' + title + ' (%d)'%id
         return path
 
     def construct_file_name(self, id):
@@ -785,7 +787,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.id = id
         for key, meta in self.field_metadata.custom_iteritems():
             mi.set_user_metadata(key, meta)
-            mi.set(key, val=self.get_custom(idx, label=meta['label'],
+            if meta['datatype'] == 'composite':
+                mi.set(key, val=row[meta['rec_index']])
+            else:
+                mi.set(key, val=self.get_custom(idx, label=meta['label'],
                                             index_is_id=index_is_id),
                         extra=self.get_custom_extra(idx, label=meta['label'],
                                                     index_is_id=index_is_id))
@@ -1692,7 +1697,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         '''
         books_to_refresh = self._set_authors(id, authors,
                                              allow_case_change=allow_case_change)
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         self.set_path(id, index_is_id=True)
@@ -1768,7 +1773,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id)
                              FROM books_publishers_link
                              WHERE publisher=publishers.id) < 1''')
-        books_to_refresh = set()
+        books_to_refresh = set([])
         if publisher:
             case_change = False
             if not isinstance(publisher, unicode):
@@ -1793,7 +1798,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 bks = self.conn.get('''SELECT book FROM books_publishers_link
                                        WHERE publisher=?''', (aid,))
                 books_to_refresh |= set([bk[0] for bk in bks])
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
@@ -2206,7 +2211,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 bks = self.conn.get('SELECT book FROM books_tags_link WHERE tag=?',
                                         (tid,))
                 books_to_refresh |= set([bk[0] for bk in bks])
-        self.dirtied([id], commit=False)
+        self.dirtied(set([id])|books_to_refresh, commit=False)
         if commit:
             self.conn.commit()
         tags = u','.join(self.get_tags(id))

src/calibre/library/server/content.py

@@ -124,8 +124,7 @@ class ContentServer(object):
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
 
-        # A better search would be great
+        want_mobile = self.is_mobile_browser(ua)
-        want_mobile = self.MOBILE_UA.search(ua) is not None
         if self.opts.develop and not want_mobile:
             cherrypy.log('User agent: '+ua)
 

src/calibre/library/server/mobile.py

@@ -169,6 +169,10 @@ class MobileServer(object):
 
     MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian|HD2|Kindle)')
 
+    def is_mobile_browser(self, ua):
+        match = self.MOBILE_UA.search(ua)
+        return match is not None and 'iPad' not in ua
+
     def add_routes(self, connect):
         connect('mobile', '/mobile', self.mobile)
         connect('mobile_css', '/mobile/style.css', self.mobile_css)

src/calibre/manual/faq.rst

@@ -182,11 +182,6 @@ If you don't want to uninstall it altogether, there are a couple of tricks you c
 simplest is to simply re-name the executable file that launches the library program. More detail
 `in the forums <http://www.mobileread.com/forums/showthread.php?t=65809>`_.
 
-Can I use the collections feature of the SONY reader?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| has full support for collections. When you add tags to a book's metadata, those tags are turned into collections when you upload the book to the SONY reader. Also, the series information is automatically
-turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does.
-
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -316,6 +311,27 @@ When you first run |app|, it will ask you for a folder in which to store your bo
 
 Metadata about the books is stored in the file ``metadata.db`` at the top level of the library folder This file is is a sqlite database. When backing up your library make sure you copy the entire folder and all its sub-folders.
 
+How does |app| manage author names and sorting?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Author names are complex, especially across cultures. |app| has a very flexible strategy for managing author names. The first thing to understand is that books and authors are separate entities in |app|. A book can have more than one author, and an author can have more than one book. You can manage the authors of a book by the edit metadata dialog. You can manage individual authors by right clicking on the author in the Tag Browser on the left of the main |app| screen and selecting :guilabel:`Manage authors`. Using this dialog you can change the name of an author and also how that name is sorted. This will automatically change the name of the author in all the books of that author. When a book has multiple authors, separate their names using the & character.
+
+Now coming to author name sorting:
+
+    * When a new author is added to |app| (this happens whenever a book by a new author is added), |app| automatically computes a sort string for both the book and the author.
+    * Authors in the Tag Browser are sorted by the sort value for the **authors**. Remember that this is different from the Author sort field for a book. 
+    * By default, this sort algorithm assumes that the author name is in ``First name Last name`` format and generates a ``Last name, First name`` sort value.
+    * You can change this algorithm by going to Preferences->Tweaks and setting the :guilabel:`author_sort_copy_method` tweak.
+    * You can force |app| to recalculate the author sort values for every author by right clicking on any author and selecting :guilabel:`Manage authors`, then pushing the `Recalculate all author sort values` button. Do this after you have set the author_sort_copy_method tweak to what you want.
+    * You can force |app| to recalculate the author sort values for all books by using the bulk metadata edit dialog (select all books and click edit metadata, check the `Automatically set author sort` checkbox, then press OK.)
+    * When recalculating the author sort values for books, |app| uses the author sort values for each individual author. Therefore, ensure that the individual author sort values are correct before recalculating the books' author sort values.
+    * You can control whether the Tag Browser display authors using their names or their sort values by setting the :guilabel:`categories_use_field_for_author_name` tweak in Preferences->Tweaks
+
+With all this flexibility, it is possible to have |app| manage your author names however you like. For example, one common request is to have |app| display author names LN, FN. To do this first set the ``author_sort_copy_method`` to ``copy``. Then change all author names to LN, FN via the Manage authors dialog. Then have |app| recalculate author sort values for both authors and books as described above.
+
+Note that you can set an individual author's sort value to whatever you want using :guilabel:`Manage authors`. This is useful when dealing with names that |app| will not get right, such as complex multi-part names like Miguel de Cervantes Saavedra or when dealing with Asian names like Sun Tzu.
+
+
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -354,13 +370,6 @@ Content From The Web
   :depth: 1
   :local:
 
-My downloaded news content causes the reader to reset.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This is a bug in the SONY firmware. The problem can be mitigated by switching the output format to EPUB
-in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software
-to transfer the files to the reader. The SONY software pre-paginates the LRF file,
-thereby reducing the number of resets.
-
 I obtained a recipe for a news site as a .py file from somewhere, how do I use it?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Start the :guilabel:`Add custom news sources` dialog (from the :guilabel:`Fetch news` menu) and click the :guilabel:`Switch to advanced mode` button. Delete everything in the box with the recipe source code and copy paste the contents of your .py file into the box. Click :guilabel:`Add/update recipe`.
@@ -370,7 +379,7 @@ I want |app| to download news from my favorite news website.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you are reasonably proficient with computers, you can teach |app| to download news from any website of your choosing. To learn how to do this see :ref:`news`.
 
-Otherwise, you can register a request for a particular news site by adding a comment `to this ticket <http://bugs.calibre-ebook.com/ticket/405>`_.
+Otherwise, you can request a particular news site by posting in the `calibre Recipes forum <http://www.mobileread.com/forums/forumdisplay.php?f=228>`_.
 
 Can I use web2disk to download an arbitrary website?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -459,7 +468,7 @@ How do I backup |app|?
 
 The most important thing to backup is the |app| library folder, that contains all your books and metadata. This is the folder you chose for your |app| library when you ran |app| for the first time. You can get the path to the library folder by clicking the |app| icon on the main toolbar. You must backup this complete folder with all its files and sub-folders.
 
-You can switch |app| to using a backed up library folder by simply clicking the |app| icon on the toolbar and choosing your backup library folder.
+You can switch |app| to using a backed up library folder by simply clicking the |app| icon on the toolbar and choosing your backup library folder. A backed up library folder backs up your custom columns and saved searches as well as all your books and metadata.
 
 If you want to backup the |app| configuration/plugins, you have to backup the config directory. You can find this config directory via :guilabel:`Preferences->Miscellaneous`. Note that restoring configuration directories is not officially supported, but should work in most cases. Just copy the contents of the backup directory into the current configuration directory to restore.
 
@@ -470,7 +479,7 @@ Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. Thi
 I am getting a "Permission Denied" error?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A permission denied error can occur because of many possible reasons, none of them having anything to do with |app|. You can get permission denied errors if you are using an SD card with write protect enabled. Or if you, or some program you used changed the file permissions of the files in question to read only. Or if there is a filesystem error on the device which caused your operating system to mount the filesystem in read only mode or mark a particular file as read only pending recovery. Or if the files have their owner set to a user other than you. You will need to fix the underlying cause of the permissions error before resuming to use |app|. Read the error message carefully, see what file it points to and fix the permissions on that file.
+A permission denied error can occur because of many possible reasons, none of them having anything to do with |app|. You can get permission denied errors if you are using an SD card with write protect enabled. Or if you, or some program you used changed the file permissions of the files in question to read only. Or if there is a filesystem error on the device which caused your operating system to mount the filesystem in read only mode or mark a particular file as read only pending recovery. Or if the files have their owner set to a user other than you. Or if your file is open in another program. You will need to fix the underlying cause of the permissions error before resuming to use |app|. Read the error message carefully, see what file it points to and fix the permissions on that file.
 
 Can I have the comment metadata show up on my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -483,7 +492,7 @@ I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 You have two choices:
  1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
- 2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first) and hopefully I will find the time to implement your feature.
+ 2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it.
 
 Can I include |app| on a CD to be distributed with my product/magazine?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -501,7 +510,7 @@ Why are there so many calibre-parallel processes on my system?
 
 In addition to this some conversion plugins run tasks in their own pool of processes, so for example if you bulk convert comics, each comic conversion will use three separate processes to render the images. The job manager knows this so it will run only a single comic conversion simultaneously.
 
-And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run the GUI thread of the main process or in a separate process.
+And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run in the GUI thread of the main process or in a separate process.
 
 Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes.
 

src/calibre/utils/config.py

@@ -728,6 +728,17 @@ def _prefs():
     c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
     c.add_opt('manage_device_metadata', default='manual',
         help=_('How and when calibre updates metadata on the device.'))
+    c.add_opt('limit_search_columns', default=False,
+            help=_('When searching for text without using lookup '
+            'prefixes, as for example, Red instead of title:Red, '
+            'limit the columns searched to those named below.'))
+    c.add_opt('limit_search_columns_to',
+            default=['title', 'authors', 'tags', 'series', 'publisher'],
+            help=_('Choose columns to be searched when not using prefixes, '
+                'as for example, when searching for Redd instead of '
+                'title:Red. Enter a list of search/lookup names '
+                'separated by commas. Only takes effect if you set the option '
+                'to limit search columns above.'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c

src/calibre/utils/html2textile.py

@@ -77,7 +77,7 @@ class EchoTarget:
             new_tag = '~'
             newline = ''
         elif tag == 'span':
-            new_tag = '%'
+            new_tag = ''
             newline = ''
         elif tag == 'a':
             self.block = True
@@ -147,7 +147,7 @@ class EchoTarget:
         elif tag == 'sub':
             self.final_output.append('~')
         elif tag == 'span':
-            self.final_output.append('%')
+            self.final_output.append('')
         elif tag == 'a':
             if self.a_part['title']:
                 textilized = ' "%s (%s)":%s ' % (

src/calibre/utils/localization.py

@@ -104,8 +104,10 @@ _extra_lang_codes = {
         'en_IN' : _('English (India)'),
         'en_TH' : _('English (Thailand)'),
         'en_CY' : _('English (Cyprus)'),
+        'en_CZ' : _('English (Czechoslovakia)'),
         'en_PK' : _('English (Pakistan)'),
         'en_HR' : _('English (Croatia)'),
+        'en_ID' : _('English (Indonesia)'),
         'en_IL' : _('English (Israel)'),
         'en_SG' : _('English (Singapore)'),
         'en_YE' : _('English (Yemen)'),