Merge from trunk, fixing editing collisions in the kobo driver

Changelog.yaml

@@ -45,7 +45,7 @@
     - title: "SONY driver: Use the tz field (available in newer readers) to set timestamps correctly, when available."
 
     - title: "Shortening file paths: Handle the case of very long filenames with periods in them."
-      title: [6566]
+      tickets: [6566]
 
   new recipes:
     - title: "The TMZ and Atlanta Journal Constitution"

resources/recipes/clarin.recipe

@@ -18,7 +18,7 @@ class Clarin(BasicNewsRecipe):
     use_embedded_content  = False
     no_stylesheets        = True
     encoding              = 'utf8'
-    language              = 'es_AR'
+    language              = 'es'
     publication_type      = 'newspaper'
     INDEX                 = 'http://www.clarin.com'
     masthead_url          = 'http://www.clarin.com/static/CLAClarin/images/logo-clarin-print.jpg'

resources/recipes/europasur.recipe

@@ -0,0 +1,68 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+europasur.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Europasur(BasicNewsRecipe):
+    title                 = 'Europa Sur'
+    __author__            = 'Darko Miletic'
+    description           = 'News in Spanish'
+    publisher             = 'Joly Digital'
+    category              = 'news, politics, Spanish'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    use_embedded_content  = False
+    remove_empty_feeds    = True
+    delay                 = 2
+    no_stylesheets        = True
+    encoding              = 'cp1252'
+    language              = 'es'
+    publication_type      = 'newspaper'
+    extra_css             = """ body{font-family: Verdana,Arial,Helvetica,sans-serif}
+                                h2{font-family: Georgia,Times New Roman,Times,serif}
+                                .subtitle{font-weight:bold}
+                                .caption{font-size: small}
+                                .body{font-size: 1.1em}
+                                .info{color: #848484}
+                            """
+
+    conversion_options = {
+                          'comment'  : description
+                        , 'tags'     : category
+                        , 'publisher': publisher
+                        , 'language' : language
+                        }
+
+    keep_only_tags = [
+                       dict(attrs={'class':['titles','current']})
+                      ,dict(attrs={'id':'newsBody'})
+                     ]
+    remove_tags = [
+                      dict(name=['iframe','base','embed','object'])
+                     ,dict(name='a', attrs={'class':'zoom thickbox'})
+                     ,dict(name='div', attrs={'class':'other'})
+                  ]
+    remove_attributes = ['width','height']
+
+    feeds = [
+                (u'Portada', u'http://www.europasur.es/rss/articles.php')
+               ,(u'Deportes', u'http://www.europasur.es/rss/articles.php?sec=1224')
+               ,(u'Economia', u'http://www.europasur.es/rss/articles.php?sec=427')
+               ,(u'Espana', u'http://www.europasur.es/rss/articles.php?sec=437')
+               ,(u'Mundo', u'http://www.europasur.es/rss/articles.php?sec=428')
+               ,(u'Pasarela', u'http://www.europasur.es/rss/articles.php?sec=1958')
+               ,(u'Ocio y cultura', u'http://www.europasur.es/rss/articles.php?sec=1210')
+               ,(u'Opinion', u'http://www.europasur.es/rss/articles.php?sec=1195')
+               ,(u'Tecnologia', u'http://www.europasur.es/rss/articles.php?sec=1681')
+               ,(u'Salud', u'http://www.europasur.es/rss/articles.php?sec=2379')
+            ]
+
+    def image_url_processor(self, baseurl, url):
+        artl, sep, width = url.rpartition('&an=')
+        artid, sep, ext = artl.rpartition('.')
+        article_id = artid.rpartition('/')[2]
+        return 'http://media.grupojoly.com/cache/' + article_id + '_' + width + 'x' + width + '_' + ext + '000.' + ext

resources/recipes/fstream.recipe

@@ -0,0 +1,64 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class FIELDSTREAM(BasicNewsRecipe):
+    title      = 'Field and Stream'
+    __author__ = 'Starson17 and Tonythebookworm'
+    description = 'Hunting and Fishing and Gun Talk'
+    language = 'en'
+    no_stylesheets = True
+    publisher           = 'Starson17 and Tonythebookworm'
+    category            = 'food recipes, hunting, fishing, guns'
+    use_embedded_content= False
+    no_stylesheets      = True
+    oldest_article      = 24
+    remove_javascript   = True
+    remove_empty_feeds  = True
+    masthead_url        = 'http://www.fieldandstream.com/sites/all/themes/fs/logo.png'
+    cover_url           = 'http://www.arrowheadflyangler.com/Portals/1/Articles/FieldStream/Field%20and%20Stream%20March%20Fishing%20Edition%20Article%20Cover.jpg'
+    # recursions          = 0
+    max_articles_per_feed = 10
+    INDEX = 'http://www.fieldandstream.com'
+
+    keep_only_tags     = [dict(name='div', attrs={'class':['interior-main']})
+                          ]
+    remove_tags = [dict(name='div', attrs={'id':['comments']})]
+
+    def parse_index(self):
+        feeds = []
+        for title, url in [
+                            (u"Wild Chef", u"http://www.fieldandstream.com/blogs/wild-chef"),
+                            (u"The Gun Nut", u"http://www.fieldandstream.com/blogs/gun-nut"),
+                            (u"Whitetail 365", u"http://www.fieldandstream.com/blogs/whitetail-365"),
+                            (u"Fly Talk",  u"http://www.fieldandstream.com/blogs/flytalk"),
+                            (u"Generation Wild",  u"http://www.fieldandstream.com/blogs/generation-wild"),
+                            (u"Conservationist",  u"http://www.fieldandstream.com/blogs/conservationist"),
+                            (u"Honest Angler",  u"http://www.fieldandstream.com/blogs/honest-angler"),
+                            (u"Mans Best Friend", u"http://www.fieldandstream.com/blogs/mans-best-friend"),
+
+                             ]:
+            articles = self.make_links(url)
+            if articles:
+                feeds.append((title, articles))
+        return feeds
+
+    def make_links(self, url):
+        title = 'Temp'
+        current_articles = []
+        soup = self.index_to_soup(url)
+        print 'The soup is: ', soup
+        for item in soup.findAll('h2'):
+            print 'item is: ', item
+            link = item.find('a')
+            print 'the link is: ', link
+            if link:
+                url         = self.INDEX + link['href']
+                title       = self.tag_to_string(link)
+                print 'the title is: ', title
+                print 'the url is: ', url
+                print 'the title is: ', title
+                current_articles.append({'title': title, 'url': url, 'description':'', 'date':''}) # append all this
+        return current_articles
+
+
+
+

resources/recipes/hawaii.recipe

@@ -0,0 +1,55 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1282101454(BasicNewsRecipe):
+    title = 'West Hawaii Today'
+    language = 'en'
+    __author__ = 'Tony Stegall'
+    description = 'Westhawaiitoday.com'
+    publisher = 'West Hawaii '
+    category = 'news,Hawaii,USA'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_javascript = True
+
+    masthead_url = 'http://images.townnews.com/westhawaiitoday.com/art/whttoplogo.gif'
+
+
+
+
+
+    feeds          = [
+                       ('Local News',     'http://www.westhawaiitoday.com/?rss=local/'),
+                       ('Local Sports',   'http://www.westhawaiitoday.com/?rss=sports/local_sports/'),
+                       ('Big Fish List',  'http://www.westhawaiitoday.com/?rss=sports/big_fish_list/'),
+                       ('Local Features'  'http://www.westhawaiitoday.com/?rss=features/'),
+                       ('Obituaries',     'http://www.westhawaiitoday.com/?rss=obituaries/'),
+                       ('Letters To Editor',  'http://www.westhawaiitoday.com/?rss=opinion/letters_-_your_voice/'),
+                       ('Editorial',     'http://www.westhawaiitoday.com/?rss=opinion/editorial/'),
+                       ('Columns',       'http://www.westhawaiitoday.com/?rss=opinion/columns/'),
+                       ('Volcano Update Sunday',   'http://www.westhawaiitoday.com/?rss=volcano/')
+                    ]
+
+    def print_version(self, url):
+        split1 = url.split("//")
+        url1 = split1[1]
+        xxx = split1[2]
+        split2 = xxx.split(".")
+        artid = split2[0]
+        print 'ARTICLE ID IS: ', artid
+
+        #example of link to convert
+        #Original link: http://www.westhawaiitoday.com/articles/2010/08/27/local//local01.txt
+        #print version: http://www.westhawaiitoday.com/articles/2010/08/27/local//local01.prt
+
+        print_url = 'http://' + url1 + '//' + artid + '.prt'
+        print 'print_url is: ', print_url
+        return print_url
+
+        #test with ebook-convert hawaii.recipe output_dir --test -vv > myrecipe.txt
+
+
+
+
+
+

resources/recipes/la_jornada.recipe

@@ -1,120 +1,64 @@
-#!/usr/bin/env  python
-
 __license__   = 'GPL v3'
-__copyright__ = '2010, Rogelio Dominguez <rogelio.dominguez at gmail.com>'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.jornada.unam.mx
 '''
 
+from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 
-import re
+class LaJornada_mx(BasicNewsRecipe):
-
+    title                 = 'La Jornada (Mexico)'
-class LaJornada(BasicNewsRecipe):
+    __author__            = 'Darko Miletic'
-    title          = u'La Jornada'
+    description           = 'Noticias del diario mexicano La Jornada'
-    language       = 'es'
+    publisher             = 'DEMOS, Desarrollo de Medios, S.A. de C.V.'
-    oldest_article = 1
+    category              = 'news, Mexico'
-    __author__ = 'rogeliodh'
+    oldest_article        = 2
-    max_articles_per_feed = 100
+    max_articles_per_feed = 200
-    remove_tags    = [dict(name='div', attrs={'class':['go gui','go gui top','comment-cont',]})]
-    remove_tags_before = dict(id='article-cont')
-    remove_tags_after = dict(id='article-cont')
     no_stylesheets        = True
-    extra_css      = ' .series{ \
+    encoding              = 'utf8'
-                               border-bottom: 1px solid #626366; \
+    use_embedded_content  = False
-                               font-weight: bold; \
+    language              = 'es'
-                               } \
+    remove_empty_feeds    = True
-                       .sumario{ \
+    cover_url             = strftime("http://www.jornada.unam.mx/%Y/%m/%d/planitas/portadita.jpg")
-                               font-weight: bold; \
+    masthead_url          = 'http://www.jornada.unam.mx/v7.0/imagenes/la-jornada-trans.png'
-                               margin-top: 2em; \
+    extra_css             = """
-                               text-align: center \
+                                body{font-family: "Times New Roman",serif }
-                               } \
+                                .cabeza{font-size: xx-large; font-weight: bold }
-                       p.sumario{ \
+                                .credito-articulo{font-size: 1.3em}
-                               text-align: center \
+                            """
-                               } \
-                       .sumarios{font-weight: bold} \
-                       .cabeza{	font-size: 1.5em} \
-                       .pie-foto { \
-                               text-align: justify; \
-                               font-size: 0.8em; \
-                               text-align: justify; \
-                               } \
-                        .pie-foto .credito { \
-                               font-weight: bold; \
-                               display: block \
-                               } \
-                       .credito-autor{ \
-                               margin-top: 1.5em; \
-                               padding-left: 0.6em; \
-                               border-bottom: 1px solid #626366; \
-                               font-variant: small-caps; \
-                               font-weight: bold \
-                               } \
-                       .credito-articulo{ \
-                               margin-top: 1.5em; \
-                               padding-left: 0.6em; \
-                               border-bottom: 1px solid #626366; \
-                               font-variant: small-caps; \
-                               font-weight: bold \
-                               } \
-                       .credito-titulo{text-align: right} \
-                       .hemero { \
-                               text-align: right; \
-                               font-size: 0.9em; \
-                               margin-bottom: 8px; \
-                               } \
-                       .loc    { \
-                               font-weight: bold; \
-                               } \
-                       .carton  { \
-                               text-align: center; \
-                               } \
-                       .credit { \
-                               font-weight: bold; \
-                               } \
-                              '
 
-    preprocess_regexps = [
+    conversion_options = {
-                         # Remove capitalized initial letter on some articles (editorial)
+                          'comment'   : description
-    		       	 (re.compile(r'<div class="inicial">(.*)</div><p class="s-s">', re.DOTALL|re.IGNORECASE),
+                        , 'tags'      : category
-        		  lambda match: match.group(1)),
+                        , 'publisher' : publisher
-                         # Cartons section uses a class instead of a div to identify the main content. Change it.
+                        , 'language'  : language
-    		       	 (re.compile(r'class="carton"', re.DOTALL|re.IGNORECASE),
+                        }
-        		  lambda match: 'id="article-cont" class="carton"'),
+
-                         # Remove <link rel="alternate"> as calibre has a bug (to report)
+    keep_only_tags = [
-    		       	 (re.compile(r'<link rel="alternate".*?/>', re.DOTALL|re.IGNORECASE),
+                         dict(name='div', attrs={'class':['documentContent','cabeza','sumarios','text']})
-        		  lambda match: ''),
+                        ,dict(name='div', attrs={'id':'renderComments'})
                      ]
+    remove_tags = [dict(name='div', attrs={'class':'buttonbar'})]
 
-    INDEX          = 'http://www.jornada.unam.mx/rss/edicion.xml'
     feeds = [
-                     (u'Opinion','http://www.jornada.unam.mx/rss/opinion.xml'),
+              (u'Ultimas noticias'    , u'http://www.jornada.unam.mx/ultimas/news/RSS'     )
-                     (u'Cartones','http://www.jornada.unam.mx/rss/cartones.xml'),
+             ,(u'Opinion'             , u'http://www.jornada.unam.mx/rss/opinion.xml'      )
-                     (u'Política','http://www.jornada.unam.mx/rss/politica.xml'),
+             ,(u'Politica'            , u'http://www.jornada.unam.mx/rss/politica.xml'     )
-                     (u'Economía','http://www.jornada.unam.mx/rss/economia.xml'),
+             ,(u'Economia'            , u'http://www.jornada.unam.mx/rss/economia.xml'     )
-                     (u'Mundo','http://www.jornada.unam.mx/rss/mundo.xml'),
+             ,(u'Mundo'               , u'http://www.jornada.unam.mx/rss/mundo.xml'        )
-                     (u'Estados','http://www.jornada.unam.mx/rss/estados.xml'),
+             ,(u'Estados'             , u'http://www.jornada.unam.mx/rss/estados.xml'      )
-                     (u'Capital','http://www.jornada.unam.mx/rss/capital.xml'),
+             ,(u'Capital'             , u'http://www.jornada.unam.mx/rss/capital.xml'      )
-                     (u'Sociedad','http://www.jornada.unam.mx/rss/sociedad.xml'),
+             ,(u'Sociedad y justicia' , u'http://www.jornada.unam.mx/rss/sociedad.xml'     )
-                     (u'Ciencias','http://www.jornada.unam.mx/rss/ciencias.xml'),
+             ,(u'Ciencias'            , u'http://www.jornada.unam.mx/rss/ciencias.xml'     )
-                     (u'Cultura','http://www.jornada.unam.mx/rss/cultura.xml'),
+             ,(u'Cultura'             , u'http://www.jornada.unam.mx/rss/cultura.xml'      )
-                     (u'Gastronomia','http://www.jornada.unam.mx/rss/gastronomia.xml'),
+             ,(u'Gastronomia'         , u'http://www.jornada.unam.mx/rss/gastronomia.xml'  )
-                     (u'Espectáculos','http://www.jornada.unam.mx/rss/espectaculos.xml'),
+             ,(u'Espectaculos'        , u'http://www.jornada.unam.mx/rss/espectaculos.xml' )
-                     (u'Deportes','http://www.jornada.unam.mx/rss/deportes.xml'),
+             ,(u'Deportes'            , u'http://www.jornada.unam.mx/rss/deportes.xml'     )
             ]
 
-    def get_cover_url(self):
+    def preprocess_html(self, soup):
-        '''
+        for item in soup.findAll(style=True):
-        Cover URL is http://www.jornada.unam.mx/YYYY/MM/DD/portada.pdf
+            del item['style']
-        '''
+        return soup
-        cover_url = None
-        soup = self.index_to_soup(self.INDEX)
-        soupstone = BeautifulStoneSoup(str(soup))
-        urlbase = str(soupstone('link')[0])
-        r= re.compile(r'.*http://www.jornada.unam.mx/([0-9]{4})/([0-9]{2})/([0-9]{2})', re.DOTALL|re.IGNORECASE)
-        m = r.match(urlbase)
-        if m:
-              cover_url = 'http://www.jornada.unam.mx/' + m.groups()[0] + '/' + m.groups()[1] + '/' + m.groups()[2] + '/portada.pdf'
 
-        return cover_url

resources/recipes/mdj.recipe

@@ -0,0 +1,50 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1282101454(BasicNewsRecipe):
+    title = 'Marietta Daily Journal'
+    __author__ = 'Tony Stegall'
+    language = 'en'
+    description = 'Marietta Ga and Metro Atlanta News'
+    publisher = 'MDJ'
+    category = 'news,politics,Georgia,USA'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_javascript = True
+
+    masthead_url = 'http://assets.matchbin.com/sites/624/assets/logo.gif'
+
+
+    keep_only_tags    = [
+                        dict(name='div', attrs={'id':['print_content_container']})
+
+                        ]
+
+
+    feeds          = [
+                       ('Local News', 'http://mdjonline.com/rss/rss/Local+News?content_type=article&tags=news&page_name=rss&tag_inclusion=or&offset=0&limit=10&instance=Local+News'),
+                       ('Sports',   'http://mdjonline.com/rss/rss/Sports?content_type=article&tags=sports&page_name=rss&tag_inclusion=or&offset=0&limit=10&instance=Sports'),
+                       ('Obits',    'http://mdjonline.com/rss/rss/Obits?content_type=article&tags=obits&page_name=rss&tag_inclusion=or&offset=0&limit=20&instance=Obits'),
+                       ('Editorial & oped',   'http://mdjonline.com/rss/rss/Editorial+and+OPED?content_type=article&tags=oped+editorial&page_name=rss&tag_inclusion=or&offset=0&limit=10&instance=Editorial+and+OPED'),
+                       ('Lifestyle',   'http://mdjonline.com/rss/rss/Lifestyle?content_type=article&tags=lifestyle&page_name=rss&tag_inclusion=or&offset=0&limit=10&instance=Lifestyle'),
+                       ('Blogs',   'http://mdjonline.com/rss/rss/Lifestyle?content_type=article&tags=lifestyle&page_name=rss&tag_inclusion=or&offset=0&limit=10&instance=Lifestyle')
+                    ]
+
+    def print_version(self, url):
+        split1 = url.split("/")
+        artid = split1[4]
+
+        #example of link to convert
+        #Original link: http://mdjonline.com/bookmark/9274197
+        #print version: http://mdjonline.com/printer_friendly/9274197
+
+        print_url = 'http://mdjonline.com/printer_friendly/' + artid
+        return print_url
+
+        #test with ebook-convert nejm.recipe output_dir --test -vv > myrecipe.txt
+
+
+
+
+
+

resources/recipes/slate.recipe

@@ -102,7 +102,6 @@ class PeriodicalNameHere(BasicNewsRecipe):
 
         todays_section = soup.find(True, attrs={'class':'todaydateline'})
         self.section_dates.append(self.tag_to_string(todays_section,use_alt=False))
-        self.section_dates.append(self.tag_to_string(todays_section,use_alt=False))
 
         older_section_dates = soup.findAll(True, attrs={'class':'maindateline'})
         for older_section in older_section_dates :

resources/recipes/winnipeg_free_press.recipe

@@ -0,0 +1,30 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class WinnipegFreePress(BasicNewsRecipe):
+    title          = u'Winnipeg Free Press'
+    __author__            = 'buyo'
+    description           = 'News from Winnipeg, Manitoba, Canada'
+    oldest_article = 1
+    max_articles_per_feed = 15
+    category              = 'News, Winnipeg, Canada'
+    cover_url             = 'http://media.winnipegfreepress.com/designimages/winnipegfreepress_WFP.gif'
+    no_stylesheets        = True
+    encoding              = 'UTF-8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    language = 'en_CA'
+
+    feeds          = [(u'Breaking News', u'http://www.winnipegfreepress.com/rss?path=/breakingnews'),
+          (u'Local News',u'http://www.winnipegfreepress.com/rss?path=/local'),
+          (u'Breaking Business News',u'http://www.winnipegfreepress.com/rss?path=/business/finance'),
+          (u'Business',u'http://www.winnipegfreepress.com/rss?path=/business'),
+          (u'Editorials',u'http://www.winnipegfreepress.com/rss?path=/opinion/editorials'),
+          (u'Views from the West',u'http://www.winnipegfreepress.com/rss?path=/opinion/westview'),
+          (u'Life & Style',u'http://www.winnipegfreepress.com/rss?path=/life'),
+          (u'Food & Drink',u'http://www.winnipegfreepress.com/rss?path=/life/food')
+         ]
+
+    keep_only_tags = [
+              dict(name='div', attrs={'id':'article_header'}),
+                              dict(name='div', attrs={'class':'article'}),
+                              ]

src/calibre/constants.py

@@ -84,6 +84,9 @@ if plugins is None:
 # }}}
 
 # config_dir {{{
+
+CONFIG_DIR_MODE = 0700
+
 if os.environ.has_key('CALIBRE_CONFIG_DIRECTORY'):
     config_dir = os.path.abspath(os.environ['CALIBRE_CONFIG_DIRECTORY'])
 elif iswindows:
@@ -98,7 +101,11 @@ elif isosx:
 else:
     bdir = os.path.abspath(os.path.expanduser(os.environ.get('XDG_CONFIG_HOME', '~/.config')))
     config_dir = os.path.join(bdir, 'calibre')
-    if not os.access(config_dir, os.W_OK):
+    try:
+        os.makedirs(config_dir, mode=CONFIG_DIR_MODE)
+    except:
+        pass
+    if not os.access(config_dir, os.W_OK) or not os.access(config_dir, os.X_OK):
         print 'No write acces to', config_dir, 'using a temporary dir instead'
         import tempfile, atexit
         config_dir = tempfile.mkdtemp(prefix='calibre-config-')

src/calibre/devices/kobo/books.py

@@ -11,7 +11,7 @@ from calibre.devices.usbms.books import Book as Book_
 class Book(Book_):
 
     def __init__(self, prefix, lpath, title, authors, mime, date, ContentType,
-                 thumbnail_name, other=None):
+                 thumbnail_name, size=None, other=None):
         Book_.__init__(self, prefix, lpath)
 
         self.title = title
@@ -20,10 +20,8 @@ class Book(Book_):
         else:
             self.authors = [authors]
         self.mime = mime
-        try:
+
-            self.size = os.path.getsize(self.path)
+        self.size = size # will be set later if None
-        except OSError:
-            self.size = 0
         try:
             if ContentType == '6':
                 self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")

src/calibre/devices/kobo/driver.py

@@ -94,19 +94,19 @@ class KOBO(USBMS):
 
                 idx = bl_cache.get(lpath, None)
                 if idx is not None:
+                    bl_cache[lpath] = None
                     if ImageID is not None:
                         imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed')
                         #print "Image name Normalized: " + imagename
                         if imagename is not None:
                             bl[idx].thumbnail = ImageWrapper(imagename)
-                    bl_cache[lpath] = None
                     if ContentType != '6':
                         if self.update_metadata_item(bl[idx]):
                             # print 'update_metadata_item returned true'
                             changed = True
                     bl[idx].device_collections = playlist_map.get(lpath, [])
                 else:
-                    book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
+                    book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID)
                     # print 'Update booklist'
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
@@ -316,10 +316,10 @@ class KOBO(USBMS):
                 lpath = lpath[1:]
             #print "path: " + lpath
             #book = self.book_class(prefix, lpath, other=info)
-            lpath = self.normalize_path(prefix + lpath)
             book = Book(prefix, lpath, '', '', '', '', '', '', other=info)
             if book.size is None:
                 book.size = os.stat(self.normalize_path(path)).st_size
+            book._new_book = True # Must be before add_book
             booklists[blist].add_book(book, replace_metadata=True)
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
 
@@ -380,3 +380,19 @@ class KOBO(USBMS):
 
         return USBMS.get_file(self, path, *args, **kwargs)
 
+    @classmethod
+    def book_from_path(cls, prefix, lpath, title, authors, mime, date, ContentType, ImageID):
+        from calibre.ebooks.metadata import MetaInformation
+
+        if cls.settings().read_metadata or cls.MUST_READ_METADATA:
+            mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, lpath)))
+        else:
+            from calibre.ebooks.metadata.meta import metadata_from_filename
+            mi = metadata_from_filename(cls.normalize_path(os.path.basename(lpath)),
+                                        cls.build_template_regexp())
+        if mi is None:
+            mi = MetaInformation(os.path.splitext(os.path.basename(lpath))[0],
+                    [_('Unknown')])
+        size = os.stat(cls.normalize_path(os.path.join(prefix, lpath))).st_size
+        book =  Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=size, other=mi)
+        return book

src/calibre/ebooks/html/input.py

@@ -491,6 +491,9 @@ class HTMLInput(InputFormatPlugin):
         return (None, raw)
 
 	def preprocess_html(self, html):
+        if not hasattr(self, 'log'):
+            from calibre.utils.logging import default_log
+            self.log = default_log
 		self.log("*********  Preprocessing HTML  *********")
 		# Detect Chapters to match the xpath in the GUI
 		chapdetect = re.compile(r'(?=</?(br|p|span))(</?(br|p|span)[^>]*>)?\s*(?P<chap>(<(i|b)><(i|b)>|<(i|b)>)?(.?Chapter|Epilogue|Prologue|Book|Part|Dedication)\s*([\d\w-]+(\s\w+)?)?(</(i|b)></(i|b)>|</(i|b)>)?)(</?(p|br|span)[^>]*>)', re.IGNORECASE)

src/calibre/gui2/actions/similar_books.py

@@ -21,7 +21,8 @@ class SimilarBooksAction(InterfaceAction):
         m = QMenu(self.gui)
         for text, icon, target, shortcut in [
         (_('Books by same author'), 'user_profile.svg', 'authors', _('Alt+A')),
-        (_('Books in this series'), 'books_in_series.svg', 'series', _('Alt+S')),
+        (_('Books in this series'), 'books_in_series.svg', 'series',
+            _('Alt+Shift+S')),
         (_('Books by this publisher'), 'publisher.png', 'publisher', _('Alt+P')),
         (_('Books with the same tags'), 'tags.svg', 'tag', _('Alt+T')),]:
             ac = self.create_action(spec=(text, icon, None, shortcut),

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -105,6 +105,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             return self.simple_error('', _('No lookup name was provided'))
         if re.match('^\w*$', col) is None or not col[0].isalpha() or col.lower() != col:
             return self.simple_error('', _('The lookup name must contain only lower case letters, digits and underscores, and start with a letter'))
+        if col.endswith('_index'):
+            return self.simple_error('', _('Lookup names cannot end with _index, because these names are reserved for the index of a series column.'))
         col_heading = unicode(self.column_heading_box.text())
         col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
         if col_type == '*text':

src/calibre/gui2/library/views.py

@@ -229,6 +229,8 @@ class BooksView(QTableView): # {{{
     def cleanup_sort_history(self, sort_history):
         history = []
         for col, order in sort_history:
+            if col == 'date':
+                col = 'timestamp'
             if col in self.column_map and (not history or history[0][0] != col):
                 history.append([col, order])
         return history

src/calibre/library/database2.py

@@ -265,7 +265,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 # account for the series index column. Field_metadata knows that
                 # the series index is one larger than the series. If you change
                 # it here, be sure to change it there as well.
-                self.FIELD_MAP[str(col)+'_s_index'] = base = base+1
+                self.FIELD_MAP[str(col)+'_index'] = base = base+1
+                self.field_metadata.set_field_record_index(
+                            self.custom_column_num_map[col]['label']+'_index',
+                            base,
+                            prefer_custom=True)
 
         self.FIELD_MAP['cover'] = base+1
         self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)

src/calibre/library/field_metadata.py

@@ -36,7 +36,7 @@ class FieldMetadata(dict):
     treated as a single term. If not None, it contains a string, and the field
     is assumed to contain a list of terms separated by that string
 
-    kind == standard: is a db field.
+    kind == field: is a db field.
     kind == category: standard tag category that isn't a field. see news.
     kind == user: user-defined tag category.
     kind == search: saved-searches category.
@@ -239,7 +239,7 @@ class FieldMetadata(dict):
                              'is_multiple':None,
                              'kind':'field',
                              'name':None,
-                             'search_terms':[],
+                             'search_terms':['series_index'],
                              'is_custom':False,
                              'is_category':False}),
             ('sort',      {'table':None,
@@ -395,6 +395,18 @@ class FieldMetadata(dict):
                              'is_editable': is_editable,}
         self._add_search_terms_to_map(key, [key])
         self.custom_label_to_key_map[label] = key
+        if datatype == 'series':
+            key += '_index'
+            self._tb_cats[key] = {'table':None,        'column':None,
+                                 'datatype':'float',   'is_multiple':False,
+                                 'kind':'field',       'name':'',
+                                 'search_terms':[key], 'label':label+'_index',
+                                 'colnum':None,        'display':{},
+                                 'is_custom':False,    'is_category':False,
+                                 'link_column':None,   'category_sort':None,
+                                 'is_editable': False,}
+            self._add_search_terms_to_map(key, [key])
+            self.custom_label_to_key_map[label+'_index'] = key
 
     def remove_custom_fields(self):
         for key in self.get_custom_fields():

src/calibre/library/server/opds.py

@@ -19,6 +19,7 @@ from calibre.ebooks.metadata import fmt_sidx
 from calibre.library.comments import comments_to_html
 from calibre import guess_type
 from calibre.utils.ordered_dict import OrderedDict
+from calibre.utils.date import format_date
 
 BASE_HREFS = {
         0 : '/stanza',
@@ -130,7 +131,7 @@ def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
             link
             )
 
-def ACQUISITION_ENTRY(item, version, FM, updated):
+def ACQUISITION_ENTRY(item, version, FM, updated, CFM, CKEYS):
     title = item[FM['title']]
     if not title:
         title = _('Unknown')
@@ -153,6 +154,21 @@ def ACQUISITION_ENTRY(item, version, FM, updated):
         extra.append(_('SERIES: %s [%s]<br />')%\
                 (series,
                 fmt_sidx(float(item[FM['series_index']]))))
+    for key in CKEYS:
+        val = item[CFM[key]['rec_index']]
+        if val is not None:
+            name = CFM[key]['name']
+            datatype = CFM[key]['datatype']
+            if datatype == 'text' and CFM[key]['is_multiple']:
+                extra.append('%s: %s<br />'%(name, ', '.join(val.split('|'))))
+            elif datatype == 'series':
+                extra.append('%s: %s [%s]<br />'%(name, val,
+                              fmt_sidx(item[CFM.cc_series_index_column_for(key)])))
+            elif datatype == 'datetime':
+                extra.append('%s: %s<br />'%(name,
+                    format_date(val, CFM[key]['display'].get('date_format','dd MMM yyyy'))))
+            else:
+                extra.append('%s: %s <br />' % (CFM[key]['name'], val))
     comments = item[FM['comments']]
     if comments:
         comments = comments_to_html(comments)
@@ -260,10 +276,14 @@ class NavFeed(Feed):
 class AcquisitionFeed(NavFeed):
 
     def __init__(self, updated, id_, items, offsets, page_url, up_url, version,
-            FM):
+            FM, CFM):
         NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+        CKEYS = [key for key in sorted(CFM.get_custom_fields(),
+                 cmp=lambda x,y: cmp(CFM[x]['name'].lower(),
+                                     CFM[y]['name'].lower()))]
         for item in items:
-            self.root.append(ACQUISITION_ENTRY(item, version, FM, updated))
+            self.root.append(ACQUISITION_ENTRY(item, version, FM, updated,
+                                               CFM, CKEYS))
 
 class CategoryFeed(NavFeed):
 
@@ -360,7 +380,7 @@ class OPDSServer(object):
         cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
         cherrypy.response.headers['Content-Type'] = 'application/atom+xml;profile=opds-catalog'
         return str(AcquisitionFeed(updated, id_, items, offsets,
-            page_url, up_url, version, self.db.FIELD_MAP))
+            page_url, up_url, version, self.db.FIELD_MAP, self.db.field_metadata))
 
     def opds_search(self, query=None, version=0, offset=0):
         try:
@@ -568,7 +588,10 @@ class OPDSServer(object):
                 (_('Newest'), _('Date'), 'Onewest'),
                 (_('Title'), _('Title'), 'Otitle'),
                 ]
-        for category in categories:
+        def getter(x):
+            return category_meta[x]['name'].lower()
+        for category in sorted(categories,
+                               cmp=lambda x,y: cmp(getter(x), getter(y))):
             if len(categories[category]) == 0:
                 continue
             if category == 'formats':

src/calibre/manual/gui.rst

@@ -166,7 +166,7 @@ Search & Sort
 
 The Search & Sort section allows you to perform several powerful actions on your book collections.
 
-    * You can sort them by title, author, date, rating etc. by clicking on the column titles.
+    * You can sort them by title, author, date, rating etc. by clicking on the column titles. You can also sub-sort (i.e. sort on multiple columns). For example, if you click on the title column and then the author column, the book will be sorted by author and then all the entries for the same author will be sorted by title.
                 
     * You can search for a particular book or set of books using the search bar. More on that below.
                 
@@ -212,9 +212,10 @@ metadata.
 You can build advanced search queries easily using the :guilabel:`Advanced Search Dialog`, accessed by 
 clicking the button |sbi|.
 
-Available fields for searching are: ``tag, title, author, publisher, series, rating, cover, comments, format,
+Available fields for searching are: ``tag, title, author, publisher, series, series_index, rating, cover,
-isbn, date, pubdate, search, size`` and custom columns. If a device is plugged in, the ``ondevice`` field
+comments, format, isbn, date, pubdate, search, size`` and custom columns. If a device is plugged in, the
-becomes available. To find the search name for a custom column, hover your mouse over the column header.
+``ondevice`` field becomes available. To find the search name for a custom column, hover your mouse over the
+column header.
 
 The syntax for searching for dates is::
 
@@ -223,9 +224,8 @@ The syntax for searching for dates is::
     pubdate:=2009 Will find all books published in 2009
 
 If the date is ambiguous, the current locale is used for date comparison. For example, in an mm/dd/yyyy
-locale, 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale, it is interpreted as 2 Jan 2009.
+locale, 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale, it is interpreted as 2 Jan 2009.  Some
-
+special date strings are available. The string ``today`` translates to today's date, whatever it is. The
-Some special date strings are available. The string ``today`` translates to today's date, whatever it is. The
 strings `yesterday`` and ``thismonth`` also work. In addition, the string ``daysago`` can be used to compare
 to a date some number of days ago, for example: date:>10daysago, date:<=45daysago.
 
@@ -234,9 +234,15 @@ You can search for books that have a format of a certain size like this::
     size:>1.1M Will find books with a format larger than 1.1MB
     size:<=1K  Will find books with a format smaller than 1KB
 
-Dates and numeric fields support the operators ``=`` (equals), ``>`` (greater than), ``>=`` (greater than or
+Dates and numeric fields support the relational operators ``=`` (equals), ``>`` (greater than), ``>=``
-equal to), ``<`` (less than), ``<=`` (less than or equal to), and ``!=`` (not equal to).  Rating fields are
+(greater than or equal to), ``<`` (less than), ``<=`` (less than or equal to), and ``!=`` (not equal to).
-considered to be numeric. For example, the search ``rating:>=3`` will find all books rated 3 or higher.
+Rating fields are considered to be numeric. For example, the search ``rating:>=3`` will find all books rated 3
+or higher.
+
+Series indices are searchable. For the standard series, the search name is 'series_index'. For
+custom series columns, use the column search name followed by _index. For example, to search the indices for a
+custom series column named ``#my_series``, you would use the search name ``#my_series_index``.
+Series indices are numbers, so you can use the relational operators described above.
 
 The special field ``search`` is used for saved searches. So if you save a search with the name
 "My spouse's books" you can enter ``search:"My spouse's books"`` in the search bar to reuse the saved
@@ -310,6 +316,70 @@ Jobs
 
 The Jobs panel shows you the number of currently running jobs. Jobs are tasks that run in a separate process, they include converting ebooks and talking to your reader device. You can click on the jobs panel to access the list of jobs. Once a job has completed, by double-clicking it in the list, you can see a detailed log from that job. This is useful to debug jobs that may not have completed successfully.
 
+Keyboard Shortcuts
+---------------------
+
+Calibre has several keyboard shortcuts to save you time and mouse movement. These shortcuts are active in the book list view (when you're not editing the details of a particular book), and most of them affect the title you have selected. The |app| e-book viewer has its own shortcuts, which can be customised by clicking the Preferences button in the viewer.
+
+.. note::
+
+    Note: The Calibre keyboard shortcuts do not require a modifier key (Command, Option, Control etc.), unless specifically noted. You only need to press the letter key, e.g. E to edit.
+
+.. list-table:: Keyboard Shortcuts
+    :widths: 10 100
+    :header-rows: 1
+
+    * - Keyboard Shortcut
+      - Action
+    * - :kbd:`A` 
+      - Add Books
+    * - :kbd:`C` 
+      - Convert selected Books
+    * - :kbd:`D` 
+      - Send to device
+    * - :kbd:`Del` 
+      - Remove selected Books
+    * - :kbd:`E` 
+      - Edit metadata of selected books
+    * - :kbd:`I` 
+      - Show book details
+    * - :kbd:`M` 
+      - Merge selected records
+    * - :kbd:`O` 
+      - Open containing folder
+    * - :kbd:`S` 
+      - Save to Disk
+    * - :kbd:`V` 
+      - View
+    * - :kbd:`Alt+V/Cmd+V in OS X` 
+      - View specific format
+    * - :kbd:`Alt+Shift+J` 
+      - Toggle jobs list
+    * - :kbd:`Alt+Shift+B` 
+      - Toggle Cover Browser
+    * - :kbd:`Alt+Shift+T` 
+      - Toggle Tag Browser
+    * - :kbd:`Alt+A` 
+      - Show books by the Same author as the current book
+    * - :kbd:`Alt+T` 
+      - Show books with the same tags as current book
+    * - :kbd:`Alt+P` 
+      - Show books by the same publisher as current book
+    * - :kbd:`Alt+Shift+S` 
+      - Show books in the same series as current book
+    * - :kbd:`/, Ctrl+F` 
+      - Focus the search bar
+    * - :kbd:`Ctrl+D` 
+      - Download metadata and shortcuts
+    * - :kbd:`Ctrl+R` 
+      - Restart calibre
+    * - :kbd:`Ctrl+Q` 
+      - Quit calibre
+
+
+
+
+
 
 
 

src/calibre/utils/config.py

@@ -13,14 +13,12 @@ from optparse import OptionParser as _OptionParser
 from optparse import IndentedHelpFormatter
 from collections import defaultdict
 
-from calibre.constants import terminal_controller, config_dir, \
+from calibre.constants import terminal_controller, config_dir, CONFIG_DIR_MODE, \
                               __appname__, __version__, __author__
 from calibre.utils.lock import LockError, ExclusiveFile
 
 plugin_dir = os.path.join(config_dir, 'plugins')
 
-CONFIG_DIR_MODE = 0700
-
 def make_config_dir():
     if not os.path.exists(plugin_dir):
         os.makedirs(plugin_dir, mode=CONFIG_DIR_MODE)

src/calibre/web/feeds/templates.py

@@ -193,9 +193,11 @@ class NavBarTemplate(Template):
         navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
             style='text-align:'+align))
         if bottom:
+            if not url.startswith('file://'):
                 navbar.append(HR())
                 text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
+                p = PT(text, STRONG(__appname__), A(url, href=url),
+                        style='text-align:left; max-width: 100%; overflow: hidden;')
                 p[0].tail = ' from '
                 navbar.append(p)
                 navbar.append(BR())