Added recipes for The Times Online and Tne Scotsman (thanks to Darko Miletic)

installer/windows/calibre/calibre.mpi

@@ -138,7 +138,7 @@ ProjectID
 DA98A0C6-9102-73EC-2516-B147E972D3F7
 
 ProjectVersion
-1.2.7.0
+1.2.10.1
 
 SaveOnlyToplevelDirs
 No
@@ -1100,6 +1100,9 @@ AAFE58A0-2DFB-CA20-1F6E-D3815F885996,Alias
 AIX-ppc,Active
 No
 
+AIX-ppc,BuildSeparateArchives
+No
+
 AIX-ppc,DefaultDirectoryPermission
 0755
 
@@ -1475,6 +1478,9 @@ FBA33088-C809-DD6B-D337-EADBF1CEE966,String
 FreeBSD-4-x86,Active
 No
 
+FreeBSD-4-x86,BuildSeparateArchives
+No
+
 FreeBSD-4-x86,DefaultDirectoryPermission
 0755
 
@@ -1526,6 +1532,9 @@ FreeBSD-4-x86,RootInstallDir
 FreeBSD-x86,Active
 No
 
+FreeBSD-x86,BuildSeparateArchives
+No
+
 FreeBSD-x86,DefaultDirectoryPermission
 0755
 
@@ -1577,6 +1586,9 @@ FreeBSD-x86,RootInstallDir
 HPUX-hppa,Active
 No
 
+HPUX-hppa,BuildSeparateArchives
+No
+
 HPUX-hppa,DefaultDirectoryPermission
 0755
 
@@ -1628,6 +1640,9 @@ HPUX-hppa,RootInstallDir
 Linux-x86,Active
 No
 
+Linux-x86,BuildSeparateArchives
+No
+
 Linux-x86,DefaultDirectoryPermission
 0755
 
@@ -1679,6 +1694,9 @@ Linux-x86,RootInstallDir
 Solaris-sparc,Active
 No
 
+Solaris-sparc,BuildSeparateArchives
+No
+
 Solaris-sparc,DefaultDirectoryPermission
 0755
 
@@ -1730,6 +1748,9 @@ Solaris-sparc,RootInstallDir
 TarArchive,Active
 No
 
+TarArchive,BuildSeparateArchives
+No
+
 TarArchive,CompressionLevel
 6
 
@@ -1790,9 +1811,15 @@ TarArchive,VirtualTextMap
 Windows,Active
 Yes
 
+Windows,BuildSeparateArchives
+No
+
 Windows,Executable
 <%AppName%>-<%Version%><%Ext%>
 
+Windows,FileDescription
+{<%AppName%> <%Version%> Setup}
+
 Windows,IncludeTWAPI
 Yes
 
@@ -1829,6 +1856,9 @@ Windows,WindowsIcon
 ZipArchive,Active
 No
 
+ZipArchive,BuildSeparateArchives
+No
+
 ZipArchive,CompressionLevel
 6
 
@@ -1921,7 +1951,7 @@ E611105F-DC85-9E20-4F7B-E63C54E5DF06,Message
 3EA07B17-04D8-6508-B535-96CC7173B49A,Message
 {<%AppName%> version <%Version%> created by Kovid Goyal
 
-Click Next to continue or Cancel to exit Setup.}
+Please ensure that calibre is not running, as this will cause the installation to fail. Click Next to continue or Cancel to exit Setup.}
 
 442920D9-8A51-9476-14E4-787D5C230E84,Message
 <%UninstallCompleteText%>

src/calibre/gui2/dialogs/scheduler.py

@@ -286,7 +286,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.username.blockSignals(False)
             self.password.blockSignals(False)
         d = datetime.utcnow() - recipe.last_downloaded
-        ld = '%.1f'%(d.days + d.seconds/(24*3600))
+        ld = '%.2f'%(d.days + d.seconds/(24.*3600))
         if d < timedelta(days=366):
             self.last_downloaded.setText(_('Last downloaded: %s days ago')%ld)
         else:

src/calibre/gui2/main.py

@@ -155,6 +155,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.sync_menu = sm # Needed
         md = QMenu()
         md.addAction(_('Edit metadata individually'))
+        md.addSeparator()
         md.addAction(_('Edit metadata in bulk'))
         self.metadata_menu = md
         self.add_menu = QMenu()

src/calibre/web/feeds/recipes/__init__.py

@@ -14,7 +14,8 @@ recipe_modules = [
            'politika', 'moscow_times', 'latimes', 'japan_times', 'san_fran_chronicle',
            'demorgen_be', 'de_standaard', 'ap', 'barrons', 'chr_mon', 'cnn', 'faznet',
            'jpost', 'jutarnji', 'nasa', 'reuters', 'spiegelde', 'wash_post', 'zeitde',
-           'blic', 'novosti', 'danas', 'vreme',
+           'blic', 'novosti', 'danas', 'vreme', 'times_online', 'the_scotsman',
+           'nytimes_sub'
           ]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/b92.py

@@ -16,28 +16,28 @@ class B92(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
-    #Locale setting to get appropriate date/month values in Serbian if possible
+#    #Locale setting to get appropriate date/month values in Serbian if possible
-    try:
+#    try:
-      #Windows seting for locale
+#      #Windows seting for locale
-      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
+#      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
-    except locale.Error:
+#    except locale.Error:
-      #Linux setting for locale -- choose one appropriate for your distribution
+#      #Linux setting for locale -- choose one appropriate for your distribution
-      try:
+#      try:
-        locale.setlocale(locale.LC_TIME,'sr_YU')
+#        locale.setlocale(locale.LC_TIME,'sr_YU')
-      except locale.Error:
+#      except locale.Error:
-        try:
+#        try:
-          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
+#          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
-        except locale.Error:
+#        except locale.Error:
-          try:
+#          try:
-            locale.setlocale(locale.LC_TIME,'sr@Latn')
+#            locale.setlocale(locale.LC_TIME,'sr@Latn')
-          except locale.Error:
+#          except locale.Error:
-            try:
+#            try:
-              locale.setlocale(locale.LC_TIME,'sr_Latn')
+#              locale.setlocale(locale.LC_TIME,'sr_Latn')
-            except locale.Error:
+#            except locale.Error:
-              try:
+#              try:
-                locale.setlocale(locale.LC_TIME,'sr_RS')
+#                locale.setlocale(locale.LC_TIME,'sr_RS')
-              except locale.Error:                  
+#              except locale.Error:                  
-                locale.setlocale(locale.LC_TIME,'C')
+#                locale.setlocale(locale.LC_TIME,'C')
 
     remove_tags_after  = dict(name='div', attrs={'class':'gas'})
     remove_tags = [

src/calibre/web/feeds/recipes/blic.py

@@ -17,28 +17,28 @@ class Blic(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%A, %d %B, %Y]' 
-    #Locale setting to get appropriate date/month values in Serbian if possible
+#    #Locale setting to get appropriate date/month values in Serbian if possible
-    try:
+#    try:
-      #Windows seting for locale
+#      #Windows seting for locale
-      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
+#      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
-    except locale.Error:
+#    except locale.Error:
-      #Linux setting for locale -- choose one appropriate for your distribution
+#      #Linux setting for locale -- choose one appropriate for your distribution
-      try:
+#      try:
-        locale.setlocale(locale.LC_TIME,'sr_YU')
+#        locale.setlocale(locale.LC_TIME,'sr_YU')
-      except locale.Error:
+#      except locale.Error:
-        try:
+#        try:
-          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
+#          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
-        except locale.Error:
+#        except locale.Error:
-          try:
+#          try:
-            locale.setlocale(locale.LC_TIME,'sr@Latn')
+#            locale.setlocale(locale.LC_TIME,'sr@Latn')
-          except locale.Error:
+#          except locale.Error:
-            try:
+#            try:
-              locale.setlocale(locale.LC_TIME,'sr_Latn')
+#              locale.setlocale(locale.LC_TIME,'sr_Latn')
-            except locale.Error:
+#            except locale.Error:
-              try:
+#              try:
-                locale.setlocale(locale.LC_TIME,'sr_RS')
+#                locale.setlocale(locale.LC_TIME,'sr_RS')
-              except locale.Error:                  
+#              except locale.Error:                  
-                locale.setlocale(locale.LC_TIME,'C')
+#                locale.setlocale(locale.LC_TIME,'C')
 
     keep_only_tags = [ dict(name='div', attrs={'class':'single_news'}) ]
 

src/calibre/web/feeds/recipes/danas.py

@@ -17,28 +17,28 @@ class Danas(BasicNewsRecipe):
     no_stylesheets        = False
     use_embedded_content  = False
     timefmt               = ' [%A, %d %B, %Y]' 
-    #Locale setting to get appropriate date/month values in Serbian if possible
+#    #Locale setting to get appropriate date/month values in Serbian if possible
-    try:
+#    try:
-      #Windows seting for locale
+#      #Windows seting for locale
-      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
+#      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
-    except locale.Error:
+#    except locale.Error:
-      #Linux setting for locale -- choose one appropriate for your distribution
+#      #Linux setting for locale -- choose one appropriate for your distribution
-      try:
+#      try:
-        locale.setlocale(locale.LC_TIME,'sr_YU')
+#        locale.setlocale(locale.LC_TIME,'sr_YU')
-      except locale.Error:
+#      except locale.Error:
-        try:
+#        try:
-          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
+#          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
-        except locale.Error:
+#        except locale.Error:
-          try:
+#          try:
-            locale.setlocale(locale.LC_TIME,'sr@Latn')
+#            locale.setlocale(locale.LC_TIME,'sr@Latn')
-          except locale.Error:
+#          except locale.Error:
-            try:
+#            try:
-              locale.setlocale(locale.LC_TIME,'sr_Latn')
+#              locale.setlocale(locale.LC_TIME,'sr_Latn')
-            except locale.Error:
+#            except locale.Error:
-              try:
+#              try:
-                locale.setlocale(locale.LC_TIME,'sr_RS')
+#                locale.setlocale(locale.LC_TIME,'sr_RS')
-              except locale.Error:                  
+#              except locale.Error:                  
-                locale.setlocale(locale.LC_TIME,'C')
+#                locale.setlocale(locale.LC_TIME,'C')
 
     keep_only_tags     = [ dict(name='div', attrs={'id':'left'}) ]
     remove_tags_after  = dict(name='div', attrs={'id':'comments'})

src/calibre/web/feeds/recipes/new_scientist.py

@@ -5,13 +5,20 @@ __copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newscientist.com
 '''
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>, AprilHare'
+'''
+newscientist.com
+'''
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
-    title                 = u'New Scientist'
+    title                 = u'New Scientist - Online News'
     __author__            = 'Darko Miletic'
-    description           = 'Science news'
+    description           = 'News from Science'
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
@@ -33,4 +40,8 @@ class NewScientist(BasicNewsRecipe):
                        ,(u'Health'           , u'http://www.newscientist.com/feed/view?id=2&type=channel' )
                        ,(u'Life'             , u'http://www.newscientist.com/feed/view?id=3&type=channel' )
                        ,(u'Space'            , u'http://www.newscientist.com/feed/view?id=6&type=channel' )
+                       ,(u'Physics and Mathematics'            , u'http://www.newscientist.com/feed/view?id=4&type=channel' )
+                       ,(u'Environment'            , u'http://www.newscientist.com/feed/view?id=1&type=channel' )
+                       ,(u'Science in Society'            , u'http://www.newscientist.com/feed/view?id=5&type=channel' )
+                       ,(u'Tech'            , u'http://www.newscientist.com/feed/view?id=7&type=channel' )
                      ]

src/calibre/web/feeds/recipes/novosti.py

@@ -17,28 +17,28 @@ class Novosti(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     timefmt               = ' [%A, %d %B, %Y]' 
-    #Locale setting to get appropriate date/month values in Serbian if possible
+#    #Locale setting to get appropriate date/month values in Serbian if possible
-    try:
+#    try:
-      #Windows seting for locale
+#      #Windows seting for locale
-      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
+#      locale.setlocale(locale.LC_TIME,'Serbian (Latin)')
-    except locale.Error:
+#    except locale.Error:
-      #Linux setting for locale -- choose one appropriate for your distribution
+#      #Linux setting for locale -- choose one appropriate for your distribution
-      try:
+#      try:
-        locale.setlocale(locale.LC_TIME,'sr_YU')
+#        locale.setlocale(locale.LC_TIME,'sr_YU')
-      except locale.Error:
+#      except locale.Error:
-        try:
+#        try:
-          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
+#          locale.setlocale(locale.LC_TIME,'sr_CS@Latn')
-        except locale.Error:
+#        except locale.Error:
-          try:
+#          try:
-            locale.setlocale(locale.LC_TIME,'sr@Latn')
+#            locale.setlocale(locale.LC_TIME,'sr@Latn')
-          except locale.Error:
+#          except locale.Error:
-            try:
+#            try:
-              locale.setlocale(locale.LC_TIME,'sr_Latn')
+#              locale.setlocale(locale.LC_TIME,'sr_Latn')
-            except locale.Error:
+#            except locale.Error:
-              try:
+#              try:
-                locale.setlocale(locale.LC_TIME,'sr_RS')
+#                locale.setlocale(locale.LC_TIME,'sr_RS')
-              except locale.Error:                  
+#              except locale.Error:                  
-                locale.setlocale(locale.LC_TIME,'C')
+#                locale.setlocale(locale.LC_TIME,'C')
 
 
     keep_only_tags     = [ dict(name='div', attrs={'class':'jednaVest'}) ]

src/calibre/web/feeds/recipes/nytimes.py

@@ -12,9 +12,9 @@ from lxml import html
 
 class NYTimesMobile(BasicNewsRecipe):
     
-    title       = 'The New York Times'
+    title       = 'The New York Times (mobile)'
     __author__  = 'Kovid Goyal'
-    description = 'Daily news from the New York Times'
+    description = 'Daily news from the New York Times (mobile version)'
     timefmt     = ' [%a, %d %b, %Y]'
     multithreaded_fetch = True
     max_articles_per_feed  = 15

src/calibre/web/feeds/recipes/nytimes_sub.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+'''
+nytimes.com
+'''
+import string, re
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+
+class NYTimes(BasicNewsRecipe):
+    
+    title       = 'The New York Times (subscription)'
+    __author__  = 'Kovid Goyal'
+    description = 'Daily news from the New York Times (subscription version)'
+    timefmt = ' [%a, %d %b, %Y]'
+    needs_subscription = True
+    remove_tags_before = dict(id='article')
+    remove_tags_after  = dict(id='article')
+    remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix']}), 
+                   dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']), 
+                   dict(name=['script', 'noscript', 'style'])]
+    encoding = 'cp1252'
+    no_stylesheets = True
+    extra_css = 'h1 {font: sans-serif large;}\n.byline {font:monospace;}'
+    
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('http://www.nytimes.com/auth/login')
+            br.select_form(name='login')
+            br['USERID']   = self.username
+            br['PASSWORD'] = self.password
+            br.submit()
+        return br
+    
+    def parse_index(self):
+        soup = self.index_to_soup('http://www.nytimes.com/pages/todayspaper/index.html')
+        
+        def feed_title(div):
+            return ''.join(div.findAll(text=True, recursive=False)).strip()
+        
+        articles = {}
+        key = None
+        ans = []
+        for div in soup.findAll(True, 
+            attrs={'class':['section-headline', 'story', 'story headline']}):
+            
+            if div['class'] == 'section-headline':
+                key = string.capwords(feed_title(div))
+                articles[key] = []
+                ans.append(key)
+            
+            elif div['class'] in ['story', 'story headline']:
+                a = div.find('a', href=True)
+                if not a:
+                    continue
+                url = re.sub(r'\?.*', '', a['href'])
+                url += '?pagewanted=all'
+                title = self.tag_to_string(a, use_alt=True).strip()
+                description = ''
+                pubdate = strftime('%a, %d %b')
+                summary = div.find(True, attrs={'class':'summary'})
+                if summary:
+                    description = self.tag_to_string(summary, use_alt=False)
+                
+                feed = key if key is not None else 'Uncategorized'
+                if not articles.has_key(feed):
+                    articles[feed] = []
+                if not 'podcasts' in url:
+                    articles[feed].append(
+                                  dict(title=title, url=url, date=pubdate, 
+                                       description=description,
+                                       content=''))
+        ans = self.sort_index_by(ans, {'The Front Page':-1, 'Dining In, Dining Out':1, 'Obituaries':2})
+        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
+        return ans
+    
+    def preprocess_html(self, soup):
+        refresh = soup.find('meta', {'http-equiv':'refresh'})
+        if refresh is None:
+            return soup
+        content = refresh.get('content').partition('=')[2]
+        raw = self.browser.open('http://www.nytimes.com'+content).read()
+        return BeautifulSoup(raw.decode('cp1252', 'replace'))

src/calibre/web/feeds/recipes/the_scotsman.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+'''
+thescotsman.scotsman.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheScotsman(BasicNewsRecipe):
+    title                 = u'The Scotsman'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Scotland'    
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    simultaneous_downloads = 1
+
+    keep_only_tags = [dict(name='div', attrs={'id':'viewarticle'})]
+    remove_tags = [
+                     dict(name='div'  , attrs={'class':'viewarticlepanel' })
+                  ]
+
+    feeds          = [
+                        (u'Latest National News', u'http://thescotsman.scotsman.com/getFeed.aspx?Format=rss&sectionid=4068'),
+                        ('UK', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=7071&format=rss'),
+                        ('Scotland', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=7042&format=rss'),
+                        ('International', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=7000&format=rss'),
+                        ('Politics', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=6990&format=rss'),
+                        ('Entertainment', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=7010&format=rss'),
+                        ('Features', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=6996&format=rss'),
+                        ('Opinion', 'http://thescotsman.scotsman.com/getfeed.aspx?sectionid=7074&format=rss'),
+                     ]

src/calibre/web/feeds/recipes/times_online.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+'''
+timesonline.co.uk
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TimesOnline(BasicNewsRecipe):
+    title                 = u'The Times Online'
+    __author__            = 'Darko Miletic'
+    description           = 'UK news'    
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    simultaneous_downloads = 1
+
+    remove_tags_after  = dict(name='div', attrs={'class':'bg-666'})
+    remove_tags = [
+                     dict(name='div'  , attrs={'class':'hide-from-print padding-bottom-7' })
+                  ]
+
+    feeds          = [
+                        (u'Top stories from Times Online', u'http://www.timesonline.co.uk/tol/feeds/rss/topstories.xml'     ),
+                        ('Latest Business News', 'http://www.timesonline.co.uk/tol/feeds/rss/business.xml'),
+                        ('Economics', 'http://www.timesonline.co.uk/tol/feeds/rss/economics.xml'),
+                        ('World News', 'http://www.timesonline.co.uk/tol/feeds/rss/worldnews.xml'),
+                        ('UK News', 'http://www.timesonline.co.uk/tol/feeds/rss/uknews.xml'),
+                        ('Travel News', 'http://www.timesonline.co.uk/tol/feeds/rss/travel.xml'),
+                        ('Sports News', 'http://www.timesonline.co.uk/tol/feeds/rss/sport.xml'),
+                        ('Film News', 'http://www.timesonline.co.uk/tol/feeds/rss/film.xml'),
+                        ('Tech news', 'http://www.timesonline.co.uk/tol/feeds/rss/tech.xml'),
+                     ]
+
+    def print_version(self, url):
+        main = url.partition('#')[0]
+        return main + '?print=yes'