Merge from trunk

Changelog.yaml

@@ -19,6 +19,57 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.59
+  date: 2012-07-06
+
+  new features:
+    - title: "Drivers for Samsung SGH-T989 and Sony Ericsson Sola"
+      tickets: [1021365]
+
+    - title: "Conversion pipeline: When removing the first image, also remove the html file the image is found in, if that file has no other content. Allows this option to be used to remove covers from EPUB files without leaving behind a blank page."
+
+    - title: "Content server: Add a navigation panel at the bottom of each page."
+      tickets: [1020225]
+
+    - title: "calibredb: Add a backup_metadata command to manually run the backup to opf from the command line"
+
+    - title: "User defined driver: Add option to swap main memory and card a."
+      tickets: [1020056]
+
+    - title: "Add new option to the series_index_auto_increment tweak, no_change, that causes calibre not to change the series_index when the series is changed"
+ 
+  bug fixes:
+    - title: "PDF Output: Resize large images so that they do not get off at the right edge of the page."
+
+    - title: "On linux ensure that WM_CLASS for the main calibre GUI is set to 'calibre-gui' to match the name of the calibre-gui.desktop file. This is apparently required by the GNOME 3 shell."
+      tickets: [1020297]
+
+    - title: "Update ICU in all builds to version 49.1"
+
+    - title: "Tag browser: Fix regression that broke drag and drop between user categories in the tag browser"
+
+    - title: "When copying to library and deleting after copy, do not place deleted files in recycle bin, as this is redundant and slow (they have already been copied into another library)"
+
+    - title: "Fix yes/no fields with value of No not showing up in the book details panel"
+
+    - title: "Catalogs: Better sorting for non English languages"
+      tickets: [930882] 
+
+    - title: "Get Books: Fix Foyles UK, Weightless books, ebooks.com and ozon.ru"
+
+    - title: "CHM Input: Fix handling of chm files that split their html into multiple sub-directories."
+      tickets: [1018792]
+
+  improved recipes:
+    - FHM UK
+    - The Age
+    - weblogs_ssl
+    - Heraldo.es
+
+  new recipes:
+    - title: CATO Institute and Heritage Foundation
+      author: _reader
+
 - version: 0.8.58
   date: 2012-06-29
 

recipes/cato.recipe

@@ -0,0 +1,75 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class CATOInstitute(BasicNewsRecipe):
+    title          		= u'The CATO Institute'
+    description      	= "The Cato Institute is a public policy research organization — a think tank — \
+dedicated to the principles of individual liberty, limited government, free markets and peace.\
+ Its scholars and analysts conduct independent, nonpartisan research on a wide range of policy issues."
+    __author__     	= '_reader'
+    __date__                	= '05 July 2012'
+    __version__            	 = '1.0'
+    cover_url        	= 'http://www.cato.org/images/logo.jpg'
+    masthead_url        	= 'http://www.cato.org/images/logo.jpg'
+    language        	= 'en'
+    oldest_article 	= 30 #days
+    max_articles_per_feed 	= 100
+    needs_subscription 	= False
+    publisher             	= 'CATO Institute'
+    category              	= 'commentary'
+    tags 		= 'commentary'
+    publication_type 	= 'blog'
+    no_stylesheets         	= True
+    use_embedded_content  = False
+    encoding        	= None
+    simultaneous_downloads = 10
+    recursions      	= 0
+    remove_javascript 	= True
+    remove_empty_feeds    	= True
+    auto_cleanup 	= True
+
+    conversion_options = {
+                           'comments'    : description,
+                           'tags'             : tags,
+                           'language'      : language,
+                           'publisher'      : publisher,
+                           'authors'        : publisher,
+                           'smarten_punctuation' : True
+                            }
+
+
+    feeds          = [
+	(u'Cato Recent Op-Eds', u'http://feeds.cato.org/CatoRecentOpeds'),
+	(u'Cato Homepage Headlines', u'http://feeds.cato.org/CatoHomepageHeadlines'),
+	(u'Cato Media Updates', u'http://feeds.cato.org/CatoMediaUpdates'),
+	(u'Cato@Liberty', u'http://feeds.cato.org/Cato-at-liberty'),
+	(u'Cato Unbound', u'http://feeds.feedburner.com/cato-unbound'),
+	(u'Education and Child Policy', u'http://www.cato.org/rss/ra.xml?name=education-child-policy'),
+	(u'Finance, Banking & Monetary Policy', u'http://www.cato.org/rss/ra.xml?name=finance-banking-monetary-policy'),
+	(u'Government and Politics', u'http://www.cato.org/rss/ra.xml?name=government-politics'),
+	(u'International Economics & Development', u'http://www.cato.org/rss/ra.xml?name=international-economics-development'),
+	(u'Political Philosophy', u'http://www.cato.org/rss/ra.xml?name=political-philosophy'),
+	(u'Social Security', u'http://www.cato.org/rss/ra.xml?name=social-security'),
+	(u'Telecom, Internet & Information Policy', u'http://www.cato.org/rss/ra.xml?name=telecom-internet-information-policy'),
+	(u'Energy and Environment', u'http://www.cato.org/rss/ra.xml?name=energy-environment'),
+	(u'Foreign Policy and National Security', u'http://www.cato.org/rss/ra.xml?name=foreign-policy-national-security'),
+	(u'Health Care', u'http://www.cato.org/rss/ra.xml?name=health-care'),
+	(u'Law and Civil Liberties', u'http://www.cato.org/rss/ra.xml?name=law-civil-liberties'),
+	(u'Regulatory Studies', u'http://www.cato.org/rss/ra.xml?name=regulatory-studies'),
+	(u'Tax and Budget Policy', u'http://www.cato.org/rss/ra.xml?name=tax-budget-policy'),
+	(u'Trade and Immigration', u'http://www.cato.org/rss/ra.xml?name=trade-immigration')
+	]
+
+
+
+
+    def print_version(self,url):
+          R_unbound = re.compile(r'(^.*cato-unbound.*)(\/\?utm_source.*$)' , re.DOTALL | re.IGNORECASE )     #CATO Unbound
+          R_pubs       = re.compile(r'(^.*\/publications\/.*$)' , re.DOTALL | re.IGNORECASE )                              #CATO Publications
+          if re.match(R_unbound, url):
+             printURL = r'\g<1>' + '/print/'
+          elif re.match(R_pubs, url):
+             printURL = url + '?print'
+          else:
+             printURL = url + '/print/'
+          return printURL

recipes/heritage_foundation.recipe

@@ -0,0 +1,71 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class HeritageFoundation(BasicNewsRecipe):
+    title		= u'The Heritage Foundation'
+    description		= 'Founded in 1973, The Heritage Foundation is a research and educational institution—a think tank—\
+whose mission is to formulate and promote conservative public policies based on the principles of free enterprise, limited government, \
+individual freedom, traditional American values, and a strong national defense.'
+    __author__ 	= '_reader'
+    __date__                	= '05 July 2012'
+    __version__            	 = '1.0'
+    oldest_article	= 30
+    max_articles_per_feed	= 100
+    publisher		= 'The Heritage Foundation'
+    category		= 'commentary'
+    tags 		= 'commentary'
+    language		= 'en'
+    publication_type	= 'blog'
+    cover_url		= 'http://www.heritage.org/static/images/logo.jpg'
+    masthead_url        	= 'http://www.heritage.org/static/images/logo.jpg'
+    encoding        	= None
+    use_embedded_content	= False
+    no_stylesheets	= True
+    remove_javascript	= True
+    recursions		= 0
+    remove_empty_feeds     = True
+    auto_cleanup = True
+
+    conversion_options = {
+                           'comments'    : description,
+                           'tags'             : tags,
+                           'language'      : language,
+                           'publisher'      : publisher,
+                           'authors'        : publisher,
+                           'smarten_punctuation' : True
+                            }
+
+
+    feeds = [
+(u'Agriculture', u'http://www.heritage.org/static/RSS/Agriculture.xml'),
+(u'Alliances', u'http://www.heritage.org/static/RSS/Alliances.xml'),
+(u'Arms Control and Non-Proliferation', u'http://www.heritage.org/static/RSS/Arms-Control-and-Non-Proliferation.xml'),
+(u'Budget and Spending', u'http://www.heritage.org/static/RSS/Budget-and-Spending.xml'),
+(u'Economic Freedom', u'http://www.heritage.org/static/RSS/Economic-Freedom.xml'),
+(u'Economy', u'http://www.heritage.org/static/RSS/Economy.xml'),
+(u'Education', u'http://www.heritage.org/static/RSS/Education.xml'),
+(u'Energy and Environment', u'http://www.heritage.org/static/RSS/Energy-and-Environment.xml'),
+(u'Family and Marriage', u'http://www.heritage.org/static/RSS/Family-And-Marriage.xml'),
+(u'Foreign Aid and Development', u'http://www.heritage.org/static/RSS/Foreign-Aid-and-Development.xml'),
+(u'Health Care', u'http://www.heritage.org/static/RSS/Health-Care.xml'),
+(u'Homeland Security', u'http://www.heritage.org/static/RSS/Homeland-Security.xml'),
+(u'Housing', u'http://www.heritage.org/static/RSS/Housing.xml'),
+(u'Immigration', u'http://www.heritage.org/static/RSS/Immigration.xml'),
+(u'International Conflicts', u'http://www.heritage.org/static/RSS/International-Conflicts.xml'),
+(u'International Law', u'http://www.heritage.org/static/RSS/International-Law.xml'),
+(u'Labor', u'http://www.heritage.org/static/RSS/Labor.xml'),
+(u'Legal Issues', u'http://www.heritage.org/static/RSS/Legal.xml'),
+(u'Missile Defense', u'http://www.heritage.org/static/RSS/Missile-Defense.xml'),
+(u'National Security and Defense', u'http://www.heritage.org/static/RSS/National-Security-and-Defense.xml'),
+(u'Political Thought', u'http://www.heritage.org/static/RSS/Political-Thought.xml'),
+(u'Public Diplomacy', u'http://www.heritage.org/static/RSS/Public-Diplomacy.xml'),
+(u'Regulation', u'http://www.heritage.org/static/RSS/Regulation.xml'),
+(u'Religion and Civil Society', u'http://www.heritage.org/static/RSS/Religion-and-Civil-Society.xml'),
+(u'Retirement Security', u'http://www.heritage.org/static/RSS/Retirement-Security.xml'),
+(u'Space Policy', u'http://www.heritage.org/static/RSS/Space-Policy.xml'),
+(u'Taxes', u'http://www.heritage.org/static/RSS/Taxes.xml'),
+(u'Terrorism', u'http://www.heritage.org/static/RSS/Terrorism.xml'),
+(u'Trade', u'http://www.heritage.org/static/RSS/Trade.xml'),
+(u'Transportation', u'http://www.heritage.org/static/RSS/Transportation.xml'),
+(u'Welfare', u'http://www.heritage.org/static/RSS/Welfare.xml'),
+(u'Worldwide Freedom and Human Rights', u'http://www.heritage.org/static/RSS/Worldwide-Freedom-and-Human-Rights.xml'),
+]
+

recipes/richmond_times_dispatch.recipe

@@ -1,59 +1,100 @@
+import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
-class AdvancedUserRecipe1335532466(BasicNewsRecipe):
+class RichmondTimesDispatch(BasicNewsRecipe):
     title               = u'Richmond Times-Dispatch'
-    description      = 'News from Richmond, Virginia, USA'
+    description         = "The Richmond Times-Dispatch is the primary daily newspaper in Richmond, \
-    __author__     = 'jde'
+                            the capital of Virginia, United States, as well as the Virginia cities of Petersburg, \
+                            Chester. Hopewell, Colonial Heights, Charlottesville, Lynchburg, Waynesboro, \
+                            and is also a default paper for rural regions of the state.  \
+                            The RTD has published in some form for more than 150 years."
+    __author__          = '_reader'
+    __date__            = '05 July 2012'
+    __version__         = '1.4'
     cover_url           = 'http://static2.dukecms.com/va_tn/timesdispatch_com/site-media/img/icons/logo252x97.png'
+    masthead_url            = 'http://static2.dukecms.com/va_tn/timesdispatch_com/site-media/img/icons/logo252x97.png'
     language            = 'en'
-    encoding        = 'utf8'
+    oldest_article      = 1.5 #days
-    oldest_article = 1 #days
+    max_articles_per_feed   = 100
-    max_articles_per_feed = 25
     needs_subscription  = False
-    remove_javascript = True
+    publisher           = 'timesdispatch.com'
-    recursions      = 0
+    category            = 'news, commentary'
-    use_embedded_content = False
+    tags                = 'news'
+    publication_type    = 'newspaper'
     no_stylesheets      = True
-    auto_cleanup = True
+    use_embedded_content= False
+    encoding            = None
+    simultaneous_downloads  = 20
+    recursions          = 0
+    remove_javascript   = True
+    remove_empty_feeds  = True
+    auto_cleanup        = False
 
-    feeds          =   [
+    conversion_options = {
+                           'comments'    : description,
+                           'tags'             : tags,
+                           'language'      : language,
+                           'publisher'      : publisher,
+                           'authors'        : publisher,
+                           'smarten_punctuation' : True
+                            }
 
-('News',
+    remove_tags_before = dict(id='hnews hentry item')
-'http://www2.timesdispatch.com/list/feed/rss/news-archive'),
-('Breaking News',
-'http://www2.timesdispatch.com/list/feed/rss/breaking-news'),
-('National News',
-'http://www2.timesdispatch.com/list/feed/rss/national-news'),
-('Local News',
-'http://www2.timesdispatch.com/list/feed/rss/local-news'),
-('Business',
-'http://www2.timesdispatch.com/list/feed/rss/business'),
-('Local Business',
-'http://www2.timesdispatch.com/list/feed/rss/local-business'),
-('Politics',
-'http://www2.timesdispatch.com/list/feed/rss/politics'),
-('Virginia Politics',
-'http://www2.timesdispatch.com/list/feed/rss/virginia-politics'),
-('Editorials',
-'http://www2.timesdispatch.com/list/feed/rss/editorial-desk'),
-('Columnists and Blogs',
-'http://www2.timesdispatch.com/list/feed/rss/news-columnists-blogs'),
-('Opinion Columnists',
-'http://www2.timesdispatch.com/list/feed/rss/opinion-editorial-columnists'),
-('Letters to the Editor',
-'http://www2.timesdispatch.com/list/feed/rss/opinion-letters'),
-('Traffic',
-'http://www2.timesdispatch.com/list/feed/rss/traffic'),
-('Sports',
-'http://www2.timesdispatch.com/list/feed/rss/sports2'),
-('Entertainment/Life',
-'http://www2.timesdispatch.com/list/feed/rss/entertainment'),
-('Movies',
-'http://www2.timesdispatch.com/list/feed/rss/movies'),
-('Music',
-'http://www2.timesdispatch.com/list/feed/rss/music'),
-('Dining & Food',
-'http://www2.timesdispatch.com/list/feed/rss/dining'),
 
+    remove_tags_after   = dict(name='hr')
+
+    remove_tags =   [
+                    dict(name='div', attrs={'id':['mg_hd', 'mg_ft', 'sr_b', 'comments_left', 'comments_right']})
+                    ,dict(name='div', attrs={'class':['bottom_social','article_bottom']})
+                    ,dict(name='table', attrs={'class':['ap-mediabox-table', 'ap-htmltable-table', 'ap-photogallery-table', 'ap-htmlfragment-table']})
                     ]
 
+
+    preprocess_regexps = [
+                   (re.compile(r'<table class="ap-story-table hnews hentry item".*?<td class="ap-story-td">', re.DOTALL|re.IGNORECASE), lambda match: ''),
+                   (re.compile(r'<p>\s*http://www2.timesdispatch.*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+                   (re.compile(r'<p>\s*<img src="http://static2.dukecms.*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+                   (re.compile(r'<p>\s*<a href="http://www2.timesdispatch.*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
+                   (re.compile(r'<hr.*?>', re.DOTALL|re.IGNORECASE), lambda match: ''),             #strip <hr /> line break
+                   (re.compile(r'<a\s*rel="item-license.*?Use</a>.', re.DOTALL|re.IGNORECASE), lambda match: ''),           #strip <hr /> line break
+                   (re.compile(r'<small>\s*Richmond Times-Dispatch.*?</small>', re.DOTALL|re.IGNORECASE), lambda match: ''),            #strip <hr /> line break
+                    ]
+
+
+    feeds          =   [
+    ('News',        'http://www2.timesdispatch.com/list/feed/rss/news-archive'),
+    ('Breaking News',   'http://www2.timesdispatch.com/list/feed/rss/breaking-news'),
+    ('National News',   'http://www2.timesdispatch.com/list/feed/rss/national-news'),
+    ('Local News',      'http://www2.timesdispatch.com/list/feed/rss/local-news'),
+    ('Business',        'http://www2.timesdispatch.com/list/feed/rss/business'),
+    ('Local Business',  'http://www2.timesdispatch.com/list/feed/rss/local-business'),
+    ('Politics',        'http://www2.timesdispatch.com/list/feed/rss/politics'),
+    ('Virginia Politics',   'http://www2.timesdispatch.com/list/feed/rss/virginia-politics'),
+    ('Sports',      'http://www2.timesdispatch.com/list/feed/rss/sports2'),
+    ('Health',      'http://www2.timesdispatch.com/feed/rss/lifestyles/health_med_fit/'),
+    ('Entertainment/Life',  'http://www2.timesdispatch.com/list/feed/rss/entertainment'),
+    ('Arts/Theatre',    'http://www2.timesdispatch.com/feed/rss/entertainment/arts_theatre/'),
+    ('Movies',      'http://www2.timesdispatch.com/list/feed/rss/movies'),
+    ('Music',       'http://www2.timesdispatch.com/list/feed/rss/music'),
+    ('Dining & Food',   'http://www2.timesdispatch.com/list/feed/rss/dining'),
+    ('Home & Garden',   'http://www2.timesdispatch.com/list/feed/rss/home-and-garden/'),
+    #inactive('Travel',     'http://www2.timesdispatch.com/feed/rss/travel/'),
+    ('Opinion',     'http://www2.timesdispatch.com/feed/rss/news/opinion/'),
+    ('Editorials',      'http://www2.timesdispatch.com/list/feed/rss/editorial-desk'),
+    ('Columnists and Blogs',    'http://www2.timesdispatch.com/list/feed/rss/news-columnists-blogs'),
+    ('Opinion Columnists',  'http://www2.timesdispatch.com/list/feed/rss/opinion-editorial-columnists'),
+    ('Letters to the Editor',   'http://www2.timesdispatch.com/list/feed/rss/opinion-letters'),
+    ('Traffic',     'http://www2.timesdispatch.com/list/feed/rss/traffic'),
+        ]
+
+    def print_version(self,url):
+        article_num = re.sub(r'(^.*)\-([0-9]{4,10})\/$', r'\g<2>', url)
+        ap_pat = re.compile('http')
+        #print '\nDEBUG>>>>>>>>: article_num: ', article_num
+        #print 'DEBUG>>>>>>>>: ap_pat.search(article_num): ', ap_pat.search(article_num)
+        if ap_pat.search(article_num):            #AP article, no print url
+            #print 'DEBUG>>>>>>>>: AP URL: ', url
+            return url
+        else:
+            printURL = 'http://www2.timesdispatch.com/member-center/share-this/print/?content=ar' + article_num
+            return printURL

setup/publish.py

@@ -40,7 +40,7 @@ class Stage2(Command):
 class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
-   sub_commands = ['upload_user_manual', 'upload_demo', 'sdist']
+   sub_commands = ['upload_user_manual', 'upload_demo', 'sdist', 'tag_release']
 
 class Stage4(Command):
 
@@ -50,7 +50,7 @@ class Stage4(Command):
 class Stage5(Command):
 
     description = 'Stage 5 of the publish process'
-    sub_commands = ['tag_release', 'upload_to_server']
+    sub_commands = ['upload_to_server']
 
     def run(self, opts):
         subprocess.check_call('rm -rf build/* dist/*', shell=True)

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 58)
+numeric_version = (0, 8, 59)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/ui.py

@@ -497,6 +497,7 @@ def initialize_plugin(plugin, path_to_zip_file):
                             %tb) + '\n'+tb)
 
 def has_external_plugins():
+    'True if there are updateable (zip file based) plugins'
     return bool(config['plugins'])
 
 def initialize_plugins(perf=False):

src/calibre/devices/android/driver.py

@@ -72,6 +72,7 @@ class ANDROID(USBMS):
 
             # Sony Ericsson
             0xfce : {
+                0xa173 : [0x216],
                 0xd12e : [0x0100],
                 0xe156 : [0x226],
                 0xe15d : [0x226],

src/calibre/gui2/store/stores/foyles_uk_plugin.py

@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import urllib2, re
+import urllib2
 from contextlib import closing
 
 from lxml import html