Merge upstream changes

Makefile

@@ -1,30 +0,0 @@
-PYTHON = python
-
-all : gui2 translations resources 
-
-clean : 
-	cd src/calibre/gui2 && ${PYTHON} make.py clean
-
-gui2 :
-	 cd src/calibre/gui2 && ${PYTHON} make.py
-
-test : gui2
-	cd src/calibre/gui2 && ${PYTHON} make.py test
-
-translations :
-	cd src/calibre/translations && ${PYTHON} __init__.py
-
-resources:	
-	${PYTHON} resources.py
-    
-manual:
-	make -C src/calibre/manual clean html
-
-pot :
-	cd src/calibre/translations && ${PYTHON} __init__.py pot
-
-egg : 
-	${PYTHON} setup.py bdist_egg --exclude-source-files
-
-linux_binary:
-	${PYTHON} -c "import upload; upload._build_linux()"

resources.py

@@ -13,7 +13,7 @@ RESOURCES = dict(
     opf_template    = '%p/ebooks/metadata/opf.xml',
     ncx_template    = '%p/ebooks/metadata/ncx.xml',
     fb2_xsl         = '%p/ebooks/lrf/fb2/fb2.xsl',
-    metadata_sqlite = '%p/library/metadata_sqlite.sql', 
+    metadata_sqlite = '%p/library/metadata_sqlite.sql',
                  )
 
 def main(args=sys.argv):
@@ -37,8 +37,10 @@ def main(args=sys.argv):
     if not translations_found:
         print 'WARNING: Could not find Qt transations'
     
-    open('src'+os.sep+__appname__+os.sep+'/resources.py', 'wb').write(data) 
+    dest = os.path.abspath(os.path.join('src', __appname__, 'resources.py'))
+    print 'Writing resources to', dest 
+    open(dest, 'wb').write(data) 
     return 0
 
 if __name__ == '__main__':
-    sys.exit(main())
+    sys.exit(main())

setup.py

@@ -47,9 +47,68 @@ main_functions = {
 
 if __name__ == '__main__':
     from setuptools import setup, find_packages, Extension
+    from distutils.command.build import build as _build
+    from distutils.core import Command
     from pyqtdistutils import PyQtExtension, build_ext
     import subprocess, glob
     
+    class pot(Command):
+        user_options = []
+        def initialize_options(self): pass
+        def finalize_options(self): pass
+        
+        def run(self):
+            from calibre.translations import create_pot
+            create_pot()
+            
+    def build_manual():
+        cwd = os.path.abspath(os.getcwd())
+        os.chdir(os.path.join('src', 'calibre', 'manual'))
+        try:
+            for d in ('.build', 'cli'):
+                if os.path.exists(d):
+                    shutil.rmtree(d)
+                os.makedirs(d)
+            if not os.path.exists('.build'+os.sep+'html'):
+                os.makedirs('.build'+os.sep+'html')
+            subprocess.check_call(['sphinx-build', '-b', 'custom', '-d', 
+                                   '.build/doctrees', '.', '.build/html'])
+        finally:
+            os.chdir(cwd)
+    
+    class manual(Command):
+        user_options = []
+        def initialize_options(self): pass
+        def finalize_options(self): pass
+        
+        def run(self):
+            build_manual()
+            
+    
+    class build(_build):
+        
+        def run(self):
+            # Build resources
+            resources = __import__('resources')
+            resources.main([sys.executable, 'resources.py'])
+            from calibre.translations import main as translations
+            cwd = os.path.abspath(os.getcwd())
+            # Build translations
+            try:
+                os.chdir(os.path.join('src', 'calibre', 'translations'))
+                translations([sys.executable])
+            finally:
+                os.chdir(cwd)
+            # Build GUI
+            from calibre.gui2.make import main as gui2
+            try:
+                os.chdir(os.path.join('src', 'calibre', 'gui2'))
+                print 'Compiling GUI resources...'
+                gui2([sys.executable])
+            finally:
+                os.chdir(cwd)
+            _build.run(self)
+    
     entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install')
     ext_modules = [
                    Extension('calibre.plugins.lzx',
@@ -125,7 +184,8 @@ if __name__ == '__main__':
             'Topic :: Software Development :: Libraries :: Python Modules',
             'Topic :: System :: Hardware :: Hardware Drivers'
             ],
-          cmdclass = {'build_ext': build_ext},
+          cmdclass = {'build_ext': build_ext, 'build' : build, 'pot' : pot,
+                      'manual' : manual},
          )
 
     if 'develop' in ' '.join(sys.argv) and islinux:

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -182,7 +182,8 @@ class PageProcessor(list):
             MagickQuantizeImage(wand, self.opts.colors, RGBColorspace, 0, 1, 0)
             dest = '%d_%d.png'%(self.num, i)
             dest = os.path.join(self.dest, dest)
-            MagickWriteImage(wand, dest)
+            MagickWriteImage(wand, dest+'8')
+            os.rename(dest+'8', dest)
             self.append(dest)
         
             DestroyPixelWand(pw)

src/calibre/ebooks/lrf/lrs/convert_from.py

@@ -110,6 +110,12 @@ class LrsParser(object):
                     tb.append(self.process_paragraph(item))
                 elif item.name == 'cr':
                     tb.append(CR())
+                elif item.name == 'charbutton': # BookDesigner does this
+                    p = Paragraph()
+                    tb.append(p)
+                    elem = self.text_tag_to_element(item)
+                    self.process_text_element(item, elem)
+                    p.append(elem)
             
     def fourth_pass(self):
         for tag in self.soup.findAll('page'):
@@ -130,7 +136,6 @@ class LrsParser(object):
         
         for tag in self.soup.findAll('textblock'):
             self.process_text_block(tag)
-        
         toc = self.soup.find('toc')
         if toc:
             for tag in toc.findAll('toclabel'):

src/calibre/ebooks/metadata/__init__.py

@@ -201,7 +201,7 @@ class MetaInformation(object):
         #: mi.cover_data = (ext, data)
         self.cover_data   = mi.cover_data if (mi and hasattr(mi, 'cover_data')) else (None, None)
         self.application_id    = mi.application_id  if (mi and hasattr(mi, 'application_id')) else None
-        self.manifest = getattr(mi, 'manifest', None) 
+        self.manifest = getattr(mi, 'manifest', None)
         self.toc      = getattr(mi, 'toc', None)
         self.spine    = getattr(mi, 'spine', None)
         self.guide    = getattr(mi, 'guide', None)

src/calibre/ebooks/metadata/fb2.py

@@ -41,6 +41,10 @@ def get_metadata(stream):
     mi.author_sort = lastname+'; '+firstname
     if series:
         mi.series = series.get('name', None)
+        try:
+            mi.series_index = int(series.get('number', None))
+        except (TypeError, ValueError):
+            pass
     if cdata:
         mi.cover_data = cdata
     return mi

src/calibre/ebooks/metadata/meta.py

@@ -12,7 +12,7 @@ from calibre.ebooks.metadata.lit  import get_metadata as lit_metadata
 from calibre.ebooks.metadata.epub import get_metadata as epub_metadata
 from calibre.ebooks.metadata.html import get_metadata as html_metadata
 from calibre.ebooks.mobi.reader   import get_metadata as mobi_metadata
-from calibre.ebooks.metadata.opf  import OPFReader 
+from calibre.ebooks.metadata.opf  import OPFReader
 from calibre.ebooks.metadata.rtf  import set_metadata as set_rtf_metadata
 from calibre.ebooks.lrf.meta      import set_metadata as set_lrf_metadata
 from calibre.ebooks.metadata.epub import set_metadata as set_epub_metadata
@@ -29,14 +29,14 @@ _METADATA_PRIORITIES = [
 # Higher values should be used to update metadata from lower values
 METADATA_PRIORITIES = collections.defaultdict(lambda:0)
 for i, ext in enumerate(_METADATA_PRIORITIES):
-    METADATA_PRIORITIES[ext] = i 
+    METADATA_PRIORITIES[ext] = i
 
 def path_to_ext(path):
     return os.path.splitext(path)[1][1:].lower()
 
 def metadata_from_formats(formats):
     mi = MetaInformation(None, None)
-    formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],  
+    formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
                                      METADATA_PRIORITIES[path_to_ext(y)]))
     for path in formats:
         ext = path_to_ext(path)

src/calibre/gui2/main.py

@@ -225,7 +225,7 @@ class Main(MainWindow, Ui_MainWindow):
             pd.show()
             db.migrate_old(self.olddb, pd)
             self.olddb = None
-            Settings().set('library path', self.library_path)
+            prefs['library_path'] = self.library_path
         self.library_view.sortByColumn(3, Qt.DescendingOrder)
         if not self.library_view.restore_column_widths():
             self.library_view.resizeColumnsToContents()
@@ -1106,7 +1106,7 @@ class Main(MainWindow, Ui_MainWindow):
                                              _('<p>An invalid database already exists at %s, delete it before trying to move the existing database.<br>Error: %s')%(newloc, str(err)))
                             d.exec_()
                     self.library_path = self.library_view.model().db.library_path
-                    prefs['library path'] =  self.library_path
+                    prefs['library_path'] =  self.library_path
                 except Exception, err:
                     traceback.print_exc()
                     d = error_dialog(self, _('Could not move database'), unicode(err))
@@ -1204,31 +1204,33 @@ class Main(MainWindow, Ui_MainWindow):
 
 
     def initialize_database(self):
-        self.library_path = prefs['library path']
+        self.library_path = prefs['library_path']
         self.olddb = None
         if self.library_path is None: # Need to migrate to new database layout
             self.database_path = prefs['database_path']
             if not os.access(os.path.dirname(self.database_path), os.W_OK):
-                error_dialog(self, _('Database does not exist'), _('The directory in which the database should be: %s no longer exists. Please choose a new database location.')%self.database_path).exec_()
+                error_dialog(self, _('Database does not exist'), 
-                self.database_path = choose_dir(self, 'database path dialog', 'Choose new location for database')
+                             _('The directory in which the database should be: %s no longer exists. Please choose a new database location.')%self.database_path).exec_()
+                self.database_path = choose_dir(self, 'database path dialog', 
+                                                _('Choose new location for database'))
                 if not self.database_path:
                     self.database_path = os.path.expanduser('~').decode(sys.getfilesystemencoding())
                 if not os.path.exists(self.database_path):
                     os.makedirs(self.database_path)
                 self.database_path = os.path.join(self.database_path, 'library1.db')
-                settings.set('database path', self.database_path)
+                prefs['database_path'] = self.database_path
             home = os.path.dirname(self.database_path)
             if not os.path.exists(home):
                 home = os.getcwd()
             from PyQt4.QtGui import QFileDialog
-            dir = qstring_to_unicode(QFileDialog.getExistingDirectory(self, _('Choose a location for your ebook library.'), home))
+            dir = unicode(QFileDialog.getExistingDirectory(self, 
+                            _('Choose a location for your ebook library.'), home))
             if not dir:
                 dir = os.path.dirname(self.database_path)
             self.library_path = os.path.abspath(dir)
             self.olddb = LibraryDatabase(self.database_path)
 
 
-
     def read_settings(self):
         self.initialize_database()
         geometry = config['main_window_geometry']

src/calibre/manual/Makefile

@@ -21,7 +21,7 @@ help:
 	@echo "  linkcheck to check all external links for integrity"
 
 clean:
-	-rm -rf .build/* cli
+	rm -rf .build/* cli
 
 html:
 	mkdir -p .build/html .build/doctrees

src/calibre/utils/config.py

@@ -451,6 +451,8 @@ def _prefs():
               help=_('Access key for isbndb.com'))
     c.add_opt('network_timeout', default=5,
               help=_('Default timeout for network operations (seconds)'))
+    c.add_opt('library_path', default=None,
+              help=_('Path to directory in which your library of books is stored'))
     
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c

src/calibre/utils/genshi/output.py

@@ -648,7 +648,7 @@ class NamespaceFlattener(object):
             elif kind is END_NS:
                 if data is '':
                     default.pop()
-                if data in prefixes:
+                if data in prefixes and prefixes.get(data):
                     uris = prefixes.get(data)
                     uri = uris.pop()
                     if not uris:

src/calibre/web/feeds/news.py

@@ -98,6 +98,10 @@ class BasicNewsRecipe(object, LoggingInterface):
     #: embedded content.
     use_embedded_content   = None
     
+    #: Set to True and implement :method:`get_obfuscated_article` to handle
+    #: websites that try to make it difficult to scrape content.
+    articles_are_obfuscated = False
+    
     #: Specify any extra :term:`CSS` that should be addded to downloaded :term:`HTML` files
     #: It will be inserted into `<style>` tags, just before the closing
     #: `</head>` tag thereby overrinding all :term:`CSS` except that which is
@@ -360,12 +364,25 @@ class BasicNewsRecipe(object, LoggingInterface):
         '''
         raise NotImplementedError
     
+    def get_obfuscated_article(self, url, logger):
+        '''
+        If you set :member:`articles_are_obfuscated` this method is called with
+        every article URL. It should return the path to a file on the filesystem
+        that contains the article HTML. That file is processed by the recursive
+        HTML fetching engine, so it can contain links to pages/images on the web.
+        
+        This method is typically useful for sites that try to make it difficult to
+        access article content automatically. See for example the 
+        :module:`calibre.web.recipes.iht` recipe.
+        '''
+        raise NotImplementedError
+    
     def __init__(self, options, parser, progress_reporter):
         '''
         Initialize the recipe.
-        @param options: Parsed commandline options 
+        :param options: Parsed commandline options 
-        @param parser:  Command line option parser. Used to intelligently merge options.
+        :param parser:  Command line option parser. Used to intelligently merge options.
-        @param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
+        :param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
         '''
         LoggingInterface.__init__(self, logging.getLogger('feeds2disk'))
         if not isinstance(self.title, unicode):
@@ -564,7 +581,11 @@ class BasicNewsRecipe(object, LoggingInterface):
     
     def fetch_article(self, url, dir, logger, f, a, num_of_feeds):
         return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
-        
+    
+    def fetch_obfuscated_article(self, url, dir, logger, f, a, num_of_feeds):
+        path = os.path.abspath(self.get_obfuscated_article(url, logger))
+        url = ('file:'+path) if iswindows else ('file://'+path)
+        return self._fetch_article(url, dir, logger, f, a, num_of_feeds)
     
     def fetch_embedded_article(self, article, dir, logger, f, a, num_of_feeds):
         pt = PersistentTemporaryFile('_feeds2disk.html')
@@ -620,7 +641,8 @@ class BasicNewsRecipe(object, LoggingInterface):
                     continue
                     
                 func, arg = (self.fetch_embedded_article, article) if self.use_embedded_content else \
-                            (self.fetch_article, url)
+                            ((self.fetch_obfuscated_article if self.articles_are_obfuscated \
+                              else self.fetch_article), url)
                 req = WorkRequest(func, (arg, art_dir, logger, f, a, len(feed)), 
                                       {}, (f, a), self.article_downloaded, 
                                       self.error_in_article_download)

src/calibre/web/feeds/recipes/__init__.py

@@ -8,7 +8,7 @@ recipes = [
            'newsweek', 'atlantic', 'economist', 'portfolio', 
            'nytimes', 'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
            'wired', 'globe_and_mail', 'smh', 'espn', 'business_week',
-           'ars_technica', 'upi', 'new_yorker',
+           'ars_technica', 'upi', 'new_yorker', 'irish_times', 'iht',
           ]
 
 import re, imp, inspect, time

src/calibre/web/feeds/recipes/iht.py

@@ -0,0 +1,51 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Derry FitzGerald'
+'''
+iht.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ptempfile import PersistentTemporaryFile
+
+
+class InternationalHeraldTribune(BasicNewsRecipe):
+    title          = u'The International Herald Tribune'
+    __author__     = 'Derry FitzGerald'
+    oldest_article = 1
+    max_articles_per_feed = 10
+    no_stylesheets = True
+
+    remove_tags    = [dict(name='div', attrs={'class':'footer'})]
+    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }' 
+
+    feeds          = [
+                      (u'Frontpage', u'http://www.iht.com/rss/frontpage.xml'), 
+                      (u'Business', u'http://www.iht.com/rss/business.xml'),
+                      (u'Americas', u'http://www.iht.com/rss/america.xml'),
+                      (u'Europe', u'http://www.iht.com/rss/europe.xml'),
+                      (u'Asia', u'http://www.iht.com/rss/asia.xml'),
+                      (u'Africa and Middle East', u'http://www.iht.com/rss/africa.xml'),
+                      (u'Opinion', u'http://www.iht.com/rss/opinion.xml'),
+                      (u'Technology', u'http://www.iht.com/rss/technology.xml'),
+                      (u'Health and Science', u'http://www.iht.com/rss/healthscience.xml'),
+                      (u'Sports', u'http://www.iht.com/rss/sports.xml'),
+                      (u'Culture', u'http://www.iht.com/rss/arts.xml'),
+                      (u'Style and Design', u'http://www.iht.com/rss/style.xml'),
+                      (u'Travel', u'http://www.iht.com/rss/travel.xml'),
+                      (u'At Home Abroad', u'http://www.iht.com/rss/athome.xml'),
+                      (u'Your Money', u'http://www.iht.com/rss/yourmoney.xml'),
+                      (u'Properties', u'http://www.iht.com/rss/properties.xml')
+                    ]
+    temp_files = []
+    articles_are_obfuscated = True
+    
+    def get_obfuscated_article(self, url, logger):
+        br = self.get_browser()
+        br.open(url)
+        br.select_form(name='printFriendly')
+        res = br.submit()
+        html = res.read()
+        self.temp_files.append(PersistentTemporaryFile('_iht.html'))
+        self.temp_files[-1].write(html)
+        self.temp_files[-1].close()
+        return self.temp_files[-1].name

src/calibre/web/feeds/recipes/irish_times.py

@@ -0,0 +1,36 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Derry FitzGerald'
+'''
+irishtimes.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class IrishTimes(BasicNewsRecipe):
+    title          = u'The Irish Times'
+    __author__     = 'Derry FitzGerald'
+    no_stylesheets = True
+
+    remove_tags    = [dict(name='div', attrs={'class':'footer'})]
+    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }' 
+
+    feeds          = [
+                      ('Frontpage', 'http://www.irishtimes.com/feeds/rss/newspaper/index.rss'), 
+                      ('Ireland', 'http://www.irishtimes.com/feeds/rss/newspaper/ireland.rss'),
+                      ('World', 'http://www.irishtimes.com/feeds/rss/newspaper/world.rss'),
+                      ('Finance', 'http://www.irishtimes.com/feeds/rss/newspaper/finance.rss'),
+                      ('Features', 'http://www.irishtimes.com/feeds/rss/newspaper/features.rss'),
+                      ('Sport', 'http://www.irishtimes.com/feeds/rss/newspaper/sport.rss'),
+                      ('Opinion', 'http://www.irishtimes.com/feeds/rss/newspaper/opinion.rss'),
+                      ('Letters', 'http://www.irishtimes.com/feeds/rss/newspaper/letters.rss'),
+                      ('Health', 'http://www.irishtimes.com/feeds/rss/newspaper/health.rss'),
+                      ('Education and Parenting', 'http://www.irishtimes.com/feeds/rss/newspaper/education.rss'),
+                      ('Science Today', 'http://www.irishtimes.com/feeds/rss/newspaper/sciencetoday.rss'),
+                      ('The Ticket', 'http://www.irishtimes.com/feeds/rss/newspaper/theticket.rss'),
+                      ('Weekend', 'http://www.irishtimes.com/feeds/rss/newspaper/weekend.rss'),
+                      ('News Features', 'http://www.irishtimes.com/feeds/rss/newspaper/newsfeatures.rss'),
+                      ('Magazine', 'http://www.irishtimes.com/feeds/rss/newspaper/magazine.rss'),
+                    ]
+
+    def print_version(self, url):
+        return url.replace('.html', '_pf.html')

upload.py

@@ -201,15 +201,9 @@ def upload_docs():
     check_call('''scp docs/pdf/api.pdf divok:%s/'''%(DOCS,))
 
 def upload_user_manual():
-    cwd = os.getcwdu()
+    check_call('python setup.py manual')
-    os.chdir('src/calibre/manual')
+    check_call('scp -r .build/html/* divok:%s'%USER_MANUAL)
-    try:
+    
-        check_call('make clean html')
-        check_call('ssh divok rm -rf %s/\\*'%USER_MANUAL)
-        check_call('scp -r .build/html/* divok:%s'%USER_MANUAL)
-    finally:
-        os.chdir(cwd)
-
 def build_src_tarball():
     check_call('bzr export dist/calibre-%s.tar.bz2'%__version__)