From c9fa1d673e34fafef47eeaba4d02f514963428ee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 16 Aug 2008 15:41:54 -0700 Subject: [PATCH 1/7] Read series number from fb2 files --- src/calibre/ebooks/metadata/__init__.py | 2 +- src/calibre/ebooks/metadata/fb2.py | 4 ++++ src/calibre/ebooks/metadata/meta.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 180a07afa8..c9468c812e 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -201,7 +201,7 @@ class MetaInformation(object): #: mi.cover_data = (ext, data) self.cover_data = mi.cover_data if (mi and hasattr(mi, 'cover_data')) else (None, None) self.application_id = mi.application_id if (mi and hasattr(mi, 'application_id')) else None - self.manifest = getattr(mi, 'manifest', None) + self.manifest = getattr(mi, 'manifest', None) self.toc = getattr(mi, 'toc', None) self.spine = getattr(mi, 'spine', None) self.guide = getattr(mi, 'guide', None) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index 672fc3e9ee..f66a25e703 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -41,6 +41,10 @@ def get_metadata(stream): mi.author_sort = lastname+'; '+firstname if series: mi.series = series.get('name', None) + try: + mi.series_index = int(series.get('number', None)) + except (TypeError, ValueError): + pass if cdata: mi.cover_data = cdata return mi diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index 4fa0f180be..35d89a51be 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -12,7 +12,7 @@ from calibre.ebooks.metadata.lit import get_metadata as lit_metadata from calibre.ebooks.metadata.epub import get_metadata as epub_metadata from calibre.ebooks.metadata.html import get_metadata as html_metadata from calibre.ebooks.mobi.reader import get_metadata as mobi_metadata -from calibre.ebooks.metadata.opf import OPFReader +from calibre.ebooks.metadata.opf import OPFReader from calibre.ebooks.metadata.rtf import set_metadata as set_rtf_metadata from calibre.ebooks.lrf.meta import set_metadata as set_lrf_metadata from calibre.ebooks.metadata.epub import set_metadata as set_epub_metadata @@ -29,14 +29,14 @@ _METADATA_PRIORITIES = [ # Higher values should be used to update metadata from lower values METADATA_PRIORITIES = collections.defaultdict(lambda:0) for i, ext in enumerate(_METADATA_PRIORITIES): - METADATA_PRIORITIES[ext] = i + METADATA_PRIORITIES[ext] = i def path_to_ext(path): return os.path.splitext(path)[1][1:].lower() def metadata_from_formats(formats): mi = MetaInformation(None, None) - formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], + formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], METADATA_PRIORITIES[path_to_ext(y)])) for path in formats: ext = path_to_ext(path) From a33b1438e77661a5ef58f72a5234519eb7a17145 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 16 Aug 2008 16:41:05 -0700 Subject: [PATCH 2/7] Switch to storing ebook files in the filesystem and only metadata in the database --- resources.py | 4 ++-- src/calibre/gui2/main.py | 18 ++++++++++-------- src/calibre/utils/config.py | 2 ++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/resources.py b/resources.py index 4eb1c15e6a..4b0bd846df 100644 --- a/resources.py +++ b/resources.py @@ -13,7 +13,7 @@ RESOURCES = dict( opf_template = '%p/ebooks/metadata/opf.xml', ncx_template = '%p/ebooks/metadata/ncx.xml', fb2_xsl = '%p/ebooks/lrf/fb2/fb2.xsl', - metadata_sqlite = '%p/library/metadata_sqlite.sql', + metadata_sqlite = '%p/library/metadata_sqlite.sql', ) def main(args=sys.argv): @@ -41,4 +41,4 @@ def main(args=sys.argv): return 0 if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 40b563f414..a8e9dca223 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -225,7 +225,7 @@ class Main(MainWindow, Ui_MainWindow): pd.show() db.migrate_old(self.olddb, pd) self.olddb = None - Settings().set('library path', self.library_path) + prefs['library_path'] = self.library_path self.library_view.sortByColumn(3, Qt.DescendingOrder) if not self.library_view.restore_column_widths(): self.library_view.resizeColumnsToContents() @@ -1106,7 +1106,7 @@ class Main(MainWindow, Ui_MainWindow): _('

An invalid database already exists at %s, delete it before trying to move the existing database.
Error: %s')%(newloc, str(err))) d.exec_() self.library_path = self.library_view.model().db.library_path - prefs['library path'] = self.library_path + prefs['library_path'] = self.library_path except Exception, err: traceback.print_exc() d = error_dialog(self, _('Could not move database'), unicode(err)) @@ -1204,31 +1204,33 @@ class Main(MainWindow, Ui_MainWindow): def initialize_database(self): - self.library_path = prefs['library path'] + self.library_path = prefs['library_path'] self.olddb = None if self.library_path is None: # Need to migrate to new database layout self.database_path = prefs['database_path'] if not os.access(os.path.dirname(self.database_path), os.W_OK): - error_dialog(self, _('Database does not exist'), _('The directory in which the database should be: %s no longer exists. Please choose a new database location.')%self.database_path).exec_() - self.database_path = choose_dir(self, 'database path dialog', 'Choose new location for database') + error_dialog(self, _('Database does not exist'), + _('The directory in which the database should be: %s no longer exists. Please choose a new database location.')%self.database_path).exec_() + self.database_path = choose_dir(self, 'database path dialog', + _('Choose new location for database')) if not self.database_path: self.database_path = os.path.expanduser('~').decode(sys.getfilesystemencoding()) if not os.path.exists(self.database_path): os.makedirs(self.database_path) self.database_path = os.path.join(self.database_path, 'library1.db') - settings.set('database path', self.database_path) + prefs['database_path'] = self.database_path home = os.path.dirname(self.database_path) if not os.path.exists(home): home = os.getcwd() from PyQt4.QtGui import QFileDialog - dir = qstring_to_unicode(QFileDialog.getExistingDirectory(self, _('Choose a location for your ebook library.'), home)) + dir = unicode(QFileDialog.getExistingDirectory(self, + _('Choose a location for your ebook library.'), home)) if not dir: dir = os.path.dirname(self.database_path) self.library_path = os.path.abspath(dir) self.olddb = LibraryDatabase(self.database_path) - def read_settings(self): self.initialize_database() geometry = config['main_window_geometry'] diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 997bfa230f..baf85186c1 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -451,6 +451,8 @@ def _prefs(): help=_('Access key for isbndb.com')) c.add_opt('network_timeout', default=5, help=_('Default timeout for network operations (seconds)')) + c.add_opt('library_path', default=None, + help=_('Path to directory in which your library of books is stored')) c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') return c From 584d89c50e4dfde7006b789ab8c6e66bf517f262 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 16 Aug 2008 18:18:17 -0700 Subject: [PATCH 3/7] IGN:Simplify build process --- Makefile | 30 ------------------ resources.py | 4 ++- setup.py | 62 ++++++++++++++++++++++++++++++++++++- src/calibre/manual/Makefile | 2 +- upload.py | 12 ++----- 5 files changed, 68 insertions(+), 42 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 1990ad22bd..0000000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -PYTHON = python - -all : gui2 translations resources - -clean : - cd src/calibre/gui2 && ${PYTHON} make.py clean - -gui2 : - cd src/calibre/gui2 && ${PYTHON} make.py - -test : gui2 - cd src/calibre/gui2 && ${PYTHON} make.py test - -translations : - cd src/calibre/translations && ${PYTHON} __init__.py - -resources: - ${PYTHON} resources.py - -manual: - make -C src/calibre/manual clean html - -pot : - cd src/calibre/translations && ${PYTHON} __init__.py pot - -egg : - ${PYTHON} setup.py bdist_egg --exclude-source-files - -linux_binary: - ${PYTHON} -c "import upload; upload._build_linux()" diff --git a/resources.py b/resources.py index 4b0bd846df..e5b426e2a4 100644 --- a/resources.py +++ b/resources.py @@ -37,7 +37,9 @@ def main(args=sys.argv): if not translations_found: print 'WARNING: Could not find Qt transations' - open('src'+os.sep+__appname__+os.sep+'/resources.py', 'wb').write(data) + dest = os.path.abspath(os.path.join('src', __appname__, 'resources.py')) + print 'Writing resources to', dest + open(dest, 'wb').write(data) return 0 if __name__ == '__main__': diff --git a/setup.py b/setup.py index 346c9059d5..0292aba9c0 100644 --- a/setup.py +++ b/setup.py @@ -47,9 +47,68 @@ main_functions = { if __name__ == '__main__': from setuptools import setup, find_packages, Extension + from distutils.command.build import build as _build + from distutils.core import Command from pyqtdistutils import PyQtExtension, build_ext import subprocess, glob + class pot(Command): + user_options = [] + def initialize_options(self): pass + def finalize_options(self): pass + + def run(self): + from calibre.translations import create_pot + create_pot() + + def build_manual(): + cwd = os.path.abspath(os.getcwd()) + os.chdir(os.path.join('src', 'calibre', 'manual')) + try: + for d in ('.build', 'cli'): + if os.path.exists(d): + shutil.rmtree(d) + os.makedirs(d) + if not os.path.exists('.build'+os.sep+'html'): + os.makedirs('.build'+os.sep+'html') + subprocess.check_call(['sphinx-build', '-b', 'custom', '-d', + '.build/doctrees', '.', '.build/html']) + finally: + os.chdir(cwd) + + class manual(Command): + user_options = [] + def initialize_options(self): pass + def finalize_options(self): pass + + def run(self): + build_manual() + + + class build(_build): + + def run(self): + # Build resources + resources = __import__('resources') + resources.main([sys.executable, 'resources.py']) + from calibre.translations import main as translations + cwd = os.path.abspath(os.getcwd()) + # Build translations + try: + os.chdir(os.path.join('src', 'calibre', 'translations')) + translations([sys.executable]) + finally: + os.chdir(cwd) + # Build GUI + from calibre.gui2.make import main as gui2 + try: + os.chdir(os.path.join('src', 'calibre', 'gui2')) + print 'Compiling GUI resources...' + gui2([sys.executable]) + finally: + os.chdir(cwd) + _build.run(self) + entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install') ext_modules = [ Extension('calibre.plugins.lzx', @@ -125,7 +184,8 @@ if __name__ == '__main__': 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Hardware :: Hardware Drivers' ], - cmdclass = {'build_ext': build_ext}, + cmdclass = {'build_ext': build_ext, 'build' : build, 'pot' : pot, + 'manual' : manual}, ) if 'develop' in ' '.join(sys.argv) and islinux: diff --git a/src/calibre/manual/Makefile b/src/calibre/manual/Makefile index 25e96dd637..b6f03e033b 100644 --- a/src/calibre/manual/Makefile +++ b/src/calibre/manual/Makefile @@ -21,7 +21,7 @@ help: @echo " linkcheck to check all external links for integrity" clean: - -rm -rf .build/* cli + rm -rf .build/* cli html: mkdir -p .build/html .build/doctrees diff --git a/upload.py b/upload.py index fc8a1754c2..e528c6f257 100644 --- a/upload.py +++ b/upload.py @@ -201,15 +201,9 @@ def upload_docs(): check_call('''scp docs/pdf/api.pdf divok:%s/'''%(DOCS,)) def upload_user_manual(): - cwd = os.getcwdu() - os.chdir('src/calibre/manual') - try: - check_call('make clean html') - check_call('ssh divok rm -rf %s/\\*'%USER_MANUAL) - check_call('scp -r .build/html/* divok:%s'%USER_MANUAL) - finally: - os.chdir(cwd) - + check_call('python setup.py manual') + check_call('scp -r .build/html/* divok:%s'%USER_MANUAL) + def build_src_tarball(): check_call('bzr export dist/calibre-%s.tar.bz2'%__version__) From c31e509de2b559a69680ba24e3daf86afcb11c0a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 17 Aug 2008 07:48:14 -0700 Subject: [PATCH 4/7] IGN:... --- src/calibre/ebooks/lrf/comic/convert_from.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/lrf/comic/convert_from.py b/src/calibre/ebooks/lrf/comic/convert_from.py index 48d19657d0..33d697bfe4 100755 --- a/src/calibre/ebooks/lrf/comic/convert_from.py +++ b/src/calibre/ebooks/lrf/comic/convert_from.py @@ -182,7 +182,8 @@ class PageProcessor(list): MagickQuantizeImage(wand, self.opts.colors, RGBColorspace, 0, 1, 0) dest = '%d_%d.png'%(self.num, i) dest = os.path.join(self.dest, dest) - MagickWriteImage(wand, dest) + MagickWriteImage(wand, dest+'8') + os.rename(dest+'8', dest) self.append(dest) DestroyPixelWand(pw) From 6334cc850c40e7a01b49573609ac3d02739b44d1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 17 Aug 2008 09:55:25 -0700 Subject: [PATCH 5/7] Add recipes for The Irish Times and The International Herald Tribune --- src/calibre/web/feeds/news.py | 32 ++++++++++-- src/calibre/web/feeds/recipes/__init__.py | 2 +- src/calibre/web/feeds/recipes/iht.py | 51 ++++++++++++++++++++ src/calibre/web/feeds/recipes/irish_times.py | 36 ++++++++++++++ 4 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 src/calibre/web/feeds/recipes/iht.py create mode 100644 src/calibre/web/feeds/recipes/irish_times.py diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index dabdd3547d..530f15b9ab 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -98,6 +98,10 @@ class BasicNewsRecipe(object, LoggingInterface): #: embedded content. use_embedded_content = None + #: Set to True and implement :method:`get_obfuscated_article` to handle + #: websites that try to make it difficult to scrape content. + articles_are_obfuscated = False + #: Specify any extra :term:`CSS` that should be addded to downloaded :term:`HTML` files #: It will be inserted into `