diff --git a/recipes/daytona_beach.recipe b/recipes/daytona_beach.recipe new file mode 100644 index 0000000000..1230c1d8ed --- /dev/null +++ b/recipes/daytona_beach.recipe @@ -0,0 +1,78 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class DaytonBeachNewsJournal(BasicNewsRecipe): + title ='Daytona Beach News Journal' + __author__ = 'BRGriff' + pubisher = 'News-JournalOnline.com' + description = 'Daytona Beach, Florida, Newspaper' + category = 'News, Daytona Beach, Florida' + oldest_article = 1 + max_articles_per_feed = 100 + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'en' + filterDuplicates = True + remove_attributes = ['style'] + + keep_only_tags = [dict(name='div', attrs={'class':'page-header'}), + dict(name='div', attrs={'class':'asset-body'}) + ] + remove_tags = [dict(name='div', attrs={'class':['byline-section', 'asset-meta']}) + ] + + feeds = [ + #####NEWS##### + (u"News", u"http://www.news-journalonline.com/rss.xml"), + (u"Breaking News", u"http://www.news-journalonline.com/breakingnews/rss.xml"), + (u"Local - East Volusia", u"http://www.news-journalonline.com/news/local/east-volusia/rss.xml"), + (u"Local - West Volusia", u"http://www.news-journalonline.com/news/local/west-volusia/rss.xml"), + (u"Local - Southeast", u"http://www.news-journalonline.com/news/local/southeast-volusia/rss.xml"), + (u"Local - Flagler", u"http://www.news-journalonline.com/news/local/flagler/rss.xml"), + (u"Florida", u"http://www.news-journalonline.com/news/florida/rss.xml"), + (u"National/World", u"http://www.news-journalonline.com/news/nationworld/rss.xml"), + (u"Politics", u"http://www.news-journalonline.com/news/politics/rss.xml"), + (u"News of Record", u"http://www.news-journalonline.com/news/news-of-record/rss.xml"), + ####BUSINESS#### + (u"Business", u"http://www.news-journalonline.com/business/rss.xml"), + #(u"Jobs", u"http://www.news-journalonline.com/business/jobs/rss.xml"), + #(u"Markets", u"http://www.news-journalonline.com/business/markets/rss.xml"), + #(u"Real Estate", u"http://www.news-journalonline.com/business/real-estate/rss.xml"), + #(u"Technology", u"http://www.news-journalonline.com/business/technology/rss.xml"), + ####SPORTS#### + (u"Sports", u"http://www.news-journalonline.com/sports/rss.xml"), + (u"Racing", u"http://www.news-journalonline.com/racing/rss.xml"), + (u"Highschool", u"http://www.news-journalonline.com/sports/highschool/rss.xml"), + (u"College", u"http://www.news-journalonline.com/sports/college/rss.xml"), + (u"Basketball", u"http://www.news-journalonline.com/sports/basketball/rss.xml"), + (u"Football", u"http://www.news-journalonline.com/sports/football/rss.xml"), + (u"Golf", u"http://www.news-journalonline.com/sports/golf/rss.xml"), + (u"Other Sports", u"http://www.news-journalonline.com/sports/other/rss.xml"), + ####LIFESTYLE#### + (u"Lifestyle", u"http://www.news-journalonline.com/lifestyle/rss.xml"), + #(u"Fashion", u"http://www.news-journalonline.com/lifestyle/fashion/rss.xml"), + (u"Food", u"http://www.news-journalonline.com/lifestyle/food/rss.xml"), + #(u"Health", u"http://www.news-journalonline.com/lifestyle/health/rss.xml"), + (u"Home and Garden", u"http://www.news-journalonline.com/lifestyle/home-and-garden/rss.xml"), + (u"Living", u"http://www.news-journalonline.com/lifestyle/living/rss.xml"), + (u"Religion", u"http://www.news-journalonline.com/lifestyle/religion/rss.xml"), + #(u"Travel", u"http://www.news-journalonline.com/lifestyle/travel/rss.xml"), + ####OPINION#### + #(u"Opinion", u"http://www.news-journalonline.com/opinion/rss.xml"), + #(u"Letters to Editor", u"http://www.news-journalonline.com/opinion/letters-to-the-editor/rss.xml"), + #(u"Columns", u"http://www.news-journalonline.com/columns/rss.xml"), + #(u"Podcasts", u"http://www.news-journalonline.com/podcasts/rss.xml"), + ####ENTERTAINMENT#### ##Weekly Feature## + (u"Entertainment", u"http://www.go386.com/rss.xml"), + (u"Go Out", u"http://www.go386.com/go/rss.xml"), + (u"Music", u"http://www.go386.com/music/rss.xml"), + (u"Movies", u"http://www.go386.com/movies/rss.xml"), + #(u"Culture", u"http://www.go386.com/culture/rss.xml"), + + ] + + extra_css = ''' + .page-header{font-family:Arial,Helvetica,sans-serif; font-style:bold;font-size:22pt;} + .asset-body{font-family:Helvetica,Arial,sans-serif; font-size:16pt;} + + ''' diff --git a/recipes/elclubdelebook.recipe b/recipes/elclubdelebook.recipe new file mode 100644 index 0000000000..e05b176cc5 --- /dev/null +++ b/recipes/elclubdelebook.recipe @@ -0,0 +1,61 @@ + +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +www.clubdelebook.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ElClubDelEbook(BasicNewsRecipe): + title = 'El club del ebook' + __author__ = 'Darko Miletic' + description = 'El Club del eBook, es la primera fuente de informacion sobre ebooks de Argentina. Aca vas a encontrar noticias, tips, tutoriales, recursos y opiniones sobre el mundo de los libros electronicos.' + tags = 'ebook, libro electronico, e-book, ebooks, libros electronicos, e-books' + oldest_article = 7 + max_articles_per_feed = 100 + language = 'es_AR' + encoding = 'utf-8' + no_stylesheets = True + use_embedded_content = True + publication_type = 'blog' + masthead_url = 'http://dl.dropbox.com/u/2845131/elclubdelebook.png' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif} + img{ margin-bottom: 0.8em; + border: 1px solid #333333; + padding: 4px; display: block + } + """ + + conversion_options = { + 'comment' : description + , 'tags' : tags + , 'publisher': title + , 'language' : language + } + + remove_tags = [dict(attrs={'id':'crp_related'})] + remove_tags_after = dict(attrs={'id':'crp_related'}) + + feeds = [(u'Articulos', u'http://feeds.feedburner.com/ElClubDelEbook')] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup diff --git a/recipes/frontlineonnet.recipe b/recipes/frontlineonnet.recipe new file mode 100644 index 0000000000..3b65e4bb18 --- /dev/null +++ b/recipes/frontlineonnet.recipe @@ -0,0 +1,81 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Darko Miletic ' +''' +frontlineonnet.com +''' + +import re +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +class Frontlineonnet(BasicNewsRecipe): + title = 'Frontline' + __author__ = 'Darko Miletic' + description = "India's national magazine" + publisher = 'Frontline' + category = 'news, politics, India' + no_stylesheets = True + delay = 1 + INDEX = 'http://frontlineonnet.com/' + use_embedded_content = False + encoding = 'cp1252' + language = 'en_IN' + publication_type = 'magazine' + masthead_url = 'http://frontlineonnet.com/images/newfline.jpg' + extra_css = """ + body{font-family: Verdana,Arial,Helvetica,sans-serif} + img{margin-top:0.5em; margin-bottom: 0.7em; display: block} + """ + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + preprocess_regexps = [ + (re.compile(r'.*?title', re.DOTALL|re.IGNORECASE),lambda match: '') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda match: '
') + ,(re.compile(r'
', re.DOTALL|re.IGNORECASE),lambda match: '') + ] + + keep_only_tags= [ + dict(name='font', attrs={'class':'storyhead'}) + ,dict(attrs={'class':'byline'}) + ] + remove_attributes=['size','noshade','border'] + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll('img'): + if not item.has_key('alt'): + item['alt'] = 'image' + return soup + + def parse_index(self): + articles = [] + soup = self.index_to_soup(self.INDEX) + for feed_link in soup.findAll('a',href=True): + if feed_link['href'].startswith('stories/'): + url = self.INDEX + feed_link['href'] + title = self.tag_to_string(feed_link) + date = strftime(self.timefmt) + articles.append({ + 'title' :title + ,'date' :date + ,'url' :url + ,'description':'' + }) + return [('Frontline', articles)] + + def print_version(self, url): + return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2] + + def image_url_processor(self, baseurl, url): + return url.replace('../images/', self.INDEX + 'images/').strip() diff --git a/recipes/icons/elclubdelebook.png b/recipes/icons/elclubdelebook.png new file mode 100644 index 0000000000..c43f045484 Binary files /dev/null and b/recipes/icons/elclubdelebook.png differ diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index 9c4f666449..aa29d4b8de 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -13,8 +13,10 @@ CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, isbn TEXT DEFAULT "" COLLATE NOCASE, lccn TEXT DEFAULT "" COLLATE NOCASE, path TEXT NOT NULL DEFAULT "", - flags INTEGER NOT NULL DEFAULT 1 - , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); + flags INTEGER NOT NULL DEFAULT 1, + uuid TEXT, + has_cover BOOL DEFAULT 0, + last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, author INTEGER NOT NULL, diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 3a35feb66f..33e80982d1 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -578,6 +578,7 @@ def url_slash_cleaner(url): def get_download_filename(url, cookie_file=None): ''' Get a local filename for a URL using the content disposition header + Returns empty string if no content disposition header present ''' from contextlib import closing from urllib2 import unquote as urllib2_unquote @@ -591,8 +592,10 @@ def get_download_filename(url, cookie_file=None): cj.load(cookie_file) br.set_cookiejar(cj) + last_part_name = '' try: with closing(br.open(url)) as r: + last_part_name = r.geturl().split('/')[-1] disposition = r.info().get('Content-disposition', '') for p in disposition.split(';'): if 'filename' in p: @@ -612,7 +615,7 @@ def get_download_filename(url, cookie_file=None): traceback.print_exc() if not filename: - filename = r.geturl().split('/')[-1] + filename = last_part_name return filename diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index d1c5b6ccd5..ce7297ef9a 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -762,99 +762,132 @@ plugins += input_profiles + output_profiles class ActionAdd(InterfaceActionBase): name = 'Add Books' actual_plugin = 'calibre.gui2.actions.add:AddAction' + description = _('Add books to calibre or the connected device') class ActionFetchAnnotations(InterfaceActionBase): name = 'Fetch Annotations' actual_plugin = 'calibre.gui2.actions.annotate:FetchAnnotationsAction' + description = _('Fetch annotations from a connected Kindle (experimental)') class ActionGenerateCatalog(InterfaceActionBase): name = 'Generate Catalog' actual_plugin = 'calibre.gui2.actions.catalog:GenerateCatalogAction' + description = _('Generate a catalog of the books in your calibre library') class ActionConvert(InterfaceActionBase): name = 'Convert Books' actual_plugin = 'calibre.gui2.actions.convert:ConvertAction' + description = _('Convert books to various ebook formats') class ActionDelete(InterfaceActionBase): name = 'Remove Books' actual_plugin = 'calibre.gui2.actions.delete:DeleteAction' + description = _('Delete books from your calibre library or connected device') class ActionEditMetadata(InterfaceActionBase): name = 'Edit Metadata' actual_plugin = 'calibre.gui2.actions.edit_metadata:EditMetadataAction' + description = _('Edit the metadata of books in your calibre library') class ActionView(InterfaceActionBase): name = 'View' actual_plugin = 'calibre.gui2.actions.view:ViewAction' + description = _('Read books in your calibre library') class ActionFetchNews(InterfaceActionBase): name = 'Fetch News' actual_plugin = 'calibre.gui2.actions.fetch_news:FetchNewsAction' + description = _('Download news from the internet in ebook form') + +class ActionQuickview(InterfaceActionBase): + name = 'Show Quickview' + actual_plugin = 'calibre.gui2.actions.show_quickview:ShowQuickviewAction' + description = _('Show a list of related books quickly') class ActionSaveToDisk(InterfaceActionBase): name = 'Save To Disk' actual_plugin = 'calibre.gui2.actions.save_to_disk:SaveToDiskAction' + description = _('Export books from your calibre library to the hard disk') class ActionShowBookDetails(InterfaceActionBase): name = 'Show Book Details' actual_plugin = 'calibre.gui2.actions.show_book_details:ShowBookDetailsAction' + description = _('Show book details in a separate popup') class ActionRestart(InterfaceActionBase): name = 'Restart' actual_plugin = 'calibre.gui2.actions.restart:RestartAction' + description = _('Restart calibre') class ActionOpenFolder(InterfaceActionBase): name = 'Open Folder' actual_plugin = 'calibre.gui2.actions.open:OpenFolderAction' + description = _('Open the folder that contains the book files in your' + ' calibre library') class ActionSendToDevice(InterfaceActionBase): name = 'Send To Device' actual_plugin = 'calibre.gui2.actions.device:SendToDeviceAction' + description = _('Send books to the connected device') class ActionConnectShare(InterfaceActionBase): name = 'Connect Share' actual_plugin = 'calibre.gui2.actions.device:ConnectShareAction' + description = _('Send books via email or the web also connect to iTunes or' + ' folders on your computer as if they are devices') class ActionHelp(InterfaceActionBase): name = 'Help' actual_plugin = 'calibre.gui2.actions.help:HelpAction' + description = _('Browse the calibre User Manual') class ActionPreferences(InterfaceActionBase): name = 'Preferences' actual_plugin = 'calibre.gui2.actions.preferences:PreferencesAction' + description = _('Customize calibre') class ActionSimilarBooks(InterfaceActionBase): name = 'Similar Books' actual_plugin = 'calibre.gui2.actions.similar_books:SimilarBooksAction' + description = _('Easily find books similar to the currently selected one') class ActionChooseLibrary(InterfaceActionBase): name = 'Choose Library' actual_plugin = 'calibre.gui2.actions.choose_library:ChooseLibraryAction' + description = _('Switch between different calibre libraries and perform' + ' maintenance on them') class ActionAddToLibrary(InterfaceActionBase): name = 'Add To Library' actual_plugin = 'calibre.gui2.actions.add_to_library:AddToLibraryAction' + description = _('Copy books from the devce to your calibre library') class ActionEditCollections(InterfaceActionBase): name = 'Edit Collections' actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction' + description = _('Edit the collections in which books are placed on your device') class ActionCopyToLibrary(InterfaceActionBase): name = 'Copy To Library' actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction' + description = _('Copy a book from one calibre library to another') class ActionTweakEpub(InterfaceActionBase): name = 'Tweak ePub' actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction' + description = _('Make small twekas to epub files in your calibre library') class ActionNextMatch(InterfaceActionBase): name = 'Next Match' actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction' + description = _('Find the next or previous match when searching in ' + 'your calibre library in highlight mode') class ActionStore(InterfaceActionBase): name = 'Store' author = 'John Schember' actual_plugin = 'calibre.gui2.actions.store:StoreAction' + description = _('Search for books from different book sellers') def customization_help(self, gui=False): return 'Customize the behavior of the store search.' @@ -870,13 +903,13 @@ class ActionStore(InterfaceActionBase): class ActionPluginUpdater(InterfaceActionBase): name = 'Plugin Updater' author = 'Grant Drake' - description = 'Queries the MobileRead forums for updates to plugins to install' + description = _('Get new calibre plugins or update your existing ones') actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdaterAction' plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionConvert, ActionDelete, ActionEditMetadata, ActionView, - ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails, - ActionRestart, ActionOpenFolder, ActionConnectShare, + ActionFetchNews, ActionSaveToDisk, ActionQuickview, + ActionShowBookDetails,ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore, @@ -1307,6 +1340,16 @@ class StoreLegimiStore(StoreBase): headquarters = 'PL' formats = ['EPUB'] +class StoreLibreDEStore(StoreBase): + name = 'Libri DE' + author = 'Charles Haley' + description = u'Sicher Bücher, Hörbücher und Downloads online bestellen.' + actual_plugin = 'calibre.gui2.store.libri_de_plugin:LibreDEStore' + + headquarters = 'DE' + formats = ['EPUB', 'PDF'] + affiliate = True + class StoreManyBooksStore(StoreBase): name = 'ManyBooks' description = u'Public domain and creative commons works from many sources.' @@ -1457,6 +1500,7 @@ plugins += [ StoreGutenbergStore, StoreKoboStore, StoreLegimiStore, + StoreLibreDEStore, StoreManyBooksStore, StoreMobileReadStore, StoreNextoStore, diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index b77ac81587..88f19df7af 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -259,6 +259,10 @@ class OutputFormatPlugin(Plugin): #: (option_name, recommended_value, recommendation_level) recommendations = set([]) + @property + def description(self): + return _('Convert ebooks to the %s format'%self.file_type) + def __init__(self, *args): Plugin.__init__(self, *args) self.report_progress = DummyReporter() diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py new file mode 100644 index 0000000000..4384cab2da --- /dev/null +++ b/src/calibre/db/__init__.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +''' +Rewrite of the calibre database backend. + +Broad Objectives: + + * Use the sqlite db only as a datastore. i.e. do not do + sorting/searching/concatenation or anything else in sqlite. Instead + mirror the sqlite tables in memory, create caches and lookup maps from + them and create a set_* API that updates the memory caches and the sqlite + correctly. + + * Move from keeping a list of books in memory as a cache to a per table + cache. This allows much faster search and sort operations at the expense + of slightly slower lookup operations. That slowdown can be mitigated by + keeping lots of maps and updating them in the set_* API. Also + get_categories becomes blazingly fast. + + * Separate the database layer from the cache layer more cleanly. Rather + than having the db layer refer to the cache layer and vice versa, the + cache layer will refer to the db layer only and the new API will be + defined on the cache layer. + + * Get rid of index_is_id and other poor design decisions + + * Minimize the API as much as possible and define it cleanly + + * Do not change the on disk format of metadata.db at all (this is for + backwards compatibility) + + * Get rid of the need for a separate db access thread by switching to apsw + to access sqlite, which is thread safe + + * The new API will have methods to efficiently do bulk operations and will + use shared/exclusive/pending locks to serialize access to the in-mem data + structures. Use the same locking scheme as sqlite itself does. + +How this will proceed: + + 1. Create the new API + 2. Create a test suite for it + 3. Write a replacement for LibraryDatabase2 that uses the new API + internally + 4. Lots of testing of calibre with the new LibraryDatabase2 + 5. Gradually migrate code to use the (much faster) new api wherever possible (the new api + will be exposed via db.new_api) + + I plan to work on this slowly, in parallel to normal calibre development + work. + +Various things that require other things before they can be migrated: + 1. From initialize_dynamic(): set_saved_searches, + load_user_template_functions. Also add custom + columns/categories/searches info into + self.field_metadata. Finally, implement metadata dirtied + functionality. + +''' diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py new file mode 100644 index 0000000000..ba683dde50 --- /dev/null +++ b/src/calibre/db/backend.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +# Imports {{{ +import os, shutil, uuid, json +from functools import partial + +import apsw + +from calibre import isbytestring, force_unicode, prints +from calibre.constants import (iswindows, filesystem_encoding, + preferred_encoding) +from calibre.ptempfile import PersistentTemporaryFile +from calibre.library.schema_upgrades import SchemaUpgrade +from calibre.library.field_metadata import FieldMetadata +from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.utils.icu import strcmp +from calibre.utils.config import to_json, from_json, prefs, tweaks +from calibre.utils.date import utcfromtimestamp +from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, + SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) +# }}} + +''' +Differences in semantics from pysqlite: + + 1. execute/executemany/executescript operate in autocommit mode + +''' + +class DynamicFilter(object): # {{{ + + 'No longer used, present for legacy compatibility' + + def __init__(self, name): + self.name = name + self.ids = frozenset([]) + + def __call__(self, id_): + return int(id_ in self.ids) + + def change(self, ids): + self.ids = frozenset(ids) +# }}} + +class DBPrefs(dict): # {{{ + + 'Store preferences as key:value pairs in the db' + + def __init__(self, db): + dict.__init__(self) + self.db = db + self.defaults = {} + self.disable_setting = False + for key, val in self.db.conn.get('SELECT key,val FROM preferences'): + try: + val = self.raw_to_object(val) + except: + prints('Failed to read value for:', key, 'from db') + continue + dict.__setitem__(self, key, val) + + def raw_to_object(self, raw): + if not isinstance(raw, unicode): + raw = raw.decode(preferred_encoding) + return json.loads(raw, object_hook=from_json) + + def to_raw(self, val): + return json.dumps(val, indent=2, default=to_json) + + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + return self.defaults[key] + + def __delitem__(self, key): + dict.__delitem__(self, key) + self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,)) + + def __setitem__(self, key, val): + if self.disable_setting: + return + raw = self.to_raw(val) + self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key, + raw)) + dict.__setitem__(self, key, val) + + def set(self, key, val): + self.__setitem__(key, val) + +# }}} + +# Extra collators {{{ +def pynocase(one, two, encoding='utf-8'): + if isbytestring(one): + try: + one = one.decode(encoding, 'replace') + except: + pass + if isbytestring(two): + try: + two = two.decode(encoding, 'replace') + except: + pass + return cmp(one.lower(), two.lower()) + +def _author_to_author_sort(x): + if not x: return '' + return author_to_author_sort(x.replace('|', ',')) + +def icu_collator(s1, s2): + return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8')) +# }}} + +class Connection(apsw.Connection): # {{{ + + BUSY_TIMEOUT = 2000 # milliseconds + + def __init__(self, path): + apsw.Connection.__init__(self, path) + + self.setbusytimeout(self.BUSY_TIMEOUT) + self.execute('pragma cache_size=5000') + self.conn.execute('pragma temp_store=2') + + encoding = self.execute('pragma encoding').fetchone()[0] + self.conn.create_collation('PYNOCASE', partial(pynocase, + encoding=encoding)) + + self.conn.create_function('title_sort', 1, title_sort) + self.conn.create_function('author_to_author_sort', 1, + _author_to_author_sort) + + self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4())) + + # Dummy functions for dynamically created filters + self.conn.create_function('books_list_filter', 1, lambda x: 1) + self.conn.create_collation('icucollate', icu_collator) + + def create_dynamic_filter(self, name): + f = DynamicFilter(name) + self.conn.create_function(name, 1, f) + + def get(self, *args, **kw): + ans = self.cursor().execute(*args) + if kw.get('all', True): + return ans.fetchall() + for row in ans: + return ans[0] + + def execute(self, sql, bindings=None): + cursor = self.cursor() + return cursor.execute(sql, bindings) + + def executemany(self, sql, sequence_of_bindings): + return self.cursor().executemany(sql, sequence_of_bindings) + + def executescript(self, sql): + with self: + # Use an explicit savepoint so that even if this is called + # while a transaction is active, it is atomic + return self.cursor().execute(sql) +# }}} + +class DB(object, SchemaUpgrade): + + PATH_LIMIT = 40 if iswindows else 100 + WINDOWS_LIBRARY_PATH_LIMIT = 75 + + # Initialize database {{{ + + def __init__(self, library_path, default_prefs=None, read_only=False): + try: + if isbytestring(library_path): + library_path = library_path.decode(filesystem_encoding) + except: + import traceback + traceback.print_exc() + + self.field_metadata = FieldMetadata() + + self.library_path = os.path.abspath(library_path) + self.dbpath = os.path.join(library_path, 'metadata.db') + self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH', + self.dbpath) + + if iswindows and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259: + raise ValueError(_( + 'Path to library too long. Must be less than' + ' %d characters.')%(259-4*self.PATH_LIMIT-10)) + exists = self._exists = os.path.exists(self.dbpath) + if not exists: + # Be more strict when creating new libraries as the old calculation + # allowed for max path lengths of 265 chars. + if (iswindows and len(self.library_path) > + self.WINDOWS_LIBRARY_PATH_LIMIT): + raise ValueError(_( + 'Path to library too long. Must be less than' + ' %d characters.')%self.WINDOWS_LIBRARY_PATH_LIMIT) + + if read_only and os.path.exists(self.dbpath): + # Work on only a copy of metadata.db to ensure that + # metadata.db is not changed + pt = PersistentTemporaryFile('_metadata_ro.db') + pt.close() + shutil.copyfile(self.dbpath, pt.name) + self.dbpath = pt.name + + self.is_case_sensitive = (not iswindows and + not os.path.exists(self.dbpath.replace('metadata.db', + 'MeTAdAtA.dB'))) + + self._conn = None + + if self.user_version == 0: + self.initialize_database() + + with self.conn: + SchemaUpgrade.__init__(self) + + # Guarantee that the library_id is set + self.library_id + + self.initialize_prefs(default_prefs) + + # Fix legacy triggers and columns + self.conn.executescript(''' + DROP TRIGGER IF EXISTS author_insert_trg; + CREATE TEMP TRIGGER author_insert_trg + AFTER INSERT ON authors + BEGIN + UPDATE authors SET sort=author_to_author_sort(NEW.name) WHERE id=NEW.id; + END; + DROP TRIGGER IF EXISTS author_update_trg; + CREATE TEMP TRIGGER author_update_trg + BEFORE UPDATE ON authors + BEGIN + UPDATE authors SET sort=author_to_author_sort(NEW.name) + WHERE id=NEW.id AND name <> NEW.name; + END; + UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL; + ''') + + def initialize_prefs(self, default_prefs): + self.prefs = DBPrefs(self) + + if default_prefs is not None and not self._exists: + # Only apply default prefs to a new database + for key in default_prefs: + # be sure that prefs not to be copied are listed below + if key not in frozenset(['news_to_be_synced']): + self.prefs[key] = default_prefs[key] + if 'field_metadata' in default_prefs: + fmvals = [f for f in default_prefs['field_metadata'].values() + if f['is_custom']] + for f in fmvals: + self.create_custom_column(f['label'], f['name'], + f['datatype'], f['is_multiple'] is not None, + f['is_editable'], f['display']) + + defs = self.prefs.defaults + defs['gui_restriction'] = defs['cs_restriction'] = '' + defs['categories_using_hierarchy'] = [] + defs['column_color_rules'] = [] + + # Migrate the bool tristate tweak + defs['bools_are_tristate'] = \ + tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes' + if self.prefs.get('bools_are_tristate') is None: + self.prefs.set('bools_are_tristate', defs['bools_are_tristate']) + + # Migrate column coloring rules + if self.prefs.get('column_color_name_1', None) is not None: + from calibre.library.coloring import migrate_old_rule + old_rules = [] + for i in range(1, 6): + col = self.prefs.get('column_color_name_'+str(i), None) + templ = self.prefs.get('column_color_template_'+str(i), None) + if col and templ: + try: + del self.prefs['column_color_name_'+str(i)] + rules = migrate_old_rule(self.field_metadata, templ) + for templ in rules: + old_rules.append((col, templ)) + except: + pass + if old_rules: + self.prefs['column_color_rules'] += old_rules + + # Migrate saved search and user categories to db preference scheme + def migrate_preference(key, default): + oldval = prefs[key] + if oldval != default: + self.prefs[key] = oldval + prefs[key] = default + if key not in self.prefs: + self.prefs[key] = default + + migrate_preference('user_categories', {}) + migrate_preference('saved_searches', {}) + + # migrate grouped_search_terms + if self.prefs.get('grouped_search_terms', None) is None: + try: + ogst = tweaks.get('grouped_search_terms', {}) + ngst = {} + for t in ogst: + ngst[icu_lower(t)] = ogst[t] + self.prefs.set('grouped_search_terms', ngst) + except: + pass + + # Rename any user categories with names that differ only in case + user_cats = self.prefs.get('user_categories', []) + catmap = {} + for uc in user_cats: + ucl = icu_lower(uc) + if ucl not in catmap: + catmap[ucl] = [] + catmap[ucl].append(uc) + cats_changed = False + for uc in catmap: + if len(catmap[uc]) > 1: + prints('found user category case overlap', catmap[uc]) + cat = catmap[uc][0] + suffix = 1 + while icu_lower((cat + unicode(suffix))) in catmap: + suffix += 1 + prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix))) + user_cats[cat + unicode(suffix)] = user_cats[cat] + del user_cats[cat] + cats_changed = True + if cats_changed: + self.prefs.set('user_categories', user_cats) + + @property + def conn(self): + if self._conn is None: + self._conn = apsw.Connection(self.dbpath) + if self._exists and self.user_version == 0: + self._conn.close() + os.remove(self.dbpath) + self._conn = apsw.Connection(self.dbpath) + return self._conn + + @dynamic_property + def user_version(self): + doc = 'The user version of this database' + + def fget(self): + return self.conn.get('pragma user_version;', all=False) + + def fset(self, val): + self.conn.execute('pragma user_version=%d'%int(val)) + + return property(doc=doc, fget=fget, fset=fset) + + def initialize_database(self): + metadata_sqlite = P('metadata_sqlite.sql', data=True, + allow_user_override=False).decode('utf-8') + self.conn.executescript(metadata_sqlite) + if self.user_version == 0: + self.user_version = 1 + # }}} + + # Database layer API {{{ + + @classmethod + def exists_at(cls, path): + return path and os.path.exists(os.path.join(path, 'metadata.db')) + + @dynamic_property + def library_id(self): + doc = ('The UUID for this library. As long as the user only operates' + ' on libraries with calibre, it will be unique') + + def fget(self): + if getattr(self, '_library_id_', None) is None: + ans = self.conn.get('SELECT uuid FROM library_id', all=False) + if ans is None: + ans = str(uuid.uuid4()) + self.library_id = ans + else: + self._library_id_ = ans + return self._library_id_ + + def fset(self, val): + self._library_id_ = unicode(val) + self.conn.execute(''' + DELETE FROM library_id; + INSERT INTO library_id (uuid) VALUES (?); + ''', self._library_id_) + + return property(doc=doc, fget=fget, fset=fset) + + def last_modified(self): + ''' Return last modified time as a UTC datetime object ''' + return utcfromtimestamp(os.stat(self.dbpath).st_mtime) + + def read_tables(self): + tables = {} + for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', + 'timestamp', 'published', 'uuid', 'path', 'cover', + 'last_modified'): + metadata = self.field_metadata[col].copy() + if metadata['table'] is None: + metadata['table'], metadata['column'] == 'books', ('has_cover' + if col == 'cover' else col) + tables[col] = OneToOneTable(col, metadata) + + for col in ('series', 'publisher', 'rating'): + tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) + + for col in ('authors', 'tags', 'formats', 'identifiers'): + cls = { + 'authors':AuthorsTable, + 'formats':FormatsTable, + 'identifiers':IdentifiersTable, + }.get(col, ManyToManyTable) + tables[col] = cls(col, self.field_metadata[col].copy()) + + tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) + + with self.conn: # Use a single transaction, to ensure nothing modifies + # the db while we are reading + for table in tables.itervalues(): + try: + table.read() + except: + prints('Failed to read table:', table.name) + raise + + return tables + + # }}} + diff --git a/src/calibre/db/errors.py b/src/calibre/db/errors.py new file mode 100644 index 0000000000..d2657f9904 --- /dev/null +++ b/src/calibre/db/errors.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +class NoSuchFormat(ValueError): + pass + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py new file mode 100644 index 0000000000..7240b3ec6e --- /dev/null +++ b/src/calibre/db/tables.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from datetime import datetime + +from dateutil.tz import tzoffset + +from calibre.constants import plugins +from calibre.utils.date import parse_date, local_tz +from calibre.ebooks.metadata import author_to_author_sort + +_c_speedup = plugins['speedup'][0] + +def _c_convert_timestamp(val): + if not val: + return None + try: + ret = _c_speedup.parse_date(val.strip()) + except: + ret = None + if ret is None: + return parse_date(val, as_utc=False) + year, month, day, hour, minutes, seconds, tzsecs = ret + return datetime(year, month, day, hour, minutes, seconds, + tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz) + +class Table(object): + + def __init__(self, name, metadata): + self.name, self.metadata = name, metadata + + # self.adapt() maps values from the db to python objects + self.adapt = \ + { + 'datetime': _c_convert_timestamp, + 'bool': bool + }.get( + metadata['datatype'], lambda x: x) + if name == 'authors': + # Legacy + self.adapt = lambda x: x.replace('|', ',') if x else None + +class OneToOneTable(Table): + + def read(self, db): + self.book_col_map = {} + idcol = 'id' if self.metadata['table'] == 'books' else 'book' + for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, + self.metadata['column'], self.metadata['table'])): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class SizeTable(OneToOneTable): + + def read(self, db): + self.book_col_map = {} + for row in db.conn.execute( + 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' + 'WHERE data.book=books.id) FROM books'): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class ManyToOneTable(Table): + + def read(self, db): + self.id_map = {} + self.extra_map = {} + self.col_book_map = {} + self.book_col_map = {} + self.read_id_maps(db) + self.read_maps(db) + + def read_id_maps(self, db): + for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( + self.metadata['name'], self.metadata['table'])): + if row[1]: + self.id_map[row[0]] = self.adapt(row[1]) + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + self.book_col_map[row[0]] = row[1] + +class ManyToManyTable(ManyToOneTable): + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append(row[1]) + +class AuthorsTable(ManyToManyTable): + + def read_id_maps(self, db): + for row in db.conn.execute( + 'SELECT id, name, sort FROM authors'): + self.id_map[row[0]] = row[1] + self.extra_map[row[0]] = (row[2] if row[2] else + author_to_author_sort(row[1])) + +class FormatsTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, format, name FROM data'): + if row[1] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) + +class IdentifiersTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, type, val FROM identifiers'): + if row[1] is not None and row[2] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) + diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index dea5844028..cc531b7476 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -107,6 +107,7 @@ class DriverBase(DeviceConfig, DevicePlugin): # Needed for config_widget to work FORMATS = ['epub', 'pdf'] USER_CAN_ADD_NEW_FORMATS = False + KEEP_TEMP_FILES_AFTER_UPLOAD = True # Hide the standard customization widgets SUPPORTS_SUB_DIRS = False diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index b265331ace..925f9eafd0 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -327,12 +327,7 @@ class DevicePlugin(Plugin): free space on the device. The text of the FreeSpaceError must contain the word "card" if ``on_card`` is not None otherwise it must contain the word "memory". - :param files: A list of paths and/or file-like objects. If they are paths and - the paths point to temporary files, they may have an additional - attribute, original_file_path pointing to the originals. They may have - another optional attribute, deleted_after_upload which if True means - that the file pointed to by original_file_path will be deleted after - being uploaded to the device. + :param files: A list of paths :param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) :param metadata: If not None, it is a list of :class:`Metadata` objects. diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 04fb3c37b0..650e965941 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -100,7 +100,7 @@ class KOBO(USBMS): for idx,b in enumerate(bl): bl_cache[b.lpath] = idx - def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType): + def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired, favouritesindex): changed = False try: lpath = path.partition(self.normalize_path(prefix))[2] @@ -111,12 +111,23 @@ class KOBO(USBMS): playlist_map = {} + if lpath not in playlist_map: + playlist_map[lpath] = [] + if readstatus == 1: - playlist_map[lpath]= "Im_Reading" + playlist_map[lpath].append('Im_Reading') elif readstatus == 2: - playlist_map[lpath]= "Read" + playlist_map[lpath].append('Read') elif readstatus == 3: - playlist_map[lpath]= "Closed" + playlist_map[lpath].append('Closed') + + # Related to a bug in the Kobo firmware that leaves an expired row for deleted books + # this shows an expired Collection so the user can decide to delete the book + if expired == 3: + playlist_map[lpath].append('Expired') + # Favourites are supported on the touch but the data field is there on most earlier models + if favouritesindex == 1: + playlist_map[lpath].append('Favourite') path = self.normalize_path(path) # print "Normalized FileName: " + path @@ -126,7 +137,13 @@ class KOBO(USBMS): bl_cache[lpath] = None if ImageID is not None: imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed') + if not os.path.exists(imagename): + # Try the Touch version if the image does not exist + imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed') + #print "Image name Normalized: " + imagename + if not os.path.exists(imagename): + debug_print("Strange - The image name does not exist - title: ", title) if imagename is not None: bl[idx].thumbnail = ImageWrapper(imagename) if (ContentType != '6' and MimeType != 'Shortcover'): @@ -138,7 +155,7 @@ class KOBO(USBMS): debug_print(" Strange: The file: ", prefix, lpath, " does mot exist!") if lpath in playlist_map and \ playlist_map[lpath] not in bl[idx].device_collections: - bl[idx].device_collections.append(playlist_map[lpath]) + bl[idx].device_collections = playlist_map.get(lpath,[]) else: if ContentType == '6' and MimeType == 'Shortcover': book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) @@ -157,7 +174,7 @@ class KOBO(USBMS): raise # print 'Update booklist' - book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else [] + book.device_collections = playlist_map.get(lpath,[])# if lpath in playlist_map else [] if bl.add_book(book, replace_metadata=False): changed = True @@ -186,24 +203,30 @@ class KOBO(USBMS): result = cursor.fetchone() self.dbversion = result[0] - query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ - 'ImageID, ReadStatus from content where BookID is Null' + if self.dbversion >= 14: + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex from content where BookID is Null' + else: + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex from content where BookID is Null' cursor.execute (query) changed = False for i, row in enumerate(cursor): # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) - + if row[3].startswith("file:///usr/local/Kobo/help/"): + # These are internal to the Kobo device and do not exist + continue path = self.path_from_contentid(row[3], row[5], row[4], oncard) mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip' # debug_print("mime:", mime) if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4]) + changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9]) # print "shortbook: " + path elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"): - changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4]) + changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8], row[9]) if changed: need_sync = True @@ -267,8 +290,12 @@ class KOBO(USBMS): cursor.execute('delete from content_keys where volumeid = ?', t) # Delete the chapters associated with the book next - t = (ContentID,ContentID,) - cursor.execute('delete from content where BookID = ? or ContentID = ?', t) + t = (ContentID,) + # Kobo does not delete the Book row (ie the row where the BookID is Null) + # The next server sync should remove the row + cursor.execute('delete from content where BookID = ?', t) + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \ + 'where BookID is Null and ContentID =?',t) connection.commit() @@ -286,7 +313,7 @@ class KOBO(USBMS): path_prefix = '.kobo/images/' path = self._main_prefix + path_prefix + ImageID - file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',) + file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed',) for ending in file_endings: fpath = path + ending @@ -450,7 +477,10 @@ class KOBO(USBMS): path = self._main_prefix + path + '.kobo' # print "Path: " + path elif (ContentType == "6" or ContentType == "10") and MimeType == 'application/x-kobo-epub+zip': - path = self._main_prefix + '.kobo/kepub/' + path + if path.startswith("file:///mnt/onboard/"): + path = self._main_prefix + path.replace("file:///mnt/onboard/", '') + else: + path = self._main_prefix + '.kobo/kepub/' + path # print "Internal: " + path else: # if path.startswith("file:///mnt/onboard/"): diff --git a/src/calibre/ebooks/epub/fix/epubcheck.py b/src/calibre/ebooks/epub/fix/epubcheck.py index 81f4ce4d80..9e812e1cf4 100644 --- a/src/calibre/ebooks/epub/fix/epubcheck.py +++ b/src/calibre/ebooks/epub/fix/epubcheck.py @@ -26,6 +26,10 @@ class Epubcheck(ePubFixer): 'significant changes to your epub, complain to the epubcheck ' 'project.') + @property + def description(self): + return self.long_description + @property def fix_name(self): return 'epubcheck' diff --git a/src/calibre/ebooks/epub/fix/unmanifested.py b/src/calibre/ebooks/epub/fix/unmanifested.py index da7a9a9d0e..98fdd4615f 100644 --- a/src/calibre/ebooks/epub/fix/unmanifested.py +++ b/src/calibre/ebooks/epub/fix/unmanifested.py @@ -22,6 +22,10 @@ class Unmanifested(ePubFixer): 'the manifest or delete them as specified by the ' 'delete unmanifested option.') + @property + def description(self): + return self.long_description + @property def fix_name(self): return 'unmanifested' diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index c335cc4c13..ca8707258b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre import prints, isbytestring from calibre.constants import filesystem_encoding - +from calibre.db.errors import NoSuchFormat def debug(*args): prints(*args) @@ -201,27 +201,35 @@ class SaveWorker(Thread): self.spare_server = spare_server self.start() - def collect_data(self, ids): + def collect_data(self, ids, tdir): from calibre.ebooks.metadata.opf2 import metadata_to_opf data = {} for i in set(ids): - mi = self.db.get_metadata(i, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(i, index_is_id=True, get_cover=True, + cover_as_data=True) opf = metadata_to_opf(mi) if isbytestring(opf): opf = opf.decode('utf-8') cpath = None - if mi.cover: - cpath = mi.cover + if mi.cover_data and mi.cover_data[1]: + cpath = os.path.join(tdir, 'cover_%s.jpg'%i) + with lopen(cpath, 'wb') as f: + f.write(mi.cover_data[1]) if isbytestring(cpath): cpath = cpath.decode(filesystem_encoding) formats = {} if mi.formats: for fmt in mi.formats: - fpath = self.db.format_abspath(i, fmt, index_is_id=True) - if fpath is not None: - if isbytestring(fpath): - fpath = fpath.decode(filesystem_encoding) - formats[fmt.lower()] = fpath + fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower())) + with lopen(fpath, 'wb') as f: + try: + self.db.copy_format_to(i, fmt, f, index_is_id=True) + except NoSuchFormat: + continue + else: + if isbytestring(fpath): + fpath = fpath.decode(filesystem_encoding) + formats[fmt.lower()] = fpath data[i] = [opf, cpath, formats, mi.last_modified.isoformat()] return data @@ -244,7 +252,7 @@ class SaveWorker(Thread): for i, task in enumerate(tasks): tids = [x[-1] for x in task] - data = self.collect_data(tids) + data = self.collect_data(tids, tdir) dpath = os.path.join(tdir, '%d.json'%i) with open(dpath, 'wb') as f: f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 7190d1486f..97880faaa1 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -53,13 +53,18 @@ class Worker(Thread): # {{{ from calibre.library.database2 import LibraryDatabase2 newdb = LibraryDatabase2(self.loc) for i, x in enumerate(self.ids): - mi = self.db.get_metadata(x, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(x, index_is_id=True, get_cover=True, + cover_as_data=True) self.progress(i, mi.title) fmts = self.db.formats(x, index_is_id=True) if not fmts: fmts = [] else: fmts = fmts.split(',') - paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in - fmts] + paths = [] + for fmt in fmts: + p = self.db.format(x, fmt, index_is_id=True, + as_path=True) + if p: + paths.append(p) added = False if prefs['add_formats_to_existing']: identical_book_list = newdb.find_identical_books(mi) @@ -75,6 +80,11 @@ class Worker(Thread): # {{{ if co is not None: newdb.set_conversion_options(x, 'PIPE', co) self.processed.add(x) + for path in paths: + try: + os.remove(path) + except: + pass # }}} class CopyToLibraryAction(InterfaceAction): diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 650947100e..718ece46d2 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key +from calibre.db.errors import NoSuchFormat class EditMetadataAction(InterfaceAction): @@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction): +'

', 'merge_too_many_books', self.gui): return - dest_id, src_books, src_ids = self.books_to_merge(rows) + dest_id, src_ids = self.books_to_merge(rows) title = self.gui.library_view.model().db.title(dest_id, index_is_id=True) if safe_merge: if not confirm('

'+_( @@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction): 'Please confirm you want to proceed.')%title +'

', 'merge_books_safe', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) elif merge_only_formats: if not confirm('

'+_( @@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_only_formats', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.delete_books_after_merge(src_ids) else: if not confirm('

'+_( @@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_books', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) self.delete_books_after_merge(src_ids) # leave the selection highlight on first selected book @@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction): self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True, notify=False, replace=replace) + def formats_for_books(self, rows): + m = self.gui.library_view.model() + ans = [] + for id_ in map(m.id, rows): + dbfmts = m.db.formats(id_, index_is_id=True) + if dbfmts: + for fmt in dbfmts.split(','): + try: + path = m.db.format(id_, fmt, index_is_id=True, + as_path=True) + ans.append(path) + except NoSuchFormat: + continue + return ans + def books_to_merge(self, rows): - src_books = [] src_ids = [] m = self.gui.library_view.model() for i, row in enumerate(rows): @@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction): dest_id = id_ else: src_ids.append(id_) - dbfmts = m.db.formats(id_, index_is_id=True) - if dbfmts: - for fmt in dbfmts.split(','): - src_books.append(m.db.format_abspath(id_, fmt, - index_is_id=True)) - return [dest_id, src_books, src_ids] + return [dest_id, src_ids] def delete_books_after_merge(self, ids_to_delete): self.gui.library_view.model().delete_books_by_id(ids_to_delete) def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db - dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True) + dest_mi = db.get_metadata(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments + dest_cover = db.cover(dest_id, index_is_id=True) + had_orig_cover = bool(dest_cover) for src_id in src_ids: - src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True) + src_mi = db.get_metadata(src_id, index_is_id=True) if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction): dest_mi.tags = src_mi.tags else: dest_mi.tags.extend(src_mi.tags) - if src_mi.cover and not dest_mi.cover: - dest_mi.cover = src_mi.cover + if not dest_cover: + src_cover = db.cover(src_id, index_is_id=True) + if src_cover: + dest_cover = src_cover if not dest_mi.publisher: dest_mi.publisher = src_mi.publisher if not dest_mi.rating: @@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction): dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index db.set_metadata(dest_id, dest_mi, ignore_errors=False) + if not had_orig_cover and dest_cover: + db.set_cover(dest_id, dest_cover) for key in db.field_metadata: #loop thru all defined fields if db.field_metadata[key]['is_custom']: diff --git a/src/calibre/gui2/actions/show_quickview.py b/src/calibre/gui2/actions/show_quickview.py new file mode 100644 index 0000000000..61a41b08ae --- /dev/null +++ b/src/calibre/gui2/actions/show_quickview.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +from calibre.gui2.actions import InterfaceAction +from calibre.gui2.dialogs.quickview import Quickview +from calibre.gui2 import error_dialog + +class ShowQuickviewAction(InterfaceAction): + + name = 'Show quickview' + action_spec = (_('Show quickview'), 'user_profile.png', None, + _('Q')) + dont_add_to = frozenset(['menubar-device', 'toolbar-device', 'context-menu-device']) + action_type = 'current' + + current_instance = None + + def genesis(self): + self.qaction.triggered.connect(self.show_quickview) + + def show_quickview(self, *args): + if self.current_instance: + if not self.current_instance.is_closed: + return + self.current_instance = None + if self.gui.current_view() is not self.gui.library_view: + error_dialog(self.gui, _('No quickview available'), + _('Quickview is not available for books ' + 'on the device.')).exec_() + return + index = self.gui.library_view.currentIndex() + if index.isValid(): + self.current_instance = \ + Quickview(self.gui, self.gui.library_view, index) + self.current_instance.show() + diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py index c9f2d7a8c6..d3924e7cd3 100755 --- a/src/calibre/gui2/actions/tweak_epub.py +++ b/src/calibre/gui2/actions/tweak_epub.py @@ -5,6 +5,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os + from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.tweak_epub import TweakEpub @@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction): # Confirm 'EPUB' in formats book_id = self.gui.library_view.model().id(row) try: - path_to_epub = self.gui.library_view.model().db.format_abspath( - book_id, 'EPUB', index_is_id=True) + path_to_epub = self.gui.library_view.model().db.format( + book_id, 'EPUB', index_is_id=True, as_path=True) except: path_to_epub = None @@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction): if dlg.exec_() == dlg.Accepted: self.update_db(book_id, dlg._output) dlg.cleanup() + os.remove(path_to_epub) def update_db(self, book_id, rebuilt): ''' diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 44b5bb446b..3dbf4b94df 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -445,6 +445,7 @@ class Saver(QObject): # {{{ self.pd.setModal(True) self.pd.show() self.pd.set_min(0) + self.pd.set_msg(_('Collecting data, please wait...')) self._parent = parent self.callback = callback self.callback_called = False diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index bf32bf472a..f79e6df3fe 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import re +import re, os from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \ @@ -134,7 +134,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): _('Cannot build regex using the GUI builder without a book.'), show=True) return False - self.open_book(db.format_abspath(book_id, format, index_is_id=True)) + fpath = db.format(book_id, format, index_is_id=True, + as_path=True) + try: + self.open_book(fpath) + finally: + os.remove(fpath) return True def open_book(self, pathtoebook): diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index 3575fb5ffb..15e670ee32 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -106,7 +106,6 @@ class Config(ResizableDialog, Ui_Dialog): Configuration dialog for single book conversion. If accepted, has the following important attributes - input_path - Path to input file output_format - Output format (without a leading .) input_format - Input format (without a leading .) opf_path - Path to OPF file with user specified metadata @@ -156,13 +155,10 @@ class Config(ResizableDialog, Ui_Dialog): oidx = self.groups.currentIndex().row() input_format = self.input_format output_format = self.output_format - input_path = self.db.format_abspath(self.book_id, input_format, - index_is_id=True) - self.input_path = input_path output_path = 'dummy.'+output_format log = Log() log.outputs = [] - self.plumber = Plumber(input_path, output_path, log) + self.plumber = Plumber('dummy.'+input_format, output_path, log) def widget_factory(cls): return cls(self.stack, self.plumber.get_option_by_name, diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 99f795cdda..d5805a8e09 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -49,16 +49,26 @@ class DeviceJob(BaseJob): # {{{ self._aborted = False def start_work(self): + if DEBUG: + prints('Job:', self.id, self.description, 'started', + safe_encode=True) self.start_time = time.time() self.job_manager.changed_queue.put(self) def job_done(self): self.duration = time.time() - self.start_time self.percent = 1 + if DEBUG: + prints('DeviceJob:', self.id, self.description, + 'done, calling callback', safe_encode=True) + try: self.callback_on_done(self) except: pass + if DEBUG: + prints('DeviceJob:', self.id, self.description, + 'callback returned', safe_encode=True) self.job_manager.changed_queue.put(self) def report_progress(self, percent, msg=''): @@ -386,8 +396,17 @@ class DeviceManager(Thread): # {{{ if DEBUG: prints(traceback.format_exc(), file=sys.__stdout__) - return self.device.upload_books(files, names, on_card, - metadata=metadata, end_session=False) + try: + return self.device.upload_books(files, names, on_card, + metadata=metadata, end_session=False) + finally: + if metadata: + for mi in metadata: + try: + if mi.cover: + os.remove(mi.cover) + except: + pass def upload_books(self, done, files, names, on_card=None, titles=None, metadata=None, plugboards=None, add_as_step_to_job=None): @@ -1062,8 +1081,6 @@ class DeviceMixin(object): # {{{ 'the device?'), autos): self.iactions['Convert Books'].auto_convert_news(auto, format) files = [f for f in files if f is not None] - for f in files: - f.deleted_after_upload = del_on_upload if not files: self.news_to_be_synced = set([]) return @@ -1305,8 +1322,17 @@ class DeviceMixin(object): # {{{ self.card_b_view if on_card == 'cardb' else self.memory_view view.model().resort(reset=False) view.model().research() - for f in files: - getattr(f, 'close', lambda : True)() + if files: + for f in files: + # Remove temporary files + try: + rem = not getattr( + self.device_manager.device, + 'KEEP_TEMP_FILES_AFTER_UPLOAD', False) + if rem and 'caltmpfmt.' in f: + os.remove(f) + except: + pass def book_on_device(self, id, reset=False): ''' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 8829dc97c0..7c7c78629c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt -def get_cover_data(path): # {{{ +def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: @@ -32,8 +32,8 @@ def get_cover_data(path): # {{{ cdata = area = None try: - mi = get_metadata(open(path, 'rb'), - os.path.splitext(path)[1][1:].lower()) + with stream: + mi = get_metadata(stream, ext) if mi.cover and os.access(mi.cover, os.R_OK): cdata = open(mi.cover).read() elif mi.cover_data[1] is not None: @@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{ if fmts: covers = [] for fmt in fmts.split(','): - fmt = self.db.format_abspath(id, fmt, index_is_id=True) - if not fmt: continue - cdata, area = get_cover_data(fmt) + fmtf = self.db.format(id, fmt, index_is_id=True, + as_file=True) + if fmtf is None: continue + cdata, area = get_cover_data(fmtf, fmt) if cdata: covers.append((cdata, area)) covers.sort(key=lambda x: x[1]) @@ -361,7 +362,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): fm = self.db.field_metadata for f in fm: if (f in ['author_sort'] or - (fm[f]['datatype'] in ['text', 'series', 'enumeration'] + (fm[f]['datatype'] in ['text', 'series', 'enumeration', 'comments'] and fm[f].get('search_terms', None) and f not in ['formats', 'ondevice']) or (fm[f]['datatype'] in ['int', 'float', 'bool'] and diff --git a/src/calibre/gui2/dialogs/quickview.py b/src/calibre/gui2/dialogs/quickview.py new file mode 100644 index 0000000000..5dcdbf04fa --- /dev/null +++ b/src/calibre/gui2/dialogs/quickview.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem, + QListWidgetItem, QByteArray, QModelIndex) + +from calibre.gui2.dialogs.quickview_ui import Ui_Quickview +from calibre.utils.icu import sort_key +from calibre.gui2 import gprefs + +class tableItem(QTableWidgetItem): + + def __init__(self, val): + QTableWidgetItem.__init__(self, val) + self.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable) + + def __ge__(self, other): + return sort_key(unicode(self.text())) >= sort_key(unicode(other.text())) + + def __lt__(self, other): + return sort_key(unicode(self.text())) < sort_key(unicode(other.text())) + +class Quickview(QDialog, Ui_Quickview): + + def __init__(self, gui, view, row): + QDialog.__init__(self, gui, flags=Qt.Window) + Ui_Quickview.__init__(self) + self.setupUi(self) + self.isClosed = False + + try: + geom = gprefs.get('quickview_dialog_geometry', bytearray('')) + self.restoreGeometry(QByteArray(geom)) + except: + pass + + icon = self.windowIcon() + self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint)) + self.setWindowIcon(icon) + + self.db = view.model().db + self.view = view + self.gui = gui + + self.items.setSelectionMode(QAbstractItemView.SingleSelection) + self.items.currentTextChanged.connect(self.item_selected) +# self.items.setFixedWidth(150) + + self.books_table.setSelectionBehavior(QAbstractItemView.SelectRows) + self.books_table.setSelectionMode(QAbstractItemView.SingleSelection) + self.books_table.setColumnCount(3) + t = QTableWidgetItem(_('Title')) + self.books_table.setHorizontalHeaderItem(0, t) + t = QTableWidgetItem(_('Authors')) + self.books_table.setHorizontalHeaderItem(1, t) + t = QTableWidgetItem(_('Series')) + self.books_table.setHorizontalHeaderItem(2, t) + self.books_table_header_height = self.books_table.height() + self.books_table.cellDoubleClicked.connect(self.book_doubleclicked) + + self.is_closed = False + self.current_book_id = None + self.current_key = None + self.use_current_key_for_next_refresh = False + self.last_search = None + + self.refresh(row) +# self.connect(self.view.selectionModel(), SIGNAL('currentChanged(QModelIndex,QModelIndex)'), self.slave) + self.view.selectionModel().currentChanged[QModelIndex,QModelIndex].connect(self.slave) + self.search_button.clicked.connect(self.do_search) + + def do_search(self): + if self.last_search is not None: + self.use_current_key_for_next_refresh = True + self.gui.search.set_search_string(self.last_search) + + def item_selected(self, txt): + self.fill_in_books_box(unicode(txt)) + + def refresh(self, idx): + bv_row = idx.row() + key = self.view.model().column_map[idx.column()] + + book_id = self.view.model().id(bv_row) + + if self.use_current_key_for_next_refresh: + key = self.current_key + self.use_current_key_for_next_refresh = False + else: + if not self.db.field_metadata[key]['is_category']: + if self.current_key is None: + return + key = self.current_key + self.items_label.setText('{0} ({1})'.format( + self.db.field_metadata[key]['name'], key)) + self.items.clear() + self.books_table.setRowCount(0) + + mi = self.db.get_metadata(book_id, index_is_id=True, get_user_categories=False) + vals = mi.get(key, None) + if not vals: + return + + if not isinstance(vals, list): + vals = [vals] + vals.sort(key=sort_key) + + self.items.blockSignals(True) + for v in vals: + a = QListWidgetItem(v) + self.items.addItem(a) + self.items.setCurrentRow(0) + self.items.blockSignals(False) + + self.current_book_id = book_id + self.current_key = key + + self.fill_in_books_box(vals[0]) + + def fill_in_books_box(self, selected_item): + if selected_item.startswith('.'): + sv = '.' + selected_item + else: + sv = selected_item + sv = sv.replace('"', r'\"') + self.last_search = self.current_key+':"=' + sv + '"' + books = self.db.search_getting_ids(self.last_search, + self.db.data.search_restriction) + self.books_table.setRowCount(len(books)) + self.books_label.setText(_('Books with selected item: {0}').format(len(books))) + + select_row = None + self.books_table.setSortingEnabled(False) + for row, b in enumerate(books): + mi = self.db.get_metadata(b, index_is_id=True, get_user_categories=False) + a = tableItem(mi.title) + a.setData(Qt.UserRole, b) + self.books_table.setItem(row, 0, a) + a = tableItem(' & '.join(mi.authors)) + self.books_table.setItem(row, 1, a) + series = mi.format_field('series')[1] + if series is None: + series = '' + a = tableItem(series) + self.books_table.setItem(row, 2, a) + if b == self.current_book_id: + select_row = row + + self.books_table.resizeColumnsToContents() +# self.books_table.resizeRowsToContents() + + if select_row is not None: + self.books_table.selectRow(select_row) + self.books_table.setSortingEnabled(True) + + def book_doubleclicked(self, row, column): + self.use_current_key_for_next_refresh = True + self.view.select_rows([self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]]) + + def slave(self, current, previous): + self.refresh(current) + self.view.activateWindow() + + def done(self, r): + geom = bytearray(self.saveGeometry()) + gprefs['quickview_dialog_geometry'] = geom + self.is_closed = True + QDialog.done(self, r) diff --git a/src/calibre/gui2/dialogs/quickview.ui b/src/calibre/gui2/dialogs/quickview.ui new file mode 100644 index 0000000000..2cdc7b7379 --- /dev/null +++ b/src/calibre/gui2/dialogs/quickview.ui @@ -0,0 +1,131 @@ + + + Quickview + + + + 0 + 0 + 768 + 342 + + + + + 0 + 0 + + + + Quickview + + + + + + Items + + + + + + + + 1 + 0 + + + + + + + + + + + + + 4 + 0 + + + + 0 + + + 0 + + + + + + + Qt::Vertical + + + + 0 + 0 + + + + + + + + + + Search + + + Search in the library view for the selected item + + + + + + + Qt::Horizontal + + + + 0 + 0 + + + + + + + + QDialogButtonBox::Close + + + false + + + + + + + + + + + buttonBox + rejected() + Quickview + reject() + + + 297 + 217 + + + 286 + 234 + + + + + diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index b82f421e1e..b9c760abff 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -174,7 +174,8 @@ class EmailMixin(object): # {{{ else: _auto_ids = [] - full_metadata = self.library_view.model().metadata_for(ids) + full_metadata = self.library_view.model().metadata_for(ids, + get_cover=False) bad, remove_ids, jobnames = [], [], [] texts, subjects, attachments, attachment_names = [], [], [], [] diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index f49c6db59a..40d6e2b6cf 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, functools, re, os, traceback -from contextlib import closing +import functools, re, os, traceback from collections import defaultdict from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, @@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y' ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center': Qt.AlignHCenter} -class FormatPath(unicode): - - def __new__(cls, path, orig_file_path): - ans = unicode.__new__(cls, path) - ans.orig_file_path = orig_file_path - ans.deleted_after_upload = False - return ans - _default_image = None def default_image(): @@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{ data = self.current_changed(index, None, False) return data - def metadata_for(self, ids): + def metadata_for(self, ids, get_cover=True): + ''' + WARNING: if get_cover=True temp files are created for mi.cover. + Remember to delete them once you are done with them. + ''' ans = [] for id in ids: - mi = self.db.get_metadata(id, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover) ans.append(mi) return ans @@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{ format = f break if format is not None: - pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(id, format, index_is_id=True, - as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() - if getattr(src, 'name', None): - pt.orig_file_path = os.path.abspath(src.name) + pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format) + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: try: - _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True), - format) + _set_metadata(pt, self.db.get_metadata( + id, get_cover=True, index_is_id=True, + cover_as_data=True), format) except: traceback.print_exc() pt.close() @@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{ if isbytestring(x): x = x.decode(filesystem_encoding) return x - name, op = map(to_uni, map(os.path.abspath, (pt.name, - pt.orig_file_path))) - ans.append(FormatPath(name, op)) + ans.append(to_uni(os.path.abspath(pt.name))) else: need_auto.append(id) if not exclude_auto: @@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{ break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(row, format, as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: - _set_metadata(pt, self.db.get_metadata(row, get_cover=True), - format) + _set_metadata(pt, self.db.get_metadata(row, get_cover=True, + cover_as_data=True), format) pt.close() if paths else pt.seek(0) ans.append(pt) else: diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 3ca898d15a..9aa926f5c5 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -584,14 +584,15 @@ class BooksView(QTableView): # {{{ m = self.model() db = m.db rows = self.selectionModel().selectedRows() - selected = map(m.id, rows) + selected = list(map(m.id, rows)) ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) fmt = prefs['output_format'] def url_for_id(i): - ans = db.format_abspath(i, fmt, index_is_id=True) + ans = db.format(i, fmt, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: fmts = db.formats(i, index_is_id=True) if fmts: @@ -599,14 +600,13 @@ class BooksView(QTableView): # {{{ else: fmts = [] for f in fmts: - ans = db.format_abspath(i, f, index_is_id=True) - if ans is not None: - break + ans = db.format(i, f, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: ans = db.abspath(i, index_is_id=True) return QUrl.fromLocalFile(ans) - md.setUrls([url_for_id(i) for i in selected]) + md.setUrls([url_for_id(i) for i in selected[:25]]) drag = QDrag(self) col = self.selectionModel().currentIndex().column() md.column_name = self.column_map[col] diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index f865a5c62c..303cc51c74 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -688,7 +688,8 @@ class FormatsManager(QWidget): # {{{ else: stream = open(fmt.path, 'r+b') try: - mi = get_metadata(stream, ext) + with stream: + mi = get_metadata(stream, ext) return mi, ext except: error_dialog(self, _('Could not read metadata'), diff --git a/src/calibre/gui2/store/beam_ebooks_de_plugin.py b/src/calibre/gui2/store/beam_ebooks_de_plugin.py index b589a8c310..6e31a76f8d 100644 --- a/src/calibre/gui2/store/beam_ebooks_de_plugin.py +++ b/src/calibre/gui2/store/beam_ebooks_de_plugin.py @@ -51,19 +51,19 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin): if counter <= 0: break - id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip() + id = ''.join(data.xpath('./tr/td[1]/a/@href')).strip() if not id: continue id = id[7:] cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src')) if cover_url: cover_url = 'http://www.beam-ebooks.de' + cover_url - title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()')) - author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/' - 'child::b/text()' - '|' - './tr/td/div[@class="stil2"]/' - 'child::strong/text()')) + temp = ''.join(data.xpath('./tr/td[1]/a/img/@alt')) + colon = temp.find(':') + if not temp.startswith('eBook') or colon < 0: + continue + author = temp[5:colon] + title = temp[colon+1:] price = ''.join(data.xpath('./tr/td[3]/text()')) pdf = data.xpath( 'boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)') diff --git a/src/calibre/gui2/store/legimi_plugin.py b/src/calibre/gui2/store/legimi_plugin.py index 2f69da24e5..792c7db4a7 100644 --- a/src/calibre/gui2/store/legimi_plugin.py +++ b/src/calibre/gui2/store/legimi_plugin.py @@ -58,6 +58,8 @@ class LegimiStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="item_cover_container"]/a/img/@src')) title = ''.join(data.xpath('.//div[@class="item_entries"]/h2/a/text()')) author = ''.join(data.xpath('.//div[@class="item_entries"]/span[1]/a/text()')) + author = re.sub(',','',author) + author = re.sub(';',',',author) price = ''.join(data.xpath('.//div[@class="item_entries"]/span[3]/text()')) price = re.sub(r'[^0-9,]*','',price) + ' zł' diff --git a/src/calibre/gui2/store/libri_de_plugin.py b/src/calibre/gui2/store/libri_de_plugin.py new file mode 100644 index 0000000000..ed93eeff0e --- /dev/null +++ b/src/calibre/gui2/store/libri_de_plugin.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class LibreDEStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://ad.zanox.com/ppc/?18817073C15644254T' + url_details = ('http://ad.zanox.com/ppc/?18845780C1371495675T&ULP=[[' + 'http://www.libri.de/shop/action/productDetails?artiId={0}]]') + + if external or self.config.get('open_external', False): + if detail_item: + url = url_details.format(detail_item) + open_url(QUrl(url)) + else: + detail_url = None + if detail_item: + detail_url = url_details.format(detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = ('http://www.libri.de/shop/action/quickSearch?facetNodeId=6' + '&mainsearchSubmit=Los!&searchString=' + urllib2.quote(query)) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[contains(@class, "item")]'): + if counter <= 0: + break + + details = data.xpath('./div[@class="beschreibungContainer"]') + if not details: + continue + details = details[0] + id = ''.join(details.xpath('./div[@class="text"]/a/@name')).strip() + if not id: + continue + cover_url = ''.join(details.xpath('./div[@class="bild"]/a/img/@src')) + title = ''.join(details.xpath('./div[@class="text"]/span[@class="titel"]/a/text()')).strip() + author = ''.join(details.xpath('./div[@class="text"]/span[@class="author"]/text()')).strip() + pdf = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "pdf")]/text())') + epub = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "epub")]/text())') + mobi = details.xpath( + 'boolean(.//span[@class="format" and contains(text(), "mobipocket")]/text())') + price = (''.join(data.xpath('.//span[@class="preis"]/text()'))).replace('*', '') + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.drm = SearchResult.DRM_UNKNOWN + s.detail_item = id + formats = [] + if epub: + formats.append('ePub') + if pdf: + formats.append('PDF') + if mobi: + formats.append('MOBI') + s.formats = ', '.join(formats) + + yield s diff --git a/src/calibre/gui2/store/woblink_plugin.py b/src/calibre/gui2/store/woblink_plugin.py index 69be8f2e94..9f2e29e825 100644 --- a/src/calibre/gui2/store/woblink_plugin.py +++ b/src/calibre/gui2/store/woblink_plugin.py @@ -57,7 +57,7 @@ class WoblinkStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/img/@src')) title = ''.join(data.xpath('.//h3[@class="title"]/a[1]/text()')) - author = ''.join(data.xpath('.//p[@class="author"]/a[1]/text()')) + author = ', '.join(data.xpath('.//p[@class="author"]/a/text()')) price = ''.join(data.xpath('.//div[@class="prices"]/p[1]/span/text()')) price = re.sub('PLN', ' zł', price) price = re.sub('\.', ',', price) diff --git a/src/calibre/gui2/store/zixo_plugin.py b/src/calibre/gui2/store/zixo_plugin.py index 419b87137a..b4e54736c0 100644 --- a/src/calibre/gui2/store/zixo_plugin.py +++ b/src/calibre/gui2/store/zixo_plugin.py @@ -53,7 +53,7 @@ class ZixoStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src')) title = ''.join(data.xpath('.//a[@class="title"]/text()')) - author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()')) + author = ','.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()')) price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()')) price = re.sub('\.', ',', price) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 39224c8b35..03b32ca2fb 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ # continue mi = db.get_metadata(book_id, True) - in_file = db.format_abspath(book_id, d.input_format, True) + in_file = PersistentTemporaryFile('.'+d.input_format) + with in_file: + db.copy_format_to(book_id, d.input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + d.output_format) out_file.write(d.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] try: dtitle = unicode(mi.title) @@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ recs.append(('cover', d.cover_file.name, OptionRecommendation.HIGH)) temp_files.append(d.cover_file) - args = [in_file, out_file.name, recs] + args = [in_file.name, out_file.name, recs] temp_files.append(out_file) jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files)) @@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog): try: input_format = get_input_format_for_book(self.db, book_id, None)[0] mi, opf_file = create_opf_file(self.db, book_id) - in_file = self.db.format_abspath(book_id, input_format, True) + in_file = PersistentTemporaryFile('.'+input_format) + with in_file: + self.db.copy_format_to(book_id, input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + self.output_format) out_file.write(self.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] combined_recs = GuiRecommendations() default_recs = bulk_defaults_for_input_format(input_format) @@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog): self.setLabelText(_('Queueing ')+dtitle) desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle) - args = [in_file, out_file.name, lrecs] + args = [in_file.name, out_file.name, lrecs] temp_files.append(out_file) self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files)) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 2e00db32c4..84a7acbc73 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{ print 'Time per record:', t/float(num_of_records) # }}} -def cover_load_timing(path=None): - from PyQt4.Qt import QApplication, QImage - import os, time - app = QApplication([]) - app - d = db(path) - paths = [d.cover(i, index_is_id=True, as_path=True) for i in - d.data.iterallids()] - paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))] - - start = time.time() - - for p in paths: - with open(p, 'rb') as f: - img = QImage() - img.loadFromData(f.read()) - - print 'Average load time:', (time.time() - start)/len(paths), 'seconds' diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 8508fb266f..1b8bb365ab 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5137,6 +5137,7 @@ Author '{0}': OptionRecommendation.HIGH)) # If cover exists, use it + cpath = None try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -5157,5 +5158,10 @@ Author '{0}': plumber.merge_ui_recommendations(recommendations) plumber.run() + try: + os.remove(cpath) + except: + pass + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 67c67b1ff7..4c61438e35 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -12,8 +12,6 @@ import threading, random from itertools import repeat from math import ceil -from PyQt4.QtGui import QImage - from calibre import prints from calibre.ebooks.metadata import (title_sort, author_to_author_sort, string_to_authors, authors_to_string) @@ -27,7 +25,7 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, base_dir from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -39,8 +37,10 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions +from calibre.db.errors import NoSuchFormat copyfile = os.link if hasattr(os, 'link') else shutil.copyfile +SPOOL_SIZE = 30*1024*1024 class Tag(object): @@ -201,16 +201,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): dbprefs = DBPrefs(self) for key in default_prefs: # be sure that prefs not to be copied are listed below - if key in ['news_to_be_synced']: + if key in frozenset(['news_to_be_synced']): continue - try: - dbprefs[key] = default_prefs[key] - except: - pass # ignore options that don't exist anymore - fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']] - for f in fmvals: - self.create_custom_column(f['label'], f['name'], f['datatype'], - f['is_multiple'] is not None, f['is_editable'], f['display']) + dbprefs[key] = default_prefs[key] + if 'field_metadata' in default_prefs: + fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']] + for f in fmvals: + self.create_custom_column(f['label'], f['name'], f['datatype'], + f['is_multiple'] is not None, f['is_editable'], f['display']) self.initialize_dynamic() def get_property(self, idx, index_is_id=False, loc=-1): @@ -603,14 +601,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - # Get data as string (can't use file as source and target files may be the same) - f = self.format(id, format, index_is_id=True, as_file=True) - if f is None: - continue - with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream: - with f: - shutil.copyfileobj(f, stream) - stream.seek(0) + with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + try: + self.copy_format_to(id, format, stream, index_is_id=True) + stream.seek(0) + except NoSuchFormat: + continue self.add_format(id, format, stream, index_is_id=True, path=tpath, notify=False) self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id)) @@ -663,32 +659,53 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): continue def cover(self, index, index_is_id=False, as_file=False, as_image=False, - as_path=False): + as_path=False): ''' Return the cover image as a bytestring (in JPEG format) or None. - `as_file` : If True return the image as an open file object - `as_image`: If True return the image as a QImage object + WARNING: Using as_path will copy the cover to a temp file and return + the path to the temp file. You should delete the temp file when you are + done with it. + + :param as_file: If True return the image as an open file object (a SpooledTemporaryFile) + :param as_image: If True return the image as a QImage object ''' - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.access(path, os.R_OK): - if as_path: - return path try: f = lopen(path, 'rb') except (IOError, OSError): time.sleep(0.2) f = lopen(path, 'rb') - if as_image: - img = QImage() - img.loadFromData(f.read()) - f.close() - return img - ans = f if as_file else f.read() - if ans is not f: - f.close() - return ans + with f: + if as_path: + pt = PersistentTemporaryFile('_dbcover.jpg') + with pt: + shutil.copyfileobj(f, pt) + return pt.name + if as_file: + ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() + if as_image: + from PyQt4.Qt import QImage + i = QImage() + i.loadFromData(ret) + ret = i + return ret + + def cover_last_modified(self, index, index_is_id=False): + id = index if index_is_id else self.id(index) + path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') + try: + return utcfromtimestamp(os.stat(path).st_mtime) + except: + # Cover doesn't exist + pass + return self.last_modified() ### The field-style interface. These use field keys. @@ -861,7 +878,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False, - get_user_categories=True): + get_user_categories=True, cover_as_data=False): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -936,7 +953,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.user_categories = user_cat_vals if get_cover: - mi.cover = self.cover(id, index_is_id=True, as_path=True) + if cover_as_data: + cdata = self.cover(id, index_is_id=True) + if cdata: + mi.cover_data = ('jpeg', cdata) + else: + mi.cover = self.cover(id, index_is_id=True, as_path=True) return mi def has_book(self, mi): @@ -1101,7 +1123,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return utcfromtimestamp(os.stat(path).st_mtime) def format_abspath(self, index, format, index_is_id=False): - 'Return absolute path to the ebook file of format `format`' + ''' + Return absolute path to the ebook file of format `format` + + WARNING: This method will return a dummy path for a network backend DB, + so do not rely on it, use format(..., as_path=True) instead. + + Currently used only in calibredb list, the viewer and the catalogs (via + get_data_as_dict()). + + Apart from the viewer, I don't believe any of the others do any file + I/O with the results of this call. + ''' id = index if index_is_id else self.id(index) try: name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) @@ -1121,25 +1154,63 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfile(candidates[0], fmt_path) return fmt_path - def format(self, index, format, index_is_id=False, as_file=False, mode='r+b'): + def copy_format_to(self, index, fmt, dest, index_is_id=False): + ''' + Copy the format ``fmt`` to the file like object ``dest``. If the + specified format does not exist, raises :class:`NoSuchFormat` error. + ''' + path = self.format_abspath(index, fmt, index_is_id=index_is_id) + if path is None: + id_ = index if index_is_id else self.id(index) + raise NoSuchFormat('Record %d has no %s file'%(id_, fmt)) + with lopen(path, 'rb') as f: + shutil.copyfileobj(f, dest) + if hasattr(dest, 'flush'): + dest.flush() + + def format(self, index, format, index_is_id=False, as_file=False, + mode='r+b', as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. - `as_file`: If True the ebook format is returned as a file object opened in `mode` + :param as_file: If True the ebook format is returned as a file object. Note + that the file object is a SpooledTemporaryFile, so if what you want to + do is copy the format to another file, use :method:`copy_format_to` + instead for performance. + :param as_path: Copies the format file to a temp file and returns the + path to the temp file + :param preserve_filename: If True and returning a path the filename is + the same as that used in the library. Note that using + this means that repeated calls yield the same + temp file (which is re-created each time) + :param mode: This is ignored (present for legacy compatibility) ''' path = self.format_abspath(index, format, index_is_id=index_is_id) if path is not None: - f = lopen(path, mode) - try: - ret = f if as_file else f.read() - except IOError: - f.seek(0) - out = cStringIO.StringIO() - shutil.copyfileobj(f, out) - ret = out.getvalue() - if not as_file: - f.close() + with lopen(path, mode) as f: + if as_path: + if preserve_filename: + bd = base_dir() + d = os.path.join(bd, 'format_abspath') + try: + os.makedirs(d) + except: + pass + fname = os.path.basename(path) + ret = os.path.join(d, fname) + with lopen(ret, 'wb') as f2: + shutil.copyfileobj(f, f2) + else: + with PersistentTemporaryFile('.'+format.lower()) as pt: + shutil.copyfileobj(f, pt) + ret = pt.name + elif as_file: + ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, @@ -1931,13 +2002,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def authors_with_sort_strings(self, id, index_is_id=False): id = id if index_is_id else self.id(id) aut_strings = self.conn.get(''' - SELECT authors.name, authors.sort + SELECT authors.id, authors.name, authors.sort FROM authors, books_authors_link as bl WHERE bl.book=? and authors.id=bl.author ORDER BY bl.id''', (id,)) result = [] - for (author, sort,) in aut_strings: - result.append((author.replace('|', ','), sort)) + for (id_, author, sort,) in aut_strings: + result.append((id_, author.replace('|', ','), sort)) return result # Given a book, return the author_sort string for authors of the book @@ -1945,6 +2016,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): auts = self.authors_sort_strings(id, index_is_id) return ' & '.join(auts).replace('|', ',') + # Given an author, return a list of books with that author + def books_for_author(self, id_, index_is_id=False): + id_ = id_ if index_is_id else self.id(id_) + books = self.conn.get(''' + SELECT bl.book + FROM books_authors_link as bl + WHERE bl.author=?''', (id_,)) + return [b[0] for b in books] + # Given a list of authors, return the author_sort string for the authors, # preferring the author sort associated with the author over the computed # string @@ -1968,7 +2048,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): aum = self.authors_with_sort_strings(id_, index_is_id=True) self.data.set(id_, self.FIELD_MAP['au_map'], - ':#:'.join([':::'.join((au.replace(',', '|'), aus)) for (au, aus) in aum]), + ':#:'.join([':::'.join((au.replace(',', '|'), aus)) for (_, au, aus) in aum]), row_is_id=True) def _set_authors(self, id, authors, allow_case_change=False): diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 5f49833564..b5c4e2faf3 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, shutil +import os, traceback, cStringIO, re from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks @@ -238,7 +238,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, def save_book_to_disk(id_, db, root, opts, length): mi = db.get_metadata(id_, index_is_id=True) - cover = db.cover(id_, index_is_id=True, as_path=True) + cover = db.cover(id_, index_is_id=True) plugboards = db.prefs.get('plugboards', {}) available_formats = db.formats(id_, index_is_id=True) @@ -252,12 +252,20 @@ def save_book_to_disk(id_, db, root, opts, length): if fmts: fmts = fmts.split(',') for fmt in fmts: - fpath = db.format_abspath(id_, fmt, index_is_id=True) + fpath = db.format(id_, fmt, index_is_id=True, as_path=True) if fpath is not None: formats[fmt.lower()] = fpath - return do_save_book_to_disk(id_, mi, cover, plugboards, + try: + return do_save_book_to_disk(id_, mi, cover, plugboards, formats, root, opts, length) + finally: + for temp in formats.itervalues(): + try: + os.remove(temp) + except: + pass + def do_save_book_to_disk(id_, mi, cover, plugboards, @@ -289,10 +297,9 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, raise ocover = mi.cover - if opts.save_cover and cover and os.access(cover, os.R_OK): + if opts.save_cover and cover: with open(base_path+'.jpg', 'wb') as f: - with open(cover, 'rb') as s: - shutil.copyfileobj(s, f) + f.write(cover) mi.cover = base_name+'.jpg' else: mi.cover = None @@ -395,8 +402,13 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): pass tb = '' try: - failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards, - format_map, root, opts, length) + with open(cover, 'rb') as f: + cover = f.read() + except: + cover = None + try: + failed, id, title = do_save_book_to_disk(x, mi, cover, + plugboards, format_map, root, opts, length) tb = _('Requested formats not available') except: failed, id, title = True, x, mi.title diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 08de4faecd..853ec7829c 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -10,12 +10,13 @@ import re, os, posixpath import cherrypy from calibre import fit_image, guess_type -from calibre.utils.date import fromtimestamp +from calibre.utils.date import fromtimestamp, utcnow from calibre.library.caches import SortKeyGenerator from calibre.library.save_to_disk import find_plugboard - -from calibre.utils.magick.draw import save_cover_data_to, Image, \ - thumbnail as generate_thumbnail +from calibre.ebooks.metadata import authors_to_string +from calibre.utils.magick.draw import (save_cover_data_to, Image, + thumbnail as generate_thumbnail) +from calibre.utils.filenames import ascii_filename plugboard_content_server_value = 'content_server' plugboard_content_server_formats = ['epub'] @@ -46,7 +47,7 @@ class ContentServer(object): # Utility methods {{{ def last_modified(self, updated): ''' - Generates a local independent, english timestamp from a datetime + Generates a locale independent, english timestamp from a datetime object ''' lm = updated.strftime('day, %d month %Y %H:%M:%S GMT') @@ -151,14 +152,12 @@ class ContentServer(object): try: cherrypy.response.headers['Content-Type'] = 'image/jpeg' cherrypy.response.timeout = 3600 - cover = self.db.cover(id, index_is_id=True, as_file=True) + cover = self.db.cover(id, index_is_id=True) if cover is None: cover = self.default_cover updated = self.build_time else: - with cover as f: - updated = fromtimestamp(os.fstat(f.fileno()).st_mtime) - cover = f.read() + updated = self.db.cover_last_modified(id, index_is_id=True) cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) if thumbnail: @@ -187,9 +186,9 @@ class ContentServer(object): mode='rb') if fmt is None: raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format)) + mi = self.db.get_metadata(id, index_is_id=True) if format == 'EPUB': # Get the original metadata - mi = self.db.get_metadata(id, index_is_id=True) # Get any EPUB plugboards for the content server plugboards = self.db.prefs.get('plugboards', {}) @@ -203,24 +202,22 @@ class ContentServer(object): newmi = mi # Write the updated file - from tempfile import TemporaryFile from calibre.ebooks.metadata.meta import set_metadata - raw = fmt.read() - fmt = TemporaryFile() - fmt.write(raw) - fmt.seek(0) set_metadata(fmt, newmi, 'epub') fmt.seek(0) mt = guess_type('dummy.'+format.lower())[0] if mt is None: mt = 'application/octet-stream' + au = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) + title = mi.title if mi.title else _('Unknown') + fname = u'%s - %s_%s.%s'%(title[:30], au[:30], id, format.lower()) + fname = ascii_filename(fname).replace('"', '_') cherrypy.response.headers['Content-Type'] = mt + cherrypy.response.headers['Content-Disposition'] = \ + b'attachment; filename="%s"'%fname cherrypy.response.timeout = 3600 - path = getattr(fmt, 'name', None) - if path and os.path.exists(path): - updated = fromtimestamp(os.stat(path).st_mtime) - cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Last-Modified'] = self.last_modified(utcnow()) return fmt # }}}