mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Merge from trunk
This commit is contained in:
Charles Haley
commit
9a90599ddb
81
recipes/frontlineonnet.recipe
Normal file
81
recipes/frontlineonnet.recipe
Normal file
@ -0,0 +1,81 @@
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
|
||||
'''
|
||||
frontlineonnet.com
|
||||
'''
|
||||
|
||||
import re
|
||||
from calibre import strftime
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
class Frontlineonnet(BasicNewsRecipe):
|
||||
title = 'Frontline'
|
||||
__author__ = 'Darko Miletic'
|
||||
description = "India's national magazine"
|
||||
publisher = 'Frontline'
|
||||
category = 'news, politics, India'
|
||||
no_stylesheets = True
|
||||
delay = 1
|
||||
INDEX = 'http://frontlineonnet.com/'
|
||||
use_embedded_content = False
|
||||
encoding = 'cp1252'
|
||||
language = 'en_IN'
|
||||
publication_type = 'magazine'
|
||||
masthead_url = 'http://frontlineonnet.com/images/newfline.jpg'
|
||||
extra_css = """
|
||||
body{font-family: Verdana,Arial,Helvetica,sans-serif}
|
||||
img{margin-top:0.5em; margin-bottom: 0.7em; display: block}
|
||||
"""
|
||||
|
||||
conversion_options = {
|
||||
'comment' : description
|
||||
, 'tags' : category
|
||||
, 'publisher' : publisher
|
||||
, 'language' : language
|
||||
, 'linearize_tables' : True
|
||||
}
|
||||
|
||||
preprocess_regexps = [
|
||||
(re.compile(r'.*?<base', re.DOTALL|re.IGNORECASE),lambda match: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"><html dir="ltr" xml:lang="en-IN"><head><title>title</title><base')
|
||||
,(re.compile(r'<base .*?>', re.DOTALL|re.IGNORECASE),lambda match: '</head><body>')
|
||||
,(re.compile(r'<byline>', re.DOTALL|re.IGNORECASE),lambda match: '<div class="byline">')
|
||||
,(re.compile(r'</byline>', re.DOTALL|re.IGNORECASE),lambda match: '</div>')
|
||||
,(re.compile(r'<center>', re.DOTALL|re.IGNORECASE),lambda match: '<div class="ctr">')
|
||||
,(re.compile(r'</center>', re.DOTALL|re.IGNORECASE),lambda match: '</div>')
|
||||
]
|
||||
|
||||
keep_only_tags= [
|
||||
dict(name='font', attrs={'class':'storyhead'})
|
||||
,dict(attrs={'class':'byline'})
|
||||
]
|
||||
remove_attributes=['size','noshade','border']
|
||||
|
||||
def preprocess_html(self, soup):
|
||||
for item in soup.findAll(style=True):
|
||||
del item['style']
|
||||
for item in soup.findAll('img'):
|
||||
if not item.has_key('alt'):
|
||||
item['alt'] = 'image'
|
||||
return soup
|
||||
|
||||
def parse_index(self):
|
||||
articles = []
|
||||
soup = self.index_to_soup(self.INDEX)
|
||||
for feed_link in soup.findAll('a',href=True):
|
||||
if feed_link['href'].startswith('stories/'):
|
||||
url = self.INDEX + feed_link['href']
|
||||
title = self.tag_to_string(feed_link)
|
||||
date = strftime(self.timefmt)
|
||||
articles.append({
|
||||
'title' :title
|
||||
,'date' :date
|
||||
,'url' :url
|
||||
,'description':''
|
||||
})
|
||||
return [('Frontline', articles)]
|
||||
|
||||
def print_version(self, url):
|
||||
return "http://www.hinduonnet.com/thehindu/thscrip/print.pl?prd=fline&file=" + url.rpartition('/')[2]
|
||||
|
||||
def image_url_processor(self, baseurl, url):
|
||||
return url.replace('../images/', self.INDEX + 'images/').strip()
|
||||
@ -13,8 +13,10 @@ CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
isbn TEXT DEFAULT "" COLLATE NOCASE,
|
||||
lccn TEXT DEFAULT "" COLLATE NOCASE,
|
||||
path TEXT NOT NULL DEFAULT "",
|
||||
flags INTEGER NOT NULL DEFAULT 1
|
||||
, uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
|
||||
flags INTEGER NOT NULL DEFAULT 1,
|
||||
uuid TEXT,
|
||||
has_cover BOOL DEFAULT 0,
|
||||
last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
|
||||
CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY,
|
||||
book INTEGER NOT NULL,
|
||||
author INTEGER NOT NULL,
|
||||
|
||||
@ -578,6 +578,7 @@ def url_slash_cleaner(url):
|
||||
def get_download_filename(url, cookie_file=None):
|
||||
'''
|
||||
Get a local filename for a URL using the content disposition header
|
||||
Returns empty string if no content disposition header present
|
||||
'''
|
||||
from contextlib import closing
|
||||
from urllib2 import unquote as urllib2_unquote
|
||||
@ -591,8 +592,10 @@ def get_download_filename(url, cookie_file=None):
|
||||
cj.load(cookie_file)
|
||||
br.set_cookiejar(cj)
|
||||
|
||||
last_part_name = ''
|
||||
try:
|
||||
with closing(br.open(url)) as r:
|
||||
last_part_name = r.geturl().split('/')[-1]
|
||||
disposition = r.info().get('Content-disposition', '')
|
||||
for p in disposition.split(';'):
|
||||
if 'filename' in p:
|
||||
@ -612,7 +615,7 @@ def get_download_filename(url, cookie_file=None):
|
||||
traceback.print_exc()
|
||||
|
||||
if not filename:
|
||||
filename = r.geturl().split('/')[-1]
|
||||
filename = last_part_name
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
@ -762,34 +762,42 @@ plugins += input_profiles + output_profiles
|
||||
class ActionAdd(InterfaceActionBase):
|
||||
name = 'Add Books'
|
||||
actual_plugin = 'calibre.gui2.actions.add:AddAction'
|
||||
description = _('Add books to calibre or the connected device')
|
||||
|
||||
class ActionFetchAnnotations(InterfaceActionBase):
|
||||
name = 'Fetch Annotations'
|
||||
actual_plugin = 'calibre.gui2.actions.annotate:FetchAnnotationsAction'
|
||||
description = _('Fetch annotations from a connected Kindle (experimental)')
|
||||
|
||||
class ActionGenerateCatalog(InterfaceActionBase):
|
||||
name = 'Generate Catalog'
|
||||
actual_plugin = 'calibre.gui2.actions.catalog:GenerateCatalogAction'
|
||||
description = _('Generate a catalog of the books in your calibre library')
|
||||
|
||||
class ActionConvert(InterfaceActionBase):
|
||||
name = 'Convert Books'
|
||||
actual_plugin = 'calibre.gui2.actions.convert:ConvertAction'
|
||||
description = _('Convert books to various ebook formats')
|
||||
|
||||
class ActionDelete(InterfaceActionBase):
|
||||
name = 'Remove Books'
|
||||
actual_plugin = 'calibre.gui2.actions.delete:DeleteAction'
|
||||
description = _('Delete books from your calibre library or connected device')
|
||||
|
||||
class ActionEditMetadata(InterfaceActionBase):
|
||||
name = 'Edit Metadata'
|
||||
actual_plugin = 'calibre.gui2.actions.edit_metadata:EditMetadataAction'
|
||||
description = _('Edit the metadata of books in your calibre library')
|
||||
|
||||
class ActionView(InterfaceActionBase):
|
||||
name = 'View'
|
||||
actual_plugin = 'calibre.gui2.actions.view:ViewAction'
|
||||
description = _('Read books in your calibre library')
|
||||
|
||||
class ActionFetchNews(InterfaceActionBase):
|
||||
name = 'Fetch News'
|
||||
actual_plugin = 'calibre.gui2.actions.fetch_news:FetchNewsAction'
|
||||
description = _('Download news from the internet in ebook form')
|
||||
|
||||
class ActionQuickview(InterfaceActionBase):
|
||||
name = 'Show Quickview'
|
||||
@ -798,67 +806,87 @@ class ActionQuickview(InterfaceActionBase):
|
||||
class ActionSaveToDisk(InterfaceActionBase):
|
||||
name = 'Save To Disk'
|
||||
actual_plugin = 'calibre.gui2.actions.save_to_disk:SaveToDiskAction'
|
||||
description = _('Export books from your calibre library to the hard disk')
|
||||
|
||||
class ActionShowBookDetails(InterfaceActionBase):
|
||||
name = 'Show Book Details'
|
||||
actual_plugin = 'calibre.gui2.actions.show_book_details:ShowBookDetailsAction'
|
||||
description = _('Show book details in a separate popup')
|
||||
|
||||
class ActionRestart(InterfaceActionBase):
|
||||
name = 'Restart'
|
||||
actual_plugin = 'calibre.gui2.actions.restart:RestartAction'
|
||||
description = _('Restart calibre')
|
||||
|
||||
class ActionOpenFolder(InterfaceActionBase):
|
||||
name = 'Open Folder'
|
||||
actual_plugin = 'calibre.gui2.actions.open:OpenFolderAction'
|
||||
description = _('Open the folder that contains the book files in your'
|
||||
' calibre library')
|
||||
|
||||
class ActionSendToDevice(InterfaceActionBase):
|
||||
name = 'Send To Device'
|
||||
actual_plugin = 'calibre.gui2.actions.device:SendToDeviceAction'
|
||||
description = _('Send books to the connected device')
|
||||
|
||||
class ActionConnectShare(InterfaceActionBase):
|
||||
name = 'Connect Share'
|
||||
actual_plugin = 'calibre.gui2.actions.device:ConnectShareAction'
|
||||
description = _('Send books via email or the web also connect to iTunes or'
|
||||
' folders on your computer as if they are devices')
|
||||
|
||||
class ActionHelp(InterfaceActionBase):
|
||||
name = 'Help'
|
||||
actual_plugin = 'calibre.gui2.actions.help:HelpAction'
|
||||
description = _('Browse the calibre User Manual')
|
||||
|
||||
class ActionPreferences(InterfaceActionBase):
|
||||
name = 'Preferences'
|
||||
actual_plugin = 'calibre.gui2.actions.preferences:PreferencesAction'
|
||||
description = _('Customize calibre')
|
||||
|
||||
class ActionSimilarBooks(InterfaceActionBase):
|
||||
name = 'Similar Books'
|
||||
actual_plugin = 'calibre.gui2.actions.similar_books:SimilarBooksAction'
|
||||
description = _('Easily find books similar to the currently selected one')
|
||||
|
||||
class ActionChooseLibrary(InterfaceActionBase):
|
||||
name = 'Choose Library'
|
||||
actual_plugin = 'calibre.gui2.actions.choose_library:ChooseLibraryAction'
|
||||
description = _('Switch between different calibre libraries and perform'
|
||||
' maintenance on them')
|
||||
|
||||
class ActionAddToLibrary(InterfaceActionBase):
|
||||
name = 'Add To Library'
|
||||
actual_plugin = 'calibre.gui2.actions.add_to_library:AddToLibraryAction'
|
||||
description = _('Copy books from the devce to your calibre library')
|
||||
|
||||
class ActionEditCollections(InterfaceActionBase):
|
||||
name = 'Edit Collections'
|
||||
actual_plugin = 'calibre.gui2.actions.edit_collections:EditCollectionsAction'
|
||||
description = _('Edit the collections in which books are placed on your device')
|
||||
|
||||
class ActionCopyToLibrary(InterfaceActionBase):
|
||||
name = 'Copy To Library'
|
||||
actual_plugin = 'calibre.gui2.actions.copy_to_library:CopyToLibraryAction'
|
||||
description = _('Copy a book from one calibre library to another')
|
||||
|
||||
class ActionTweakEpub(InterfaceActionBase):
|
||||
name = 'Tweak ePub'
|
||||
actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction'
|
||||
description = _('Make small twekas to epub files in your calibre library')
|
||||
|
||||
class ActionNextMatch(InterfaceActionBase):
|
||||
name = 'Next Match'
|
||||
actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction'
|
||||
description = _('Find the next or previous match when searching in '
|
||||
'your calibre library in highlight mode')
|
||||
|
||||
class ActionStore(InterfaceActionBase):
|
||||
name = 'Store'
|
||||
author = 'John Schember'
|
||||
actual_plugin = 'calibre.gui2.actions.store:StoreAction'
|
||||
description = _('Search for books from different book sellers')
|
||||
|
||||
def customization_help(self, gui=False):
|
||||
return 'Customize the behavior of the store search.'
|
||||
@ -874,7 +902,7 @@ class ActionStore(InterfaceActionBase):
|
||||
class ActionPluginUpdater(InterfaceActionBase):
|
||||
name = 'Plugin Updater'
|
||||
author = 'Grant Drake'
|
||||
description = 'Queries the MobileRead forums for updates to plugins to install'
|
||||
description = _('Get new calibre plugins or update your existing ones')
|
||||
actual_plugin = 'calibre.gui2.actions.plugin_updates:PluginUpdaterAction'
|
||||
|
||||
plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
|
||||
@ -1311,6 +1339,16 @@ class StoreLegimiStore(StoreBase):
|
||||
headquarters = 'PL'
|
||||
formats = ['EPUB']
|
||||
|
||||
class StoreLibreDEStore(StoreBase):
|
||||
name = 'Libri DE'
|
||||
author = 'Charles Haley'
|
||||
description = u'Sicher Bücher, Hörbücher und Downloads online bestellen.'
|
||||
actual_plugin = 'calibre.gui2.store.libri_de_plugin:LibreDEStore'
|
||||
|
||||
headquarters = 'DE'
|
||||
formats = ['EPUB', 'PDF']
|
||||
affiliate = True
|
||||
|
||||
class StoreManyBooksStore(StoreBase):
|
||||
name = 'ManyBooks'
|
||||
description = u'Public domain and creative commons works from many sources.'
|
||||
@ -1461,6 +1499,7 @@ plugins += [
|
||||
StoreGutenbergStore,
|
||||
StoreKoboStore,
|
||||
StoreLegimiStore,
|
||||
StoreLibreDEStore,
|
||||
StoreManyBooksStore,
|
||||
StoreMobileReadStore,
|
||||
StoreNextoStore,
|
||||
|
||||
@ -259,6 +259,10 @@ class OutputFormatPlugin(Plugin):
|
||||
#: (option_name, recommended_value, recommendation_level)
|
||||
recommendations = set([])
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return _('Convert ebooks to the %s format'%self.file_type)
|
||||
|
||||
def __init__(self, *args):
|
||||
Plugin.__init__(self, *args)
|
||||
self.report_progress = DummyReporter()
|
||||
|
||||
67
src/calibre/db/__init__.py
Normal file
67
src/calibre/db/__init__.py
Normal file
@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
|
||||
'''
|
||||
Rewrite of the calibre database backend.
|
||||
|
||||
Broad Objectives:
|
||||
|
||||
* Use the sqlite db only as a datastore. i.e. do not do
|
||||
sorting/searching/concatenation or anything else in sqlite. Instead
|
||||
mirror the sqlite tables in memory, create caches and lookup maps from
|
||||
them and create a set_* API that updates the memory caches and the sqlite
|
||||
correctly.
|
||||
|
||||
* Move from keeping a list of books in memory as a cache to a per table
|
||||
cache. This allows much faster search and sort operations at the expense
|
||||
of slightly slower lookup operations. That slowdown can be mitigated by
|
||||
keeping lots of maps and updating them in the set_* API. Also
|
||||
get_categories becomes blazingly fast.
|
||||
|
||||
* Separate the database layer from the cache layer more cleanly. Rather
|
||||
than having the db layer refer to the cache layer and vice versa, the
|
||||
cache layer will refer to the db layer only and the new API will be
|
||||
defined on the cache layer.
|
||||
|
||||
* Get rid of index_is_id and other poor design decisions
|
||||
|
||||
* Minimize the API as much as possible and define it cleanly
|
||||
|
||||
* Do not change the on disk format of metadata.db at all (this is for
|
||||
backwards compatibility)
|
||||
|
||||
* Get rid of the need for a separate db access thread by switching to apsw
|
||||
to access sqlite, which is thread safe
|
||||
|
||||
* The new API will have methods to efficiently do bulk operations and will
|
||||
use shared/exclusive/pending locks to serialize access to the in-mem data
|
||||
structures. Use the same locking scheme as sqlite itself does.
|
||||
|
||||
How this will proceed:
|
||||
|
||||
1. Create the new API
|
||||
2. Create a test suite for it
|
||||
3. Write a replacement for LibraryDatabase2 that uses the new API
|
||||
internally
|
||||
4. Lots of testing of calibre with the new LibraryDatabase2
|
||||
5. Gradually migrate code to use the (much faster) new api wherever possible (the new api
|
||||
will be exposed via db.new_api)
|
||||
|
||||
I plan to work on this slowly, in parallel to normal calibre development
|
||||
work.
|
||||
|
||||
Various things that require other things before they can be migrated:
|
||||
1. From initialize_dynamic(): set_saved_searches,
|
||||
load_user_template_functions. Also add custom
|
||||
columns/categories/searches info into
|
||||
self.field_metadata. Finally, implement metadata dirtied
|
||||
functionality.
|
||||
|
||||
'''
|
||||
441
src/calibre/db/backend.py
Normal file
441
src/calibre/db/backend.py
Normal file
@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
# Imports {{{
|
||||
import os, shutil, uuid, json
|
||||
from functools import partial
|
||||
|
||||
import apsw
|
||||
|
||||
from calibre import isbytestring, force_unicode, prints
|
||||
from calibre.constants import (iswindows, filesystem_encoding,
|
||||
preferred_encoding)
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.library.schema_upgrades import SchemaUpgrade
|
||||
from calibre.library.field_metadata import FieldMetadata
|
||||
from calibre.ebooks.metadata import title_sort, author_to_author_sort
|
||||
from calibre.utils.icu import strcmp
|
||||
from calibre.utils.config import to_json, from_json, prefs, tweaks
|
||||
from calibre.utils.date import utcfromtimestamp
|
||||
from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
|
||||
SizeTable, FormatsTable, AuthorsTable, IdentifiersTable)
|
||||
# }}}
|
||||
|
||||
'''
|
||||
Differences in semantics from pysqlite:
|
||||
|
||||
1. execute/executemany/executescript operate in autocommit mode
|
||||
|
||||
'''
|
||||
|
||||
class DynamicFilter(object): # {{{
|
||||
|
||||
'No longer used, present for legacy compatibility'
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.ids = frozenset([])
|
||||
|
||||
def __call__(self, id_):
|
||||
return int(id_ in self.ids)
|
||||
|
||||
def change(self, ids):
|
||||
self.ids = frozenset(ids)
|
||||
# }}}
|
||||
|
||||
class DBPrefs(dict): # {{{
|
||||
|
||||
'Store preferences as key:value pairs in the db'
|
||||
|
||||
def __init__(self, db):
|
||||
dict.__init__(self)
|
||||
self.db = db
|
||||
self.defaults = {}
|
||||
self.disable_setting = False
|
||||
for key, val in self.db.conn.get('SELECT key,val FROM preferences'):
|
||||
try:
|
||||
val = self.raw_to_object(val)
|
||||
except:
|
||||
prints('Failed to read value for:', key, 'from db')
|
||||
continue
|
||||
dict.__setitem__(self, key, val)
|
||||
|
||||
def raw_to_object(self, raw):
|
||||
if not isinstance(raw, unicode):
|
||||
raw = raw.decode(preferred_encoding)
|
||||
return json.loads(raw, object_hook=from_json)
|
||||
|
||||
def to_raw(self, val):
|
||||
return json.dumps(val, indent=2, default=to_json)
|
||||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return dict.__getitem__(self, key)
|
||||
except KeyError:
|
||||
return self.defaults[key]
|
||||
|
||||
def __delitem__(self, key):
|
||||
dict.__delitem__(self, key)
|
||||
self.db.conn.execute('DELETE FROM preferences WHERE key=?', (key,))
|
||||
|
||||
def __setitem__(self, key, val):
|
||||
if self.disable_setting:
|
||||
return
|
||||
raw = self.to_raw(val)
|
||||
self.db.conn.execute('INSERT OR REPLACE INTO preferences (key,val) VALUES (?,?)', (key,
|
||||
raw))
|
||||
dict.__setitem__(self, key, val)
|
||||
|
||||
def set(self, key, val):
|
||||
self.__setitem__(key, val)
|
||||
|
||||
# }}}
|
||||
|
||||
# Extra collators {{{
|
||||
def pynocase(one, two, encoding='utf-8'):
|
||||
if isbytestring(one):
|
||||
try:
|
||||
one = one.decode(encoding, 'replace')
|
||||
except:
|
||||
pass
|
||||
if isbytestring(two):
|
||||
try:
|
||||
two = two.decode(encoding, 'replace')
|
||||
except:
|
||||
pass
|
||||
return cmp(one.lower(), two.lower())
|
||||
|
||||
def _author_to_author_sort(x):
|
||||
if not x: return ''
|
||||
return author_to_author_sort(x.replace('|', ','))
|
||||
|
||||
def icu_collator(s1, s2):
|
||||
return strcmp(force_unicode(s1, 'utf-8'), force_unicode(s2, 'utf-8'))
|
||||
# }}}
|
||||
|
||||
class Connection(apsw.Connection): # {{{
|
||||
|
||||
BUSY_TIMEOUT = 2000 # milliseconds
|
||||
|
||||
def __init__(self, path):
|
||||
apsw.Connection.__init__(self, path)
|
||||
|
||||
self.setbusytimeout(self.BUSY_TIMEOUT)
|
||||
self.execute('pragma cache_size=5000')
|
||||
self.conn.execute('pragma temp_store=2')
|
||||
|
||||
encoding = self.execute('pragma encoding').fetchone()[0]
|
||||
self.conn.create_collation('PYNOCASE', partial(pynocase,
|
||||
encoding=encoding))
|
||||
|
||||
self.conn.create_function('title_sort', 1, title_sort)
|
||||
self.conn.create_function('author_to_author_sort', 1,
|
||||
_author_to_author_sort)
|
||||
|
||||
self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
|
||||
|
||||
# Dummy functions for dynamically created filters
|
||||
self.conn.create_function('books_list_filter', 1, lambda x: 1)
|
||||
self.conn.create_collation('icucollate', icu_collator)
|
||||
|
||||
def create_dynamic_filter(self, name):
|
||||
f = DynamicFilter(name)
|
||||
self.conn.create_function(name, 1, f)
|
||||
|
||||
def get(self, *args, **kw):
|
||||
ans = self.cursor().execute(*args)
|
||||
if kw.get('all', True):
|
||||
return ans.fetchall()
|
||||
for row in ans:
|
||||
return ans[0]
|
||||
|
||||
def execute(self, sql, bindings=None):
|
||||
cursor = self.cursor()
|
||||
return cursor.execute(sql, bindings)
|
||||
|
||||
def executemany(self, sql, sequence_of_bindings):
|
||||
return self.cursor().executemany(sql, sequence_of_bindings)
|
||||
|
||||
def executescript(self, sql):
|
||||
with self:
|
||||
# Use an explicit savepoint so that even if this is called
|
||||
# while a transaction is active, it is atomic
|
||||
return self.cursor().execute(sql)
|
||||
# }}}
|
||||
|
||||
class DB(object, SchemaUpgrade):
|
||||
|
||||
PATH_LIMIT = 40 if iswindows else 100
|
||||
WINDOWS_LIBRARY_PATH_LIMIT = 75
|
||||
|
||||
# Initialize database {{{
|
||||
|
||||
def __init__(self, library_path, default_prefs=None, read_only=False):
|
||||
try:
|
||||
if isbytestring(library_path):
|
||||
library_path = library_path.decode(filesystem_encoding)
|
||||
except:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
self.field_metadata = FieldMetadata()
|
||||
|
||||
self.library_path = os.path.abspath(library_path)
|
||||
self.dbpath = os.path.join(library_path, 'metadata.db')
|
||||
self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH',
|
||||
self.dbpath)
|
||||
|
||||
if iswindows and len(self.library_path) + 4*self.PATH_LIMIT + 10 > 259:
|
||||
raise ValueError(_(
|
||||
'Path to library too long. Must be less than'
|
||||
' %d characters.')%(259-4*self.PATH_LIMIT-10))
|
||||
exists = self._exists = os.path.exists(self.dbpath)
|
||||
if not exists:
|
||||
# Be more strict when creating new libraries as the old calculation
|
||||
# allowed for max path lengths of 265 chars.
|
||||
if (iswindows and len(self.library_path) >
|
||||
self.WINDOWS_LIBRARY_PATH_LIMIT):
|
||||
raise ValueError(_(
|
||||
'Path to library too long. Must be less than'
|
||||
' %d characters.')%self.WINDOWS_LIBRARY_PATH_LIMIT)
|
||||
|
||||
if read_only and os.path.exists(self.dbpath):
|
||||
# Work on only a copy of metadata.db to ensure that
|
||||
# metadata.db is not changed
|
||||
pt = PersistentTemporaryFile('_metadata_ro.db')
|
||||
pt.close()
|
||||
shutil.copyfile(self.dbpath, pt.name)
|
||||
self.dbpath = pt.name
|
||||
|
||||
self.is_case_sensitive = (not iswindows and
|
||||
not os.path.exists(self.dbpath.replace('metadata.db',
|
||||
'MeTAdAtA.dB')))
|
||||
|
||||
self._conn = None
|
||||
|
||||
if self.user_version == 0:
|
||||
self.initialize_database()
|
||||
|
||||
SchemaUpgrade.__init__(self)
|
||||
# Guarantee that the library_id is set
|
||||
self.library_id
|
||||
|
||||
self.initialize_prefs(default_prefs)
|
||||
|
||||
# Fix legacy triggers and columns
|
||||
self.conn.executescript('''
|
||||
DROP TRIGGER IF EXISTS author_insert_trg;
|
||||
CREATE TEMP TRIGGER author_insert_trg
|
||||
AFTER INSERT ON authors
|
||||
BEGIN
|
||||
UPDATE authors SET sort=author_to_author_sort(NEW.name) WHERE id=NEW.id;
|
||||
END;
|
||||
DROP TRIGGER IF EXISTS author_update_trg;
|
||||
CREATE TEMP TRIGGER author_update_trg
|
||||
BEFORE UPDATE ON authors
|
||||
BEGIN
|
||||
UPDATE authors SET sort=author_to_author_sort(NEW.name)
|
||||
WHERE id=NEW.id AND name <> NEW.name;
|
||||
END;
|
||||
UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL;
|
||||
''')
|
||||
|
||||
def initialize_prefs(self, default_prefs):
|
||||
self.prefs = DBPrefs(self)
|
||||
|
||||
if default_prefs is not None and not self._exists:
|
||||
# Only apply default prefs to a new database
|
||||
for key in default_prefs:
|
||||
# be sure that prefs not to be copied are listed below
|
||||
if key not in frozenset(['news_to_be_synced']):
|
||||
self.prefs[key] = default_prefs[key]
|
||||
if 'field_metadata' in default_prefs:
|
||||
fmvals = [f for f in default_prefs['field_metadata'].values()
|
||||
if f['is_custom']]
|
||||
for f in fmvals:
|
||||
self.create_custom_column(f['label'], f['name'],
|
||||
f['datatype'], f['is_multiple'] is not None,
|
||||
f['is_editable'], f['display'])
|
||||
|
||||
defs = self.prefs.defaults
|
||||
defs['gui_restriction'] = defs['cs_restriction'] = ''
|
||||
defs['categories_using_hierarchy'] = []
|
||||
defs['column_color_rules'] = []
|
||||
|
||||
# Migrate the bool tristate tweak
|
||||
defs['bools_are_tristate'] = \
|
||||
tweaks.get('bool_custom_columns_are_tristate', 'yes') == 'yes'
|
||||
if self.prefs.get('bools_are_tristate') is None:
|
||||
self.prefs.set('bools_are_tristate', defs['bools_are_tristate'])
|
||||
|
||||
# Migrate column coloring rules
|
||||
if self.prefs.get('column_color_name_1', None) is not None:
|
||||
from calibre.library.coloring import migrate_old_rule
|
||||
old_rules = []
|
||||
for i in range(1, 6):
|
||||
col = self.prefs.get('column_color_name_'+str(i), None)
|
||||
templ = self.prefs.get('column_color_template_'+str(i), None)
|
||||
if col and templ:
|
||||
try:
|
||||
del self.prefs['column_color_name_'+str(i)]
|
||||
rules = migrate_old_rule(self.field_metadata, templ)
|
||||
for templ in rules:
|
||||
old_rules.append((col, templ))
|
||||
except:
|
||||
pass
|
||||
if old_rules:
|
||||
self.prefs['column_color_rules'] += old_rules
|
||||
|
||||
# Migrate saved search and user categories to db preference scheme
|
||||
def migrate_preference(key, default):
|
||||
oldval = prefs[key]
|
||||
if oldval != default:
|
||||
self.prefs[key] = oldval
|
||||
prefs[key] = default
|
||||
if key not in self.prefs:
|
||||
self.prefs[key] = default
|
||||
|
||||
migrate_preference('user_categories', {})
|
||||
migrate_preference('saved_searches', {})
|
||||
|
||||
# migrate grouped_search_terms
|
||||
if self.prefs.get('grouped_search_terms', None) is None:
|
||||
try:
|
||||
ogst = tweaks.get('grouped_search_terms', {})
|
||||
ngst = {}
|
||||
for t in ogst:
|
||||
ngst[icu_lower(t)] = ogst[t]
|
||||
self.prefs.set('grouped_search_terms', ngst)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Rename any user categories with names that differ only in case
|
||||
user_cats = self.prefs.get('user_categories', [])
|
||||
catmap = {}
|
||||
for uc in user_cats:
|
||||
ucl = icu_lower(uc)
|
||||
if ucl not in catmap:
|
||||
catmap[ucl] = []
|
||||
catmap[ucl].append(uc)
|
||||
cats_changed = False
|
||||
for uc in catmap:
|
||||
if len(catmap[uc]) > 1:
|
||||
prints('found user category case overlap', catmap[uc])
|
||||
cat = catmap[uc][0]
|
||||
suffix = 1
|
||||
while icu_lower((cat + unicode(suffix))) in catmap:
|
||||
suffix += 1
|
||||
prints('Renaming user category %s to %s'%(cat, cat+unicode(suffix)))
|
||||
user_cats[cat + unicode(suffix)] = user_cats[cat]
|
||||
del user_cats[cat]
|
||||
cats_changed = True
|
||||
if cats_changed:
|
||||
self.prefs.set('user_categories', user_cats)
|
||||
|
||||
@property
|
||||
def conn(self):
|
||||
if self._conn is None:
|
||||
self._conn = apsw.Connection(self.dbpath)
|
||||
if self._exists and self.user_version == 0:
|
||||
self._conn.close()
|
||||
os.remove(self.dbpath)
|
||||
self._conn = apsw.Connection(self.dbpath)
|
||||
return self._conn
|
||||
|
||||
@dynamic_property
|
||||
def user_version(self):
|
||||
doc = 'The user version of this database'
|
||||
|
||||
def fget(self):
|
||||
return self.conn.get('pragma user_version;', all=False)
|
||||
|
||||
def fset(self, val):
|
||||
self.conn.execute('pragma user_version=%d'%int(val))
|
||||
|
||||
return property(doc=doc, fget=fget, fset=fset)
|
||||
|
||||
def initialize_database(self):
|
||||
metadata_sqlite = P('metadata_sqlite.sql', data=True,
|
||||
allow_user_override=False).decode('utf-8')
|
||||
self.conn.executescript(metadata_sqlite)
|
||||
if self.user_version == 0:
|
||||
self.user_version = 1
|
||||
# }}}
|
||||
|
||||
# Database layer API {{{
|
||||
|
||||
@classmethod
|
||||
def exists_at(cls, path):
|
||||
return path and os.path.exists(os.path.join(path, 'metadata.db'))
|
||||
|
||||
@dynamic_property
|
||||
def library_id(self):
|
||||
doc = ('The UUID for this library. As long as the user only operates'
|
||||
' on libraries with calibre, it will be unique')
|
||||
|
||||
def fget(self):
|
||||
if getattr(self, '_library_id_', None) is None:
|
||||
ans = self.conn.get('SELECT uuid FROM library_id', all=False)
|
||||
if ans is None:
|
||||
ans = str(uuid.uuid4())
|
||||
self.library_id = ans
|
||||
else:
|
||||
self._library_id_ = ans
|
||||
return self._library_id_
|
||||
|
||||
def fset(self, val):
|
||||
self._library_id_ = unicode(val)
|
||||
self.conn.execute('''
|
||||
DELETE FROM library_id;
|
||||
INSERT INTO library_id (uuid) VALUES (?);
|
||||
''', self._library_id_)
|
||||
|
||||
return property(doc=doc, fget=fget, fset=fset)
|
||||
|
||||
def last_modified(self):
|
||||
''' Return last modified time as a UTC datetime object '''
|
||||
return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
|
||||
|
||||
def read_tables(self):
|
||||
tables = {}
|
||||
for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
|
||||
'timestamp', 'published', 'uuid', 'path', 'cover',
|
||||
'last_modified'):
|
||||
metadata = self.field_metadata[col].copy()
|
||||
if metadata['table'] is None:
|
||||
metadata['table'], metadata['column'] == 'books', ('has_cover'
|
||||
if col == 'cover' else col)
|
||||
tables[col] = OneToOneTable(col, metadata)
|
||||
|
||||
for col in ('series', 'publisher', 'rating'):
|
||||
tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
|
||||
|
||||
for col in ('authors', 'tags', 'formats', 'identifiers'):
|
||||
cls = {
|
||||
'authors':AuthorsTable,
|
||||
'formats':FormatsTable,
|
||||
'identifiers':IdentifiersTable,
|
||||
}.get(col, ManyToManyTable)
|
||||
tables[col] = cls(col, self.field_metadata[col].copy())
|
||||
|
||||
tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
|
||||
|
||||
with self.conn: # Use a single transaction, to ensure nothing modifies
|
||||
# the db while we are reading
|
||||
for table in tables.itervalues():
|
||||
try:
|
||||
table.read()
|
||||
except:
|
||||
prints('Failed to read table:', table.name)
|
||||
raise
|
||||
|
||||
return tables
|
||||
|
||||
# }}}
|
||||
|
||||
143
src/calibre/db/tables.py
Normal file
143
src/calibre/db/tables.py
Normal file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from dateutil.tz import tzoffset
|
||||
|
||||
from calibre.constants import plugins
|
||||
from calibre.utils.date import parse_date, local_tz
|
||||
from calibre.ebooks.metadata import author_to_author_sort
|
||||
|
||||
_c_speedup = plugins['speedup'][0]
|
||||
|
||||
def _c_convert_timestamp(val):
|
||||
if not val:
|
||||
return None
|
||||
try:
|
||||
ret = _c_speedup.parse_date(val.strip())
|
||||
except:
|
||||
ret = None
|
||||
if ret is None:
|
||||
return parse_date(val, as_utc=False)
|
||||
year, month, day, hour, minutes, seconds, tzsecs = ret
|
||||
return datetime(year, month, day, hour, minutes, seconds,
|
||||
tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz)
|
||||
|
||||
class Table(object):
|
||||
|
||||
def __init__(self, name, metadata):
|
||||
self.name, self.metadata = name, metadata
|
||||
|
||||
# self.adapt() maps values from the db to python objects
|
||||
self.adapt = \
|
||||
{
|
||||
'datetime': _c_convert_timestamp,
|
||||
'bool': bool
|
||||
}.get(
|
||||
metadata['datatype'], lambda x: x)
|
||||
if name == 'authors':
|
||||
# Legacy
|
||||
self.adapt = lambda x: x.replace('|', ',') if x else None
|
||||
|
||||
class OneToOneTable(Table):
|
||||
|
||||
def read(self, db):
|
||||
self.book_col_map = {}
|
||||
idcol = 'id' if self.metadata['table'] == 'books' else 'book'
|
||||
for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol,
|
||||
self.metadata['column'], self.metadata['table'])):
|
||||
self.book_col_map[row[0]] = self.adapt(row[1])
|
||||
|
||||
class SizeTable(OneToOneTable):
|
||||
|
||||
def read(self, db):
|
||||
self.book_col_map = {}
|
||||
for row in db.conn.execute(
|
||||
'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data '
|
||||
'WHERE data.book=books.id) FROM books'):
|
||||
self.book_col_map[row[0]] = self.adapt(row[1])
|
||||
|
||||
class ManyToOneTable(Table):
|
||||
|
||||
def read(self, db):
|
||||
self.id_map = {}
|
||||
self.extra_map = {}
|
||||
self.col_book_map = {}
|
||||
self.book_col_map = {}
|
||||
self.read_id_maps(db)
|
||||
self.read_maps(db)
|
||||
|
||||
def read_id_maps(self, db):
|
||||
for row in db.conn.execute('SELECT id, {0} FROM {1}'.format(
|
||||
self.metadata['name'], self.metadata['table'])):
|
||||
if row[1]:
|
||||
self.id_map[row[0]] = self.adapt(row[1])
|
||||
|
||||
def read_maps(self, db):
|
||||
for row in db.conn.execute(
|
||||
'SELECT book, {0} FROM books_{1}_link'.format(
|
||||
self.metadata['link_column'], self.metadata['table'])):
|
||||
if row[1] not in self.col_book_map:
|
||||
self.col_book_map[row[1]] = []
|
||||
self.col_book_map.append(row[0])
|
||||
self.book_col_map[row[0]] = row[1]
|
||||
|
||||
class ManyToManyTable(ManyToOneTable):
|
||||
|
||||
def read_maps(self, db):
|
||||
for row in db.conn.execute(
|
||||
'SELECT book, {0} FROM books_{1}_link'.format(
|
||||
self.metadata['link_column'], self.metadata['table'])):
|
||||
if row[1] not in self.col_book_map:
|
||||
self.col_book_map[row[1]] = []
|
||||
self.col_book_map.append(row[0])
|
||||
if row[0] not in self.book_col_map:
|
||||
self.book_col_map[row[0]] = []
|
||||
self.book_col_map[row[0]].append(row[1])
|
||||
|
||||
class AuthorsTable(ManyToManyTable):
|
||||
|
||||
def read_id_maps(self, db):
|
||||
for row in db.conn.execute(
|
||||
'SELECT id, name, sort FROM authors'):
|
||||
self.id_map[row[0]] = row[1]
|
||||
self.extra_map[row[0]] = (row[2] if row[2] else
|
||||
author_to_author_sort(row[1]))
|
||||
|
||||
class FormatsTable(ManyToManyTable):
|
||||
|
||||
def read_id_maps(self, db):
|
||||
pass
|
||||
|
||||
def read_maps(self, db):
|
||||
for row in db.conn.execute('SELECT book, format, name FROM data'):
|
||||
if row[1] is not None:
|
||||
if row[1] not in self.col_book_map:
|
||||
self.col_book_map[row[1]] = []
|
||||
self.col_book_map.append(row[0])
|
||||
if row[0] not in self.book_col_map:
|
||||
self.book_col_map[row[0]] = []
|
||||
self.book_col_map[row[0]].append((row[1], row[2]))
|
||||
|
||||
class IdentifiersTable(ManyToManyTable):
|
||||
|
||||
def read_id_maps(self, db):
|
||||
pass
|
||||
|
||||
def read_maps(self, db):
|
||||
for row in db.conn.execute('SELECT book, type, val FROM identifiers'):
|
||||
if row[1] is not None and row[2] is not None:
|
||||
if row[1] not in self.col_book_map:
|
||||
self.col_book_map[row[1]] = []
|
||||
self.col_book_map.append(row[0])
|
||||
if row[0] not in self.book_col_map:
|
||||
self.book_col_map[row[0]] = []
|
||||
self.book_col_map[row[0]].append((row[1], row[2]))
|
||||
|
||||
@ -100,7 +100,7 @@ class KOBO(USBMS):
|
||||
for idx,b in enumerate(bl):
|
||||
bl_cache[b.lpath] = idx
|
||||
|
||||
def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType):
|
||||
def update_booklist(prefix, path, title, authors, mime, date, ContentType, ImageID, readstatus, MimeType, expired):
|
||||
changed = False
|
||||
try:
|
||||
lpath = path.partition(self.normalize_path(prefix))[2]
|
||||
@ -118,6 +118,11 @@ class KOBO(USBMS):
|
||||
elif readstatus == 3:
|
||||
playlist_map[lpath]= "Closed"
|
||||
|
||||
# Related to a bug in the Kobo firmware that leaves an expired row for deleted books
|
||||
# this shows an expired Collection so the user can decide to delete the book
|
||||
if expired == 3:
|
||||
playlist_map[lpath] = "Expired"
|
||||
|
||||
path = self.normalize_path(path)
|
||||
# print "Normalized FileName: " + path
|
||||
|
||||
@ -126,7 +131,13 @@ class KOBO(USBMS):
|
||||
bl_cache[lpath] = None
|
||||
if ImageID is not None:
|
||||
imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - NickelBookCover.parsed')
|
||||
if not os.path.exists(imagename):
|
||||
# Try the Touch version if the image does not exist
|
||||
imagename = self.normalize_path(self._main_prefix + '.kobo/images/' + ImageID + ' - N3_LIBRARY_FULL.parsed')
|
||||
|
||||
#print "Image name Normalized: " + imagename
|
||||
if not os.path.exists(imagename):
|
||||
debug_print("Strange - The image name does not exist - title: ", title)
|
||||
if imagename is not None:
|
||||
bl[idx].thumbnail = ImageWrapper(imagename)
|
||||
if (ContentType != '6' and MimeType != 'Shortcover'):
|
||||
@ -187,23 +198,25 @@ class KOBO(USBMS):
|
||||
self.dbversion = result[0]
|
||||
|
||||
query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \
|
||||
'ImageID, ReadStatus from content where BookID is Null'
|
||||
'ImageID, ReadStatus, ___ExpirationStatus from content where BookID is Null'
|
||||
|
||||
cursor.execute (query)
|
||||
|
||||
changed = False
|
||||
for i, row in enumerate(cursor):
|
||||
# self.report_progress((i+1) / float(numrows), _('Getting list of books on device...'))
|
||||
|
||||
if row[3].startswith("file:///usr/local/Kobo/help/"):
|
||||
# These are internal to the Kobo device and do not exist
|
||||
continue
|
||||
path = self.path_from_contentid(row[3], row[5], row[4], oncard)
|
||||
mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip'
|
||||
# debug_print("mime:", mime)
|
||||
|
||||
if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"):
|
||||
changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4])
|
||||
changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8])
|
||||
# print "shortbook: " + path
|
||||
elif oncard == 'carda' and row[3].startswith("file:///mnt/sd/"):
|
||||
changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4])
|
||||
changed = update_booklist(self._card_a_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7], row[4], row[8])
|
||||
|
||||
if changed:
|
||||
need_sync = True
|
||||
@ -267,8 +280,12 @@ class KOBO(USBMS):
|
||||
cursor.execute('delete from content_keys where volumeid = ?', t)
|
||||
|
||||
# Delete the chapters associated with the book next
|
||||
t = (ContentID,ContentID,)
|
||||
cursor.execute('delete from content where BookID = ? or ContentID = ?', t)
|
||||
t = (ContentID,)
|
||||
# Kobo does not delete the Book row (ie the row where the BookID is Null)
|
||||
# The next server sync should remove the row
|
||||
cursor.execute('delete from content where BookID = ?', t)
|
||||
cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \
|
||||
'where BookID is Null and ContentID =?',t)
|
||||
|
||||
connection.commit()
|
||||
|
||||
@ -286,7 +303,7 @@ class KOBO(USBMS):
|
||||
path_prefix = '.kobo/images/'
|
||||
path = self._main_prefix + path_prefix + ImageID
|
||||
|
||||
file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed',)
|
||||
file_endings = (' - iPhoneThumbnail.parsed', ' - bbMediumGridList.parsed', ' - NickelBookCover.parsed', ' - N3_LIBRARY_FULL.parsed', ' - N3_LIBRARY_GRID.parsed', ' - N3_LIBRARY_LIST.parsed', ' - N3_SOCIAL_CURRENTREAD.parsed',)
|
||||
|
||||
for ending in file_endings:
|
||||
fpath = path + ending
|
||||
@ -450,6 +467,9 @@ class KOBO(USBMS):
|
||||
path = self._main_prefix + path + '.kobo'
|
||||
# print "Path: " + path
|
||||
elif (ContentType == "6" or ContentType == "10") and MimeType == 'application/x-kobo-epub+zip':
|
||||
if path.startswith("file:///mnt/onboard/"):
|
||||
path = self._main_prefix + path.replace("file:///mnt/onboard/", '')
|
||||
else:
|
||||
path = self._main_prefix + '.kobo/kepub/' + path
|
||||
# print "Internal: " + path
|
||||
else:
|
||||
|
||||
@ -26,6 +26,10 @@ class Epubcheck(ePubFixer):
|
||||
'significant changes to your epub, complain to the epubcheck '
|
||||
'project.')
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.long_description
|
||||
|
||||
@property
|
||||
def fix_name(self):
|
||||
return 'epubcheck'
|
||||
|
||||
@ -22,6 +22,10 @@ class Unmanifested(ePubFixer):
|
||||
'the manifest or delete them as specified by the '
|
||||
'delete unmanifested option.')
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.long_description
|
||||
|
||||
@property
|
||||
def fix_name(self):
|
||||
return 'unmanifested'
|
||||
|
||||
@ -49,16 +49,26 @@ class DeviceJob(BaseJob): # {{{
|
||||
self._aborted = False
|
||||
|
||||
def start_work(self):
|
||||
if DEBUG:
|
||||
prints('Job:', self.id, self.description, 'started',
|
||||
safe_encode=True)
|
||||
self.start_time = time.time()
|
||||
self.job_manager.changed_queue.put(self)
|
||||
|
||||
def job_done(self):
|
||||
self.duration = time.time() - self.start_time
|
||||
self.percent = 1
|
||||
if DEBUG:
|
||||
prints('DeviceJob:', self.id, self.description,
|
||||
'done, calling callback', safe_encode=True)
|
||||
|
||||
try:
|
||||
self.callback_on_done(self)
|
||||
except:
|
||||
pass
|
||||
if DEBUG:
|
||||
prints('DeviceJob:', self.id, self.description,
|
||||
'callback returned', safe_encode=True)
|
||||
self.job_manager.changed_queue.put(self)
|
||||
|
||||
def report_progress(self, percent, msg=''):
|
||||
|
||||
@ -51,19 +51,19 @@ class BeamEBooksDEStore(BasicStoreConfig, StorePlugin):
|
||||
if counter <= 0:
|
||||
break
|
||||
|
||||
id = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/@href')).strip()
|
||||
id = ''.join(data.xpath('./tr/td[1]/a/@href')).strip()
|
||||
if not id:
|
||||
continue
|
||||
id = id[7:]
|
||||
cover_url = ''.join(data.xpath('./tr/td[1]/a/img/@src'))
|
||||
if cover_url:
|
||||
cover_url = 'http://www.beam-ebooks.de' + cover_url
|
||||
title = ''.join(data.xpath('./tr/td/div[@class="stil2"]/a/b/text()'))
|
||||
author = ' '.join(data.xpath('./tr/td/div[@class="stil2"]/'
|
||||
'child::b/text()'
|
||||
'|'
|
||||
'./tr/td/div[@class="stil2"]/'
|
||||
'child::strong/text()'))
|
||||
temp = ''.join(data.xpath('./tr/td[1]/a/img/@alt'))
|
||||
colon = temp.find(':')
|
||||
if not temp.startswith('eBook') or colon < 0:
|
||||
continue
|
||||
author = temp[5:colon]
|
||||
title = temp[colon+1:]
|
||||
price = ''.join(data.xpath('./tr/td[3]/text()'))
|
||||
pdf = data.xpath(
|
||||
'boolean(./tr/td[3]/a/img[contains(@alt, "PDF")]/@alt)')
|
||||
|
||||
@ -58,6 +58,8 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
|
||||
cover_url = ''.join(data.xpath('.//div[@class="item_cover_container"]/a/img/@src'))
|
||||
title = ''.join(data.xpath('.//div[@class="item_entries"]/h2/a/text()'))
|
||||
author = ''.join(data.xpath('.//div[@class="item_entries"]/span[1]/a/text()'))
|
||||
author = re.sub(',','',author)
|
||||
author = re.sub(';',',',author)
|
||||
price = ''.join(data.xpath('.//div[@class="item_entries"]/span[3]/text()'))
|
||||
price = re.sub(r'[^0-9,]*','',price) + ' zł'
|
||||
|
||||
|
||||
90
src/calibre/gui2/store/libri_de_plugin.py
Normal file
90
src/calibre/gui2/store/libri_de_plugin.py
Normal file
@ -0,0 +1,90 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import (unicode_literals, division, absolute_import, print_function)
|
||||
|
||||
__license__ = 'GPL 3'
|
||||
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import urllib2
|
||||
from contextlib import closing
|
||||
|
||||
from lxml import html
|
||||
|
||||
from PyQt4.Qt import QUrl
|
||||
|
||||
from calibre import browser
|
||||
from calibre.gui2 import open_url
|
||||
from calibre.gui2.store import StorePlugin
|
||||
from calibre.gui2.store.basic_config import BasicStoreConfig
|
||||
from calibre.gui2.store.search_result import SearchResult
|
||||
from calibre.gui2.store.web_store_dialog import WebStoreDialog
|
||||
|
||||
class LibreDEStore(BasicStoreConfig, StorePlugin):
|
||||
|
||||
def open(self, parent=None, detail_item=None, external=False):
|
||||
url = 'http://ad.zanox.com/ppc/?18817073C15644254T'
|
||||
url_details = ('http://ad.zanox.com/ppc/?18845780C1371495675T&ULP=[['
|
||||
'http://www.libri.de/shop/action/productDetails?artiId={0}]]')
|
||||
|
||||
if external or self.config.get('open_external', False):
|
||||
if detail_item:
|
||||
url = url_details.format(detail_item)
|
||||
open_url(QUrl(url))
|
||||
else:
|
||||
detail_url = None
|
||||
if detail_item:
|
||||
detail_url = url_details.format(detail_item)
|
||||
d = WebStoreDialog(self.gui, url, parent, detail_url)
|
||||
d.setWindowTitle(self.name)
|
||||
d.set_tags(self.config.get('tags', ''))
|
||||
d.exec_()
|
||||
|
||||
def search(self, query, max_results=10, timeout=60):
|
||||
url = ('http://www.libri.de/shop/action/quickSearch?facetNodeId=6'
|
||||
'&mainsearchSubmit=Los!&searchString=' + urllib2.quote(query))
|
||||
br = browser()
|
||||
|
||||
counter = max_results
|
||||
with closing(br.open(url, timeout=timeout)) as f:
|
||||
doc = html.fromstring(f.read())
|
||||
for data in doc.xpath('//div[contains(@class, "item")]'):
|
||||
if counter <= 0:
|
||||
break
|
||||
|
||||
details = data.xpath('./div[@class="beschreibungContainer"]')
|
||||
if not details:
|
||||
continue
|
||||
details = details[0]
|
||||
id = ''.join(details.xpath('./div[@class="text"]/a/@name')).strip()
|
||||
if not id:
|
||||
continue
|
||||
cover_url = ''.join(details.xpath('./div[@class="bild"]/a/img/@src'))
|
||||
title = ''.join(details.xpath('./div[@class="text"]/span[@class="titel"]/a/text()')).strip()
|
||||
author = ''.join(details.xpath('./div[@class="text"]/span[@class="author"]/text()')).strip()
|
||||
pdf = details.xpath(
|
||||
'boolean(.//span[@class="format" and contains(text(), "pdf")]/text())')
|
||||
epub = details.xpath(
|
||||
'boolean(.//span[@class="format" and contains(text(), "epub")]/text())')
|
||||
mobi = details.xpath(
|
||||
'boolean(.//span[@class="format" and contains(text(), "mobipocket")]/text())')
|
||||
price = (''.join(data.xpath('.//span[@class="preis"]/text()'))).replace('*', '')
|
||||
counter -= 1
|
||||
|
||||
s = SearchResult()
|
||||
s.cover_url = cover_url
|
||||
s.title = title.strip()
|
||||
s.author = author.strip()
|
||||
s.price = price
|
||||
s.drm = SearchResult.DRM_UNKNOWN
|
||||
s.detail_item = id
|
||||
formats = []
|
||||
if epub:
|
||||
formats.append('ePub')
|
||||
if pdf:
|
||||
formats.append('PDF')
|
||||
if mobi:
|
||||
formats.append('MOBI')
|
||||
s.formats = ', '.join(formats)
|
||||
|
||||
yield s
|
||||
@ -57,7 +57,7 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
|
||||
|
||||
cover_url = ''.join(data.xpath('.//td[@class="w10 va-t"]/a[1]/img/@src'))
|
||||
title = ''.join(data.xpath('.//h3[@class="title"]/a[1]/text()'))
|
||||
author = ''.join(data.xpath('.//p[@class="author"]/a[1]/text()'))
|
||||
author = ', '.join(data.xpath('.//p[@class="author"]/a/text()'))
|
||||
price = ''.join(data.xpath('.//div[@class="prices"]/p[1]/span/text()'))
|
||||
price = re.sub('PLN', ' zł', price)
|
||||
price = re.sub('\.', ',', price)
|
||||
|
||||
@ -53,7 +53,7 @@ class ZixoStore(BasicStoreConfig, StorePlugin):
|
||||
|
||||
cover_url = ''.join(data.xpath('.//a[@class="productThumb"]/img/@src'))
|
||||
title = ''.join(data.xpath('.//a[@class="title"]/text()'))
|
||||
author = ''.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()'))
|
||||
author = ','.join(data.xpath('.//div[@class="productDescription"]/span[1]/a/text()'))
|
||||
price = ''.join(data.xpath('.//div[@class="priceList"]/span/text()'))
|
||||
price = re.sub('\.', ',', price)
|
||||
|
||||
|
||||
@ -4728,7 +4728,7 @@ Author '{0}':
|
||||
pass
|
||||
else:
|
||||
# uuid found in cache with matching crc
|
||||
thumb_data = zf.read(title['uuid'])
|
||||
thumb_data = zf.read(title['uuid']+cover_crc)
|
||||
with open(os.path.join(image_dir, thumb_file), 'wb') as f:
|
||||
f.write(thumb_data)
|
||||
return
|
||||
|
||||
@ -201,12 +201,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
|
||||
dbprefs = DBPrefs(self)
|
||||
for key in default_prefs:
|
||||
# be sure that prefs not to be copied are listed below
|
||||
if key in ['news_to_be_synced']:
|
||||
if key in frozenset(['news_to_be_synced']):
|
||||
continue
|
||||
try:
|
||||
dbprefs[key] = default_prefs[key]
|
||||
except:
|
||||
pass # ignore options that don't exist anymore
|
||||
if 'field_metadata' in default_prefs:
|
||||
fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']]
|
||||
for f in fmvals:
|
||||
self.create_custom_column(f['label'], f['name'], f['datatype'],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user