From c06f894229a15d35cc88c98b52177a76399dcd7f Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 10:31:47 -0400 Subject: [PATCH 1/8] USBMS: Do not check for books in subdirs on devices that do not support subdirs. --- src/calibre/devices/usbms/driver.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 68041a19cd..0a66b78014 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -57,12 +57,17 @@ class USBMS(Device): prefix = self._card_prefix if oncard else self._main_prefix ebook_dir = self.EBOOK_DIR_CARD if oncard else self.EBOOK_DIR_MAIN - # Get all books in all directories under the root ebook_dir directory - for path, dirs, files in os.walk(os.path.join(prefix, ebook_dir)): - # Filter out anything that isn't in the list of supported ebook - # types - for book_type in self.FORMATS: - for filename in fnmatch.filter(files, '*.%s' % (book_type)): + # Get all books in the ebook_dir directory + if self.SUPPORTS_SUB_DIRS: + for path, dirs, files in os.walk(os.path.join(prefix, ebook_dir)): + # Filter out anything that isn't in the list of supported ebook types + for book_type in self.FORMATS: + for filename in fnmatch.filter(files, '*.%s' % (book_type)): + bl.append(self.__class__.book_from_path(os.path.join(path, filename))) + else: + path = os.path.join(prefix, ebook_dir) + for filename in os.listdir(path): + if path_to_ext(filename) in self.FORMATS: bl.append(self.__class__.book_from_path(os.path.join(path, filename))) return bl From 8d774b6e7c6c3cc41f2d41594622fa5bf6ba26c0 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 11:46:55 -0400 Subject: [PATCH 2/8] EPubInput: make convert conform to interface. --- src/calibre/ebooks/epub/input.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/epub/input.py b/src/calibre/ebooks/epub/input.py index 4c1cdbfcf5..5c8a5c9d89 100644 --- a/src/calibre/ebooks/epub/input.py +++ b/src/calibre/ebooks/epub/input.py @@ -51,8 +51,7 @@ class EPUBInput(InputFormatPlugin): traceback.print_exc() return False - def convert(self, stream, options, file_ext, parse_cache, log, - accelerators): + def convert(self, stream, options, file_ext, log, accelerators): from calibre.utils.zipfile import ZipFile from calibre import walk from calibre.ebooks import DRMError @@ -72,6 +71,5 @@ class EPUBInput(InputFormatPlugin): if os.path.exists(encfile): if not self.process_encryption(encfile, opf, log): raise DRMError(os.path.basename(path)) - - return opf - + + return os.path.join(os.getcwd(), opf) From 316b5670c5030ec61df704c15dbeb1791b49d3af Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 17:38:12 -0400 Subject: [PATCH 3/8] space symbol --- src/calibre/ebooks/htmlsymbols.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/htmlsymbols.py b/src/calibre/ebooks/htmlsymbols.py index fa10873845..d46e4c707a 100644 --- a/src/calibre/ebooks/htmlsymbols.py +++ b/src/calibre/ebooks/htmlsymbols.py @@ -306,5 +306,7 @@ HTML_SYMBOLS = { u'ý' : ['ý', 'ý'], # latin small letter y with acute u'þ' : ['þ', 'þ'], # latin small letter thorn u'ÿ' : ['ÿ', 'ÿ'], # latin small letter y with diaeresis + # More + u' ' : [' '], } From b963cdc58104300189b5bb258b09e7065e0e0639 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 19:32:39 -0400 Subject: [PATCH 4/8] MobiReader only reads part of file when getting metadata --- src/calibre/ebooks/mobi/reader.py | 101 +++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 22 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index fa43a7af42..38d8255348 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -157,6 +157,62 @@ class BookHeader(object): self.exth.mi.language = self.language +class MetadataHeader(BookHeader): + def __init__(self, stream, log): + self.stream = stream + + self.ident = self.identity() + self.num_sections = self.section_count() + + if self.num_sections >= 2: + header = self.header() + BookHeader.__init__(self, header, self.ident, None, log) + else: + self.exth = None + + def identity(self): + self.stream.seek(60) + ident = self.stream.read(8).upper() + + if ident not in ['BOOKMOBI', 'TEXTREAD']: + raise MobiError('Unknown book type: %s' % ident) + return ident + + def section_count(self): + self.stream.seek(76) + return struct.unpack('>H', self.stream.read(2))[0] + + def section_offset(self, number): + self.stream.seek(78+number*8) + return struct.unpack('>LBBBB', self.stream.read(8))[0] + + def header(self): + section_headers = [] + + # First section with the metadata + section_headers.append(self.section_offset(0)) + # Second section used to get the lengh of the first + section_headers.append(self.section_offset(1)) + + end_off = section_headers[1] + off = section_headers[0] + + self.stream.seek(off) + return self.stream.read(end_off - off) + + def section_data(self, number): + start = self.section_offset(number) + + if number == self.num_sections -1: + end = os.stat(self.stream.name).st_size + else: + end = self.section_offset(number + 1) + + self.stream.seek(start) + + return self.stream.read(end - start) + + class MobiReader(object): PAGE_BREAK_PAT = re.compile(r'(<[/]{0,1}mbp:pagebreak\s*[/]{0,1}>)+', re.IGNORECASE) IMAGE_ATTRS = ('lowrecindex', 'recindex', 'hirecindex') @@ -414,7 +470,7 @@ class MobiReader(object): def create_opf(self, htmlfile, guide=None, root=None): mi = getattr(self.book_header.exth, 'mi', self.embedded_mi) if mi is None: - mi = MetaInformation(self.title, [_('Unknown')]) + mi = MetaInformation(self.book_header.title, [_('Unknown')]) opf = OPFCreator(os.path.dirname(htmlfile), mi) if hasattr(self.book_header.exth, 'cover_offset'): opf.cover = 'images/%05d.jpg'%(self.book_header.exth.cover_offset+1) @@ -595,25 +651,26 @@ class MobiReader(object): def get_metadata(stream): from calibre.utils.logging import Log log = Log() - mr = MobiReader(stream, log) - if mr.book_header.exth is None: - mi = MetaInformation(mr.name, [_('Unknown')]) - else: - mi = mr.create_opf('dummy.html')[0] - try: - if hasattr(mr.book_header.exth, 'cover_offset'): - cover_index = mr.book_header.first_image_index + \ - mr.book_header.exth.cover_offset - data = mr.sections[int(cover_index)][0] - else: - data = mr.sections[mr.book_header.first_image_index][0] - buf = cStringIO.StringIO(data) - im = PILImage.open(buf) - obuf = cStringIO.StringIO() - im.convert('RGBA').save(obuf, format='JPEG') - mi.cover_data = ('jpg', obuf.getvalue()) - except: - log.exception() + + mi = MetaInformation(stream.name, [_('Unknown')]) + try: + mh = MetadataHeader(stream, log) + + if mh.exth is not None: + if mh.exth.mi is not None: + mi = mh.exth.mi + + if hasattr(mh.exth, 'cover_offset'): + cover_index = mh.first_image_index + mh.exth.cover_offset + data = mh.section_data(int(cover_index)) + else: + data = mh.section_data(mh.first_image_index) + buf = cStringIO.StringIO(data) + im = PILImage.open(buf) + obuf = cStringIO.StringIO() + im.convert('RGBA').save(obuf, format='JPEG') + mi.cover_data = ('jpg', obuf.getvalue()) + except: + log.exception() + return mi - - From eed1f6923191b86f53ed8c489d98f4385384a0e9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 20:22:03 -0400 Subject: [PATCH 5/8] MobiReader read metadata from content with older prc files. --- src/calibre/ebooks/mobi/reader.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 38d8255348..9e29ea09b3 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -15,7 +15,8 @@ except ImportError: from lxml import html, etree -from calibre import entity_to_unicode +from calibre import entity_to_unicode, sanitize_file_name +from calibre.ptempfile import TemporaryDirectory from calibre.ebooks import DRMError from calibre.ebooks.chardet import ENCODING_PATS from calibre.ebooks.mobi import MobiError @@ -25,7 +26,6 @@ from calibre.ebooks.mobi.langcodes import main_language, sub_language from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata.opf2 import OPFCreator, OPF from calibre.ebooks.metadata.toc import TOC -from calibre import sanitize_file_name class EXTHHeader(object): @@ -659,6 +659,13 @@ def get_metadata(stream): if mh.exth is not None: if mh.exth.mi is not None: mi = mh.exth.mi + else: + with TemporaryDirectory('_mobi_meta_reader') as tdir: + mr = MobiReader(stream, log) + parse_cache = {} + mr.extract_content(tdir, parse_cache) + if mr.embedded_mi is not None: + mi = mr.embedded_mi if hasattr(mh.exth, 'cover_offset'): cover_index = mh.first_image_index + mh.exth.cover_offset From 632db425a2be1ccb36749bb3f029b75e3cb8a26e Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 11 Apr 2009 20:32:00 -0400 Subject: [PATCH 6/8] MobiReader do not include file path in default metadata title. --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 9e29ea09b3..161a6995ba 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -652,7 +652,7 @@ def get_metadata(stream): from calibre.utils.logging import Log log = Log() - mi = MetaInformation(stream.name, [_('Unknown')]) + mi = MetaInformation(os.path.basename(stream.name), [_('Unknown')]) try: mh = MetadataHeader(stream, log) From a48dd172db1e35f5cf54628051e8e7d44852217d Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 12 Apr 2009 10:03:46 -0400 Subject: [PATCH 7/8] PDFMetadataWriter working --- src/calibre/customize/builtins.py | 11 +++++++++++ src/calibre/ebooks/metadata/pdf.py | 31 +++++++++++++++++++----------- src/calibre/ebooks/oeb/iterator.py | 1 + 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 484d46dc36..a9fc342059 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -262,6 +262,17 @@ class MOBIMetadataWriter(MetadataWriterPlugin): def set_metadata(self, stream, mi, type): from calibre.ebooks.metadata.mobi import set_metadata set_metadata(stream, mi) + +class PDFMetadataWriter(MetadataWriterPlugin): + + name = 'Set PDF metadata' + file_types = set(['pdf']) + description = _('Set metadata in %s files') % 'PDF' + author = 'John Schember' + + def set_metadata(self, stream, mi, type): + from calibre.ebooks.metadata.pdf import set_metadata + set_metadata(stream, mi) from calibre.ebooks.epub.input import EPUBInput diff --git a/src/calibre/ebooks/metadata/pdf.py b/src/calibre/ebooks/metadata/pdf.py index 6b94b07275..06a02939ba 100644 --- a/src/calibre/ebooks/metadata/pdf.py +++ b/src/calibre/ebooks/metadata/pdf.py @@ -5,7 +5,7 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import sys, os, re, StringIO +import sys, os, StringIO from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.ptempfile import TemporaryDirectory @@ -52,18 +52,27 @@ def get_metadata(stream, extract_cover=True): def set_metadata(stream, mi): stream.seek(0) - raw = stream.read() - if mi.title: - tit = mi.title.encode('utf-8') if isinstance(mi.title, unicode) else mi.title - raw = re.compile(r'<<.*?/Title\((.+?)\)', re.DOTALL).sub(lambda m: m.group().replace(m.group(1), tit), raw) - if mi.authors: - au = authors_to_string(mi.authors) - if isinstance(au, unicode): - au = au.encode('utf-8') - raw = re.compile(r'<<.*?/Author\((.+?)\)', re.DOTALL).sub(lambda m: m.group().replace(m.group(1), au), raw) + + # Use a StringIO object for the pdf because we will want to over + # write it later and if we are working on the stream directly it + # could cause some issues. + raw = StringIO.StringIO(stream.read()) + orig_pdf = PdfFileReader(raw) + + title = mi.title if mi.title else orig_pdf.documentInfo.title + author = authors_to_string(mi.authors) if mi.authors else orig_pdf.documentInfo.author + + out_pdf = PdfFileWriter(title=title, author=author) + for page in orig_pdf.pages: + out_pdf.addPage(page) + + out_str = StringIO.StringIO() + out_pdf.write(out_str) + stream.seek(0) stream.truncate() - stream.write(raw) + out_str.seek(0) + stream.write(out_str.read()) stream.seek(0) def get_cover(stream): diff --git a/src/calibre/ebooks/oeb/iterator.py b/src/calibre/ebooks/oeb/iterator.py index ec0eda908a..88fffc604a 100644 --- a/src/calibre/ebooks/oeb/iterator.py +++ b/src/calibre/ebooks/oeb/iterator.py @@ -1,3 +1,4 @@ +from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008 Kovid Goyal ' From 1bd3829c6b4442283541c31e3630c0ca16f643da Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 12 Apr 2009 12:48:16 -0400 Subject: [PATCH 8/8] Don't allow editing of config category names. Fix import. --- src/calibre/gui2/dialogs/config.py | 2 +- src/calibre/gui2/dialogs/config.ui | 794 +++++++++++++++-------------- 2 files changed, 399 insertions(+), 397 deletions(-) diff --git a/src/calibre/gui2/dialogs/config.py b/src/calibre/gui2/dialogs/config.py index 831d44251e..1b2a2b8702 100644 --- a/src/calibre/gui2/dialogs/config.py +++ b/src/calibre/gui2/dialogs/config.py @@ -18,7 +18,7 @@ from calibre.utils.config import prefs from calibre.gui2.widgets import FilenamePattern from calibre.gui2.library import BooksModel from calibre.ebooks import BOOK_EXTENSIONS -from calibre.ebooks.epub.iterator import is_supported +from calibre.ebooks.oeb.iterator import is_supported from calibre.library import server_config from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \ disable_plugin, customize_plugin, \ diff --git a/src/calibre/gui2/dialogs/config.ui b/src/calibre/gui2/dialogs/config.ui index 9afcac8914..ac432c52c0 100644 --- a/src/calibre/gui2/dialogs/config.ui +++ b/src/calibre/gui2/dialogs/config.ui @@ -1,9 +1,8 @@ - - + Kovid Goyal Dialog - - + + 0 0 @@ -11,108 +10,111 @@ 557 - + Configuration - - + + :/images/config.svg:/images/config.svg - - - + + + - - - + + + 1 0 - + 75 true - + + QAbstractItemView::NoEditTriggers + + true - + false - + 48 48 - + QAbstractItemView::ScrollPerItem - + QAbstractItemView::ScrollPerPixel - + QListView::TopToBottom - + 20 - + QListView::ListMode - - - + + + 100 0 - + 0 - - + + - + - - + + 16777215 70 - + &Location of ebooks (The ebooks are stored in folders sorted by author and metadata is stored in the file metadata.db) - + true - + location - + - + - - + + Browse for the new database location - + ... - - + + :/images/mimetypes/dir.svg:/images/mimetypes/dir.svg @@ -122,107 +124,107 @@ - - + + Show notification when &new version is available - - + + If you disable this setting, metadata is guessed from the filename instead. This can be configured in the Advanced section. - + Read &metadata from files - + true - - - - + + + + Format for &single file save: - + single_format - - + + - - - + + + Default network &timeout: - + timeout - - - + + + Set the default timeout for network fetches (i.e. anytime we go out to the internet to get information) - + seconds - + 2 - + 120 - + 5 - - + + - - - + + + Choose &language (requires restart): - + language - - + + - + Normal - + High - + Low - - - + + + Job &priority: - + priority @@ -230,19 +232,19 @@ - - + + Frequently used directories - - - + + + - - + + true - + 22 22 @@ -251,13 +253,13 @@ - + - + Qt::Vertical - + 20 40 @@ -266,25 +268,25 @@ - - + + Add a directory to the frequently used directories list - + ... - - + + :/images/plus.svg:/images/plus.svg - + Qt::Vertical - + 20 40 @@ -293,25 +295,25 @@ - - + + Remove a directory from the frequently used directories list - + ... - - + + :/images/list_remove.svg:/images/list_remove.svg - + Qt::Vertical - + 20 40 @@ -328,111 +330,111 @@ - - + + - - + + Use &Roman numerals for series number - + true - - + + Enable system &tray icon (needs restart) - - + + Show &notifications in system tray - - + + Show cover &browser in a separate window (needs restart) - - + + Automatically send downloaded &news to ebook reader - - + + &Delete news from library when it is automatically sent to reader - + - - + + &Number of covers to show in browse mode (needs restart): - + cover_browse - + - - + + Toolbar - - - + + + - + Large - + Medium - + Small - - - + + + &Button size in toolbar - + toolbar_button_size - - - + + + Show &text in toolbar buttons - + true @@ -441,44 +443,44 @@ - + - - + + Select visible &columns in library view - + - + - - + + true - + QAbstractItemView::SelectRows - + - - + + ... - - + + :/images/arrow-up.svg:/images/arrow-up.svg - - + + Qt::Vertical - + 20 40 @@ -487,12 +489,12 @@ - - + + ... - - + + :/images/arrow-down.svg:/images/arrow-down.svg @@ -505,17 +507,17 @@ - - + + Use internal &viewer for: - - - - + + + + true - + QAbstractItemView::NoSelection @@ -536,99 +538,99 @@ - - - - - + + + + + calibre can send your books to you (or your reader) by email - + true - - + + - - + + Send email &from: - + email_from - - - <p>This is what will be present in the From: field of emails sent by calibre.<br> Set it to your email address + + + <p>This is what will be present in the From: field of emails sent by calibre.<br> Set it to your email address - - + + - - + + QAbstractItemView::SingleSelection - + QAbstractItemView::SelectRows - + - - + + Add an email address to which to send books - + &Add email - - + + :/images/plus.svg:/images/plus.svg - + 24 24 - + Qt::ToolButtonTextUnderIcon - - + + Make &default - - + + &Remove email - - + + :/images/minus.svg:/images/minus.svg - + 24 24 - + Qt::ToolButtonTextUnderIcon @@ -637,155 +639,155 @@ - - - - <p>A mail server is useful if the service you are sending mail to only accepts email from well know mail services. + + + + <p>A mail server is useful if the service you are sending mail to only accepts email from well know mail services. - + Mail &Server - - - - - calibre can <b>optionally</b> use a server to send mail + + + + + calibre can <b>optionally</b> use a server to send mail - + true - - - + + + &Hostname: - + relay_host - - - + + + The hostname of your mail server. For e.g. smtp.gmail.com - - + + - - + + &Port: - + relay_port - - + + The port your mail server listens for connections on. The default is 25 - + 1 - + 65555 - + 25 - - - + + + &Username: - + relay_username - - - + + + Your username on the mail server - - - + + + &Password: - + relay_password - - - + + + Your password on the mail server - + QLineEdit::Password - - - + + + &Show - - - + + + &Encryption: - + relay_tls - - - + + + Use TLS encryption when connecting to the mail server. This is the most common. - + &TLS - + true - - - + + + Use SSL encryption when connecting to the mail server. - + &SSL - - - + + + Qt::Horizontal - + 40 20 @@ -796,31 +798,31 @@ - - + + - - + + Use Gmail - - + + :/images/gmail_logo.png:/images/gmail_logo.png - + 48 48 - + Qt::ToolButtonTextUnderIcon - - + + &Test email @@ -829,16 +831,16 @@ - - + + - + - + Qt::Horizontal - + 40 20 @@ -847,21 +849,21 @@ - - + + Free unused diskspace from the database - + &Compact database - + Qt::Horizontal - + 40 20 @@ -872,17 +874,17 @@ - - + + &Metadata from file name - + - + Qt::Vertical - + 20 40 @@ -895,96 +897,96 @@ - - + + - - + + calibre contains a network server that allows you to access your book collection using a browser from anywhere in the world. Any changes to the settings will only take effect after a server restart. - + true - - - - + + + + Server &port: - + port - - - + + + 1025 - + 16000 - + 8080 - - - + + + &Username: - + username - - + + - - - + + + &Password: - + password - - - + + + If you leave the password blank, anyone will be able to access your book collection using the web interface. - - - + + + &Show password - - - + + + The maximum size (widthxheight) for displayed covers. Larger covers are resized. - + - - - + + + Max. &cover size: - + max_cover_size @@ -992,27 +994,27 @@ - + - - + + &Start Server - - + + St&op Server - - + + Qt::Horizontal - + 40 20 @@ -1021,8 +1023,8 @@ - - + + &Test Server @@ -1030,25 +1032,25 @@ - - + + Run server &automatically on startup - - + + View &server logs - - + + Qt::Vertical - + 20 40 @@ -1057,21 +1059,21 @@ - - + + If you want to use the content server to access your ebook collection on your iphone with Stanza, you will need to add the URL http://myhostname:8080/stanza as a new catalog in the stanza reader on your iphone. Here myhostname should be the fully qualified hostname or the IP address of this computer. - + true - - + + Qt::Vertical - + 20 40 @@ -1081,53 +1083,53 @@ - - + + - - + + Here you can customize the behavior of Calibre by controlling what plugins it uses. - + true - - + + 32 32 - + true - + true - + - - + + Enable/&Disable plugin - - + + &Customize plugin - - + + &Remove plugin @@ -1135,33 +1137,33 @@ - - + + Add new plugin - + - + - - + + Plugin &file: - + plugin_path - + - - + + ... - - + + :/images/document_open.svg:/images/document_open.svg @@ -1169,13 +1171,13 @@ - + - - + + Qt::Horizontal - + 40 20 @@ -1184,8 +1186,8 @@ - - + + &Add @@ -1201,12 +1203,12 @@ - - - + + + Qt::Horizontal - + QDialogButtonBox::Cancel|QDialogButtonBox::Ok @@ -1214,7 +1216,7 @@ - + @@ -1223,11 +1225,11 @@ Dialog accept() - + 239 558 - + 157 274 @@ -1239,11 +1241,11 @@ Dialog reject() - + 307 558 - + 286 274