diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000000..b546204f50 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,363 @@ +Files: * +Copyright: Copyright (C) 2008 Kovid Goyal +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/BeautifulSoup.py +Copyright: Copyright (c) 2004-2007, Leonard Richardson +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/chardet/* +Copyright: Copyright (C) 1998-2001 Netscape Communications Corporation +License: LGPL-2.1+ + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/calibre/ebooks/hyphenate.py +Copyright: Copyright (C) 1990, 2004, 2005 Gerard D.C. Kuiken. +License: other + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. + +Files: /src/cherrypy/* +Copyright: Copyright (c) 2004-2007, CherryPy Team (team@cherrypy.org) +Copyright: Copyright (C) 2005, Tiago Cogumbreiro +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/odf/* +Copyright: Copyright (C) 2006-2008 Søren Roug, European Environment Agency +License: LGPL2.1+ + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/odf/teletype.py +Files: src/odf/easyliststyle.py +Copyright: Copyright (C) 2008, J. David Eisenberg +License: GPL2+ + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-2 on Debian systems. + +Files: src/pyPdf/* +Copyright: Copyright (c) 2006, Mathieu Fenniak +Copyright: Copyright (c) 2007, Ashish Kulkarni +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/genshi/* +Copyright: Copyright (C) 2006-2008 Edgewall Software +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/lzx/* +Copyright: Copyright (C) 2002, Matthew T. Russotto +Copyright: Copyright (C) 2008, Marshall T. Vandegrift +Copyright: Copyright (C) 2006-2008, Alexander Chemeris +License: LGPL-2.1 + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-2.1 on Debian systems. + +Files: src/calibre/utils/lzx/msstdint.h +Copyright: Copyright (C) 2006-2008, Alexander Chemeris +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/utils/pyparsing.py +Copyright: Copyright (c) 2003-2008, Paul T. McGuire +License: MIT + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Files: src/calibre/utils/PythonMagickWand.py +Copyright: (c) 2007 - Achim Domma - domma@procoders.net +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +Files: src/calibre/utils/msdes/d3des.h: +Files: src/calibre/utils/msdes/des.c: +Copyright: Copyright (C) 1988,1989,1990,1991,1992, Richard Outerbridge +License: Other + THIS SOFTWARE PLACED IN THE PUBLIC DOMAIN BY THE AUTHOUR + +Files: src/calibre/utils/msdes/msdesmodule.c +Copyright: Copyright (C) 2008, Marshall T. Vandegrift +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/utils/msdes/spr.h +Copyright: Copyright (C) 2002, Dan A. Jackson +License: GPL2+ + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-2 on Debian systems. + +Files: src/calibre/gui2/pictureflow/* +Copyright: (C) Copyright 2007 Trolltech ASA +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/lit/* +Copyright: 2008, Marshall T. Vandegrift +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/lrf/* +Copyright: 2008, Anatoly Shipitsin +Copyright: copyright 2002 Paul Henry Tremblay +Copyright: Copyright (C) 2008 B.Scott Wxby [bswxby] +Copyright: Copyright (C) 2007 David Chen SonyReaderDaveChenorg +Copyright: Copyright (c) 2007 Mike Higgins (Falstaff) +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/BeautifulSoup.py +Copyright: Copyright (c) 2004-2007, Leonard Richardson +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/ebooks/rtf2xml/* +Copyright: copyright 2002 Paul Henry Tremblay +License: GPL + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL on Debian systems. + +Files: src/calibre/web/feeds/feedparser.py +Copyright: Copyright (c) 2002-2006, Mark Pilgrim +License: BSD + The full text of the BSD license is distributed as in + /usr/share/common-licenses/BSD on Debian systems. + +Files: src/calibre/web/feeds/recipes/* +Copyright: 2008, Darko Miletic +Copyright: 2008, Mathieu Godlewski +Copyright: Copyright (C) 2008 B.Scott Wxby [bswxby] +Copyright: Copyright (C) 2007 David Chen SonyReaderDaveChenorg +Copyright: 2008, Derry FitzGerald +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/ebooks/metadata/* +Copyright: 2008, Ashish Kulkarni +Copyright: Copyright (C) 2006 Søren Roug, European Environment Agency +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/encutils/__init__.py +Copyright: 2005-2008: Christof Hoeke +License: LGPL-3+, CC-BY-3.0 + The full text of the LGPL is distributed as in + /usr/share/common-licenses/LGPL-3 on Debian systems. + +Files: src/calibre/translations/* +Copyright: Copyright (C) 2007, Kovid Goyal +Copyright: Copyright (C) 2008, Rosetta Contributors and Canonical Ltd. +License: GPL-3 + The full text of the GPL is distributed as in + /usr/share/common-licenses/GPL-3 on Debian systems. + +Files: src/calibre/gui2/viewer/jquery.js +Files: src/calibre/gui2/viewer/jquery_scrollTo.js +Files: src/calibre/library/static/date.js +Copyright: Copyright (C) 2008, John Resig (jquery.com) +Copyright: Copyright (C) 2007-2008, Ariel Flesler - aflesler@gmail.com | http://flesler.blogspot.com +Copyright: Copyright (C) 2006-2007, Coolite Inc. (http://www.coolite.com/) +License: MIT + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Files: src/calibre/ebooks/lrf/fonts/liberation/* +Copyright: Copyright (C) 2007, Red Hat, Inc. All rights reserved. +License: Other + Copyright (C) 2007, Red Hat, Inc. All rights reserved. + LIBERATION is a trademark of Red Hat, Inc. + + This agreement governs the use of the Software and any updates to the Software, + regardless of the delivery mechanism. Subject to the following terms, Red Hat, Inc. + ("Red Hat") grants to the user ("Client") a license to this work pursuant to + the GNU General Public License v.2 with the exceptions set forth below and such + other terms as our set forth in this End User License Agreement. + + 1. The Software and License Exception. LIBERATION font software (the "Software") + consists of TrueType-OpenType formatted font software for rendering LIBERATION + typefaces in sans serif, serif, and monospaced character styles. You are licensed + to use, modify, copy, and distribute the Software pursuant to the GNU General + Public License v.2 with the following exceptions: + + (a) As a special exception, if you create a document which uses this font, and + embed this font or unaltered portions of this font into the document, this + font does not by itself cause the resulting document to be covered by the + GNU General Public License. This exception does not however invalidate any + other reasons why the document might be covered by the GNU General Public + License. If you modify this font, you may extend this exception to your + version of the font, but you are not obligated to do so. If you do not + wish to do so, delete this exception statement from your version. + + (b) As a further exception, any distribution of the object code of the Software + in a physical product must provide you the right to access and modify the + source code for the Software and to reinstall that modified version of the + Software in object code form on the same physical product on which you + received it. + + 2. Intellectual Property Rights. The Software and each of its components, including + the source code, documentation, appearance, structure and organization are owned + by Red Hat and others and are protected under copyright and other laws. Title to + the Software and any component, or to any copy, modification, or merged portion + shall remain with the aforementioned, subject to the applicable license. + The "LIBERATION" trademark is a trademark of Red Hat, Inc. in the U.S. and other + countries. This agreement does not permit Client to distribute modified versions + of the Software using Red Hat's trademarks. If Client makes a redistribution of + a modified version of the Software, then Client must modify the files names to + remove any reference to the Red Hat trademarks and must not use the Red Hat + trademarks in any way to reference or promote the modified Software. + + 3. Limited Warranty. To the maximum extent permitted under applicable law, the + Software is provided and licensed "as is" without warranty of any kind, + expressed or implied, including the implied warranties of merchantability, + non-infringement or fitness for a particular purpose. Red Hat does not warrant + that the functions contained in the Software will meet Client's requirements or + that the operation of the Software will be entirely error free or appear precisely + as described in the accompanying documentation. + + 4. Limitation of Remedies and Liability. To the maximum extent permitted by applicable + law, Red Hat or any Red Hat authorized dealer will not be liable to Client for any + incidental or consequential damages, including lost profits or lost savings arising + out of the use or inability to use the Software, even if Red Hat or such dealer has + been advised of the possibility of such damages. + + 5. General. If any provision of this agreement is held to be unenforceable, that shall + not affect the enforceability of the remaining provisions. This agreement shall be + governed by the laws of the State of North Carolina and of the United States, without + regard to any conflict of laws provisions, except that the United Nations Convention + on the International Sale of Goods shall not apply. + + +Files: installer/cx_Freeze/* +Copyright: Copyright © 2007-2008, Colt Engineering, Edmonton, Alberta, Canada. +Copyright: Copyright © 2001-2006, Computronix (Canada) Ltd., Edmonton, Alberta, Canada. +License: other + All rights reserved. + + NOTE: this license is derived from the Python Software Foundation License + which can be found at http://www.python.org/psf/license + + License for cx_Freeze 4.0.1 + --------------------------- + + 1. This LICENSE AGREEMENT is between the copyright holders and the Individual + or Organization ("Licensee") accessing and otherwise using cx_Freeze + software in source or binary form and its associated documentation. + + 2. Subject to the terms and conditions of this License Agreement, the + copyright holders hereby grant Licensee a nonexclusive, royalty-free, + world-wide license to reproduce, analyze, test, perform and/or display + publicly, prepare derivative works, distribute, and otherwise use cx_Freeze + alone or in any derivative version, provided, however, that this License + Agreement and this notice of copyright are retained in cx_Freeze alone or in + any derivative version prepared by Licensee. + + 3. In the event Licensee prepares a derivative work that is based on or + incorporates cx_Freeze or any part thereof, and wants to make the derivative + work available to others as provided herein, then Licensee hereby agrees to + include in any such work a brief summary of the changes made to cx_Freeze. + + 4. The copyright holders are making cx_Freeze available to Licensee on an + "AS IS" basis. THE COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, + EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, THE COPYRIGHT + HOLDERS MAKE NO AND DISCLAIM ANY REPRESENTATION OR WARRANTY OF + MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF + CX_FREEZE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + + 5. THE COPYRIGHT HOLDERS SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF + CX_FREEZE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING CX_FREEZE, OR ANY + DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + + 6. This License Agreement will automatically terminate upon a material breach + of its terms and conditions. + + 7. Nothing in this License Agreement shall be deemed to create any relationship + of agency, partnership, or joint venture between the copyright holders and + Licensee. This License Agreement does not grant permission to use + copyright holder's trademarks or trade name in a trademark sense to endorse + or promote products or services of Licensee, or any third party. + + 8. By copying, installing or otherwise using cx_Freeze, Licensee agrees to be + bound by the terms and conditions of this License Agreement. + + Computronix® is a registered trademark of Computronix (Canada) Ltd. + + +BSD License (for all the BSD licensed code indicated above) +----------------------------------------------------------- +Copyright (c) , +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/setup.py b/setup.py index 5e018ebd20..64283521ea 100644 --- a/setup.py +++ b/setup.py @@ -65,10 +65,10 @@ if __name__ == '__main__': '/Users/kovid/podofo/include/podofo' podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \ '/Users/kovid/podofo/lib' - if os.path.exists(os.path.join(podofo_inc, 'PdfString.h')): + if os.path.exists(os.path.join(podofo_inc, 'podofo.h')): eca = ['/EHsc'] if iswindows else [] - optional.append(PyQtExtension('calibre.plugins.podofo', [], - ['src/calibre/utils/podofo/podofo.sip'], + optional.append(Extension('calibre.plugins.podofo', + sources=['src/calibre/utils/podofo/podofo.cpp'], libraries=['podofo'], extra_compile_args=eca, library_dirs=[os.environ.get('PODOFO_LIB_DIR', podofo_lib)], include_dirs=\ diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 280bf02cae..a08f0417ee 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, re, logging, time, subprocess, mimetypes, \ +import sys, os, re, logging, time, mimetypes, \ __builtin__, warnings, multiprocessing __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint @@ -91,11 +91,16 @@ def prints(*args, **kwargs): file = kwargs.get('file', sys.stdout) sep = kwargs.get('sep', ' ') end = kwargs.get('end', '\n') + enc = preferred_encoding + if 'CALIBRE_WORKER' in os.environ: + enc = 'utf-8' for i, arg in enumerate(args): if isinstance(arg, unicode): - arg = arg.encode(preferred_encoding) + arg = arg.encode(enc) if not isinstance(arg, str): arg = str(arg) + if not isinstance(arg, unicode): + arg = arg.decode(preferred_encoding, 'replace').encode(enc) file.write(arg) if i != len(args)-1: file.write(sep) diff --git a/src/calibre/devices/bebook/driver.py b/src/calibre/devices/bebook/driver.py index 0980554387..d3e887eb74 100644 --- a/src/calibre/devices/bebook/driver.py +++ b/src/calibre/devices/bebook/driver.py @@ -18,9 +18,9 @@ class BEBOOK(USBMS): VENDOR_ID = [0x0525] PRODUCT_ID = [0x8803, 0x6803] - BCD = [0x312] + BCD = [0x312] - VENDOR_NAME = 'LINUX' + VENDOR_NAME = 'LINUX' WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET' WINDOWS_CARD_MEM = 'FILE-STOR_GADGET' @@ -51,7 +51,7 @@ class BEBOOK_MINI(BEBOOK): VENDOR_ID = [0x0492] PRODUCT_ID = [0x8813] - BCD = [0x319] + BCD = [0x319] OSX_MAIN_MEM = 'BeBook Mini Internal Memory' OSX_CARD_MEM = 'BeBook Mini Storage Card' diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 61393f9988..ff9d71f370 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -9,43 +9,43 @@ a backend that implement the Device interface for the SONY PRS500 Reader. from calibre.customize import Plugin class DevicePlugin(Plugin): - """ - Defines the interface that should be implemented by backends that - communicate with an ebook reader. - + """ + Defines the interface that should be implemented by backends that + communicate with an ebook reader. + The C{end_session} variables are used for USB session management. Sometimes - the front-end needs to call several methods one after another, in which case + the front-end needs to call several methods one after another, in which case the USB session should not be closed after each method call. """ type = _('Device Interface') - + # Ordered list of supported formats FORMATS = ["lrf", "rtf", "pdf", "txt"] VENDOR_ID = 0x0000 PRODUCT_ID = 0x0000 - # BCD can be either None to not distinguish between devices based on BCD, or + # BCD can be either None to not distinguish between devices based on BCD, or # it can be a list of the BCD numbers of all devices supported by this driver. BCD = None THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device # Whether the metadata on books can be set via the GUI. CAN_SET_METADATA = True - + def reset(self, key='-1', log_packets=False, report_progress=None) : """ @param key: The key to unlock the device - @param log_packets: If true the packet stream to/from the device is logged - @param report_progress: Function that is called with a % progress + @param log_packets: If true the packet stream to/from the device is logged + @param report_progress: Function that is called with a % progress (number between 0 and 100) for various tasks - If it is called with -1 that means that the + If it is called with -1 that means that the task does not have any progress information """ raise NotImplementedError() - + @classmethod def get_fdi(cls): '''Return the FDI description of this device for HAL on linux.''' return '' - + @classmethod def can_handle(cls, device_info): ''' @@ -54,40 +54,40 @@ class DevicePlugin(Plugin): is only called after the vendor, product ids and the bcd have matched, so it can do some relatively time intensive checks. The default implementation returns True. - - :param device_info: On windows a device ID string. On Unix a tuple of - ``(vendor_id, product_id, bcd)``. + + :param device_info: On windows a device ID string. On Unix a tuple of + ``(vendor_id, product_id, bcd)``. ''' return True - + def open(self): ''' Perform any device specific initialization. Called after the device is detected but before any other functions that communicate with the device. For example: For devices that present themselves as USB Mass storage devices, this method would be responsible for mounting the device or - if the device has been automounted, for finding out where it has been + if the device has been automounted, for finding out where it has been mounted. The driver for the PRS505 has a implementation of this function that should serve as a good example for USB Mass storage devices. ''' raise NotImplementedError() - + def set_progress_reporter(self, report_progress): ''' - @param report_progress: Function that is called with a % progress + @param report_progress: Function that is called with a % progress (number between 0 and 100) for various tasks - If it is called with -1 that means that the + If it is called with -1 that means that the task does not have any progress information ''' raise NotImplementedError() - + def get_device_information(self, end_session=True): - """ - Ask device for device information. See L{DeviceInfoQuery}. + """ + Ask device for device information. See L{DeviceInfoQuery}. @return: (device name, device version, software version on device, mime type) """ raise NotImplementedError() - + def card_prefix(self, end_session=True): ''' Return a 2 element list of the prefix to paths on the cards. @@ -99,9 +99,9 @@ class DevicePlugin(Plugin): (None, None) ''' raise NotImplementedError() - + def total_space(self, end_session=True): - """ + """ Get total space available on the mountpoints: 1. Main memory 2. Memory Card A @@ -111,9 +111,9 @@ class DevicePlugin(Plugin): particular device doesn't have any of these locations it should return 0. """ raise NotImplementedError() - + def free_space(self, end_session=True): - """ + """ Get free space available on the mountpoints: 1. Main memory 2. Card A @@ -121,20 +121,20 @@ class DevicePlugin(Plugin): @return: A 3 element list with free space in bytes of (1, 2, 3). If a particular device doesn't have any of these locations it should return -1. - """ + """ raise NotImplementedError() - + def books(self, oncard=None, end_session=True): - """ + """ Return a list of ebooks on the device. @param oncard: If 'carda' or 'cardb' return a list of ebooks on the specific storage card, otherwise return list of ebooks in main memory of device. If a card is specified and no - books are on the card return empty list. - @return: A BookList. - """ + books are on the card return empty list. + @return: A BookList. + """ raise NotImplementedError() - + def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): ''' @@ -144,26 +144,26 @@ class DevicePlugin(Plugin): free space on the device. The text of the FreeSpaceError must contain the word "card" if C{on_card} is not None otherwise it must contain the word "memory". @param files: A list of paths and/or file-like objects. - @param names: A list of file names that the books should have + @param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) - @return: A list of 3-element tuples. The list is meant to be passed + @return: A list of 3-element tuples. The list is meant to be passed to L{add_books_to_metadata}. - @param metadata: If not None, it is a list of dictionaries. Each dictionary + @param metadata: If not None, it is a list of dictionaries. Each dictionary will have at least the key tags to allow the driver to choose book location based on tags. len(metadata) == len(files). If your device does not support hierarchical ebook folders, you can safely ignore this parameter. ''' raise NotImplementedError() - + @classmethod def add_books_to_metadata(cls, locations, metadata, booklists): ''' - Add locations to the booklists. This function must not communicate with - the device. + Add locations to the booklists. This function must not communicate with + the device. @param locations: Result of a call to L{upload_books} @param metadata: List of dictionaries. Each dictionary must have the - keys C{title}, C{authors}, C{author_sort}, C{cover}, C{tags}. - The value of the C{cover} + keys C{title}, C{authors}, C{author_sort}, C{cover}, C{tags}. + The value of the C{cover} element can be None or a three element tuple (width, height, data) where data is the image data in JPEG format as a string. C{tags} must be a possibly empty list of strings. C{authors} must be a string. @@ -172,22 +172,22 @@ class DevicePlugin(Plugin): The dictionary can also have an optional key "tag order" which should be another dictionary that maps tag names to lists of book ids. The ids are ids from the book database. - @param booklists: A tuple containing the result of calls to + @param booklists: A tuple containing the result of calls to (L{books}(oncard=None), L{books}(oncard='carda'), L{books}(oncard='cardb')). ''' raise NotImplementedError - + def delete_books(self, paths, end_session=True): ''' Delete books at paths on device. ''' raise NotImplementedError() - + @classmethod def remove_books_from_metadata(cls, paths, booklists): ''' - Remove books from the metadata list. This function must not communicate + Remove books from the metadata list. This function must not communicate with the device. @param paths: paths to books on the device. @param booklists: A tuple containing the result of calls to @@ -195,7 +195,7 @@ class DevicePlugin(Plugin): L{books}(oncard='cardb')). ''' raise NotImplementedError() - + def sync_booklists(self, booklists, end_session=True): ''' Update metadata on device. @@ -204,8 +204,8 @@ class DevicePlugin(Plugin): L{books}(oncard='cardb')). ''' raise NotImplementedError() - - def get_file(self, path, outfile, end_session=True): + + def get_file(self, path, outfile, end_session=True): ''' Read the file at C{path} on the device and write it to outfile. @param outfile: file object like C{sys.stdout} or the result of an C{open} call @@ -231,13 +231,13 @@ class DevicePlugin(Plugin): def settings(cls): ''' Should return an opts object. The opts object should have one attribute - `formats` whihc is an ordered list of formats for the device. + `format_map` which is an ordered list of formats for the device. ''' raise NotImplementedError() - + class BookList(list): ''' A list of books. Each Book object must have the fields: @@ -247,21 +247,21 @@ class BookList(list): 4. datetime (a UTC time tuple) 5. path (path on the device to the book) 6. thumbnail (can be None) - 7. tags (a list of strings, can be empty). + 7. tags (a list of strings, can be empty). ''' - + __getslice__ = None __setslice__ = None - + def supports_tags(self): ''' Return True if the the device supports tags (collections) for this book list. ''' raise NotImplementedError() - + def set_tags(self, book, tags): ''' - Set the tags for C{book} to C{tags}. + Set the tags for C{book} to C{tags}. @param tags: A list of strings. Can be empty. - @param book: A book object that is in this BookList. + @param book: A book object that is in this BookList. ''' raise NotImplementedError() diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 6471112c1a..de8cf0272c 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -53,7 +53,7 @@ class KINDLE(USBMS): @classmethod def metadata_from_path(cls, path): - from calibre.devices.usbms.driver import metadata_from_formats + from calibre.ebooks.metadata.meta import metadata_from_formats mi = metadata_from_formats([path]) if mi.title == _('Unknown') or ('-asin' in mi.title and '-type' in mi.title): match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path)) diff --git a/src/calibre/devices/prs500/books.py b/src/calibre/devices/prs500/books.py index d567511ec6..5eb8d7f011 100644 --- a/src/calibre/devices/prs500/books.py +++ b/src/calibre/devices/prs500/books.py @@ -1,8 +1,8 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -""" -This module contains the logic for dealing with XML book lists found -in the reader cache. +""" +This module contains the logic for dealing with XML book lists found +in the reader cache. """ import xml.dom.minidom as dom from base64 import b64decode as decode @@ -25,16 +25,16 @@ def sortable_title(title): class book_metadata_field(object): """ Represents metadata stored as an attribute """ - def __init__(self, attr, formatter=None, setter=None): - self.attr = attr + def __init__(self, attr, formatter=None, setter=None): + self.attr = attr self.formatter = formatter self.setter = setter - + def __get__(self, obj, typ=None): """ Return a string. String may be empty if self.attr is absent """ return self.formatter(obj.elem.getAttribute(self.attr)) if \ self.formatter else obj.elem.getAttribute(self.attr).strip() - + def __set__(self, obj, val): """ Set the attribute """ val = self.setter(val) if self.setter else val @@ -44,7 +44,7 @@ class book_metadata_field(object): class Book(object): """ Provides a view onto the XML element that represents a book """ - + title = book_metadata_field("title") authors = book_metadata_field("author", \ formatter=lambda x: x if x and x.strip() else "Unknown") @@ -66,12 +66,12 @@ class Book(object): def fset(self, val): self.elem.setAttribute('titleSorter', sortable_title(unicode(val))) return property(doc=doc, fget=fget, fset=fset) - + @dynamic_property def thumbnail(self): doc = \ - """ - The thumbnail. Should be a height 68 image. + """ + The thumbnail. Should be a height 68 image. Setting is not supported. """ def fget(self): @@ -83,18 +83,18 @@ class Book(object): break rc = "" for node in th.childNodes: - if node.nodeType == node.TEXT_NODE: + if node.nodeType == node.TEXT_NODE: rc += node.data return decode(rc) return property(fget=fget, doc=doc) - + @dynamic_property def path(self): doc = """ Absolute path to book on device. Setting not supported. """ - def fget(self): + def fget(self): return self.root + self.rpath return property(fget=fget, doc=doc) - + @dynamic_property def db_id(self): doc = '''The database id in the application database that this file corresponds to''' @@ -103,20 +103,20 @@ class Book(object): if match: return int(match.group(1)) return property(fget=fget, doc=doc) - + def __init__(self, node, tags=[], prefix="", root="/Data/media/"): self.elem = node self.prefix = prefix self.root = root self.tags = tags - + def __str__(self): """ Return a utf-8 encoded string with title author and path information """ return self.title.encode('utf-8') + " by " + \ self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') -def fix_ids(media, cache): +def fix_ids(media, cache, *args): ''' Adjust ids in cache to correspond with media. ''' @@ -131,16 +131,16 @@ def fix_ids(media, cache): child.setAttribute("id", str(cid)) cid += 1 media.set_next_id(str(cid)) - - + + class BookList(_BookList): - """ - A list of L{Book}s. Created from an XML file. Can write list + """ + A list of L{Book}s. Created from an XML file. Can write list to an XML file. """ __getslice__ = None __setslice__ = None - + def __init__(self, root="/Data/media/", sfile=None): _BookList.__init__(self) self.tag_order = {} @@ -163,25 +163,25 @@ class BookList(_BookList): if records: self.prefix = 'xs1:' self.root = records[0] - self.proot = root - + self.proot = root + for book in self.document.getElementsByTagName(self.prefix + "text"): id = book.getAttribute('id') pl = [i.getAttribute('title') for i in self.get_playlists(id)] self.append(Book(book, root=root, prefix=self.prefix, tags=pl)) - + def supports_tags(self): return bool(self.prefix) - + def playlists(self): return self.root.getElementsByTagName(self.prefix+'playlist') - - def playlist_items(self): + + def playlist_items(self): plitems = [] for pl in self.playlists(): plitems.extend(pl.getElementsByTagName(self.prefix+'item')) return plitems - + def purge_corrupted_files(self): if not self.root: return [] @@ -193,32 +193,32 @@ class BookList(_BookList): c.parentNode.removeChild(c) c.unlink() return paths - + def purge_empty_playlists(self): ''' Remove all playlist entries that have no children. ''' for pl in self.playlists(): if not pl.getElementsByTagName(self.prefix + 'item'): pl.parentNode.removeChild(pl) pl.unlink() - + def _delete_book(self, node): nid = node.getAttribute('id') node.parentNode.removeChild(node) node.unlink() self.remove_from_playlists(nid) - - + + def delete_book(self, cid): - ''' + ''' Remove DOM node corresponding to book with C{id == cid}. Also remove book from any collections it is part of. ''' for book in self: if str(book.id) == str(cid): self.remove(book) - self._delete_book(book.elem) + self._delete_book(book.elem) break - + def remove_book(self, path): ''' Remove DOM node corresponding to book with C{path == path}. @@ -227,15 +227,15 @@ class BookList(_BookList): for book in self: if path.endswith(book.rpath): self.remove(book) - self._delete_book(book.elem) + self._delete_book(book.elem) break - + def next_id(self): return self.document.documentElement.getAttribute('nextID') - + def set_next_id(self, id): self.document.documentElement.setAttribute('nextID', str(id)) - + def max_id(self): max = 0 for child in self.root.childNodes: @@ -243,15 +243,15 @@ class BookList(_BookList): nid = int(child.getAttribute('id')) if nid > max: max = nid - return max - + return max + def book_by_path(self, path): for child in self.root.childNodes: if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"): if path == child.getAttribute('path'): return child return None - + def add_book(self, info, name, size, ctime): """ Add a node into DOM tree representing a book """ book = self.book_by_path(name) @@ -262,23 +262,23 @@ class BookList(_BookList): cid = self.max_id()+1 sourceid = str(self[0].sourceid) if len(self) else "1" attrs = { - "title" : info["title"], + "title" : info["title"], 'titleSorter' : sortable_title(info['title']), "author" : info["authors"] if info['authors'] else 'Unknown', \ "page":"0", "part":"0", "scale":"0", \ "sourceid":sourceid, "id":str(cid), "date":"", \ "mime":mime, "path":name, "size":str(size) - } + } for attr in attrs.keys(): node.setAttributeNode(self.document.createAttribute(attr)) - node.setAttribute(attr, attrs[attr]) + node.setAttribute(attr, attrs[attr]) try: - w, h, data = info["cover"] + w, h, data = info["cover"] except TypeError: w, h, data = None, None, None - + if data: - th = self.document.createElement(self.prefix + "thumbnail") + th = self.document.createElement(self.prefix + "thumbnail") th.setAttribute("width", str(w)) th.setAttribute("height", str(h)) jpeg = self.document.createElement(self.prefix + "jpeg") @@ -294,15 +294,15 @@ class BookList(_BookList): if info.has_key('tag order'): self.tag_order.update(info['tag order']) self.set_playlists(book.id, info['tags']) - - + + def playlist_by_title(self, title): for pl in self.playlists(): if pl.getAttribute('title').lower() == title.lower(): return pl - + def add_playlist(self, title): - cid = self.max_id()+1 + cid = self.max_id()+1 pl = self.document.createElement(self.prefix+'playlist') pl.setAttribute('sourceid', '0') pl.setAttribute('id', str(cid)) @@ -316,18 +316,18 @@ class BookList(_BookList): except AttributeError: continue return pl - - + + def remove_from_playlists(self, id): for pli in self.playlist_items(): if pli.getAttribute('id') == str(id): pli.parentNode.removeChild(pli) pli.unlink() - + def set_tags(self, book, tags): book.tags = tags self.set_playlists(book.id, tags) - + def set_playlists(self, id, collections): self.remove_from_playlists(id) for collection in set(collections): @@ -337,7 +337,7 @@ class BookList(_BookList): item = self.document.createElement(self.prefix+'item') item.setAttribute('id', str(id)) coll.appendChild(item) - + def get_playlists(self, id): ans = [] for pl in self.playlists(): @@ -346,12 +346,12 @@ class BookList(_BookList): ans.append(pl) continue return ans - + def book_by_id(self, id): for book in self: if str(book.id) == str(id): return book - + def reorder_playlists(self): for title in self.tag_order.keys(): pl = self.playlist_by_title(title) @@ -364,7 +364,7 @@ class BookList(_BookList): map[i] = j pl_book_ids = [i for i in pl_book_ids if i is not None] ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids] - + if len(ordered_ids) < len(pl.childNodes): continue children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] @@ -374,8 +374,8 @@ class BookList(_BookList): for id in ordered_ids: item = self.document.createElement(self.prefix+'item') item.setAttribute('id', str(map[id])) - pl.appendChild(item) - + pl.appendChild(item) + def write(self, stream): """ Write XML representation of DOM tree to C{stream} """ - stream.write(self.document.toxml('utf-8')) \ No newline at end of file + stream.write(self.document.toxml('utf-8')) diff --git a/src/calibre/devices/prs500/driver.py b/src/calibre/devices/prs500/driver.py index 874ed494e4..c6cd120283 100644 --- a/src/calibre/devices/prs500/driver.py +++ b/src/calibre/devices/prs500/driver.py @@ -47,6 +47,7 @@ from calibre.devices.prs500.prstypes import * from calibre.devices.errors import * from calibre.devices.prs500.books import BookList, fix_ids from calibre import __author__, __appname__ +from calibre.devices.usbms.deviceconfig import DeviceConfig # Protocol versions this driver has been tested with KNOWN_USB_PROTOCOL_VERSIONS = [0x3030303030303130L] @@ -76,7 +77,7 @@ class File(object): return self.name -class PRS500(DevicePlugin): +class PRS500(DeviceConfig, DevicePlugin): """ Implements the backend for communication with the SONY Reader. @@ -624,6 +625,8 @@ class PRS500(DevicePlugin): data_type=FreeSpaceAnswer, \ command_number=FreeSpaceQuery.NUMBER)[0] data.append( pkt.free ) + data = [x for x in data if x != 0] + data.append(0) return data def _exists(self, path): diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index b47ab5ae67..528770d3c5 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -15,11 +15,11 @@ from calibre.devices import strptime strftime = functools.partial(_strftime, zone=time.gmtime) -MIME_MAP = { +MIME_MAP = { "lrf" : "application/x-sony-bbeb", - 'lrx' : 'application/x-sony-bbeb', - "rtf" : "application/rtf", - "pdf" : "application/pdf", + 'lrx' : 'application/x-sony-bbeb', + "rtf" : "application/rtf", + "pdf" : "application/pdf", "txt" : "text/plain" , 'epub': 'application/epub+zip', } @@ -32,16 +32,16 @@ def sortable_title(title): class book_metadata_field(object): """ Represents metadata stored as an attribute """ - def __init__(self, attr, formatter=None, setter=None): - self.attr = attr + def __init__(self, attr, formatter=None, setter=None): + self.attr = attr self.formatter = formatter self.setter = setter - + def __get__(self, obj, typ=None): """ Return a string. String may be empty if self.attr is absent """ return self.formatter(obj.elem.getAttribute(self.attr)) if \ self.formatter else obj.elem.getAttribute(self.attr).strip() - + def __set__(self, obj, val): """ Set the attribute """ val = self.setter(val) if self.setter else val @@ -52,7 +52,7 @@ class book_metadata_field(object): class Book(object): """ Provides a view onto the XML element that represents a book """ - + title = book_metadata_field("title") authors = book_metadata_field("author", \ formatter=lambda x: x if x and x.strip() else _('Unknown')) @@ -63,7 +63,7 @@ class Book(object): size = book_metadata_field("size", formatter=lambda x : int(float(x))) # When setting this attribute you must use an epoch datetime = book_metadata_field("date", formatter=strptime, setter=strftime) - + @dynamic_property def title_sorter(self): doc = '''String to sort the title. If absent, title is returned''' @@ -75,12 +75,12 @@ class Book(object): def fset(self, val): self.elem.setAttribute('titleSorter', sortable_title(unicode(val))) return property(doc=doc, fget=fget, fset=fset) - + @dynamic_property def thumbnail(self): doc = \ - """ - The thumbnail. Should be a height 68 image. + """ + The thumbnail. Should be a height 68 image. Setting is not supported. """ def fget(self): @@ -94,18 +94,18 @@ class Book(object): break rc = "" for node in th.childNodes: - if node.nodeType == node.TEXT_NODE: + if node.nodeType == node.TEXT_NODE: rc += node.data return decode(rc) return property(fget=fget, doc=doc) - + @dynamic_property def path(self): doc = """ Absolute path to book on device. Setting not supported. """ - def fget(self): + def fget(self): return self.mountpath + self.rpath return property(fget=fget, doc=doc) - + @dynamic_property def db_id(self): doc = '''The database id in the application database that this file corresponds to''' @@ -114,13 +114,13 @@ class Book(object): if match: return int(match.group(1)) return property(fget=fget, doc=doc) - + def __init__(self, node, mountpath, tags, prefix=""): self.elem = node self.prefix = prefix self.tags = tags self.mountpath = mountpath - + def __str__(self): """ Return a utf-8 encoded string with title author and path information """ return self.title.encode('utf-8') + " by " + \ @@ -128,7 +128,7 @@ class Book(object): class BookList(_BookList): - + def __init__(self, xml_file, mountpath, report_progress=None): _BookList.__init__(self) xml_file.seek(0) @@ -143,15 +143,15 @@ class BookList(_BookList): self.root_element = records[0] else: self.prefix = '' - + nodes = self.root_element.childNodes for i, book in enumerate(nodes): if report_progress: - self.report_progress((i+1) / float(len(nodes)), _('Getting list of books on device...')) + report_progress((i+1) / float(len(nodes)), _('Getting list of books on device...')) if hasattr(book, 'tagName') and book.tagName.endswith('text'): tags = [i.getAttribute('title') for i in self.get_playlists(book.getAttribute('id'))] self.append(Book(book, mountpath, tags, prefix=self.prefix)) - + def max_id(self): max = 0 for child in self.root_element.childNodes: @@ -160,7 +160,7 @@ class BookList(_BookList): if nid > max: max = nid return max - + def is_id_valid(self, id): '''Return True iff there is an element with C{id==id}.''' id = str(id) @@ -169,23 +169,23 @@ class BookList(_BookList): if child.getAttribute('id') == id: return True return False - + def supports_tags(self): return True - + def book_by_path(self, path): for child in self.root_element.childNodes: if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"): if path == child.getAttribute('path'): return child return None - + def add_book(self, info, name, size, ctime): """ Add a node into the DOM tree, representing a book """ book = self.book_by_path(name) if book is not None: self.remove_book(name) - + node = self.document.createElement(self.prefix + "text") mime = MIME_MAP[name.rpartition('.')[-1].lower()] cid = self.max_id()+1 @@ -194,23 +194,23 @@ class BookList(_BookList): except: sourceid = '1' attrs = { - "title" : info["title"], + "title" : info["title"], 'titleSorter' : sortable_title(info['title']), - "author" : info["authors"] if info['authors'] else _('Unknown'), + "author" : info["authors"] if info['authors'] else _('Unknown'), "page":"0", "part":"0", "scale":"0", \ "sourceid":sourceid, "id":str(cid), "date":"", \ "mime":mime, "path":name, "size":str(size) - } + } for attr in attrs.keys(): node.setAttributeNode(self.document.createAttribute(attr)) - node.setAttribute(attr, attrs[attr]) + node.setAttribute(attr, attrs[attr]) try: - w, h, data = info["cover"] + w, h, data = info["cover"] except TypeError: w, h, data = None, None, None - + if data: - th = self.document.createElement(self.prefix + "thumbnail") + th = self.document.createElement(self.prefix + "thumbnail") th.setAttribute("width", str(w)) th.setAttribute("height", str(h)) jpeg = self.document.createElement(self.prefix + "jpeg") @@ -225,24 +225,24 @@ class BookList(_BookList): if info.has_key('tag order'): self.tag_order.update(info['tag order']) self.set_tags(book, info['tags']) - + def _delete_book(self, node): nid = node.getAttribute('id') self.remove_from_playlists(nid) node.parentNode.removeChild(node) node.unlink() - + def delete_book(self, cid): - ''' + ''' Remove DOM node corresponding to book with C{id == cid}. Also remove book from any collections it is part of. ''' for book in self: if str(book.id) == str(cid): self.remove(book) - self._delete_book(book.elem) + self._delete_book(book.elem) break - + def remove_book(self, path): ''' Remove DOM node corresponding to book with C{path == path}. @@ -251,24 +251,24 @@ class BookList(_BookList): for book in self: if path.endswith(book.rpath): self.remove(book) - self._delete_book(book.elem) + self._delete_book(book.elem) break - + def playlists(self): ans = [] for c in self.root_element.childNodes: if hasattr(c, 'tagName') and c.tagName.endswith('playlist'): ans.append(c) return ans - - def playlist_items(self): + + def playlist_items(self): plitems = [] for pl in self.playlists(): for c in pl.childNodes: if hasattr(c, 'tagName') and c.tagName.endswith('item'): plitems.append(c) return plitems - + def purge_corrupted_files(self): if not self.root_element: return [] @@ -279,7 +279,7 @@ class BookList(_BookList): c.parentNode.removeChild(c) c.unlink() return paths - + def purge_empty_playlists(self): ''' Remove all playlists that have no children. Also removes any invalid playlist items.''' for pli in self.playlist_items(): @@ -298,32 +298,32 @@ class BookList(_BookList): if empty: pl.parentNode.removeChild(pl) pl.unlink() - + def playlist_by_title(self, title): for pl in self.playlists(): if pl.getAttribute('title').lower() == title.lower(): return pl - + def add_playlist(self, title): - cid = self.max_id()+1 + cid = self.max_id()+1 pl = self.document.createElement(self.prefix+'playlist') pl.setAttribute('id', str(cid)) pl.setAttribute('title', title) pl.setAttribute('uuid', uuid()) self.root_element.insertBefore(pl, self.root_element.childNodes[-1]) return pl - + def remove_from_playlists(self, id): for pli in self.playlist_items(): if pli.getAttribute('id') == str(id): pli.parentNode.removeChild(pli) pli.unlink() - + def set_tags(self, book, tags): tags = [t for t in tags if t] book.tags = tags self.set_playlists(book.id, tags) - + def set_playlists(self, id, collections): self.remove_from_playlists(id) for collection in set(collections): @@ -333,7 +333,7 @@ class BookList(_BookList): item = self.document.createElement(self.prefix+'item') item.setAttribute('id', str(id)) coll.appendChild(item) - + def get_playlists(self, bookid): ans = [] for pl in self.playlists(): @@ -342,23 +342,23 @@ class BookList(_BookList): if item.getAttribute('id') == str(bookid): ans.append(pl) return ans - + def next_id(self): return self.document.documentElement.getAttribute('nextID') - + def set_next_id(self, id): self.document.documentElement.setAttribute('nextID', str(id)) - + def write(self, stream): """ Write XML representation of DOM tree to C{stream} """ src = self.document.toxml('utf-8') + '\n' stream.write(src.replace("'", ''')) - + def book_by_id(self, id): for book in self: if str(book.id) == str(id): return book - + def reorder_playlists(self): for title in self.tag_order.keys(): pl = self.playlist_by_title(title) @@ -371,7 +371,7 @@ class BookList(_BookList): map[i] = j pl_book_ids = [i for i in pl_book_ids if i is not None] ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids] - + if len(ordered_ids) < len(pl.childNodes): continue children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] @@ -382,7 +382,7 @@ class BookList(_BookList): item = self.document.createElement(self.prefix+'item') item.setAttribute('id', str(map[id])) pl.appendChild(item) - + def fix_ids(main, carda, cardb): ''' Adjust ids the XML databases. @@ -393,7 +393,7 @@ def fix_ids(main, carda, cardb): carda.purge_empty_playlists() if hasattr(cardb, 'purge_empty_playlists'): cardb.purge_empty_playlists() - + def regen_ids(db): if not hasattr(db, 'root_element'): return @@ -402,11 +402,11 @@ def fix_ids(main, carda, cardb): cid = 0 if db == main else 1 for child in db.root_element.childNodes: if child.nodeType == child.ELEMENT_NODE and child.hasAttribute('id'): - id_map[child.getAttribute('id')] = str(cid) + id_map[child.getAttribute('id')] = str(cid) child.setAttribute("sourceid", '1') - child.setAttribute('id', str(cid)) + child.setAttribute('id', str(cid)) cid += 1 - + for item in db.playlist_items(): oid = item.getAttribute('id') try: @@ -414,11 +414,11 @@ def fix_ids(main, carda, cardb): except KeyError: item.parentNode.removeChild(item) item.unlink() - + db.reorder_playlists() - + regen_ids(main) regen_ids(carda) regen_ids(cardb) - + main.set_next_id(str(main.max_id()+1)) diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index a93ef54c32..c73b1cd2f6 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -47,8 +47,8 @@ class DeviceScanner(object): rev = ('rev_%4.4x'%c).replace('a', ':') if rev in device_id: return True - return False - + return False + def test_bcd(self, bcdDevice, bcd): if bcd is None or len(bcd) == 0: return True @@ -56,19 +56,20 @@ class DeviceScanner(object): if c == bcdDevice: return True return False - + def is_device_connected(self, device): vendor_ids = device.VENDOR_ID if hasattr(device.VENDOR_ID, '__len__') else [device.VENDOR_ID] product_ids = device.PRODUCT_ID if hasattr(device.PRODUCT_ID, '__len__') else [device.PRODUCT_ID] if iswindows: - for vendor_id, product_id in zip(vendor_ids, product_ids): - vid, pid = 'vid_%4.4x'%vendor_id, 'pid_%4.4x'%product_id - vidd, pidd = 'vid_%i'%vendor_id, 'pid_%i'%product_id - for device_id in self.devices: - if (vid in device_id or vidd in device_id) and (pid in device_id or pidd in device_id): - if self.test_bcd_windows(device_id, getattr(device, 'BCD', None)): - if device.can_handle(device_id): - return True + for vendor_id in vendor_ids: + for product_id in product_ids: + vid, pid = 'vid_%4.4x'%vendor_id, 'pid_%4.4x'%product_id + vidd, pidd = 'vid_%i'%vendor_id, 'pid_%i'%product_id + for device_id in self.devices: + if (vid in device_id or vidd in device_id) and (pid in device_id or pidd in device_id): + if self.test_bcd_windows(device_id, getattr(device, 'BCD', None)): + if device.can_handle(device_id): + return True else: for vendor, product, bcdDevice in self.devices: if vendor in vendor_ids and product in product_ids: diff --git a/src/calibre/devices/usbms/deviceconfig.py b/src/calibre/devices/usbms/deviceconfig.py index bf955e1cee..bbe3a13646 100644 --- a/src/calibre/devices/usbms/deviceconfig.py +++ b/src/calibre/devices/usbms/deviceconfig.py @@ -12,7 +12,8 @@ class DeviceConfig(object): @classmethod def _config(cls): - c = Config('device_drivers_%s' % cls.__class__.__name__, _('settings for device drivers')) + klass = cls if isinstance(cls, type) else cls.__class__ + c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers')) c.add_opt('format_map', default=cls.FORMATS, help=cls.HELP_MESSAGE) return c @@ -33,7 +34,7 @@ class DeviceConfig(object): @classmethod def settings(cls): return cls._config().parse() - + def customization_help(cls, gui=False): return cls.HELP_MESSAGE diff --git a/src/calibre/ebooks/comic/input.py b/src/calibre/ebooks/comic/input.py index b228182ef4..bf2aac1162 100755 --- a/src/calibre/ebooks/comic/input.py +++ b/src/calibre/ebooks/comic/input.py @@ -7,12 +7,14 @@ __docformat__ = 'restructuredtext en' Based on ideas from comiclrf created by FangornUK. ''' -import os, shutil, traceback, textwrap +import os, shutil, traceback, textwrap, time +from Queue import Empty from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre import extract, CurrentDir +from calibre import extract, CurrentDir, prints from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.parallel import Server, ParallelJob +from calibre.utils.ipc.server import Server +from calibre.utils.ipc.job import ParallelJob def extract_comic(path_to_comic_file): ''' @@ -47,8 +49,8 @@ def find_pages(dir, sort_on_mtime=False, verbose=False): pages.sort(cmp=comparator) if verbose: - print 'Found comic pages...' - print '\t'+'\n\t'.join([os.path.basename(p) for p in pages]) + prints('Found comic pages...') + prints('\t'+'\n\t'.join([os.path.basename(p) for p in pages])) return pages class PageProcessor(list): @@ -181,7 +183,7 @@ class PageProcessor(list): p.DestroyPixelWand(pw) p.DestroyMagickWand(wand) -def render_pages(tasks, dest, opts, notification=None): +def render_pages(tasks, dest, opts, notification=lambda x, y: x): ''' Entry point for the job server. ''' @@ -197,30 +199,23 @@ def render_pages(tasks, dest, opts, notification=None): msg = _('Failed %s')%path if opts.verbose: msg += '\n' + traceback.format_exc() - if notification is not None: - notification(0.5, msg) + prints(msg) + notification(0.5, msg) return pages, failures -class JobManager(object): - ''' - Simple job manager responsible for keeping track of overall progress. - ''' +class Progress(object): def __init__(self, total, update): self.total = total self.update = update self.done = 0 - self.add_job = lambda j: j - self.output = lambda j: j - self.start_work = lambda j: j - self.job_done = lambda j: j - def status_update(self, job): + def __call__(self, percent, msg=''): self.done += 1 #msg = msg%os.path.basename(job.args[0]) - self.update(float(self.done)/self.total, job.msg) + self.update(float(self.done)/self.total, msg) def process_pages(pages, opts, update, tdir): ''' @@ -229,22 +224,38 @@ def process_pages(pages, opts, update, tdir): from calibre.utils.PythonMagickWand import ImageMagick ImageMagick - job_manager = JobManager(len(pages), update) + progress = Progress(len(pages), update) server = Server() jobs = [] + tasks = [(p, os.path.join(tdir, os.path.basename(p))) for p in pages] tasks = server.split(pages) for task in tasks: - jobs.append(ParallelJob('render_pages', lambda s:s, job_manager=job_manager, + jobs.append(ParallelJob('render_pages', '', progress, args=[task, tdir, opts])) server.add_job(jobs[-1]) - server.wait() - server.killall() + while True: + time.sleep(1) + running = False + for job in jobs: + while True: + try: + x = job.notifications.get_nowait() + progress(*x) + except Empty: + break + job.update() + if not job.is_finished: + running = True + if not running: + break server.close() ans, failures = [], [] for job in jobs: - if job.result is None: - raise Exception(_('Failed to process comic: %s\n\n%s')%(job.exception, job.traceback)) + if job.failed: + raw_input() + raise Exception(_('Failed to process comic: \n\n%s')% + job.log_file.read()) pages, failures_ = job.result ans += pages failures += failures_ diff --git a/src/calibre/ebooks/conversion/config.py b/src/calibre/ebooks/conversion/config.py new file mode 100644 index 0000000000..e8b923a1d7 --- /dev/null +++ b/src/calibre/ebooks/conversion/config.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre.utils.config import config_dir +from calibre.utils.lock import ExclusiveFile +from calibre import sanitize_file_name +from calibre.customize.conversion import OptionRecommendation + + +config_dir = os.path.join(config_dir, 'conversion') +if not os.path.exists(config_dir): + os.makedirs(config_dir) + +def name_to_path(name): + return os.path.join(config_dir, sanitize_file_name(name)+'.py') + +def save_defaults(name, recs): + path = name_to_path(name) + raw = str(recs) + with open(path, 'wb'): + pass + with ExclusiveFile(path) as f: + f.write(raw) + +def load_defaults(name): + path = name_to_path(name) + if not os.path.exists(path): + open(path, 'wb').close() + with ExclusiveFile(path) as f: + raw = f.read() + r = GuiRecommendations() + if raw: + r.from_string(raw) + return r + +def save_specifics(db, book_id, recs): + raw = str(recs) + db.set_conversion_options(book_id, 'PIPE', raw) + +def load_specifics(db, book_id): + raw = db.conversion_options(book_id, 'PIPE') + r = GuiRecommendations() + if raw: + r.from_string(raw) + return r + +class GuiRecommendations(dict): + + def __new__(cls, *args): + dict.__new__(cls) + obj = super(GuiRecommendations, cls).__new__(cls, *args) + obj.disabled_options = set([]) + return obj + + def to_recommendations(self, level=OptionRecommendation.LOW): + ans = [] + for key, val in self.items(): + ans.append((key, val, level)) + return ans + + def __str__(self): + ans = ['{'] + for key, val in self.items(): + ans.append('\t'+repr(key)+' : '+repr(val)+',') + ans.append('}') + return '\n'.join(ans) + + def from_string(self, raw): + try: + d = eval(raw) + except SyntaxError: + d = None + if d: + self.update(d) + + def merge_recommendations(self, get_option, level, options, + only_existing=False): + for name in options: + if only_existing and name not in self: + continue + opt = get_option(name) + if opt is None: continue + if opt.level == OptionRecommendation.HIGH: + self[name] = opt.recommended_value + self.disabled_options.add(name) + elif opt.level > level or name not in self: + self[name] = opt.recommended_value + + diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index ee26c3001c..7387cf158e 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -668,6 +668,7 @@ OptionRecommendation(name='list_recipes', self.output_plugin.convert(self.oeb, self.output, self.input_plugin, self.opts, self.log) self.ui_reporter(1.) + self.log(self.output_fmt.upper(), 'output written to', self.output) def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None): ''' diff --git a/src/calibre/ebooks/lit/output.py b/src/calibre/ebooks/lit/output.py index 42be1ecac7..c72d8eae96 100644 --- a/src/calibre/ebooks/lit/output.py +++ b/src/calibre/ebooks/lit/output.py @@ -7,7 +7,8 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import OutputFormatPlugin, \ + OptionRecommendation class LITOutput(OutputFormatPlugin): @@ -15,12 +16,23 @@ class LITOutput(OutputFormatPlugin): author = 'Marshall T. Vandegrift' file_type = 'lit' + recommendations = set([ + ('dont_split_on_page_breaks', False, OptionRecommendation.HIGH), + ]) + def convert(self, oeb, output_path, input_plugin, opts, log): self.log, self.opts, self.oeb = log, opts, oeb from calibre.ebooks.oeb.transforms.manglecase import CaseMangler from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder from calibre.ebooks.lit.writer import LitWriter + from calibre.ebooks.oeb.transforms.split import Split + split = Split(not self.opts.dont_split_on_page_breaks, + max_flow_size=0 + ) + split(self.oeb, self.opts) + + tocadder = HTMLTOCAdder() tocadder(oeb, opts) mangler = CaseMangler() diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 793c607527..0174bf1ec3 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -241,8 +241,13 @@ class MetaInformation(object): self.tags += mi.tags self.tags = list(set(self.tags)) - if getattr(mi, 'cover_data', None) and mi.cover_data[0] is not None: - self.cover_data = mi.cover_data + if getattr(mi, 'cover_data', False): + other_cover = mi.cover_data[-1] + self_cover = self.cover_data[-1] if self.cover_data else '' + if not self_cover: self_cover = '' + if not other_cover: other_cover = '' + if len(other_cover) > len(self_cover): + self.cover_data = mi.cover_data my_comments = getattr(self, 'comments', '') other_comments = getattr(mi, 'comments', '') diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index a176c12c2b..1460a97eee 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -28,6 +28,14 @@ def path_to_ext(path): return os.path.splitext(path)[1][1:].lower() def metadata_from_formats(formats): + try: + return _metadata_from_formats(formats) + except: + mi = metadata_from_filename(formats[0]) + if not mi.authors: + mi.authors = [_('Unknown')] + +def _metadata_from_formats(formats): mi = MetaInformation(None, None) formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], METADATA_PRIORITIES[path_to_ext(y)])) diff --git a/src/calibre/ebooks/metadata/mobi.py b/src/calibre/ebooks/metadata/mobi.py index e58cfbca4d..f49a57a7c6 100644 --- a/src/calibre/ebooks/metadata/mobi.py +++ b/src/calibre/ebooks/metadata/mobi.py @@ -9,12 +9,9 @@ __copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net and ' \ 'Marshall T. Vandegrift ' __docformat__ = 'restructuredtext en' -import sys -import os from struct import pack, unpack from cStringIO import StringIO from calibre.ebooks.mobi import MobiError -from calibre.ebooks.mobi.reader import get_metadata from calibre.ebooks.mobi.writer import rescale_image, MAX_THUMB_DIMEN from calibre.ebooks.mobi.langcodes import iana2mobi @@ -116,8 +113,13 @@ class MetadataUpdater(object): def update(self, mi): recs = [] - from calibre.ebooks.mobi.from_any import config - if mi.author_sort and config().parse().prefer_author_sort: + try: + from calibre.ebooks.conversion.config import load_defaults + prefs = load_defaults('mobi_output') + pas = prefs.get('prefer_author_sort', False) + except: + pas = False + if mi.author_sort and pas: authors = mi.author_sort recs.append((100, authors.encode(self.codec, 'replace'))) elif mi.authors: diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 4b7648d81f..c735fbe940 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -387,6 +387,8 @@ class MetadataField(object): ans = self.formatter(ans) except: return None + if hasattr(ans, 'strip'): + ans = ans.strip() return ans def __get__(self, obj, type=None): diff --git a/src/calibre/ebooks/metadata/pdf.py b/src/calibre/ebooks/metadata/pdf.py index 038418cd41..0ffa0fb5a3 100644 --- a/src/calibre/ebooks/metadata/pdf.py +++ b/src/calibre/ebooks/metadata/pdf.py @@ -18,13 +18,13 @@ except: from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.utils.pdftk import set_metadata as pdftk_set_metadata from calibre.utils.podofo import get_metadata as podofo_get_metadata, \ - set_metadata as podofo_set_metadata + set_metadata as podofo_set_metadata, Unavailable def get_metadata(stream, extract_cover=True): try: mi = podofo_get_metadata(stream) - except: + except Unavailable: mi = get_metadata_pypdf(stream) stream.seek(0) @@ -43,7 +43,7 @@ def set_metadata(stream, mi): stream.seek(0) try: return podofo_set_metadata(stream, mi) - except: + except Unavailable: pass try: return pdftk_set_metadata(stream, mi) diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py new file mode 100644 index 0000000000..66a992df98 --- /dev/null +++ b/src/calibre/ebooks/metadata/worker.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from threading import Thread +from Queue import Empty +import os, time, sys + +from calibre.utils.ipc.job import ParallelJob +from calibre.utils.ipc.server import Server +from calibre.ptempfile import PersistentTemporaryDirectory +from calibre import prints + + +def debug(*args): + prints(*args) + sys.stdout.flush() + +def read_metadata_(task, tdir, notification=lambda x,y:x): + from calibre.ebooks.metadata.meta import metadata_from_formats + from calibre.ebooks.metadata.opf2 import OPFCreator + for x in task: + id, formats = x + if isinstance(formats, basestring): formats = [formats] + mi = metadata_from_formats(formats) + mi.cover = None + cdata = None + if mi.cover_data: + cdata = mi.cover_data[-1] + mi.cover_data = None + opf = OPFCreator(tdir, mi) + with open(os.path.join(tdir, '%s.opf'%id), 'wb') as f: + opf.render(f) + if cdata: + with open(os.path.join(tdir, str(id)), 'wb') as f: + f.write(cdata) + notification(0.5, id) + +class Progress(object): + + def __init__(self, result_queue, tdir): + self.result_queue = result_queue + self.tdir = tdir + + def __call__(self, id): + cover = os.path.join(self.tdir, str(id)) + if not os.path.exists(cover): cover = None + self.result_queue.put((id, os.path.join(self.tdir, '%s.opf'%id), cover)) + +class ReadMetadata(Thread): + + def __init__(self, tasks, result_queue): + self.tasks, self.result_queue = tasks, result_queue + self.canceled = False + Thread.__init__(self) + self.daemon = True + self.tdir = PersistentTemporaryDirectory('_rm_worker') + + + def run(self): + jobs, ids = set([]), set([]) + for t in self.tasks: + for b in t: + ids.add(b[0]) + progress = Progress(self.result_queue, self.tdir) + server = Server() + for i, task in enumerate(self.tasks): + job = ParallelJob('read_metadata', + 'Read metadata (%d of %d)'%(i, len(self.tasks)), + lambda x,y:x, args=[task, self.tdir]) + jobs.add(job) + server.add_job(job) + + while not self.canceled: + time.sleep(0.2) + running = False + for job in jobs: + while True: + try: + id = job.notifications.get_nowait()[-1] + if id in ids: + progress(id) + ids.remove(id) + except Empty: + break + job.update() + if not job.is_finished: + running = True + + if not running: + break + + server.close() + time.sleep(1) + + if self.canceled: + return + + for id in ids: + print 11111111, id + progress(id) + + for job in jobs: + if job.failed: + prints(job.details) + if os.path.exists(job.log_path): + os.remove(job.log_path) + + +def read_metadata(paths, result_queue, chunk=50): + tasks = [] + pos = 0 + while pos < len(paths): + tasks.append(paths[pos:pos+chunk]) + pos += chunk + t = ReadMetadata(tasks, result_queue) + t.start() + return t diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index bbac34f0b1..bdf78f96e4 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -96,8 +96,12 @@ def iterlinks(root): for el in root.iter(): attribs = el.attrib + try: + tag = el.tag + except UnicodeDecodeError: + continue - if el.tag == XHTML('object'): + if tag == XHTML('object'): codebase = None ## tags have attributes that are relative to ## codebase @@ -122,7 +126,7 @@ def iterlinks(root): yield (el, attr, attribs[attr], 0) - if el.tag == XHTML('style') and el.text: + if tag == XHTML('style') and el.text: for match in _css_url_re.finditer(el.text): yield (el, None, match.group(1), match.start(1)) for match in _css_import_re.finditer(el.text): @@ -801,6 +805,11 @@ class Manifest(object): self.oeb.logger.warn( 'File %r missing element' % self.href) etree.SubElement(data, XHTML('body')) + + # Remove microsoft office markup + r = [x for x in data.iterdescendants(etree.Element) if 'microsoft-com' in x.tag] + for x in r: + x.tag = XHTML('span') return data def _parse_css(self, data): diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index a6873fac2a..8cac967890 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -18,7 +18,7 @@ from xml.dom import SyntaxErr as CSSSyntaxError import cssutils from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \ CSSValueList, cssproperties -from cssutils.profiles import profiles as cssprofiles +from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES diff --git a/src/calibre/ebooks/oeb/transforms/structure.py b/src/calibre/ebooks/oeb/transforms/structure.py index 8ec3c7737a..5e3322a57a 100644 --- a/src/calibre/ebooks/oeb/transforms/structure.py +++ b/src/calibre/ebooks/oeb/transforms/structure.py @@ -12,7 +12,13 @@ from lxml import etree from urlparse import urlparse from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML -XPath = lambda x: etree.XPath(x, namespaces=XPNSMAP) +from calibre.ebooks import ConversionError +def XPath(x): + try: + return etree.XPath(x, namespaces=XPNSMAP) + except etree.XPathSyntaxError: + raise ConversionError( + 'The syntax of the XPath expression %s is invalid.' % repr(x)) class DetectStructure(object): diff --git a/src/calibre/ebooks/pdf/pdftohtml.py b/src/calibre/ebooks/pdf/pdftohtml.py index e03d7d0647..2c0daf05ca 100644 --- a/src/calibre/ebooks/pdf/pdftohtml.py +++ b/src/calibre/ebooks/pdf/pdftohtml.py @@ -6,7 +6,7 @@ __copyright__ = '2008, Kovid Goyal , ' \ '2009, John Schember ' __docformat__ = 'restructuredtext en' -import errno, os, re, sys, subprocess +import errno, os, sys, subprocess from functools import partial from calibre.ebooks import ConversionError, DRMError @@ -33,7 +33,7 @@ def pdftohtml(pdf_path): if isinstance(pdf_path, unicode): pdf_path = pdf_path.encode(sys.getfilesystemencoding()) if not os.access(pdf_path, os.R_OK): - raise ConversionError, 'Cannot read from ' + pdf_path + raise ConversionError('Cannot read from ' + pdf_path) with TemporaryDirectory('_pdftohtml') as tdir: index = os.path.join(tdir, 'index.html') @@ -47,7 +47,7 @@ def pdftohtml(pdf_path): p = popen(cmd, stderr=subprocess.PIPE) except OSError, err: if err.errno == 2: - raise ConversionError(_('Could not find pdftohtml, check it is in your PATH'), True) + raise ConversionError(_('Could not find pdftohtml, check it is in your PATH')) else: raise @@ -63,13 +63,13 @@ def pdftohtml(pdf_path): if ret != 0: err = p.stderr.read() - raise ConversionError, err + raise ConversionError(err) if not os.path.exists(index) or os.stat(index).st_size < 100: raise DRMError() - + with open(index, 'rb') as i: raw = i.read() if not '\n' + raw diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 2afa65e690..2a5a02495a 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -122,7 +122,7 @@ def question_dialog(parent, title, msg, det_msg=''): parent) d.setDetailedText(det_msg) d.setIconPixmap(QPixmap(':/images/dialog_information.svg')) - return d + return d.exec_() == QMessageBox.Yes def info_dialog(parent, title, msg, det_msg='', show=False): d = QMessageBox(QMessageBox.Information, title, msg, QMessageBox.NoButton, diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index bd99864dbd..d0aeb3b300 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -2,238 +2,163 @@ UI for adding books to the database ''' import os +from Queue import Queue, Empty -from PyQt4.Qt import QThread, SIGNAL, QMutex, QWaitCondition, Qt +from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt from calibre.gui2.dialogs.progress import ProgressDialog +from calibre.gui2 import question_dialog +from calibre.ebooks.metadata.opf2 import OPF +from calibre.ebooks.metadata import MetaInformation from calibre.constants import preferred_encoding -from calibre.gui2 import warning_dialog -class Add(QThread): - - def __init__(self): - QThread.__init__(self) - self._lock = QMutex() - self._waiting = QWaitCondition() - - def is_canceled(self): - if self.pd.canceled: - self.canceled = True - return self.canceled - - def wait_for_condition(self): - self._lock.lock() - self._waiting.wait(self._lock) - self._lock.unlock() - - def wake_up(self): - self._waiting.wakeAll() - -class AddFiles(Add): - - def __init__(self, paths, default_thumbnail, get_metadata, db=None): - Add.__init__(self) - self.paths = paths - self.get_metadata = get_metadata - self.default_thumbnail = default_thumbnail +class RecursiveFind(QThread): + + def __init__(self, parent, db, root, single): + QThread.__init__(self, parent) self.db = db - self.formats, self.metadata, self.names, self.infos = [], [], [], [] - self.duplicates = [] + self.path = root + self.single_book_per_directory = single + self.canceled = False + + def run(self): + root = os.path.abspath(self.path) + self.books = [] + for dirpath in os.walk(root): + if self.canceled: + return + self.emit(SIGNAL('update(PyQt_PyObject)'), + _('Searching in')+' '+dirpath[0]) + self.books += list(self.db.find_books_in_directory(dirpath[0], + self.single_book_per_directory)) + self.books = [formats for formats in self.books if formats] + + if not self.canceled: + self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) + + +class Adder(QObject): + + def __init__(self, parent, db, callback): + QObject.__init__(self, parent) + self.pd = ProgressDialog(_('Add books'), parent=parent) + self.db = db + self.pd.setModal(True) + self.pd.show() + self._parent = parent self.number_of_books_added = 0 - self.connect(self.get_metadata, - SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), - self.metadata_delivered) - - def metadata_delivered(self, id, mi): - if self.is_canceled(): - self.wake_up() + self.rfind = self.worker = self.timer = None + self.callback = callback + self.callback_called = False + self.infos, self.paths, self.names = [], [], [] + self.connect(self.pd, SIGNAL('canceled()'), self.canceled) + + def add_recursive(self, root, single=True): + self.path = root + self.pd.set_msg(_('Searching for books in all sub-directories...')) + self.pd.set_min(0) + self.pd.set_max(0) + self.pd.value = 0 + self.rfind = RecursiveFind(self, self.db, root, single) + self.connect(self.rfind, SIGNAL('update(PyQt_PyObject)'), + self.pd.set_msg, Qt.QueuedConnection) + self.connect(self.rfind, SIGNAL('found(PyQt_PyObject)'), + self.add, Qt.QueuedConnection) + self.rfind.start() + + def add(self, books): + books = [[b] if isinstance(b, basestring) else b for b in books] + self.rfind = None + from calibre.ebooks.metadata.worker import read_metadata + self.rq = Queue() + tasks = [] + self.ids = {} + self.nmap = {} + self.duplicates = [] + for i, b in enumerate(books): + tasks.append((i, b)) + self.ids[i] = b + self.nmap[i] = os.path.basename(b[0]) + self.worker = read_metadata(tasks, self.rq) + self.pd.set_min(0) + self.pd.set_max(len(self.ids)) + self.pd.value = 0 + self.timer = QTimer(self) + self.connect(self.timer, SIGNAL('timeout()'), self.update) + self.timer.start(200) + + def add_formats(self, id, formats): + for path in formats: + fmt = os.path.splitext(path)[-1].replace('.', '').upper() + self.db.add_format(id, fmt, open(path, 'rb'), index_is_id=True, + notify=False) + + def canceled(self): + if self.rfind is not None: + self.rfind.cenceled = True + if self.timer is not None: + self.timer.stop() + if self.worker is not None: + self.worker.canceled = True + self.pd.hide() + if not self.callback_called: + self.callback(self.paths, self.names, self.infos) + self.callback_called = True + + + + def update(self): + if not self.ids: + self.timer.stop() + self.process_duplicates() + self.pd.hide() + if not self.callback_called: + self.callback(self.paths, self.names, self.infos) + self.callback_called = True return + + try: + id, opf, cover = self.rq.get_nowait() + except Empty: + return + self.pd.value += 1 + formats = self.ids.pop(id) + mi = MetaInformation(OPF(opf)) + name = self.nmap.pop(id) if not mi.title: - mi.title = os.path.splitext(self.names[id])[0] + mi.title = os.path.splitext(name)[0] mi.title = mi.title if isinstance(mi.title, unicode) else \ mi.title.decode(preferred_encoding, 'replace') - self.metadata.append(mi) - self.infos.append({'title':mi.title, - 'authors':', '.join(mi.authors), - 'cover':self.default_thumbnail, 'tags':[]}) + self.pd.set_msg(_('Added')+' '+mi.title) + if self.db is not None: - duplicates, num = self.db.add_books(self.paths[id:id+1], - self.formats[id:id+1], [mi], - add_duplicates=False) - self.number_of_books_added += num - if duplicates: - if not self.duplicates: - self.duplicates = [[], [], [], []] - for i in range(4): - self.duplicates[i] += duplicates[i] - self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), - mi.title, id) - self.wake_up() - - def create_progress_dialog(self, title, msg, parent): - self._parent = parent - self.pd = ProgressDialog(title, msg, -1, len(self.paths)-1, parent) - self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), - self.update_progress_dialog) - self.pd.setModal(True) - self.pd.show() - self.connect(self, SIGNAL('finished()'), self.pd.hide) - return self.pd - - - def update_progress_dialog(self, title, count): - self.pd.set_value(count) - if self.db is not None: - self.pd.set_msg(_('Added %s to library')%title) + if cover: + cover = open(cover, 'rb').read() + id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False) + self.number_of_books_added += 1 + if id is None: + self.duplicates.append((mi, cover, formats)) + else: + self.add_formats(id, formats) else: - self.pd.set_msg(_('Read metadata from ')+title) - - - def run(self): - try: - self.canceled = False - for c, book in enumerate(self.paths): - if self.pd.canceled: - self.canceled = True - break - format = os.path.splitext(book)[1] - format = format[1:] if format else None - stream = open(book, 'rb') - self.formats.append(format) - self.names.append(os.path.basename(book)) - self.get_metadata(c, stream, stream_type=format, - use_libprs_metadata=True) - self.wait_for_condition() - finally: - self.disconnect(self.get_metadata, - SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), - self.metadata_delivered) - self.get_metadata = None - - + self.names.append(name) + self.paths.append(formats[0]) + self.infos.append({'title':mi.title, + 'authors':', '.join(mi.authors), + 'cover':None, + 'tags':mi.tags if mi.tags else []}) + def process_duplicates(self): - if self.duplicates: - files = '' - for mi in self.duplicates[2]: - files += mi.title+'\n' - d = warning_dialog(_('Duplicates found!'), - _('Books with the same title as the following already ' - 'exist in the database. Add them anyway?'), - files, parent=self._parent) - if d.exec_() == d.Accepted: - num = self.db.add_books(*self.duplicates, - **dict(add_duplicates=True))[1] - self.number_of_books_added += num - - -class AddRecursive(Add): - - def __init__(self, path, db, get_metadata, single_book_per_directory, parent): - self.path = path - self.db = db - self.get_metadata = get_metadata - self.single_book_per_directory = single_book_per_directory - self.duplicates, self.books, self.metadata = [], [], [] - self.number_of_books_added = 0 - self.canceled = False - Add.__init__(self) - self.connect(self.get_metadata, - SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), - self.metadata_delivered, Qt.QueuedConnection) - self.connect(self, SIGNAL('searching_done()'), self.searching_done, - Qt.QueuedConnection) - self._parent = parent - self.pd = ProgressDialog(_('Adding books recursively...'), - _('Searching for books in all sub-directories...'), - 0, 0, parent) - self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), - self.update_progress_dialog) - self.connect(self, SIGNAL('update(PyQt_PyObject)'), self.pd.set_msg, - Qt.QueuedConnection) - self.connect(self, SIGNAL('pupdate(PyQt_PyObject)'), self.pd.set_value, - Qt.QueuedConnection) - self.pd.setModal(True) - self.pd.show() - self.connect(self, SIGNAL('finished()'), self.pd.hide) - - def update_progress_dialog(self, title, count): - self.pd.set_value(count) - if title: - self.pd.set_msg(_('Read metadata from ')+title) - - def metadata_delivered(self, id, mi): - if self.is_canceled(): - self.wake_up() + if not self.duplicates: return - self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), - mi.title, id) - self.metadata.append((mi if mi.title else None, self.books[id])) - if len(self.metadata) >= len(self.books): - self.metadata = [x for x in self.metadata if x[0] is not None] - self.pd.set_min(-1) - self.pd.set_max(len(self.metadata)-1) - self.pd.set_value(-1) - self.pd.set_msg(_('Adding books to database...')) - self.wake_up() - - def searching_done(self): - self.pd.set_min(-1) - self.pd.set_max(len(self.books)-1) - self.pd.set_value(-1) - self.pd.set_msg(_('Reading metadata...')) - - - def run(self): - try: - root = os.path.abspath(self.path) - for dirpath in os.walk(root): - if self.is_canceled(): - return - self.emit(SIGNAL('update(PyQt_PyObject)'), - _('Searching in')+' '+dirpath[0]) - self.books += list(self.db.find_books_in_directory(dirpath[0], - self.single_book_per_directory)) - self.books = [formats for formats in self.books if formats] - # Reset progress bar - self.emit(SIGNAL('searching_done()')) - - for c, formats in enumerate(self.books): - self.get_metadata.from_formats(c, formats) - self.wait_for_condition() - - # Add books to database - for c, x in enumerate(self.metadata): - mi, formats = x - if self.is_canceled(): - break - if self.db.has_book(mi): - self.duplicates.append((mi, formats)) - else: - self.db.import_book(mi, formats, notify=False) - self.number_of_books_added += 1 - self.emit(SIGNAL('pupdate(PyQt_PyObject)'), c) - finally: - self.disconnect(self.get_metadata, - SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), - self.metadata_delivered) - self.get_metadata = None - - - def process_duplicates(self): - if self.duplicates: - files = '' - for mi in self.duplicates: - title = mi[0].title - if not isinstance(title, unicode): - title = title.decode(preferred_encoding, 'replace') - files += title+'\n' - d = warning_dialog(_('Duplicates found!'), + files = [x[0].title for x in self.duplicates] + if question_dialog(self._parent, _('Duplicates found!'), _('Books with the same title as the following already ' - 'exist in the database. Add them anyway?'), - files, parent=self._parent) - if d.exec_() == d.Accepted: - for mi, formats in self.duplicates: - self.db.import_book(mi, formats, notify=False) - self.number_of_books_added += 1 - - + 'exist in the database. Add them anyway?'), + '\n'.join(files)): + for mi, cover, formats in self.duplicates: + id = self.db.create_book_entry(mi, cover=cover, + add_duplicates=True) + self.add_formats(id, formats) + self.number_of_books_added += 1 + diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index 67b6e47aa9..2050108bde 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -6,96 +6,13 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os - from PyQt4.Qt import QWidget, QSpinBox, QDoubleSpinBox, QLineEdit, QTextEdit, \ QCheckBox, QComboBox, Qt, QIcon, SIGNAL from calibre.customize.conversion import OptionRecommendation -from calibre.utils.config import config_dir -from calibre.utils.lock import ExclusiveFile -from calibre import sanitize_file_name - -config_dir = os.path.join(config_dir, 'conversion') -if not os.path.exists(config_dir): - os.makedirs(config_dir) - -def name_to_path(name): - return os.path.join(config_dir, sanitize_file_name(name)+'.py') - -def save_defaults(name, recs): - path = name_to_path(name) - raw = str(recs) - with open(path, 'wb'): - pass - with ExclusiveFile(path) as f: - f.write(raw) -save_defaults_ = save_defaults - -def load_defaults(name): - path = name_to_path(name) - if not os.path.exists(path): - open(path, 'wb').close() - with ExclusiveFile(path) as f: - raw = f.read() - r = GuiRecommendations() - if raw: - r.from_string(raw) - return r - -def save_specifics(db, book_id, recs): - raw = str(recs) - db.set_conversion_options(book_id, 'PIPE', raw) - -def load_specifics(db, book_id): - raw = db.conversion_options(book_id, 'PIPE') - r = GuiRecommendations() - if raw: - r.from_string(raw) - return r - -class GuiRecommendations(dict): - - def __new__(cls, *args): - dict.__new__(cls) - obj = super(GuiRecommendations, cls).__new__(cls, *args) - obj.disabled_options = set([]) - return obj - - def to_recommendations(self, level=OptionRecommendation.LOW): - ans = [] - for key, val in self.items(): - ans.append((key, val, level)) - return ans - - def __str__(self): - ans = ['{'] - for key, val in self.items(): - ans.append('\t'+repr(key)+' : '+repr(val)+',') - ans.append('}') - return '\n'.join(ans) - - def from_string(self, raw): - try: - d = eval(raw) - except SyntaxError: - d = None - if d: - self.update(d) - - def merge_recommendations(self, get_option, level, options, - only_existing=False): - for name in options: - if only_existing and name not in self: - continue - opt = get_option(name) - if opt is None: continue - if opt.level == OptionRecommendation.HIGH: - self[name] = opt.recommended_value - self.disabled_options.add(name) - elif opt.level > level or name not in self: - self[name] = opt.recommended_value - +from calibre.ebooks.conversion.config import load_defaults, \ + save_defaults as save_defaults_, \ + load_specifics, GuiRecommendations class Widget(QWidget): @@ -176,7 +93,7 @@ class Widget(QWidget): elif isinstance(g, QCheckBox): return bool(g.isChecked()) elif isinstance(g, XPathEdit): - return g.xpath + return g.xpath if g.xpath else None else: raise Exception('Can\'t get value from %s'%type(g)) diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index c0da36e5dd..8aaa9de7cd 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -11,7 +11,7 @@ import sys, cPickle from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont from calibre.gui2 import ResizableDialog, NONE -from calibre.gui2.convert import GuiRecommendations, save_specifics, \ +from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics, \ load_specifics from calibre.gui2.convert.single_ui import Ui_Dialog from calibre.gui2.convert.metadata import MetadataWidget @@ -189,6 +189,8 @@ class Config(ResizableDialog, Ui_Dialog): def setup_input_output_formats(self, db, book_id, preferred_input_format, preferred_output_format): + if preferred_output_format: + preferred_output_format = preferred_output_format.lower() available_formats = db.formats(book_id, index_is_id=True) if not available_formats: available_formats = '' diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a22703c748..b58f90daf6 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1,7 +1,7 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import os, traceback, Queue, time, socket +import os, traceback, Queue, time, socket, cStringIO from threading import Thread, RLock from itertools import repeat from functools import partial @@ -15,7 +15,7 @@ from calibre.customize.ui import available_input_formats, available_output_forma from calibre.devices.interface import DevicePlugin from calibre.constants import iswindows from calibre.gui2.dialogs.choose_format import ChooseFormatDialog -from calibre.parallel import Job +from calibre.utils.ipc.job import BaseJob from calibre.devices.scanner import DeviceScanner from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \ pixmap_to_data, warning_dialog, \ @@ -27,22 +27,46 @@ from calibre.devices.errors import FreeSpaceError from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config -class DeviceJob(Job): +class DeviceJob(BaseJob): - def __init__(self, func, *args, **kwargs): - Job.__init__(self, *args, **kwargs) + def __init__(self, func, done, job_manager, args=[], kwargs={}, + description=''): + BaseJob.__init__(self, description, done=done) self.func = func + self.args, self.kwargs = args, kwargs + self.exception = None + self.job_manager = job_manager + self._details = _('No details available.') + + def start_work(self): + self.start_time = time.time() + self.job_manager.changed_queue.put(self) + + def job_done(self): + self.duration = time.time() - self.start_time + self.percent = 1 + self.job_manager.changed_queue.put(self) + + def report_progress(self, percent, msg=''): + self.notifications.put((percent, msg)) + self.job_manager.changed_queue.put(self) def run(self): self.start_work() try: self.result = self.func(*self.args, **self.kwargs) except (Exception, SystemExit), err: + self.failed = True + self._details = unicode(err) + '\n\n' + \ + traceback.format_exc() self.exception = err - self.traceback = traceback.format_exc() finally: self.job_done() + @property + def log_file(self): + return cStringIO.StringIO(self._details.encode('utf-8')) + class DeviceManager(Thread): @@ -113,7 +137,7 @@ class DeviceManager(Thread): job = self.next() if job is not None: self.current_job = job - self.device.set_progress_reporter(job.update_status) + self.device.set_progress_reporter(job.report_progress) self.current_job.run() self.current_job = None else: @@ -206,7 +230,6 @@ class DeviceManager(Thread): def _view_book(self, path, target): f = open(target, 'wb') - print self.device self.device.get_file(path, f) f.close() return target @@ -355,12 +378,12 @@ class DeviceMenu(QMenu): if action.dest == 'main:': action.setEnabled(True) elif action.dest == 'carda:0': - if card_prefix[0] != None: + if card_prefix and card_prefix[0] != None: action.setEnabled(True) else: action.setEnabled(False) elif action.dest == 'cardb:0': - if card_prefix[1] != None: + if card_prefix and card_prefix[1] != None: action.setEnabled(True) else: action.setEnabled(False) @@ -463,7 +486,8 @@ class DeviceGUI(object): fmts = [x.strip().lower() for x in fmts.split(',')] self.send_by_mail(to, fmts, delete) - def send_by_mail(self, to, fmts, delete_from_library, send_ids=None, do_auto_convert=True): + def send_by_mail(self, to, fmts, delete_from_library, send_ids=None, + do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids if not ids or len(ids) == 0: return @@ -474,7 +498,7 @@ class DeviceGUI(object): ids = list(set(ids).difference(_auto_ids)) else: _auto_ids = [] - + full_metadata = self.library_view.model().get_metadata( ids, full_metadata=True, rows_are_ids=True)[-1] files = [getattr(f, 'name', None) for f in files] @@ -537,10 +561,10 @@ class DeviceGUI(object): bad += auto else: autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto] - autos = '\n'.join('
  • %s
    • '%(i,) for i in autos) - d = info_dialog(self, _('No suitable formats'), - _('Auto converting the following books before uploading to the device:
      %s
    ')%(autos,)) - d.exec_() + autos = '\n'.join('%s'%i for i in autos) + info_dialog(self, _('No suitable formats'), + _('Auto converting the following books before uploading to ' + 'the device:'), det_msg=autos, show=True) self.auto_convert_mail(to, delete_from_library, auto, format) if bad: @@ -724,10 +748,10 @@ class DeviceGUI(object): bad += auto else: autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto] - autos = '\n'.join('
  • %s
    • '%(i,) for i in autos) - d = info_dialog(self, _('No suitable formats'), - _('Auto converting the following books before uploading to the device:
      %s
    ')%(autos,)) - d.exec_() + autos = '\n'.join('%s'%i for i in autos) + info_dialog(self, _('No suitable formats'), + _('Auto converting the following books before uploading to ' + 'the device:'), det_msg=autos, show=True) self.auto_convert(auto, on_card, format) if bad: @@ -750,7 +774,7 @@ class DeviceGUI(object): ''' Called once metadata has been uploaded. ''' - if job.exception is not None: + if job.failed: self.device_job_exception(job) return cp, fs = job.result diff --git a/src/calibre/gui2/dialogs/jobs.py b/src/calibre/gui2/dialogs/jobs.py deleted file mode 100644 index c6907c6b60..0000000000 --- a/src/calibre/gui2/dialogs/jobs.py +++ /dev/null @@ -1,66 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' -'''Display active jobs''' - -from PyQt4.QtCore import Qt, QObject, SIGNAL, QSize, QString, QTimer -from PyQt4.QtGui import QDialog, QAbstractItemDelegate, QStyleOptionProgressBarV2, \ - QApplication, QStyle - -from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog -from calibre import __appname__ - -class ProgressBarDelegate(QAbstractItemDelegate): - - def sizeHint(self, option, index): - return QSize(120, 30) - - def paint(self, painter, option, index): - opts = QStyleOptionProgressBarV2() - opts.rect = option.rect - opts.minimum = 1 - opts.maximum = 100 - opts.textVisible = True - percent, ok = index.model().data(index, Qt.DisplayRole).toInt() - if not ok: - percent = 0 - opts.progress = percent - opts.text = QString(_('Unavailable') if percent == 0 else '%d%%'%percent) - QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) - -class JobsDialog(QDialog, Ui_JobsDialog): - def __init__(self, window, model): - QDialog.__init__(self, window) - Ui_JobsDialog.__init__(self) - self.setupUi(self) - self.jobs_view.setModel(model) - self.model = model - self.setWindowModality(Qt.NonModal) - self.setWindowTitle(__appname__ + _(' - Jobs')) - QObject.connect(self.jobs_view.model(), SIGNAL('modelReset()'), - self.jobs_view.resizeColumnsToContents) - QObject.connect(self.kill_button, SIGNAL('clicked()'), - self.kill_job) - QObject.connect(self, SIGNAL('kill_job(int, PyQt_PyObject)'), - self.jobs_view.model().kill_job) - self.pb_delegate = ProgressBarDelegate(self) - self.jobs_view.setItemDelegateForColumn(2, self.pb_delegate) - - self.running_time_timer = QTimer(self) - self.connect(self.running_time_timer, SIGNAL('timeout()'), self.update_running_time) - self.running_time_timer.start(1000) - - def update_running_time(self, *args): - try: - self.model.running_time_updated() - except: # Raises random exceptions on OS X - pass - - def kill_job(self): - for index in self.jobs_view.selectedIndexes(): - row = index.row() - self.model.kill_job(row, self) - return - - def closeEvent(self, e): - self.jobs_view.write_settings() - e.accept() diff --git a/src/calibre/gui2/dialogs/jobs.ui b/src/calibre/gui2/dialogs/jobs.ui index a14be17f78..3716c9fbb9 100644 --- a/src/calibre/gui2/dialogs/jobs.ui +++ b/src/calibre/gui2/dialogs/jobs.ui @@ -1,7 +1,8 @@ - + + JobsDialog - - + + 0 0 @@ -9,31 +10,32 @@ 542 - + Active Jobs - - :/images/jobs.svg + + + :/images/jobs.svg:/images/jobs.svg - + - - + + Qt::NoContextMenu - + QAbstractItemView::NoEditTriggers - + true - + QAbstractItemView::SingleSelection - + QAbstractItemView::SelectRows - + 32 32 @@ -42,12 +44,19 @@ - - + + &Stop selected job + + + + Show job &details + + + @@ -58,7 +67,7 @@ - + diff --git a/src/calibre/gui2/dialogs/progress.py b/src/calibre/gui2/dialogs/progress.py index 0f64d7b041..39057c5374 100644 --- a/src/calibre/gui2/dialogs/progress.py +++ b/src/calibre/gui2/dialogs/progress.py @@ -10,7 +10,7 @@ from PyQt4.Qt import QDialog, SIGNAL, Qt from calibre.gui2.dialogs.progress_ui import Ui_Dialog class ProgressDialog(QDialog, Ui_Dialog): - + def __init__(self, title, msg='', min=0, max=99, parent=None): QDialog.__init__(self, parent) self.setupUi(self) @@ -22,28 +22,39 @@ class ProgressDialog(QDialog, Ui_Dialog): self.set_max(max) self.bar.setValue(min) self.canceled = False - + self.connect(self.button_box, SIGNAL('rejected()'), self._canceled) - + def set_msg(self, msg=''): self.message.setText(msg) - + def set_value(self, val): self.bar.setValue(val) - + + @dynamic_property + def value(self): + def fset(self, val): + return self.bar.setValue(val) + def fget(self): + return self.bar.value() + return property(fget=fget, fset=fset) + + + def set_min(self, min): self.bar.setMinimum(min) - + def set_max(self, max): self.bar.setMaximum(max) - + def _canceled(self, *args): self.canceled = True self.button_box.setDisabled(True) self.title.setText(_('Aborting...')) - + self.emit(SIGNAL('canceled()')) + def keyPressEvent(self, ev): if ev.key() == Qt.Key_Escape: self._canceled() else: - QDialog.keyPressEvent(self, ev) \ No newline at end of file + QDialog.keyPressEvent(self, ev) diff --git a/src/calibre/gui2/dialogs/tag_editor.py b/src/calibre/gui2/dialogs/tag_editor.py index b50649f761..5d17f3a693 100644 --- a/src/calibre/gui2/dialogs/tag_editor.py +++ b/src/calibre/gui2/dialogs/tag_editor.py @@ -1,19 +1,19 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' from PyQt4.QtCore import SIGNAL, Qt -from PyQt4.QtGui import QDialog, QMessageBox +from PyQt4.QtGui import QDialog from calibre.gui2.dialogs.tag_editor_ui import Ui_TagEditor from calibre.gui2 import qstring_to_unicode from calibre.gui2 import question_dialog, error_dialog class TagEditor(QDialog, Ui_TagEditor): - + def __init__(self, window, db, index=None): QDialog.__init__(self, window) Ui_TagEditor.__init__(self) self.setupUi(self) - + self.db = db self.index = index if self.index is not None: @@ -27,16 +27,16 @@ class TagEditor(QDialog, Ui_TagEditor): self.applied_tags.addItem(tag) else: tags = [] - + self.tags = tags - + all_tags = [tag for tag in self.db.all_tags()] all_tags = list(set(all_tags)) all_tags.sort() for tag in all_tags: if tag not in tags: self.available_tags.addItem(tag) - + self.connect(self.apply_button, SIGNAL('clicked()'), self.apply_tags) self.connect(self.unapply_button, SIGNAL('clicked()'), self.unapply_tags) self.connect(self.add_tag_button, SIGNAL('clicked()'), self.add_tag) @@ -44,8 +44,8 @@ class TagEditor(QDialog, Ui_TagEditor): self.connect(self.add_tag_input, SIGNAL('returnPressed()'), self.add_tag) self.connect(self.available_tags, SIGNAL('itemActivated(QListWidgetItem*)'), self.apply_tags) self.connect(self.applied_tags, SIGNAL('itemActivated(QListWidgetItem*)'), self.unapply_tags) - - + + def delete_tags(self, item=None): confirms, deletes = [], [] items = self.available_tags.selectedItems() if item is None else [item] @@ -56,47 +56,47 @@ class TagEditor(QDialog, Ui_TagEditor): if self.db.is_tag_used(qstring_to_unicode(item.text())): confirms.append(item) else: - deletes.append(item) + deletes.append(item) if confirms: ct = ', '.join([qstring_to_unicode(item.text()) for item in confirms]) - d = question_dialog(self, 'Are your sure?', - '

    The following tags are used by one or more books. Are you certain you want to delete them?
    '+ct) - if d.exec_() == QMessageBox.Yes: + if question_dialog(self, _('Are your sure?'), + '

    '+_('The following tags are used by one or more books. ' + 'Are you certain you want to delete them?')+'
    '+ct): deletes += confirms - + for item in deletes: self.db.delete_tag(qstring_to_unicode(item.text())) self.available_tags.takeItem(self.available_tags.row(item)) - - + + def apply_tags(self, item=None): - items = self.available_tags.selectedItems() if item is None else [item] + items = self.available_tags.selectedItems() if item is None else [item] for item in items: tag = qstring_to_unicode(item.text()) self.tags.append(tag) self.available_tags.takeItem(self.available_tags.row(item)) - + self.tags.sort() self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) - - - + + + def unapply_tags(self, item=None): - items = self.applied_tags.selectedItems() if item is None else [item] + items = self.applied_tags.selectedItems() if item is None else [item] for item in items: tag = qstring_to_unicode(item.text()) self.tags.remove(tag) self.available_tags.addItem(tag) - + self.tags.sort() self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) - + self.available_tags.sortItems() - + def add_tag(self): tags = qstring_to_unicode(self.add_tag_input.text()).split(',') for tag in tags: @@ -105,10 +105,10 @@ class TagEditor(QDialog, Ui_TagEditor): self.available_tags.takeItem(self.available_tags.row(item)) if tag not in self.tags: self.tags.append(tag) - + self.tags.sort() self.applied_tags.clear() for tag in self.tags: self.applied_tags.addItem(tag) - + self.add_tag_input.setText('') diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index cab28c66eb..6a4a3275a8 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal ' import time, os from PyQt4.QtCore import SIGNAL, QUrl -from PyQt4.QtGui import QMessageBox, QDesktopServices +from PyQt4.QtGui import QDesktopServices from calibre.web.feeds.recipes import compile_recipe from calibre.web.feeds.news import AutomaticNewsRecipe @@ -86,7 +86,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog): self.source_code.setPlainText('') else: self.source_code.setPlainText(src) - #self.highlighter = PythonHighlighter(self.source_code.document()) + self.highlighter = PythonHighlighter(self.source_code.document()) self.stacks.setCurrentIndex(1) self.toggle_mode_button.setText(_('Switch to Basic mode')) @@ -175,9 +175,9 @@ class %(classname)s(%(base_class)s): try: self.available_profiles.add_item(title, (title, profile), replace=False) except ValueError: - d = question_dialog(self, _('Replace recipe?'), - _('A custom recipe named %s already exists. Do you want to replace it?')%title) - if d.exec_() == QMessageBox.Yes: + if question_dialog(self, _('Replace recipe?'), + _('A custom recipe named %s already exists. Do you want to ' + 'replace it?')%title): self.available_profiles.add_item(title, (title, profile), replace=True) else: return @@ -207,9 +207,9 @@ class %(classname)s(%(base_class)s): try: self.available_profiles.add_item(title, (title, r.text), replace=False) except ValueError: - d = question_dialog(self, _('Replace recipe?'), - _('A custom recipe named %s already exists. Do you want to replace it?')%title) - if d.exec_() == QMessageBox.Yes: + if question_dialog(self, _('Replace recipe?'), + _('A custom recipe named %s already exists. Do you ' + 'want to replace it?')%title): self.available_profiles.add_item(title, (title, r.text), replace=True) else: return @@ -231,9 +231,9 @@ class %(classname)s(%(base_class)s): try: self.available_profiles.add_item(title, (title, src), replace=False) except ValueError: - d = question_dialog(self, _('Replace recipe?'), - _('A custom recipe named %s already exists. Do you want to replace it?')%title) - if d.exec_() == QMessageBox.Yes: + if question_dialog(self, _('Replace recipe?'), + _('A custom recipe named %s already exists. Do you want to ' + 'replace it?')%title): self.available_profiles.add_item(title, (title, src), replace=True) else: return diff --git a/src/calibre/gui2/images/news/hrt.png b/src/calibre/gui2/images/news/hrt.png new file mode 100644 index 0000000000..828819e226 Binary files /dev/null and b/src/calibre/gui2/images/news/hrt.png differ diff --git a/src/calibre/gui2/images/news/rts.png b/src/calibre/gui2/images/news/rts.png new file mode 100644 index 0000000000..278f45edd7 Binary files /dev/null and b/src/calibre/gui2/images/news/rts.png differ diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py new file mode 100644 index 0000000000..b57e1836d2 --- /dev/null +++ b/src/calibre/gui2/jobs.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Job management. +''' + +from Queue import Empty, Queue + +from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ + QTimer, SIGNAL, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ + QSize, QStyleOptionProgressBarV2, QString, QStyle + +from calibre.utils.ipc.server import Server +from calibre.utils.ipc.job import ParallelJob +from calibre.gui2 import Dispatcher, error_dialog, NONE +from calibre.gui2.device import DeviceJob +from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog +from calibre import __appname__ + +class JobManager(QAbstractTableModel): + + def __init__(self): + QAbstractTableModel.__init__(self) + self.wait_icon = QVariant(QIcon(':/images/jobs.svg')) + self.running_icon = QVariant(QIcon(':/images/exec.svg')) + self.error_icon = QVariant(QIcon(':/images/dialog_error.svg')) + self.done_icon = QVariant(QIcon(':/images/ok.svg')) + + self.jobs = [] + self.add_job = Dispatcher(self._add_job) + self.server = Server() + self.changed_queue = Queue() + + self.timer = QTimer(self) + self.connect(self.timer, SIGNAL('timeout()'), self.update, + Qt.QueuedConnection) + self.timer.start(1000) + + def columnCount(self, parent=QModelIndex()): + return 4 + + def rowCount(self, parent=QModelIndex()): + return len(self.jobs) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + if orientation == Qt.Horizontal: + if section == 0: text = _('Job') + elif section == 1: text = _('Status') + elif section == 2: text = _('Progress') + elif section == 3: text = _('Running time') + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + try: + if role not in (Qt.DisplayRole, Qt.DecorationRole): + return NONE + row, col = index.row(), index.column() + job = self.jobs[row] + + if role == Qt.DisplayRole: + if col == 0: + desc = job.description + if not desc: + desc = _('Unknown job') + return QVariant(desc) + if col == 1: + return QVariant(job.status_text) + if col == 2: + return QVariant(job.percent) + if col == 3: + rtime = job.running_time + if rtime is None: + return NONE + return QVariant('%dm %ds'%(int(rtime)//60, int(rtime)%60)) + if role == Qt.DecorationRole and col == 0: + state = job.run_state + if state == job.WAITING: + return self.wait_icon + if state == job.RUNNING: + return self.running_icon + if job.killed or job.failed: + return self.error_icon + return self.done_icon + except: + import traceback + traceback.print_exc() + return NONE + + def update(self): + try: + self._update() + except BaseException: + import traceback + traceback.print_exc() + + def _update(self): + # Update running time + rows = set([]) + for i, j in enumerate(self.jobs): + if j.run_state == j.RUNNING: + idx = self.index(i, 3) + self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'), + idx, idx) + + # Update parallel jobs + jobs = set([]) + while True: + try: + jobs.add(self.server.changed_jobs_queue.get_nowait()) + except Empty: + break + while True: + try: + jobs.add(self.changed_queue.get_nowait()) + except Empty: + break + + if jobs: + needs_reset = False + for job in jobs: + orig_state = job.run_state + job.update() + if orig_state != job.run_state: + needs_reset = True + if needs_reset: + self.jobs.sort() + self.reset() + if job.is_finished: + self.emit(SIGNAL('job_done(int)'), len(self.unfinished_jobs())) + else: + for job in jobs: + idx = self.jobs.index(job) + self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'), + self.index(idx, 0), self.index(idx, 3)) + + + def _add_job(self, job): + self.emit(SIGNAL('layoutAboutToBeChanged()')) + self.jobs.append(job) + self.jobs.sort() + self.emit(SIGNAL('job_added(int)'), len(self.unfinished_jobs())) + self.emit(SIGNAL('layoutChanged()')) + + def done_jobs(self): + return [j for j in self.jobs if j.is_finished] + + def unfinished_jobs(self): + return [j for j in self.jobs if not j.is_finished] + + def row_to_job(self, row): + return self.jobs[row] + + def has_device_jobs(self): + for job in self.jobs: + if job.is_running and isinstance(job, DeviceJob): + return True + return False + + def has_jobs(self): + for job in self.jobs: + if job.is_running: + return True + return False + + def run_job(self, done, name, args=[], kwargs={}, + description=''): + job = ParallelJob(name, description, done, args=args, kwargs=kwargs) + self.add_job(job) + self.server.add_job(job) + return job + + def launch_gui_app(self, name, args=[], kwargs={}, description=''): + job = ParallelJob(name, description, lambda x: x, + args=args, kwargs=kwargs) + self.server.run_job(job, gui=True, redirect_output=False) + + + def kill_job(self, row, view): + job = self.jobs[row] + if isinstance(job, DeviceJob): + return error_dialog(view, _('Cannot kill job'), + _('Cannot kill jobs that communicate with the device')).exec_() + if job.duration is not None: + return error_dialog(view, _('Cannot kill job'), + _('Job has already run')).exec_() + self.server.kill_job(job) + + def terminate_all_jobs(self): + self.server.killall() + + +class ProgressBarDelegate(QAbstractItemDelegate): + + def sizeHint(self, option, index): + return QSize(120, 30) + + def paint(self, painter, option, index): + opts = QStyleOptionProgressBarV2() + opts.rect = option.rect + opts.minimum = 1 + opts.maximum = 100 + opts.textVisible = True + percent, ok = index.model().data(index, Qt.DisplayRole).toInt() + if not ok: + percent = 0 + opts.progress = percent + opts.text = QString(_('Unavailable') if percent == 0 else '%d%%'%percent) + QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) + +class JobsDialog(QDialog, Ui_JobsDialog): + def __init__(self, window, model): + QDialog.__init__(self, window) + Ui_JobsDialog.__init__(self) + self.setupUi(self) + self.jobs_view.setModel(model) + self.model = model + self.setWindowModality(Qt.NonModal) + self.setWindowTitle(__appname__ + _(' - Jobs')) + self.connect(self.jobs_view.model(), SIGNAL('modelReset()'), + self.jobs_view.resizeColumnsToContents) + self.connect(self.kill_button, SIGNAL('clicked()'), + self.kill_job) + self.connect(self.details_button, SIGNAL('clicked()'), + self.show_details) + self.connect(self, SIGNAL('kill_job(int, PyQt_PyObject)'), + self.jobs_view.model().kill_job) + self.pb_delegate = ProgressBarDelegate(self) + self.jobs_view.setItemDelegateForColumn(2, self.pb_delegate) + + + def kill_job(self): + for index in self.jobs_view.selectedIndexes(): + row = index.row() + self.model.kill_job(row, self) + return + + def show_details(self): + for index in self.jobs_view.selectedIndexes(): + self.jobs_view.show_details(index) + return + + + + def closeEvent(self, e): + self.jobs_view.write_settings() + e.accept() diff --git a/src/calibre/gui2/jobs2.py b/src/calibre/gui2/jobs2.py deleted file mode 100644 index fc6ddb642e..0000000000 --- a/src/calibre/gui2/jobs2.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Job management. -''' -import time -from PyQt4.QtCore import QAbstractTableModel, QVariant, QModelIndex, Qt, SIGNAL -from PyQt4.QtGui import QIcon, QDialog - -from calibre.parallel import ParallelJob, Server -from calibre.gui2 import Dispatcher, error_dialog -from calibre.gui2.device import DeviceJob -from calibre.gui2.dialogs.job_view_ui import Ui_Dialog - -NONE = QVariant() - -class JobManager(QAbstractTableModel): - - def __init__(self): - QAbstractTableModel.__init__(self) - self.wait_icon = QVariant(QIcon(':/images/jobs.svg')) - self.running_icon = QVariant(QIcon(':/images/exec.svg')) - self.error_icon = QVariant(QIcon(':/images/dialog_error.svg')) - self.done_icon = QVariant(QIcon(':/images/ok.svg')) - - self.jobs = [] - self.server = Server() - self.add_job = Dispatcher(self._add_job) - self.status_update = Dispatcher(self._status_update) - self.start_work = Dispatcher(self._start_work) - self.job_done = Dispatcher(self._job_done) - - def columnCount(self, parent=QModelIndex()): - return 4 - - def rowCount(self, parent=QModelIndex()): - return len(self.jobs) - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - if orientation == Qt.Horizontal: - if section == 0: text = _("Job") - elif section == 1: text = _("Status") - elif section == 2: text = _("Progress") - elif section == 3: text = _('Running time') - return QVariant(text) - else: - return QVariant(section+1) - - def data(self, index, role): - try: - if role not in (Qt.DisplayRole, Qt.DecorationRole): - return NONE - row, col = index.row(), index.column() - job = self.jobs[row] - - if role == Qt.DisplayRole: - if col == 0: - desc = job.description - if not desc: - desc = _('Unknown job') - return QVariant(desc) - if col == 1: - status = job.status() - if status == 'DONE': - st = _('Finished') - elif status == 'ERROR': - st = _('Error') - elif status == 'WAITING': - st = _('Waiting') - else: - st = _('Working') - return QVariant(st) - if col == 2: - pc = job.percent - if pc <=0: - percent = 0 - else: - percent = int(100*pc) - return QVariant(percent) - if col == 3: - if job.start_time is None: - return NONE - rtime = job.running_time if job.running_time is not None else \ - time.time() - job.start_time - return QVariant('%dm %ds'%(int(rtime)//60, int(rtime)%60)) - if role == Qt.DecorationRole and col == 0: - status = job.status() - if status == 'WAITING': - return self.wait_icon - if status == 'WORKING': - return self.running_icon - if status == 'ERROR': - return self.error_icon - if status == 'DONE': - return self.done_icon - except: - import traceback - traceback.print_exc() - return NONE - - def _add_job(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.append(job) - self.jobs.sort() - self.emit(SIGNAL('job_added(int)'), self.rowCount()) - self.emit(SIGNAL('layoutChanged()')) - - def done_jobs(self): - return [j for j in self.jobs if j.status() in ['DONE', 'ERROR']] - - def row_to_job(self, row): - return self.jobs[row] - - def _start_work(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.sort() - self.emit(SIGNAL('layoutChanged()')) - - def _job_done(self, job): - self.emit(SIGNAL('layoutAboutToBeChanged()')) - self.jobs.sort() - self.emit(SIGNAL('job_done(int)'), len(self.jobs) - len(self.done_jobs())) - self.emit(SIGNAL('layoutChanged()')) - - def _status_update(self, job): - try: - row = self.jobs.index(job) - except ValueError: # Job has been stopped - return - self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), - self.index(row, 0), self.index(row, 3)) - - def running_time_updated(self, *args): - for job in self.jobs: - if not job.is_running: - continue - row = self.jobs.index(job) - self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'), - self.index(row, 3), self.index(row, 3)) - - def has_device_jobs(self): - for job in self.jobs: - if job.is_running and isinstance(job, DeviceJob): - return True - return False - - def has_jobs(self): - for job in self.jobs: - if job.is_running: - return True - return False - - def run_job(self, done, func, args=[], kwargs={}, - description=None): - job = ParallelJob(func, done, self, args=args, kwargs=kwargs, - description=description) - self.server.add_job(job) - return job - - - def output(self, job): - self.emit(SIGNAL('output_received()')) - - def kill_job(self, row, view): - job = self.jobs[row] - if isinstance(job, DeviceJob): - error_dialog(view, _('Cannot kill job'), - _('Cannot kill jobs that communicate with the device')).exec_() - return - if job.has_run: - error_dialog(view, _('Cannot kill job'), - _('Job has already run')).exec_() - return - if not job.is_running: - self.jobs.remove(job) - self.reset() - return - - - self.server.kill(job) - - def terminate_all_jobs(self): - pass - -class DetailView(QDialog, Ui_Dialog): - - def __init__(self, parent, job): - QDialog.__init__(self, parent) - self.setupUi(self) - self.setWindowTitle(job.description) - self.job = job - self.update() - - - def update(self): - self.log.setPlainText(self.job.console_text()) - vbar = self.log.verticalScrollBar() - vbar.setValue(vbar.maximum()) diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index f35a478847..e036223815 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -723,9 +723,6 @@ class BooksView(TableView): def close(self): self._model.close() - - def set_editable(self, editable): - self._model.set_editable(editable) def set_editable(self, editable): self._model.set_editable(editable) @@ -1019,10 +1016,6 @@ class DeviceBooksModel(BooksModel): self.sort(col, self.sorted_on[1]) done = True return done - - def set_editable(self, editable): - self.editable = editable - def set_editable(self, editable): self.editable = editable diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 83190898ff..8f632ad10e 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -13,7 +13,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \ from PyQt4.QtSvg import QSvgRenderer from calibre import __version__, __appname__, islinux, sanitize_file_name, \ - iswindows, isosx + iswindows, isosx, prints from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \ @@ -32,10 +32,9 @@ from calibre.gui2.main_window import MainWindow, option_parser as _option_parser from calibre.gui2.main_ui import Ui_MainWindow from calibre.gui2.device import DeviceManager, DeviceMenu, DeviceGUI, Emailer from calibre.gui2.status import StatusBar -from calibre.gui2.jobs2 import JobManager +from calibre.gui2.jobs import JobManager, JobsDialog from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog -from calibre.gui2.dialogs.jobs import JobsDialog from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \ fetch_scheduled_recipe from calibre.gui2.dialogs.config import ConfigDialog @@ -44,7 +43,6 @@ from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.gui2.dialogs.book_info import BookInfo from calibre.ebooks import BOOK_EXTENSIONS from calibre.library.database2 import LibraryDatabase2, CoverCache -from calibre.parallel import JobKilled from calibre.gui2.dialogs.confirm_delete import confirm class SaveMenu(QMenu): @@ -202,17 +200,22 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.delete_books) QObject.connect(self.action_edit, SIGNAL("triggered(bool)"), self.edit_metadata) + self.__em1__ = partial(self.edit_metadata, bulk=False) QObject.connect(md.actions()[0], SIGNAL('triggered(bool)'), - partial(self.edit_metadata, bulk=False)) + self.__em1__) + self.__em2__ = partial(self.edit_metadata, bulk=True) QObject.connect(md.actions()[2], SIGNAL('triggered(bool)'), - partial(self.edit_metadata, bulk=True)) + self.__em2__) + self.__em3__ = partial(self.download_metadata, covers=True) QObject.connect(md.actions()[4], SIGNAL('triggered(bool)'), - partial(self.download_metadata, covers=True)) + self.__em3__) + self.__em4__ = partial(self.download_metadata, covers=False) QObject.connect(md.actions()[5], SIGNAL('triggered(bool)'), - partial(self.download_metadata, covers=False)) + self.__em4__) + self.__em5__ = partial(self.download_metadata, covers=True, + set_metadata=False) QObject.connect(md.actions()[6], SIGNAL('triggered(bool)'), - partial(self.download_metadata, covers=True, - set_metadata=False)) + self.__em5__) @@ -626,9 +629,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' Called once device information has been read. ''' - if job.exception is not None: - self.device_job_exception(job) - return + if job.failed: + return self.device_job_exception(job) info, cp, fs = job.result self.location_view.model().update_devices(cp, fs) self.device_info = _('Connected ')+info[0] @@ -641,7 +643,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' Called once metadata has been read for all books on the device. ''' - if job.exception is not None: + if job.failed: if isinstance(job.exception, ExpatError): error_dialog(self, _('Device database corrupted'), _(''' @@ -679,21 +681,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): 'Select root folder') if not root: return - from calibre.gui2.add import AddRecursive - self._add_recursive_thread = AddRecursive(root, - self.library_view.model().db, self.get_metadata, - single, self) - self.connect(self._add_recursive_thread, SIGNAL('finished()'), - self._recursive_files_added) - self._add_recursive_thread.start() - - def _recursive_files_added(self): - self._add_recursive_thread.process_duplicates() - if self._add_recursive_thread.number_of_books_added > 0: - self.library_view.model().resort(reset=False) - self.library_view.model().research() - self.library_view.model().count_changed() - self._add_recursive_thread = None + from calibre.gui2.add import Adder + self._adder = Adder(self, + self.library_view.model().db, + Dispatcher(self._files_added)) + self._adder.add_recursive(root, single) def add_recursive_single(self, checked): ''' @@ -734,10 +726,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): (_('LRF Books'), ['lrf']), (_('HTML Books'), ['htm', 'html', 'xhtm', 'xhtml']), (_('LIT Books'), ['lit']), - (_('MOBI Books'), ['mobi', 'prc']), + (_('MOBI Books'), ['mobi', 'prc', 'azw']), (_('Text books'), ['txt', 'rtf']), (_('PDF Books'), ['pdf']), - (_('Comics'), ['cbz', 'cbr']), + (_('Comics'), ['cbz', 'cbr', 'cbc']), (_('Archives'), ['zip', 'rar']), ]) if not books: @@ -748,40 +740,30 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def _add_books(self, paths, to_device, on_card=None): if on_card is None: - on_card = self.stack.currentIndex() == 2 + on_card = self.stack.currentIndex() >= 2 if not paths: return - from calibre.gui2.add import AddFiles - self._add_files_thread = AddFiles(paths, self.default_thumbnail, - self.get_metadata, - None if to_device else \ - self.library_view.model().db - ) - self._add_files_thread.send_to_device = to_device - self._add_files_thread.on_card = on_card - self._add_files_thread.create_progress_dialog(_('Adding books...'), - _('Reading metadata...'), self) - self.connect(self._add_files_thread, SIGNAL('finished()'), - self._files_added) - self._add_files_thread.start() + from calibre.gui2.add import Adder + self.__adder_func = partial(self._files_added, on_card=on_card) + self._adder = Adder(self, + None if to_device else self.library_view.model().db, + Dispatcher(self.__adder_func)) + self._adder.add(paths) - def _files_added(self): - t = self._add_files_thread - self._add_files_thread = None - if not t.canceled: - if t.send_to_device: - self.upload_books(t.paths, - list(map(sanitize_file_name, t.names)), - t.infos, on_card=t.on_card) - self.status_bar.showMessage( - _('Uploading books to device.'), 2000) - else: - t.process_duplicates() - if t.number_of_books_added > 0: - self.library_view.model().books_added(t.number_of_books_added) + def _files_added(self, paths=[], names=[], infos=[], on_card=False): + if paths: + self.upload_books(paths, + list(map(sanitize_file_name, names)), + infos, on_card=on_card) + self.status_bar.showMessage( + _('Uploading books to device.'), 2000) + if self._adder.number_of_books_added > 0: + self.library_view.model().books_added(self._adder.number_of_books_added) if hasattr(self, 'db_images'): self.db_images.reset() + self._adder = None + ############################################################################ @@ -823,8 +805,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): Called once deletion is done on the device ''' for view in (self.memory_view, self.card_a_view, self.card_b_view): - view.model().deletion_done(job, bool(job.exception)) - if job.exception is not None: + view.model().deletion_done(job, job.failed) + if job.failed: self.device_job_exception(job) return @@ -993,9 +975,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): progress.hide() def books_saved(self, job): - if job.exception is not None: - self.device_job_exception(job) - return + if job.failed: + return self.device_job_exception(job) ############################################################################ @@ -1013,9 +994,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def scheduled_recipe_fetched(self, job): temp_files, fmt, recipe, callback = self.conversion_jobs.pop(job) pt = temp_files[0] - if job.exception is not None: - self.job_exception(job) - return + if job.failed: + return self.job_exception(job) id = self.library_view.model().add_news(pt.name, recipe) self.library_view.model().reset() sync = dynamic.get('news_to_be_synced', set([])) @@ -1047,7 +1027,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): current = self.library_view.currentIndex() self.library_view.model().current_changed(current, previous) - def auto_convert_mail(to, delete_from_library, book_ids, format): + def auto_convert_mail(self, to, delete_from_library, book_ids, format): previous = self.library_view.currentIndex() rows = [x.row() for x in \ self.library_view.selectionModel().selectedRows()] @@ -1057,7 +1037,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): if id not in bad: job = self.job_manager.run_job(Dispatcher(self.book_auto_converted_mail), func, args=args, description=desc) - self.conversion_jobs[job] = (temp_files, fmt, id, delete_from_library) + self.conversion_jobs[job] = (temp_files, fmt, id, + delete_from_library, to) if changed: self.library_view.model().refresh_rows(rows) @@ -1115,9 +1096,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def book_auto_converted(self, job): temp_files, fmt, book_id, on_card = self.conversion_jobs.pop(job) try: - if job.exception is not None: - self.job_exception(job) - return + if job.failed: + return self.job_exception(job) data = open(temp_files[0].name, 'rb') self.library_view.model().db.add_format(book_id, fmt, data, index_is_id=True) data.close() @@ -1137,9 +1117,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.sync_to_device(on_card, False, specific_format=fmt, send_ids=[book_id], do_auto_convert=False) def book_auto_converted_mail(self, job): - temp_files, fmt, book_id, delete_from_library = self.conversion_jobs.pop(job) + temp_files, fmt, book_id, delete_from_library, to = self.conversion_jobs.pop(job) try: - if job.exception is not None: + if job.failed: self.job_exception(job) return data = open(temp_files[0].name, 'rb') @@ -1163,7 +1143,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def book_converted(self, job): temp_files, fmt, book_id = self.conversion_jobs.pop(job) try: - if job.exception is not None: + if job.failed: self.job_exception(job) return data = open(temp_files[-1].name, 'rb') @@ -1192,7 +1172,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self._view_file(fmt_path) def book_downloaded_for_viewing(self, job): - if job.exception: + if job.failed: self.device_job_exception(job) return self._view_file(job.result) @@ -1206,12 +1186,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): args.append('--raise-window') if name is not None: args.append(name) - self.job_manager.server.run_free_job(viewer, - kwdargs=dict(args=args)) + self.job_manager.launch_gui_app(viewer, + kwargs=dict(args=args)) else: QDesktopServices.openUrl(QUrl.fromLocalFile(name))#launch(name) - - time.sleep(5) # User feedback + time.sleep(2) # User feedback finally: self.unsetCursor() @@ -1436,7 +1415,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' try: if 'Could not read 32 bytes on the control bus.' in \ - unicode(job.exception): + unicode(job.details): error_dialog(self, _('Error talking to device'), _('There was a temporary error talking to the ' 'device. Please unplug and reconnect the device ' @@ -1445,16 +1424,16 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): except: pass try: - print >>sys.stderr, job.console_text() + prints(job.details, file=sys.stderr) except: pass if not self.device_error_dialog.isVisible(): - self.device_error_dialog.set_message(job.gui_text()) + self.device_error_dialog.setDetailedText(job.details) self.device_error_dialog.show() def job_exception(self, job): try: - if job.exception[0] == 'DRMError': + if 'calibre.ebooks.DRMError' in job.details: error_dialog(self, _('Conversion Error'), _('

    Could not convert: %s

    It is a ' 'DRMed book. You must first remove the ' @@ -1464,23 +1443,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): return except: pass - only_msg = getattr(job.exception, 'only_msg', False) + if job.killed: + return try: - print job.console_text() + prints(job.details, file=sys.stderr) except: pass - if only_msg: - try: - exc = unicode(job.exception) - except: - exc = repr(job.exception) - error_dialog(self, _('Conversion Error'), exc).exec_() - return - if isinstance(job.exception, JobKilled): - return error_dialog(self, _('Conversion Error'), - _('Failed to process')+': '+unicode(job.description), - det_msg=job.console_text()).exec_() + _('Failed')+': '+unicode(job.description), + det_msg=job.details).exec_() def initialize_database(self): @@ -1581,8 +1552,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): if self.job_manager.has_device_jobs(): msg = '

    '+__appname__ + \ _(''' is communicating with the device!
    - 'Quitting may cause corruption on the device.
    - 'Are you sure you want to quit?''')+'

    ' + Quitting may cause corruption on the device.
    + Are you sure you want to quit?''')+'

    ' d = QMessageBox(QMessageBox.Warning, _('WARNING: Active jobs'), msg, QMessageBox.Yes|QMessageBox.No, self) @@ -1596,7 +1567,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def shutdown(self, write_settings=True): if write_settings: self.write_settings() - self.job_manager.terminate_all_jobs() + self.job_manager.server.close() self.device_manager.keep_going = False self.cover_cache.stop() self.hide() @@ -1647,12 +1618,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.vanity.update() if config.get('new_version_notification') and \ dynamic.get('update to version %s'%version, True): - d = question_dialog(self, _('Update available'), + if question_dialog(self, _('Update available'), _('%s has been updated to version %s. ' 'See the new features. Visit the download pa' - 'ge?')%(__appname__, version)) - if d.exec_() == QMessageBox.Yes: + 'ge?')%(__appname__, version)): url = 'http://calibre.kovidgoyal.net/download_'+\ ('windows' if iswindows else 'osx' if isosx else 'linux') QDesktopServices.openUrl(QUrl(url)) diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py index b77a3fe7c8..5988ff2026 100644 --- a/src/calibre/gui2/main_window.py +++ b/src/calibre/gui2/main_window.py @@ -8,6 +8,7 @@ from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\ from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog from calibre.utils.config import OptionParser from calibre.gui2 import error_dialog +from calibre import prints def option_parser(usage='''\ Usage: %prog [options] @@ -79,8 +80,8 @@ class MainWindow(QMainWindow): sio = StringIO.StringIO() traceback.print_exception(type, value, tb, file=sio) fe = sio.getvalue() - print >>sys.stderr, fe - msg = unicode(str(value), 'utf8', 'replace') + prints(fe, file=sys.stderr) + msg = '%s:'%type.__name__ + unicode(str(value), 'utf8', 'replace') error_dialog(self, _('ERROR: Unhandled exception'), msg, det_msg=fe, show=True) except: diff --git a/src/calibre/gui2/status.py b/src/calibre/gui2/status.py index 7b3dac4dc8..11b442fd17 100644 --- a/src/calibre/gui2/status.py +++ b/src/calibre/gui2/status.py @@ -10,10 +10,10 @@ from calibre.gui2 import qstring_to_unicode, config class BookInfoDisplay(QWidget): class BookCoverDisplay(QLabel): - + WIDTH = 81 HEIGHT = 108 - + def __init__(self, coverpath=':/images/book.svg'): QLabel.__init__(self) self.default_pixmap = QPixmap(coverpath).scaled(self.__class__.WIDTH, @@ -23,42 +23,42 @@ class BookInfoDisplay(QWidget): self.setScaledContents(True) self.setMaximumHeight(self.HEIGHT) self.setPixmap(self.default_pixmap) - - + + def setPixmap(self, pixmap): width, height = fit_image(pixmap.width(), pixmap.height(), self.WIDTH, self.HEIGHT)[1:] self.setMaximumHeight(height) self.setMaximumWidth(width) QLabel.setPixmap(self, pixmap) - + try: aspect_ratio = pixmap.width()/float(pixmap.height()) except ZeroDivisionError: aspect_ratio = 1 self.setMaximumWidth(int(aspect_ratio*self.HEIGHT)) - + def sizeHint(self): return QSize(self.__class__.WIDTH, self.__class__.HEIGHT) - - + + class BookDataDisplay(QLabel): def __init__(self): QLabel.__init__(self) self.setText('') self.setWordWrap(True) self.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)) - + def mouseReleaseEvent(self, ev): self.emit(SIGNAL('mr(int)'), 1) - + WEIGHTS = collections.defaultdict(lambda : 100) WEIGHTS[_('Path')] = 0 WEIGHTS[_('Formats')] = 1 WEIGHTS[_('Comments')] = 2 WEIGHTS[_('Series')] = 3 WEIGHTS[_('Tags')] = 4 - + def __init__(self, clear_message): QWidget.__init__(self) self.setCursor(Qt.PointingHandCursor) @@ -74,16 +74,16 @@ class BookInfoDisplay(QWidget): self.data = {} self.setVisible(False) self._layout.setAlignment(self.cover_display, Qt.AlignTop|Qt.AlignLeft) - + def mouseReleaseEvent(self, ev): self.emit(SIGNAL('show_book_info()')) - + def show_data(self, data): if data.has_key('cover'): self.cover_display.setPixmap(QPixmap.fromImage(data.pop('cover'))) else: self.cover_display.setPixmap(self.cover_display.default_pixmap) - + rows = u'' self.book_data.setText('') self.data = data.copy() @@ -97,7 +97,7 @@ class BookInfoDisplay(QWidget): txt = txt.decode(preferred_encoding, 'replace') rows += u'%s:%s'%(key, txt) self.book_data.setText(u''+rows+u'
    ') - + self.clear_message() self.book_data.updateGeometry() self.updateGeometry() @@ -113,7 +113,7 @@ class MovieButton(QFrame): self.movie = movie self.layout().addWidget(self.movie_widget) self.jobs = QLabel(''+_('Jobs:')+' 0') - self.jobs.setAlignment(Qt.AlignHCenter|Qt.AlignBottom) + self.jobs.setAlignment(Qt.AlignHCenter|Qt.AlignBottom) self.layout().addWidget(self.jobs) self.layout().setAlignment(self.jobs, Qt.AlignHCenter) self.jobs.setMargin(0) @@ -125,8 +125,8 @@ class MovieButton(QFrame): movie.start() movie.setPaused(True) self.jobs_dialog.jobs_view.restore_column_widths() - - + + def mouseReleaseEvent(self, event): if self.jobs_dialog.isVisible(): self.jobs_dialog.jobs_view.write_settings() @@ -137,7 +137,7 @@ class MovieButton(QFrame): self.jobs_dialog.jobs_view.restore_column_widths() class CoverFlowButton(QToolButton): - + def __init__(self, parent=None): QToolButton.__init__(self, parent) self.setIconSize(QSize(80, 80)) @@ -149,17 +149,17 @@ class CoverFlowButton(QToolButton): self.connect(self, SIGNAL('toggled(bool)'), self.adjust_tooltip) self.adjust_tooltip(False) self.setCursor(Qt.PointingHandCursor) - + def adjust_tooltip(self, on): tt = _('Click to turn off Cover Browsing') if on else _('Click to browse books by their covers') self.setToolTip(tt) - + def disable(self, reason): self.setDisabled(True) self.setToolTip(_('

    Browsing books by their covers is disabled.
    Import of pictureflow module failed:
    ')+reason) - + class TagViewButton(QToolButton): - + def __init__(self, parent=None): QToolButton.__init__(self, parent) self.setIconSize(QSize(80, 80)) @@ -170,10 +170,10 @@ class TagViewButton(QToolButton): self.setCheckable(True) self.setChecked(False) self.setAutoRaise(True) - + class StatusBar(QStatusBar): - + def __init__(self, jobs_dialog, systray=None): QStatusBar.__init__(self) self.systray = systray @@ -192,11 +192,11 @@ class StatusBar(QStatusBar): self.addWidget(self.scroll_area, 100) self.setMinimumHeight(120) self.setMaximumHeight(120) - - + + def reset_info(self): self.book_info.show_data({}) - + def showMessage(self, msg, timeout=0): ret = QStatusBar.showMessage(self, msg, timeout) if self.systray is not None and not config['disable_tray_notification']: @@ -207,39 +207,38 @@ class StatusBar(QStatusBar): msg = msg.encode('utf-8') self.systray.showMessage('calibre', msg, self.systray.Information, 10000) return ret - + def jobs(self): src = qstring_to_unicode(self.movie_button.jobs.text()) return int(re.search(r'\d+', src).group()) - + def show_book_info(self): self.emit(SIGNAL('show_book_info()')) - + def job_added(self, nnum): jobs = self.movie_button.jobs src = qstring_to_unicode(jobs.text()) num = self.jobs() - nnum = num + 1 text = src.replace(str(num), str(nnum)) jobs.setText(text) if self.movie_button.movie.state() == QMovie.Paused: self.movie_button.movie.setPaused(False) - - def job_done(self, running): + + def job_done(self, nnum): jobs = self.movie_button.jobs src = qstring_to_unicode(jobs.text()) num = self.jobs() - text = src.replace(str(num), str(running)) + text = src.replace(str(num), str(nnum)) jobs.setText(text) - if running == 0: + if nnum == 0: self.no_more_jobs() - + def no_more_jobs(self): if self.movie_button.movie.state() == QMovie.Running: self.movie_button.movie.jumpToFrame(0) self.movie_button.movie.setPaused(True) QCoreApplication.instance().alert(self, 5000) - + if __name__ == '__main__': # Used to create the animated status icon from PyQt4.Qt import QApplication, QPainter, QSvgRenderer, QColor @@ -280,4 +279,4 @@ if __name__ == '__main__': os.remove(file) import sys create_mng(sys.argv[1]) - + diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index db36daa784..d6df39b5d2 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -4,16 +4,16 @@ __copyright__ = '2008, Kovid Goyal ' Miscellaneous widgets used in the GUI ''' import re, os, traceback -from PyQt4.QtGui import QListView, QIcon, QFont, QLabel, QListWidget, \ +from PyQt4.Qt import QListView, QIcon, QFont, QLabel, QListWidget, \ QListWidgetItem, QTextCharFormat, QApplication, \ - QSyntaxHighlighter, QCursor, QColor, QWidget, QDialog, \ - QPixmap, QMovie, QPalette -from PyQt4.QtCore import QAbstractListModel, QVariant, Qt, SIGNAL, \ + QSyntaxHighlighter, QCursor, QColor, QWidget, \ + QPixmap, QMovie, QPalette, QTimer, QDialog, \ + QAbstractListModel, QVariant, Qt, SIGNAL, \ QRegExp, QSettings, QSize, QModelIndex -from calibre.gui2.jobs2 import DetailView from calibre.gui2 import human_readable, NONE, TableView, \ qstring_to_unicode, error_dialog +from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.filename_pattern_ui import Ui_Form from calibre import fit_image from calibre.utils.fontconfig import find_font_families @@ -176,8 +176,8 @@ class LocationModel(QAbstractListModel): _('Click to see the list of books on storage card B in your reader') ] - def rowCount(self, parent): - return 1 + sum([1 for i in self.free if i >= 0]) + def rowCount(self, *args): + return 1 + len([i for i in self.free if i >= 0]) def data(self, index, role): row = index.row() @@ -249,6 +249,31 @@ class LocationView(QListView): if 0 <= row and row <= 3: self.model().location_changed(row) +class DetailView(QDialog, Ui_Dialog): + + def __init__(self, parent, job): + QDialog.__init__(self, parent) + self.setupUi(self) + self.setWindowTitle(job.description) + self.job = job + self.next_pos = 0 + self.update() + self.timer = QTimer(self) + self.connect(self.timer, SIGNAL('timeout()'), self.update) + self.timer.start(1000) + + + def update(self): + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) + vbar = self.log.verticalScrollBar() + vbar.setValue(vbar.maximum()) + + class JobsView(TableView): def __init__(self, parent): @@ -259,8 +284,8 @@ class JobsView(TableView): row = index.row() job = self.model().row_to_job(row) d = DetailView(self, job) - self.connect(self.model(), SIGNAL('output_received()'), d.update) d.exec_() + d.timer.stop() class FontFamilyModel(QAbstractListModel): @@ -539,12 +564,12 @@ class PythonHighlighter(QSyntaxHighlighter): return for regex, format in PythonHighlighter.Rules: - i = text.indexOf(regex) + i = regex.indexIn(text) while i >= 0: length = regex.matchedLength() self.setFormat(i, length, PythonHighlighter.Formats[format]) - i = text.indexOf(regex, i + length) + i = regex.indexIn(text, i + length) # Slow but good quality highlighting for comments. For more # speed, comment this out and add the following to __init__: @@ -569,12 +594,12 @@ class PythonHighlighter(QSyntaxHighlighter): self.setCurrentBlockState(NORMAL) - if text.indexOf(self.stringRe) != -1: + if self.stringRe.indexIn(text) != -1: return # This is fooled by triple quotes inside single quoted strings - for i, state in ((text.indexOf(self.tripleSingleRe), + for i, state in ((self.tripleSingleRe.indexIn(text), TRIPLESINGLE), - (text.indexOf(self.tripleDoubleRe), + (self.tripleDoubleRe.indexIn(text), TRIPLEDOUBLE)): if self.previousBlockState() == state: if i == -1: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index c18de5b336..b502520088 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1183,6 +1183,28 @@ class LibraryDatabase2(LibraryDatabase): path = path_or_stream return run_plugins_on_import(path, format) + def create_book_entry(self, mi, cover=None, add_duplicates=True): + if not add_duplicates and self.has_book(mi): + return None + series_index = 1 if mi.series_index is None else mi.series_index + aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors) + title = mi.title + if isinstance(aus, str): + aus = aus.decode(preferred_encoding, 'replace') + if isinstance(title, str): + title = title.decode(preferred_encoding) + obj = self.conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)', + (title, series_index, aus)) + id = obj.lastrowid + self.data.books_added([id], self.conn) + self.set_path(id, True) + self.conn.commit() + self.set_metadata(id, mi) + if cover: + self.set_cover(id, cover) + return id + + def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True): ''' Add a book to the database. The result cache is not updated. diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 215236e83d..9582651ca0 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -29,7 +29,7 @@ entry_points = { 'calibre-debug = calibre.debug:main', 'calibredb = calibre.library.cli:main', 'calibre-fontconfig = calibre.utils.fontconfig:main', - 'calibre-parallel = calibre.parallel:main', + 'calibre-parallel = calibre.utils.ipc.worker:main', 'calibre-customize = calibre.customize.ui:main', 'calibre-complete = calibre.utils.complete:main', 'pdfmanipulate = calibre.ebooks.pdf.manipulate.cli:main', diff --git a/src/calibre/manual/custom.py b/src/calibre/manual/custom.py index 1b0a6334d7..00897b3fff 100644 --- a/src/calibre/manual/custom.py +++ b/src/calibre/manual/custom.py @@ -4,9 +4,10 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' import sys, os, inspect, re -from sphinx.builder import StandaloneHTMLBuilder, bold +from sphinx.builder import StandaloneHTMLBuilder from sphinx.util import rpartition -from sphinx.ext.autodoc import get_module_charset, prepare_docstring +from sphinx.util.console import bold +from sphinx.ext.autodoc import prepare_docstring from docutils.statemachine import ViewList from docutils import nodes @@ -181,7 +182,7 @@ def auto_member(dirname, arguments, options, content, lineno, docstring = '\n'.join(comment_lines) if module is not None and docstring is not None: - docstring = docstring.decode(get_module_charset(mod)) + docstring = docstring.decode('utf-8') result = ViewList() result.append('.. attribute:: %s.%s'%(cls.__name__, obj), '') diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 717effd455..cf46e060cf 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -17,39 +17,11 @@ E-book Format Conversion What formats does |app| support conversion to/from? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|app| supports the conversion of the following formats: +|app| supports the conversion of many input formats to many output formats. +It can convert every input format in the following list, to every output format. -+----------------------------+------------------------------------------------------------------+ -| | **Output formats** | -| +------------------+-----------------------+-----------------------+ -| | EPUB | LRF | MOBI | -+===================+========+==================+=======================+=======================+ -| | MOBI | ✔ | ✔ | ✔ | -| | | | | | -| | LIT | ✔ | ✔ | ✔ | -| | | | | | -| | PRC** | ✔ | ✔ | ✔ | -| | | | | | -| | EPUB | ✔ | ✔ | ✔ | -| | | | | | -| | ODT | ✔ | ✔ | ✔ | -| | | | | | -| | FB2 | ✔ | ✔ | ✔ | -| | | | | | -| | HTML | ✔ | ✔ | ✔ | -| | | | | | -| **Input formats** | CBR | ✔ | ✔ | ✔ | -| | | | | | -| | CBZ | ✔ | ✔ | ✔ | -| | | | | | -| | RTF | ✔ | ✔ | ✔ | -| | | | | | -| | TXT | ✔ | ✔ | ✔ | -| | | | | | -| | PDF | ✔ | ✔ | ✔ | -| | | | | | -| | LRS | | ✔ | | -+-------------------+--------+------------------+-----------------------+-----------------------+ +*Input Formats:* CBZ, CBR, CBC, EPUB, FB2, HTML, LIT, MOBI, ODT, PDF, PRC**, RTF, TXT +*Output Formats:* EPUB, FB2, OEB, LIT, LRF, MOBI, PDB, PDF, TXT ** PRC is a generic format, |app| supports PRC files with TextRead and MOBIBook headers @@ -64,7 +36,7 @@ The PDF conversion tries to extract the text and images from the PDF file and co are also represented as vector diagrams, thus they cannot be extracted. How do I convert a collection of HTML files in a specific order? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like:: @@ -105,7 +77,7 @@ Device Integration What devices does |app| support? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1/2, Netronix EB600, Ectaco Jetbook and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk. +At the moment |app| has full support for the SONY PRS 500/505/700, Cybook Gen 3, Amazon Kindle 1/2, Netronix EB600, Ectaco Jetbook, BeBook/BeBook Mini and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk. I used |app| to transfer some books to my reader, and now the SONY software hangs every time I connect the reader? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/manual/news.rst b/src/calibre/manual/news.rst index 3c682ad648..f31928ed69 100644 --- a/src/calibre/manual/news.rst +++ b/src/calibre/manual/news.rst @@ -257,38 +257,26 @@ The final new feature is the :meth:`calibre.web.feeds.news.BasicNewsRecipe.prepr Tips for developing new recipes --------------------------------- -The best way to develop new recipes is to use the command line interface. Create the recipe using your favorite python editor and save it to a file say :file:`myrecipe.py`. You can download content using this recipe with the command:: +The best way to develop new recipes is to use the command line interface. Create the recipe using your favorite python editor and save it to a file say :file:`myrecipe.recipe`. The `.recipe` extension is required. You can download content using this recipe with the command:: - feeds2disk --debug --test myrecipe.py + ebook-convert myrecipe.recipe output_dir --test -vv -The :command:`feeds2disk` will download all the webpages and save them to the current directory. The :option:`--debug` makes feeds2disk spit out a lot of information about what it is doing. The :option:`--test` makes it download only a couple of articles from at most two feeds. +The :command:`ebook-convert` will download all the webpages and save them to the directory :file:`output_dir`, creating it if necessary. The :option:`-vv` makes ebook-convert spit out a lot of information about what it is doing. The :option:`--test` makes it download only a couple of articles from at most two feeds. -Once the download is complete, you can look at the downloaded :term:`HTML` by opening the file :file:`index.html` in a browser. Once you're satisfied that the download and preprocessing is happening correctly, you can generate an LRF ebook with the command:: +Once the download is complete, you can look at the downloaded :term:`HTML` by opening the file :file:`index.html` in a browser. Once you're satisfied that the download and preprocessing is happening correctly, you can generate ebooks in different formats as shown below:: - html2lrf --use-spine --page-break-before "$" index.html - -If the generated :term:`LRF` looks good, you can finally, run:: - - feeds2lrf myrecipe.py - -to see the final :term:`LRF` format e-book generated from your recipe. If you're satisfied with your recipe, consider attaching it to `the wiki `_, so that others can use it as well. If you feel there is enough demand to justify its inclusion into the set of built-in recipes, add a comment to the ticket http://calibre.kovidgoyal.net/ticket/405 + ebook-convert myrecipe.recipe myrecipe.epub + ebook-convert myrecipe.recipe myrecipe.mobi + ... -If you just want to quickly test a couple of feeds, you can use the :option:`--feeds` option:: - - feeds2disk --feeds "['http://feeds.newsweek.com/newsweek/TopNews', 'http://feeds.newsweek.com/headlines/politics']" +If you're satisfied with your recipe, consider attaching it to `the wiki `_, so that others can use it as well. If you feel there is enough demand to justify its inclusion into the set of built-in recipes, add a comment to the ticket http://calibre.kovidgoyal.net/ticket/405 .. seealso:: - :ref:`feeds2disk` - The command line interfce for downloading content from the internet - - :ref:`feeds2lrf` - The command line interface for downloading content fro the internet and converting it into a :term:`LRF` e-book. - - :ref:`html2lrf` - The command line interface for converting :term:`HTML` into a :term:`LRF` e-book. + :ref:`ebook-convert` + The command line interface for all ebook conversion. Further reading @@ -305,16 +293,6 @@ To learn more about writing advanced recipes using some of the facilities, avail `Built-in recipes `_ The source code for the built-in recipes that come with |app| -Migrating old style profiles to recipes ----------------------------------------- - -In earlier versions of |app| there was a similar, if less powerful, framework for fetching news based on *Profiles*. If you have a profile that you would like to migrate to a recipe, the basic technique is simple, as they are very similar (on the surface). Common changes you have to make include: - - * Replace ``DefaultProfile`` with ``BasicNewsRecipe`` - - * Remove ``max_recursions`` - - * If the server you're downloading from doesn't like multiple connects, set ``simultaneous_downloads = 1``. API documentation -------------------- diff --git a/src/calibre/manual/news_recipe.rst b/src/calibre/manual/news_recipe.rst index 6872d8e532..289da4b28a 100644 --- a/src/calibre/manual/news_recipe.rst +++ b/src/calibre/manual/news_recipe.rst @@ -54,6 +54,8 @@ Customizing e-book download .. automember:: BasicNewsRecipe.timefmt +.. automember:: BasicNewsRecipe.conversion_options + .. automember:: BasicNewsRecipe.feeds .. automember:: BasicNewsRecipe.no_stylesheets diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py deleted file mode 100644 index 0ec7ed09cc..0000000000 --- a/src/calibre/parallel.py +++ /dev/null @@ -1,980 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Used to run jobs in parallel in separate processes. Features output streaming, -support for progress notification as well as job killing. The worker processes -are controlled via a simple protocol run over sockets. The control happens -mainly in two class, :class:`Server` and :class:`Overseer`. The worker is -encapsulated in the function :function:`worker`. Every worker process -has the environment variable :envvar:`CALIBRE_WORKER` defined. - -The worker control protocol has two modes of operation. In the first mode, the -worker process listens for commands from the controller process. The controller -process can either hand off a job to the worker or tell the worker to die. -Once a job is handed off to the worker, the protocol enters the second mode, where -the controller listens for messages from the worker. The worker can send progress updates -as well as console output (i.e. text that would normally have been written to stdout -or stderr by the job). Once the job completes (or raises an exception) the worker -returns the result (or exception) to the controller and the protocol reverts to the first mode. - -In the second mode, the controller can also send the worker STOP messages, in which case -the worker interrupts the job and dies. The sending of progress and console output messages -is buffered and asynchronous to prevent the job from being IO bound. -''' -import sys, os, gc, cPickle, traceback, cStringIO, time, signal, \ - subprocess, socket, collections, binascii, re, thread, tempfile, atexit -from select import select -from threading import RLock, Thread, Event -from math import ceil - -from calibre.ptempfile import PersistentTemporaryFile -from calibre import iswindows, detect_ncpus, isosx, preferred_encoding -from calibre.utils.config import prefs - -DEBUG = False - -#: A mapping from job names to functions that perform the jobs -PARALLEL_FUNCS = { - 'lrfviewer' : - ('calibre.gui2.lrf_renderer.main', 'main', {}, None), - - 'ebook-viewer' : - ('calibre.gui2.viewer.main', 'main', {}, None), - - 'render_pages' : - ('calibre.ebooks.comic.input', 'render_pages', {}, 'notification'), - - 'ebook-convert' : - ('calibre.ebooks.conversion.cli', 'main', {}, None), - - 'gui_convert' : - ('calibre.gui2.convert.gui_conversion', 'gui_convert', {}, 'notification'), -} - - -isfrozen = hasattr(sys, 'frozen') -isworker = False - -win32event = __import__('win32event') if iswindows else None -win32process = __import__('win32process') if iswindows else None -msvcrt = __import__('msvcrt') if iswindows else None - -SOCKET_TYPE = socket.AF_UNIX if not iswindows else socket.AF_INET - -class WorkerStatus(object): - ''' - A platform independent class to control child processes. Provides the - methods: - - .. method:: WorkerStatus.is_alive() - - Return True is the child process is alive (i.e. it hasn't exited and returned a return code). - - .. method:: WorkerStatus.returncode() - - Wait for the child process to exit and return its return code (blocks until child returns). - - .. method:: WorkerStatus.kill() - - Forcibly terminates child process using operating system specific semantics. - ''' - - def __init__(self, obj): - ''' - `obj`: On windows a process handle, on unix a subprocess.Popen object. - ''' - self.obj = obj - self.win32process = win32process # Needed if kill is called during shutdown of interpreter - self.os = os - self.signal = signal - ext = 'windows' if iswindows else 'unix' - for func in ('is_alive', 'returncode', 'kill'): - setattr(self, func, getattr(self, func+'_'+ext)) - - def is_alive_unix(self): - return self.obj.poll() == None - - def returncode_unix(self): - return self.obj.wait() - - def kill_unix(self): - os.kill(self.obj.pid, self.signal.SIGKILL) - - def is_alive_windows(self): - return win32event.WaitForSingleObject(self.obj, 0) != win32event.WAIT_OBJECT_0 - - def returncode_windows(self): - return win32process.GetExitCodeProcess(self.obj) - - def kill_windows(self, returncode=-1): - self.win32process.TerminateProcess(self.obj, returncode) - -class WorkerMother(object): - ''' - Platform independent object for launching child processes. All processes - have the environment variable :envvar:`CALIBRE_WORKER` set. - - ..method:: WorkerMother.spawn_free_spirit(arg) - - Launch a non monitored process with argument `arg`. - - ..method:: WorkerMother.spawn_worker(arg) - - Launch a monitored and controllable process with argument `arg`. - ''' - - def __init__(self): - ext = 'windows' if iswindows else 'osx' if isosx else 'linux' - self.os = os # Needed incase cleanup called when interpreter is shutting down - self.env = {} - if iswindows: - self.executable = os.path.join(os.path.dirname(sys.executable), - 'calibre-parallel.exe' if isfrozen else 'Scripts\\calibre-parallel.exe') - elif isosx: - self.executable = self.gui_executable = sys.executable - self.prefix = '' - if isfrozen: - fd = os.path.realpath(getattr(sys, 'frameworks_dir')) - contents = os.path.dirname(fd) - self.gui_executable = os.path.join(contents, 'MacOS', - os.path.basename(sys.executable)) - contents = os.path.join(contents, 'console.app', 'Contents') - exe = os.path.basename(sys.executable) - if 'python' not in exe: - exe = 'python' - self.executable = os.path.join(contents, 'MacOS', exe) - - resources = os.path.join(contents, 'Resources') - fd = os.path.join(contents, 'Frameworks') - sp = os.path.join(resources, 'lib', 'python'+sys.version[:3], 'site-packages.zip') - self.prefix += 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd - self.prefix += 'sys.path.insert(0, %s); '%repr(sp) - if fd not in os.environ['PATH']: - self.env['PATH'] = os.environ['PATH']+':'+fd - self.env['PYTHONHOME'] = resources - self.env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick') - self.env['DYLD_LIBRARY_PATH'] = os.path.join(fd, 'ImageMagick', 'lib') - else: - self.executable = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \ - if isfrozen else 'calibre-parallel' - if isfrozen: - self.env['LD_LIBRARY_PATH'] = getattr(sys, 'frozen_path') + ':' + os.environ.get('LD_LIBRARY_PATH', '') - - self.spawn_worker_windows = lambda arg : self.spawn_free_spirit_windows(arg, type='worker') - self.spawn_worker_linux = lambda arg : self.spawn_free_spirit_linux(arg, type='worker') - self.spawn_worker_osx = lambda arg : self.spawn_free_spirit_osx(arg, type='worker') - - for func in ('spawn_free_spirit', 'spawn_worker'): - setattr(self, func, getattr(self, func+'_'+ext)) - - - def cleanup_child_windows(self, child, name=None, fd=None): - try: - child.kill() - except: - pass - try: - if fd is not None: - self.os.close(fd) - except: - pass - try: - if name is not None and os.path.exists(name): - self.os.unlink(name) - except: - pass - - def cleanup_child_linux(self, child): - try: - child.kill() - except: - pass - - def get_env(self): - env = dict(os.environ) - env['CALIBRE_WORKER'] = '1' - env['ORIGWD'] = os.path.abspath(os.getcwd()) - if hasattr(self, 'env'): - env.update(self.env) - return env - - def spawn_free_spirit_osx(self, arg, type='free_spirit'): - script = ('from calibre.parallel import main; ' - 'main(args=["calibre-parallel", %s]);')%repr(arg) - exe = self.gui_executable if type == 'free_spirit' else self.executable - cmdline = [exe, '-c', self.prefix+script] - child = WorkerStatus(subprocess.Popen(cmdline, env=self.get_env())) - atexit.register(self.cleanup_child_linux, child) - return child - - def spawn_free_spirit_linux(self, arg, type='free_spirit'): - cmdline = [self.executable, arg] - child = WorkerStatus(subprocess.Popen(cmdline, - env=self.get_env(), cwd=getattr(sys, 'frozen_path', None))) - atexit.register(self.cleanup_child_linux, child) - return child - - def spawn_free_spirit_windows(self, arg, type='free_spirit'): - priority = {'high':win32process.HIGH_PRIORITY_CLASS, 'normal':win32process.NORMAL_PRIORITY_CLASS, - 'low':win32process.IDLE_PRIORITY_CLASS}[prefs['worker_process_priority']] - fd, name = tempfile.mkstemp('.log', 'calibre_'+type+'_') - handle = msvcrt.get_osfhandle(fd) - si = win32process.STARTUPINFO() - si.hStdOutput = handle - si.hStdError = handle - cmdline = self.executable + ' ' + str(arg) - hProcess = \ - win32process.CreateProcess( - None, # Application Name - cmdline, # Command line - None, # processAttributes - None, # threadAttributes - 1, # bInheritHandles - win32process.CREATE_NO_WINDOW|priority, # Dont want ugly console popping up - self.get_env(), # New environment - None, # Current directory - si - )[0] - child = WorkerStatus(hProcess) - atexit.register(self.cleanup_child_windows, child, name, fd) - return child - - -mother = WorkerMother() - -_comm_lock = RLock() -def write(socket, msg, timeout=5): - ''' - Write a message on socket. If `msg` is unicode, it is encoded in utf-8. - Raises a `RuntimeError` if the socket is not ready for writing or the writing fails. - `msg` is broken into chunks of size 4096 and sent. The :function:`read` function - automatically re-assembles the chunks into whole message. - ''' - if isworker: - _comm_lock.acquire() - try: - if isinstance(msg, unicode): - msg = msg.encode('utf-8') - if DEBUG: - print >>sys.__stdout__, 'write(%s):'%('worker' if isworker else 'overseer'), repr(msg) - length = None - while len(msg) > 0: - if length is None: - length = len(msg) - chunk = ('%-12d'%length) + msg[:4096-12] - msg = msg[4096-12:] - else: - chunk, msg = msg[:4096], msg[4096:] - w = select([], [socket], [], timeout)[1] - if not w: - raise RuntimeError('Write to socket timed out') - if socket.sendall(chunk) is not None: - raise RuntimeError('Failed to write chunk to socket') - finally: - if isworker: - _comm_lock.release() - -def read(socket, timeout=5): - ''' - Read a message from `socket`. The message must have been sent with the :function:`write` - function. Raises a `RuntimeError` if the message is corrupted. Can return an - empty string. - ''' - if isworker: - _comm_lock.acquire() - try: - buf = cStringIO.StringIO() - length = None - while select([socket],[],[],timeout)[0]: - msg = socket.recv(4096) - if not msg: - break - if length is None: - try: - length, msg = int(msg[:12]), msg[12:] - except ValueError: - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), 'no length in', msg - return '' - buf.write(msg) - if buf.tell() >= length: - break - if not length: - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), 'nothing' - return '' - msg = buf.getvalue()[:length] - if len(msg) < length: - raise RuntimeError('Corrupted packet received') - if DEBUG: - print >>sys.__stdout__, 'read(%s):'%('worker' if isworker else 'overseer'), repr(msg) - return msg - finally: - if isworker: - _comm_lock.release() - -class RepeatingTimer(Thread): - ''' - Calls a specified function repeatedly at a specified interval. Runs in a - daemon thread (i.e. the interpreter can exit while it is still running). - Call :meth:`start()` to start it. - ''' - - def repeat(self): - while True: - self.event.wait(self.interval) - if self.event.isSet(): - break - self.action() - - def __init__(self, interval, func, name): - self.event = Event() - self.interval = interval - self.action = func - Thread.__init__(self, target=self.repeat, name=name) - self.setDaemon(True) - -class ControlError(Exception): - pass - -class Overseer(object): - ''' - Responsible for controlling worker processes. The main interface is the - methods, :meth:`initialize_job`, :meth:`control`. - ''' - - KILL_RESULT = 'Server: job killed by user|||#@#$%&*)*(*$#$%#$@&' - INTERVAL = 0.1 - - def __init__(self, server, port, timeout=5): - self.worker_status = mother.spawn_worker('127.0.0.1:'+str(port)) - self.socket = server.accept()[0] - # Needed if terminate called when interpreter is shutting down - self.os = os - self.signal = signal - self.on_probation = False - self.terminated = False - - self.working = False - self.timeout = timeout - self.last_job_time = time.time() - self._stop = False - if not select([self.socket], [], [], 120)[0]: - raise RuntimeError(_('Could not launch worker process.')) - ID = self.read().split(':') - if ID[0] != 'CALIBRE_WORKER': - raise RuntimeError('Impostor') - self.worker_pid = int(ID[1]) - self.write('OK') - if self.read() != 'WAITING': - raise RuntimeError('Worker sulking') - - def terminate(self): - 'Kill worker process.' - self.terminated = True - try: - if self.socket: - self.write('STOP:') - time.sleep(1) - self.socket.shutdown(socket.SHUT_RDWR) - except: - pass - if iswindows: - win32api = __import__('win32api') - try: - handle = win32api.OpenProcess(1, False, self.worker_pid) - win32api.TerminateProcess(handle, -1) - except: - pass - else: - try: - try: - self.os.kill(self.worker_pid, self.signal.SIGKILL) - time.sleep(0.5) - finally: - self.worker_status.kill() - except: - pass - - - def write(self, msg, timeout=None): - write(self.socket, msg, timeout=self.timeout if timeout is None else timeout) - - def read(self, timeout=None): - return read(self.socket, timeout=self.timeout if timeout is None else timeout) - - def __eq__(self, other): - return hasattr(other, 'process') and hasattr(other, 'worker_pid') and self.worker_pid == other.worker_pid - - def is_viable(self): - if self.terminated: - return False - return self.worker_status.is_alive() - - def select(self, timeout=0): - return select([self.socket], [self.socket], [self.socket], timeout) - - def initialize_job(self, job): - ''' - Sends `job` to worker process. Can raise `ControlError` if worker process - does not respond appropriately. In this case, this Overseer is useless - and should be discarded. - - `job`: An instance of :class:`Job`. - ''' - self.working = True - self.write('JOB:'+cPickle.dumps((job.func, job.args, job.kwargs), -1)) - msg = self.read() - if msg != 'OK': - raise ControlError('Failed to initialize job on worker %d:%s'%(self.worker_pid, msg)) - self.job = job - self.last_report = time.time() - job.start_work() - - def control(self): - ''' - Listens for messages from the worker process and dispatches them - appropriately. If the worker process dies unexpectedly, returns a result - of None with a ControlError indicating the worker died. - - Returns a :class:`Result` instance or None, if the worker is still working. - ''' - if select([self.socket],[],[],0)[0]: - msg = self.read() - if msg: - self.on_probation = False - self.last_report = time.time() - else: - if self.on_probation: - self.terminate() - self.job.result = None - self.job.exception = ControlError('Worker process died unexpectedly') - return - else: - self.on_probation = True - return - word, msg = msg.partition(':')[0], msg.partition(':')[-1] - if word == 'PING': - self.write('OK') - return - elif word == 'RESULT': - self.write('OK') - self.job.result = cPickle.loads(msg) - return True - elif word == 'OUTPUT': - self.write('OK') - try: - self.job.output(''.join(cPickle.loads(msg))) - except: - self.job.output('Bad output message: '+ repr(msg)) - elif word == 'PROGRESS': - self.write('OK') - percent = None - try: - percent, msg = cPickle.loads(msg)[-1] - except: - print 'Bad progress update:', repr(msg) - if percent is not None: - self.job.update_status(percent, msg) - elif word == 'ERROR': - self.write('OK') - exception, tb = cPickle.loads(msg) - self.job.output(u'%s\n%s'%(exception, tb)) - self.job.exception, self.job.traceback = exception, tb - return True - else: - self.terminate() - self.job.exception = ControlError('Worker sent invalid msg: %s'%repr(msg)) - return - if not self.worker_status.is_alive() or time.time() - self.last_report > 380: - self.terminate() - self.job.exception = ControlError('Worker process died unexpectedly') - return - -class JobKilled(Exception): - pass - -class Job(object): - - def __init__(self, job_done, job_manager=None, - args=[], kwargs={}, description=None): - self.args = args - self.kwargs = kwargs - self._job_done = job_done - self.job_manager = job_manager - self.is_running = False - self.has_run = False - self.percent = -1 - self.msg = None - self.description = description - self.start_time = None - self.running_time = None - - self.result = self.exception = self.traceback = self.log = None - - def __cmp__(self, other): - sstatus, ostatus = self.status(), other.status() - if sstatus == ostatus or (self.has_run and other.has_run): - if self.start_time == other.start_time: - return cmp(id(self), id(other)) - return cmp(self.start_time, other.start_time) - if sstatus == 'WORKING': - return -1 - if ostatus == 'WORKING': - return 1 - if sstatus == 'WAITING': - return -1 - if ostatus == 'WAITING': - return 1 - - - def job_done(self): - self.is_running, self.has_run = False, True - self.running_time = (time.time() - self.start_time) if \ - self.start_time is not None else 0 - if self.job_manager is not None: - self.job_manager.job_done(self) - self._job_done(self) - - def start_work(self): - self.is_running = True - self.has_run = False - self.start_time = time.time() - if self.job_manager is not None: - self.job_manager.start_work(self) - - def update_status(self, percent, msg=None): - self.percent = percent - self.msg = msg - if self.job_manager is not None: - try: - self.job_manager.status_update(self) - except: - traceback.print_exc() - - def status(self): - if self.is_running: - return 'WORKING' - if not self.has_run: - return 'WAITING' - if self.has_run: - if self.exception is None: - return 'DONE' - return 'ERROR' - - def console_text(self): - ans = [u'Job: '] - if self.description: - ans[0] += self.description - if self.exception is not None: - header = unicode(self.exception.__class__.__name__) if \ - hasattr(self.exception, '__class__') else u'Error' - header = u'**%s**'%header - header += u': ' - try: - header += unicode(self.exception) - except: - header += unicode(repr(self.exception)) - ans.append(header) - if self.traceback: - ans.append(u'**Traceback**:') - ans.extend(self.traceback.split('\n')) - - if self.log: - if isinstance(self.log, str): - self.log = unicode(self.log, 'utf-8', 'replace') - ans.append(self.log) - return (u'\n'.join(ans)).encode('utf-8') - - def gui_text(self): - ans = [u'Job: '] - if self.description: - if not isinstance(self.description, unicode): - self.description = self.description.decode('utf-8', 'replace') - ans[0] += u'**%s**'%self.description - if self.exception is not None: - header = unicode(self.exception.__class__.__name__) if \ - hasattr(self.exception, '__class__') else u'Error' - header = u'**%s**'%header - header += u': ' - try: - header += unicode(self.exception) - except: - header += unicode(repr(self.exception)) - ans.append(header) - if self.traceback: - ans.append(u'**Traceback**:') - ans.extend(self.traceback.split('\n')) - if self.log: - ans.append(u'**Log**:') - if isinstance(self.log, str): - self.log = unicode(self.log, 'utf-8', 'replace') - ans.extend(self.log.split('\n')) - - ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, str) else x for x in ans] - - return u'
    '.join(ans) - - -class ParallelJob(Job): - - def __init__(self, func, *args, **kwargs): - Job.__init__(self, *args, **kwargs) - self.func = func - self.done = self.job_done - - def output(self, msg): - if not self.log: - self.log = u'' - if not isinstance(msg, unicode): - msg = msg.decode('utf-8', 'replace') - if msg: - self.log += msg - if self.job_manager is not None: - self.job_manager.output(self) - - -def remove_ipc_socket(path): - os = __import__('os') - if os.path.exists(path): - os.unlink(path) - -class Server(Thread): - - KILL_RESULT = Overseer.KILL_RESULT - START_PORT = 10013 - PID = os.getpid() - - - def __init__(self, number_of_workers=detect_ncpus()): - Thread.__init__(self) - self.setDaemon(True) - self.server_socket = socket.socket(SOCKET_TYPE, socket.SOCK_STREAM) - self.port = tempfile.mktemp(prefix='calibre_server')+'_%d_'%self.PID if not iswindows else self.START_PORT - while True: - try: - address = ('localhost', self.port) if iswindows else self.port - self.server_socket.bind(address) - break - except socket.error: - self.port += (1 if iswindows else '1') - if not iswindows: - atexit.register(remove_ipc_socket, self.port) - self.server_socket.listen(5) - self.number_of_workers = number_of_workers - self.pool, self.jobs, self.working = [], collections.deque(), [] - atexit.register(self.killall) - atexit.register(self.close) - self.job_lock = RLock() - self.overseer_lock = RLock() - self.working_lock = RLock() - self.result_lock = RLock() - self.pool_lock = RLock() - self.start() - - def split(self, tasks): - ''' - Split a list into a list of sub lists, with the number of sub lists being - no more than the number of workers this server supports. Each sublist contains - two tuples of the form (i, x) where x is an element fro the original list - and i is the index of the element x in the original list. - ''' - ans, count, pos = [], 0, 0 - delta = int(ceil(len(tasks)/float(self.number_of_workers))) - while count < len(tasks): - section = [] - for t in tasks[pos:pos+delta]: - section.append((count, t)) - count += 1 - ans.append(section) - pos += delta - return ans - - - def close(self): - try: - self.server_socket.shutdown(socket.SHUT_RDWR) - except: - pass - - def add_job(self, job): - with self.job_lock: - self.jobs.append(job) - if job.job_manager is not None: - job.job_manager.add_job(job) - - def poll(self): - ''' - Return True if the server has either working or queued jobs - ''' - with self.job_lock: - with self.working_lock: - return len(self.jobs) + len(self.working) > 0 - - def wait(self, sleep=1): - ''' - Wait until job queue is empty - ''' - while self.poll(): - time.sleep(sleep) - - def run(self): - while True: - job = None - with self.job_lock: - if len(self.jobs) > 0 and len(self.working) < self.number_of_workers: - job = self.jobs.popleft() - with self.pool_lock: - o = None - while self.pool: - o = self.pool.pop() - try: - o.initialize_job(job) - break - except: - o.terminate() - if o is None: - o = Overseer(self.server_socket, self.port) - try: - o.initialize_job(job) - except Exception, err: - o.terminate() - job.exception = err - job.traceback = traceback.format_exc() - job.done() - o = None - if o and o.is_viable(): - with self.working_lock: - self.working.append(o) - - with self.working_lock: - done = [] - for o in self.working: - try: - if o.control() is not None or o.job.exception is not None: - o.job.done() - done.append(o) - except Exception, err: - o.job.exception = err - o.job.traceback = traceback.format_exc() - o.terminate() - o.job.done() - done.append(o) - for o in done: - self.working.remove(o) - if o and o.is_viable(): - with self.pool_lock: - self.pool.append(o) - - try: - time.sleep(1) - except: - return - - - def killall(self): - with self.pool_lock: - map(lambda x: x.terminate(), self.pool) - self.pool = [] - - - def kill(self, job): - with self.working_lock: - pop = None - for o in self.working: - if o.job == job or o == job: - try: - o.terminate() - except: pass - o.job.exception = JobKilled(_('Job stopped by user')) - try: - o.job.done() - except: pass - pop = o - break - if pop is not None: - self.working.remove(pop) - - def run_free_job(self, func, args=[], kwdargs={}): - pt = PersistentTemporaryFile('.pickle', '_IPC_') - pt.write(cPickle.dumps((func, args, kwdargs))) - pt.close() - mother.spawn_free_spirit(binascii.hexlify(pt.name)) - - -########################################################################################## -##################################### CLIENT CODE ##################################### -########################################################################################## - -class BufferedSender(object): - - def __init__(self, socket): - self.socket = socket - self.wbuf, self.pbuf = [], [] - self.wlock, self.plock = RLock(), RLock() - self.last_report = None - self.timer = RepeatingTimer(0.5, self.send, 'BufferedSender') - self.timer.start() - - - def write(self, msg): - if not isinstance(msg, basestring): - msg = unicode(msg) - with self.wlock: - self.wbuf.append(msg) - - def send(self): - if callable(select) and select([self.socket], [], [], 0)[0]: - msg = read(self.socket) - if msg == 'PING:': - write(self.socket, 'OK') - elif msg: - self.socket.shutdown(socket.SHUT_RDWR) - thread.interrupt_main() - time.sleep(1) - raise SystemExit - if not select([], [self.socket], [], 30)[1]: - print >>sys.__stderr__, 'Cannot pipe to overseer' - return - - reported = False - with self.wlock: - if self.wbuf: - msg = cPickle.dumps(self.wbuf, -1) - self.wbuf = [] - write(self.socket, 'OUTPUT:'+msg) - read(self.socket, 10) - reported = True - - with self.plock: - if self.pbuf: - msg = cPickle.dumps(self.pbuf, -1) - self.pbuf = [] - write(self.socket, 'PROGRESS:'+msg) - read(self.socket, 10) - reported = True - - if self.last_report is not None: - if reported: - self.last_report = time.time() - elif time.time() - self.last_report > 60: - write(self.socket, 'PING:') - read(self.socket, 10) - self.last_report = time.time() - - def notify(self, percent, msg=''): - with self.plock: - self.pbuf.append((percent, msg)) - - def flush(self): - pass - -def get_func(name): - module, func, kwdargs, notification = PARALLEL_FUNCS[name] - module = __import__(module, fromlist=[1]) - func = getattr(module, func) - return func, kwdargs, notification - -_atexit = collections.deque() -def myatexit(func, *args, **kwargs): - _atexit.append((func, args, kwargs)) - -def work(client_socket, func, args, kwdargs): - sys.stdout.last_report = time.time() - orig = atexit.register - atexit.register = myatexit - try: - func, kargs, notification = get_func(func) - if notification is not None and hasattr(sys.stdout, 'notify'): - kargs[notification] = sys.stdout.notify - kargs.update(kwdargs) - res = func(*args, **kargs) - if hasattr(sys.stdout, 'send'): - sys.stdout.send() - return res - finally: - atexit.register = orig - sys.stdout.last_report = None - while True: - try: - func, args, kwargs = _atexit.pop() - except IndexError: - break - try: - func(*args, **kwargs) - except (Exception, SystemExit): - continue - - time.sleep(5) # Give any in progress BufferedSend time to complete - - -def worker(host, port): - client_socket = socket.socket(SOCKET_TYPE, socket.SOCK_STREAM) - address = (host, port) if iswindows else port - client_socket.connect(address) - write(client_socket, 'CALIBRE_WORKER:%d'%os.getpid()) - msg = read(client_socket, timeout=10) - if msg != 'OK': - return 1 - write(client_socket, 'WAITING') - - sys.stdout = BufferedSender(client_socket) - sys.stderr = sys.stdout - - while True: - if not select([client_socket], [], [], 60)[0]: - time.sleep(1) - continue - msg = read(client_socket, timeout=60) - if msg.startswith('JOB:'): - func, args, kwdargs = cPickle.loads(msg[4:]) - write(client_socket, 'OK') - try: - result = work(client_socket, func, args, kwdargs) - write(client_socket, 'RESULT:'+ cPickle.dumps(result)) - except BaseException, err: - exception = (err.__class__.__name__, unicode(str(err), 'utf-8', 'replace')) - tb = unicode(traceback.format_exc(), 'utf-8', 'replace') - msg = 'ERROR:'+cPickle.dumps((exception, tb),-1) - write(client_socket, msg) - res = read(client_socket, 10) - if res != 'OK': - break - gc.collect() - elif msg == 'PING:': - write(client_socket, 'OK') - elif msg == 'STOP:': - client_socket.shutdown(socket.SHUT_RDWR) - return 0 - elif not msg: - time.sleep(1) - else: - print >>sys.__stderr__, 'Invalid protocols message', msg - return 1 - -def free_spirit(path): - func, args, kwdargs = cPickle.load(open(path, 'rb')) - try: - os.unlink(path) - except: - pass - func, kargs = get_func(func)[:2] - kargs.update(kwdargs) - func(*args, **kargs) - -def main(args=sys.argv): - global isworker - isworker = True - args = args[1].split(':') - if len(args) == 1: - free_spirit(binascii.unhexlify(re.sub(r'[^a-f0-9A-F]', '', args[0]))) - else: - worker(args[0].replace("'", ''), int(args[1]) if iswindows else args[1]) - return 0 - -if __name__ == '__main__': - sys.exit(main()) - diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index 03a6676e7b..dd25279071 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -20,7 +20,7 @@ DEPENDENCIES = [ ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'), ('dnspython', '1.6.0', 'dnspython', 'dnspython', 'dnspython', 'dnspython'), ('poppler', '0.10.5', 'poppler', 'poppler', 'poppler', 'poppler'), - ('pdftk', '1.12', 'pdftk', 'pdftk', 'pdftk', 'pdftk'), + ('podofo', '0.7', 'podofo', 'podofo', 'podofo', 'podofo'), ] diff --git a/src/calibre/trac/plugins/templates/linux.html b/src/calibre/trac/plugins/templates/linux.html index 96881aa108..ffaa1d8394 100644 --- a/src/calibre/trac/plugins/templates/linux.html +++ b/src/calibre/trac/plugins/templates/linux.html @@ -49,7 +49,7 @@

    ${app} is available in the software repositories of the following - linux distributions: + supported linux distributions: diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py new file mode 100644 index 0000000000..f5991740f6 --- /dev/null +++ b/src/calibre/utils/ipc/job.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +_count = 0 + +import time, cStringIO +from Queue import Queue, Empty + +class BaseJob(object): + + WAITING = 0 + RUNNING = 1 + FINISHED = 2 + + def __init__(self, description, done=lambda x: x): + global _count + _count += 1 + + self.id = _count + self.description = description + self.done = done + self.done2 = None + self.killed = False + self.failed = False + self.start_time = None + self.result = None + self.duration = None + self.log_path = None + self.notifications = Queue() + + self._run_state = self.WAITING + self.percent = 0 + self._message = None + self._status_text = _('Waiting...') + self._done_called = False + + def update(self): + if self.duration is not None: + self._run_state = self.FINISHED + self.percent = 100 + if self.killed: + self._status_text = _('Stopped') + else: + self._status_text = _('Error') if self.failed else _('Finished') + if not self._done_called: + self._done_called = True + try: + self.done(self) + except: + pass + try: + if callable(self.done2): + self.done2(self) + except: + pass + elif self.start_time is not None: + self._run_state = self.RUNNING + self._status_text = _('Working...') + + while True: + try: + self.percent, self._message = self.notifications.get_nowait() + self.percent *= 100. + except Empty: + break + + @property + def status_text(self): + if self._run_state == self.FINISHED or not self._message: + return self._status_text + return self._message + + @property + def run_state(self): + return self._run_state + + @property + def running_time(self): + if self.duration is not None: + return self.duration + if self.start_time is not None: + return time.time() - self.start_time + return None + + @property + def is_finished(self): + return self._run_state == self.FINISHED + + @property + def is_started(self): + return self._run_state != self.WAITING + + @property + def is_running(self): + return self.is_started and not self.is_finished + + def __cmp__(self, other): + if self.is_finished == other.is_finished: + if self.start_time is None: + if other.start_time is None: # Both waiting + return cmp(other.id, self.id) + else: + return 1 + else: + if other.start_time is None: + return -1 + else: # Both running + return cmp(other.start_time, self.start_time) + + else: + return 1 if self.is_finished else -1 + return 0 + + @property + def log_file(self): + if self.log_path: + return open(self.log_path, 'rb') + return cStringIO.StringIO(_('No details available.')) + + @property + def details(self): + return self.log_file.read().decode('utf-8') + + +class ParallelJob(BaseJob): + + def __init__(self, name, description, done, args=[], kwargs={}): + self.name, self.args, self.kwargs = name, args, kwargs + BaseJob.__init__(self, description, done) + + + diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index 6c0ba46885..14530d7fea 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -70,7 +70,7 @@ class Worker(object): @property def is_alive(self): - return hasattr(self, 'child') and self.child.poll() is not None + return hasattr(self, 'child') and self.child.poll() is None @property def returncode(self): @@ -144,6 +144,7 @@ class Worker(object): self.child = subprocess.Popen(cmd, **args) + self.log_path = ret return ret diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 3d1a86922e..0d7f307a9d 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -6,5 +6,241 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os, cPickle, time, tempfile +from math import ceil +from threading import Thread, RLock +from Queue import Queue, Empty +from multiprocessing.connection import Listener +from multiprocessing import cpu_count +from collections import deque +from binascii import hexlify + +from calibre.utils.ipc.launch import Worker +from calibre.utils.ipc.worker import PARALLEL_FUNCS + +_counter = 0 + +class ConnectedWorker(Thread): + + def __init__(self, worker, conn, rfile): + Thread.__init__(self) + self.daemon = True + self.conn = conn + self.worker = worker + self.notifications = Queue() + self._returncode = 'dummy' + self.killed = False + self.log_path = worker.log_path + self.rfile = rfile + + def start_job(self, job): + notification = PARALLEL_FUNCS[job.name][-1] is not None + self.conn.send((job.name, job.args, job.kwargs)) + if notification: + self.start() + else: + self.conn.close() + self.job = job + + def run(self): + while True: + try: + x = self.conn.recv() + self.notifications.put(x) + except BaseException: + break + try: + self.conn.close() + except BaseException: + pass + + def kill(self): + self.killed = True + try: + self.worker.kill() + except BaseException: + pass + + @property + def is_alive(self): + return not self.killed and self.worker.is_alive + + @property + def returncode(self): + if self._returncode != 'dummy': + return self._returncode + r = self.worker.returncode + if self.killed and r is None: + self._returncode = 1 + return 1 + if r is not None: + self._returncode = r + return r + +class Server(Thread): + + def __init__(self, notify_on_job_done=lambda x: x, pool_size=None): + Thread.__init__(self) + self.daemon = True + global _counter + self.id = _counter+1 + _counter += 1 + + self.pool_size = cpu_count() if pool_size is None else pool_size + self.notify_on_job_done = notify_on_job_done + self.auth_key = os.urandom(32) + self.listener = Listener(authkey=self.auth_key, backlog=4) + self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() + self.kill_queue = Queue() + self.waiting_jobs, self.processing_jobs = deque(), deque() + self.pool, self.workers = deque(), deque() + self.launched_worker_count = 0 + self._worker_launch_lock = RLock() + + self.start() + + def launch_worker(self, gui=False, redirect_output=None): + with self._worker_launch_lock: + self.launched_worker_count += 1 + id = self.launched_worker_count + rfile = os.path.join(tempfile.gettempdir(), + 'calibre_ipc_result_%d_%d.pickle'%(self.id, id)) + + env = { + 'CALIBRE_WORKER_ADDRESS' : + hexlify(cPickle.dumps(self.listener.address, -1)), + 'CALIBRE_WORKER_KEY' : hexlify(self.auth_key), + 'CALIBRE_WORKER_RESULT' : hexlify(rfile), + } + w = Worker(env, gui=gui) + if redirect_output is None: + redirect_output = not gui + w(redirect_output=redirect_output) + conn = self.listener.accept() + if conn is None: + raise Exception('Failed to launch worker process') + return ConnectedWorker(w, conn, rfile) + + def add_job(self, job): + job.done2 = self.notify_on_job_done + self.add_jobs_queue.put(job) + + def run_job(self, job, gui=True, redirect_output=False): + w = self.launch_worker(gui=gui, redirect_output=redirect_output) + w.start_job(job) + + + def run(self): + while True: + try: + job = self.add_jobs_queue.get(True, 0.2) + if job is None: + break + self.waiting_jobs.append(job) + except Empty: + pass + + for worker in self.workers: + while True: + try: + n = worker.notifications.get_nowait() + worker.job.notifications.put(n) + self.changed_jobs_queue.put(job) + except Empty: + break + + for worker in [w for w in self.workers if not w.is_alive]: + self.workers.remove(worker) + job = worker.job + if worker.returncode != 0: + job.failed = True + job.returncode = worker.returncode + elif os.path.exists(worker.rfile): + job.result = cPickle.load(open(worker.rfile, 'rb')) + os.remove(worker.rfile) + job.duration = time.time() - job.start_time + self.changed_jobs_queue.put(job) + + if len(self.pool) + len(self.workers) < self.pool_size: + try: + self.pool.append(self.launch_worker()) + except Exception, err: + pass + + if len(self.pool) > 0 and len(self.waiting_jobs) > 0: + job = self.waiting_jobs.pop() + worker = self.pool.pop() + job.start_time = time.time() + worker.start_job(job) + self.workers.append(worker) + job.log_path = worker.log_path + self.changed_jobs_queue.put(job) + + while True: + try: + j = self.kill_queue.get_nowait() + self._kill_job(j) + except Empty: + break + + def kill_job(self, job): + self.kill_queue.put(job) + + def killall(self): + for job in self.workers: + self.kill_queue.put(job) + + def _kill_job(self, job): + if job.start_time is None: return + for worker in self.workers: + if job is worker.job: + worker.kill() + job.killed = True + break + + def split(self, tasks): + ''' + Split a list into a list of sub lists, with the number of sub lists being + no more than the number of workers this server supports. Each sublist contains + two tuples of the form (i, x) where x is an element from the original list + and i is the index of the element x in the original list. + ''' + ans, count, pos = [], 0, 0 + delta = int(ceil(len(tasks)/float(self.pool_size))) + while count < len(tasks): + section = [] + for t in tasks[pos:pos+delta]: + section.append((count, t)) + count += 1 + ans.append(section) + pos += delta + return ans + + + + def close(self): + try: + self.add_jobs_queue.put(None) + self.listener.close() + except: + pass + time.sleep(0.2) + for worker in self.workers: + try: + worker.kill() + except: + pass + for worker in self.pool: + try: + worker.kill() + except: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index 75b42c9a25..82bfff0cbf 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -6,11 +6,12 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, cPickle +import os, cPickle, sys from multiprocessing.connection import Client from threading import Thread -from queue import Queue +from Queue import Queue from contextlib import closing +from binascii import unhexlify PARALLEL_FUNCS = { 'lrfviewer' : @@ -24,13 +25,16 @@ PARALLEL_FUNCS = { 'gui_convert' : ('calibre.gui2.convert.gui_conversion', 'gui_convert', 'notification'), + + 'read_metadata' : + ('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'), } class Progress(Thread): def __init__(self, conn): - self.daemon = True Thread.__init__(self) + self.daemon = True self.conn = conn self.queue = Queue() @@ -56,23 +60,30 @@ def get_func(name): return func, notification def main(): - address = cPickle.loads(os.environ['CALIBRE_WORKER_ADDRESS']) - key = os.environ['CALIBRE_WORKER_KEY'] + address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS'])) + key = unhexlify(os.environ['CALIBRE_WORKER_KEY']) + resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']) with closing(Client(address, authkey=key)) as conn: name, args, kwargs = conn.recv() + #print (name, args, kwargs) + #sys.stdout.flush() func, notification = get_func(name) notifier = Progress(conn) if notification: kwargs[notification] = notifier notifier.start() - func(*args, **kwargs) + result = func(*args, **kwargs) + if result is not None: + cPickle.dump(result, open(resultf, 'wb'), -1) notifier.queue.put(None) + sys.stdout.flush() + sys.stderr.flush() return 0 if __name__ == '__main__': - raise SystemExit(main()) + sys.exit(main()) diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 6c2846cee2..3db6699dfe 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -21,17 +21,16 @@ def get_metadata(stream): raise Unavailable(podofo_err) raw = stream.read() stream.seek(0) - p = podofo.PdfMemDocument() - p.Load(raw, len(raw)) - info = p.GetInfo() - title = info.GetTitle().decode('utf-8').strip() + p = podofo.PDFDoc() + p.load(raw) + title = p.title if not title: title = getattr(stream, 'name', _('Unknown')) title = os.path.splitext(os.path.basename(title))[0] - author = info.GetAuthor().decode('utf-8').strip() + author = p.author authors = string_to_authors(author) if author else [_('Unknown')] mi = MetaInformation(title, authors) - creator = info.GetCreator().decode('utf-8').strip() + creator = p.creator if creator: mi.book_producer = creator return mi @@ -47,31 +46,28 @@ def set_metadata(stream, mi): if not podofo: raise Unavailable(podofo_err) raw = stream.read() - p = podofo.PdfMemDocument() - p.Load(raw, len(raw)) - info = p.GetInfo() + p = podofo.PDFDoc() + p.load(raw) title = prep(mi.title) touched = False if title: - info.SetTitle(title) + p.title = title touched = True author = prep(authors_to_string(mi.authors)) if author: - print repr(author) - info.SetAuthor(author) + p.author = author touched = True bkp = prep(mi.book_producer) if bkp: - info.SetCreator(bkp) + p.creator = bkp touched = True if touched: - p.SetInfo(info) from calibre.ptempfile import TemporaryFile with TemporaryFile('_pdf_set_metadata.pdf') as f: - p.Write(f) + p.save(f) raw = open(f, 'rb').read() stream.seek(0) stream.truncate() diff --git a/src/calibre/utils/podofo/podofo.cpp b/src/calibre/utils/podofo/podofo.cpp new file mode 100644 index 0000000000..11d9b94868 --- /dev/null +++ b/src/calibre/utils/podofo/podofo.cpp @@ -0,0 +1,330 @@ +#define UNICODE +#define PY_SSIZE_T_CLEAN +#include + +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; + +#include + +class podofo_pdfmem_wrapper : public PdfMemDocument { + public: + inline void set_info(PdfInfo *i) { this->SetInfo(i); } +}; + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + podofo_pdfmem_wrapper *doc; + +} podofo_PDFDoc; + +extern "C" { +static void +podofo_PDFDoc_dealloc(podofo_PDFDoc* self) +{ + if (self->doc != NULL) delete self->doc; + self->ob_type->tp_free((PyObject*)self); +} + +static PyObject * +podofo_PDFDoc_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + podofo_PDFDoc *self; + + self = (podofo_PDFDoc *)type->tp_alloc(type, 0); + if (self != NULL) { + self->doc = new podofo_pdfmem_wrapper(); + if (self->doc == NULL) { Py_DECREF(self); return NULL; } + } + + return (PyObject *)self; +} + +static PyObject * +podofo_PDFDoc_load(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { + char *buffer; Py_ssize_t size; + + if (PyArg_ParseTuple(args, "s#", &buffer, &size)) { + try { + self->doc->Load(buffer, size); + } catch(const PdfError & err) { + PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError())); + return NULL; + } + } else return NULL; + + + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +podofo_PDFDoc_save(podofo_PDFDoc *self, PyObject *args, PyObject *kwargs) { + char *buffer; + + if (PyArg_ParseTuple(args, "s", &buffer)) { + try { + self->doc->Write(buffer); + } catch(const PdfError & err) { + PyErr_SetString(PyExc_ValueError, PdfError::ErrorMessage(err.GetError())); + return NULL; + } + } else return NULL; + + + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +podofo_convert_pdfstring(const PdfString &s) { + std::string raw = s.GetStringUtf8(); + return PyString_FromStringAndSize(raw.c_str(), raw.length()); +} + +static PdfString * +podofo_convert_pystring(PyObject *py) { + Py_UNICODE* u = PyUnicode_AS_UNICODE(py); + PyObject *u8 = PyUnicode_EncodeUTF8(u, PyUnicode_GET_SIZE(py), "replace"); + if (u8 == NULL) { PyErr_NoMemory(); return NULL; } + pdf_utf8 *s8 = (pdf_utf8 *)PyString_AS_STRING(u8); + PdfString *ans = new PdfString(s8); + Py_DECREF(u8); + if (ans == NULL) PyErr_NoMemory(); + return ans; +} + +static PyObject * +podofo_PDFDoc_getter(podofo_PDFDoc *self, int field) +{ + PyObject *ans; + PdfString s; + PdfInfo *info = self->doc->GetInfo(); + if (info == NULL) { + PyErr_SetString(PyExc_Exception, "You must first load a PDF Document"); + return NULL; + } + switch (field) { + case 0: + s = info->GetTitle(); break; + case 1: + s = info->GetAuthor(); break; + case 2: + s = info->GetSubject(); break; + case 3: + s = info->GetKeywords(); break; + case 4: + s = info->GetCreator(); break; + case 5: + s = info->GetProducer(); break; + default: + PyErr_SetString(PyExc_Exception, "Bad field"); + return NULL; + } + + ans = podofo_convert_pdfstring(s); + if (ans == NULL) {PyErr_NoMemory(); return NULL;} + PyObject *uans = PyUnicode_FromEncodedObject(ans, "utf-8", "replace"); + Py_DECREF(ans); + if (uans == NULL) {return NULL;} + Py_INCREF(uans); + return uans; +} + +static int +podofo_PDFDoc_setter(podofo_PDFDoc *self, PyObject *val, int field) { + if (val == NULL || !PyUnicode_Check(val)) { + PyErr_SetString(PyExc_ValueError, "Must use unicode objects to set metadata"); + return -1; + } + PdfInfo *info = new PdfInfo(*self->doc->GetInfo()); + if (info == NULL) { + PyErr_SetString(PyExc_Exception, "You must first load a PDF Document"); + return -1; + } + + PdfString *s = podofo_convert_pystring(val); + if (s == NULL) return -1; + switch (field) { + case 0: + info->SetTitle(*s); break; + case 1: + info->SetAuthor(*s); break; + case 2: + info->SetSubject(*s); break; + case 3: + info->SetKeywords(*s); break; + case 4: + info->SetCreator(*s); break; + case 5: + info->SetProducer(*s); break; + default: + PyErr_SetString(PyExc_Exception, "Bad field"); + return -1; + } + + self->doc->set_info(info); + return 0; +} + +static PyObject * +podofo_PDFDoc_title_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 0); +} +static PyObject * +podofo_PDFDoc_author_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 1); +} +static PyObject * +podofo_PDFDoc_subject_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 2); +} +static PyObject * +podofo_PDFDoc_keywords_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 3); +} +static PyObject * +podofo_PDFDoc_creator_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 4); +} +static PyObject * +podofo_PDFDoc_producer_getter(podofo_PDFDoc *self, void *closure) { + return podofo_PDFDoc_getter(self, 5); +} +static int +podofo_PDFDoc_title_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 0); +} +static int +podofo_PDFDoc_author_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 1); +} +static int +podofo_PDFDoc_subject_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 2); +} +static int +podofo_PDFDoc_keywords_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 3); +} +static int +podofo_PDFDoc_creator_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 4); +} +static int +podofo_PDFDoc_producer_setter(podofo_PDFDoc *self, PyObject *val, void *closure) { + return podofo_PDFDoc_setter(self, val, 5); +} + + + + + +} /* extern "C" */ + +static PyMethodDef podofo_PDFDoc_methods[] = { + {"load", (PyCFunction)podofo_PDFDoc_load, METH_VARARGS, + "Load a PDF document from a byte buffer (string)" + }, + {"save", (PyCFunction)podofo_PDFDoc_save, METH_VARARGS, + "Save the PDF document to a path on disk" + }, + {NULL} /* Sentinel */ +}; + +static PyGetSetDef podofo_PDFDoc_getseters[] = { + {"title", + (getter)podofo_PDFDoc_title_getter, (setter)podofo_PDFDoc_title_setter, + "Document title", + NULL}, + {"author", + (getter)podofo_PDFDoc_author_getter, (setter)podofo_PDFDoc_author_setter, + "Document author", + NULL}, + {"subject", + (getter)podofo_PDFDoc_subject_getter, (setter)podofo_PDFDoc_subject_setter, + "Document subject", + NULL}, + {"keywords", + (getter)podofo_PDFDoc_keywords_getter, (setter)podofo_PDFDoc_keywords_setter, + "Document keywords", + NULL}, + {"creator", + (getter)podofo_PDFDoc_creator_getter, (setter)podofo_PDFDoc_creator_setter, + "Document creator", + NULL}, + {"producer", + (getter)podofo_PDFDoc_producer_getter, (setter)podofo_PDFDoc_producer_setter, + "Document producer", + NULL}, + + {NULL} /* Sentinel */ +}; + +static PyTypeObject podofo_PDFDocType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "podofo.PDFDoc", /*tp_name*/ + sizeof(podofo_PDFDoc), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)podofo_PDFDoc_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "PDF Documents", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + podofo_PDFDoc_methods, /* tp_methods */ + 0, /* tp_members */ + podofo_PDFDoc_getseters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + podofo_PDFDoc_new, /* tp_new */ + +}; + +static PyMethodDef podofo_methods[] = { + {NULL} /* Sentinel */ +}; + +extern "C" { + + +PyMODINIT_FUNC +initpodofo(void) +{ + PyObject* m; + + if (PyType_Ready(&podofo_PDFDocType) < 0) + return; + + m = Py_InitModule3("podofo", podofo_methods, + "Wrapper for the PoDoFo pDF library"); + + Py_INCREF(&podofo_PDFDocType); + PyModule_AddObject(m, "PDFDoc", (PyObject *)&podofo_PDFDocType); +} +} diff --git a/src/calibre/utils/podofo/podofo.sip b/src/calibre/utils/podofo/podofo.sip deleted file mode 100644 index 0964e3a7e6..0000000000 --- a/src/calibre/utils/podofo/podofo.sip +++ /dev/null @@ -1,128 +0,0 @@ -%Module podofo 0 - -%MappedType PdfString -{ -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -using namespace PoDoFo; -%End -%ConvertFromTypeCode - if (sipCpp -> IsValid()) { - std::string raw = sipCpp->GetStringUtf8(); - return PyString_FromStringAndSize(raw.c_str(), raw.length()); - } else return PyString_FromString(""); -%End -%ConvertToTypeCode - if (sipIsErr == NULL) { - if (sipIsErr == NULL) - return (PyUnicode_Check(sipPy) || PyString_Check(sipPy)); - } - if (sipPy == Py_None) { - *sipCppPtr = NULL; - return 0; - } - if (PyString_Check(sipPy)) { - *sipCppPtr = new PdfString((pdf_utf8 *)PyString_AS_STRING(sipPy)); - return sipGetState(sipTransferObj); - } - if (PyUnicode_Check(sipPy)) { - Py_UNICODE* u = PyUnicode_AS_UNICODE(sipPy); - PyObject *u8 = PyUnicode_EncodeUTF8(u, PyUnicode_GET_SIZE(sipPy), "replace"); - pdf_utf8 *s8 = (pdf_utf8 *)PyString_AS_STRING(u8); - *sipCppPtr = new PdfString(s8); - return sipGetState(sipTransferObj); - } - *sipCppPtr = (PdfString *)sipForceConvertTo_PdfString(sipPy,sipIsErr); - return 1; -%End -}; - -class PdfObject { -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -using namespace PoDoFo; -%End - public: - PdfObject(); - -}; - -class PdfInfo { -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -using namespace PoDoFo; -%End - public: - PdfInfo(PdfObject *); - - PdfString GetAuthor() const; - PdfString GetSubject() const; - PdfString GetTitle() const; - PdfString GetKeywords() const; - PdfString GetCreator() const; - PdfString GetProducer() const; - - void SetAuthor(PdfString &); - void SetSubject(PdfString &); - void SetTitle(PdfString &); - void SetKeywords(PdfString &); - void SetCreator(PdfString &); - void SetProducer(PdfString &); - -}; - -class PdfOutputDevice { -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -using namespace PoDoFo; -%End - public: - PdfOutputDevice(char *, long); - unsigned long GetLength(); - unsigned long Tell(); - void Flush(); -}; - - -class PdfMemDocument { -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -using namespace PoDoFo; -%End - public: - PdfMemDocument(); - - void Load(const char *filename); - void Load(const char *buffer, long size); - void Write(const char *filename); - PdfInfo *GetInfo() const; - - protected: - void SetInfo(PdfInfo * /TransferThis/); - - private: - PdfMemDocument(PdfMemDocument &); - -}; - - -%Exception PoDoFo::PdfError /PyName=PdfError/ -{ -%TypeHeaderCode -#define USING_SHARED_PODOFO -#include -%End -%RaiseCode - const char *detail = sipExceptionRef.what(); - - SIP_BLOCK_THREADS - PyErr_SetString(sipException_PoDoFo_PdfError, detail); - SIP_UNBLOCK_THREADS -%End -}; - diff --git a/src/calibre/web/feeds/input.py b/src/calibre/web/feeds/input.py index ee003be0da..3052ffebed 100644 --- a/src/calibre/web/feeds/input.py +++ b/src/calibre/web/feeds/input.py @@ -57,6 +57,8 @@ class RecipeInput(InputFormatPlugin): ro = recipe(opts, log, self.report_progress) ro.download() + for key, val in recipe.conversion_options.items(): + setattr(opts, key, val) opts.output_profile.flow_size = 0 diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 3f6b9b9ae1..e15e6eaa36 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -156,13 +156,17 @@ class BasicNewsRecipe(Recipe): #: :attr:`BasicNewsRecipe.filter_regexps` should be defined. filter_regexps = [] - #: List of options to pass to html2lrf, to customize generation of LRF ebooks. - html2lrf_options = [] - - #: Options to pass to html2epub to customize generation of EPUB ebooks. - html2epub_options = '' - #: Options to pass to oeb2mobi to customize generation of MOBI ebooks. - oeb2mobi_options = '' + #: Recipe specific options to control the conversion of the downloaded + #: content into an e-book. These will override any user or plugin specified + #: values, so only use if absolutely necessary. For example:: + #: conversion_options = { + #: 'base_font_size' : 16, + #: 'tags' : 'mytag1,mytag2', + #: 'title' : 'My Title', + #: 'linearize_tables' : True, + #: } + #: + conversion_options = {} #: List of tags to be removed. Specified tags are removed from downloaded HTML. #: A tag is specified as a dictionary of the form:: diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index 6b7bc6d2ac..bda81ddc89 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -42,7 +42,7 @@ recipe_modules = ['recipe_' + r for r in ( 'moneynews', 'der_standard', 'diepresse', 'nzz_ger', 'hna', 'seattle_times', 'scott_hanselman', 'coding_horror', 'twitchfilms', 'stackoverflow', 'telepolis_artikel', 'zaobao', 'usnews', - 'straitstimes', + 'straitstimes', 'index_hu', 'pcworld_hu', 'hrt', 'rts', )] import re, imp, inspect, time, os diff --git a/src/calibre/web/feeds/recipes/recipe_barrons.py b/src/calibre/web/feeds/recipes/recipe_barrons.py index 164be20d3e..f9f501a9c3 100644 --- a/src/calibre/web/feeds/recipes/recipe_barrons.py +++ b/src/calibre/web/feeds/recipes/recipe_barrons.py @@ -1,76 +1,76 @@ ## -## web2lrf profile to download articles from Barrons.com -## can download subscriber-only content if username and +## web2lrf profile to download articles from Barrons.com +## can download subscriber-only content if username and ## password are supplied. ## -''' -''' - -import re - -from calibre.web.feeds.news import BasicNewsRecipe - -class Barrons(BasicNewsRecipe): - - title = 'Barron\'s' +''' +''' + +import re + +from calibre.web.feeds.news import BasicNewsRecipe + +class Barrons(BasicNewsRecipe): + + title = 'Barron\'s' max_articles_per_feed = 50 needs_subscription = True language = _('English') __author__ = 'Kovid Goyal' description = 'Weekly publication for investors from the publisher of the Wall Street Journal' - timefmt = ' [%a, %b %d, %Y]' - use_embedded_content = False + timefmt = ' [%a, %b %d, %Y]' + use_embedded_content = False no_stylesheets = False match_regexps = ['http://online.barrons.com/.*?html\?mod=.*?|file:.*'] - html2lrf_options = [('--ignore-tables'),('--base-font-size=10')] + conversion_options = {'linearize_tables': True} ##delay = 1 - - ## Don't grab articles more than 7 days old - oldest_article = 7 + + ## Don't grab articles more than 7 days old + oldest_article = 7 - preprocess_regexps = [(re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in - [ - ## Remove anything before the body of the article. - (r'