Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

sync with Kovid's branch

Browse Source
This commit is contained in:
Tomasz Długosz 2013-05-05 21:19:11 +02:00
commit 840f41017a
282 changed files with 119232 additions and 84187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
COPYRIGHT
View File

@ -79,13 +79,6 @@ License: GPL2+
The full text of the GPL is distributed as in
/usr/share/common-licenses/GPL-2 on Debian systems.
Files: src/pyPdf/*
Copyright: Copyright (c) 2006, Mathieu Fenniak
Copyright: Copyright (c) 2007, Ashish Kulkarni <kulkarni.ashish@gmail.com>
License: BSD
The full text of the BSD license is distributed as in
/usr/share/common-licenses/BSD on Debian systems.
Files: src/calibre/utils/lzx/*
Copyright: Copyright (C) 2002, Matthew T. Russotto
Copyright: Copyright (C) 2008, Marshall T. Vandegrift <llasram@gmail.com>
@ -100,49 +93,6 @@ License: BSD
The full text of the BSD license is distributed as in
/usr/share/common-licenses/BSD on Debian systems.
Files: src/calibre/utils/pyparsing.py
Copyright: Copyright (c) 2003-2008, Paul T. McGuire
License: MIT
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Files: src/calibre/utils/PythonMagickWand.py
Copyright: (c) 2007 - Achim Domma - domma@procoders.net
License: MIT
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
Files: src/calibre/utils/msdes/d3des.h:
Files: src/calibre/utils/msdes/des.c:
Copyright: Copyright (C) 1988,1989,1990,1991,1992, Richard Outerbridge

186
Changelog.yaml
View File

@ -1,4 +1,4 @@
# vim:fileencoding=UTF-8:ts=2:sw=2:sta:et:sts=2:ai
# vim:fileencoding=utf-8:ts=2:sw=2:sta:et:sts=2:ai
# Each release can have new features and bug fixes. Each of which
# must have a title and can optionally have linked tickets and a description.
# In addition they can have a type field which defaults to minor, but should be major
@ -20,6 +20,190 @@
# new recipes:
# - title:
- version: 0.9.29
date: 2013-05-03
new features:
- title: "Bulk metadata download: Allow reviewing of the downloaded metadata before it is applied"
- title: "FB2 Output: Write ISBN, pubdate, tags and publisher metadata when creating fb2 files"
tickets: [1174047]
bug fixes:
- title: "When reading metadata from EPUB 3 files, use the first <dc:title> element rather than the last."
tickets: [1175184]
- title: "Fix regression causing the search query parser to not parse search string containing newlines/tabs instead of spaces correctly"
tickets: [1174629]
- title: "Kobo driver: Fix covers written to wrong place on OS X/linux when books sent to SD card. Fix covers not sent to SD card is images directory missing."
tickets: [1174147,1174126]
- title: "Fix 'Preferences->Behavior->Virtual library to use when this library is opened' being applied only on calibre startup and not when switching to the library"
- title: "PDF metadata: When rendering the first page as the cover, respect the PDF CropBox."
tickets: [1173795]
- title: "PDF Output: Fix link generation broken on windows when converting epubs if the filenames contained uppercase letters."
tickets: [1169795]
- title: "Tolino driver: Fix card and main memory swapped on windows"
tickets: [1173544]
- title: "FB2 Output: Fix images being ignored when converting a EPUB with image filenames that contain URL unsafe characters."
tickets: [1173351]
- title: "EPUB Input: Fix page margins specified in Adobe page template files with incorrect mime-types not being removed."
improved recipes:
- The New Republic
- io9
- What if
- Orlando Sentinel
- Read It Later recipe
- Smithsonian
- Business Week Magazine
new recipes:
- title: Diario Extra
author: Douglas Delgado
- version: 0.9.28
date: 2013-04-26
new features:
- title: "Virtual Libraries: Easily partition your large calibre library into smaller 'virtual' libraries"
type: major
description: "A virtual library is a way to tell calibre to open only a subset of a normal library. For example, you might want to only work with books by a certain author, or books having only a certain tag. To use this feature, click the button labeled 'Virtual Library' to the left of the search bar. For details, see http://manual.calibre-ebook.com/virtual_libraries.html. This feature used to be called 'Search restriction', the new virtual libraries are easier to use, but otherwise fulfil the same function."
- title: "Book details panel: Allow copying of links in the book details panel by right clicking on them."
tickets: [1171963]
- title: "Kobo driver: Add support for the new Kobo Aura HD and firmware version 2.5.0"
tickets: [1169571,1169968]
- title: "Metadata download: When showing downloaded covers, allow right clicking on a cover to view a full size version."
tickets: [1170544]
- title: "Driver for Easy player cyber book e touch and Droid 4"
tickets: [1171633,1170763]
- title: "Edit ToC: Allow the size of the panels in the location view to be adjusted"
- title: "When copying to a library by path, make it more efficient to choose between moving and copying"
tickets: [1168231]
- title: "When checking if a zip/rar file is a comic or contains a single ebook to be auto-extracted, ignore thumbs.db files inside the archive"
bug fixes:
- title: "EPUB Input: Fix handling of EPUB files that contain images with non-ascii filenames."
tickets: [1171186]
- title: "Device driver: Detect Laser EB720 with newer firmware."
tickets: [1171341]
- title: "Fix bug in Danish translation causing books with language Ingush being incorrectly translated as Engelsk"
- title: "PDF Output: Fix hyperlinks not working when converting an EPUB whose individual files have names with URL unsafe characters."
tickets: [1169795]
- title: "Book polishing: Fix inserting cover into an epub with no cover could lead to incorrect guide entry if the opf is not at the root of the epub."
tickets: [1167941]
- title: "ZIP Output: Fix links containing backslashes on windows"
tickets: [1169910]
- title: "Fix polishing of AZW3 files not working on OS X."
tickets: [1168789]
- title: "Polishing books: Fix polishing erroring out if the book being polished has no cover"
- title: "RTF Input: Add partial support for hyperlinks to web resources."
tickets: [1167562]
- title: "Fix book details panel showing incorrect info after deleting books from a connected device"
tickets: [1172839]
improved recipes:
- NZZ Online
- Baltimore Sun
- Metro NL
- Financial Times
- EcoGeek
- comics.com
- Psychology Today
- Science News
new recipes:
- title: Voice of America
author: Krittika Goyal
- title: Lightspeed Magazine
author: Jose Pinto
- title: The Feature
author: Jose Pinto
- version: 0.9.27
date: 2013-04-12
new features:
- title: "Metadata download: Add two new sources for covers: Google Image Search and bigbooksearch.com."
description: "To enable them go to Preferences->Metadata download and enable the 'Google Image' and 'Big Book Search' sources. Google Images is useful for finding larger covers as well as alternate versions of the cover. Big Book Search searches for alternate covers from amazon.com. It can occasionally find nicer covers than the direct Amazon source. Note that both these sources download multiple covers for a single book. Some of these covers can be wrong (i.e. they may be of a different book or not covers at all, so you should inspect the results and manually pick the best match). When bulk downloading, these sources are only used if the other sources find no covers."
type: major
- title: "Content server: Allow specifying a restriction to use for the server when embedding it as a WSGI app."
tickets: [1167951]
- title: "Get Books: Add a plugin for the Koobe Polish book store"
- title: "calibredb add_format: Add an option to not replace existing formats. Also pep8 compliance."
- title: "Allow restoring of the ORIGINAL_XXX format by right-clicking it in the book details panel"
bug fixes:
- title: "AZW3 Input: Do not fail to identify JPEG images with 8BIM headers created with Adobe Photoshop."
tickets: [1167985]
- title: "Amazon metadata download: Ignore Spanish edition entries when searching for a book on amazon.com"
- title: "TXT Input: When converting a txt file with a Byte Order Mark, remove the Byte Order Mark before further processing as it can cause the first line of the text to be mis-interpreted."
- title: "Get Books: Fix searching for current book/title/author by right clicking the get books icon"
- title: "Get Books: Update nexto, gutenberg, and virtualo store plugins for website changes"
- title: "Amazon metadata download: When downloading from amazon.co.jp handle the 'Black curtain redirect' for adult titles."
tickets: [1165628]
- title: "When extracting zip files do not allow maliciously created zip files to overwrite other files on the system"
- title: "RTF Input: Handle RTF files with invalid border style specifications"
tickets: [1021270]
improved recipes:
- The Escapist
- San Francisco Chronicle
- The Onion
- Fronda
- Tom's Hardware
- New Yorker
- Financial Times UK
- Business Week Magazine
- Victoria Times
- tvxs
- The Independent
new recipes:
- title: Economia
author: Manish Bhattarai
- title: Universe Today
author: seird
- title: The Galaxy's Edge
author: Krittika Goyal
- version: 0.9.26
date: 2013-04-05

4
manual/conversion.rst
View File

@ -436,8 +436,8 @@ generate a Table of Contents in the converted ebook, based on the actual content
.. note:: Using these options can be a little challenging to get exactly right.
If you prefer creating/editing the Table of Contents by hand, convert to
the EPUB or AZW3 formats and select the checkbox at the bottom of the
screen that says
the EPUB or AZW3 formats and select the checkbox at the bottom of the Table
of Contents section of the conversion dialog that says
:guilabel:`Manually fine-tune the Table of Contents after conversion`.
This will launch the ToC Editor tool after the conversion. It allows you to
create entries in the Table of Contents by simply clicking the place in the

14
manual/develop.rst
View File

@ -67,8 +67,12 @@ and you will most likely get help from one of |app|'s many developers.
Getting the code
------------------
|app| uses `Bazaar <http://bazaar-vcs.org/>`_, a distributed version control system. Bazaar is available on all the platforms |app| supports.
After installing Bazaar, you can get the |app| source code with the command::
You can get the |app| source code in two ways, using a version control system or
directly downloading a `tarball <http://status.calibre-ebook.com/dist/src>`_.
|app| uses `Bazaar <http://bazaar-vcs.org/>`_, a distributed version control
system. Bazaar is available on all the platforms |app| supports. After
installing Bazaar, you can get the |app| source code with the command::
bzr branch lp:calibre
@ -124,6 +128,8 @@ discuss them in the forum or contact Kovid directly (his email address is all ov
Windows development environment
---------------------------------
.. note:: You must also get the |app| source code separately as described above.
Install |app| normally, using the Windows installer. Then open a Command Prompt and change to
the previously checked out |app| code directory. For example::
@ -153,6 +159,8 @@ near the top of the file. Now run the command :command:`calibredb`. The very fir
OS X development environment
------------------------------
.. note:: You must also get the |app| source code separately as described above.
Install |app| normally using the provided .dmg. Then open a Terminal and change to
the previously checked out |app| code directory, for example::
@ -183,6 +191,8 @@ window, indicating that you are running from source.
Linux development environment
------------------------------
.. note:: You must also get the |app| source code separately as described above.
|app| is primarily developed on Linux. You have two choices in setting up the development environment. You can install the
|app| binary as normal and use that as a runtime environment to do your development. This approach is similar to that
used in Windows and OS X. Alternatively, you can install |app| from source. Instructions for setting up a development

23
manual/faq.rst
View File

@ -647,12 +647,17 @@ computers. Run |app| on a single computer and access it via the Content Server
or a Remote Desktop solution.
If you must share the actual library, use a file syncing tool like
DropBox or rsync or Microsoft SkyDrive instead of a networked drive. Even with
these tools there is danger of data corruption/loss, so only do this if you are
willing to live with that risk. In particular, be aware that **Google Drive**
is incompatible with |app|, if you put your |app| library in Google Drive, you
*will* suffer data loss. See
`this thread <http://www.mobileread.com/forums/showthread.php?t=205581>`_ for details.
DropBox or rsync or Microsoft SkyDrive instead of a networked drive. If you are
using a file-syncing tool it is **essential** that you make sure that both
|app| and the file syncing tool do not try to access the |app| library at the
same time. In other words, **do not** run the file syncing tool and |app| at
the same time.
Even with these tools there is danger of data corruption/loss, so only do this
if you are willing to live with that risk. In particular, be aware that
**Google Drive** is incompatible with |app|, if you put your |app| library in
Google Drive, **you will suffer data loss**. See `this thread
<http://www.mobileread.com/forums/showthread.php?t=205581>`_ for details.
Content From The Web
---------------------
@ -797,6 +802,12 @@ Downloading from the Internet can sometimes result in a corrupted download. If t
* Try temporarily disabling your antivirus program (Microsoft Security Essentials, or Kaspersky or Norton or McAfee or whatever). This is most likely the culprit if the upgrade process is hanging in the middle.
* Try rebooting your computer and running a registry cleaner like `Wise registry cleaner <http://www.wisecleaner.com>`_.
* Try downloading the installer with an alternate browser. For example if you are using Internet Explorer, try using Firefox or Chrome instead.
* If you get an error about a missing DLL on windows, then most likely, the
permissions on your temporary folder are incorrect. Go to the folder
:file:`C:\\Users\\USERNAME\\AppData\\Local` in Windows explorer and then
right click on the :file:`Temp` folder and select :guilabel:`Properties` and go to
the :guilabel:`Security` tab. Make sure that your user account has full control
for this folder.
If you still cannot get the installer to work and you are on windows, you can use the `calibre portable install <http://calibre-ebook.com/download_portable>`_, which does not need an installer (it is just a zip file).

19
manual/gui.rst
View File

@ -367,6 +367,8 @@ For example::
date:>10daysago
date:<=45daysago
To avoid potential problems with translated strings when using a non-English version of calibre, the strings ``_today``, ``_yesterday``, ``_thismonth``, and ``_daysago`` are always available. They are not translated.
You can search for books that have a format of a certain size like this::
@ -424,6 +426,8 @@ Identifiers (e.g., isbn, doi, lccn etc) also use an extended syntax. First, note
:guilabel:`Advanced Search Dialog`
.. _saved_searches:
Saving searches
-----------------
@ -433,6 +437,15 @@ Now you can access your saved search in the Tag Browser under "Searches". A sing
.. _config_filename_metadata:
Virtual Libraries
-------------------
A :guilabel:`Virtual Library` is a way to pretend that your |app| library has
only a few books instead of its full collection. This is an excellent way to
partition your large collection of books into smaller, manageable chunks. To
learn how to create and use virtual libraries, see the tutorial:
:ref:`virtual_libraries`.
Guessing metadata from file names
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In the :guilabel:`Add/Save` section of the configuration dialog, you can specify a regular expression that |app| will use to try and guess metadata from the names of ebook files
@ -569,6 +582,12 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
- Open the advanced search dialog
* - :kbd:`Esc`
- Clear the current search
* - :kbd:`Shift+Esc`
- Focus the book list
* - :kbd:`Ctrl+Esc`
- Clear the virtual library
* - :kbd:`Alt+Esc`
- Clear the additional restriction
* - :kbd:`N or F3`
- Find the next book that matches the current search (only works if the highlight checkbox next to the search bar is checked)
* - :kbd:`Shift+N or Shift+F3`

BIN
manual/images/virtual_library_button.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

BIN
manual/images/vl_by_author.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 65 KiB

6
manual/server.rst
View File

@ -91,7 +91,11 @@ First, we have to create a WSGI *adapter* for the calibre content server. Here i
# Path to the calibre library to be served
# The server process must have write permission for all files/dirs
# in this directory or BAD things will happen
path_to_library='/home/kovid/documents/demo library'
path_to_library='/home/kovid/documents/demo library',
# The virtual library (restriction) to be used when serving this
# library.
virtual_library=None
)
del create_wsgi_app

1
manual/tutorials.rst
View File

@ -20,4 +20,5 @@ Here you will find tutorials to get you started using |app|'s more advanced feat
creating_plugins
typesetting_math
catalogs
virtual_libraries

89
manual/virtual_libraries.rst Normal file
View File

@ -0,0 +1,89 @@
.. include:: global.rst
.. _virtual_libraries:
Virtual Libraries
============================
In |app|, a virtual library is a way to tell |app| to open only a subset of a
normal library. For example, you might want to only work with books by a certain
author, or books having only a certain tag. Using virtual libraries is the
preferred way of partitioning your large book collection into smaller sub
collections. It is superior to splitting up your library into multiple smaller
libraries as, when you want to search through your entire collection, you can
simply go back to the full library. There is no way to search through multiple
separate libraries simultaneously in |app|.
A virtual library is different from a simple search. A search will only restrict
the list of books shown in the book list. A virtual library does that, and in
addition it also restricts the entries shown in the :guilabel:`Tag Browser` to
the left. The Tag Browser will only show tags, authors, series, publishers, etc.
that come from the books in the virtual library. A virtual library thus behaves
as though the actual library contains only the restricted set of books.
Creating Virtual Libraries
----------------------------
.. |vlb| image:: images/virtual_library_button.png
:class: float-left-img
|vlb| To use a virtual library click the :guilabel:`Virtual Library` button located
to the left of the search bar and select the :guilabel:`Create Virtual Library`
option. As a first example, let's create a virtual library that shows us only
the books by a particular author. Click the :guilabel:`Authors` link as shown
in the image below and choose the author you want to use and click OK.
.. image:: images/vl_by_author.png
:align: center
The Create Virtual Library dialog has been filled in for you. Click OK and you
will see that a new Virtual Library has been created, and automatically
switched to, that displays only the books by the selected author. As far as
|app| is concerned, it is as if your library contains only the books by the
selected author.
You can switch back to the full library at any time by once again clicking the
:guilabel:`Virtual Library` and selecting the entry named :guilabel:`<None>`.
Virtual Libraries are based on *searches*. You can use any search as the
basis of a virtual library. The virtual library will contain only the
books matched by that search. First, type in the search you want to use
in the search bar or build a search using the :guilabel:`Tag Browser`.
When you are happy with the returned results, click the Virtual Library
button, choose Create Library and enter a name for the new virtual
library. The virtual library will then be created based on the search
you just typed in. Searches are very powerful, for examples of the kinds
of things you can do with them, see :ref:`search_interface`.
Working with Virtual Libraries
-------------------------------------
You can edit a previously created virtual library or remove it, by clicking the
:guilabel:`Virtual Library` and choosing the appropriate action.
You can tell |app| that you always want to apply a particular virtual library
when the current library is opened, by going to
:guilabel:`Preferences->Behavior`.
If you use the |app| Content Server, you can have it share a virtual library
instead of the full library by going to :guilabel:`Preferences->Sharing over the net`.
You can quickly use the current search as a temporary virtual library by
clicking the :guilabel:`Virtual Library` button and choosing the
:guilabel:`*current search` entry.
Using additional restrictions
-------------------------------
You can further restrict the books shown in a Virtual Library by using
:guilabel:`Additional restrictions`. An additional restriction is saved search
you previously created that can be applied to the current Virtual Library to
further restrict the books shown in a virtual library. For example, say you
have a Virtual Library for books tagged as :guilabel:`Historical Fiction` and a
saved search that shows you unread books, you can click the :guilabel:`Virtual
Library` button and choose the :guilabel:`Additional restriction` option to
show only unread Historical Fiction books. To learn about saved searches, see
:ref:`saved_searches`.

95
recipes/baltimore_sun.recipe
View File

@ -13,13 +13,13 @@ class BaltimoreSun(BasicNewsRecipe):
__author__ = 'Josh Hall'
description = 'Complete local news and blogs from Baltimore'
language = 'en'
version = 2.1
version = 2.5
oldest_article = 1
max_articles_per_feed = 100
use_embedded_content = False
no_stylesheets = True
remove_javascript = True
#auto_cleanup = True
remove_empty_feeds= True
recursions = 1
ignore_duplicate_articles = {'title'}
@ -31,7 +31,7 @@ class BaltimoreSun(BasicNewsRecipe):
match_regexps = [r'page=[0-9]+']
remove_tags = [{'id':["moduleArticleTools","content-bottom","rail","articleRelates module","toolSet","relatedrailcontent","div-wrapper","beta","atp-comments","footer",'gallery-subcontent','subFooter']},
{'class':["clearfix","relatedTitle","articleRelates module","asset-footer","tools","comments","featurePromo","featurePromo fp-topjobs brownBackground","clearfix fullSpan brownBackground","curvedContent",'nextgen-share-tools','outbrainTools', 'google-ad-story-bottom']},
{'class':["clearfix","relatedTitle","articleRelates module","asset-footer","tools","comments","featurePromo","featurePromo fp-topjobs brownBackground","clearfix fullSpan brownBackground","curvedContent",'nextgen-share-tools','nextgen-comments-container','nextgen-comments-content','outbrainTools','fb-like' 'google-ad-story-bottom']},
dict(name='font',attrs={'id':["cr-other-headlines"]})]
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
@ -49,40 +49,39 @@ class BaltimoreSun(BasicNewsRecipe):
'''
feeds = [
## News ##
(u'Top Headlines', u'http://www.baltimoresun.com/rss2.0.xml'),
(u'Breaking News', u'http://www.baltimoresun.com/news/breaking/rss2.0.xml'),
(u'Top Maryland', u'http://www.baltimoresun.com/news/maryland/rss2.0.xml'),
#(u'Anne Arundel County', u'http://www.baltimoresun.com/news/maryland/anne-arundel/rss2.0.xml'),
(u'Baltimore City', u'http://www.baltimoresun.com/news/maryland/baltimore-city/rss2.0.xml'),
#(u'Baltimore County', u'http://www.baltimoresun.com/news/maryland/baltimore-county/rss2.0.xml'),
#(u'Carroll County', u'http://www.baltimoresun.com/news/maryland/carroll/rss2.0.xml'),
#(u'Harford County', u'http://www.baltimoresun.com/news/maryland/harford/rss2.0.xml'),
#(u'Howard County', u'http://www.baltimoresun.com/news/maryland/howard/rss2.0.xml'),
(u'Education', u'http://www.baltimoresun.com/news/education/rss2.0.xml'),
#(u'Obituaries', u'http://www.baltimoresun.com/news/obituaries/rss2.0.xml'),
(u'Local Politics', u'http://www.baltimoresun.com/news/maryland/politics/rss2.0.xml'),
(u'Weather', u'http://www.baltimoresun.com/news/weather/rss2.0.xml'),
#(u'Traffic', u'http://www.baltimoresun.com/features/commuting/rss2.0.xml'),
(u'Top Headlines', u'http://feeds.feedburner.com/baltimoresun/news/rss2'),
(u'Breaking News', u'http://feeds.feedburner.com/baltimoresun/news/local/annearundel/rss2'),
(u'Top Maryland', u'http://feeds.feedburner.com/baltimoresun/news/local/rss2'),
#(u'Anne Arundel County', u'http://feeds.feedburner.com/baltimoresun/news/local/annearundel/rss2'),
(u'Baltimore City', u'http://feeds.feedburner.com/baltimoresun/news/local/baltimore_city/rss20xml'),
#(u'Baltimore County', u'http://feeds.feedburner.com/baltimoresun/news/local/baltimore_county/rss2'),
#(u'Carroll County', u'http://feeds.feedburner.com/baltimoresun/news/local/carroll/rss2'),
#(u'Harford County', u'http://feeds.feedburner.com/baltimoresun/news/local/harford/rss2),
#(u'Howard County', u'http://feeds.feedburner.com/baltimoresun/news/local/howard/rss2'),
(u'Education', u'http://feeds.feedburner.com/baltimoresun/news/education/rss2'),
#(u'Obituaries', u'http://feeds.feedburner.com/baltimoresun/news/obituaries/rss2'),
(u'Local Politics', u'http://feeds.feedburner.com/baltimoresun/news/local/politics/rss2'),
(u'Weather', u'http://feeds.feedburner.com/baltimoresun/news/weather/site/rss2'),
#(u'Traffic', u'http://feeds.feedburner.com/baltimoresun/news/traffic/rss2'),
(u'Nation/world', u'http://feeds.feedburner.com/baltimoresun/news/nationworld/rss2'),
(u'Weird News', u'http://www.baltimoresun.com/news/offbeat/rss2.0.xml'),
#(u'Weird News', u'http://feeds.feedburner.com/baltsun-weirdnews'),
##Sports##
(u'Top Sports', u'http://www.baltimoresun.com/sports/rss2.0.xml'),
(u'Top Sports', u'http://feeds.feedburner.com/baltimoresun/sports/rss2'),
(u'Orioles/Baseball', u'http://www.baltimoresun.com/sports/orioles/rss2.0.xml'),
(u'Ravens/Football', u'http://www.baltimoresun.com/sports/ravens/rss2.0.xml'),
#(u'Terps', u'http://www.baltimoresun.com/sports/terps/rss2.0.xml'),
#(u'College Football', u'http://www.baltimoresun.com/sports/college/football/rss2.0.xml'),
#(u'Lacrosse', u'http://www.baltimoresun.com/sports/college/lacrosse/rss2.0.xml'),
#(u'Horse Racing', u'http://www.baltimoresun.com/sports/horse-racing/rss2.0.xml'),
#(u'Golf', u'http://www.baltimoresun.com/sports/golf/rss2.0.xml'),
#(u'NBA', u'http://www.baltimoresun.com/sports/nba/rss2.0.xml'),
#(u'High School', u'http://www.baltimoresun.com/sports/high-school/rss2.0.xml'),
#(u'Outdoors', u'http://www.baltimoresun.com/sports/outdoors/rss2.0.xml'),
(u'Ravens/Football', u'http://feeds.feedburner.com/baltimoresun/sports/football/rss2'),
#(u'Terps', u''http://feeds.feedburner.com/baltimoresun/sports/terps/rss2'),
#(u'College Football', u''feed://feeds.feedburner.com/baltimoresun/sports/college/football/rss2'),
#(u'Lacrosse', u'http://feeds.feedburner.com/baltimoresun/sports/college/lacrosse/rss2'),
#(u'Horse Racing', u'http://feeds.feedburner.com/baltimoresun/sports/horseracing/rss2'),
#(u'Golf', u'http://feeds.feedburner.com/baltimoresun/sports/golf/rss2'),
#(u'NBA', u'http://feeds.feedburner.com/baltimoresun/sports/basketball/rss2'),
#(u'High School', u'http://feeds.feedburner.com/baltimoresun/sports/highschool/rss2'),
#(u'Outdoors', u'http://feeds.feedburner.com/baltimoresun/sports/outdoors/rss2'),
## Entertainment ##
(u'Celebrity News', u'http://www.baltimoresun.com/entertainment/celebrities/rss2.0.xml'),
(u'Arts & Theater', u'http://www.baltimoresun.com/entertainment/arts/rss2.0.xml'),
(u'Celebrity News', u'http://baltimore.feedsportal.com/c/34255/f/623042/index.rss'),
(u'Arts & Theater', u'http://feeds.feedburner.com/baltimoresun/entertainment/galleriesmuseums/rss2'),
(u'Movies', u'http://www.baltimoresun.com/entertainment/movies/rss2.0.xml'),
(u'Music & Nightlife', u'http://www.baltimoresun.com/entertainment/music/rss2.0.xml'),
(u'Restaurants & Food', u'http://www.baltimoresun.com/entertainment/dining/rss2.0.xml'),
@ -92,7 +91,6 @@ class BaltimoreSun(BasicNewsRecipe):
(u'Health&Wellness', u'http://www.baltimoresun.com/health/rss2.0.xml'),
(u'Home & Garden', u'http://www.baltimoresun.com/features/home-garden/rss2.0.xml'),
(u'Living Green', u'http://www.baltimoresun.com/features/green/rss2.0.xml'),
(u'Parenting', u'http://www.baltimoresun.com/features/parenting/rss2.0.xml'),
(u'Fashion', u'http://www.baltimoresun.com/features/fashion/rss2.0.xml'),
(u'Travel', u'http://www.baltimoresun.com/travel/rss2.0.xml'),
#(u'Faith', u'http://www.baltimoresun.com/features/faith/rss2.0.xml'),
@ -100,17 +98,17 @@ class BaltimoreSun(BasicNewsRecipe):
## Business ##
(u'Top Business', u'http://www.baltimoresun.com/business/rss2.0.xml'),
(u'Technology', u'http://www.baltimoresun.com/business/technology/rss2.0.xml'),
(u'Personal finance', u'http://www.baltimoresun.com/business/money/rss2.0.xml'),
(u'Personal finance', u'http://baltimore.feedsportal.com/c/34255/f/623057/index.rss'),
(u'Real Estate', u'http://www.baltimoresun.com/classified/realestate/rss2.0.xml'),
(u'Jobs', u'http://www.baltimoresun.com/classified/jobs/rss2.0.xml'),
(u'DIY', u'http://www.baltimoresun.com/features/do-it-yourself/rss2.0.xml'),
(u'Consumer Safety', u'http://www.baltimoresun.com/business/consumer-safety/rss2.0.xml'),
(u'Jobs', u'http://baltimore.feedsportal.com/c/34255/f/623059/index.rss'),
#(u'DIY', u'http://baltimore.feedsportal.com/c/34255/f/623060/index.rss'),
#(u'Consumer Safety', u'http://baltimore.feedsportal.com/c/34255/f/623061/index.rss'),
(u'Investing', u'http://www.baltimoresun.com/business/money/rss2.0.xml'),
## Opinion##
(u'Sun Editorials', u'http://www.baltimoresun.com/news/opinion/editorial/rss2.0.xml'),
(u'Op/Ed', u'http://www.baltimoresun.com/news/opinion/oped/rss2.0.xml'),
(u'Readers Respond', u'http://www.baltimoresun.com/news/opinion/readersrespond/'),
(u'Readers Respond', u'http://baltimore.feedsportal.com/c/34255/f/623065/index.rss'),
## Columnists ##
(u'Kevin Cowherd', u'http://www.baltimoresun.com/sports/bal-columnist-cowherd,0,6829726.columnist-rss2.0.xml'),
@ -138,30 +136,26 @@ class BaltimoreSun(BasicNewsRecipe):
(u'The Real Estate Wonk', u'http://www.baltimoresun.com/business/real-estate/wonk/rss2.0.xml'),
## Entertainment Blogs ##
(u'Clef Notes & Drama Queens', 'http://weblogs.baltimoresun.com/entertainment/classicalmusic/index.xml'),
(u'Baltimore Diner', u'http://baltimore.feedsportal.com/c/34255/f/623088/index.rss'),
(u'ArtSmash', 'http://www.baltimoresun.com/entertainment/arts/artsmash/rss2.0.xml'),
(u'Baltimore Diner', u'http://baltimore.feedsportal.com/c/34255/f/623088/index.rss'),
(u'Midnight Sun', u'http://www.baltimoresun.com/entertainment/music/midnight-sun-blog/rss2.0.xml'),
(u'Read Street', u'http://www.baltimoresun.com/features/books/read-street/rss2.0.xml'),
(u'Z on TV', u'http://www.baltimoresun.com/entertainment/tv/z-on-tv-blog/rss2.0.xml'),
### Life Blogs ##
## Life Blogs ##
#(u'BMore Green', u'http://weblogs.baltimoresun.com/features/green/index.xml'),
#(u'Baltimore Insider',u'http://www.baltimoresun.com/features/baltimore-insider-blog/rss2.0.xml'),
#(u'Homefront', u'http://www.baltimoresun.com/features/parenting/homefront/rss2.0.xml'),
#(u'Picture of Health', u'http://www.baltimoresun.com/health/blog/rss2.0.xml'),
#(u'Unleashed', u'http://weblogs.baltimoresun.com/features/mutts/blog/index.xml'),
(u'Baltimore Insider',u'http://www.baltimoresun.com/features/baltimore-insider-blog/rss2.0.xml'),
(u'Picture of Health', u'http://www.baltimoresun.com/health/blog/rss2.0.xml'),
#(u'Unleashed', u'http://weblogs.baltimoresun.com/features/mutts/blog/index.xml'),
## b the site blogs ##
(u'Game Cache', u'http://www.baltimoresun.com/entertainment/bthesite/game-cache/rss2.0.xml'),
(u'TV Lust', u'http://www.baltimoresun.com/entertainment/bthesite/tv-lust/rss2.0.xml'),
(u'TV Lust', u'http://baltimore.feedsportal.com/c/34255/f/623096/index.rss'),
## Sports Blogs ##
(u'Baltimore Sports Blitz', u'http://baltimore.feedsportal.com/c/34255/f/623097/index.rss'),
#(u'Faceoff', u'http://weblogs.baltimoresun.com/sports/lacrosse/blog/index.xml'),
#(u'MMA Stomping Grounds', u'http://weblogs.baltimoresun.com/sports/mma/blog/index.xml'),
## (u'Lacrosse Insider',u'http://www.baltimoresun.com/sports/lacrosse-blog/rss2.0.xml'),
(u'Orioles Insider', u'http://baltimore.feedsportal.com/c/34255/f/623100/index.rss'),
(u'Ravens Insider', u'http://www.baltimoresun.com/sports/ravens/ravens-insider/rss2.0.xml'),
#(u'Recruiting Report', u'http://weblogs.baltimoresun.com/sports/college/recruiting/index.xml'),
#(u'Ring Posts', u'http://weblogs.baltimoresun.com/sports/wrestling/blog/index.xml'),
(u'The Schmuck Stops Here', u'http://www.baltimoresun.com/sports/schmuck-blog/rss2.0.xml'),
#(u'Tracking the Terps', u'http://weblogs.baltimoresun.com/sports/college/maryland_terps/blog/index.xml'),
@ -169,7 +163,6 @@ class BaltimoreSun(BasicNewsRecipe):
]
def get_article_url(self, article):
ans = None
try:
@ -190,6 +183,8 @@ class BaltimoreSun(BasicNewsRecipe):
url = a.get('href')
if url:
return self.index_to_soup(url, raw=True)
def print_version(self, url):
return self.browser.open_novisit(url).geturl()
def postprocess_html(self, soup, first_fetch):
# Remove the navigation bar. It was kept until now to be able to follow

34
recipes/bwmagazine2.recipe
View File

@ -12,7 +12,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [
dict(name='div', attrs={'id':'article_body_container'}),
dict(name='div', attrs={'id':['article_body_container','story_body']}),
]
remove_tags = [dict(name='ui'),dict(name='li'),dict(name='div', attrs={'id':['share-email']})]
no_javascript = True
@ -26,43 +26,45 @@ class BusinessWeekMagazine(BasicNewsRecipe):
#Find date
mag=soup.find('h2',text='Magazine')
self.log(mag)
dates=self.tag_to_string(mag.findNext('h3'))
self.timefmt = u' [%s]'%dates
#Go to the main body
div0 = soup.find ('div', attrs={'class':'column left'})
div0 = soup.find('div', attrs={'class':'column left'})
section_title = ''
feeds = OrderedDict()
for div in div0.findAll(['h4','h5']):
for div in div0.findAll('a', attrs={'class': None}):
articles = []
section_title = self.tag_to_string(div.findPrevious('h3')).strip()
title=self.tag_to_string(div.a).strip()
url=div.a['href']
title=self.tag_to_string(div).strip()
url=div['href']
soup0 = self.index_to_soup(url)
urlprint=soup0.find('a', attrs={'href':re.compile('.*printer.*')})['href']
articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''})
urlprint=soup0.find('a', attrs={'href':re.compile('.*printer.*')})
if urlprint is not None:
url=urlprint['href']
articles.append({'title':title, 'url':url, 'description':'', 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
div1 = soup.find ('div', attrs={'class':'column center'})
div1 = soup.find('div', attrs={'class':'column center'})
section_title = ''
for div in div1.findAll(['h4','h5']):
for div in div1.findAll('a'):
articles = []
desc=self.tag_to_string(div.findNext('p')).strip()
section_title = self.tag_to_string(div.findPrevious('h3')).strip()
title=self.tag_to_string(div.a).strip()
url=div.a['href']
title=self.tag_to_string(div).strip()
url=div['href']
soup0 = self.index_to_soup(url)
urlprint=soup0.find('a', attrs={'href':re.compile('.*printer.*')})['href']
articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''})
urlprint=soup0.find('a', attrs={'href':re.compile('.*printer.*')})
if urlprint is not None:
url=urlprint['href']
articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans

47
recipes/diario_extra.recipe Normal file
View File

@ -0,0 +1,47 @@
from calibre.web.feeds.news import BasicNewsRecipe
class goonews(BasicNewsRecipe):
__author__ = 'Douglas Delgado'
title = u'Diario Extra'
publisher = 'Sociedad Periodistica Extra Limitada'
description = 'Diario de circulacion nacional de Costa Rica.'
category = 'Spanish, Entertainment'
masthead_url = 'http://www.diarioextra.com/img/apariencia/logo.png'
oldest_article = 7
delay = 1
max_articles_per_feed = 100
auto_cleanup = True
encoding = 'utf-8'
language = 'es_CR'
use_embedded_content = False
remove_empty_feeds = True
remove_javascript = True
no_stylesheets = True
feeds = [(u'Nacionales',
u'http://www.diarioextra.com/includes/rss_text.php?id=1'),
(u'Internacionales',
u'http://www.diarioextra.com/includes/rss_text.php?id=2'),
(u'Sucesos',
u'http://www.diarioextra.com/includes/rss_text.php?id=3'),
(u'Deportes',
u'http://www.diarioextra.com/includes/rss_text.php?id=6'),
(u'Espectaculos',
u'http://www.diarioextra.com/includes/rss_text.php?id=7'),
(u'Opinion',
u'http://www.diarioextra.com/includes/rss_text.php?id=4')]
def get_cover_url(self):
index = 'http://kiosko.net/cr/np/cr_extra.html'
soup = self.index_to_soup(index)
for image in soup.findAll('img', src=True):
if image['src'].endswith('cr_extra.750.jpg'):
return image['src']
return None
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:30px;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal; font-style:italic; font-size:18px;}
'''

10
recipes/ecogeek.recipe
View File

@ -11,22 +11,22 @@ from calibre.web.feeds.news import BasicNewsRecipe
class EcoGeek(BasicNewsRecipe):
title = 'EcoGeek'
__author__ = 'Darko Miletic'
description = 'EcoGeek - Technology for the Environment Blog Feed'
description = 'EcoGeek - Technology for the Environment Blog Feed'
publisher = 'EcoGeek'
language = 'en'
category = 'news, ecology, blog'
oldest_article = 7
oldest_article = 30
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = True
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
feeds = [(u'Posts', u'http://feeds2.feedburner.com/EcoGeek')]

6
recipes/financial_times_uk.recipe
View File

@ -1,7 +1,7 @@
__license__ = 'GPL v3'
__copyright__ = '2010-2012, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2013, Darko Miletic <darko.miletic at gmail.com>'
'''
www.ft.com/uk-edition
www.ft.com/intl/uk-edition
'''
import datetime
@ -29,7 +29,7 @@ class FinancialTimes(BasicNewsRecipe):
masthead_url = 'http://im.media.ft.com/m/img/masthead_main.jpg'
LOGIN = 'https://registration.ft.com/registration/barrier/login'
LOGIN2 = 'http://media.ft.com/h/subs3.html'
INDEX = 'http://www.ft.com/uk-edition'
INDEX = 'http://www.ft.com/intl/uk-edition'
PREFIX = 'http://www.ft.com'
conversion_options = {

63
recipes/financial_times_us.recipe
View File

@ -1,20 +1,21 @@
__license__ = 'GPL v3'
__copyright__ = '2013, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2013, Darko Miletic <darko.miletic at gmail.com>'
'''
http://www.ft.com/intl/us-edition
www.ft.com/intl/international-edition
'''
import datetime
from calibre.ptempfile import PersistentTemporaryFile
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
from collections import OrderedDict
class FinancialTimes(BasicNewsRecipe):
title = 'Financial Times (US) printed edition'
title = 'Financial Times (International) printed edition'
__author__ = 'Darko Miletic'
description = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
publisher = 'The Financial Times Ltd.'
category = 'news, finances, politics, UK, World'
category = 'news, finances, politics, World'
oldest_article = 2
language = 'en'
max_articles_per_feed = 250
@ -28,7 +29,7 @@ class FinancialTimes(BasicNewsRecipe):
masthead_url = 'http://im.media.ft.com/m/img/masthead_main.jpg'
LOGIN = 'https://registration.ft.com/registration/barrier/login'
LOGIN2 = 'http://media.ft.com/h/subs3.html'
INDEX = 'http://www.ft.com/intl/us-edition'
INDEX = 'http://www.ft.com/intl/international-edition'
PREFIX = 'http://www.ft.com'
conversion_options = {
@ -93,7 +94,7 @@ class FinancialTimes(BasicNewsRecipe):
try:
urlverified = self.browser.open_novisit(url).geturl() # resolve redirect.
except:
continue
continue
title = self.tag_to_string(item)
date = strftime(self.timefmt)
articles.append({
@ -105,29 +106,30 @@ class FinancialTimes(BasicNewsRecipe):
return articles
def parse_index(self):
feeds = []
feeds = OrderedDict()
soup = self.index_to_soup(self.INDEX)
dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div'))
self.timefmt = ' [%s]'%dates
wide = soup.find('div',attrs={'class':'wide'})
if not wide:
return feeds
allsections = wide.findAll(attrs={'class':lambda x: x and 'footwell' in x.split()})
if not allsections:
return feeds
count = 0
for item in allsections:
count = count + 1
if self.test and count > 2:
return feeds
fitem = item.h3
if not fitem:
fitem = item.h4
ftitle = self.tag_to_string(fitem)
self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))
feedarts = self.get_artlinks(item.ul)
feeds.append((ftitle,feedarts))
return feeds
#dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div'))
#self.timefmt = ' [%s]'%dates
section_title = 'Untitled'
for column in soup.findAll('div', attrs = {'class':'feedBoxes clearfix'}):
for section in column. findAll('div', attrs = {'class':'feedBox'}):
sectiontitle=self.tag_to_string(section.find('h4'))
if '...' not in sectiontitle: section_title=sectiontitle
for article in section.ul.findAll('li'):
articles = []
title=self.tag_to_string(article.a)
url=article.a['href']
articles.append({'title':title, 'url':url, 'description':'', 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans
def preprocess_html(self, soup):
items = ['promo-box','promo-title',
@ -174,9 +176,6 @@ class FinancialTimes(BasicNewsRecipe):
count += 1
tfile = PersistentTemporaryFile('_fa.html')
tfile.write(html)
tfile.close()
tfile.close()
self.temp_files.append(tfile)
return tfile.name
def cleanup(self):
self.browser.open('https://registration.ft.com/registration/login/logout?location=')

1
recipes/fronda.recipe
View File

@ -6,6 +6,7 @@ __copyright__ = u'2010-2013, Tomasz Dlugosz <tomek3d@gmail.com>'
fronda.pl
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
from datetime import timedelta, date

90
recipes/givemesomethingtoread.recipe
View File

@ -1,90 +0,0 @@
import re
from calibre.web.feeds.news import BasicNewsRecipe
class GiveMeSomethingToRead(BasicNewsRecipe):
title = u'Give Me Something To Read'
description = 'Curation / aggregation of articles on diverse topics'
language = 'en'
__author__ = 'barty on mobileread.com forum'
max_articles_per_feed = 100
no_stylesheets = False
timefmt = ' [%a, %d %b, %Y]'
oldest_article = 365
auto_cleanup = True
INDEX = 'http://givemesomethingtoread.com'
CATEGORIES = [
# comment out categories you don't want
# (user friendly name, system name, max number of articles to load)
('The Arts','arts',25),
('Science','science',30),
('Technology','technology',30),
('Politics','politics',20),
('Media','media',30),
('Crime','crime',15),
('Other articles','',10)
]
def parse_index(self):
self.cover_url = 'http://thegretchenshow.files.wordpress.com/2009/12/well-read-cat-small.jpg'
feeds = []
seen_urls = set([])
regex = re.compile( r'http://(www\.)?([^/:]+)', re.I)
for category in self.CATEGORIES:
(cat_name, tag, max_articles) = category
tagurl = '' if tag=='' else '/tagged/'+tag
self.log('Reading category:', cat_name)
articles = []
pageno = 1
while len(articles) < max_articles and pageno < 100:
page = "%s%s/page/%d" % (self.INDEX, tagurl, pageno) if pageno > 1 else self.INDEX + tagurl
pageno += 1
self.log('\tReading page:', page)
try:
soup = self.index_to_soup(page)
except:
break
headers = soup.findAll('h2')
if len(headers) == .0:
break
for header in headers:
atag = header.find('a')
url = atag['href']
# skip promotionals and duplicate
if url.startswith('http://givemesomethingtoread') or url.startswith('/') or url in seen_urls:
continue
seen_urls.add(url)
title = self.tag_to_string(header)
self.log('\tFound article:', title)
#self.log('\t', url)
desc = header.parent.find('blockquote')
desc = self.tag_to_string(desc) if desc else ''
m = regex.match( url)
if m:
desc = "[%s] %s" % (m.group(2), desc)
#self.log('\t', desc)
date = ''
p = header.parent.previousSibling
# navigate up to find h3, which contains the date
while p:
if hasattr(p,'name') and p.name == 'h3':
date = self.tag_to_string(p)
break
p = p.previousSibling
articles.append({'title':title,'url':url,'description':desc,'date':date})
if len(articles) >= max_articles:
break
if articles:
feeds.append((cat_name, articles))
return feeds

617
recipes/go_comics.recipe
View File

@ -1,448 +1,229 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = 'Copyright 2010 Starson17'
'''
www.gocomics.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
import mechanize, re
class GoComics(BasicNewsRecipe):
title = 'GoComics'
class Comics(BasicNewsRecipe):
title = 'Comics.com'
__author__ = 'Starson17'
__version__ = '1.06'
__date__ = '07 June 2011'
description = u'200+ Comics - Customize for more days/comics: Defaults to 7 days, 25 comics - 20 general, 5 editorial.'
category = 'news, comics'
description = 'Comics from comics.com. You should customize this recipe to fetch only the comics you are interested in'
language = 'en'
use_embedded_content= False
no_stylesheets = True
oldest_article = 24
remove_javascript = True
cover_url = 'http://paulbuckley14059.files.wordpress.com/2008/06/calvin-and-hobbes.jpg'
remove_attributes = ['style']
####### USER PREFERENCES - COMICS, IMAGE SIZE AND NUMBER OF COMICS TO RETRIEVE ########
# num_comics_to_get - I've tried up to 99 on Calvin&Hobbes
cover_url = 'http://www.bsb.lib.tx.us/images/comics.com.gif'
recursions = 0
max_articles_per_feed = 10
num_comics_to_get = 7
# comic_size 300 is small, 600 is medium, 900 is large, 1500 is extra-large
comic_size = 900
# CHOOSE COMIC STRIPS BELOW - REMOVE COMMENT '# ' FROM IN FRONT OF DESIRED STRIPS
# Please do not overload their servers by selecting all comics and 1000 strips from each!
simultaneous_downloads = 1
# delay = 3
conversion_options = {'linearize_tables' : True
, 'comment' : description
, 'tags' : category
, 'language' : language
}
keep_only_tags = [dict(name='div', attrs={'class':['feature','banner']}),
keep_only_tags = [dict(name='h1'),
dict(name='p', attrs={'class':'feature_item'})
]
remove_tags = [dict(name='a', attrs={'class':['beginning','prev','cal','next','newest']}),
dict(name='div', attrs={'class':['tag-wrapper']}),
dict(name='a', attrs={'href':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
dict(name='img', attrs={'src':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
dict(name='ul', attrs={'class':['share-nav','feature-nav']}),
]
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
cookies = mechanize.CookieJar()
br = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))
br.addheaders = [('Referer','http://www.gocomics.com/')]
return br
def parse_index(self):
feeds = []
for title, url in [
(u"2 Cows and a Chicken", u"http://www.gocomics.com/2cowsandachicken"),
#(u"9 Chickweed Lane", u"http://www.gocomics.com/9chickweedlane"),
(u"9 to 5", u"http://www.gocomics.com/9to5"),
#(u"Adam At Home", u"http://www.gocomics.com/adamathome"),
(u"Agnes", u"http://www.gocomics.com/agnes"),
#(u"Alley Oop", u"http://www.gocomics.com/alleyoop"),
#(u"Andy Capp", u"http://www.gocomics.com/andycapp"),
#(u"Animal Crackers", u"http://www.gocomics.com/animalcrackers"),
#(u"Annie", u"http://www.gocomics.com/annie"),
#(u"Arlo & Janis", u"http://www.gocomics.com/arloandjanis"),
#(u"Ask Shagg", u"http://www.gocomics.com/askshagg"),
(u"B.C.", u"http://www.gocomics.com/bc"),
#(u"Back in the Day", u"http://www.gocomics.com/backintheday"),
#(u"Bad Reporter", u"http://www.gocomics.com/badreporter"),
#(u"Baldo", u"http://www.gocomics.com/baldo"),
#(u"Ballard Street", u"http://www.gocomics.com/ballardstreet"),
#(u"Barkeater Lake", u"http://www.gocomics.com/barkeaterlake"),
#(u"Basic Instructions", u"http://www.gocomics.com/basicinstructions"),
#(u"Ben", u"http://www.gocomics.com/ben"),
#(u"Betty", u"http://www.gocomics.com/betty"),
#(u"Bewley", u"http://www.gocomics.com/bewley"),
#(u"Big Nate", u"http://www.gocomics.com/bignate"),
#(u"Big Top", u"http://www.gocomics.com/bigtop"),
#(u"Biographic", u"http://www.gocomics.com/biographic"),
#(u"Birdbrains", u"http://www.gocomics.com/birdbrains"),
#(u"Bleeker: The Rechargeable Dog", u"http://www.gocomics.com/bleeker"),
#(u"Bliss", u"http://www.gocomics.com/bliss"),
(u"Bloom County", u"http://www.gocomics.com/bloomcounty"),
#(u"Bo Nanas", u"http://www.gocomics.com/bonanas"),
#(u"Bob the Squirrel", u"http://www.gocomics.com/bobthesquirrel"),
#(u"Boomerangs", u"http://www.gocomics.com/boomerangs"),
#(u"Bottomliners", u"http://www.gocomics.com/bottomliners"),
#(u"Bound and Gagged", u"http://www.gocomics.com/boundandgagged"),
#(u"Brainwaves", u"http://www.gocomics.com/brainwaves"),
#(u"Brenda Starr", u"http://www.gocomics.com/brendastarr"),
#(u"Brevity", u"http://www.gocomics.com/brevity"),
#(u"Brewster Rockit", u"http://www.gocomics.com/brewsterrockit"),
#(u"Broom Hilda", u"http://www.gocomics.com/broomhilda"),
(u"Calvin and Hobbes", u"http://www.gocomics.com/calvinandhobbes"),
#(u"Candorville", u"http://www.gocomics.com/candorville"),
#(u"Cathy", u"http://www.gocomics.com/cathy"),
#(u"C'est la Vie", u"http://www.gocomics.com/cestlavie"),
#(u"Cheap Thrills", u"http://www.gocomics.com/cheapthrills"),
#(u"Chuckle Bros", u"http://www.gocomics.com/chucklebros"),
#(u"Citizen Dog", u"http://www.gocomics.com/citizendog"),
#(u"Cleats", u"http://www.gocomics.com/cleats"),
#(u"Close to Home", u"http://www.gocomics.com/closetohome"),
#(u"Committed", u"http://www.gocomics.com/committed"),
#(u"Compu-toon", u"http://www.gocomics.com/compu-toon"),
#(u"Cornered", u"http://www.gocomics.com/cornered"),
#(u"Cow & Boy", u"http://www.gocomics.com/cow&boy"),
#(u"Cul de Sac", u"http://www.gocomics.com/culdesac"),
#(u"Daddy's Home", u"http://www.gocomics.com/daddyshome"),
#(u"Deep Cover", u"http://www.gocomics.com/deepcover"),
#(u"Dick Tracy", u"http://www.gocomics.com/dicktracy"),
(u"Dog Eat Doug", u"http://www.gocomics.com/dogeatdoug"),
#(u"Domestic Abuse", u"http://www.gocomics.com/domesticabuse"),
(u"Doodles", u"http://www.gocomics.com/doodles"),
(u"Doonesbury", u"http://www.gocomics.com/doonesbury"),
#(u"Drabble", u"http://www.gocomics.com/drabble"),
#(u"Eek!", u"http://www.gocomics.com/eek"),
#(u"F Minus", u"http://www.gocomics.com/fminus"),
#(u"Family Tree", u"http://www.gocomics.com/familytree"),
#(u"Farcus", u"http://www.gocomics.com/farcus"),
(u"Fat Cats Classics", u"http://www.gocomics.com/fatcatsclassics"),
#(u"Ferd'nand", u"http://www.gocomics.com/ferdnand"),
#(u"Flight Deck", u"http://www.gocomics.com/flightdeck"),
(u"Flo and Friends", u"http://www.gocomics.com/floandfriends"),
#(u"For Better or For Worse", u"http://www.gocomics.com/forbetterorforworse"),
#(u"For Heaven's Sake", u"http://www.gocomics.com/forheavenssake"),
#(u"Fort Knox", u"http://www.gocomics.com/fortknox"),
#(u"FoxTrot Classics", u"http://www.gocomics.com/foxtrotclassics"),
(u"FoxTrot", u"http://www.gocomics.com/foxtrot"),
#(u"Frank & Ernest", u"http://www.gocomics.com/frankandernest"),
#(u"Frazz", u"http://www.gocomics.com/frazz"),
#(u"Fred Basset", u"http://www.gocomics.com/fredbasset"),
#(u"Free Range", u"http://www.gocomics.com/freerange"),
#(u"Frog Applause", u"http://www.gocomics.com/frogapplause"),
#(u"Garfield Minus Garfield", u"http://www.gocomics.com/garfieldminusgarfield"),
(u"Garfield", u"http://www.gocomics.com/garfield"),
#(u"Gasoline Alley", u"http://www.gocomics.com/gasolinealley"),
#(u"Geech Classics", u"http://www.gocomics.com/geechclassics"),
#(u"Get Fuzzy", u"http://www.gocomics.com/getfuzzy"),
#(u"Gil Thorp", u"http://www.gocomics.com/gilthorp"),
#(u"Ginger Meggs", u"http://www.gocomics.com/gingermeggs"),
#(u"Girls & Sports", u"http://www.gocomics.com/girlsandsports"),
#(u"Graffiti", u"http://www.gocomics.com/graffiti"),
#(u"Grand Avenue", u"http://www.gocomics.com/grandavenue"),
#(u"Haiku Ewe", u"http://www.gocomics.com/haikuewe"),
#(u"Heart of the City", u"http://www.gocomics.com/heartofthecity"),
(u"Heathcliff", u"http://www.gocomics.com/heathcliff"),
#(u"Herb and Jamaal", u"http://www.gocomics.com/herbandjamaal"),
#(u"Herman", u"http://www.gocomics.com/herman"),
#(u"Home and Away", u"http://www.gocomics.com/homeandaway"),
#(u"Housebroken", u"http://www.gocomics.com/housebroken"),
#(u"Hubert and Abby", u"http://www.gocomics.com/hubertandabby"),
#(u"Imagine This", u"http://www.gocomics.com/imaginethis"),
#(u"In the Bleachers", u"http://www.gocomics.com/inthebleachers"),
#(u"In the Sticks", u"http://www.gocomics.com/inthesticks"),
#(u"Ink Pen", u"http://www.gocomics.com/inkpen"),
#(u"It's All About You", u"http://www.gocomics.com/itsallaboutyou"),
#(u"Jane's World", u"http://www.gocomics.com/janesworld"),
#(u"Joe Vanilla", u"http://www.gocomics.com/joevanilla"),
#(u"Jump Start", u"http://www.gocomics.com/jumpstart"),
#(u"Kit 'N' Carlyle", u"http://www.gocomics.com/kitandcarlyle"),
#(u"La Cucaracha", u"http://www.gocomics.com/lacucaracha"),
#(u"Last Kiss", u"http://www.gocomics.com/lastkiss"),
#(u"Legend of Bill", u"http://www.gocomics.com/legendofbill"),
#(u"Liberty Meadows", u"http://www.gocomics.com/libertymeadows"),
#(u"Li'l Abner Classics", u"http://www.gocomics.com/lilabnerclassics"),
#(u"Lio", u"http://www.gocomics.com/lio"),
#(u"Little Dog Lost", u"http://www.gocomics.com/littledoglost"),
#(u"Little Otto", u"http://www.gocomics.com/littleotto"),
#(u"Lola", u"http://www.gocomics.com/lola"),
#(u"Loose Parts", u"http://www.gocomics.com/looseparts"),
#(u"Love Is...", u"http://www.gocomics.com/loveis"),
#(u"Luann", u"http://www.gocomics.com/luann"),
#(u"Maintaining", u"http://www.gocomics.com/maintaining"),
(u"Marmaduke", u"http://www.gocomics.com/marmaduke"),
#(u"Meg! Classics", u"http://www.gocomics.com/megclassics"),
#(u"Middle-Aged White Guy", u"http://www.gocomics.com/middleagedwhiteguy"),
#(u"Minimum Security", u"http://www.gocomics.com/minimumsecurity"),
#(u"Moderately Confused", u"http://www.gocomics.com/moderatelyconfused"),
(u"Momma", u"http://www.gocomics.com/momma"),
#(u"Monty", u"http://www.gocomics.com/monty"),
#(u"Motley Classics", u"http://www.gocomics.com/motleyclassics"),
(u"Mutt & Jeff", u"http://www.gocomics.com/muttandjeff"),
#(u"Mythtickle", u"http://www.gocomics.com/mythtickle"),
#(u"Nancy", u"http://www.gocomics.com/nancy"),
#(u"Natural Selection", u"http://www.gocomics.com/naturalselection"),
#(u"Nest Heads", u"http://www.gocomics.com/nestheads"),
#(u"NEUROTICA", u"http://www.gocomics.com/neurotica"),
#(u"New Adventures of Queen Victoria", u"http://www.gocomics.com/thenewadventuresofqueenvictoria"),
#(u"Non Sequitur", u"http://www.gocomics.com/nonsequitur"),
#(u"Off The Mark", u"http://www.gocomics.com/offthemark"),
#(u"On A Claire Day", u"http://www.gocomics.com/onaclaireday"),
#(u"One Big Happy Classics", u"http://www.gocomics.com/onebighappyclassics"),
#(u"One Big Happy", u"http://www.gocomics.com/onebighappy"),
#(u"Out of the Gene Pool Re-Runs", u"http://www.gocomics.com/outofthegenepool"),
#(u"Over the Hedge", u"http://www.gocomics.com/overthehedge"),
#(u"Overboard", u"http://www.gocomics.com/overboard"),
#(u"PC and Pixel", u"http://www.gocomics.com/pcandpixel"),
(u"Peanuts", u"http://www.gocomics.com/peanuts"),
#(u"Pearls Before Swine", u"http://www.gocomics.com/pearlsbeforeswine"),
#(u"Pibgorn Sketches", u"http://www.gocomics.com/pibgornsketches"),
#(u"Pibgorn", u"http://www.gocomics.com/pibgorn"),
(u"Pickles", u"http://www.gocomics.com/pickles"),
#(u"Pinkerton", u"http://www.gocomics.com/pinkerton"),
#(u"Pluggers", u"http://www.gocomics.com/pluggers"),
#(u"Pooch Cafe", u"http://www.gocomics.com/poochcafe"),
#(u"PreTeena", u"http://www.gocomics.com/preteena"),
#(u"Prickly City", u"http://www.gocomics.com/pricklycity"),
#(u"Rabbits Against Magic", u"http://www.gocomics.com/rabbitsagainstmagic"),
#(u"Raising Duncan Classics", u"http://www.gocomics.com/raisingduncanclassics"),
#(u"Real Life Adventures", u"http://www.gocomics.com/reallifeadventures"),
#(u"Reality Check", u"http://www.gocomics.com/realitycheck"),
#(u"Red and Rover", u"http://www.gocomics.com/redandrover"),
#(u"Red Meat", u"http://www.gocomics.com/redmeat"),
#(u"Reynolds Unwrapped", u"http://www.gocomics.com/reynoldsunwrapped"),
#(u"Rip Haywire", u"http://www.gocomics.com/riphaywire"),
#(u"Ripley's Believe It or Not!", u"http://www.gocomics.com/ripleysbelieveitornot"),
#(u"Ronaldinho Gaucho", u"http://www.gocomics.com/ronaldinhogaucho"),
#(u"Rose Is Rose", u"http://www.gocomics.com/roseisrose"),
#(u"Rubes", u"http://www.gocomics.com/rubes"),
#(u"Rudy Park", u"http://www.gocomics.com/rudypark"),
#(u"Scary Gary", u"http://www.gocomics.com/scarygary"),
#(u"Shirley and Son Classics", u"http://www.gocomics.com/shirleyandsonclassics"),
#(u"Shoe", u"http://www.gocomics.com/shoe"),
#(u"Shoecabbage", u"http://www.gocomics.com/shoecabbage"),
#(u"Skin Horse", u"http://www.gocomics.com/skinhorse"),
#(u"Slowpoke", u"http://www.gocomics.com/slowpoke"),
#(u"Soup To Nutz", u"http://www.gocomics.com/souptonutz"),
#(u"Speed Bump", u"http://www.gocomics.com/speedbump"),
#(u"Spot The Frog", u"http://www.gocomics.com/spotthefrog"),
#(u"State of the Union", u"http://www.gocomics.com/stateoftheunion"),
#(u"Stone Soup", u"http://www.gocomics.com/stonesoup"),
#(u"Strange Brew", u"http://www.gocomics.com/strangebrew"),
#(u"Sylvia", u"http://www.gocomics.com/sylvia"),
#(u"Tank McNamara", u"http://www.gocomics.com/tankmcnamara"),
#(u"Tarzan Classics", u"http://www.gocomics.com/tarzanclassics"),
#(u"That's Life", u"http://www.gocomics.com/thatslife"),
#(u"The Academia Waltz", u"http://www.gocomics.com/academiawaltz"),
#(u"The Argyle Sweater", u"http://www.gocomics.com/theargylesweater"),
#(u"The Barn", u"http://www.gocomics.com/thebarn"),
#(u"The Boiling Point", u"http://www.gocomics.com/theboilingpoint"),
#(u"The Boondocks", u"http://www.gocomics.com/boondocks"),
#(u"The Born Loser", u"http://www.gocomics.com/thebornloser"),
#(u"The Buckets", u"http://www.gocomics.com/thebuckets"),
#(u"The City", u"http://www.gocomics.com/thecity"),
#(u"The Dinette Set", u"http://www.gocomics.com/dinetteset"),
#(u"The Doozies", u"http://www.gocomics.com/thedoozies"),
#(u"The Duplex", u"http://www.gocomics.com/duplex"),
#(u"The Elderberries", u"http://www.gocomics.com/theelderberries"),
#(u"The Flying McCoys", u"http://www.gocomics.com/theflyingmccoys"),
#(u"The Fusco Brothers", u"http://www.gocomics.com/thefuscobrothers"),
#(u"The Grizzwells", u"http://www.gocomics.com/thegrizzwells"),
#(u"The Humble Stumble", u"http://www.gocomics.com/thehumblestumble"),
#(u"The Knight Life", u"http://www.gocomics.com/theknightlife"),
#(u"The Meaning of Lila", u"http://www.gocomics.com/meaningoflila"),
#(u"The Middletons", u"http://www.gocomics.com/themiddletons"),
#(u"The Norm", u"http://www.gocomics.com/thenorm"),
#(u"The Other Coast", u"http://www.gocomics.com/theothercoast"),
#(u"The Quigmans", u"http://www.gocomics.com/thequigmans"),
#(u"The Sunshine Club", u"http://www.gocomics.com/thesunshineclub"),
#(u"Tiny Sepuk", u"http://www.gocomics.com/tinysepuk"),
#(u"TOBY", u"http://www.gocomics.com/toby"),
#(u"Tom the Dancing Bug", u"http://www.gocomics.com/tomthedancingbug"),
#(u"Too Much Coffee Man", u"http://www.gocomics.com/toomuchcoffeeman"),
#(u"Unstrange Phenomena", u"http://www.gocomics.com/unstrangephenomena"),
#(u"W.T. Duck", u"http://www.gocomics.com/wtduck"),
#(u"Watch Your Head", u"http://www.gocomics.com/watchyourhead"),
#(u"Wee Pals", u"http://www.gocomics.com/weepals"),
#(u"Winnie the Pooh", u"http://www.gocomics.com/winniethepooh"),
#(u"Wizard of Id", u"http://www.gocomics.com/wizardofid"),
#(u"Working Daze", u"http://www.gocomics.com/workingdaze"),
#(u"Working It Out", u"http://www.gocomics.com/workingitout"),
#(u"Yenny", u"http://www.gocomics.com/yenny"),
#(u"Zack Hill", u"http://www.gocomics.com/zackhill"),
(u"Ziggy", u"http://www.gocomics.com/ziggy"),
#
######## EDITORIAL CARTOONS #####################
(u"Adam Zyglis", u"http://www.gocomics.com/adamzyglis"),
#(u"Andy Singer", u"http://www.gocomics.com/andysinger"),
#(u"Ben Sargent",u"http://www.gocomics.com/bensargent"),
#(u"Bill Day", u"http://www.gocomics.com/billday"),
#(u"Bill Schorr", u"http://www.gocomics.com/billschorr"),
#(u"Bob Englehart", u"http://www.gocomics.com/bobenglehart"),
(u"Bob Gorrell",u"http://www.gocomics.com/bobgorrell"),
#(u"Brian Fairrington", u"http://www.gocomics.com/brianfairrington"),
#(u"Bruce Beattie", u"http://www.gocomics.com/brucebeattie"),
#(u"Cam Cardow", u"http://www.gocomics.com/camcardow"),
#(u"Chan Lowe",u"http://www.gocomics.com/chanlowe"),
#(u"Chip Bok",u"http://www.gocomics.com/chipbok"),
#(u"Chris Britt",u"http://www.gocomics.com/chrisbritt"),
#(u"Chuck Asay",u"http://www.gocomics.com/chuckasay"),
#(u"Clay Bennett",u"http://www.gocomics.com/claybennett"),
#(u"Clay Jones",u"http://www.gocomics.com/clayjones"),
#(u"Dan Wasserman",u"http://www.gocomics.com/danwasserman"),
#(u"Dana Summers",u"http://www.gocomics.com/danasummers"),
#(u"Daryl Cagle", u"http://www.gocomics.com/darylcagle"),
#(u"David Fitzsimmons", u"http://www.gocomics.com/davidfitzsimmons"),
(u"Dick Locher",u"http://www.gocomics.com/dicklocher"),
#(u"Don Wright",u"http://www.gocomics.com/donwright"),
#(u"Donna Barstow",u"http://www.gocomics.com/donnabarstow"),
#(u"Drew Litton", u"http://www.gocomics.com/drewlitton"),
#(u"Drew Sheneman",u"http://www.gocomics.com/drewsheneman"),
#(u"Ed Stein", u"http://www.gocomics.com/edstein"),
#(u"Eric Allie", u"http://www.gocomics.com/ericallie"),
#(u"Gary Markstein", u"http://www.gocomics.com/garymarkstein"),
#(u"Gary McCoy", u"http://www.gocomics.com/garymccoy"),
#(u"Gary Varvel", u"http://www.gocomics.com/garyvarvel"),
#(u"Glenn McCoy",u"http://www.gocomics.com/glennmccoy"),
#(u"Henry Payne", u"http://www.gocomics.com/henrypayne"),
#(u"Jack Ohman",u"http://www.gocomics.com/jackohman"),
#(u"JD Crowe", u"http://www.gocomics.com/jdcrowe"),
#(u"Jeff Danziger",u"http://www.gocomics.com/jeffdanziger"),
#(u"Jeff Parker", u"http://www.gocomics.com/jeffparker"),
#(u"Jeff Stahler", u"http://www.gocomics.com/jeffstahler"),
#(u"Jerry Holbert", u"http://www.gocomics.com/jerryholbert"),
#(u"Jim Morin",u"http://www.gocomics.com/jimmorin"),
#(u"Joel Pett",u"http://www.gocomics.com/joelpett"),
#(u"John Cole", u"http://www.gocomics.com/johncole"),
#(u"John Darkow", u"http://www.gocomics.com/johndarkow"),
#(u"John Deering",u"http://www.gocomics.com/johndeering"),
#(u"John Sherffius", u"http://www.gocomics.com/johnsherffius"),
#(u"Ken Catalino",u"http://www.gocomics.com/kencatalino"),
#(u"Kerry Waghorn",u"http://www.gocomics.com/facesinthenews"),
#(u"Kevin Kallaugher",u"http://www.gocomics.com/kevinkallaugher"),
#(u"Lalo Alcaraz",u"http://www.gocomics.com/laloalcaraz"),
#(u"Larry Wright", u"http://www.gocomics.com/larrywright"),
#(u"Lisa Benson", u"http://www.gocomics.com/lisabenson"),
#(u"Marshall Ramsey", u"http://www.gocomics.com/marshallramsey"),
#(u"Matt Bors", u"http://www.gocomics.com/mattbors"),
#(u"Matt Davies",u"http://www.gocomics.com/mattdavies"),
#(u"Michael Ramirez", u"http://www.gocomics.com/michaelramirez"),
#(u"Mike Keefe", u"http://www.gocomics.com/mikekeefe"),
#(u"Mike Luckovich", u"http://www.gocomics.com/mikeluckovich"),
#(u"MIke Thompson", u"http://www.gocomics.com/mikethompson"),
#(u"Monte Wolverton", u"http://www.gocomics.com/montewolverton"),
#(u"Mr. Fish", u"http://www.gocomics.com/mrfish"),
#(u"Nate Beeler", u"http://www.gocomics.com/natebeeler"),
#(u"Nick Anderson", u"http://www.gocomics.com/nickanderson"),
#(u"Pat Bagley", u"http://www.gocomics.com/patbagley"),
#(u"Pat Oliphant",u"http://www.gocomics.com/patoliphant"),
#(u"Paul Conrad",u"http://www.gocomics.com/paulconrad"),
#(u"Paul Szep", u"http://www.gocomics.com/paulszep"),
#(u"RJ Matson", u"http://www.gocomics.com/rjmatson"),
#(u"Rob Rogers", u"http://www.gocomics.com/robrogers"),
#(u"Robert Ariail", u"http://www.gocomics.com/robertariail"),
#(u"Scott Stantis", u"http://www.gocomics.com/scottstantis"),
#(u"Signe Wilkinson", u"http://www.gocomics.com/signewilkinson"),
#(u"Small World",u"http://www.gocomics.com/smallworld"),
#(u"Steve Benson", u"http://www.gocomics.com/stevebenson"),
#(u"Steve Breen", u"http://www.gocomics.com/stevebreen"),
#(u"Steve Kelley", u"http://www.gocomics.com/stevekelley"),
#(u"Steve Sack", u"http://www.gocomics.com/stevesack"),
#(u"Stuart Carlson",u"http://www.gocomics.com/stuartcarlson"),
#(u"Ted Rall",u"http://www.gocomics.com/tedrall"),
#(u"(Th)ink", u"http://www.gocomics.com/think"),
#(u"Tom Toles",u"http://www.gocomics.com/tomtoles"),
(u"Tony Auth",u"http://www.gocomics.com/tonyauth"),
#(u"Views of the World",u"http://www.gocomics.com/viewsoftheworld"),
#(u"ViewsAfrica",u"http://www.gocomics.com/viewsafrica"),
#(u"ViewsAmerica",u"http://www.gocomics.com/viewsamerica"),
#(u"ViewsAsia",u"http://www.gocomics.com/viewsasia"),
#(u"ViewsBusiness",u"http://www.gocomics.com/viewsbusiness"),
#(u"ViewsEurope",u"http://www.gocomics.com/viewseurope"),
#(u"ViewsLatinAmerica",u"http://www.gocomics.com/viewslatinamerica"),
#(u"ViewsMidEast",u"http://www.gocomics.com/viewsmideast"),
(u"Walt Handelsman",u"http://www.gocomics.com/walthandelsman"),
#(u"Wayne Stayskal",u"http://www.gocomics.com/waynestayskal"),
#(u"Wit of the World",u"http://www.gocomics.com/witoftheworld"),
]:
print 'Working on: ', title
("9 Chickweed Lane", "http://gocomics.com/9_chickweed_lane"),
("Agnes", "http://gocomics.com/agnes"),
("Alley Oop", "http://gocomics.com/alley_oop"),
("Andy Capp", "http://gocomics.com/andy_capp"),
("Arlo & Janis", "http://gocomics.com/arlo&janis"),
("B.C.", "http://gocomics.com/bc"),
("Ballard Street", "http://gocomics.com/ballard_street"),
# ("Ben", "http://comics.com/ben"),
# ("Betty", "http://comics.com/betty"),
# ("Big Nate", "http://comics.com/big_nate"),
# ("Brevity", "http://comics.com/brevity"),
# ("Candorville", "http://comics.com/candorville"),
# ("Cheap Thrills", "http://comics.com/cheap_thrills"),
# ("Committed", "http://comics.com/committed"),
# ("Cow & Boy", "http://comics.com/cow&boy"),
# ("Daddy's Home", "http://comics.com/daddys_home"),
# ("Dog eat Doug", "http://comics.com/dog_eat_doug"),
# ("Drabble", "http://comics.com/drabble"),
# ("F Minus", "http://comics.com/f_minus"),
# ("Family Tree", "http://comics.com/family_tree"),
# ("Farcus", "http://comics.com/farcus"),
# ("Fat Cats Classics", "http://comics.com/fat_cats_classics"),
# ("Ferd'nand", "http://comics.com/ferdnand"),
# ("Flight Deck", "http://comics.com/flight_deck"),
# ("Flo & Friends", "http://comics.com/flo&friends"),
# ("Fort Knox", "http://comics.com/fort_knox"),
# ("Frank & Ernest", "http://comics.com/frank&ernest"),
# ("Frazz", "http://comics.com/frazz"),
# ("Free Range", "http://comics.com/free_range"),
# ("Geech Classics", "http://comics.com/geech_classics"),
# ("Get Fuzzy", "http://comics.com/get_fuzzy"),
# ("Girls & Sports", "http://comics.com/girls&sports"),
# ("Graffiti", "http://comics.com/graffiti"),
# ("Grand Avenue", "http://comics.com/grand_avenue"),
# ("Heathcliff", "http://comics.com/heathcliff"),
# "Heathcliff, a street-smart and mischievous cat with many adventures."
# ("Herb and Jamaal", "http://comics.com/herb_and_jamaal"),
# ("Herman", "http://comics.com/herman"),
# ("Home and Away", "http://comics.com/home_and_away"),
# ("It's All About You", "http://comics.com/its_all_about_you"),
# ("Jane's World", "http://comics.com/janes_world"),
# ("Jump Start", "http://comics.com/jump_start"),
# ("Kit 'N' Carlyle", "http://comics.com/kit_n_carlyle"),
# ("Li'l Abner Classics", "http://comics.com/lil_abner_classics"),
# ("Liberty Meadows", "http://comics.com/liberty_meadows"),
# ("Little Dog Lost", "http://comics.com/little_dog_lost"),
# ("Lola", "http://comics.com/lola"),
# ("Luann", "http://comics.com/luann"),
# ("Marmaduke", "http://comics.com/marmaduke"),
# ("Meg! Classics", "http://comics.com/meg_classics"),
# ("Minimum Security", "http://comics.com/minimum_security"),
# ("Moderately Confused", "http://comics.com/moderately_confused"),
# ("Momma", "http://comics.com/momma"),
# ("Monty", "http://comics.com/monty"),
# ("Motley Classics", "http://comics.com/motley_classics"),
# ("Nancy", "http://comics.com/nancy"),
# ("Natural Selection", "http://comics.com/natural_selection"),
# ("Nest Heads", "http://comics.com/nest_heads"),
# ("Off The Mark", "http://comics.com/off_the_mark"),
# ("On a Claire Day", "http://comics.com/on_a_claire_day"),
# ("One Big Happy Classics", "http://comics.com/one_big_happy_classics"),
# ("Over the Hedge", "http://comics.com/over_the_hedge"),
# ("PC and Pixel", "http://comics.com/pc_and_pixel"),
# ("Peanuts", "http://comics.com/peanuts"),
# ("Pearls Before Swine", "http://comics.com/pearls_before_swine"),
# ("Pickles", "http://comics.com/pickles"),
# ("Prickly City", "http://comics.com/prickly_city"),
# ("Raising Duncan Classics", "http://comics.com/raising_duncan_classics"),
# ("Reality Check", "http://comics.com/reality_check"),
# ("Red & Rover", "http://comics.com/red&rover"),
# ("Rip Haywire", "http://comics.com/rip_haywire"),
# ("Ripley's Believe It or Not!", "http://comics.com/ripleys_believe_it_or_not"),
# ("Rose Is Rose", "http://comics.com/rose_is_rose"),
# ("Rubes", "http://comics.com/rubes"),
# ("Rudy Park", "http://comics.com/rudy_park"),
# ("Scary Gary", "http://comics.com/scary_gary"),
# ("Shirley and Son Classics", "http://comics.com/shirley_and_son_classics"),
# ("Soup To Nutz", "http://comics.com/soup_to_nutz"),
# ("Speed Bump", "http://comics.com/speed_bump"),
# ("Spot The Frog", "http://comics.com/spot_the_frog"),
# ("State of the Union", "http://comics.com/state_of_the_union"),
# ("Strange Brew", "http://comics.com/strange_brew"),
# ("Tarzan Classics", "http://comics.com/tarzan_classics"),
# ("That's Life", "http://comics.com/thats_life"),
# ("The Barn", "http://comics.com/the_barn"),
# ("The Born Loser", "http://comics.com/the_born_loser"),
# ("The Buckets", "http://comics.com/the_buckets"),
# ("The Dinette Set", "http://comics.com/the_dinette_set"),
# ("The Grizzwells", "http://comics.com/the_grizzwells"),
# ("The Humble Stumble", "http://comics.com/the_humble_stumble"),
# ("The Knight Life", "http://comics.com/the_knight_life"),
# ("The Meaning of Lila", "http://comics.com/the_meaning_of_lila"),
# ("The Other Coast", "http://comics.com/the_other_coast"),
# ("The Sunshine Club", "http://comics.com/the_sunshine_club"),
# ("Unstrange Phenomena", "http://comics.com/unstrange_phenomena"),
# ("Watch Your Head", "http://comics.com/watch_your_head"),
# ("Wizard of Id", "http://comics.com/wizard_of_id"),
# ("Working Daze", "http://comics.com/working_daze"),
# ("Working It Out", "http://comics.com/working_it_out"),
# ("Zack Hill", "http://comics.com/zack_hill"),
# ("(Th)ink", "http://comics.com/think"),
# "Tackling the political and social issues impacting communities of color."
# ("Adam Zyglis", "http://comics.com/adam_zyglis"),
# "Known for his excellent caricatures, as well as independent and incisive imagery. "
# ("Andy Singer", "http://comics.com/andy_singer"),
# ("Bill Day", "http://comics.com/bill_day"),
# "Powerful images on sensitive issues."
# ("Bill Schorr", "http://comics.com/bill_schorr"),
# ("Bob Englehart", "http://comics.com/bob_englehart"),
# ("Brian Fairrington", "http://comics.com/brian_fairrington"),
# ("Bruce Beattie", "http://comics.com/bruce_beattie"),
# ("Cam Cardow", "http://comics.com/cam_cardow"),
# ("Chip Bok", "http://comics.com/chip_bok"),
# ("Chris Britt", "http://comics.com/chris_britt"),
# ("Chuck Asay", "http://comics.com/chuck_asay"),
# ("Clay Bennett", "http://comics.com/clay_bennett"),
# ("Daryl Cagle", "http://comics.com/daryl_cagle"),
# ("David Fitzsimmons", "http://comics.com/david_fitzsimmons"),
# "David Fitzsimmons is a new editorial cartoons on comics.com. He is also a staff writer and editorial cartoonist for the Arizona Daily Star. "
# ("Drew Litton", "http://comics.com/drew_litton"),
# "Drew Litton is an artist who is probably best known for his sports cartoons. He received the National Cartoonist Society Sports Cartoon Award for 1993. "
# ("Ed Stein", "http://comics.com/ed_stein"),
# "Winner of the Fischetti Award in 2006 and the Scripps Howard National Journalism Award, 1999, Ed Stein has been the editorial cartoonist for the Rocky Mountain News since 1978. "
# ("Eric Allie", "http://comics.com/eric_allie"),
# "Eric Allie is an editorial cartoonist with the Pioneer Press and CNS News. "
# ("Gary Markstein", "http://comics.com/gary_markstein"),
# ("Gary McCoy", "http://comics.com/gary_mccoy"),
# "Gary McCoy is known for his editorial cartoons, humor and inane ramblings. He is a 2 time nominee for Best Magazine Cartoonist of the Year by the National Cartoonists Society. He resides in Belleville, IL. "
# ("Gary Varvel", "http://comics.com/gary_varvel"),
# ("Henry Payne", "http://comics.com/henry_payne"),
# ("JD Crowe", "http://comics.com/jd_crowe"),
# ("Jeff Parker", "http://comics.com/jeff_parker"),
# ("Jeff Stahler", "http://comics.com/jeff_stahler"),
# ("Jerry Holbert", "http://comics.com/jerry_holbert"),
# ("John Cole", "http://comics.com/john_cole"),
# ("John Darkow", "http://comics.com/john_darkow"),
# "John Darkow is a contributing editorial cartoonist for the Humor Times as well as editoiral cartoonist for the Columbia Daily Tribune, Missouri"
# ("John Sherffius", "http://comics.com/john_sherffius"),
# ("Larry Wright", "http://comics.com/larry_wright"),
# ("Lisa Benson", "http://comics.com/lisa_benson"),
# ("Marshall Ramsey", "http://comics.com/marshall_ramsey"),
# ("Matt Bors", "http://comics.com/matt_bors"),
# ("Michael Ramirez", "http://comics.com/michael_ramirez"),
# ("Mike Keefe", "http://comics.com/mike_keefe"),
# ("Mike Luckovich", "http://comics.com/mike_luckovich"),
# ("MIke Thompson", "http://comics.com/mike_thompson"),
# ("Monte Wolverton", "http://comics.com/monte_wolverton"),
# "Unique mix of perspectives"
# ("Mr. Fish", "http://comics.com/mr_fish"),
# "Side effects may include swelling"
# ("Nate Beeler", "http://comics.com/nate_beeler"),
# "Middle America meets the Beltway."
# ("Nick Anderson", "http://comics.com/nick_anderson"),
# ("Pat Bagley", "http://comics.com/pat_bagley"),
# "Unfair and Totally Unbalanced."
# ("Paul Szep", "http://comics.com/paul_szep"),
# ("RJ Matson", "http://comics.com/rj_matson"),
# "Power cartoons from NYC and Capitol Hill"
# ("Rob Rogers", "http://comics.com/rob_rogers"),
# "Humorous slant on current events"
# ("Robert Ariail", "http://comics.com/robert_ariail"),
# "Clever and unpredictable"
# ("Scott Stantis", "http://comics.com/scott_stantis"),
# ("Signe Wilkinson", "http://comics.com/signe_wilkinson"),
# ("Steve Benson", "http://comics.com/steve_benson"),
# ("Steve Breen", "http://comics.com/steve_breen"),
# ("Steve Kelley", "http://comics.com/steve_kelley"),
# ("Steve Sack", "http://comics.com/steve_sack"),
]:
articles = self.make_links(url)
if articles:
feeds.append((title, articles))
return feeds
def make_links(self, url):
title = 'Temp'
soup = self.index_to_soup(url)
# print 'soup: ', soup
title = ''
current_articles = []
pages = range(1, self.num_comics_to_get+1)
for page in pages:
page_soup = self.index_to_soup(url)
if page_soup:
try:
strip_title = page_soup.find(name='div', attrs={'class':'top'}).h1.a.string
except:
strip_title = 'Error - no Title found'
try:
date_title = page_soup.find('ul', attrs={'class': 'feature-nav'}).li.string
if not date_title:
date_title = page_soup.find('ul', attrs={'class': 'feature-nav'}).li.string
except:
date_title = 'Error - no Date found'
title = strip_title + ' - ' + date_title
for i in range(2):
try:
strip_url_date = page_soup.find(name='div', attrs={'class':'top'}).h1.a['href']
break #success - this is normal exit
except:
strip_url_date = None
continue #try to get strip_url_date again
for i in range(2):
try:
prev_strip_url_date = page_soup.find('a', attrs={'class': 'prev'})['href']
break #success - this is normal exit
except:
prev_strip_url_date = None
continue #try to get prev_strip_url_date again
if strip_url_date:
page_url = 'http://www.gocomics.com' + strip_url_date
else:
continue
if prev_strip_url_date:
prev_page_url = 'http://www.gocomics.com' + prev_strip_url_date
else:
continue
from datetime import datetime, timedelta
now = datetime.now()
dates = [(now-timedelta(days=d)).strftime('%Y/%m/%d') for d in range(self.num_comics_to_get)]
for page in dates:
page_url = url + '/' + str(page)
print(page_url)
soup = self.index_to_soup(page_url)
if soup:
strip_tag = self.tag_to_string(soup.find('a'))
if strip_tag:
print 'strip_tag: ', strip_tag
title = strip_tag
print 'title: ', title
current_articles.append({'title': title, 'url': page_url, 'description':'', 'date':''})
url = prev_page_url
current_articles.reverse()
return current_articles
def preprocess_html(self, soup):
if soup.title:
title_string = soup.title.string.strip()
_cd = title_string.split(',',1)[1]
comic_date = ' '.join(_cd.split(' ', 4)[0:-1])
if soup.h1.span:
artist = soup.h1.span.string
soup.h1.span.string.replaceWith(comic_date + artist)
feature_item = soup.find('p',attrs={'class':'feature_item'})
if feature_item.a:
a_tag = feature_item.a
a_href = a_tag["href"]
img_tag = a_tag.img
img_tag["src"] = a_href
img_tag["width"] = self.comic_size
img_tag["height"] = None
return self.adeify_images(soup)
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
img {max-width:100%; min-width:100%;}
p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
'''
'''

3
recipes/independent.recipe
View File

@ -41,6 +41,7 @@ class TheIndependentNew(BasicNewsRecipe):
publication_type = 'newspaper'
masthead_url = 'http://www.independent.co.uk/independent.co.uk/editorial/logo/independent_Masthead.png'
encoding = 'utf-8'
compress_news_images = True
remove_tags =[
dict(attrs={'id' : ['RelatedArtTag','renderBiography']}),
dict(attrs={'class' : ['autoplay','openBiogPopup']}),
@ -343,7 +344,7 @@ class TheIndependentNew(BasicNewsRecipe):
if 'class' in subtitle_div:
clazz = subtitle_div['class'] + ' '
clazz = clazz + 'subtitle'
subtitle_div['class'] = clazz
subtitle_div['class'] = clazz
#find broken images and remove captions
items_to_extract = []

15
recipes/io9.recipe
View File

@ -16,14 +16,15 @@ class i09(BasicNewsRecipe):
max_articles_per_feed = 100
no_stylesheets = True
encoding = 'utf-8'
use_embedded_content = True
use_embedded_content = False
auto_cleanup = True
language = 'en'
masthead_url = 'http://cache.gawkerassets.com/assets/io9.com/img/logo.png'
extra_css = '''
body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif}
img{margin-bottom: 1em}
h1{font-family :Arial,Helvetica,sans-serif; font-size:large}
'''
body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif}
img{margin-bottom: 1em}
h1{font-family :Arial,Helvetica,sans-serif; font-size:large}
'''
conversion_options = {
'comment' : description
, 'tags' : category
@ -33,10 +34,6 @@ class i09(BasicNewsRecipe):
feeds = [(u'Articles', u'http://feeds.gawker.com/io9/vip?format=xml')]
remove_tags = [
{'class': 'feedflare'},
]
def preprocess_html(self, soup):
return self.adeify_images(soup)

11
recipes/lightspeed_magazine.recipe Normal file
View File

@ -0,0 +1,11 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1366025923(BasicNewsRecipe):
title = u'Lightspeed Magazine'
language = 'en'
__author__ = 'Jose Pinto'
oldest_article = 31
max_articles_per_feed = 100
auto_cleanup = True
use_embedded_content = False
feeds = [(u'Lastest Stories', u'http://www.lightspeedmagazine.com/rss-2/')]

17
recipes/metro_news_nl.recipe
View File

@ -36,6 +36,9 @@ from BeautifulSoup import BeautifulSoup
Changed order of regex to speedup proces
Version 1.9.3 23-05-2012
Updated Cover image
Version 1.9.4 19-04-2013
Added regex filter for mailto
Updated for new layout of metro-site
'''
class AdvancedUserRecipe1306097511(BasicNewsRecipe):
@ -43,7 +46,7 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
oldest_article = 1.2
max_articles_per_feed = 25
__author__ = u'DrMerry'
description = u'Metro Nederland'
description = u'Metro Nederland v1.9.4 2013-04-19'
language = u'nl'
simultaneous_downloads = 5
masthead_url = 'http://blog.metronieuws.nl/wp-content/themes/metro/images/header.gif'
@ -68,13 +71,17 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
#(re.compile('(</?)h2', re.DOTALL|re.IGNORECASE),lambda match:'\1em')
]
remove_tags_before= dict(id='date')
remove_tags_after = [dict(name='div', attrs={'class':['column-1-3','gallery-text']})]#id='share-and-byline')]
remove_tags_before= dict(id='subwrapper')
remove_tags_after = dict(name='div', attrs={'class':['body-area','article-main-area']})
#name='div', attrs={'class':['subwrapper']})]
#'column-1-3','gallery-text']})]#id='share-and-byline')]
filter_regexps = [r'mailto:.*']
remove_tags = [
dict(name=['iframe','script','noscript','style']),
dict(name='div', attrs={'class':['column-4-5','column-1-5','ad-msg','col-179 ','col-373 ','clear','ad','navigation',re.compile('share-tools(-top)?'),'tools','metroCommentFormWrap','article-tools-below-title','related-links','padding-top-15',re.compile('^promo.*?$'),'teaser-component',re.compile('fb(-comments|_iframe_widget)'),'promos','header-links','promo-2']}),
dict(id=['column-1-5-bottom','column-4-5',re.compile('^ad(\d+|adcomp.*?)?$'),'adadcomp-4','margin-5','sidebar',re.compile('^article-\d'),'comments','gallery-1']),
dict(name='div', attrs={'class':['aside clearfix','aside clearfix middle-col-line','comments','share-tools','article-right-column','column-4-5','column-1-5','ad-msg','col-179 ','col-373 ','clear','ad','navigation',re.compile('share-tools(-top)?'),'tools','metroCommentFormWrap','article-tools-below-title','related-links','padding-top-15',re.compile('^promo.*?$'),'teaser-component',re.compile('fb(-comments|_iframe_widget)'),'promos','header-links','promo-2']}),
dict(id=['article-2','googleads','column-1-5-bottom','column-4-5',re.compile('^ad(\d+|adcomp.*?)?$'),'adadcomp-4','margin-5','sidebar',re.compile('^article-\d'),'comments','gallery-1','sharez_container','ts-container','topshares','ts-title']),
dict(name='a', attrs={'name':'comments'}),
#dict(name='div', attrs={'data-href'}),
dict(name='img', attrs={'class':'top-line','title':'volledig scherm'}),

29
recipes/nzz_webpaper.recipe
View File

@ -42,7 +42,6 @@ class Nzz(BasicNewsRecipe):
soup = self.index_to_soup(baseref)
articles = {}
key = None
ans = []
issuelist = soup.find(id="issueSelectorList")
@ -52,27 +51,25 @@ class Nzz(BasicNewsRecipe):
section = f.string
sectionref = baseref + f['href']
# print "section is "+section +" and ref is "+sectionref
ans.append(section)
articlesoup = self.index_to_soup(sectionref)
articlesoup = articlesoup.findAll('article','article')
for a in articlesoup:
artlink = a.find('a')
artlink = a.find('a')
arthref = baseref + artlink['href']
arthead = a.find('h2')
artcaption = arthead.string
arthref = baseref + artlink['href']
arthead = a.find('h2')
artcaption = arthead.string
pubdate = strftime('%a, %d %b')
pubdate = strftime('%a, %d %b')
if not artcaption is None:
# print " found article named "+artcaption+" at "+arthref
if not articles.has_key(section):
articles[section] = []
articles[section].append(
dict(title=artcaption, url=arthref, date=pubdate, description='', content=''))
if not artcaption is None:
if not articles.has_key(section):
articles[section] = []
articles[section].append(
dict(title=artcaption, url=arthref, date=pubdate, description='', content=''))
ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
return ans
@ -80,10 +77,10 @@ class Nzz(BasicNewsRecipe):
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
if self.username is not None and self.password is not None:
br.open('https://webpaper.nzz.ch/login')
br.open('https://cas.nzz.ch/cas/login')
br.select_form(nr=0)
br['_username'] = self.username
br['_password'] = self.password
br['username'] = self.username
br['password'] = self.password
br.submit()
return br

21
recipes/orlando_sentinel.recipe
View File

@ -7,27 +7,26 @@ class AdvancedUserRecipe1279258912(BasicNewsRecipe):
max_articles_per_feed = 100
feeds = [
(u'News', u'http://feeds.feedburner.com/orlandosentinel/news'),
(u'Opinion', u'http://feeds.feedburner.com/orlandosentinel/news/opinion'),
(u'Business', u'http://feeds.feedburner.com/orlandosentinel/business'),
(u'Technology', u'http://feeds.feedburner.com/orlandosentinel/technology'),
(u'Space and Science', u'http://feeds.feedburner.com/orlandosentinel/news/space'),
(u'Entertainment', u'http://feeds.feedburner.com/orlandosentinel/entertainment'),
(u'Life and Family', u'http://feeds.feedburner.com/orlandosentinel/features/lifestyle'),
]
(u'News', u'http://feeds.feedburner.com/orlandosentinel/news'),
(u'Opinion', u'http://feeds.feedburner.com/orlandosentinel/news/opinion'),
(u'Business', u'http://feeds.feedburner.com/orlandosentinel/business'),
(u'Technology', u'http://feeds.feedburner.com/orlandosentinel/technology'),
(u'Space and Science', u'http://feeds.feedburner.com/orlandosentinel/news/space'),
(u'Entertainment', u'http://feeds.feedburner.com/orlandosentinel/entertainment'),
(u'Life and Family', u'http://feeds.feedburner.com/orlandosentinel/features/lifestyle'),
]
__author__ = 'rty'
pubisher = 'OrlandoSentinel.com'
description = 'Orlando, Florida, Newspaper'
category = 'News, Orlando, Florida'
remove_javascript = True
use_embedded_content = False
no_stylesheets = True
language = 'en'
encoding = 'utf-8'
conversion_options = {'linearize_tables':True}
masthead_url = 'http://www.orlandosentinel.com/media/graphic/2009-07/46844851.gif'
remove_empty_feeds = True
auto_cleanup = True
@ -45,7 +44,7 @@ class AdvancedUserRecipe1279258912(BasicNewsRecipe):
link=link.split('/')[-2]
encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&',
'0D': '?', '0E': '-', '0N': '.com', '0L': 'http:',
'0S':'//'}
'0S':'//', '0H':','}
for k, v in encoding.iteritems():
link = link.replace(k, v)
ans = link

55
recipes/psych.recipe
View File

@ -11,7 +11,8 @@ class PsychologyToday(BasicNewsRecipe):
language = 'en'
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [dict(attrs={'class':['print-title', 'print-submitted', 'print-content', 'print-footer', 'print-source_url', 'print-links']})]
auto_cleanup = True
#keep_only_tags = [dict(attrs={'class':['print-title', 'print-submitted', 'print-content', 'print-footer', 'print-source_url', 'print-links']})]
no_javascript = True
no_stylesheets = True
@ -31,50 +32,32 @@ class PsychologyToday(BasicNewsRecipe):
self.timefmt = u' [%s]'%date
articles = []
for post in div.findAll('div', attrs={'class':'collections-node-feature-info'}):
for post in div.findAll('div', attrs={'class':'collections-node-feature collection-node-even'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
url= 'http://www.psychologytoday.com'+post.find('a', href=True)['href']
#print_page=article_page.find('li', attrs={'class':'print_html first'})
#url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('div', attrs={'class':'collections-node-feature collection-node-odd'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
url= 'http://www.psychologytoday.com'+post.find('a', href=True)['href']
#print_page=article_page.find('li', attrs={'class':'print_html first'})
#url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('div', attrs={'class':'collections-node-thumbnail-info'}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
description = post.find('div', attrs={'class':'collection-node-description'})
author = re.sub(r'.*by\s',"",self.tag_to_string(description.nextSibling).strip())
desc = self.tag_to_string(description).strip()
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
title = title + u' (%s)'%author
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
for post in div.findAll('li', attrs={'class':['collection-item-list-odd','collection-item-list-even']}):
title = self.tag_to_string(post.find('h2'))
author_item=post.find('div', attrs={'class':'collection-node-byline'})
author = re.sub(r'.*by\s',"",self.tag_to_string(author_item).strip())
title = title + u' (%s)'%author
article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
print_page=article_page.find('li', attrs={'class':'print_html first'})
if print_page is not None:
url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
self.log('Found article:', title)
self.log('\t', url)
self.log('\t', desc)
articles.append({'title':title, 'url':url, 'date':'','description':desc})
return [('Current Issue', articles)]

222
recipes/readitlater.recipe
View File

@ -1,6 +1,15 @@
"""
Pocket Calibre Recipe v1.2
Pocket Calibre Recipe v1.3
"""
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
import urllib2
import urllib
import json
import operator
import tempfile
import re
__license__ = 'GPL v3'
__copyright__ = '''
2010, Darko Miletic <darko.miletic at gmail.com>
@ -8,9 +17,6 @@ __copyright__ = '''
2012, tBunnyMan <Wag That Tail At Me dot com>
'''
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
class Pocket(BasicNewsRecipe):
title = 'Pocket'
@ -21,109 +27,150 @@ class Pocket(BasicNewsRecipe):
read after downloading.'''
publisher = 'getpocket.com'
category = 'news, custom'
oldest_article = 7
max_articles_per_feed = 50
minimum_articles = 10
mark_as_read_after_dl = True
#Set this to False for testing
mark_as_read_after_dl = False
#MUST be either 'oldest' or 'newest'
sort_method = 'oldest'
#To filter by tag this needs to be a single tag in quotes; IE 'calibre'
only_pull_tag = None
#You don't want to change anything under here unless you REALLY know what you are doing
no_stylesheets = True
use_embedded_content = False
needs_subscription = True
INDEX = u'http://getpocket.com'
LOGIN = INDEX + u'/l'
readList = []
articles_are_obfuscated = True
apikey = '19eg0e47pbT32z4793Tf021k99Afl889'
index_url = u'http://getpocket.com'
ajax_url = u'http://getpocket.com/a/x/getArticle.php'
read_api_url = index_url + u'/v3/get'
modify_api_url = index_url + u'/v3/send'
legacy_login_url = index_url + u'/l' # We use this to cheat oAuth
articles = []
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
if self.username is not None:
br.open(self.LOGIN)
def get_browser(self, *args, **kwargs):
"""
We need to pretend to be a recent version of safari for the mac to prevent User-Agent checks
Pocket api requires username and password so fail loudly if it's missing from the config.
"""
br = BasicNewsRecipe.get_browser(self,
user_agent='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4')
if self.username is not None and self.password is not None:
br.open(self.legacy_login_url)
br.select_form(nr=0)
br['feed_id'] = self.username
if self.password is not None:
br['password'] = self.password
br['password'] = self.password
br.submit()
else:
self.user_error("This Recipe requires authentication, please configured user & pass")
return br
def get_feeds(self):
self.report_progress(0, ('Fetching list of pages...'))
lfeeds = []
i = 1
feedurl = self.INDEX + u'/unread/1'
while True:
title = u'Unread articles, page ' + str(i)
lfeeds.insert(0, (title, feedurl))
self.report_progress(0, ('Got ') + str(i) + (' pages'))
i += 1
soup = self.index_to_soup(feedurl)
ritem = soup.find('a', attrs={'id':'next', 'class':'active'})
if ritem is None:
break
feedurl = self.INDEX + ritem['href']
return lfeeds
def get_auth_uri(self):
"""Quick function to return the authentication part of the url"""
uri = ""
uri = u'{0}&apikey={1!s}'.format(uri, self.apikey)
if self.username is None or self.password is None:
self.user_error("Username or password is blank. Pocket no longer supports blank passwords")
else:
uri = u'{0}&username={1!s}'.format(uri, self.username)
uri = u'{0}&password={1!s}'.format(uri, self.password)
return uri
def get_pull_articles_uri(self):
"""Return the part of the uri that has all of the get request settings"""
uri = ""
uri = u'{0}&state={1}'.format(uri, u'unread') # TODO This could be modded to allow pulling archives
uri = u'{0}&contentType={1}'.format(uri, u'article') # TODO This COULD return images too
uri = u'{0}&sort={1}'.format(uri, self.sort_method)
uri = u'{0}&count={1!s}'.format(uri, self.max_articles_per_feed)
if self.only_pull_tag is not None:
uri = u'{0}tag={1}'.format(uri, self.only_pull_tag)
return uri
def parse_index(self):
totalfeeds = []
articlesToGrab = self.max_articles_per_feed
lfeeds = self.get_feeds()
for feedobj in lfeeds:
if articlesToGrab < 1:
break
feedtitle, feedurl = feedobj
self.report_progress(0, ('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
articles = []
soup = self.index_to_soup(feedurl)
ritem = soup.find('ul', attrs={'id':'list'})
if ritem is None:
self.log.exception("Page %s skipped: invalid HTML" % (feedtitle if feedtitle else feedurl))
continue
for item in reversed(ritem.findAll('li')):
if articlesToGrab < 1:
break
else:
articlesToGrab -= 1
description = ''
atag = item.find('a', attrs={'class':'text'})
if atag and atag.has_key('href'):
url = self.INDEX + atag['href']
title = self.tag_to_string(item.div)
date = strftime(self.timefmt)
articles.append({
'title' :title
,'date' :date
,'url' :url
,'description':description
})
readLink = item.find('a', attrs={'class':'check'})['href']
self.readList.append(readLink)
totalfeeds.append((feedtitle, articles))
if len(self.readList) < self.minimum_articles:
pocket_feed = []
fetch_url = u"{0}?{1}{2}".format(
self.read_api_url,
self.get_auth_uri(),
self.get_pull_articles_uri()
)
try:
request = urllib2.Request(fetch_url)
response = urllib2.urlopen(request)
pocket_feed = json.load(response)['list']
except urllib2.HTTPError as e:
self.log.exception("Pocket returned an error: {0}\nurl: {1}".format(e, fetch_url))
return []
except urllib2.URLError as e:
self.log.exception("Unable to connect to getpocket.com's api: {0}\nurl: {1}".format(e, fetch_url))
return []
if len(pocket_feed) < self.minimum_articles:
self.mark_as_read_after_dl = False
if hasattr(self, 'abort_recipe_processing'):
self.abort_recipe_processing("Only %d articles retrieved, minimum_articles not reached" % len(self.readList))
else:
self.log.exception("Only %d articles retrieved, minimum_articles not reached" % len(self.readList))
return []
return totalfeeds
self.user_error("Only {0} articles retrieved, minimum_articles not reached".format(len(pocket_feed)))
def mark_as_read(self, markList):
br = self.get_browser()
for link in markList:
url = self.INDEX + link
print 'Marking read: ', url
response = br.open(url)
print response.info()
for pocket_article in pocket_feed.iteritems():
self.articles.append({
'item_id': pocket_article[0],
'title': pocket_article[1]['resolved_title'],
'date': pocket_article[1]['time_updated'],
'url': u'{0}/a/read/{1}'.format(self.index_url, pocket_article[0]),
'real_url': pocket_article[1]['resolved_url'],
'description': pocket_article[1]['excerpt'],
'sort': pocket_article[1]['sort_id']
})
self.articles = sorted(self.articles, key=operator.itemgetter('sort'))
print self.articles
return [("My Pocket Articles for {0}".format(strftime('[%I:%M %p]')), self.articles)]
def get_obfuscated_article(self, url):
soup = self.index_to_soup(url)
formcheck_script_tag = soup.find('script', text=re.compile("formCheck"))
form_check = formcheck_script_tag.split("=")[1].replace("'", "").replace(";", "").strip()
article_id = url.split("/")[-1]
data = urllib.urlencode({'itemId': article_id, 'formCheck': form_check})
response = self.browser.open(self.ajax_url, data)
article_json = json.load(response)['article']['article']
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf.write(article_json)
return tf.name
def mark_as_read(self, mark_list):
formatted_list = []
for article_id in mark_list:
formatted_list.append({
'action': 'archive',
'item_id': article_id
})
command = {
'actions': formatted_list
}
mark_read_url = u'{0}?{1}'.format(
self.modify_api_url,
self.get_auth_uri()
)
try:
request = urllib2.Request(mark_read_url, json.dumps(command))
response = urllib2.urlopen(request)
print u'response = {0}'.format(response.info())
except urllib2.HTTPError as e:
self.log.exception('Pocket returned an error while archiving articles: {0}'.format(e))
return []
except urllib2.URLError as e:
self.log.exception("Unable to connect to getpocket.com's modify api: {0}".format(e))
return []
def cleanup(self):
if self.mark_as_read_after_dl:
self.mark_as_read(self.readList)
self.mark_as_read([x[1]['item_id'] for x in self.articles])
else:
pass
def default_cover(self, cover_file):
'''
"""
Create a generic cover for recipes that don't have a cover
This override adds time to the cover
'''
"""
try:
from calibre.ebooks import calibre_cover
title = self.title if isinstance(self.title, unicode) else \
@ -137,3 +184,12 @@ class Pocket(BasicNewsRecipe):
self.log.exception('Failed to generate default cover')
return False
return True
def user_error(self, error_message):
if hasattr(self, 'abort_recipe_processing'):
self.abort_recipe_processing(error_message)
else:
self.log.exception(error_message)
raise RuntimeError(error_message)
# vim:ft=python

46
recipes/san_fran_chronicle.recipe
View File

@ -7,7 +7,6 @@ sfgate.com
'''
from calibre.web.feeds.news import BasicNewsRecipe
import re
class SanFranciscoChronicle(BasicNewsRecipe):
title = u'San Francisco Chronicle'
@ -19,16 +18,7 @@ class SanFranciscoChronicle(BasicNewsRecipe):
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = False
remove_tags_before = {'id':'printheader'}
remove_tags = [
dict(name='div',attrs={'id':'printheader'})
,dict(name='a', attrs={'href':re.compile('http://ads\.pheedo\.com.*')})
,dict(name='div',attrs={'id':'footer'})
]
auto_cleanup = True
extra_css = '''
h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
@ -43,33 +33,13 @@ class SanFranciscoChronicle(BasicNewsRecipe):
'''
feeds = [
(u'Top News Stories', u'http://www.sfgate.com/rss/feeds/news.xml')
(u'Bay Area News', u'http://www.sfgate.com/bayarea/feed/Bay-Area-News-429.php'),
(u'City Insider', u'http://www.sfgate.com/default/feed/City-Insider-Blog-573.php'),
(u'Crime Scene', u'http://www.sfgate.com/rss/feed/Crime-Scene-Blog-599.php'),
(u'Education News', u'http://www.sfgate.com/education/feed/Education-News-from-SFGate-430.php'),
(u'National News', u'http://www.sfgate.com/rss/feed/National-News-RSS-Feed-435.php'),
(u'Weird News', u'http://www.sfgate.com/weird/feed/Weird-News-RSS-Feed-433.php'),
(u'World News', u'http://www.sfgate.com/rss/feed/World-News-From-SFGate-432.php'),
]
def print_version(self,url):
url= url +"&type=printable"
return url
def get_article_url(self, article):
print str(article['title_detail']['value'])
url = article.get('guid',None)
url = "http://www.sfgate.com/cgi-bin/article.cgi?f="+url
if "Presented By:" in str(article['title_detail']['value']):
url = ''
return url

11
recipes/science_news.recipe
View File

@ -50,6 +50,10 @@ class ScienceNewsIssue(BasicNewsRecipe):
dict(name='ul', attrs={'id':'toc'})
]
remove_tags= [ dict(name='a', attrs={'class':'enlarge print-no'}),
dict(name='a', attrs={'rel':'shadowbox'})
]
feeds = [(u"Science News Current Issues", u'http://www.sciencenews.org/view/feed/type/edition/name/issues.rss')]
match_regexps = [
@ -57,6 +61,12 @@ class ScienceNewsIssue(BasicNewsRecipe):
r'www.sciencenews.org/view/generic/id'
]
def image_url_processor(self, baseurl, url):
x = url.split('/')
if x[4] == u'scale':
url = u'http://www.sciencenews.org/view/download/id/' + x[6] + u'/name/' + x[-1]
return url
def get_cover_url(self):
cover_url = None
index = 'http://www.sciencenews.org/view/home'
@ -64,7 +74,6 @@ class ScienceNewsIssue(BasicNewsRecipe):
link_item = soup.find(name = 'img',alt = "issue")
if link_item:
cover_url = 'http://www.sciencenews.org' + link_item['src'] + '.jpg'
return cover_url
def preprocess_html(self, soup):

20
recipes/smith.recipe
View File

@ -25,7 +25,7 @@ class Smithsonian(BasicNewsRecipe):
soup = self.index_to_soup(current_issue_url)
#Go to the main body
div = soup.find ('div', attrs={'id':'article-body'})
div = soup.find('div', attrs={'id':'article-body'})
#Find date
date = re.sub('.*\:\W*', "", self.tag_to_string(div.find('h2')).strip())
@ -49,16 +49,20 @@ class Smithsonian(BasicNewsRecipe):
self.log('Found section:', section_title)
else:
link=post.find('a',href=True)
article_cat=link.findPrevious('p', attrs={'class':'article-cat'})
url=link['href']+'?c=y&story=fullstory'
description=self.tag_to_string(post.find('p')).strip()
desc=re.sub('\sBy\s.*', '', description, re.DOTALL)
author=re.sub('.*By\s', '', description, re.DOTALL)
title=self.tag_to_string(link).strip()+ u' (%s)'%author
description=self.tag_to_string(post.findAll('p')[-1]).strip()
title=self.tag_to_string(link).strip()
if article_cat is not None:
title += u' (%s)'%self.tag_to_string(article_cat).strip()
self.log('\tFound article:', title)
articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
articles.append({'title':title, 'url':url, 'description':description, 'date':''})
if articles:
feeds[section_title] = articles
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
articles = []
ans = [(key, val) for key, val in feeds.iteritems()]
return ans

29
recipes/the_escapist.recipe
View File

@ -1,8 +1,11 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__author__ = 'Lorenzo Vigentini'
__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>'
description = 'the Escapist Magazine - v1.02 (09, January 2010)'
__author__ = 'Lorenzo Vigentini and Tom Surace'
__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>, 2013 Tom Surace <tekhedd@byteheaven.net>'
description = 'The Escapist Magazine - v1.3 (2013, April 2013)'
#
# Based on 'the Escapist Magazine - v1.02 (09, January 2010)'
'''
http://www.escapistmagazine.com/
@ -11,12 +14,11 @@ http://www.escapistmagazine.com/
from calibre.web.feeds.news import BasicNewsRecipe
class al(BasicNewsRecipe):
author = 'Lorenzo Vigentini'
author = 'Lorenzo Vigentini and Tom Surace'
description = 'The Escapist Magazine'
cover_url = 'http://cdn.themis-media.com/themes/escapistmagazine/default/images/logo.png'
title = u'The Escapist Magazine'
publisher = 'Themis media'
publisher = 'Themis Media'
category = 'Video games news, lifestyle, gaming culture'
language = 'en'
@ -36,18 +38,19 @@ class al(BasicNewsRecipe):
]
def print_version(self,url):
# Expect article url in the format:
# http://www.escapistmagazine.com/news/view/123198-article-name?utm_source=rss&utm_medium=rss&utm_campaign=news
#
baseURL='http://www.escapistmagazine.com'
segments = url.split('/')
#basename = '/'.join(segments[:3]) + '/'
subPath= '/'+ segments[3] + '/'
articleURL=(segments[len(segments)-1])[0:5]
if articleURL[4] =='-':
articleURL=articleURL[:4]
# The article number is the "number" that starts the name
articleNumber = segments[len(segments)-1]; # the "article name"
articleNumber = articleNumber.split('-')[0]; # keep part before hyphen
printVerString='print/'+ articleURL
s= baseURL + subPath + printVerString
return s
fullUrl = baseURL + subPath + 'print/' + articleNumber
return fullUrl
keep_only_tags = [
dict(name='div', attrs={'id':'article'})

11
recipes/the_feature.recipe Normal file
View File

@ -0,0 +1,11 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1365777047(BasicNewsRecipe):
title = u'The Feature'
__author__ = 'Jose Pinto'
language = 'en'
oldest_article = 30
max_articles_per_feed = 100
auto_cleanup = True
use_embedded_content = False
feeds = [(u'Latest', u'http://thefeature.net/rss/links')]

131
recipes/the_new_republic.recipe
View File

@ -1,68 +1,63 @@
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict
class TNR(BasicNewsRecipe):
title = 'The New Republic'
__author__ = 'Rick Shang'
description = 'The New Republic is a journal of opinion with an emphasis on politics and domestic and international affairs. It carries feature articles by staff and contributing editors. The second half of each issue is devoted to book and the arts, theater, motion pictures, music and art.'
language = 'en'
category = 'news'
encoding = 'UTF-8'
remove_tags = [dict(attrs={'class':['print-logo','print-site_name','print-hr']})]
no_javascript = True
no_stylesheets = True
def parse_index(self):
#Go to the issue
soup0 = self.index_to_soup('http://www.tnr.com/magazine-issues')
issue = soup0.find('div',attrs={'id':'current_issue'})
#Find date
date = self.tag_to_string(issue.find('div',attrs={'class':'date'})).strip()
self.timefmt = u' [%s]'%date
#Go to the main body
current_issue_url = 'http://www.tnr.com' + issue.find('a', href=True)['href']
soup = self.index_to_soup(current_issue_url)
div = soup.find ('div', attrs={'class':'article_detail_body'})
#Find cover
self.cover_url = div.find('img',src=True)['src']
feeds = OrderedDict()
section_title = ''
subsection_title = ''
for post in div.findAll('p'):
articles = []
em=post.find('em')
b=post.find('b')
a=post.find('a',href=True)
p=post.find('img', src=True)
#Find cover
if p is not None:
self.cover_url = p['src'].strip()
if em is not None:
section_title = self.tag_to_string(em).strip()
subsection_title = ''
elif b is not None:
subsection_title=self.tag_to_string(b).strip()
elif a is not None:
prefix = (subsection_title+': ') if subsection_title else ''
url=re.sub('www.tnr.com','www.tnr.com/print', a['href'])
author=re.sub('.*by\s', '', self.tag_to_string(post), re.DOTALL)
title=prefix + self.tag_to_string(a).strip()+ u' (%s)'%author
articles.append({'title':title, 'url':url, 'description':'', 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
class TNR(BasicNewsRecipe):
title = 'The New Republic'
__author__ = 'Krittika Goyal'
description = '''The New Republic is a journal of opinion with an emphasis
on politics and domestic and international affairs. It carries feature
articles by staff and contributing editors. The second half of each issue
is devoted to book and the arts, theater, motion pictures, music and art.'''
language = 'en'
encoding = 'UTF-8'
needs_subscription = True
preprocess_regexps = [
(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: ''),
(re.compile(r'<script.*?</script>', re.DOTALL), lambda m: ''),
]
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
br.open('http://www.newrepublic.com/user')
br.select_form(nr=1)
try:
br['user'] = self.username
except:
br['name'] = self.username
br['pass'] = self.password
self.log('Logging in...')
raw = br.submit().read()
if 'SIGN OUT' not in raw:
raise ValueError('Failed to log in to tnr.com, check your username and password')
self.log('Logged in successfully')
return br
def parse_index(self):
raw = self.index_to_soup('http://www.newrepublic.com/current-issue', raw=True)
# raw = self.index_to_soup(open('/t/raw.html').read().decode('utf-8'), raw=True)
for pat, sub in self.preprocess_regexps:
raw = pat.sub(sub, raw)
soup = self.index_to_soup(raw)
feed_title = 'The New Republic Magazine Articles'
articles = []
for div in soup.findAll('div', attrs={'class':lambda x: x and 'field-item' in x.split()}):
a = div.find('a', href=True, attrs={'class':lambda x: x != 'author'})
if a is not None:
art_title = self.tag_to_string(a)
url = a.get('href')
num = re.search(r'/(\d+)/', url)
if num is not None:
art = num.group(1)
url = 'http://www.newrepublic.com/node/%s/print'%art
self.log.info('\tFound article:', art_title, 'at', url)
article = {'title':art_title, 'url':url, 'description':'', 'date':''}
articles.append(article)
return [(feed_title, articles)]

58
recipes/the_sun.recipe
View File

@ -1,4 +1,4 @@
import re, random
import random
from calibre import browser
from calibre.web.feeds.recipes import BasicNewsRecipe
@ -8,7 +8,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
title = u'The Sun UK'
description = 'Articles from The Sun tabloid UK'
__author__ = 'Dave Asbury'
# last updated 19/10/12 better cover fetch
# last updated 5/5/13 better cover fetch
language = 'en_GB'
oldest_article = 1
max_articles_per_feed = 15
@ -29,16 +29,12 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
dict(name='div',attrs={'class' : 'intro'}),
dict(name='h3'),
dict(name='div',attrs={'id' : 'articlebody'}),
#dict(attrs={'class' : ['right_col_branding','related-stories','mystery-meat-link','ltbx-container','ltbx-var ltbx-hbxpn','ltbx-var ltbx-nav-loop','ltbx-var ltbx-url']}),
# dict(name='div',attrs={'class' : 'cf'}),
# dict(attrs={'title' : 'download flash'}),
# dict(attrs={'style' : 'padding: 5px'})
]
]
remove_tags_after = [dict(id='bodyText')]
remove_tags=[
dict(name='li'),
dict(attrs={'class' : 'grid-4 right-hand-column'}),
dict(name='li'),
dict(attrs={'class' : 'grid-4 right-hand-column'}),
]
feeds = [
@ -47,40 +43,24 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
(u'Showbiz', u'http://www.thesun.co.uk/sol/homepage/showbiz/rss'),
(u'Woman', u'http://www.thesun.co.uk/sol/homepage/woman/rss'),
]
# starsons code
def parse_feeds (self):
feeds = BasicNewsRecipe.parse_feeds(self)
for feed in feeds:
for article in feed.articles[:]:
print 'article.title is: ', article.title
if 'Try out The Sun' in article.title.upper() or 'Try-out-The-Suns' in article.url:
feed.articles.remove(article)
if 'Web porn harms kids' in article.title.upper() or 'Sun-says-Web-porn' in article.url:
feed.articles.remove(article)
return feeds
# starsons code
def parse_feeds(self):
feeds = BasicNewsRecipe.parse_feeds(self)
for feed in feeds:
for article in feed.articles[:]:
if 'Try out The Sun' in article.title.upper() or 'Try-out-The-Suns' in article.url:
feed.articles.remove(article)
if 'Web porn harms kids' in article.title.upper() or 'Sun-says-Web-porn' in article.url:
feed.articles.remove(article)
return feeds
def get_cover_url(self):
soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html')
# look for the block containing the sun button and url
cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'})
#cov = soup.find(attrs={'id' : 'large'})
cov2 = str(cov)
cov2='http://www.politicshome.com'+cov2[9:-133]
#cov2 now contains url of the page containing pic
#cov2 now contains url of the page containing pic
soup = self.index_to_soup(cov2)
cov = soup.find(attrs={'id' : 'large'})
cov=str(cov)
cov2 = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', cov)
cov2 = str(cov2)
cov2=cov2[2:len(cov2)-2]
br = browser()
br.set_handle_redirect(False)
cover_url = 'http://www.thepaperboy.com/frontpages/current/The_Sun_newspaper_front_page.jpg'
try:
br.open_novisit(cov2)
cover_url = cov2
br.open_novisit('http://www.thepaperboy.com/frontpages/current/The_Sun_newspaper_front_page.jpg')
except:
cover_url = random.choice([
'http://img.thesun.co.uk/multimedia/archive/00905/errorpage6_677961a_905507a.jpg'
@ -88,6 +68,6 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage5_677960a_905512a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage2_677957a_905502a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage3_677958a_905503a.jpg'
])
])
return cover_url

64
recipes/theonion.recipe
View File

@ -1,5 +1,5 @@
__license__ = 'GPL v3'
__copyright__ = '2009-2011, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2009-2013, Darko Miletic <darko.miletic at gmail.com>'
'''
theonion.com
@ -10,7 +10,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
class TheOnion(BasicNewsRecipe):
title = 'The Onion'
__author__ = 'Darko Miletic'
description = "America's finest news source"
description = "The Onion, America's Finest News Source, is an award-winning publication covering world, national, and * local issues. It is updated daily online and distributed weekly in select American cities."
oldest_article = 2
max_articles_per_feed = 100
publisher = 'Onion, Inc.'
@ -20,7 +20,8 @@ class TheOnion(BasicNewsRecipe):
use_embedded_content = False
encoding = 'utf-8'
publication_type = 'newsportal'
masthead_url = 'http://o.onionstatic.com/img/headers/onion_190.png'
needs_subscription = 'optional'
masthead_url = 'http://www.theonion.com/static/onion/img/logo_1x.png'
extra_css = """
body{font-family: Helvetica,Arial,sans-serif}
.section_title{color: gray; text-transform: uppercase}
@ -36,21 +37,56 @@ class TheOnion(BasicNewsRecipe):
, 'publisher': publisher
, 'language' : language
}
keep_only_tags = [dict(name='article', attrs={'class':'full-article'})]
remove_tags = [
dict(name=['nav', 'aside', 'section', 'meta']),
{'attrs':{'class':lambda x: x and ('share-tools' in x or 'ad-zone' in x)}},
]
keep_only_tags = [dict(attrs={'class':'full-article'})]
remove_attributes = ['lang','rel']
remove_tags = [
dict(name=['object','link','iframe','base','meta'])
,dict(attrs={'class':lambda x: x and 'share-tools' in x.split()})
]
feeds = [
(u'Daily' , u'http://feeds.theonion.com/theonion/daily' )
,(u'Sports' , u'http://feeds.theonion.com/theonion/sports' )
]
def preprocess_html(self, soup, *args):
for img in soup.findAll('img', attrs={'data-src':True}):
if img['data-src']:
img['src'] = img['data-src']
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
br.open('http://www.theonion.com/')
if self.username is not None and self.password is not None:
br.open('https://ui.ppjol.com/login/onion/u/j_spring_security_check')
br.select_form(name='f')
br['j_username'] = self.username
br['j_password'] = self.password
br.submit()
return br
def get_article_url(self, article):
artl = BasicNewsRecipe.get_article_url(self, article)
if artl.startswith('http://www.theonion.com/audio/'):
artl = None
return artl
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
if not limg.has_key('alt'):
limg['alt'] = 'image'
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if item.has_key('data-src'):
item['src'] = item['data-src']
return soup

28
recipes/tomshardware.recipe
View File

@ -1,7 +1,5 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2008-2013, Darko Miletic <darko.miletic at gmail.com>'
'''
tomshardware.com/us
'''
@ -16,22 +14,20 @@ class Tomshardware(BasicNewsRecipe):
publisher = "Tom's Hardware"
category = 'news, IT, hardware, USA'
no_stylesheets = True
needs_subscription = True
language = 'en'
needs_subscription = 'optional'
language = 'en'
INDEX = 'http://www.tomshardware.com'
LOGIN = INDEX + '/membres/'
remove_javascript = True
use_embedded_content= False
html2lrf_options = [
'--comment', description
, '--category', category
, '--publisher', publisher
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
br.open(self.INDEX+'/us/')
@ -50,8 +46,8 @@ class Tomshardware(BasicNewsRecipe):
]
feeds = [
(u'Latest Articles', u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-2.xml' )
,(u'Latest News' , u'http://www.tomshardware.com/feeds/atom/tom-s-hardware-us,18-1.xml')
(u'Reviews', u'http://www.tomshardware.com/feeds/rss2/tom-s-hardware-us,18-2.xml')
,(u'News' , u'http://www.tomshardware.com/feeds/rss2/tom-s-hardware-us,18-1.xml')
]
def print_version(self, url):

37
recipes/tvxs.recipe
View File

@ -1,5 +1,6 @@
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
class TVXS(BasicNewsRecipe):
@ -8,19 +9,30 @@ class TVXS(BasicNewsRecipe):
description = 'News from Greece'
max_articles_per_feed = 100
oldest_article = 3
simultaneous_downloads = 1
publisher = 'TVXS'
category = 'news, GR'
category = 'news, sport, greece'
language = 'el'
encoding = None
use_embedded_content = False
remove_empty_feeds = True
#conversion_options = { 'linearize_tables': True}
conversion_options = {'smarten_punctuation': True}
no_stylesheets = True
publication_type = 'newspaper'
remove_tags_before = dict(name='h1',attrs={'class':'print-title'})
remove_tags_after = dict(name='div',attrs={'class':'field field-type-relevant-content field-field-relevant-articles'})
remove_attributes = ['width', 'src', 'header', 'footer']
remove_tags = [dict(name='div',attrs={'class':'field field-type-relevant-content field-field-relevant-articles'}),
dict(name='div',attrs={'class':'field field-type-filefield field-field-image-gallery'}),
dict(name='div',attrs={'class':'filefield-file'})]
remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
extra_css = 'body { font-family: verdana, helvetica, sans-serif; } \
table { width: 100%; } \
td img { display: block; margin: 5px auto; } \
ul { padding-top: 10px; } \
ol { padding-top: 10px; } \
li { padding-top: 5px; padding-bottom: 5px; } \
h1 { text-align: center; font-size: 125%; font-weight: bold; } \
h2, h3, h4, h5, h6 { text-align: center; font-size: 100%; font-weight: bold; }'
preprocess_regexps = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''), (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
feeds = [(u'Ελλάδα', 'http://tvxs.gr/feeds/2/feed.xml'),
(u'Κόσμος', 'http://tvxs.gr/feeds/5/feed.xml'),
@ -35,17 +47,10 @@ class TVXS(BasicNewsRecipe):
(u'Ιστορία', 'http://tvxs.gr/feeds/1573/feed.xml'),
(u'Χιούμορ', 'http://tvxs.gr/feeds/692/feed.xml')]
def print_version(self, url):
import urllib2, urlparse, StringIO, gzip
fp = urllib2.urlopen(url)
data = fp.read()
if fp.info()['content-encoding'] == 'gzip':
gzip_data = StringIO.StringIO(data)
gzipper = gzip.GzipFile(fileobj=gzip_data)
data = gzipper.read()
fp.close()
br = self.get_browser()
response = br.open(url)
data = response.read()
pos_1 = data.find('<a href="/print/')
if pos_1 == -1:
@ -57,5 +62,5 @@ class TVXS(BasicNewsRecipe):
pos_1 += len('<a href="')
new_url = data[pos_1:pos_2]
print_url = urlparse.urljoin(url, new_url)
print_url = "http://tvxs.gr" + new_url
return print_url

27
recipes/voice_of_america.recipe Normal file
View File

@ -0,0 +1,27 @@
from calibre.web.feeds.news import BasicNewsRecipe
class HindustanTimes(BasicNewsRecipe):
title = u'Voice of America'
language = 'en'
__author__ = 'Krittika Goyal'
oldest_article = 15 #days
max_articles_per_feed = 25
#encoding = 'cp1252'
use_embedded_content = False
no_stylesheets = True
auto_cleanup = True
feeds = [
('All Zones',
'http://learningenglish.voanews.com/rss/?count=20'),
('World',
'http://learningenglish.voanews.com/rss/?count=20&zoneid=957'),
('USA',
'http://learningenglish.voanews.com/rss/?count=20&zoneid=958'),
('Health',
'http://learningenglish.voanews.com/rss/?count=20&zoneid=955'),
]

5
recipes/what_if.recipe
View File

@ -13,7 +13,7 @@ class XkcdCom(BasicNewsRecipe):
use_embedded_content = False
oldest_article = 60
# add image and text
# add an horizontal line after the question
# add an horizontal line after the question
preprocess_regexps = [
(re.compile(r'(<img.*title=")([^"]+)(".*>)'),
lambda m: '<div>%s%s<p id="photo_text">(%s)</p></div>' % (m.group(1), m.group(3), m.group(2))),
@ -22,3 +22,6 @@ class XkcdCom(BasicNewsRecipe):
]
extra_css = "#photo_text{font-size:small;}"
feeds = [(u'What If...', u'http://what-if.xkcd.com/feed.atom')]

5
resources/default_tweaks.py
View File

@ -531,3 +531,8 @@ numeric_collation = False
# number here. The default is ten libraries.
many_libraries = 10
#: Highlight the count of books when using a Virtual Library
# The count of books next to the Virtual Library button is highlighted in
# yellow when using a Virtual Library. By setting this to False, you can turn
# that off.
highlight_virtual_library_book_count = True

BIN
resources/images/devices/ipad.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 62 KiB

After

Width:  |  Height:  |  Size: 61 KiB

BIN
resources/images/devices/italica.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.7 KiB

After

Width:  |  Height:  |  Size: 5.2 KiB

BIN
resources/images/library.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 396 KiB

After

Width:  |  Height:  |  Size: 397 KiB

11
resources/templates/rtf.xsl
View File

@ -448,8 +448,15 @@
<xsl:template match = "rtf:field[@type='hyperlink']">
<xsl:element name ="a">
<xsl:attribute name = "href">
<xsl:if test = "not(contains(@link, '/'))">#</xsl:if>
<xsl:value-of select = "@link"/>
<xsl:choose>
<xsl:when test="@argument">
<xsl:value-of select="@argument"/>
</xsl:when>
<xsl:otherwise>
<xsl:if test = "not(contains(@link, '/'))">#</xsl:if>
<xsl:value-of select = "@link"/>
</xsl:otherwise>
</xsl:choose>
</xsl:attribute>
<xsl:apply-templates/>
</xsl:element>

3
session.vim
View File

@ -1,6 +1,3 @@
" Project wide builtins
let $PYFLAKES_BUILTINS = "_,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext"
" Include directories for C++ modules
let g:syntastic_cpp_include_dirs = [
\'/usr/include/python2.7',

4
setup.cfg Normal file
View File

@ -0,0 +1,4 @@
[flake8]
max-line-length = 160
builtins = _,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext
ignore = E12,E203,E22,E231,E241,E301,E302,E304,E401,W391

2
setup.py
View File

@ -63,7 +63,7 @@ def main(args=sys.argv):
parser = option_parser()
command.add_all_options(parser)
parser.set_usage('Usage: python setup.py %s [options]\n\n'%args[1]+\
parser.set_usage('Usage: python setup.py %s [options]\n\n'%args[1]+
command.description)
opts, args = parser.parse_args(args)

49
setup/check.py
View File

@ -22,40 +22,12 @@ class Message:
self.filename, self.lineno, self.msg = filename, lineno, msg
def __str__(self):
return '%s:%s: %s'%(self.filename, self.lineno, self.msg)
def check_for_python_errors(code_string, filename):
import _ast
# First, compile into an AST and handle syntax errors.
try:
tree = compile(code_string, filename, "exec", _ast.PyCF_ONLY_AST)
except (SyntaxError, IndentationError) as value:
msg = value.args[0]
(lineno, offset, text) = value.lineno, value.offset, value.text
# If there's an encoding problem with the file, the text is None.
if text is None:
# Avoid using msg, since for the only known case, it contains a
# bogus message that claims the encoding the file declared was
# unknown.
msg = "%s: problem decoding source" % filename
return [Message(filename, lineno, msg)]
else:
checker = __import__('pyflakes.checker').checker
# Okay, it's syntactically valid. Now check it.
w = checker.Checker(tree, filename)
w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
return [Message(x.filename, x.lineno, x.message%x.message_args) for x in
w.messages]
return '%s:%s: %s' % (self.filename, self.lineno, self.msg)
class Check(Command):
description = 'Check for errors in the calibre source code'
BUILTINS = ['_', '__', 'dynamic_property', 'I', 'P', 'lopen', 'icu_lower',
'icu_upper', 'icu_title', 'ngettext']
CACHE = '.check-cache.pickle'
def get_files(self, cache):
@ -65,10 +37,10 @@ class Check(Command):
mtime = os.stat(y).st_mtime
if cache.get(y, 0) == mtime:
continue
if (f.endswith('.py') and f not in ('feedparser.py',
'pyparsing.py', 'markdown.py') and
'prs500/driver.py' not in y):
yield y, mtime
if (f.endswith('.py') and f not in (
'feedparser.py', 'markdown.py') and
'prs500/driver.py' not in y):
yield y, mtime
if f.endswith('.coffee'):
yield y, mtime
@ -79,25 +51,22 @@ class Check(Command):
if f.endswith('.recipe') and cache.get(f, 0) != mtime:
yield f, mtime
def run(self, opts):
cache = {}
if os.path.exists(self.CACHE):
cache = cPickle.load(open(self.CACHE, 'rb'))
builtins = list(set_builtins(self.BUILTINS))
for f, mtime in self.get_files(cache):
self.info('\tChecking', f)
errors = False
ext = os.path.splitext(f)[1]
if ext in {'.py', '.recipe'}:
w = check_for_python_errors(open(f, 'rb').read(), f)
if w:
p = subprocess.Popen(['flake8', '--ignore=E,W', f])
if p.wait() != 0:
errors = True
self.report_errors(w)
else:
from calibre.utils.serve_coffee import check_coffeescript
try:
check_coffeescript(f)
check_coffeescript(f)
except:
errors = True
if errors:
@ -106,8 +75,6 @@ class Check(Command):
self.j(self.SRC, '../session.vim'), '-f', f])
raise SystemExit(1)
cache[f] = mtime
for x in builtins:
delattr(__builtin__, x)
cPickle.dump(cache, open(self.CACHE, 'wb'), -1)
wn_path = os.path.expanduser('~/work/servers/src/calibre_servers/main')
if os.path.exists(wn_path):

11
setup/installer/linux/freeze2.py
View File

@ -48,7 +48,7 @@ binary_includes = [
'/usr/lib/libpng14.so.14',
'/usr/lib/libexslt.so.0',
# Ensure that libimobiledevice is compiled against openssl, not gnutls
'/usr/lib/libimobiledevice.so.3',
'/usr/lib/libimobiledevice.so.4',
'/usr/lib/libusbmuxd.so.2',
'/usr/lib/libplist.so.1',
MAGICK_PREFIX+'/lib/libMagickWand.so.5',
@ -112,7 +112,6 @@ class LinuxFreeze(Command):
else:
ffi = glob.glob('/usr/lib/libffi.so.?')[-1]
for x in binary_includes + [stdcpp, ffi]:
dest = self.bin_dir if '/bin/' in x else self.lib_dir
shutil.copy2(x, dest)
@ -226,7 +225,6 @@ class LinuxFreeze(Command):
except:
self.warn('Failed to byte-compile', y)
def run_builder(self, cmd, verbose=True):
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
@ -256,7 +254,6 @@ class LinuxFreeze(Command):
self.info('Archive %s created: %.2f MB'%(dist,
os.stat(dist).st_size/(1024.**2)))
def build_launchers(self):
self.obj_dir = self.j(self.src_root, 'build', 'launcher')
if not os.path.exists(self.obj_dir):
@ -268,7 +265,8 @@ class LinuxFreeze(Command):
cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%self.py_ver
cflags = cflags.split() + ['-I/usr/include/python'+self.py_ver]
for src, obj in zip(sources, objects):
if not self.newer(obj, headers+[src, __file__]): continue
if not self.newer(obj, headers+[src, __file__]):
continue
cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
self.run_builder(cmd)
@ -330,8 +328,7 @@ class LinuxFreeze(Command):
self.run_builder(cmd, verbose=False)
def create_site_py(self): # {{{
def create_site_py(self): # {{{
with open(self.j(self.py_dir, 'site.py'), 'wb') as f:
f.write(textwrap.dedent('''\
import sys

13
setup/installer/osx/app/main.py
View File

@ -37,7 +37,6 @@ class OSX32_Freeze(Command):
action='store_true',
help='Only build launchers')
def run(self, opts):
global info, warn
info, warn = self.info, self.warn
@ -332,7 +331,7 @@ class Py2App(object):
def create_plist(self):
from calibre.ebooks import BOOK_EXTENSIONS
env = dict(**ENV)
env['CALIBRE_LAUNCHED_FROM_BUNDLE']='1';
env['CALIBRE_LAUNCHED_FROM_BUNDLE']='1'
docs = [{'CFBundleTypeName':'E-book',
'CFBundleTypeExtensions':list(BOOK_EXTENSIONS),
'CFBundleTypeRole':'Viewer',
@ -395,12 +394,11 @@ class Py2App(object):
self.install_dylib(os.path.join(SW, 'lib', 'libpng12.0.dylib'))
self.install_dylib(os.path.join(SW, 'lib', 'libpng.3.dylib'))
@flush
def add_fontconfig(self):
info('\nAdding fontconfig')
for x in ('fontconfig.1', 'freetype.6', 'expat.1',
'plist.1', 'usbmuxd.2', 'imobiledevice.3'):
'plist.1', 'usbmuxd.2', 'imobiledevice.4'):
src = os.path.join(SW, 'lib', 'lib'+x+'.dylib')
self.install_dylib(src)
dst = os.path.join(self.resources_dir, 'fonts')
@ -568,7 +566,7 @@ class Py2App(object):
@flush
def compile_py_modules(self):
info( '\nCompiling Python modules')
info('\nCompiling Python modules')
base = join(self.resources_dir, 'Python')
for x in os.walk(base):
root = x[0]
@ -584,7 +582,7 @@ class Py2App(object):
@flush
def create_console_app(self):
info( '\nCreating console.app')
info('\nCreating console.app')
cc_dir = os.path.join(self.contents_dir, 'console.app', 'Contents')
os.makedirs(cc_dir)
for x in os.listdir(self.contents_dir):
@ -607,7 +605,6 @@ class Py2App(object):
shutil.copy2(join(base, 'site.py'), join(self.resources_dir, 'Python',
'lib', 'python'+self.version_info))
@flush
def makedmg(self, d, volname,
destdir='dist',
@ -630,7 +627,7 @@ class Py2App(object):
'-volname', volname, '-format', format, dmg])
shutil.rmtree(tdir)
if internet_enable:
subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
size = os.stat(dmg).st_size/(1024*1024.)
info('\nInstaller size: %.2fMB\n'%size)
return dmg

352
setup/installer/windows/libimobiledevice_notes.rst Normal file
View File

@ -0,0 +1,352 @@
Notes on building libiMobileDevice for Windows
=========================================================
1. Get source files, set up VS project
2. Build libcnary
3. Build libgen
4. Build libplist
5. Build libusbmuxd
6. Build libimobiledevice
7. Exporting libimobiledevice entry points
8. Finished
Get source files, set up VS project
-------------------------------------
Starting with source downloaded from https://github.com/storoj/libimobiledevice-win32
Now create a new directory in which we will work::
mkdir imobiledevice
cp -r libcnary libgen vendors/include libimobiledevice libplist libusbmuxd imobiledevice
cd imobiledevice
rm `find . -name '*.props'`
rm `find . -name *.vcxproj*`
rm `find . -name *.txt`
cd ..
mv imobiledevice ~/sw/private/
In include/unistd.h, comment out line 11::
// #include <getopt.h> /* getopt from: http://www.pwilson.net/sample.html.
Create a new VS 2008 Project
- File|New|Project…
- Visual C++: Win32
- Template: Win32Project
- Name: imobiledevice
- Location: Choose ~/sw/private
- Solution: (Uncheck the create directory for solution checkbox)
- Click OK
- Next screen, select Application Settings tab
- Application type: DLL
- Additional options: Empty project
- Click Finish
In the tool bar Solution Configurations dropdown, select Release.
In the tool bar Solution Platforms dropdown, select Win32.
(For 64 bit choose new configuration and create x64 with properties copied from
win32).
Build libcnary
-------------------------
In VS Solution Explorer, right-click Solution 'imobiledevice', then click
Add|New Project.
- Name: libcnary
- Location: Add \imobiledevice to the end of the default location
- Visual C++: Win32, Template: Win32 Project
- Click OK
- Application Settings: Static library (not using precompiled headers)
- Click Finish
In VS Solution Explorer, select the libcnary project, Project->Show All files.
- Right-click the include folder, select 'Include In Project'.
- Select all the .c files, right click, select 'Include In Project'
- Select all the .c files, right click -> Properties -> C/C++ -> Advanced -> Compile as C++ code
- Properties|Configuration Properties|C/C++:
General|Additional Include Directories:
"$(ProjectDir)\include"
- If 64bits, then Right click->Properties->Configuration Manager change
Win32 to x64 for the libcnary project and check the Build checkbox
- Right-click libcnary, Build. Should build with 0 errors, 0 warnings.
Build libplist
---------------------
In VS Solution Explorer, right-click Solution 'imobiledevice', then click
Add|New Project.
- Name: libplist
- Visual C++: Win32, Template: Win32 Project
- Location: Add \imobiledevice to the end of the default location
- Click OK
- Application Settings: DLL (Empty project)
- Click Finish
In VS Solution Explorer, select the libplist project, then click the 'Show all files'
button.
- Right-click the include folder, select Include In Project
- Right-click the src folder, select Include In Project
- Set 7 C files to compile as C++
Advanced|Compile As: Compile as C++ Code (/TP)
base64.c, bplist.c, bytearray.c, hashtable.c, plist.c, ptarray.c, xplist.c
- Properties|Configuration Properties|C/C++:
General|Additional Include Directories:
$(ProjectDir)\include
$(SolutionDir)\include
$(SolutionDir)\libcnary\include
$SW\include\libxml2 (if it exists)
$SW\include (make sure this is last in the list)
- Properties|C/C++|Preprocessor
Preprocessor Definitions: Add the following items
__STDC_FORMAT_MACROS
plist_EXPORTS
- Properties -> Linker -> General -> Additional Library directories: ~/sw/lib (for libxml2.lib)
- Properties -> Linker -> Input -> Additional Dependencies: libxml2.lib
- Project Dependencies:
Depends on: libcnary
- If 64bits, then Right click->Properties->Configuration Manager change
Win32 to x64 for the libcnary project and check the Build checkbox
- Right-click libplist, Build. Should build with 0 errors (there will be
warnings about datatype conversion for the 64 bit build)
Build libusbmuxd
----------------------
In VS Solution Explorer, right-click Solution 'imobiledevice', then click
Add|New Project.
- Name: libusbmuxd
- Visual C++: Win32, Template: Win32 Project
- Location: Add \imobiledevice to the end of the default location
- Click OK
- Application Settings: DLL (Empty project)
- Click Finish
In VS Solution Explorer, select the libusbmuxd project, then click the 'Show all files'
button.
- Select all 7 files, right-click, Include In Project.
- Set 3 C files to compile as C++
Advanced|Compile As: Compile as C++ Code (/TP)
libusbmuxd.c, sock_stuff.c, utils.c
- Properties|Configuration Properties|C/C++:
General|Additional Include Directories:
$(SolutionDir)\include
$(SolutionDir)\libplist\include
- Properties|Linker|Input|Additional Dependencies:
ws2_32.lib
- Properties|C/C++|Preprocessor
Preprocessor Definitions: add 'HAVE_PLIST'
- Project Dependencies:
Depends on: libplist
- Edit sock_stuff.c #227:
fprintf(stderr, "%s: gethostbyname returned NULL address!\n",
__FUNCTION__);
- Edit libusbmuxd\usbmuxd.h, insert at #26:
#ifdef LIBUSBMUXD_EXPORTS
# define LIBUSBMUXD_API __declspec( dllexport )
#else
# define LIBUSBMUXD_API __declspec( dllimport )
#endif
Then, at each function, insert LIBUSBMUXD_API ahead of declaration:
usbmuxd_subscribe
usbmuxd_unsubscribe
usbmuxd_get_device_list
usbmuxd_device_list_free
usbmuxd_get_device_by_udid
usbmuxd_connect
usbmuxd_disconnect
usbmuxd_send
usbmuxd_recv_timeout
usbmuxd_recv
usbmuxd_set_use_inotify
usbmuxd_set_debug_level
- If 64bits, then Right click->Properties->Configuration Manager change
Win32 to x64 for the libcnary project and check the Build checkbox
- Right-click libusbmuxd, Build. Should build with 0 errors, 10 or 14 warnings
Build libgen
-----------------------
In VS Solution Explorer, right-click Solution 'imobiledevice', then click
Add|New Project.
- Name: libgen
- Visual C++: Win32, Template: Win32 Project
- Location: Add \imobiledevice to the end of the default location
- Click OK
- Application Settings: Static library (not using precompiled headers)
- Click Finish
In VS Solution Explorer, select the libgen project, then click the 'Show all files'
button.
- Select libgen.cpp and libgen.h, right click, select 'Include In Project'
- Open libgen.cpp, comment out line 5::
// #include <fileapi.h>
(This is a Windows 8 include file, not needed to build in Win 7)
- If 64bits, then Right click->Properties->Configuration Manager change
Win32 to x64 for the libcnary project and check the Build checkbox
- Right-click libgen, Build. Should build with 0 errors, 0 warnings.
Build libimobiledevice
----------------------------
In VS Solution Explorer, right-click Solution 'imobiledevice', then click
Add|New Project.
- Name: libimobiledevice
- Visual C++: Win32, Template: Win32 Project
- Location: Add \imobiledevice to the end of the default location
- Click OK
- Application Settings: DLL (Empty project)
- Click Finish
- Right-click the include folder, select Include In Project
- Right-click the src folder, select Include In Project
- Set .c files to compile as C++
Advanced|Compile As: Compile as C++ Code (/TP)
- Properties|Configuration Properties|C/C++:
General|Additional Include Directories:
$(ProjectDir)\include
$(SolutionDir)\include
$(SolutionDir)\libplist\include
$(SolutionDir)\libgen
$(SolutionDir)\libusbmuxd
$SW\private\openssl\include
- Properties -> Linker -> General -> Additional library directories:
$SW\private\openssl\lib
$(OutDir)
- Properties|Linker|Input|Additional Dependencies:
libeay32.lib
ssleay32.lib
libplist.lib
libgen.lib
libusbmuxd.lib
ws2_32.lib
- Properties|C/C++|Preprocessor
Preprocessor Definitions:
ASN1_STATIC
HAVE_OPENSSL
__LITTLE_ENDIAN__
_LIB
- Project Dependencies:
libcnary
libgen
libplist
libusbmuxd
- Edit afc.c #35:
Comment out lines 35-37 (Synchapi.h is a Windows 8 include file)
- Edit userprofile.c and add at line 25:
#include <Windows.h>
- Edit libimobiledevice\include\libimobiledevice\afc.h
At #26, insert
#define AFC_API __declspec( dllexport )
Then, at each function, insert AFC_API ahead of declaration
afc_client_new
afc_client_free
afc_get_device_info
afc_read_directory
afc_get_file_info
afc_file_open
afc_file_close
afc_file_lock
afc_file_read
afc_file_write
afc_file_seek
afc_file_tell
afc_file_truncate
afc_remove_path
afc_rename_path
afc_make_directory
afc_truncate
afc_make_link
afc_set_file_time
afc_get_device_info_key
- Edit libimobiledevice\include\libimobiledevice\housearrest.h
At #26, insert
#define HOUSE_ARREST_API __declspec( dllexport )
Then, at each function, insert HOUSE_ARREST_API ahead of declaration
house_arrest_client_new
house_arrest_client_free
house_arrest_send_request
house_arrest_send_command
house_arrest_get_result
afc_client_new_from_house_arrest_client
- Edit libimobiledevice\include\libimobiledevice\installation_proxy.h
At #26, insert
#define INSTALLATION_PROXY_API __declspec( dllexport )
Then, at each function, insert INSTALLATION_PROXY_API ahead of declaration
instproxy_client_new
instproxy_client_free
instproxy_browse
instproxy_install
instproxy_upgrade
instproxy_uninstall
instproxy_lookup_archives
instproxy_archive
instproxy_restore
instproxy_remove_archive
instproxy_client_options_new
instproxy_client_options_add
instproxy_client_options_free
- Edit libimobiledevice\include\libimobiledevice\libimobiledevice.h
At #26, insert
#define LIBIMOBILEDEVICE_API __declspec( dllexport )
Then, at each function, insert LIBIMOBILEDEVICE_API ahead of declaration
idevice_set_debug_level
idevice_event_subscribe
idevice_event_unsubscribe
idevice_get_device_list
idevice_device_list_free
idevice_new
idevice_free
idevice_connect
idevice_disconnect
idevice_connection_send
idevice_connection_receive_timeout
idevice_connection_receive
idevice_get_handle
idevice_get_udid
- Edit libimobiledevice\include\libimobiledevice\lockdown.h
At #27, insert
#define LOCKDOWN_API __declspec( dllexport )
Then, at each function, insert LOCKDOWN_API ahead of declaration
lockdownd_client_new
lockdownd_client_new_with_handshake
lockdownd_client_free
lockdownd_query_type
lockdownd_get_value
lockdownd_set_value
lockdownd_remove_value
lockdownd_start_service
lockdownd_start_session
lockdownd_stop_session
lockdownd_send
lockdownd_receive
lockdownd_pair
lockdownd_validate_pair
lockdownd_unpair
lockdownd_activate
lockdownd_deactivate
lockdownd_enter_recovery
lockdownd_goodbye
lockdownd_getdevice_udid
lockdownd_get_device_name
lockdownd_get_sync_data
lockdownd_data_classes_free
lockdownd_service_descriptor_free
- If 64bits, then Right click->Properties->Configuration Manager change
Win32 to x64 for the libcnary project and check the Build checkbox
- Right-click libimobiledevice, Build.
0 errors, 60 warnings.
Copy the DLLs
-----------------
Run::
cp `find . -name '*.dll'` ~/sw/bin/

5
setup/installer/windows/notes.rst
View File

@ -540,6 +540,11 @@ Then open ChmLib.dsw in Visual Studio, change the configuration to Release
(Win32|x64) and build solution, this will generate a static library in
Release/ChmLib.lib
libimobiledevice
------------------
See libimobiledevice_notes.rst
calibre
---------

19
setup/iso_639/ar.po
View File

@ -10,15 +10,18 @@ msgstr ""
"Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
"devel@lists.alioth.debian.org>\n"
"POT-Creation-Date: 2011-11-25 14:01+0000\n"
"PO-Revision-Date: 2011-08-27 05:57+0000\n"
"Last-Translator: Mohammad Gamal <f2c2001@yahoo.com>\n"
"Language-Team: Arabic <support@arabeyes.org>\n"
"PO-Revision-Date: 2013-04-15 10:56+0000\n"
"Last-Translator: LADHARI <nader.ladhari@gmail.com>\n"
"Language-Team: awadh alghaamdi <awadh_al_ghaamdi@hotmail.com>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Launchpad-Export-Date: 2011-11-26 05:06+0000\n"
"X-Generator: Launchpad (build 14381)\n"
"X-Launchpad-Export-Date: 2013-04-16 04:37+0000\n"
"X-Generator: Launchpad (build 16564)\n"
"X-Poedit-Country: SAUDI ARABIA\n"
"Language: ar\n"
"X-Poedit-Language: Arabic\n"
"X-Poedit-SourceCharset: utf-8\n"
#. name for aaa
msgid "Ghotuo"
@ -66,7 +69,7 @@ msgstr ""
#. name for aam
msgid "Aramanik"
msgstr ""
msgstr "ارامانيك"
#. name for aan
msgid "Anambé"
@ -110,7 +113,7 @@ msgstr ""
#. name for aaz
msgid "Amarasi"
msgstr ""
msgstr "أماراسي"
#. name for aba
msgid "Abé"
@ -294,7 +297,7 @@ msgstr ""
#. name for acx
msgid "Arabic; Omani"
msgstr ""
msgstr "عماني"
#. name for acy
msgid "Arabic; Cypriot"

1128
setup/iso_639/ca.po
View File

File diff suppressed because it is too large Load Diff

2640
setup/iso_639/cs.po
View File

File diff suppressed because it is too large Load Diff

8
setup/iso_639/da.po
View File

@ -17,14 +17,14 @@ msgstr ""
"Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
"devel@lists.alioth.debian.org>\n"
"POT-Creation-Date: 2011-11-25 14:01+0000\n"
"PO-Revision-Date: 2011-09-27 18:12+0000\n"
"PO-Revision-Date: 2013-04-21 09:31+0000\n"
"Last-Translator: Kovid Goyal <Unknown>\n"
"Language-Team: Danish <dansk@klid.dk>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Launchpad-Export-Date: 2011-11-26 05:11+0000\n"
"X-Generator: Launchpad (build 14381)\n"
"X-Launchpad-Export-Date: 2013-04-22 05:23+0000\n"
"X-Generator: Launchpad (build 16567)\n"
"Language: da\n"
#. name for aaa
@ -10253,7 +10253,7 @@ msgstr ""
#. name for inh
msgid "Ingush"
msgstr "Engelsk"
msgstr "Ingush"
#. name for inj
msgid "Inga; Jungle"

10
setup/iso_639/de.po
View File

@ -18,14 +18,14 @@ msgstr ""
"Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
"devel@lists.alioth.debian.org>\n"
"POT-Creation-Date: 2011-11-25 14:01+0000\n"
"PO-Revision-Date: 2013-03-15 22:01+0000\n"
"Last-Translator: Hendrik Knackstedt <Unknown>\n"
"PO-Revision-Date: 2013-04-11 13:29+0000\n"
"Last-Translator: Simon Schütte <simonschuette@arcor.de>\n"
"Language-Team: Ubuntu German Translators\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Launchpad-Export-Date: 2013-03-16 04:55+0000\n"
"X-Generator: Launchpad (build 16532)\n"
"X-Launchpad-Export-Date: 2013-04-12 05:20+0000\n"
"X-Generator: Launchpad (build 16564)\n"
"Language: de\n"
#. name for aaa
@ -58,7 +58,7 @@ msgstr "Ambrak"
#. name for aah
msgid "Arapesh; Abu'"
msgstr "Arapesh;Abu' (Papua-Neuguinea)"
msgstr "Arapesh; Abu' (Papua-Neuguinea)"
#. name for aai
msgid "Arifama-Miniafia"

10
setup/iso_639/el.po
View File

@ -12,14 +12,14 @@ msgstr ""
"Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
"devel@lists.alioth.debian.org>\n"
"POT-Creation-Date: 2011-11-25 14:01+0000\n"
"PO-Revision-Date: 2011-09-27 17:41+0000\n"
"Last-Translator: Kovid Goyal <Unknown>\n"
"PO-Revision-Date: 2013-04-12 15:49+0000\n"
"Last-Translator: Costis Aspiotis <aspiotisk@gmail.com>\n"
"Language-Team: Greek <debian-l10n-greek@lists.debian.org>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Launchpad-Export-Date: 2011-11-26 05:17+0000\n"
"X-Generator: Launchpad (build 14381)\n"
"X-Launchpad-Export-Date: 2013-04-13 05:32+0000\n"
"X-Generator: Launchpad (build 16564)\n"
"Language: el\n"
#. name for aaa
@ -30825,7 +30825,7 @@ msgstr ""
#. name for zxx
msgid "No linguistic content"
msgstr ""
msgstr "Χωρίς γλωσσολογικό περιεχόμενο"
#. name for zyb
msgid "Zhuang; Yongbei"

404
setup/iso_639/es.po
View File

File diff suppressed because it is too large Load Diff

40
setup/iso_639/sv.po
View File

@ -30,23 +30,23 @@ msgstr ""
"Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
"devel@lists.alioth.debian.org>\n"
"POT-Creation-Date: 2011-11-25 14:01+0000\n"
"PO-Revision-Date: 2011-09-27 16:53+0000\n"
"Last-Translator: Christian Rose <menthos@menthos.com>\n"
"PO-Revision-Date: 2013-04-28 21:03+0000\n"
"Last-Translator: Merarom <Unknown>\n"
"Language-Team: Swedish <sv@li.org>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"X-Launchpad-Export-Date: 2011-11-26 05:39+0000\n"
"X-Generator: Launchpad (build 14381)\n"
"X-Launchpad-Export-Date: 2013-04-29 04:38+0000\n"
"X-Generator: Launchpad (build 16580)\n"
"Language: sv\n"
#. name for aaa
msgid "Ghotuo"
msgstr ""
msgstr "Ghotuo"
#. name for aab
msgid "Alumu-Tesu"
msgstr ""
msgstr "Alumu-Tesu"
#. name for aac
msgid "Ari"
@ -58,7 +58,7 @@ msgstr ""
#. name for aae
msgid "Albanian; Arbëreshë"
msgstr ""
msgstr "Albanska; Arbëreshë"
#. name for aaf
msgid "Aranadan"
@ -78,7 +78,7 @@ msgstr ""
#. name for aak
msgid "Ankave"
msgstr ""
msgstr "Ankave"
#. name for aal
msgid "Afade"
@ -94,7 +94,7 @@ msgstr ""
#. name for aao
msgid "Arabic; Algerian Saharan"
msgstr ""
msgstr "Arabiska;algeriska Sahara"
#. name for aap
msgid "Arára; Pará"
@ -114,7 +114,7 @@ msgstr ""
#. name for aat
msgid "Albanian; Arvanitika"
msgstr ""
msgstr "Albanska; Arvanitika"
#. name for aau
msgid "Abau"
@ -218,7 +218,7 @@ msgstr ""
#. name for abv
msgid "Arabic; Baharna"
msgstr ""
msgstr "Arabiska; Baharna"
#. name for abw
msgid "Pal"
@ -311,7 +311,7 @@ msgstr ""
#. name for acw
msgid "Arabic; Hijazi"
msgstr ""
msgstr "Arabiska; Hijazi"
#. name for acx
msgid "Arabic; Omani"
@ -319,7 +319,7 @@ msgstr ""
#. name for acy
msgid "Arabic; Cypriot"
msgstr ""
msgstr "Arabiska; Cypriotiska"
#. name for acz
msgid "Acheron"
@ -343,7 +343,7 @@ msgstr ""
#. name for adf
msgid "Arabic; Dhofari"
msgstr ""
msgstr "Arabiska; Dhofari"
#. name for adg
msgid "Andegerebinha"
@ -419,11 +419,11 @@ msgstr ""
#. name for aeb
msgid "Arabic; Tunisian"
msgstr ""
msgstr "Arabiska; Tunisiska"
#. name for aec
msgid "Arabic; Saidi"
msgstr ""
msgstr "Arabiska,; Saidi"
#. name for aed
msgid "Argentine Sign Language"
@ -479,7 +479,7 @@ msgstr ""
#. name for afb
msgid "Arabic; Gulf"
msgstr ""
msgstr "Arabiska,; Gulf"
#. name for afd
msgid "Andai"
@ -803,7 +803,7 @@ msgstr ""
#. name for ajt
msgid "Arabic; Judeo-Tunisian"
msgstr ""
msgstr "Arabiska; judisk-tunisiska"
#. name for aju
msgid "Arabic; Judeo-Moroccan"
@ -963,7 +963,7 @@ msgstr ""
#. name for aln
msgid "Albanian; Gheg"
msgstr ""
msgstr "Albanska; Gheg"
#. name for alo
msgid "Larike-Wakasihu"
@ -9431,7 +9431,7 @@ msgstr ""
#. name for hlb
msgid "Halbi"
msgstr ""
msgstr "Halbi"
#. name for hld
msgid "Halang Doan"

11
src/calibre/constants.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = u'calibre'
numeric_version = (0, 9, 26)
numeric_version = (0, 9, 29)
__version__ = u'.'.join(map(unicode, numeric_version))
__author__ = u"Kovid Goyal <kovid@kovidgoyal.net>"
@ -29,7 +29,7 @@ isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None
ispy3 = sys.version_info.major > 2
isxp = iswindows and sys.getwindowsversion().major < 6
is64bit = sys.maxsize > (1 << 32)
isworker = os.environ.has_key('CALIBRE_WORKER') or os.environ.has_key('CALIBRE_SIMPLE_WORKER')
isworker = 'CALIBRE_WORKER' in os.environ or 'CALIBRE_SIMPLE_WORKER' in os.environ
if isworker:
os.environ.pop('CALIBRE_FORCE_ANSI', None)
@ -58,7 +58,8 @@ def get_osx_version():
return _osx_ver
filesystem_encoding = sys.getfilesystemencoding()
if filesystem_encoding is None: filesystem_encoding = 'utf-8'
if filesystem_encoding is None:
filesystem_encoding = 'utf-8'
else:
try:
if codecs.lookup(filesystem_encoding).name == 'ascii':
@ -85,7 +86,7 @@ def _get_cache_dir():
confcache = os.path.join(config_dir, u'caches')
if isportable:
return confcache
if os.environ.has_key('CALIBRE_CACHE_DIRECTORY'):
if 'CALIBRE_CACHE_DIRECTORY' in os.environ:
return os.path.abspath(os.environ['CALIBRE_CACHE_DIRECTORY'])
if iswindows:
@ -184,7 +185,7 @@ if plugins is None:
CONFIG_DIR_MODE = 0700
if os.environ.has_key('CALIBRE_CONFIG_DIRECTORY'):
if 'CALIBRE_CONFIG_DIRECTORY' in os.environ:
config_dir = os.path.abspath(os.environ['CALIBRE_CONFIG_DIRECTORY'])
elif iswindows:
if plugins['winutil'][0] is None:

73
src/calibre/customize/builtins.py
View File

@ -68,7 +68,7 @@ class TXT2TXTZ(FileTypePlugin):
images.append(path)
# Markdown inline
for m in re.finditer(ur'(?mu)\!\[([^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*)\]\s*\((?P<path>[^\)]*)\)', txt):
for m in re.finditer(ur'(?mu)\!\[([^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*)\]\s*\((?P<path>[^\)]*)\)', txt): # noqa
path = m.group('path')
if path and not os.path.isabs(path) and guess_type(path)[0] in OEB_IMAGES and os.path.exists(os.path.join(base_dir, path)):
images.append(path)
@ -78,7 +78,7 @@ class TXT2TXTZ(FileTypePlugin):
for m in re.finditer(ur'(?mu)^(\ ?\ ?\ ?)\[(?P<id>[^\]]*)\]:\s*(?P<path>[^\s]*)$', txt):
if m.group('id') and m.group('path'):
refs[m.group('id')] = m.group('path')
for m in re.finditer(ur'(?mu)\!\[([^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*)\]\s*\[(?P<id>[^\]]*)\]', txt):
for m in re.finditer(ur'(?mu)\!\[([^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*(\[[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*\])*[^\]\[]*)\]\s*\[(?P<id>[^\]]*)\]', txt): # noqa
path = refs.get(m.group('id'), None)
if path and not os.path.isabs(path) and guess_type(path)[0] in OEB_IMAGES and os.path.exists(os.path.join(base_dir, path)):
images.append(path)
@ -414,7 +414,7 @@ class ZipMetadataReader(MetadataReaderPlugin):
from calibre.ebooks.metadata.zip import get_metadata
return get_metadata(stream)
plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
plugins += [x for x in list(locals().values()) if isinstance(x, type) and
x.__name__.endswith('MetadataReader')]
# }}}
@ -527,7 +527,7 @@ class TXTZMetadataWriter(MetadataWriterPlugin):
from calibre.ebooks.metadata.extz import set_metadata
set_metadata(stream, mi)
plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
plugins += [x for x in list(locals().values()) if isinstance(x, type) and
x.__name__.endswith('MetadataWriter')]
# }}}
@ -630,7 +630,6 @@ plugins += input_profiles + output_profiles
# }}}
# Device driver plugins {{{
from calibre.devices.apple.driver import ITUNES
from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA
from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK
from calibre.devices.cybook.driver import CYBOOK, ORIZON
@ -644,6 +643,7 @@ from calibre.devices.jetbook.driver import (JETBOOK, MIBUK, JETBOOK_MINI,
JETBOOK_COLOR)
from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX,
KINDLE_FIRE)
from calibre.devices.apple.driver import ITUNES
from calibre.devices.nook.driver import NOOK, NOOK_COLOR
from calibre.devices.prs505.driver import PRS505
from calibre.devices.prst1.driver import PRST1
@ -1263,7 +1263,7 @@ class StoreAmazonUKKindleStore(StoreBase):
class StoreArchiveOrgStore(StoreBase):
name = 'Archive.org'
description = u'An Internet library offering permanent access for researchers, historians, scholars, people with disabilities, and the general public to historical collections that exist in digital format.'
description = u'An Internet library offering permanent access for researchers, historians, scholars, people with disabilities, and the general public to historical collections that exist in digital format.' # noqa
actual_plugin = 'calibre.gui2.store.stores.archive_org_plugin:ArchiveOrgStore'
drm_free_only = True
@ -1290,7 +1290,7 @@ class StoreBNStore(StoreBase):
class StoreBeamEBooksDEStore(StoreBase):
name = 'Beam EBooks DE'
author = 'Charles Haley'
description = u'Bei uns finden Sie: Tausende deutschsprachige eBooks; Alle eBooks ohne hartes DRM; PDF, ePub und Mobipocket Format; Sofortige Verfügbarkeit - 24 Stunden am Tag; Günstige Preise; eBooks für viele Lesegeräte, PC,Mac und Smartphones; Viele Gratis eBooks'
description = u'Bei uns finden Sie: Tausende deutschsprachige eBooks; Alle eBooks ohne hartes DRM; PDF, ePub und Mobipocket Format; Sofortige Verfügbarkeit - 24 Stunden am Tag; Günstige Preise; eBooks für viele Lesegeräte, PC,Mac und Smartphones; Viele Gratis eBooks' # noqa
actual_plugin = 'calibre.gui2.store.stores.beam_ebooks_de_plugin:BeamEBooksDEStore'
drm_free_only = True
@ -1310,7 +1310,7 @@ class StoreBiblioStore(StoreBase):
class StoreBookotekaStore(StoreBase):
name = 'Bookoteka'
author = u'Tomasz Długosz'
description = u'E-booki w Bookotece dostępne są w formacie EPUB oraz PDF. Publikacje sprzedawane w Bookotece są objęte prawami autorskimi. Zobowiązaliśmy się chronić te prawa, ale bez ograniczania dostępu do książki użytkownikowi, który nabył ją w legalny sposób. Dlatego też Bookoteka stosuje tak zwany „watermarking transakcyjny” czyli swego rodzaju znaki wodne.'
description = u'E-booki w Bookotece dostępne są w formacie EPUB oraz PDF. Publikacje sprzedawane w Bookotece są objęte prawami autorskimi. Zobowiązaliśmy się chronić te prawa, ale bez ograniczania dostępu do książki użytkownikowi, który nabył ją w legalny sposób. Dlatego też Bookoteka stosuje tak zwany „watermarking transakcyjny” czyli swego rodzaju znaki wodne.' # noqa
actual_plugin = 'calibre.gui2.store.stores.bookoteka_plugin:BookotekaStore'
drm_free_only = True
@ -1329,7 +1329,7 @@ class StoreChitankaStore(StoreBase):
class StoreDieselEbooksStore(StoreBase):
name = 'Diesel eBooks'
description = u'Instant access to over 2.4 million titles from hundreds of publishers including Harlequin, HarperCollins, John Wiley & Sons, McGraw-Hill, Simon & Schuster and Random House.'
description = u'Instant access to over 2.4 million titles from hundreds of publishers including Harlequin, HarperCollins, John Wiley & Sons, McGraw-Hill, Simon & Schuster and Random House.' # noqa
actual_plugin = 'calibre.gui2.store.stores.diesel_ebooks_plugin:DieselEbooksStore'
headquarters = 'US'
@ -1358,7 +1358,7 @@ class StoreEbookpointStore(StoreBase):
class StoreEbookscomStore(StoreBase):
name = 'eBooks.com'
description = u'Sells books in multiple electronic formats in all categories. Technical infrastructure is cutting edge, robust and scalable, with servers in the US and Europe.'
description = u'Sells books in multiple electronic formats in all categories. Technical infrastructure is cutting edge, robust and scalable, with servers in the US and Europe.' # noqa
actual_plugin = 'calibre.gui2.store.stores.ebooks_com_plugin:EbookscomStore'
headquarters = 'US'
@ -1386,7 +1386,7 @@ class StoreEbooksGratuitsStore(StoreBase):
class StoreEHarlequinStore(StoreBase):
name = 'eHarlequin'
description = u'A global leader in series romance and one of the world\'s leading publishers of books for women. Offers women a broad range of reading from romance to bestseller fiction, from young adult novels to erotic literature, from nonfiction to fantasy, from African-American novels to inspirational romance, and more.'
description = u'A global leader in series romance and one of the world\'s leading publishers of books for women. Offers women a broad range of reading from romance to bestseller fiction, from young adult novels to erotic literature, from nonfiction to fantasy, from African-American novels to inspirational romance, and more.' # noqa
actual_plugin = 'calibre.gui2.store.stores.eharlequin_plugin:EHarlequinStore'
headquarters = 'CA'
@ -1406,7 +1406,7 @@ class StoreEKnigiStore(StoreBase):
class StoreEmpikStore(StoreBase):
name = 'Empik'
author = u'Tomasz Długosz'
description = u'Empik to marka o unikalnym dziedzictwie i legendarne miejsce, dawne “okno na świat”. Jest obecna w polskim krajobrazie kulturalnym od 60 lat (wcześniej jako Kluby Międzynarodowej Prasy i Książki).'
description = u'Empik to marka o unikalnym dziedzictwie i legendarne miejsce, dawne “okno na świat”. Jest obecna w polskim krajobrazie kulturalnym od 60 lat (wcześniej jako Kluby Międzynarodowej Prasy i Książki).' # noqa
actual_plugin = 'calibre.gui2.store.stores.empik_plugin:EmpikStore'
headquarters = 'PL'
@ -1425,7 +1425,7 @@ class StoreEscapeMagazineStore(StoreBase):
class StoreFeedbooksStore(StoreBase):
name = 'Feedbooks'
description = u'Feedbooks is a cloud publishing and distribution service, connected to a large ecosystem of reading systems and social networks. Provides a variety of genres from independent and classic books.'
description = u'Feedbooks is a cloud publishing and distribution service, connected to a large ecosystem of reading systems and social networks. Provides a variety of genres from independent and classic books.' # noqa
actual_plugin = 'calibre.gui2.store.stores.feedbooks_plugin:FeedbooksStore'
headquarters = 'FR'
@ -1448,11 +1448,10 @@ class StoreGoogleBooksStore(StoreBase):
headquarters = 'US'
formats = ['EPUB', 'PDF', 'TXT']
affiliate = True
class StoreGutenbergStore(StoreBase):
name = 'Project Gutenberg'
description = u'The first producer of free ebooks. Free in the United States because their copyright has expired. They may not be free of copyright in other countries. Readers outside of the United States must check the copyright laws of their countries before downloading or redistributing our ebooks.'
description = u'The first producer of free ebooks. Free in the United States because their copyright has expired. They may not be free of copyright in other countries. Readers outside of the United States must check the copyright laws of their countries before downloading or redistributing our ebooks.' # noqa
actual_plugin = 'calibre.gui2.store.stores.gutenberg_plugin:GutenbergStore'
drm_free_only = True
@ -1461,13 +1460,23 @@ class StoreGutenbergStore(StoreBase):
class StoreKoboStore(StoreBase):
name = 'Kobo'
description = u'With over 2.3 million eBooks to browse we have engaged readers in over 200 countries in Kobo eReading. Our eBook listings include New York Times Bestsellers, award winners, classics and more!'
description = u'With over 2.3 million eBooks to browse we have engaged readers in over 200 countries in Kobo eReading. Our eBook listings include New York Times Bestsellers, award winners, classics and more!' # noqa
actual_plugin = 'calibre.gui2.store.stores.kobo_plugin:KoboStore'
headquarters = 'CA'
formats = ['EPUB']
affiliate = True
class StoreKoobeStore(StoreBase):
name = 'Koobe'
author = u'Tomasz Długosz'
description = u'Księgarnia internetowa oferuje ebooki (książki elektroniczne) w postaci plików epub, mobi i pdf.'
actual_plugin = 'calibre.gui2.store.stores.koobe_plugin:KoobeStore'
drm_free_only = True
headquarters = 'PL'
formats = ['EPUB', 'MOBI', 'PDF']
class StoreLegimiStore(StoreBase):
name = 'Legimi'
author = u'Tomasz Długosz'
@ -1540,7 +1549,7 @@ class StoreNextoStore(StoreBase):
class StoreNookUKStore(StoreBase):
name = 'Nook UK'
author = 'John Schember'
description = u'Barnes & Noble S.à r.l, a subsidiary of Barnes & Noble, Inc., a leading retailer of content, digital media and educational products, is proud to bring the award-winning NOOK® reading experience and a leading digital bookstore to the UK.'
description = u'Barnes & Noble S.à r.l, a subsidiary of Barnes & Noble, Inc., a leading retailer of content, digital media and educational products, is proud to bring the award-winning NOOK® reading experience and a leading digital bookstore to the UK.' # noqa
actual_plugin = 'calibre.gui2.store.stores.nook_uk_plugin:NookUKStore'
headquarters = 'UK'
@ -1617,7 +1626,7 @@ class StoreVirtualoStore(StoreBase):
class StoreWaterstonesUKStore(StoreBase):
name = 'Waterstones UK'
author = 'Charles Haley'
description = u'Waterstone\'s mission is to be the leading Bookseller on the High Street and online providing customers the widest choice, great value and expert advice from a team passionate about Bookselling.'
description = u'Waterstone\'s mission is to be the leading Bookseller on the High Street and online providing customers the widest choice, great value and expert advice from a team passionate about Bookselling.' # noqa
actual_plugin = 'calibre.gui2.store.stores.waterstones_uk_plugin:WaterstonesUKStore'
headquarters = 'UK'
@ -1687,6 +1696,7 @@ plugins += [
StoreGoogleBooksStore,
StoreGutenbergStore,
StoreKoboStore,
StoreKoobeStore,
StoreLegimiStore,
StoreLibreDEStore,
StoreLitResStore,
@ -1715,6 +1725,28 @@ plugins += [
if __name__ == '__main__':
# Test load speed
import subprocess, textwrap
try:
subprocess.check_call(['python', '-c', textwrap.dedent(
'''
import init_calibre # noqa
def doit():
import calibre.customize.builtins as b # noqa
def show_stats():
from pstats import Stats
s = Stats('/tmp/calibre_stats')
s.sort_stats('cumulative')
s.print_stats(30)
import cProfile
cProfile.run('doit()', '/tmp/calibre_stats')
show_stats()
'''
)])
except subprocess.CalledProcessError:
raise SystemExit(1)
try:
subprocess.check_call(['python', '-c', textwrap.dedent(
'''
@ -1727,7 +1759,10 @@ if __name__ == '__main__':
for x in ('lxml', 'calibre.ebooks.BeautifulSoup', 'uuid',
'calibre.utils.terminal', 'calibre.utils.magick', 'PIL', 'Image',
'sqlite3', 'mechanize', 'httplib', 'xml'):
'sqlite3', 'mechanize', 'httplib', 'xml', 'inspect', 'urllib',
'calibre.utils.date', 'calibre.utils.config', 'platform',
'calibre.utils.zipfile', 'calibre.utils.formatter',
):
if x in sys.modules:
ret = 1
print (x, 'has been loaded by a plugin')

45
src/calibre/db/__init__.py
View File

@ -9,6 +9,50 @@ __docformat__ = 'restructuredtext en'
SPOOL_SIZE = 30*1024*1024
def _get_next_series_num_for_list(series_indices):
from calibre.utils.config_base import tweaks
from math import ceil, floor
if not series_indices:
if isinstance(tweaks['series_index_auto_increment'], (int, float)):
return float(tweaks['series_index_auto_increment'])
return 1.0
series_indices = [x[0] for x in series_indices]
if tweaks['series_index_auto_increment'] == 'next':
return floor(series_indices[-1]) + 1
if tweaks['series_index_auto_increment'] == 'first_free':
for i in xrange(1, 10000):
if i not in series_indices:
return i
# really shouldn't get here.
if tweaks['series_index_auto_increment'] == 'next_free':
for i in xrange(int(ceil(series_indices[0])), 10000):
if i not in series_indices:
return i
# really shouldn't get here.
if tweaks['series_index_auto_increment'] == 'last_free':
for i in xrange(int(ceil(series_indices[-1])), 0, -1):
if i not in series_indices:
return i
return series_indices[-1] + 1
if isinstance(tweaks['series_index_auto_increment'], (int, float)):
return float(tweaks['series_index_auto_increment'])
return 1.0
def _get_series_values(val):
import re
series_index_pat = re.compile(r'(.*)\s+\[([.0-9]+)\]$')
if not val:
return (val, None)
match = series_index_pat.match(val.strip())
if match is not None:
idx = match.group(2)
try:
idx = float(idx)
return (match.group(1).strip(), idx)
except:
pass
return (val, None)
'''
Rewrite of the calibre database backend.
@ -68,4 +112,5 @@ Various things that require other things before they can be migrated:
libraries/switching/on calibre startup.
3. From refresh in the legacy interface: Rember to flush the composite
column template cache.
4. Replace the metadatabackup thread with the new implementation when using the new backend.
'''

131
src/calibre/db/backend.py
View File

@ -25,6 +25,7 @@ from calibre.utils.config import to_json, from_json, prefs, tweaks
from calibre.utils.date import utcfromtimestamp, parse_date
from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename,
WindowsAtomicFolderMove)
from calibre.utils.magick.draw import save_cover_data_to
from calibre.utils.recycle_bin import delete_tree
from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable,
@ -41,8 +42,7 @@ Differences in semantics from pysqlite:
'''
class DynamicFilter(object): # {{{
class DynamicFilter(object): # {{{
'No longer used, present for legacy compatibility'
@ -57,7 +57,7 @@ class DynamicFilter(object): # {{{
self.ids = frozenset(ids)
# }}}
class DBPrefs(dict): # {{{
class DBPrefs(dict): # {{{
'Store preferences as key:value pairs in the db'
@ -114,9 +114,10 @@ class DBPrefs(dict): # {{{
return default
def set_namespaced(self, namespace, key, val):
if u':' in key: raise KeyError('Colons are not allowed in keys')
if u':' in namespace: raise KeyError('Colons are not allowed in'
' the namespace')
if u':' in key:
raise KeyError('Colons are not allowed in keys')
if u':' in namespace:
raise KeyError('Colons are not allowed in the namespace')
key = u'namespaced:%s:%s'%(namespace, key)
self[key] = val
@ -170,7 +171,8 @@ def pynocase(one, two, encoding='utf-8'):
return cmp(one.lower(), two.lower())
def _author_to_author_sort(x):
if not x: return ''
if not x:
return ''
return author_to_author_sort(x.replace('|', ','))
def icu_collator(s1, s2):
@ -239,9 +241,9 @@ def AumSortedConcatenate():
# }}}
class Connection(apsw.Connection): # {{{
class Connection(apsw.Connection): # {{{
BUSY_TIMEOUT = 2000 # milliseconds
BUSY_TIMEOUT = 2000 # milliseconds
def __init__(self, path):
apsw.Connection.__init__(self, path)
@ -257,7 +259,7 @@ class Connection(apsw.Connection): # {{{
self.createscalarfunction('title_sort', title_sort, 1)
self.createscalarfunction('author_to_author_sort',
_author_to_author_sort, 1)
self.createscalarfunction('uuid4', lambda : str(uuid.uuid4()),
self.createscalarfunction('uuid4', lambda: str(uuid.uuid4()),
0)
# Dummy functions for dynamically created filters
@ -305,7 +307,8 @@ class DB(object):
# Initialize database {{{
def __init__(self, library_path, default_prefs=None, read_only=False):
def __init__(self, library_path, default_prefs=None, read_only=False,
restore_all_prefs=False, progress_callback=lambda x, y:True):
try:
if isbytestring(library_path):
library_path = library_path.decode(filesystem_encoding)
@ -376,23 +379,27 @@ class DB(object):
UPDATE authors SET sort=author_to_author_sort(name) WHERE sort IS NULL;
''')
self.initialize_prefs(default_prefs)
self.initialize_prefs(default_prefs, restore_all_prefs, progress_callback)
self.initialize_custom_columns()
self.initialize_tables()
def initialize_prefs(self, default_prefs): # {{{
def initialize_prefs(self, default_prefs, restore_all_prefs, progress_callback): # {{{
self.prefs = DBPrefs(self)
if default_prefs is not None and not self._exists:
progress_callback(None, len(default_prefs))
# Only apply default prefs to a new database
for key in default_prefs:
for i, key in enumerate(default_prefs):
# be sure that prefs not to be copied are listed below
if key not in frozenset(['news_to_be_synced']):
if restore_all_prefs or key not in frozenset(['news_to_be_synced']):
self.prefs[key] = default_prefs[key]
progress_callback(_('restored preference ') + key, i+1)
if 'field_metadata' in default_prefs:
fmvals = [f for f in default_prefs['field_metadata'].values()
if f['is_custom']]
for f in fmvals:
progress_callback(None, len(fmvals))
for i, f in enumerate(fmvals):
progress_callback(_('creating custom column ') + f['label'], i)
self.create_custom_column(f['label'], f['name'],
f['datatype'],
(f['is_multiple'] is not None and
@ -421,6 +428,8 @@ class DB(object):
('uuid', False), ('comments', True), ('id', False), ('pubdate', False),
('last_modified', False), ('size', False), ('languages', False),
]
defs['virtual_libraries'] = {}
defs['virtual_lib_on_startup'] = defs['cs_virtual_lib_on_startup'] = ''
# Migrate the bool tristate tweak
defs['bools_are_tristate'] = \
@ -469,6 +478,24 @@ class DB(object):
except:
pass
# migrate the gui_restriction preference to a virtual library
gr_pref = self.prefs.get('gui_restriction', None)
if gr_pref:
virt_libs = self.prefs.get('virtual_libraries', {})
virt_libs[gr_pref] = 'search:"' + gr_pref + '"'
self.prefs['virtual_libraries'] = virt_libs
self.prefs['gui_restriction'] = ''
self.prefs['virtual_lib_on_startup'] = gr_pref
# migrate the cs_restriction preference to a virtual library
gr_pref = self.prefs.get('cs_restriction', None)
if gr_pref:
virt_libs = self.prefs.get('virtual_libraries', {})
virt_libs[gr_pref] = 'search:"' + gr_pref + '"'
self.prefs['virtual_libraries'] = virt_libs
self.prefs['cs_restriction'] = ''
self.prefs['cs_virtual_lib_on_startup'] = gr_pref
# Rename any user categories with names that differ only in case
user_cats = self.prefs.get('user_categories', [])
catmap = {}
@ -493,7 +520,7 @@ class DB(object):
self.prefs.set('user_categories', user_cats)
# }}}
def initialize_custom_columns(self): # {{{
def initialize_custom_columns(self): # {{{
with self.conn:
# Delete previously marked custom columns
for record in self.conn.get(
@ -634,11 +661,11 @@ class DB(object):
self.custom_data_adapters = {
'float': adapt_number,
'int': adapt_number,
'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
'bool': adapt_bool,
'int': adapt_number,
'rating':lambda x,d: x if x is None else min(10., max(0., float(x))),
'bool': adapt_bool,
'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
'datetime' : adapt_datetime,
'datetime': adapt_datetime,
'text':adapt_text,
'series':adapt_text,
'enumeration': adapt_enum
@ -661,7 +688,7 @@ class DB(object):
# }}}
def initialize_tables(self): # {{{
def initialize_tables(self): # {{{
tables = self.tables = {}
for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
'timestamp', 'pubdate', 'uuid', 'path', 'cover',
@ -690,11 +717,13 @@ class DB(object):
tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3,
'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17,
'au_map':18, 'last_modified':19, 'identifiers':20}
self.FIELD_MAP = {
'id':0, 'title':1, 'authors':2, 'timestamp':3, 'size':4,
'rating':5, 'tags':6, 'comments':7, 'series':8, 'publisher':9,
'series_index':10, 'sort':11, 'author_sort':12, 'formats':13,
'path':14, 'pubdate':15, 'uuid':16, 'cover':17, 'au_map':18,
'last_modified':19, 'identifiers':20, 'languages':21,
}
for k,v in self.FIELD_MAP.iteritems():
self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@ -740,6 +769,8 @@ class DB(object):
self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False)
self.FIELD_MAP['marked'] = base = base+1
self.field_metadata.set_field_record_index('marked', base, prefer_custom=False)
self.FIELD_MAP['series_sort'] = base = base+1
self.field_metadata.set_field_record_index('series_sort', base, prefer_custom=False)
# }}}
@ -753,6 +784,11 @@ class DB(object):
self._conn = Connection(self.dbpath)
return self._conn
def close(self):
if self._conn is not None:
self._conn.close()
del self._conn
@dynamic_property
def user_version(self):
doc = 'The user version of this database'
@ -866,8 +902,8 @@ class DB(object):
Read all data from the db into the python in-memory tables
'''
with self.conn: # Use a single transaction, to ensure nothing modifies
# the db while we are reading
with self.conn: # Use a single transaction, to ensure nothing modifies
# the db while we are reading
for table in self.tables.itervalues():
try:
table.read(self)
@ -885,7 +921,7 @@ class DB(object):
return fmt_path
try:
candidates = glob.glob(os.path.join(path, '*'+fmt))
except: # If path contains strange characters this throws an exc
except: # If path contains strange characters this throws an exc
candidates = []
if fmt and candidates and os.path.exists(candidates[0]):
shutil.copyfile(candidates[0], fmt_path)
@ -938,6 +974,23 @@ class DB(object):
return True
return False
def set_cover(self, book_id, path, data):
path = os.path.abspath(os.path.join(self.library_path, path))
if not os.path.exists(path):
os.makedirs(path)
path = os.path.join(path, 'cover.jpg')
if callable(getattr(data, 'save', None)):
from calibre.gui2 import pixmap_to_data
data = pixmap_to_data(data)
else:
if callable(getattr(data, 'read', None)):
data = data.read()
try:
save_cover_data_to(data, path)
except (IOError, OSError):
time.sleep(0.2)
save_cover_data_to(data, path)
def copy_format_to(self, book_id, fmt, fname, path, dest,
windows_atomic_move=None, use_hardlink=False):
path = self.format_abspath(book_id, fmt, fname, path)
@ -954,7 +1007,7 @@ class DB(object):
if path != dest:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
windows_atomic_move.copy_path_to(path, dest)
else:
@ -970,7 +1023,7 @@ class DB(object):
try:
os.rename(path, dest)
except:
pass # Nothing too catastrophic happened, the cases mismatch, that's all
pass # Nothing too catastrophic happened, the cases mismatch, that's all
else:
if use_hardlink:
try:
@ -1021,7 +1074,7 @@ class DB(object):
if not os.path.exists(tpath):
os.makedirs(tpath)
if source_ok: # Migrate existing files
if source_ok: # Migrate existing files
dest = os.path.join(tpath, 'cover.jpg')
self.copy_cover_to(current_path, dest,
windows_atomic_move=wam, use_hardlink=True)
@ -1064,8 +1117,18 @@ class DB(object):
os.rename(os.path.join(curpath, oldseg),
os.path.join(curpath, newseg))
except:
break # Fail silently since nothing catastrophic has happened
break # Fail silently since nothing catastrophic has happened
curpath = os.path.join(curpath, newseg)
def write_backup(self, path, raw):
path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf'))
with lopen(path, 'wb') as f:
f.write(raw)
def read_backup(self, path):
path = os.path.abspath(os.path.join(self.library_path, path, 'metadata.opf'))
with lopen(path, 'rb') as f:
return f.read()
# }}}

115
src/calibre/db/backup.py Normal file
View File

@ -0,0 +1,115 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import weakref, traceback
from threading import Thread, Event
from calibre import prints
from calibre.ebooks.metadata.opf2 import metadata_to_opf
class Abort(Exception):
pass
class MetadataBackup(Thread):
'''
Continuously backup changed metadata into OPF files
in the book directory. This class runs in its own
thread.
'''
def __init__(self, db, interval=2, scheduling_interval=0.1):
Thread.__init__(self)
self.daemon = True
self._db = weakref.ref(db)
self.stop_running = Event()
self.interval = interval
self.scheduling_interval = scheduling_interval
@property
def db(self):
ans = self._db()
if ans is None:
raise Abort()
return ans
def stop(self):
self.stop_running.set()
def wait(self, interval):
if self.stop_running.wait(interval):
raise Abort()
def run(self):
while not self.stop_running.is_set():
try:
self.wait(self.interval)
self.do_one()
except Abort:
break
def do_one(self):
try:
book_id = self.db.get_a_dirtied_book()
if book_id is None:
return
except Abort:
raise
except:
# Happens during interpreter shutdown
return
self.wait(0)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'again, giving up')
traceback.print_exc()
return
if mi is None:
self.db.clear_dirtied(book_id, sequence)
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The wait makes such an event
self.wait(self.scheduling_interval)
try:
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', book_id)
traceback.print_exc()
return
self.wait(self.scheduling_interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'once')
self.wait(self.interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'again, giving up')
return
self.db.clear_dirtied(book_id, sequence)
def break_cycles(self):
# Legacy compatibility
pass

369
src/calibre/db/cache.py
View File

@ -7,7 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os, traceback
import os, traceback, random
from io import BytesIO
from collections import defaultdict
from functools import wraps, partial
@ -15,17 +15,19 @@ from functools import wraps, partial
from calibre.constants import iswindows
from calibre.db import SPOOL_SIZE
from calibre.db.categories import get_categories
from calibre.db.locking import create_locks, RecordLock
from calibre.db.locking import create_locks
from calibre.db.errors import NoSuchFormat
from calibre.db.fields import create_field
from calibre.db.search import Search
from calibre.db.tables import VirtualTable
from calibre.db.write import get_series_values
from calibre.db.lazy import FormatMetadata, FormatsList
from calibre.ebooks.metadata import string_to_authors
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ptempfile import (base_dir, PersistentTemporaryFile,
SpooledTemporaryFile)
from calibre.utils.date import now
from calibre.utils.date import now as nowf
from calibre.utils.icu import sort_key
def api(f):
@ -57,9 +59,10 @@ class Cache(object):
self.fields = {}
self.composites = set()
self.read_lock, self.write_lock = create_locks()
self.record_lock = RecordLock(self.read_lock)
self.format_metadata_cache = defaultdict(dict)
self.formatter_template_cache = {}
self.dirtied_cache = {}
self.dirtied_sequence = 0
self._search_api = Search(self.field_metadata.get_search_terms())
# Implement locking for all simple read/write API methods
@ -78,17 +81,18 @@ class Cache(object):
self.initialize_dynamic()
@write_api
def initialize_dynamic(self):
# Reconstruct the user categories, putting them into field_metadata
# Assumption is that someone else will fix them if they change.
self.field_metadata.remove_dynamic_categories()
for user_cat in sorted(self.pref('user_categories', {}).iterkeys(), key=sort_key):
cat_name = '@' + user_cat # add the '@' to avoid name collision
for user_cat in sorted(self._pref('user_categories', {}).iterkeys(), key=sort_key):
cat_name = '@' + user_cat # add the '@' to avoid name collision
self.field_metadata.add_user_category(label=cat_name, name=user_cat)
# add grouped search term user categories
muc = frozenset(self.pref('grouped_search_make_user_categories', []))
for cat in sorted(self.pref('grouped_search_terms', {}).iterkeys(), key=sort_key):
muc = frozenset(self._pref('grouped_search_make_user_categories', []))
for cat in sorted(self._pref('grouped_search_terms', {}).iterkeys(), key=sort_key):
if cat in muc:
# There is a chance that these can be duplicates of an existing
# user category. Print the exception and continue.
@ -102,15 +106,33 @@ class Cache(object):
# self.field_metadata.add_search_category(label='search', name=_('Searches'))
self.field_metadata.add_grouped_search_terms(
self.pref('grouped_search_terms', {}))
self._pref('grouped_search_terms', {}))
self._search_api.change_locations(self.field_metadata.get_search_terms())
self.dirtied_cache = {x:i for i, (x,) in enumerate(
self.backend.conn.execute('SELECT book FROM metadata_dirtied'))}
if self.dirtied_cache:
self.dirtied_sequence = max(self.dirtied_cache.itervalues())+1
@write_api
def initialize_template_cache(self):
self.formatter_template_cache = {}
@write_api
def refresh(self):
self._initialize_template_cache()
for field in self.fields.itervalues():
if hasattr(field, 'clear_cache'):
field.clear_cache() # Clear the composite cache
if hasattr(field, 'table'):
field.table.read(self.backend) # Reread data from metadata.db
@property
def field_metadata(self):
return self.backend.field_metadata
def _get_metadata(self, book_id, get_user_categories=True): # {{{
def _get_metadata(self, book_id, get_user_categories=True): # {{{
mi = Metadata(None, template_cache=self.formatter_template_cache)
author_ids = self._field_ids_for('authors', book_id)
aut_list = [self._author_data(i) for i in author_ids]
@ -131,7 +153,7 @@ class Cache(object):
mi.author_link_map = aul
mi.comments = self._field_for('comments', book_id)
mi.publisher = self._field_for('publisher', book_id)
n = now()
n = nowf()
mi.timestamp = self._field_for('timestamp', book_id, default_value=n)
mi.pubdate = self._field_for('pubdate', book_id, default_value=n)
mi.uuid = self._field_for('uuid', book_id,
@ -395,16 +417,19 @@ class Cache(object):
'''
if as_file:
ret = SpooledTemporaryFile(SPOOL_SIZE)
if not self.copy_cover_to(book_id, ret): return
if not self.copy_cover_to(book_id, ret):
return
ret.seek(0)
elif as_path:
pt = PersistentTemporaryFile('_dbcover.jpg')
with pt:
if not self.copy_cover_to(book_id, pt): return
if not self.copy_cover_to(book_id, pt):
return
ret = pt.name
else:
buf = BytesIO()
if not self.copy_cover_to(book_id, buf): return
if not self.copy_cover_to(book_id, buf):
return
ret = buf.getvalue()
if as_image:
from PyQt4.Qt import QImage
@ -413,7 +438,7 @@ class Cache(object):
ret = i
return ret
@api
@read_api
def copy_cover_to(self, book_id, dest, use_hardlink=False):
'''
Copy the cover to the file like object ``dest``. Returns False
@ -422,17 +447,15 @@ class Cache(object):
copied to it iff the path is different from the current path (taking
case sensitivity into account).
'''
with self.read_lock:
try:
path = self._field_for('path', book_id).replace('/', os.sep)
except:
return False
try:
path = self._field_for('path', book_id).replace('/', os.sep)
except AttributeError:
return False
with self.record_lock.lock(book_id):
return self.backend.copy_cover_to(path, dest,
return self.backend.copy_cover_to(path, dest,
use_hardlink=use_hardlink)
@api
@read_api
def copy_format_to(self, book_id, fmt, dest, use_hardlink=False):
'''
Copy the format ``fmt`` to the file like object ``dest``. If the
@ -441,15 +464,13 @@ class Cache(object):
the path is different from the current path (taking case sensitivity
into account).
'''
with self.read_lock:
try:
name = self.fields['formats'].format_fname(book_id, fmt)
path = self._field_for('path', book_id).replace('/', os.sep)
except:
raise NoSuchFormat('Record %d has no %s file'%(book_id, fmt))
try:
name = self.fields['formats'].format_fname(book_id, fmt)
path = self._field_for('path', book_id).replace('/', os.sep)
except (KeyError, AttributeError):
raise NoSuchFormat('Record %d has no %s file'%(book_id, fmt))
with self.record_lock.lock(book_id):
return self.backend.copy_format_to(book_id, fmt, name, path, dest,
return self.backend.copy_format_to(book_id, fmt, name, path, dest,
use_hardlink=use_hardlink)
@read_api
@ -520,16 +541,16 @@ class Cache(object):
this means that repeated calls yield the same
temp file (which is re-created each time)
'''
with self.read_lock:
ext = ('.'+fmt.lower()) if fmt else ''
try:
fname = self.fields['formats'].format_fname(book_id, fmt)
except:
return None
fname += ext
ext = ('.'+fmt.lower()) if fmt else ''
if as_path:
if preserve_filename:
with self.read_lock:
try:
fname = self.fields['formats'].format_fname(book_id, fmt)
except:
return None
fname += ext
bd = base_dir()
d = os.path.join(bd, 'format_abspath')
try:
@ -537,36 +558,40 @@ class Cache(object):
except:
pass
ret = os.path.join(d, fname)
with self.record_lock.lock(book_id):
try:
self.copy_format_to(book_id, fmt, ret)
except NoSuchFormat:
return None
try:
self.copy_format_to(book_id, fmt, ret)
except NoSuchFormat:
return None
else:
with PersistentTemporaryFile(ext) as pt, self.record_lock.lock(book_id):
with PersistentTemporaryFile(ext) as pt:
try:
self.copy_format_to(book_id, fmt, pt)
except NoSuchFormat:
return None
ret = pt.name
elif as_file:
ret = SpooledTemporaryFile(SPOOL_SIZE)
with self.record_lock.lock(book_id):
with self.read_lock:
try:
self.copy_format_to(book_id, fmt, ret)
except NoSuchFormat:
fname = self.fields['formats'].format_fname(book_id, fmt)
except:
return None
fname += ext
ret = SpooledTemporaryFile(SPOOL_SIZE)
try:
self.copy_format_to(book_id, fmt, ret)
except NoSuchFormat:
return None
ret.seek(0)
# Various bits of code try to use the name as the default
# title when reading metadata, so set it
ret.name = fname
else:
buf = BytesIO()
with self.record_lock.lock(book_id):
try:
self.copy_format_to(book_id, fmt, buf)
except NoSuchFormat:
return None
try:
self.copy_format_to(book_id, fmt, buf)
except NoSuchFormat:
return None
ret = buf.getvalue()
@ -621,7 +646,31 @@ class Cache(object):
icon_map=icon_map)
@write_api
def set_field(self, name, book_id_to_val_map, allow_case_change=True):
def update_last_modified(self, book_ids, now=None):
if now is None:
now = nowf()
if book_ids:
f = self.fields['last_modified']
f.writer.set_books({book_id:now for book_id in book_ids}, self.backend)
@write_api
def mark_as_dirty(self, book_ids):
self._update_last_modified(book_ids)
already_dirtied = set(self.dirtied_cache).intersection(book_ids)
new_dirtied = book_ids - already_dirtied
already_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(already_dirtied)}
if already_dirtied:
self.dirtied_sequence = max(already_dirtied.itervalues()) + 1
self.dirtied_cache.update(already_dirtied)
if new_dirtied:
self.backend.conn.executemany('INSERT OR IGNORE INTO metadata_dirtied (book) VALUES (?)',
((x,) for x in new_dirtied))
new_dirtied = {book_id:self.dirtied_sequence+i for i, book_id in enumerate(new_dirtied)}
self.dirtied_sequence = max(new_dirtied.itervalues()) + 1
self.dirtied_cache.update(new_dirtied)
@write_api
def set_field(self, name, book_id_to_val_map, allow_case_change=True, do_path_update=True):
f = self.fields[name]
is_series = f.metadata['datatype'] == 'series'
update_path = name in {'title', 'authors'}
@ -637,7 +686,7 @@ class Cache(object):
else:
v = sid = None
if name.startswith('#') and sid is None:
sid = 1.0 # The value will be set to 1.0 in the db table
sid = 1.0 # The value will be set to 1.0 in the db table
bimap[k] = v
if sid is not None:
simap[k] = sid
@ -654,10 +703,10 @@ class Cache(object):
for name in self.composites:
self.fields[name].pop_cache(dirtied)
if dirtied and update_path:
if dirtied and update_path and do_path_update:
self._update_path(dirtied, mark_as_dirtied=False)
# TODO: Mark these as dirtied so that the opf is regenerated
self._mark_as_dirty(dirtied)
return dirtied
@ -668,13 +717,211 @@ class Cache(object):
author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0]
self.backend.update_path(book_id, title, author, self.fields['path'], self.fields['formats'])
if mark_as_dirtied:
self._mark_as_dirty(book_ids)
@read_api
def get_a_dirtied_book(self):
if self.dirtied_cache:
return random.choice(tuple(self.dirtied_cache.iterkeys()))
return None
@read_api
def get_metadata_for_dump(self, book_id):
mi = None
# get the current sequence number for this book to pass back to the
# backup thread. This will avoid double calls in the case where the
# thread has not done the work between the put and the get_metadata
sequence = self.dirtied_cache.get(book_id, None)
if sequence is not None:
try:
# While a book is being created, the path is empty. Don't bother to
# try to write the opf, because it will go to the wrong folder.
if self._field_for('path', book_id):
mi = self._get_metadata(book_id)
# Always set cover to cover.jpg. Even if cover doesn't exist,
# no harm done. This way no need to call dirtied when
# cover is set/removed
mi.cover = 'cover.jpg'
except:
# This almost certainly means that the book has been deleted while
# the backup operation sat in the queue.
pass
# TODO: Mark these books as dirtied so that metadata.opf is
# re-created
return mi, sequence
@write_api
def clear_dirtied(self, book_id, sequence):
'''
Clear the dirtied indicator for the books. This is used when fetching
metadata, creating an OPF, and writing a file are separated into steps.
The last step is clearing the indicator
'''
dc_sequence = self.dirtied_cache.get(book_id, None)
if dc_sequence is None or sequence is None or dc_sequence == sequence:
self.backend.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
(book_id,))
self.dirtied_cache.pop(book_id, None)
@write_api
def write_backup(self, book_id, raw):
try:
path = self._field_for('path', book_id).replace('/', os.sep)
except:
return
self.backend.write_backup(path, raw)
@read_api
def dirty_queue_length(self):
return len(self.dirtied_cache)
@read_api
def read_backup(self, book_id):
''' Return the OPF metadata backup for the book as a bytestring or None
if no such backup exists. '''
try:
path = self._field_for('path', book_id).replace('/', os.sep)
except:
return
try:
return self.backend.read_backup(path)
except EnvironmentError:
return None
@write_api
def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
callback=None):
'''
Write metadata for each record to an individual OPF file. If callback
is not None, it is called once at the start with the number of book_ids
being processed. And once for every book_id, with arguments (book_id,
mi, ok).
'''
if book_ids is None:
book_ids = set(self.dirtied_cache)
if callback is not None:
callback(len(book_ids), True, False)
for book_id in book_ids:
if self._field_for('path', book_id) is None:
if callback is not None:
callback(book_id, None, False)
continue
mi, sequence = self._get_metadata_for_dump(book_id)
if mi is None:
if callback is not None:
callback(book_id, mi, False)
continue
try:
raw = metadata_to_opf(mi)
self._write_backup(book_id, raw)
if remove_from_dirtied:
self._clear_dirtied(book_id, sequence)
except:
pass
if callback is not None:
callback(book_id, mi, True)
@write_api
def set_cover(self, book_id_data_map):
''' Set the cover for this book. data can be either a QImage,
QPixmap, file object or bytestring '''
for book_id, data in book_id_data_map.iteritems():
try:
path = self._field_for('path', book_id).replace('/', os.sep)
except AttributeError:
self._update_path((book_id,))
path = self._field_for('path', book_id).replace('/', os.sep)
self.backend.set_cover(book_id, path, data)
self._set_field('cover', {book_id:1 for book_id in book_id_data_map})
@write_api
def set_metadata(self, book_id, mi, ignore_errors=False, force_changes=False,
set_title=True, set_authors=True):
if callable(getattr(mi, 'to_book_metadata', None)):
# Handle code passing in an OPF object instead of a Metadata object
mi = mi.to_book_metadata()
def set_field(name, val, **kwargs):
self._set_field(name, {book_id:val}, **kwargs)
path_changed = False
if set_title and mi.title:
path_changed = True
set_field('title', mi.title, do_path_update=False)
if set_authors:
path_changed = True
if not mi.authors:
mi.authors = [_('Unknown')]
authors = []
for a in mi.authors:
authors += string_to_authors(a)
set_field('authors', authors, do_path_update=False)
if path_changed:
self._update_path((book_id,))
def protected_set_field(name, val, **kwargs):
try:
set_field(name, val, **kwargs)
except:
if ignore_errors:
traceback.print_exc()
else:
raise
for field in ('rating', 'series_index', 'timestamp'):
val = getattr(mi, field)
if val is not None:
protected_set_field(field, val)
# force_changes has no effect on cover manipulation
cdata = mi.cover_data[1]
if cdata is None and isinstance(mi.cover, basestring) and mi.cover and os.access(mi.cover, os.R_OK):
with lopen(mi.cover, 'rb') as f:
raw = f.read()
if raw:
cdata = raw
if cdata is not None:
self._set_cover({book_id: cdata})
for field in ('title_sort', 'author_sort', 'publisher', 'series',
'tags', 'comments', 'languages', 'pubdate'):
val = mi.get(field, None)
if (force_changes and val is not None) or not mi.is_null(field):
protected_set_field(field, val)
# identifiers will always be replaced if force_changes is True
mi_idents = mi.get_identifiers()
if force_changes:
protected_set_field('identifiers', mi_idents)
elif mi_idents:
identifiers = self._field_for('identifiers', book_id, default_value={})
for key, val in mi_idents.iteritems():
if val and val.strip(): # Don't delete an existing identifier
identifiers[icu_lower(key)] = val
protected_set_field('identifiers', identifiers)
user_mi = mi.get_all_user_metadata(make_copy=False)
fm = self.field_metadata
for key in user_mi.iterkeys():
if (key in fm and
user_mi[key]['datatype'] == fm[key]['datatype'] and
(user_mi[key]['datatype'] != 'text' or
user_mi[key]['is_multiple'] == fm[key]['is_multiple'])):
val = mi.get(key, None)
if force_changes or val is not None:
protected_set_field(key, val)
extra = mi.get_extra(key)
if extra is not None:
protected_set_field(key+'_index', extra)
# }}}
class SortKey(object): # {{{
class SortKey(object): # {{{
def __init__(self, fields, sort_keys, book_id):
self.orders = tuple(1 if f[1] else -1 for f in fields)

4
src/calibre/db/categories.py
View File

@ -18,7 +18,7 @@ from calibre.utils.config_base import tweaks
from calibre.utils.icu import sort_key
from calibre.utils.search_query_parser import saved_searches
CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not a set
CATEGORY_SORTS = ('name', 'popularity', 'rating') # This has to be a tuple not a set
class Tag(object):
@ -218,7 +218,7 @@ def get_categories(dbcache, sort='name', book_ids=None, icon_map=None):
else:
items.append(taglist[label][n])
# else: do nothing, to not include nodes w zero counts
cat_name = '@' + user_cat # add the '@' to avoid name collision
cat_name = '@' + user_cat # add the '@' to avoid name collision
# Not a problem if we accumulate entries in the icon map
if icon_map is not None:
icon_map[cat_name] = icon_map['user:']

36
src/calibre/db/fields.py
View File

@ -31,7 +31,7 @@ class Field(object):
self.table_type = self.table.table_type
self._sort_key = (sort_key if dt in ('text', 'series', 'enumeration') else lambda x: x)
self._default_sort_key = ''
if dt in { 'int', 'float', 'rating' }:
if dt in {'int', 'float', 'rating'}:
self._default_sort_key = 0
elif dt == 'bool':
self._default_sort_key = None
@ -138,7 +138,7 @@ class OneToOneField(Field):
return self.table.book_col_map.iterkeys()
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
return {id_ : self._sort_key(self.table.book_col_map.get(id_,
return {id_: self._sort_key(self.table.book_col_map.get(id_,
self._default_sort_key)) for id_ in all_book_ids}
def iter_searchable_values(self, get_metadata, candidates, default_value=None):
@ -183,7 +183,7 @@ class CompositeField(OneToOneField):
return ans
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in
return {id_: sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in
all_book_ids}
def iter_searchable_values(self, get_metadata, candidates, default_value=None):
@ -245,7 +245,7 @@ class OnDeviceField(OneToOneField):
return iter(())
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
return {id_ : self.for_book(id_) for id_ in
return {id_: self.for_book(id_) for id_ in
all_book_ids}
def iter_searchable_values(self, get_metadata, candidates, default_value=None):
@ -280,12 +280,12 @@ class ManyToOneField(Field):
return self.table.id_map.iterkeys()
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
ans = {id_ : self.table.book_col_map.get(id_, None)
ans = {id_: self.table.book_col_map.get(id_, None)
for id_ in all_book_ids}
sk_map = {cid : (self._default_sort_key if cid is None else
sk_map = {cid: (self._default_sort_key if cid is None else
self._sort_key(self.table.id_map[cid]))
for cid in ans.itervalues()}
return {id_ : sk_map[cid] for id_, cid in ans.iteritems()}
return {id_: sk_map[cid] for id_, cid in ans.iteritems()}
def iter_searchable_values(self, get_metadata, candidates, default_value=None):
cbm = self.table.col_book_map
@ -327,14 +327,14 @@ class ManyToManyField(Field):
return self.table.id_map.iterkeys()
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
ans = {id_ : self.table.book_col_map.get(id_, ())
ans = {id_: self.table.book_col_map.get(id_, ())
for id_ in all_book_ids}
all_cids = set()
for cids in ans.itervalues():
all_cids = all_cids.union(set(cids))
sk_map = {cid : self._sort_key(self.table.id_map[cid])
sk_map = {cid: self._sort_key(self.table.id_map[cid])
for cid in all_cids}
return {id_ : (tuple(sk_map[cid] for cid in cids) if cids else
return {id_: (tuple(sk_map[cid] for cid in cids) if cids else
(self._default_sort_key,))
for id_, cids in ans.iteritems()}
@ -369,9 +369,9 @@ class IdentifiersField(ManyToManyField):
def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):
'Sort by identifier keys'
ans = {id_ : self.table.book_col_map.get(id_, ())
ans = {id_: self.table.book_col_map.get(id_, ())
for id_ in all_book_ids}
return {id_ : (tuple(sorted(cids.iterkeys())) if cids else
return {id_: (tuple(sorted(cids.iterkeys())) if cids else
(self._default_sort_key,))
for id_, cids in ans.iteritems()}
@ -397,9 +397,9 @@ class AuthorsField(ManyToManyField):
def author_data(self, author_id):
return {
'name' : self.table.id_map[author_id],
'sort' : self.table.asort_map[author_id],
'link' : self.table.alink_map[author_id],
'name': self.table.id_map[author_id],
'sort': self.table.asort_map[author_id],
'link': self.table.alink_map[author_id],
}
def category_sort_value(self, item_id, book_ids, lang_map):
@ -505,9 +505,9 @@ class TagsField(ManyToManyField):
def create_field(name, table):
cls = {
ONE_ONE : OneToOneField,
MANY_ONE : ManyToOneField,
MANY_MANY : ManyToManyField,
ONE_ONE: OneToOneField,
MANY_ONE: ManyToOneField,
MANY_MANY: ManyToManyField,
}[table.table_type]
if name == 'authors':
cls = AuthorsField

170
src/calibre/db/legacy.py Normal file
View File

@ -0,0 +1,170 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import os, traceback
from functools import partial
from calibre.db import _get_next_series_num_for_list, _get_series_values
from calibre.db.backend import DB
from calibre.db.cache import Cache
from calibre.db.categories import CATEGORY_SORTS
from calibre.db.view import View
from calibre.utils.date import utcnow
class LibraryDatabase(object):
''' Emulate the old LibraryDatabase2 interface '''
PATH_LIMIT = DB.PATH_LIMIT
WINDOWS_LIBRARY_PATH_LIMIT = DB.WINDOWS_LIBRARY_PATH_LIMIT
CATEGORY_SORTS = CATEGORY_SORTS
MATCH_TYPE = ('any', 'all')
CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
'int', 'float', 'bool', 'series', 'composite', 'enumeration'])
@classmethod
def exists_at(cls, path):
return path and os.path.exists(os.path.join(path, 'metadata.db'))
def __init__(self, library_path,
default_prefs=None, read_only=False, is_second_db=False,
progress_callback=lambda x, y:True, restore_all_prefs=False):
self.is_second_db = is_second_db # TODO: Use is_second_db
self.listeners = set([])
backend = self.backend = DB(library_path, default_prefs=default_prefs,
read_only=read_only, restore_all_prefs=restore_all_prefs,
progress_callback=progress_callback)
cache = self.new_api = Cache(backend)
cache.init()
self.data = View(cache)
self.get_property = self.data.get_property
for prop in (
'author_sort', 'authors', 'comment', 'comments',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp', 'uuid', 'pubdate', 'ondevice',
'metadata_last_modified', 'languages',
):
fm = {'comment':'comments', 'metadata_last_modified':
'last_modified', 'title_sort':'sort'}.get(prop, prop)
setattr(self, prop, partial(self.get_property,
loc=self.FIELD_MAP[fm]))
self.last_update_check = self.last_modified()
def close(self):
self.backend.close()
def break_cycles(self):
self.data.cache.backend = None
self.data.cache = None
self.data = self.backend = self.new_api = self.field_metadata = self.prefs = self.listeners = self.refresh_ondevice = None
# Library wide properties {{{
@property
def field_metadata(self):
return self.backend.field_metadata
@property
def user_version(self):
return self.backend.user_version
@property
def library_id(self):
return self.backend.library_id
@property
def library_path(self):
return self.backend.library_path
@property
def dbpath(self):
return self.backend.dbpath
def last_modified(self):
return self.backend.last_modified()
def check_if_modified(self):
if self.last_modified() > self.last_update_check:
self.refresh()
self.last_update_check = utcnow()
@property
def custom_column_num_map(self):
return self.backend.custom_column_num_map
@property
def custom_column_label_map(self):
return self.backend.custom_column_label_map
@property
def FIELD_MAP(self):
return self.backend.FIELD_MAP
@property
def formatter_template_cache(self):
return self.data.cache.formatter_template_cache
def initialize_template_cache(self):
self.data.cache.initialize_template_cache()
def all_ids(self):
for book_id in self.data.cache.all_book_ids():
yield book_id
def refresh(self, field=None, ascending=True):
self.data.cache.refresh()
self.data.refresh(field=field, ascending=ascending)
def add_listener(self, listener):
'''
Add a listener. Will be called on change events with two arguments.
Event name and list of affected ids.
'''
self.listeners.add(listener)
def notify(self, event, ids=[]):
'Notify all listeners'
for listener in self.listeners:
try:
listener(event, ids)
except:
traceback.print_exc()
continue
# }}}
def path(self, index, index_is_id=False):
'Return the relative path to the directory containing this books files as a unicode string.'
book_id = index if index_is_id else self.data.index_to_id(index)
return self.data.cache.field_for('path', book_id).replace('/', os.sep)
def abspath(self, index, index_is_id=False, create_dirs=True):
'Return the absolute path to the directory containing this books files as a unicode string.'
path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id))
if create_dirs and not os.path.exists(path):
os.makedirs(path)
return path
# Private interface {{{
def __iter__(self):
for row in self.data.iterall():
yield row
def _get_next_series_num_for_list(self, series_indices):
return _get_next_series_num_for_list(series_indices)
def _get_series_values(self, val):
return _get_series_values(val)
# }}}

4
src/calibre/db/locking.py
View File

@ -39,7 +39,7 @@ def create_locks():
l = SHLock()
return RWLockWrapper(l), RWLockWrapper(l, is_shared=False)
class SHLock(object): # {{{
class SHLock(object): # {{{
'''
Shareable lock class. Used to implement the Multiple readers-single writer
paradigm. As best as I can tell, neither writer nor reader starvation
@ -191,7 +191,7 @@ class SHLock(object): # {{{
try:
return self._free_waiters.pop()
except IndexError:
return Condition(self._lock)#, verbose=True)
return Condition(self._lock)
def _return_waiter(self, waiter):
self._free_waiters.append(waiter)

3
src/calibre/db/schema_upgrades.py
View File

@ -172,7 +172,6 @@ class SchemaUpgrade(object):
'''
)
def upgrade_version_6(self):
'Show authors in order'
self.conn.execute('''
@ -337,7 +336,7 @@ class SchemaUpgrade(object):
FROM {tn};
'''.format(tn=table_name, cn=column_name,
vcn=view_column_name, scn= sort_column_name))
vcn=view_column_name, scn=sort_column_name))
self.conn.execute(script)
def create_cust_tag_browser_view(table_name, link_table_name):

84
src/calibre/db/search.py
View File

@ -64,7 +64,7 @@ def _match(query, value, matchkind, use_primary_find_in_search=True):
else:
internal_match_ok = False
for t in value:
try: ### ignore regexp exceptions, required because search-ahead tries before typing is finished
try: # ignore regexp exceptions, required because search-ahead tries before typing is finished
t = icu_lower(t)
if (matchkind == EQUALS_MATCH):
if internal_match_ok:
@ -95,20 +95,20 @@ def _match(query, value, matchkind, use_primary_find_in_search=True):
return False
# }}}
class DateSearch(object): # {{{
class DateSearch(object): # {{{
def __init__(self):
self.operators = {
'=' : (1, self.eq),
'!=' : (2, self.ne),
'>' : (1, self.gt),
'>=' : (2, self.ge),
'<' : (1, self.lt),
'<=' : (2, self.le),
'=': (1, self.eq),
'!=': (2, self.ne),
'>': (1, self.gt),
'>=': (2, self.ge),
'<': (1, self.lt),
'<=': (2, self.le),
}
self.local_today = { '_today', 'today', icu_lower(_('today')) }
self.local_yesterday = { '_yesterday', 'yesterday', icu_lower(_('yesterday')) }
self.local_thismonth = { '_thismonth', 'thismonth', icu_lower(_('thismonth')) }
self.local_today = {'_today', 'today', icu_lower(_('today'))}
self.local_yesterday = {'_yesterday', 'yesterday', icu_lower(_('yesterday'))}
self.local_thismonth = {'_thismonth', 'thismonth', icu_lower(_('thismonth'))}
self.daysago_pat = re.compile(r'(%s|daysago|_daysago)$'%_('daysago'))
def eq(self, dbdate, query, field_count):
@ -195,13 +195,13 @@ class DateSearch(object): # {{{
try:
qd = now() - timedelta(int(num))
except:
raise ParseException(query, len(query), 'Number conversion error')
raise ParseException(_('Number conversion error: {0}').format(num))
field_count = 3
else:
try:
qd = parse_date(query, as_utc=False)
except:
raise ParseException(query, len(query), 'Date conversion error')
raise ParseException(_('Date conversion error: {0}').format(query))
if '-' in query:
field_count = query.count('-') + 1
else:
@ -216,16 +216,16 @@ class DateSearch(object): # {{{
return matches
# }}}
class NumericSearch(object): # {{{
class NumericSearch(object): # {{{
def __init__(self):
self.operators = {
'=':( 1, lambda r, q: r == q ),
'>':( 1, lambda r, q: r is not None and r > q ),
'<':( 1, lambda r, q: r is not None and r < q ),
'!=':( 2, lambda r, q: r != q ),
'>=':( 2, lambda r, q: r is not None and r >= q ),
'<=':( 2, lambda r, q: r is not None and r <= q )
'=':(1, lambda r, q: r == q),
'>':(1, lambda r, q: r is not None and r > q),
'<':(1, lambda r, q: r is not None and r < q),
'!=':(2, lambda r, q: r != q),
'>=':(2, lambda r, q: r is not None and r >= q),
'<=':(2, lambda r, q: r is not None and r <= q)
}
def __call__(self, query, field_iter, location, datatype, candidates, is_many=False):
@ -267,7 +267,7 @@ class NumericSearch(object): # {{{
p, relop = self.operators['=']
cast = int
if dt == 'rating':
if dt == 'rating':
cast = lambda x: 0 if x is None else int(x)
adjust = lambda x: x/2
elif dt in ('float', 'composite'):
@ -285,8 +285,8 @@ class NumericSearch(object): # {{{
try:
q = cast(query) * mult
except:
raise ParseException(query, len(query),
'Non-numeric value in query: %r'%query)
raise ParseException(
_('Non-numeric value in query: {0}').format(query))
for val, book_ids in field_iter():
if val is None:
@ -303,7 +303,7 @@ class NumericSearch(object): # {{{
# }}}
class BooleanSearch(object): # {{{
class BooleanSearch(object): # {{{
def __init__(self):
self.local_no = icu_lower(_('no'))
@ -324,35 +324,35 @@ class BooleanSearch(object): # {{{
for val, book_ids in field_iter():
val = force_to_bool(val)
if not bools_are_tristate:
if val is None or not val: # item is None or set to false
if query in { self.local_no, self.local_unchecked, 'no', '_no', 'false' }:
if val is None or not val: # item is None or set to false
if query in {self.local_no, self.local_unchecked, 'no', '_no', 'false'}:
matches |= book_ids
else: # item is explicitly set to true
if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }:
else: # item is explicitly set to true
if query in {self.local_yes, self.local_checked, 'yes', '_yes', 'true'}:
matches |= book_ids
else:
if val is None:
if query in { self.local_empty, self.local_blank, 'empty', '_empty', 'false' }:
if query in {self.local_empty, self.local_blank, 'empty', '_empty', 'false'}:
matches |= book_ids
elif not val: # is not None and false
if query in { self.local_no, self.local_unchecked, 'no', '_no', 'true' }:
elif not val: # is not None and false
if query in {self.local_no, self.local_unchecked, 'no', '_no', 'true'}:
matches |= book_ids
else: # item is not None and true
if query in { self.local_yes, self.local_checked, 'yes', '_yes', 'true' }:
else: # item is not None and true
if query in {self.local_yes, self.local_checked, 'yes', '_yes', 'true'}:
matches |= book_ids
return matches
# }}}
class KeyPairSearch(object): # {{{
class KeyPairSearch(object): # {{{
def __call__(self, query, field_iter, candidates, use_primary_find):
matches = set()
if ':' in query:
q = [q.strip() for q in query.split(':')]
if len(q) != 2:
raise ParseException(query, len(query),
'Invalid query format for colon-separated search')
raise ParseException(
_('Invalid query format for colon-separated search: {0}').format(query))
keyq, valq = q
keyq_mkind, keyq = _matchkind(keyq)
valq_mkind, valq = _matchkind(valq)
@ -465,7 +465,8 @@ class Parser(SearchQueryParser):
if invert:
matches = self.all_book_ids - matches
return matches
raise ParseException(query, len(query), 'Recursive query group detected')
raise ParseException(
_('Recursive query group detected: {0}').format(query))
# If the user has asked to restrict searching over all field, apply
# that restriction
@ -547,11 +548,12 @@ class Parser(SearchQueryParser):
field_metadata = {}
for x, fm in self.field_metadata.iteritems():
if x.startswith('@'): continue
if x.startswith('@'):
continue
if fm['search_terms'] and x != 'series_sort':
all_locs.add(x)
field_metadata[x] = fm
if fm['datatype'] in { 'composite', 'text', 'comments', 'series', 'enumeration' }:
if fm['datatype'] in {'composite', 'text', 'comments', 'series', 'enumeration'}:
text_fields.add(x)
locations = all_locs if location == 'all' else {location}
@ -687,8 +689,8 @@ class Search(object):
dbcache, all_book_ids, dbcache.pref('grouped_search_terms'),
self.date_search, self.num_search, self.bool_search,
self.keypair_search,
prefs[ 'limit_search_columns' ],
prefs[ 'limit_search_columns_to' ], self.all_search_locations,
prefs['limit_search_columns'],
prefs['limit_search_columns_to'], self.all_search_locations,
virtual_fields)
try:

2
src/calibre/db/tables.py
View File

@ -82,7 +82,7 @@ class OneToOneTable(Table):
self.metadata['column'], self.metadata['table'])):
self.book_col_map[row[0]] = self.unserialize(row[1])
class PathTable(OneToOneTable):
class PathTable(OneToOneTable):
def set_path(self, book_id, path, db):
self.book_col_map[book_id] = path

18
src/calibre/db/tests/base.py
View File

@ -16,6 +16,9 @@ rmtree = partial(shutil.rmtree, ignore_errors=True)
class BaseTest(unittest.TestCase):
longMessage = True
maxDiff = None
def setUp(self):
self.library_path = self.mkdtemp()
self.create_db(self.library_path)
@ -40,10 +43,10 @@ class BaseTest(unittest.TestCase):
db.conn.close()
return dest
def init_cache(self, library_path):
def init_cache(self, library_path=None):
from calibre.db.backend import DB
from calibre.db.cache import Cache
backend = DB(library_path)
backend = DB(library_path or self.library_path)
cache = Cache(backend)
cache.init()
return cache
@ -53,9 +56,13 @@ class BaseTest(unittest.TestCase):
atexit.register(rmtree, ans)
return ans
def init_old(self, library_path):
def init_old(self, library_path=None):
from calibre.library.database2 import LibraryDatabase2
return LibraryDatabase2(library_path)
return LibraryDatabase2(library_path or self.library_path)
def init_legacy(self, library_path=None):
from calibre.db.legacy import LibraryDatabase
return LibraryDatabase(library_path or self.library_path)
def clone_library(self, library_path):
if not hasattr(self, 'clone_dir'):
@ -81,7 +88,8 @@ class BaseTest(unittest.TestCase):
'ondevice_col', 'last_modified', 'has_cover',
'cover_data'}.union(allfk1)
for attr in all_keys:
if attr == 'user_metadata': continue
if attr == 'user_metadata':
continue
attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
if attr == 'formats':
attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2))

133
src/calibre/db/tests/legacy.py Normal file
View File

@ -0,0 +1,133 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import inspect
from calibre.db.tests.base import BaseTest
class LegacyTest(BaseTest):
''' Test the emulation of the legacy interface. '''
def test_library_wide_properties(self): # {{{
'Test library wide properties'
def get_props(db):
props = ('user_version', 'is_second_db', 'library_id', 'field_metadata',
'custom_column_label_map', 'custom_column_num_map', 'library_path', 'dbpath')
fprops = ('last_modified', )
ans = {x:getattr(db, x) for x in props}
ans.update({x:getattr(db, x)() for x in fprops})
ans['all_ids'] = frozenset(db.all_ids())
return ans
old = self.init_old()
oldvals = get_props(old)
old.close()
del old
db = self.init_legacy()
newvals = get_props(db)
self.assertEqual(oldvals, newvals)
db.close()
# }}}
def test_get_property(self): # {{{
'Test the get_property interface for reading data'
def get_values(db):
ans = {}
for label, loc in db.FIELD_MAP.iteritems():
if isinstance(label, int):
label = '#'+db.custom_column_num_map[label]['label']
label = type('')(label)
ans[label] = tuple(db.get_property(i, index_is_id=True, loc=loc)
for i in db.all_ids())
if label in ('id', 'title', '#tags'):
with self.assertRaises(IndexError):
db.get_property(9999, loc=loc)
with self.assertRaises(IndexError):
db.get_property(9999, index_is_id=True, loc=loc)
if label in {'tags', 'formats'}:
# Order is random in the old db for these
ans[label] = tuple(set(x.split(',')) if x else x for x in ans[label])
if label == 'series_sort':
# The old db code did not take book language into account
# when generating series_sort values (the first book has
# lang=deu)
ans[label] = ans[label][1:]
return ans
old = self.init_old()
old_vals = get_values(old)
old.close()
old = None
db = self.init_legacy()
new_vals = get_values(db)
db.close()
self.assertEqual(old_vals, new_vals)
# }}}
def test_refresh(self): # {{{
' Test refreshing the view after a change to metadata.db '
db = self.init_legacy()
db2 = self.init_legacy()
self.assertEqual(db2.data.cache.set_field('title', {1:'xxx'}), set([1]))
db2.close()
del db2
self.assertNotEqual(db.title(1, index_is_id=True), 'xxx')
db.check_if_modified()
self.assertEqual(db.title(1, index_is_id=True), 'xxx')
# }}}
def test_legacy_getters(self): # {{{
' Test various functions to get individual bits of metadata '
old = self.init_old()
getters = ('path', 'abspath', 'title', 'authors', 'series',
'publisher', 'author_sort', 'authors', 'comments',
'comment', 'publisher', 'rating', 'series_index', 'tags',
'timestamp', 'uuid', 'pubdate', 'ondevice',
'metadata_last_modified', 'languages')
oldvals = {g:tuple(getattr(old, g)(x) for x in xrange(3)) + tuple(getattr(old, g)(x, True) for x in (1,2,3)) for g in getters}
old_rows = {tuple(r)[:5] for r in old}
old.close()
db = self.init_legacy()
newvals = {g:tuple(getattr(db, g)(x) for x in xrange(3)) + tuple(getattr(db, g)(x, True) for x in (1,2,3)) for g in getters}
new_rows = {tuple(r)[:5] for r in db}
for x in (oldvals, newvals):
x['tags'] = tuple(set(y.split(',')) if y else y for y in x['tags'])
self.assertEqual(oldvals, newvals)
self.assertEqual(old_rows, new_rows)
# }}}
def test_legacy_coverage(self): # {{{
' Check that the emulation of the legacy interface is (almost) total '
cl = self.cloned_library
db = self.init_old(cl)
ndb = self.init_legacy()
SKIP_ATTRS = {
'TCat_Tag', '_add_newbook_tag', '_clean_identifier', '_library_id_', '_set_authors',
'_set_title', '_set_custom', '_update_author_in_cache',
}
SKIP_ARGSPEC = {
'__init__',
}
for attr in dir(db):
if attr in SKIP_ATTRS:
continue
self.assertTrue(hasattr(ndb, attr), 'The attribute %s is missing' % attr)
obj, nobj = getattr(db, attr), getattr(ndb, attr)
if attr not in SKIP_ARGSPEC:
try:
argspec = inspect.getargspec(obj)
except TypeError:
pass
else:
self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr)
# }}}

25
src/calibre/db/tests/main.py
View File

@ -9,15 +9,32 @@ __docformat__ = 'restructuredtext en'
import unittest, os, argparse
try:
import init_calibre # noqa
except ImportError:
pass
def find_tests():
return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('name', nargs='?', default=None, help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic')
parser.add_argument('name', nargs='?', default=None,
help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic or .many_many_basic for a shortcut')
args = parser.parse_args()
if args.name:
unittest.TextTestRunner(verbosity=4).run(unittest.defaultTestLoader.loadTestsFromName(args.name))
if args.name and args.name.startswith('.'):
tests = find_tests()
ans = None
try:
for suite in tests:
for test in suite._tests:
for s in test:
if s._testMethodName == args.name[1:]:
tests = s
raise StopIteration()
except StopIteration:
pass
else:
unittest.TextTestRunner(verbosity=4).run(find_tests())
tests = unittest.defaultTestLoader.loadTestsFromName(args.name) if args.name else find_tests()
unittest.TextTestRunner(verbosity=4).run(tests)

8
src/calibre/db/tests/reading.py
View File

@ -8,6 +8,7 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import datetime
from io import BytesIO
from calibre.utils.date import utc_tz
from calibre.db.tests.base import BaseTest
@ -205,6 +206,9 @@ class ReadingTest(BaseTest):
else:
self.assertEqual(cdata, cache.cover(book_id, as_path=True),
'Reading of null cover as path failed')
buf = BytesIO()
self.assertFalse(cache.copy_cover_to(99999, buf), 'copy_cover_to() did not return False for non-existent book_id')
self.assertFalse(cache.copy_cover_to(3, buf), 'copy_cover_to() did not return False for non-existent cover')
# }}}
@ -305,6 +309,7 @@ class ReadingTest(BaseTest):
def test_get_formats(self): # {{{
'Test reading ebook formats using the format() method'
from calibre.library.database2 import LibraryDatabase2
from calibre.db.cache import NoSuchFormat
old = LibraryDatabase2(self.library_path)
ids = old.all_ids()
lf = {i:set(old.formats(i, index_is_id=True).split(',')) if old.formats(
@ -332,6 +337,9 @@ class ReadingTest(BaseTest):
self.assertEqual(old, f.read(),
'Failed to read format as path')
buf = BytesIO()
self.assertRaises(NoSuchFormat, cache.copy_format_to, 99999, 'X', buf, 'copy_format_to() failed to raise an exception for non-existent book')
self.assertRaises(NoSuchFormat, cache.copy_format_to, 1, 'X', buf, 'copy_format_to() failed to raise an exception for non-existent format')
# }}}

77
src/calibre/db/tests/writing.py
View File

@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
from collections import namedtuple
from functools import partial
from io import BytesIO
from calibre.ebooks.metadata import author_to_author_sort
from calibre.utils.date import UNDEFINED_DATE
@ -16,6 +17,7 @@ from calibre.db.tests.base import BaseTest
class WritingTest(BaseTest):
# Utils {{{
def create_getter(self, name, getter=None):
if getter is None:
if name.endswith('_index'):
@ -35,7 +37,7 @@ class WritingTest(BaseTest):
ans = lambda db:partial(getattr(db, setter), commit=True)
return ans
def create_test(self, name, vals, getter=None, setter=None ):
def create_test(self, name, vals, getter=None, setter=None):
T = namedtuple('Test', 'name vals getter setter')
return T(name, vals, self.create_getter(name, getter),
self.create_setter(name, setter))
@ -70,8 +72,9 @@ class WritingTest(BaseTest):
'Failed setting for %s, sqlite value not the same: %r != %r'%(
test.name, old_sqlite_res, sqlite_res))
del db
# }}}
def test_one_one(self): # {{{
def test_one_one(self): # {{{
'Test setting of values in one-one fields'
tests = [self.create_test('#yesno', (True, False, 'true', 'false', None))]
for name, getter, setter in (
@ -112,7 +115,7 @@ class WritingTest(BaseTest):
self.run_tests(tests)
# }}}
def test_many_one_basic(self): # {{{
def test_many_one_basic(self): # {{{
'Test the different code paths for writing to a many-one field'
cl = self.cloned_library
cache = self.init_cache(cl)
@ -199,7 +202,7 @@ class WritingTest(BaseTest):
# }}}
def test_many_many_basic(self): # {{{
def test_many_many_basic(self): # {{{
'Test the different code paths for writing to a many-many field'
cl = self.cloned_library
cache = self.init_cache(cl)
@ -289,6 +292,70 @@ class WritingTest(BaseTest):
ae(c.field_for('sort', 1), 'Moose, The')
ae(c.field_for('sort', 2), 'Cat')
# }}}
def test_dirtied(self): # {{{
'Test the setting of the dirtied flag and the last_modified column'
cl = self.cloned_library
cache = self.init_cache(cl)
ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field
# First empty dirtied
cache.dump_metadata()
af(cache.dirtied_cache)
af(self.init_cache(cl).dirtied_cache)
prev = cache.field_for('last_modified', 3)
import calibre.db.cache as c
from datetime import timedelta
utime = prev+timedelta(days=1)
onowf = c.nowf
c.nowf = lambda: utime
try:
ae(sf('title', {3:'xxx'}), set([3]))
self.assertTrue(3 in cache.dirtied_cache)
ae(cache.field_for('last_modified', 3), utime)
cache.dump_metadata()
raw = cache.read_backup(3)
from calibre.ebooks.metadata.opf2 import OPF
opf = OPF(BytesIO(raw))
ae(opf.title, 'xxx')
finally:
c.nowf = onowf
# }}}
def test_backup(self): # {{{
'Test the automatic backup of changed metadata'
cl = self.cloned_library
cache = self.init_cache(cl)
ae, af, sf, ff = self.assertEqual, self.assertFalse, cache.set_field, cache.field_for
# First empty dirtied
cache.dump_metadata()
af(cache.dirtied_cache)
from calibre.db.backup import MetadataBackup
interval = 0.01
mb = MetadataBackup(cache, interval=interval, scheduling_interval=0)
mb.start()
try:
ae(sf('title', {1:'title1', 2:'title2', 3:'title3'}), {1,2,3})
ae(sf('authors', {1:'author1 & author2', 2:'author1 & author2', 3:'author1 & author2'}), {1,2,3})
count = 6
while cache.dirty_queue_length() and count > 0:
mb.join(interval)
count -= 1
af(cache.dirty_queue_length())
finally:
mb.stop()
mb.join(interval)
af(mb.is_alive())
from calibre.ebooks.metadata.opf2 import OPF
for book_id in (1, 2, 3):
raw = cache.read_backup(book_id)
opf = OPF(BytesIO(raw))
ae(opf.title, 'title%d'%book_id)
ae(opf.authors, ['author1', 'author2'])
# }}}
def test_set_cover(self):
' Test setting of cover '
self.assertTrue(False, 'TODO: test set_cover() and set_metadata()')

169
src/calibre/db/view.py
View File

@ -11,6 +11,9 @@ import weakref
from functools import partial
from itertools import izip, imap
from calibre.ebooks.metadata import title_sort
from calibre.utils.config_base import tweaks
def sanitize_sort_field_name(field_metadata, field):
field = field_metadata.search_term_to_field_key(field.lower().strip())
# translate some fields to their hidden equivalent
@ -26,11 +29,12 @@ class MarkedVirtualField(object):
for book_id in candidates:
yield self.marked_ids.get(book_id, default_value), {book_id}
class TableRow(list):
class TableRow(object):
def __init__(self, book_id, view):
self.book_id = book_id
self.view = weakref.ref(view)
self.column_count = view.column_count
def __getitem__(self, obj):
view = self.view()
@ -40,6 +44,25 @@ class TableRow(list):
else:
return view._field_getters[obj](self.book_id)
def __len__(self):
return self.column_count
def __iter__(self):
for i in xrange(self.column_count):
yield self[i]
def format_is_multiple(x, sep=',', repl=None):
if not x:
return None
if repl is not None:
x = (y.replace(sep, repl) for y in x)
return sep.join(x)
def format_identifiers(x):
if not x:
return None
return ','.join('%s:%s'%(k, v) for k, v in x.iteritems())
class View(object):
''' A table view of the database, with rows and columns. Also supports
@ -49,51 +72,82 @@ class View(object):
self.cache = cache
self.marked_ids = {}
self.search_restriction_book_count = 0
self.search_restriction = ''
self.search_restriction = self.base_restriction = ''
self.search_restriction_name = self.base_restriction_name = ''
self._field_getters = {}
self.column_count = len(cache.backend.FIELD_MAP)
for col, idx in cache.backend.FIELD_MAP.iteritems():
label, fmt = col, lambda x:x
func = {
'id': self._get_id,
'au_map': self.get_author_data,
'ondevice': self.get_ondevice,
'marked': self.get_marked,
'series_sort':self.get_series_sort,
}.get(col, self._get)
if isinstance(col, int):
label = self.cache.backend.custom_column_num_map[col]['label']
label = (self.cache.backend.field_metadata.custom_field_prefix
+ label)
self._field_getters[idx] = partial(self.get, label)
else:
if label.endswith('_index'):
try:
self._field_getters[idx] = {
'id' : self._get_id,
'au_map' : self.get_author_data,
'ondevice': self.get_ondevice,
'marked' : self.get_marked,
}[col]
except KeyError:
self._field_getters[idx] = partial(self.get, col)
num = int(label.partition('_')[0])
except ValueError:
pass # series_index
else:
label = self.cache.backend.custom_column_num_map[num]['label']
label = (self.cache.backend.field_metadata.custom_field_prefix
+ label + '_index')
self._map = tuple(self.cache.all_book_ids())
fm = self.field_metadata[label]
fm
if label == 'authors':
fmt = partial(format_is_multiple, repl='|')
elif label in {'tags', 'languages', 'formats'}:
fmt = format_is_multiple
elif label == 'cover':
fmt = bool
elif label == 'identifiers':
fmt = format_identifiers
elif fm['datatype'] == 'text' and fm['is_multiple']:
sep = fm['is_multiple']['cache_to_list']
if sep not in {'&','|'}:
sep = '|'
fmt = partial(format_is_multiple, sep=sep)
self._field_getters[idx] = partial(func, label, fmt=fmt) if func == self._get else func
self._map = tuple(sorted(self.cache.all_book_ids()))
self._map_filtered = tuple(self._map)
def get_property(self, id_or_index, index_is_id=False, loc=-1):
book_id = id_or_index if index_is_id else self._map_filtered[id_or_index]
return self._field_getters[loc](book_id)
@property
def field_metadata(self):
return self.cache.field_metadata
def _get_id(self, idx, index_is_id=True):
if index_is_id and idx not in self.cache.all_book_ids():
raise IndexError('No book with id %s present'%idx)
return idx if index_is_id else self.index_to_id(idx)
def __getitem__(self, row):
return TableRow(self._map_filtered[row], self.cache)
return TableRow(self._map_filtered[row], self)
def __len__(self):
return len(self._map_filtered)
def __iter__(self):
for book_id in self._map_filtered:
yield self._data[book_id]
yield TableRow(book_id, self)
def iterall(self):
for book_id in self._map:
yield self[book_id]
for book_id in self.iterallids():
yield TableRow(book_id, self)
def iterallids(self):
for book_id in self._map:
for book_id in sorted(self._map):
yield book_id
def get_field_map_field(self, row, col, index_is_id=True):
@ -107,9 +161,21 @@ class View(object):
def index_to_id(self, idx):
return self._map_filtered[idx]
def get(self, field, idx, index_is_id=True, default_value=None):
def _get(self, field, idx, index_is_id=True, default_value=None, fmt=lambda x:x):
id_ = idx if index_is_id else self.index_to_id(idx)
return self.cache.field_for(field, id_)
if index_is_id and id_ not in self.cache.all_book_ids():
raise IndexError('No book with id %s present'%idx)
return fmt(self.cache.field_for(field, id_, default_value=default_value))
def get_series_sort(self, idx, index_is_id=True, default_value=''):
book_id = idx if index_is_id else self.index_to_id(idx)
with self.cache.read_lock:
lang_map = self.cache.fields['languages'].book_value_map
lang = lang_map.get(book_id, None) or None
if lang:
lang = lang[0]
return title_sort(self.cache._field_for('series', book_id, default_value=''),
order=tweaks['title_series_sorting'], lang=lang)
def get_ondevice(self, idx, index_is_id=True, default_value=''):
id_ = idx if index_is_id else self.index_to_id(idx)
@ -119,26 +185,15 @@ class View(object):
id_ = idx if index_is_id else self.index_to_id(idx)
return self.marked_ids.get(id_, default_value)
def get_author_data(self, idx, index_is_id=True, default_value=()):
'''
Return author data for all authors of the book identified by idx as a
tuple of dictionaries. The dictionaries should never be empty, unless
there is a bug somewhere. The list could be empty if idx point to an
non existent book, or book with no authors (though again a book with no
authors should never happen).
Each dictionary has the keys: name, sort, link. Link can be an empty
string.
default_value is ignored, this method always returns a tuple
'''
def get_author_data(self, idx, index_is_id=True, default_value=None):
id_ = idx if index_is_id else self.index_to_id(idx)
with self.cache.read_lock:
ids = self.cache._field_ids_for('authors', id_)
ans = []
for id_ in ids:
ans.append(self.cache._author_data(id_))
return tuple(ans)
data = self.cache._author_data(id_)
ans.append(':::'.join((data['name'], data['sort'], data['link'])))
return ':#:'.join(ans) if ans else default_value
def multisort(self, fields=[], subsort=False, only_ids=None):
fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields]
@ -168,8 +223,19 @@ class View(object):
return ans
self._map_filtered = tuple(ans)
def _build_restriction_string(self, restriction):
if self.base_restriction:
if restriction:
return u'(%s) and (%s)' % (self.base_restriction, restriction)
else:
return self.base_restriction
else:
return restriction
def search_getting_ids(self, query, search_restriction,
set_restriction_count=False):
set_restriction_count=False, use_virtual_library=True):
if use_virtual_library:
search_restriction = self._build_restriction_string(search_restriction)
q = ''
if not query or not query.strip():
q = search_restriction
@ -188,11 +254,32 @@ class View(object):
self.search_restriction_book_count = len(rv)
return rv
def get_search_restriction(self):
return self.search_restriction
def set_search_restriction(self, s):
self.search_restriction = s
def get_base_restriction(self):
return self.base_restriction
def set_base_restriction(self, s):
self.base_restriction = s
def get_base_restriction_name(self):
return self.base_restriction_name
def set_base_restriction_name(self, s):
self.base_restriction_name = s
def get_search_restriction_name(self):
return self.search_restriction_name
def set_search_restriction_name(self, s):
self.search_restriction_name = s
def search_restriction_applied(self):
return bool(self.search_restriction)
return bool(self.search_restriction) or bool(self.base_restriction)
def get_search_restriction_book_count(self):
return self.search_restriction_book_count
@ -216,3 +303,11 @@ class View(object):
self.marked_ids = dict(izip(id_dict.iterkeys(), imap(unicode,
id_dict.itervalues())))
def refresh(self, field=None, ascending=True):
self._map = tuple(self.cache.all_book_ids())
self._map_filtered = tuple(self._map)
if field is not None:
self.sort(field, ascending)
if self.search_restriction or self.base_restriction:
self.search('', return_matches=False)

2
src/calibre/db/write.py
View File

@ -417,7 +417,7 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args):
# }}}
def identifiers(book_id_val_map, db, field, *args): # {{{
def identifiers(book_id_val_map, db, field, *args): # {{{
table = field.table
updates = set()
for book_id, identifiers in book_id_val_map.iteritems():

5
src/calibre/devices/android/driver.py
View File

@ -71,6 +71,7 @@ class ANDROID(USBMS):
0x42f7 : [0x216],
0x4365 : [0x216],
0x4366 : [0x216],
0x4371 : [0x216],
},
# Freescale
0x15a2 : {
@ -239,7 +240,7 @@ class ANDROID(USBMS):
'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E',
'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS',
'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD']
'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894']
WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@ -250,7 +251,7 @@ class ANDROID(USBMS):
'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E',
'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1']
'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894']
OSX_MAIN_MEM = 'Android Device Main Memory'

24
src/calibre/devices/apple/driver.py
View File

@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
__docformat__ = 'restructuredtext en'
import cStringIO, ctypes, datetime, os, platform, re, shutil, sys, tempfile, time
import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time
from calibre import fit_image, confirm_config_name, strftime as _strftime
from calibre.constants import (
@ -17,13 +17,12 @@ from calibre.devices.interface import DevicePlugin
from calibre.ebooks.metadata import (author_to_author_sort, authors_to_string,
MetaInformation, title_sort)
from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.config import config_dir, dynamic, prefs
from calibre.utils.date import now, parse_date
from calibre.utils.zipfile import ZipFile
from calibre.utils.config_base import config_dir, prefs
DEBUG = CALIBRE_DEBUG
def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
from calibre.utils.date import now
if not hasattr(dt, 'timetuple'):
dt = now()
@ -96,6 +95,7 @@ class AppleOpenFeedback(OpenFeedback):
def do_it(self, return_code):
from calibre.utils.logging import default_log
from calibre.utils.config import dynamic
if return_code == self.Accepted:
default_log.info(" Apple driver ENABLED")
dynamic[confirm_config_name(self.cd.plugin.DISPLAY_DISABLE_DIALOG)] = False
@ -413,6 +413,7 @@ class ITUNES(DriverBase):
list of device books.
"""
from calibre.utils.date import parse_date
if not oncard:
if DEBUG:
logger().info("%s.books():" % self.__class__.__name__)
@ -860,6 +861,7 @@ class ITUNES(DriverBase):
Note that most of the initialization is necessarily performed in can_handle(), as
we need to talk to iTunes to discover if there's a connected iPod
'''
from calibre.utils.zipfile import ZipFile
if self.iTunes is None:
raise OpenFeedback(self.ITUNES_SANDBOX_LOCKOUT_MESSAGE)
@ -881,6 +883,7 @@ class ITUNES(DriverBase):
if False:
# Display a dialog recommending using 'Connect to iTunes' if user hasn't
# previously disabled the dialog
from calibre.utils.config import dynamic
if dynamic.get(confirm_config_name(self.DISPLAY_DISABLE_DIALOG), True):
raise AppleOpenFeedback(self)
else:
@ -930,6 +933,7 @@ class ITUNES(DriverBase):
NB: This will not find books that were added by a different installation of calibre
as uuids are different
'''
from calibre.utils.zipfile import ZipFile
if DEBUG:
logger().info("%s.remove_books_from_metadata()" % self.__class__.__name__)
for path in paths:
@ -1429,6 +1433,7 @@ class ITUNES(DriverBase):
as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
'''
from PIL import Image as PILImage
from calibre.utils.zipfile import ZipFile
if DEBUG:
logger().info(" %s._cover_to_thumb()" % self.__class__.__name__)
@ -1557,6 +1562,7 @@ class ITUNES(DriverBase):
def _create_new_book(self, fpath, metadata, path, db_added, lb_added, thumb, format):
'''
'''
from calibre.utils.date import parse_date
if DEBUG:
logger().info(" %s._create_new_book()" % self.__class__.__name__)
@ -1761,6 +1767,7 @@ class ITUNES(DriverBase):
'''
'''
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.utils.zipfile import ZipFile
logger().info(" %s.__get_epub_metadata()" % self.__class__.__name__)
title = None
@ -2014,6 +2021,7 @@ class ITUNES(DriverBase):
as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
'''
from PIL import Image as PILImage
from calibre.utils.zipfile import ZipFile
if not self.settings().extra_customization[self.CACHE_COVERS]:
thumb_data = None
@ -2126,6 +2134,7 @@ class ITUNES(DriverBase):
'''
Calculate the exploded size of file
'''
from calibre.utils.zipfile import ZipFile
exploded_file_size = compressed_size
format = file.rpartition('.')[2].lower()
if format == 'epub':
@ -2478,6 +2487,7 @@ class ITUNES(DriverBase):
'''
if DEBUG:
import platform
logger().info(" %s %s" % (__appname__, __version__))
logger().info(" [OSX %s, %s %s (%s), %s driver version %d.%d.%d]" %
(platform.mac_ver()[0],
@ -2622,7 +2632,7 @@ class ITUNES(DriverBase):
# for deletion from booklist[0] during add_books_to_metadata
for book in self.cached_books:
if (self.cached_books[book]['uuid'] == metadata.uuid or
(self.cached_books[book]['title'] == metadata.title and \
(self.cached_books[book]['title'] == metadata.title and
self.cached_books[book]['author'] == metadata.author)):
self.update_list.append(self.cached_books[book])
if DEBUG:
@ -2776,8 +2786,10 @@ class ITUNES(DriverBase):
def _update_epub_metadata(self, fpath, metadata):
'''
'''
from calibre.utils.date import parse_date, now
from calibre.ebooks.metadata.epub import set_metadata
from lxml import etree
from calibre.utils.zipfile import ZipFile
if DEBUG:
logger().info(" %s._update_epub_metadata()" % self.__class__.__name__)
@ -3248,6 +3260,7 @@ class ITUNES_ASYNC(ITUNES):
list of device books.
"""
from calibre.utils.date import parse_date
if not oncard:
if DEBUG:
logger().info("%s.books()" % self.__class__.__name__)
@ -3418,6 +3431,7 @@ class ITUNES_ASYNC(ITUNES):
Note that most of the initialization is necessarily performed in can_handle(), as
we need to talk to iTunes to discover if there's a connected iPod
'''
from calibre.utils.zipfile import ZipFile
if self.iTunes is None:
raise OpenFeedback(self.ITUNES_SANDBOX_LOCKOUT_MESSAGE)

19
src/calibre/devices/eb600/driver.py
View File

@ -60,13 +60,24 @@ class TOLINO(EB600):
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['_TELEKOMTOLINO']
def linux_swap_drives(self, drives):
if len(drives) < 2 or not drives[1] or not drives[2]: return drives
if len(drives) < 2 or not drives[1] or not drives[2]:
return drives
drives = list(drives)
t = drives[0]
drives[0] = drives[1]
drives[1] = t
return tuple(drives)
def windows_sort_drives(self, drives):
if len(drives) < 2:
return drives
main = drives.get('main', None)
carda = drives.get('carda', None)
if main and carda:
drives['main'] = carda
drives['carda'] = main
return drives
class COOL_ER(EB600):
name = 'Cool-er device interface'
@ -94,13 +105,11 @@ class SHINEBOOK(EB600):
MAIN_MEMORY_VOLUME_LABEL = 'ShineBook Main Memory'
STORAGE_CARD_VOLUME_LABEL = 'ShineBook Storage Card'
@classmethod
def can_handle(cls, dev, debug=False):
return dev[4] == 'ShineBook'
class POCKETBOOK360(EB600):
# Device info on OS X
@ -113,7 +122,6 @@ class POCKETBOOK360(EB600):
PRODUCT_ID = [0x1688, 0xa4a5]
BCD = [0x110]
FORMATS = ['epub', 'fb2', 'prc', 'mobi', 'pdf', 'djvu', 'rtf', 'chm', 'txt']
VENDOR_NAME = ['PHILIPS', '__POCKET', 'POCKETBO']
@ -312,7 +320,8 @@ class POCKETBOOK701(USBMS):
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '__UMS_COMPOSITE'
def windows_sort_drives(self, drives):
if len(drives) < 2: return drives
if len(drives) < 2:
return drives
main = drives.get('main', None)
carda = drives.get('carda', None)
if main and carda:

4
src/calibre/devices/kobo/books.py
View File

@ -10,8 +10,7 @@ from calibre.utils.icu import sort_key
from calibre.devices.usbms.books import Book as Book_
from calibre.devices.usbms.books import CollectionsBookList
from calibre.utils.config import prefs
from calibre.utils.date import parse_date
from calibre.utils.config_base import prefs
from calibre.devices.usbms.driver import debug_print
from calibre.ebooks.metadata import author_to_author_sort
@ -19,6 +18,7 @@ class Book(Book_):
def __init__(self, prefix, lpath, title=None, authors=None, mime=None, date=None, ContentType=None,
thumbnail_name=None, size=None, other=None):
from calibre.utils.date import parse_date
# debug_print('Book::__init__ - title=', title)
show_debug = title is not None and title.lower().find("xxxxx") >= 0
if show_debug:

180
src/calibre/devices/kobo/driver.py
View File

@ -26,7 +26,7 @@ from calibre.devices.usbms.driver import USBMS, debug_print
from calibre import prints
from calibre.ptempfile import PersistentTemporaryFile
from calibre.constants import DEBUG
from calibre.utils.config import prefs
from calibre.utils.config_base import prefs
class KOBO(USBMS):
@ -35,11 +35,11 @@ class KOBO(USBMS):
gui_name = 'Kobo Reader'
description = _('Communicate with the Kobo Reader')
author = 'Timothy Legge and David Forrester'
version = (2, 0, 7)
version = (2, 0, 9)
dbversion = 0
fwversion = 0
supported_dbversion = 75
supported_dbversion = 80
has_kepubs = False
supported_platforms = ['windows', 'osx', 'linux']
@ -419,7 +419,7 @@ class KOBO(USBMS):
# If all this succeeds we need to delete the images files via the ImageID
return ImageID
def delete_images(self, ImageID):
def delete_images(self, ImageID, book_path):
if ImageID != None:
path_prefix = '.kobo/images/'
path = self._main_prefix + path_prefix + ImageID
@ -449,7 +449,7 @@ class KOBO(USBMS):
ImageID = self.delete_via_sql(ContentID, ContentType)
#print " We would now delete the Images for" + ImageID
self.delete_images(ImageID)
self.delete_images(ImageID, path)
if os.path.exists(path):
# Delete the ebook
@ -1193,21 +1193,30 @@ class KOBO(USBMS):
db.set_comment(db_id, mi.comments)
# Add bookmark file to db_id
db.add_format_with_hooks(db_id, bm.value.bookmark_extension,
bm.value.path, index_is_id=True)
# NOTE: As it is, this copied the book from the device back to the library. That meant it replaced the
# existing file. Taking this out for that reason, but some books have a ANNOT file that could be
# copied.
# db.add_format_with_hooks(db_id, bm.value.bookmark_extension,
# bm.value.path, index_is_id=True)
class KOBOTOUCH(KOBO):
name = 'KoboTouch'
gui_name = 'Kobo Touch'
gui_name = 'Kobo Touch/Glo/Mini/Aura HD'
author = 'David Forrester'
description = 'Communicate with the Kobo Touch, Glo and Mini firmware. Based on the existing Kobo driver by %s.' % (KOBO.author)
description = 'Communicate with the Kobo Touch, Glo, Mini and Aura HD ereaders. Based on the existing Kobo driver by %s.' % (KOBO.author)
# icon = I('devices/kobotouch.jpg')
supported_dbversion = 75
min_supported_dbversion = 53
min_dbversion_series = 65
min_dbversion_archive = 71
supported_dbversion = 80
min_supported_dbversion = 53
min_dbversion_series = 65
min_dbversion_archive = 71
min_dbversion_images_on_sdcard = 77
max_supported_fwversion = (2,5,1)
min_fwversion_images_on_sdcard = (2,4,1)
has_kepubs = True
booklist_class = KTCollectionsBookList
book_class = Book
@ -1291,12 +1300,13 @@ class KOBOTOUCH(KOBO):
TIMESTAMP_STRING = "%Y-%m-%dT%H:%M:%SZ"
GLO_PRODUCT_ID = [0x4173]
MINI_PRODUCT_ID = [0x4183]
TOUCH_PRODUCT_ID = [0x4163]
PRODUCT_ID = GLO_PRODUCT_ID + MINI_PRODUCT_ID + TOUCH_PRODUCT_ID
AURA_HD_PRODUCT_ID = [0x4193]
GLO_PRODUCT_ID = [0x4173]
MINI_PRODUCT_ID = [0x4183]
TOUCH_PRODUCT_ID = [0x4163]
PRODUCT_ID = AURA_HD_PRODUCT_ID + GLO_PRODUCT_ID + MINI_PRODUCT_ID + TOUCH_PRODUCT_ID
BCD = [0x0110, 0x0326]
BCD = [0x0110, 0x0326]
# Image file name endings. Made up of: image size, min_dbversion, max_dbversion,
COVER_FILE_ENDINGS = {
@ -1313,6 +1323,11 @@ class KOBOTOUCH(KOBO):
# ' - N3_LIBRARY_LIST.parsed':[(60,90),0, 53,],
# ' - N3_LIBRARY_SHELF.parsed': [(40,60),0, 52,],
}
AURA_HD_COVER_FILE_ENDINGS = {
' - N3_FULL.parsed': [(1080,1440), 0, 99,True,], # Used for screensaver, home screen
' - N3_LIBRARY_FULL.parsed':[(355, 471), 0, 99,False,], # Used for Details screen
' - N3_LIBRARY_GRID.parsed':[(149, 198), 0, 99,False,], # Used for library lists
}
#Following are the sizes used with pre2.1.4 firmware
# COVER_FILE_ENDINGS = {
# ' - N3_LIBRARY_FULL.parsed':[(355,530),0, 99,], # Used for Details screen
@ -1328,6 +1343,10 @@ class KOBOTOUCH(KOBO):
super(KOBOTOUCH, self).initialize()
self.bookshelvelist = []
def get_device_information(self, end_session=True):
self.set_device_name()
return super(KOBOTOUCH, self).get_device_information(end_session)
def books(self, oncard=None, end_session=True):
debug_print("KoboTouch:books - oncard='%s'"%oncard)
from calibre.ebooks.metadata.meta import path_to_ext
@ -1350,18 +1369,17 @@ class KOBOTOUCH(KOBO):
prefix = self._card_a_prefix if oncard == 'carda' else \
self._card_b_prefix if oncard == 'cardb' \
else self._main_prefix
debug_print("KoboTouch:books - prefix='%s'"%oncard)
debug_print("KoboTouch:books - oncard='%s', prefix='%s'"%(oncard, prefix))
# Determine the firmware version
try:
with open(self.normalize_path(self._main_prefix + '.kobo/version'),
'rb') as f:
with open(self.normalize_path(self._main_prefix + '.kobo/version'), 'rb') as f:
self.fwversion = f.readline().split(',')[2]
self.fwversion = tuple((int(x) for x in self.fwversion.split('.')))
except:
self.fwversion = 'unknown'
self.fwversion = (0,0,0)
if self.fwversion != '1.0' and self.fwversion != '1.4':
self.has_kepubs = True
debug_print('Kobo device: %s' % self.gui_name)
debug_print('Version of driver:', self.version, 'Has kepubs:', self.has_kepubs)
debug_print('Version of firmware:', self.fwversion, 'Has kepubs:', self.has_kepubs)
@ -1374,7 +1392,7 @@ class KOBOTOUCH(KOBO):
debug_print(opts.extra_customization)
if opts.extra_customization:
debugging_title = opts.extra_customization[self.OPT_DEBUGGING_TITLE]
debug_print("KoboTouch:books - set_debugging_title to", debugging_title )
debug_print("KoboTouch:books - set_debugging_title to '%s'" % debugging_title )
bl.set_debugging_title(debugging_title)
debug_print("KoboTouch:books - length bl=%d"%len(bl))
need_sync = self.parse_metadata_cache(bl, prefix, self.METADATA_CACHE)
@ -1466,6 +1484,7 @@ class KOBOTOUCH(KOBO):
if show_debug:
self.debug_index = idx
debug_print("KoboTouch:update_booklist - idx=%d"%idx)
debug_print("KoboTouch:update_booklist - lpath=%s"%lpath)
debug_print('KoboTouch:update_booklist - bl[idx].device_collections=', bl[idx].device_collections)
debug_print('KoboTouch:update_booklist - playlist_map=', playlist_map)
debug_print('KoboTouch:update_booklist - bookshelves=', bookshelves)
@ -1477,7 +1496,7 @@ class KOBOTOUCH(KOBO):
bl_cache[lpath] = None
if ImageID is not None:
imagename = self.imagefilename_from_imageID(ImageID)
imagename = self.imagefilename_from_imageID(prefix, ImageID)
if imagename is not None:
bl[idx].thumbnail = ImageWrapper(imagename)
if (ContentType == '6' and MimeType != 'application/x-kobo-epub+zip'):
@ -1717,12 +1736,14 @@ class KOBOTOUCH(KOBO):
debug_print("KoboTouch:books - end - oncard='%s'"%oncard)
return bl
def imagefilename_from_imageID(self, ImageID):
def imagefilename_from_imageID(self, prefix, ImageID):
show_debug = self.is_debugging_title(ImageID)
path = self.images_path(prefix)
path = self.normalize_path(path.replace('/', os.sep))
for ending, cover_options in self.cover_file_endings().items():
fpath = self._main_prefix + '.kobo/images/' + ImageID + ending
fpath = self.normalize_path(fpath.replace('/', os.sep))
fpath = path + ImageID + ending
if os.path.exists(fpath):
if show_debug:
debug_print("KoboTouch:imagefilename_from_imageID - have cover image fpath=%s" % (fpath))
@ -1764,7 +1785,7 @@ class KOBOTOUCH(KOBO):
if not self.copying_covers():
imageID = self.imageid_from_contentid(contentID)
self.delete_images(imageID)
self.delete_images(imageID, fname)
connection.commit()
cursor.close()
@ -1821,11 +1842,11 @@ class KOBOTOUCH(KOBO):
return imageId
def delete_images(self, ImageID):
def delete_images(self, ImageID, book_path):
debug_print("KoboTouch:delete_images - ImageID=", ImageID)
if ImageID != None:
path_prefix = '.kobo/images/'
path = self._main_prefix + path_prefix + ImageID
path = self.images_path(book_path)
path = path + ImageID
for ending in self.cover_file_endings().keys():
fpath = path + ending
@ -1872,12 +1893,14 @@ class KOBOTOUCH(KOBO):
def get_content_type_from_extension(self, extension):
debug_print("KoboTouch:get_content_type_from_extension - start")
# With new firmware, ContentType appears to be 6 for all types of sideloaded books.
if self.fwversion.startswith('2.'):
if self.fwversion >= (1,9,17) or extension == '.kobo' or extension == '.mobi':
debug_print("KoboTouch:get_content_type_from_extension - V2 firmware")
ContentType = 6
# For older firmware, it depends on the type of file.
elif extension == '.kobo' or extension == '.mobi':
ContentType = 6
else:
debug_print("KoboTouch:get_content_type_from_extension - calling super")
ContentType = super(KOBOTOUCH, self).get_content_type_from_extension(extension)
ContentType = 901
return ContentType
def update_device_database_collections(self, booklists, collections_attributes, oncard):
@ -1920,7 +1943,7 @@ class KOBOTOUCH(KOBO):
delete_empty_shelves = opts.extra_customization[self.OPT_DELETE_BOOKSHELVES] and self.supports_bookshelves()
update_series_details = opts.extra_customization[self.OPT_UPDATE_SERIES_DETAILS] and self.supports_series()
debugging_title = opts.extra_customization[self.OPT_DEBUGGING_TITLE]
debug_print("KoboTouch:update_device_database_collections - set_debugging_title to", debugging_title )
debug_print("KoboTouch:update_device_database_collections - set_debugging_title to '%s'" % debugging_title )
booklists.set_debugging_title(debugging_title)
else:
delete_empty_shelves = False
@ -2080,7 +2103,8 @@ class KOBOTOUCH(KOBO):
:param filepath: The full path to the ebook file
'''
# debug_print("KoboTouch:upload_cover - path='%s' filename='%s'"%(path, filename))
debug_print("KoboTouch:upload_cover - path='%s' filename='%s' "%(path, filename))
debug_print(" filepath='%s' "%(filepath))
opts = self.settings()
if not self.copying_covers():
@ -2088,8 +2112,8 @@ class KOBOTOUCH(KOBO):
# debug_print('KoboTouch: not uploading cover')
return
# Don't upload covers if book is on the SD card
if self._card_a_prefix and path.startswith(self._card_a_prefix):
# Only upload covers to SD card if that is supported
if self._card_a_prefix and os.path.abspath(path).startswith(os.path.abspath(self._card_a_prefix)) and not self.supports_covers_on_sdcard():
return
if not opts.extra_customization[self.OPT_UPLOAD_GRAYSCALE_COVERS]:
@ -2111,6 +2135,17 @@ class KOBOTOUCH(KOBO):
ImageID = ImageID.replace('.', '_')
return ImageID
def images_path(self, path):
if self._card_a_prefix and os.path.abspath(path).startswith(os.path.abspath(self._card_a_prefix)) and self.supports_covers_on_sdcard():
path_prefix = 'koboExtStorage/images/'
path = os.path.join(self._card_a_prefix, path_prefix)
else:
path_prefix = '.kobo/images/'
path = os.path.join(self._main_prefix, path_prefix)
return path
def _upload_cover(self, path, filename, metadata, filepath, uploadgrayscale, keep_cover_aspect=False):
from calibre.utils.magick.draw import save_cover_data_to, identify_data
debug_print("KoboTouch:_upload_cover - filename='%s' uploadgrayscale='%s' "%(filename, uploadgrayscale))
@ -2151,11 +2186,16 @@ class KOBOTOUCH(KOBO):
cursor.close()
if ImageID != None:
path_prefix = '.kobo/images/'
path = self._main_prefix + path_prefix + ImageID
path = os.path.join(self.images_path(path), ImageID)
if show_debug:
debug_print("KoboTouch:_upload_cover - About to loop over cover endings")
image_dir = os.path.dirname(os.path.abspath(path))
if not os.path.exists(image_dir):
debug_print("KoboTouch:_upload_cover - Image directory does not exust. Creating path='%s'" % (image_dir))
os.makedirs(image_dir)
for ending, cover_options in self.cover_file_endings().items():
resize, min_dbversion, max_dbversion, isFullsize = cover_options
if show_debug:
@ -2496,6 +2536,8 @@ class KOBOTOUCH(KOBO):
return opts
def isAuraHD(self):
return self.detected_device.idProduct in self.AURA_HD_PRODUCT_ID
def isGlo(self):
return self.detected_device.idProduct in self.GLO_PRODUCT_ID
def isMini(self):
@ -2504,7 +2546,21 @@ class KOBOTOUCH(KOBO):
return self.detected_device.idProduct in self.TOUCH_PRODUCT_ID
def cover_file_endings(self):
return self.GLO_COVER_FILE_ENDINGS if self.isGlo() else self.COVER_FILE_ENDINGS
return self.GLO_COVER_FILE_ENDINGS if self.isGlo() else self.AURA_HD_COVER_FILE_ENDINGS if self.isAuraHD() else self.COVER_FILE_ENDINGS
def set_device_name(self):
device_name = self.gui_name
if self.isAuraHD():
device_name = 'Kobo Aura HD'
elif self.isGlo():
device_name = 'Kobo Glo'
elif self.isMini():
device_name = 'Kobo Mini'
elif self.isTouch():
device_name = 'Kobo Touch'
self.__class__.gui_name = device_name
return device_name
def copying_covers(self):
opts = self.settings()
@ -2524,6 +2580,44 @@ class KOBOTOUCH(KOBO):
def supports_kobo_archive(self):
return self.dbversion >= self.min_dbversion_archive
def supports_covers_on_sdcard(self):
return self.dbversion >= 77 and self.fwversion >= self.min_fwversion_images_on_sdcard
def modify_database_check(self, function):
# Checks to see whether the database version is supported
# and whether the user has chosen to support the firmware version
# debug_print("KoboTouch:modify_database_check - self.fwversion <= self.max_supported_fwversion=", self.fwversion > self.max_supported_fwversion)
if self.dbversion > self.supported_dbversion or self.fwversion > self.max_supported_fwversion:
# Unsupported database
opts = self.settings()
if not opts.extra_customization[self.OPT_SUPPORT_NEWER_FIRMWARE]:
debug_print('The database has been upgraded past supported version')
self.report_progress(1.0, _('Removing books from device...'))
from calibre.devices.errors import UserFeedback
raise UserFeedback(_("Kobo database version unsupported - See details"),
_('Your Kobo is running an updated firmware/database version.'
' As calibre does not know about this updated firmware,'
' database editing is disabled, to prevent corruption.'
' You can still send books to your Kobo with calibre, '
' but deleting books and managing collections is disabled.'
' If you are willing to experiment and know how to reset'
' your Kobo to Factory defaults, you can override this'
' check by right clicking the device icon in calibre and'
' selecting "Configure this device" and then the '
' "Attempt to support newer firmware" option.'
' Doing so may require you to perform a factory reset of'
' your Kobo.'
),
UserFeedback.WARN)
return False
else:
# The user chose to edit the database anyway
return True
else:
# Supported database version
return True
@classmethod
def is_debugging_title(cls, title):

20
src/calibre/devices/misc.py
View File

@ -95,7 +95,6 @@ class PDNOVEL(USBMS):
SUPPORTS_SUB_DIRS = False
DELETE_EXTS = ['.jpg', '.jpeg', '.png']
def upload_cover(self, path, filename, metadata, filepath):
coverdata = getattr(metadata, 'thumbnail', None)
if coverdata and coverdata[2]:
@ -226,9 +225,9 @@ class TREKSTOR(USBMS):
VENDOR_ID = [0x1e68]
PRODUCT_ID = [0x0041, 0x0042, 0x0052, 0x004e, 0x0056,
0x0067, # This is for the Pyrus Mini
0x003e, # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
0x5cL, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=191318
0x0067, # This is for the Pyrus Mini
0x003e, # This is for the EBOOK_PLAYER_5M https://bugs.launchpad.net/bugs/792091
0x5cL, # This is for the 4ink http://www.mobileread.com/forums/showthread.php?t=191318
]
BCD = [0x0002, 0x100]
@ -427,8 +426,8 @@ class WAYTEQ(USBMS):
EBOOK_DIR_MAIN = 'Documents'
SCAN_FROM_ROOT = True
VENDOR_NAME = 'ROCKCHIP'
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'RK28_SDK_DEMO'
VENDOR_NAME = ['ROCKCHIP', 'CBR']
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['RK28_SDK_DEMO', 'EINK_EBOOK_READE']
SUPPORTS_SUB_DIRS = True
def get_gui_name(self):
@ -445,7 +444,8 @@ class WAYTEQ(USBMS):
return self.EBOOK_DIR_CARD_A
def windows_sort_drives(self, drives):
if len(drives) < 2: return drives
if len(drives) < 2:
return drives
main = drives.get('main', None)
carda = drives.get('carda', None)
if main and carda:
@ -455,7 +455,8 @@ class WAYTEQ(USBMS):
def linux_swap_drives(self, drives):
# See https://bugs.launchpad.net/bugs/1151901
if len(drives) < 2 or not drives[1] or not drives[2]: return drives
if len(drives) < 2 or not drives[1] or not drives[2]:
return drives
drives = list(drives)
t = drives[0]
drives[0] = drives[1]
@ -463,7 +464,8 @@ class WAYTEQ(USBMS):
return tuple(drives)
def osx_sort_names(self, names):
if len(names) < 2: return names
if len(names) < 2:
return names
main = names.get('main', None)
card = names.get('carda', None)

6
src/calibre/devices/mtp/driver.py
View File

@ -17,8 +17,6 @@ from calibre.devices.errors import PathError
from calibre.devices.mtp.base import debug
from calibre.devices.mtp.defaults import DeviceDefaults
from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory
from calibre.utils.config import from_json, to_json, JSONConfig
from calibre.utils.date import now, isoformat, utcnow
from calibre.utils.filenames import shorten_components_to
BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%(
@ -57,6 +55,7 @@ class MTP_DEVICE(BASE):
@property
def prefs(self):
from calibre.utils.config import JSONConfig
if self._prefs is None:
self._prefs = p = JSONConfig('mtp_devices')
p.defaults['format_map'] = self.FORMATS
@ -103,6 +102,7 @@ class MTP_DEVICE(BASE):
del self.prefs[x]
def open(self, device, library_uuid):
from calibre.utils.date import isoformat, utcnow
self.current_library_uuid = library_uuid
self.location_paths = None
self.driveinfo = {}
@ -128,6 +128,8 @@ class MTP_DEVICE(BASE):
# Device information {{{
def _update_drive_info(self, storage, location_code, name=None):
from calibre.utils.date import isoformat, now
from calibre.utils.config import from_json, to_json
import uuid
f = storage.find_path((self.DRIVEINFO,))
dinfo = {}

2
src/calibre/devices/prs505/driver.py
View File

@ -11,7 +11,6 @@ import os, time, re
from calibre.devices.usbms.driver import USBMS, debug_print
from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \
MEDIA_THUMBNAIL, CACHE_THUMBNAIL
from calibre.devices.prs505.sony_cache import XMLCache
from calibre import __appname__, prints
from calibre.devices.usbms.books import CollectionsBookList
@ -178,6 +177,7 @@ class PRS505(USBMS):
return fname
def initialize_XML_cache(self):
from calibre.devices.prs505.sony_cache import XMLCache
paths, prefixes, ext_paths = {}, {}, {}
for prefix, path, ext_path, source_id in [
('main', MEDIA_XML, MEDIA_EXT, 0),

9
src/calibre/devices/smart_device_app/driver.py
View File

@ -7,7 +7,7 @@ Created on 29 Jun 2012
@author: charles
'''
import socket, select, json, inspect, os, traceback, time, sys, random
import socket, select, json, os, traceback, time, sys, random
import posixpath
import hashlib, threading
import Queue
@ -34,8 +34,7 @@ from calibre.library import current_library_name
from calibre.library.server import server_config as content_server_config
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.ipc import eintr_retry_call
from calibre.utils.config import from_json, tweaks
from calibre.utils.date import isoformat, now
from calibre.utils.config_base import tweaks
from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
unpublish_zeroconf, get_all_ips)
@ -345,6 +344,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
def _debug(self, *args):
# manual synchronization so we don't lose the calling method name
import inspect
with self.sync_lock:
if not DEBUG:
return
@ -373,6 +373,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# copied from USBMS. Perhaps this could be a classmethod in usbms?
def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
from calibre.utils.date import isoformat, now
import uuid
if not isinstance(dinfo, dict):
dinfo = {}
@ -593,6 +594,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
raise ControlError(desc='Device responded with incorrect information')
def _receive_from_client(self, print_debug_info=True):
from calibre.utils.config import from_json
extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
try:
v = self._read_string_from_net()
@ -816,6 +818,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock')
def open(self, connected_device, library_uuid):
from calibre.utils.date import isoformat, now
self._debug()
if not self.is_connected:
# We have been called to retry the connection. Give up immediately

4
src/calibre/devices/teclast/driver.py
View File

@ -58,8 +58,8 @@ class PICO(NEWSMY):
gui_name = 'Pico'
description = _('Communicate with the Pico reader.')
VENDOR_NAME = ['TECLAST', 'IMAGIN', 'LASER-', '']
WINDOWS_MAIN_MEM = ['USBDISK__USER', 'EB720']
VENDOR_NAME = ['TECLAST', 'IMAGIN', 'LASER-', 'LASER', '']
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['USBDISK__USER', 'EB720', 'EBOOK-EB720']
EBOOK_DIR_MAIN = 'Books'
FORMATS = ['EPUB', 'FB2', 'TXT', 'LRC', 'PDB', 'PDF', 'HTML', 'WTXT']
SCAN_FROM_ROOT = True

5
src/calibre/devices/usbms/books.py
View File

@ -12,9 +12,8 @@ from calibre.devices.mime import mime_type_ext
from calibre.devices.interface import BookList as _BookList
from calibre.constants import preferred_encoding
from calibre import isbytestring, force_unicode
from calibre.utils.config import device_prefs, tweaks
from calibre.utils.config_base import tweaks
from calibre.utils.icu import sort_key
from calibre.utils.formatter import EvalFormatter
class Book(Metadata):
def __init__(self, prefix, lpath, size=None, other=None):
@ -109,6 +108,7 @@ class CollectionsBookList(BookList):
return None
def compute_category_name(self, field_key, field_value, field_meta):
from calibre.utils.formatter import EvalFormatter
renames = tweaks['sony_collection_renaming_rules']
field_name = renames.get(field_key, None)
if field_name is None:
@ -124,6 +124,7 @@ class CollectionsBookList(BookList):
def get_collections(self, collection_attributes):
from calibre.devices.usbms.driver import debug_print
from calibre.utils.config import device_prefs
debug_print('Starting get_collections:', device_prefs['manage_device_metadata'])
debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
debug_print('Formatting template:', tweaks['sony_collection_name_template'])

2
src/calibre/devices/usbms/deviceconfig.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL 3'
__copyright__ = '2009, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from calibre.utils.config import Config, ConfigProxy
from calibre.utils.config_base import Config, ConfigProxy
class DeviceConfig(object):

4
src/calibre/devices/usbms/driver.py
View File

@ -20,8 +20,6 @@ from calibre.devices.usbms.cli import CLI
from calibre.devices.usbms.device import Device
from calibre.devices.usbms.books import BookList, Book
from calibre.ebooks.metadata.book.json_codec import JsonCodec
from calibre.utils.config import from_json, to_json
from calibre.utils.date import now, isoformat
BASE_TIME = None
def debug_print(*args):
@ -58,6 +56,7 @@ class USBMS(CLI, Device):
SCAN_FROM_ROOT = False
def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
from calibre.utils.date import now, isoformat
import uuid
if not isinstance(dinfo, dict):
dinfo = {}
@ -75,6 +74,7 @@ class USBMS(CLI, Device):
return dinfo
def _update_driveinfo_file(self, prefix, location_code, name=None):
from calibre.utils.config import from_json, to_json
if os.path.exists(os.path.join(prefix, self.DRIVEINFO)):
with open(os.path.join(prefix, self.DRIVEINFO), 'rb') as f:
try:

9
src/calibre/ebooks/conversion/plugins/epub_input.py
View File

@ -188,7 +188,6 @@ class EPUBInput(InputFormatPlugin):
raise DRMError(os.path.basename(path))
self.encrypted_fonts = self._encrypted_font_uris
if len(parts) > 1 and parts[0]:
delta = '/'.join(parts[:-1])+'/'
for elem in opf.itermanifest():
@ -207,9 +206,11 @@ class EPUBInput(InputFormatPlugin):
not_for_spine = set()
for y in opf.itermanifest():
id_ = y.get('id', None)
if id_ and y.get('media-type', None) in \
('application/vnd.adobe-page-template+xml','application/text'):
not_for_spine.add(id_)
if id_ and y.get('media-type', None) in {
'application/vnd.adobe-page-template+xml', 'application/vnd.adobe.page-template+xml',
'application/adobe-page-template+xml', 'application/adobe.page-template+xml',
'application/text'}:
not_for_spine.add(id_)
seen = set()
for x in list(opf.iterspine()):

2
src/calibre/ebooks/conversion/plugins/html_input.py
View File

@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'
import re, tempfile, os
from functools import partial
from itertools import izip
from urllib import quote
from calibre.constants import islinux, isbsd
from calibre.customize.conversion import (InputFormatPlugin,
@ -223,6 +222,7 @@ class HTMLInput(InputFormatPlugin):
return link, frag
def resource_adder(self, link_, base=None):
from urllib import quote
link, frag = self.link_to_local_path(link_, base=base)
if link is None:
return link_

5
src/calibre/ebooks/conversion/plugins/html_output.py
View File

@ -4,12 +4,15 @@ __copyright__ = '2010, Fabian Grassl <fg@jusmeum.de>'
__docformat__ = 'restructuredtext en'
import os, re, shutil
from os.path import dirname, abspath, relpath, exists, basename
from os.path import dirname, abspath, relpath as _relpath, exists, basename
from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation
from calibre import CurrentDir
from calibre.ptempfile import PersistentTemporaryDirectory
def relpath(*args):
return _relpath(*args).replace(os.sep, '/')
class HTMLOutput(OutputFormatPlugin):
name = 'HTML Output'

2
src/calibre/ebooks/conversion/plugins/pml_input.py
View File

@ -10,7 +10,6 @@ import shutil
from calibre.customize.conversion import InputFormatPlugin
from calibre.ptempfile import TemporaryDirectory
from calibre.utils.zipfile import ZipFile
class PMLInput(InputFormatPlugin):
@ -86,6 +85,7 @@ class PMLInput(InputFormatPlugin):
accelerators):
from calibre.ebooks.metadata.toc import TOC
from calibre.ebooks.metadata.opf2 import OPFCreator
from calibre.utils.zipfile import ZipFile
self.options = options
self.log = log

7
src/calibre/ebooks/conversion/plugins/txt_input.py
View File

@ -63,7 +63,6 @@ class TXTInput(InputFormatPlugin):
normalize_line_endings, convert_textile, remove_indents,
block_to_single_line, separate_hard_scene_breaks)
self.log = log
txt = ''
log.debug('Reading text from file...')
@ -92,6 +91,12 @@ class TXTInput(InputFormatPlugin):
log.debug('Using user specified input encoding of %s' % ienc)
else:
det_encoding = detect(txt)
if det_encoding and det_encoding.lower().replace('_', '-').strip() in (
'gb2312', 'chinese', 'csiso58gb231280', 'euc-cn', 'euccn',
'eucgb2312-cn', 'gb2312-1980', 'gb2312-80', 'iso-ir-58'):
# Microsoft Word exports to HTML with encoding incorrectly set to
# gb2312 instead of gbk. gbk is a superset of gb2312, anyway.
det_encoding = 'gbk'
ienc = det_encoding['encoding']
log.debug('Detected input encoding as %s with a confidence of %s%%' % (ienc, det_encoding['confidence'] * 100))
if not ienc:

11
src/calibre/ebooks/docx/__init__.py Normal file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
class InvalidDOCX(ValueError):
pass

Some files were not shown because too many files have changed in this diff Show More