Merge from Calibre Root

.bzrignore

@@ -3,9 +3,9 @@
 src/calibre/plugins
 resources/images.qrc
 src/calibre/ebooks/oeb/display/test/*.js
-src/calibre/manual/.build/
+manual/.build/
-src/calibre/manual/cli/
+manual/cli/
-src/calibre/manual/template_ref.rst
+manual/template_ref.rst
 build
 dist
 docs
@@ -16,7 +16,6 @@ resources/ebook-convert-complete.pickle
 resources/builtin_recipes.xml
 resources/builtin_recipes.zip
 resources/template-functions.json
-resources/display/*.js
 setup/installer/windows/calibre/build.log
 src/calibre/translations/.errors
 src/cssutils/.svn/

COPYRIGHT

@@ -4,11 +4,6 @@ License: GPL-3
  The full text of the GPL is distributed as in
  /usr/share/common-licenses/GPL-3 on Debian systems.
 
-Files: src/calibre/ebooks/pdf/*.h,*.cpp
-License: GPL-2 or later
- The full text of the GPL is distributed as in
- /usr/share/common-licenses/GPL-2 on Debian systems.
-
 Files: setup/iso_639/*
 Copyright: Various
 License: LGPL 2.1

Changelog.yaml

@@ -19,6 +19,428 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.54
+  date: 2012-05-31
+
+  new features:
+    - title: "E-book viewer: The Table of contents panel now tracks the current position in the book. As you scroll through the book, the entry you are currently on is highlighted."
+      type: major
+      description: "To see this feature in action, open the Table of Contents panel in the viewer by clicking the button with three blue lines on it. As you page through the book, the chapter you are reading currently is highlighted in the Table of Contents Panel. Obviously, this will only work if the book you are reading has a Table of Contents. You can also use the Ctrl+PgUp and Ctrl+PgDn keys to quickly skip between chapters."
+
+    - title: "calibredb: Allow setting metadata for individual fields with the set_metadata command"
+
+    - title: "Make it a little harder to accidentally change the sorting of items in the Tag Browser. Also frees up more vertical space for the Tag Browser itself."
+
+    - title: "The calibre user manual is now available in AZW3 format as well as EPUB"
+
+  bug fixes:
+    - title: "Automatic titlecasing: No longer try to capitalize scottish names, as there are too many special cases." 
+      tickets: [775825] 
+
+    - title: "Never crash when reading metadata from PDF files (reading now always happens in a worker process)"
+      tickets: [1006452]
+
+    - title: "EPUB Input: Do no skip the valid children of an NCX node that has no text/href"
+
+    - title: "Archos driver: Detect SD card"
+      tickets: [1005650] 
+
+    - title: "When bulk downloading metadata and the user deletes one of the books for which metadata is being downloaded, just ignore it, instead of erroring out"
+
+    - title: "When deleting books from the bottom of the booklist, ensure that the bottom book after deleting is selected"
+
+    - title: "Fix regression in 0.8.53 that broke sending APNX files to older Kindle devices"
+
+    - title: "Use correct text color for selected rows in the list of matches when downloading metadata and showing results in get books."
+      tickets: [1004568]
+
+  improved recipes:
+    - The Independent
+    - Welt der Physik
+    - China Daily
+    - The Grid
+    - Prospect Magazine
+
+  new recipes:
+    - title: La gazetta del Mezzogiorno 
+      author: faber1971
+
+- version: 0.8.53
+  date: 2012-05-25
+
+  new features:
+    - title: "Kindle Touch/4 driver: Upload cover thumbnails when sending books to device by USB to workaround Amazon bug of not displaying covers for sync-enabled books"
+
+    - title: "Support for updating metadata in FB2 files"
+
+    - title: "Set a different background color when choosing formats to not delete as opposed to choosing format to delete."
+      tickets: [ 1001741 ] 
+
+    - title: "E-book viewer: Add an option to prevent the up and down arrow keys from scrolling past page breaks"
+
+    - title: "Get Books: Remove ebookshoppe.com at the website's request"
+ 
+  bug fixes:
+    - title: "PDF Input: Support image rotation commands in PDF files. Fixes the long standing problem of some  images being flipped when converting from PDF in calibre."
+
+    - title: "Fix a regression in 0.8.51 that caused conversion to HTMLZ to not have any CSS"
+
+    - title: "Get Books: Fix website change at kobo.com causing prices not to be found"
+
+    - title: "Edit the time in the 24 hour clock when calibre's interface language is set to German."
+      tickets: [ 1001809 ] 
+
+    - title: "MOBI Output: When generating joint KF8/MOBI6 .mobi files set the text length field in the MOBI 6 header correctly. "
+      tickets: [ 1003489 ] 
+
+    - title: "ODT Input: More workarounds for LibreOffice 3.5's habit of inserting pointless margin:100% directives everywhere."
+      tickets: [ 1002702 ] 
+
+    - title: "Fix regression that broke smarten punctuation when quotes were next to html tags."
+      tickets: [ 998900 ] 
+
+    - title: "Fix published date from ozon.ru wrong in some timezones"
+      tickets: [ 975338 ]
+
+    - title: "Catalogs: Handle the use of custom columns with non-ascii names correctly"
+      tickets: [1001437]
+
+    - title: "Conversion pipeline: Remove the attempt to detect and autocorrect if text will go off the left edge of the page, as it was a rather crude heuristic. Also do not remove fake margins if the book uses negative text indents on the margined elements."
+
+    - title: "KF8 Output: Set offsets to tags in the skeleton the same way kindlegen does. Also linearize non linear ToCs to ensure section to section jumping works."
+
+    - title: "Conversion pipeline: Use correct default value of 'inherit' for font-family and font-size when normalizing the shorthand font property."
+
+    - title: "When running python scripts via calibre-debug ensure that user plugins are loaded"
+
+  improved recipes:
+    - Business Week Magazine
+    - Metro Nieuws NL
+
+  new recipes:
+    - title: Attac.es 
+      author: Marc Busque
+
+    - title: Drytooling.com
+      author: Damian Granowski
+
+    - title: Shortlist.com 
+      author: Dave ASbury
+
+    - title: National Geographic (es) 
+      author: vakya
+
+- version: 0.8.52
+  date: 2012-05-18
+
+  new features:
+    - title: "EPUB Input: When setting the cover for a book that identifies its cover image, but not the html wrapper around the cover, try to detect and remove that wrapper automatically."
+      tickets: [ 999959 ] 
+
+    - title: "When deleting books of a specific format, show the number of books with each format available"
+
+    - title: "Linux install: No longer create MAN pages as all utilities have more comprehensive command line --help anyway"
+
+    - title: "Add a tweak Preferences->Tweaks to control the default choice of format for the Tweak Book feature"
+
+    - title: "Conversion: Allow setting negative page margins. A negative page margin means that calibre will not specify any page margin in the output document (for formats that support this)"
+ 
+  bug fixes:
+    - title: "Tweak book: Fix handling of covers when tweaking KF8 books"
+
+    - title: "KF8 Output: Handle input documents with out of sequence ToC entries. Note that currently section jumping in the KF8 output produced by calibre for such files does not work."
+      tickets: [1000493]
+
+    - title: "Edit metadata dialog: Fix the edit values button for custom tag-like columns showing a unneeded warning about changed values"
+
+    - title: "EPUB Output: Be a little more conservative when removing <form> tags. Only remove them if they have actual forms inside. "
+      tickets: [ 1000384 ]
+
+    - title: "EPUB Input: Correctly update the Cover entry in the ToC even when the entry has a fragment reference. "
+      tickets: [ 999973 ]
+
+    - title: "Update ImagMagick DLLs in all calibre binary builds to fix security vulnerabilities in ImageMagick"
+      tickets: [ 999496 ] 
+
+    - title: "Advanced search dialog: Fix equals and regex matching not being applied for custom column searches."
+      tickets: [ 980221 ] 
+
+    - title: "RTF Input: Handle old RTF files that have commands without braces."
+      tickets: [ 994133 ] 
+
+    - title: "Get Books: Diesel, fix results not showing when only a single match is found"
+
+    - title: "Get Books: Fix DRM status indicators for Kobo and Diesel stores. Fix smashwords not returning results."
+      tickets: [ 993755 ] 
+
+    - title: "Fix regression in 0.8.51 that broke viewing of LIT and some EPUB files"
+      tickets: [998248, 998216] 
+
+  improved recipes:
+    - Clarin
+    - Spiegel
+    - Spiegel International
+    - Montreal Gazette
+    - Gosc Niedzelny
+    - Ars Technica
+
+  new recipes:
+    - title: "Army/Navy/Air force/Marine Times and News busters" 
+      author: jde
+
+    - title: "Ads of the World, Heavy Meta (Italian) and Juve La Stampa"
+      author: faber1971
+
+    - title: "Revista Summa"
+      author: Vakya
+
+    - title: "Strategic culture"
+      author: Darko Miletic
+
+    - title: Stars and Stripes 
+      author: adoucette
+
+    - title: Nackdenkseiten 
+      author: jrda
+
+
+- version: 0.8.51
+  date: 2012-05-11
+
+  new features:
+    - title: "When switching libraries preserve the position and selected books if you switch back to a previously opened library."
+      tickets: [994514]
+
+    - title: "Conversion pipeline: Filter out the useless font-face rules inserted by Microsoft Word for every font on the system"
+
+    - title: "Driver for Motorola XT875 and Pandigital SuperNova"
+      tickets: [996890]
+
+    - title: "Add a colour swatch the the dialog for creating column coloring rules, to ease selection of colors"
+      tickets: [994811] 
+
+    - title: "EPUB Output: Consolidate internal CSS generated by calibre into external stylesheets for ease of editing the EPUB"
+
+    - title: "List EPUB and MOBI at the top of the dropdown list fo formats to convert to, as they are the most common choices"
+      tickets: [994838] 
+
+  bug fixes:
+    - title: "E-book viewer: Improve performance when switching between normal and fullscreen views."
+      tickets: [996102]
+
+    - title: "Edit metadata dialog: When running download metadata do not insert duplicate tags into the list of tags"
+
+    - title: "KF8 Input: Do not error out if the file has a few invalidly encoded bytes."
+      tickets: [997034]
+
+    - title: "Fix download of news in AZW3 format not working"
+      tickets: [996439]
+
+    - title: "Pocketbook driver: Update for new PB 611 firmware."
+      tickets: [903079]
+
+    - title: "ebook-convert: Error out if the user prvides extra command line args instead of silently ignoring them"
+      tickets: [994939] 
+
+    - title: "EPUB Output: Do not self close any container tags to prevent artifacts when EPUBs are viewed using buggy browser based viewers."
+      tickets: [994861]
+
+    - title: "Fix regression in 0.8.50 that broke the conversion of HTML files that contained non-ascii font-face declarations, typically produced by Microsoft Word"
+
+  improved recipes:
+    - Mainichi news
+    - derStandard
+    - Endgadget Japan
+
+  new recipes:
+    - title: Mainichi English
+      author: Hiroshi Miura
+
+    - title: The Grid TO
+      author: Yusuf W
+
+    - title: National Geographic (Italy)
+      author: faber1971
+
+    - title: Rebelion 
+      author: Marc Busque
+
+- version: 0.8.50
+  date: 2012-05-04
+
+  new features:
+    - title: "Tweak Book: Allow tweaking of KF8 MOBI files. Useful to fine-tune the result of a conversion. Right click on the book and select Tweak Book to use the feature. Note that tweaking a MOBI file that contains both KF8 and older MOBI6 will cause the MOBI6 version to be discarded."
+
+    - title: "AZW3 output plugin. This output plugin generates pure KF8 mobi files. These only work on the Kindle Fire and Kindle Touch with latest firmware."
+
+    - title: "Conversion: Allow easy re-ordering of the search and replace expressions in the conversion dialog. Also apply the expressions in the same order that they were entered when doing the conversion."
+
+    - title: "Automatically add the Tag 'Sample Book' when an Amazon sample is added to calibre"
+
+    - title: "FB2 Input: Better handling of inline images."
+      tickets: [989869]
+ 
+  bug fixes:
+    - title: "KF8 Output: Fix section to section jumps not working for documents with multi-level ToCs"
+
+    - title: "EPUB Input: Handle the case of the metadata ToC containing a reference to the cover HTML file."
+      tickets: [993812]
+
+    - title: "CHM Input: Handle files with deeply nested markup and non html files listed at the start of the manifest."
+      tickets: [993607]
+
+    - title: "KF8 Output: Workaround Kindle Touch bug that causes the book to be rendered as black pages when a height is specified for <body>"
+
+    - title: "Fix regression in 0.8.49 that broke italics detection in heuristic processing on 32-bit systems."
+      tickets: [991380] 
+
+    - title: "KF8 Output: Fix joint MOBI6/KF8 books not being recognized as MOBI files by older Kindles"
+
+    - title: "KF8 Output: Fix errors when processing documents with HTML comments and/or XML processing instructions"
+
+    - title: "Get Books: Amazon fix prices not being found. B&N fix details link. ebooks.com: fix cover image. Website changes to various EU stores"
+
+    - title: "FB2 Input: More robust base64 decoding to handle embedded images that are incorrectly encoded."
+      tickets: [990929]
+
+    - title: "Fix scrolling with the cover browser updating only the selection in the book list, not the current book."
+      tickets: [990881] 
+
+    - title: "Save to Disk: Do not run out memory when saving very large files on systems with low RAM."
+      tickets: [990741]
+
+    - title: "FB2 Output: Use 2 letter language codes in preference to 3-letter ones to not break poorly implemented FB2 readers"
+      tickets: [990026] 
+
+    - title: "EPUB Input: Auto set the media-type for OPF manifest entries with an empty media-type"
+
+  improved recipes:
+    - National Post
+    - Daily Mirror
+    - Sun
+    - Newsweek Polska
+    - Max-Planck
+    - derStandard
+    - tweakers.net
+
+  new recipes:
+    - title: George Monbiot 
+      author: Darko Miletic
+
+    - title: El Mundo
+      author: atordo
+
+    - title: AraInfo and Diagonal 
+      author: Ruben Pollan
+
+
+- version: 0.8.49
+  date: 2012-04-27
+
+  new features:
+    - title: "Experimental support for generating Amazon's new KF8 format MOBI files"
+      description: "calibre can now generate Amazon's new KF8 format MOBI files.
+      To turn on this feature, go to Preferences->Tweaks and click Plugin Tweaks. In the box add:
+      test_mobi_output_type = 'both'
+      calibre will now produce MOBI files that have both the old MOBI format and the new KF8 format in them. 
+      To learn more about KF8, see: http://www.amazon.com/gp/feature.html?docId=1000729511
+      Note that calibre support for KF8 is still experimental and there will likely be bugs."
+
+    - title: "Upgrade to using cssutils 0.9.9 for CSS parsing. Improved speed and robustness."
+
+    - title: "Show cover size in a tooltip in the conversion dialog"
+      tickets: [986958]
+
+    - title: "Driver for Nook Simple Touch with Glow Light"
+      tickets: [989264]
+ 
+  bug fixes:
+    - title: "Heuristics: When italicizing words do not operate on words not in between HTML tags."
+      tickets: [986298]
+
+    - title: "Fix (I hope) the bulk metadata download process crashing for some people on OS X when clicking the Yes button to apply the updates."
+      tickets: [986658] 
+
+    - title: "Fix tooltip not being updated in the book details panel when pasting in a new cover"
+      tickets: [986958]
+
+    - title: "Cover Browser: Wrap the title on space only, not in between words."
+      tickets: [986516]
+
+    - title: "Edit metadata dialog: If a permission denied error occurs when clicking the next or prev buttons, stay on the current book."
+      tickets: [986903]
+
+    - title: "Fix heuristics not removing unnecessary hyphens from the end of lines."
+      tickets: [822744]
+
+  improved recipes:
+    - Metro Nieuws NL
+    - Der Tagesspiegel
+
+  new recipes:
+    - title: Berria 
+      author: Alayn Gortazar
+
+    - title: Sol Haber 
+      author: Onur Gungor
+
+    - title: Telam
+      author: Darko Miletic
+
+    - title: Richmond Times-Dispatch
+      author: jde
+
+- version: 0.8.48
+  date: 2012-04-20
+
+  new features:
+    - title: "Conversion: The search and replace feature has been completely revamped."
+      description: "You can now use any number of search and replace
+      expression, not just three. You can also store and load frequently used
+      sets of search and replace expressions. Also, the wizard generates its
+      preview in a separate process to protect against crashes/memory leaks."
+      tickets: [983476,983484,983478]
+
+    - title: "Support for the new '.azw3' files that Amazon recently started generating. calibre will now detect them as ebooks. It can also view/convert them, if they are DRM free."
+
+    - title: "Drivers for Samsung Galaxy ACE GT-S5830L and HTC One X"
+      tickets: [981185] 
+ 
+  bug fixes:
+    - title: "Get Books: Support the new website design of Barnes & Noble"
+
+    - title: "T1 driver: Fix books sent to SD card sometimes resulting problems when deleted."
+      tickets: [943586]
+
+    - title: "Do not allow author names to be set to blank via the Manage authors function. Blank authors are now automatically set to 'Unknown'"
+
+    - title: "MOBI Output: Handle background color specified on <td> and <tr> in addition to <table> tags."
+      tickets: [980813]
+
+    - title: "MOBI Output: Fix underline style applied to parent element not getting inherited by <a> children."
+      tickets: [985711]
+
+  improved recipes:
+    - xkcd
+    - Metro Nieuws
+    - Calgary Herald
+    - Orlando Sentinel
+    - countryfile
+    - Heise
+
+  new recipes:
+    - title: Various new Polish news sources
+      author: fenuks
+
+    - title: Various Italian news sources 
+      author: faber1971
+
+    - title: Jakarta Globe 
+      author: rty
+
+    - title: Acim Bilim Dergisi
+      author: thomass
+
 - version: 0.8.47
   date: 2012-04-13
 

manual/conf.py

@@ -14,7 +14,7 @@
 import sys, os
 
 # If your extensions are in another directory, add it here.
-sys.path.append(os.path.abspath('../../../'))
+sys.path.append(os.path.abspath('../src'))
 sys.path.append(os.path.abspath('.'))
 __appname__ = os.environ.get('__appname__', 'calibre')
 __version__ = os.environ.get('__version__', '0.0.0')
@@ -98,7 +98,7 @@ html_favicon = 'favicon.ico'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the built-in static files,
 # so a file named "default.css" will overwrite the built-in "default.css".
-html_static_path = ['resources', '../../../icons/favicon.ico']
+html_static_path = ['resources', '../icons/favicon.ico']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

manual/conversion.rst

@@ -573,6 +573,18 @@ There is a Word macro package that can automate the conversion of Word documents
 generating the Table of Contents much simpler. It is called BookCreator and is available for free
 at `mobileread <http://www.mobileread.com/forums/showthread.php?t=28313>`_.
 
+An easy way to generate a Table of Contents when converting a Word document is:
+
+ 1. Mark your Chapters and sub-Chapters in the doc file with one of the MS built-in styles called 'Heading 1', 'Heading 2', ..., 'Heading 6'. 'Heading 1' equates to the HTML tag <h1>, 'Heading 2' to <h2> etc
+
+ 2. Save the doc as Webpage-filtered (rather than Webpage) and import the html file into |app|
+
+ 3. When you convert in |app| you use what you did in step 1 to set the box called 'Detect chapters at' on the Convert - Structure Detection page. For example:
+
+  * If you mark Chapters with style 'Heading 2' then set the 'Detect chapters at' box to //h:h2 This will give you a proper external metadata TOC in the converted epub.
+  * A slightly more complex example...if your book has Sections and Chapters and you want a 2-level nested metadata TOC. Mark the doc Sections with style 'Heading 2' and the Chapters with style 'Heading 3'. When you convert set the 'Detect chapters at' box to //h:h2|//h:h3. On the Convert - TOC page set the 'Level 1 TOC' box to //h:h2 and the 'Level 2 TOC' box to //h:h3.
+
+
 Convert TXT documents
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -657,7 +669,6 @@ Some limitations of PDF input are:
     * Complex, multi-column, and image based documents are not supported.
     * Extraction of vector images and tables from within the document is also not supported.
     * Some PDFs use special glyphs to represent ll or ff or fi, etc. Conversion of these may or may not work depending on just how they are represented internally in the PDF.
-    * Some PDFs store their images upside down with a rotation instruction, |app| currently doesn't support that instruction, so the images will be rotated in the output as well. 
     * Links and Tables of Contents are not supported
     * PDFs that use embedded non-unicode fonts to represent non-English characters will result in garbled output for those characters
     * Some PDFs are made up of photographs of the page with OCRed text behind them. In such cases |app| uses the OCRed text, which can be very different from what you see when you view the PDF file

manual/custom.py

@@ -5,9 +5,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, re, textwrap
 
-sys.path.insert(0, os.path.abspath('../../'))
+sys.path.insert(0, os.path.abspath('../src'))
-sys.extensions_location = '../plugins'
+sys.extensions_location = '../src/calibre/plugins'
-sys.resources_location  = '../../../resources'
+sys.resources_location  = '../resources'
 
 from sphinx.util.console import bold
 
@@ -116,44 +116,42 @@ def generate_ebook_convert_help(preamble, info):
     from calibre.utils.logging import default_log
     preamble = re.sub(r'http.*\.html', ':ref:`conversion`', preamble)
     raw = preamble + textwrap.dedent('''
-    Since the options supported by ebook-convert vary depending on both the
+    The options and default values for the options change depending on both the
-    input and the output formats, the various combinations are listed below:
+    input and output formats, so you should always check with::
+
+        ebook-convert myfile.input_format myfile.output_format -h
+
+    Below are the options that are common to all conversion, followed by the
+    options specific to every input and output format
 
     ''')
-    toc = {}
-    sec_templ = textwrap.dedent('''\
-        .. include:: ../global.rst
-
-        {0}
-        ================================================================
-
-        .. contents:: Contents
-          :depth: 1
-          :local:
-
-    ''')
-    for i, ip in enumerate(input_format_plugins()):
-        sraw = sec_templ.format(ip.name)
-        toc[ip.name] = 'ebook-convert-%d'%i
-        for op in output_format_plugins():
-            title = ip.name + ' to ' + op.name
     parser, plumber = create_option_parser(['ebook-convert',
-                'dummyi.'+list(ip.file_types)[0],
+        'dummyi.mobi', 'dummyo.epub', '-h'], default_log)
-                'dummyo.'+op.file_type, '-h'], default_log)
-            cmd = 'ebook-convert '+list(ip.file_types)[0]+' '+op.file_type
     groups = [(None, None, parser.option_list)]
     for grp in parser.option_groups:
-                groups.append((grp.title, grp.description, grp.option_list))
+        if grp.title not in {'INPUT OPTIONS', 'OUTPUT OPTIONS'}:
-            options = '\n'.join(render_options(cmd, groups, False))
+            groups.append((grp.title.title(), grp.description, grp.option_list))
-            sraw += title+'\n------------------------------------------------------\n\n'
+    options = '\n'.join(render_options('ebook-convert', groups, False))
-            sraw += options + '\n\n'
+
-        update_cli_doc(os.path.join('cli', toc[ip.name]+'.rst'), sraw, info)
+    raw += '\n\n.. contents::\n  :local:'
+
+    raw += '\n\n' + options
+    for pl in sorted(input_format_plugins(), key=lambda x:x.name):
+        parser, plumber = create_option_parser(['ebook-convert',
+            'dummyi.'+list(pl.file_types)[0], 'dummyo.epub', '-h'], default_log)
+        groups = [(pl.name+ ' Options', '', g.option_list) for g in
+                parser.option_groups if g.title == "INPUT OPTIONS"]
+        prog = 'ebook-convert-'+(pl.name.lower().replace(' ', '-'))
+        raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
+    for pl in sorted(output_format_plugins(), key=lambda x: x.name):
+        parser, plumber = create_option_parser(['ebook-convert', 'd.epub',
+            'dummyi.'+pl.file_type, '-h'], default_log)
+        groups = [(pl.name+ ' Options', '', g.option_list) for g in
+                parser.option_groups if g.title == "OUTPUT OPTIONS"]
+        prog = 'ebook-convert-'+(pl.name.lower().replace(' ', '-'))
+        raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
 
-    toct = '\n\n.. toctree::\n    :maxdepth: 2\n\n'
-    for ip in sorted(toc):
-        toct += '    ' + toc[ip]+'\n'
 
-    raw += toct+'\n\n'
     update_cli_doc(os.path.join('cli', 'ebook-convert.rst'), raw, info)
 
 def update_cli_doc(path, raw, info):

manual/customize.rst

@@ -39,7 +39,7 @@ Tweaks
 Tweaks are small changes that you can specify to control various aspects of |app|'s behavior. You can change them by going to Preferences->Advanced->Tweaks.
 The default values for the tweaks are reproduced below
 
-.. literalinclude:: ../../../resources/default_tweaks.py
+.. literalinclude:: ../resources/default_tweaks.py
 
 
 Overriding icons, templates, et cetera

manual/develop.rst

@@ -45,6 +45,16 @@ All the |app| python code is in the ``calibre`` package. This package contains t
       The format independent code is all in ebooks.oeb and the format dependent code is in ebooks.format_name.
 
         * Metadata reading, writing, and downloading is all in ebooks.metadata
+        * Conversion happens in a pipeline, for the structure of the pipeline,
+          see :ref:`conversion-introduction`. The pipeline consists of an input
+          plugin, various transforms and an output plugin. The code constructs
+          and drives the pipeline is in plumber.py. The pipeline works on a
+          representation of an ebook that is like an unzipped epub, with
+          manifest, spine, toc, guide, html content, etc. The
+          class that manages this representation is OEBBook in oeb/base.py. The
+          various transformations that are applied to the book during
+          conversions live in `oeb/transforms/*.py`. And the input and output
+          plugins live in `conversion/plugins/*.py`.
 
     * library - The database back-end and the content server. See library.database2 for the interface to the |app| library. library.server is the |app| Content Server.
     * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer.

manual/faq.rst

@@ -22,20 +22,20 @@ It can convert every input format in the following list, to every output format.
 
 *Input Formats:* CBZ, CBR, CBC, CHM, DJVU, EPUB, FB2, HTML, HTMLZ, LIT, LRF, MOBI, ODT, PDF, PRC, PDB, PML, RB, RTF, SNB, TCR, TXT, TXTZ
 
-*Output Formats:* EPUB, FB2, OEB, LIT, LRF, MOBI, HTMLZ, PDB, PML, RB, PDF, RTF, SNB, TCR, TXT, TXTZ
+*Output Formats:* AZW3, EPUB, FB2, OEB, LIT, LRF, MOBI, HTMLZ, PDB, PML, RB, PDF, RTF, SNB, TCR, TXT, TXTZ
 
 .. note ::
 
     PRC is a generic format, |app| supports PRC files with TextRead and MOBIBook headers.
     PDB is also a generic format. |app| supports eReder, Plucker, PML and zTxt PDB files.
     DJVU support is only for converting DJVU files that contain embedded text. These are typically generated by OCR software.
-    MOBI books can be of two types Mobi6 and KF8. |app| currently fully supports Mobi6 and supports conversion from, but not to, KF8
+    MOBI books can be of two types Mobi6 and KF8. |app| fully supports both. MOBI files often have .azw or .azw3 file extensions
 
 .. _best-source-formats:
 
 What are the best source formats to convert?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order of decreasing preference: LIT, MOBI, EPUB, FB2, HTML, PRC, RTF, PDB, TXT, PDF
+In order of decreasing preference: LIT, MOBI, AZW, EPUB, AZW3, FB2, HTML, PRC, RTF, PDB, TXT, PDF
 
 I converted a PDF file, but the result has various problems?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -69,6 +69,22 @@ If you have a hand edited TOC in the input document, you can use the TOC detecti
 
 Finally, I encourage you to ditch the content TOC and only have a metadata TOC in your ebooks. Metadata TOCs will give the people reading your ebooks a much superior navigation experience (except on the Kindle, where they are essentially the same as a content TOC).
 
+The covers for my MOBI files have stopped showing up in Kindle for PC/Kindle for Android/etc.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is caused by a bug in the Amazon software. You can work around it by going
+to Preferences->Output Options->MOBI output and setting the "Enable sharing
+of book content" option. If you are reconverting a previously converted book,
+you will also have to enable the option in the conversion dialog for that
+individual book (as per book conversion settings are saved and take
+precedence). 
+
+Note that doing this will mean that the generated MOBI will show up under
+personal documents instead of Books on the Kindle Fire and Amazon whispersync
+will not work, but the covers will. It's your choice which functionality is
+more important to you. I encourage you to contact Amazon and ask them to fix
+this bug.
+
 How do I convert a collection of HTML files in a specific order?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::

manual/index.rst

@@ -17,7 +17,7 @@ To get started with more advanced usage, you should read about the :ref:`Graphic
 
 .. only:: online
     
-    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_.
+    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_ and `AZW3 (Kindle Fire) format <calibre.azw3>`_.
     
 Sections
 ------------

manual/template_lang.rst

@@ -245,7 +245,7 @@ The following functions are available in addition to those described in single-f
     * ``current_library_name() -- `` return the last name on the path to the current calibre library. This function can be called in template program mode using the template ``{:'current_library_name()'}``.
     * ``days_between(date1, date2)`` -- return the number of days between ``date1`` and ``date2``. The number is positive if ``date1`` is greater than ``date2``, otherwise negative. If either ``date1`` or ``date2`` are not dates, the function returns the empty string.
     * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers.
-    * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables.
+    * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. Note also that prefixes and suffixes (the `|prefix|suffix` syntax) cannot be used in the argument to this function when using template program mode.
     * ``field(name)`` -- returns the metadata field named by ``name``.
     * ``first_non_empty(value, value, ...)`` -- returns the first value that is not empty. If all values are empty, then the empty value is returned. You can have as many values as you want.
     * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are::
@@ -306,7 +306,7 @@ The following functions are available in addition to those described in single-f
     * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
     * ``subtract(x, y)`` -- returns x - y. Throws an exception if either x or y are not numbers.
     * ``today()`` -- return a date string for today. This value is designed for use in format_date or days_between, but can be manipulated like any other string. The date is in ISO format.
-    * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value.
+    * ``template(x)`` -- evaluates x as a template. The evaluation is done in its own context, meaning that variables are not shared between the caller and the template evaluation. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. For example, ``template('[[title_sort]]') will evaluate the template ``{title_sort}`` and return its value. Note also that prefixes and suffixes (the `|prefix|suffix` syntax) cannot be used in the argument to this function when using template program mode.
 
 .. _template_functions_reference:
     

manual/template_ref_generate.py

@@ -55,7 +55,7 @@ The python implementation of the template functions is passed in a Metadata obje
 
     The set of standard metadata fields.
 
-.. literalinclude:: ../ebooks/metadata/book/__init__.py
+.. literalinclude:: ../src/calibre/ebooks/metadata/book/__init__.py
    :lines: 7-
 '''
 

recipes/acim_bilim_dergisi.recipe

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1334868409(BasicNewsRecipe):
+    title          = u'AÇIK BİLİM DERGİSİ'
+    description            =  ' Aylık çevrimiçi bilim dergisi'
+    __author__             = u'thomass'
+    oldest_article = 30
+    max_articles_per_feed = 300
+    auto_cleanup = True
+    encoding               = 'UTF-8'
+    publisher              = 'açık bilim'
+    category               = 'haber, bilim,TR,dergi'
+    language               = 'tr'
+    publication_type = 'magazine '
+    conversion_options = {
+                            'tags'            : category
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                            ,'linearize_tables': True
+                         }
+    cover_img_url = 'http://www.acikbilim.com/wp-content/themes/Equilibrium/images/logodene.jpg'
+    masthead_url = 'http://www.acikbilim.com/wp-content/themes/Equilibrium/images/logodene.jpg'
+
+
+    feeds          = [(u'Tüm Yayınlar', u'http://www.acikbilim.com/feed')]

recipes/ads_of_the_world.recipe

@@ -0,0 +1,26 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1336986047(BasicNewsRecipe):
+    title          = u'Ads of the World'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    auto_cleanup = False
+    description   = 'The best international advertising campaigns'
+    language = 'en'
+    __author__ = 'faber1971'
+
+    no_stylesheets = True
+    keep_only_tags = [
+                       dict(name='div', attrs={'id':'primary'})
+                     ]
+
+    remove_tags = [
+                       dict(name='ul', attrs={'class':'links inline'})
+                      ,dict(name='div', attrs={'class':'form-item'})
+                      ,dict(name='div', attrs={'id':['options', 'comments']})
+                      ,dict(name='ul', attrs={'id':'nodePager'})
+                     ]
+
+    reverse_article_order = True
+    masthead_url            = 'http://bigcatgroup.co.uk/files/2011/01/05-ads-of-the-world.png'
+    feeds          = [(u'Ads of the world', u'http://feeds.feedburner.com/adsoftheworld-latest')]

recipes/adventure_zone_pl.recipe

@@ -9,6 +9,7 @@ class Adventure_zone(BasicNewsRecipe):
     no_stylesheets = True
     oldest_article = 20
     max_articles_per_feed = 100
+    index='http://www.adventure-zone.info/fusion/'
     use_embedded_content=False
     preprocess_regexps     = [(re.compile(r"<td class='capmain'>Komentarze</td>", re.IGNORECASE), lambda m: '')]
     remove_tags_before= dict(name='td', attrs={'class':'main-bg'})
@@ -45,6 +46,19 @@ class Adventure_zone(BasicNewsRecipe):
         skip_tag = skip_tag.findAll(name='a')
         for r in skip_tag:
            if r.strong:
-                 word=r.strong.string
+                 word=r.strong.string.lower()
-                 if word and (('zapowied' in word) or ('recenzj' in word)  or ('solucj' in word)):
+                 if word and (('zapowied' in word) or ('recenzj' in word)  or ('solucj' in word) or ('poradnik' in word)):
                    return self.index_to_soup('http://www.adventure-zone.info/fusion/print.php?type=A&item'+r['href'][r['href'].find('article_id')+7:], raw=True)
+
+    def preprocess_html(self, soup):
+        footer=soup.find(attrs={'class':'news-footer middle-border'})
+        if footer and len(footer('a'))>=2:
+            footer('a')[1].extract()
+        for item in soup.findAll(style=True):
+            del item['style']
+        for a in soup('a'):
+            if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
+                a['href']=self.index + a['href']
+        return soup
+           
+            

recipes/air_force_times.recipe

@@ -0,0 +1,43 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AirForceTimes(BasicNewsRecipe):
+    title = 'Air Force Times'
+    __author__ = 'jde'
+    __date__ = '16 May 2012'
+    __version__ = '1.0'
+    description = 'News of the U.S. Air Force'
+    language = 'en'
+    publisher = 'AirForceTimes.com'
+    category = 'news, U.S. Air Force'
+    tags = 'news, U.S. Air Force'
+    cover_url = 'http://www.airforcetimes.com/images/logo_airforcetimes_alert.jpg'
+    masthead_url = 'http://www.airforcetimes.com/images/logo_airforcetimes_alert.jpg'
+    oldest_article = 7 #days
+    max_articles_per_feed = 25
+    publication_type = 'newspaper'
+    no_stylesheets = True
+    use_embedded_content = False
+    encoding = None
+    recursions = 0
+    needs_subscription = False
+    remove_javascript = True
+    remove_empty_feeds = True
+    auto_cleanup = True
+
+
+
+    feeds = [
+
+	('News', 		'http://www.airforcetimes.com/rss_news.php'),
+	('Benefits', 		'http://www.airforcetimes.com/rss_benefits.php'),
+	('Money', 		'http://www.airforcetimes.com/rss_money.php'),
+	('Careers & Education', 	'http://www.airforcetimes.com/rss_careers.php'),
+	('Community', 	'http://www.airforcetimes.com/rss_community.php'),
+	('Off Duty', 		'http://www.airforcetimes.com/rss_off_duty.php'),
+	('Entertainment', 	'http://www.airforcetimes.com/rss_entertainment.php'),
+	('Guard & Reserve', 	'http://www.airforcetimes.com/rss_guard.php'),
+              ]
+
+
+
+