Sync to trunk.

installer/linux/freeze.py

@@ -9,7 +9,7 @@ Create linux binary.
 '''
 
 def freeze():
-    import glob, sys, tarfile, os, textwrap, shutil
+    import glob, sys, tarfile, os, textwrap, shutil, platform
     from contextlib import closing
     from cx_Freeze import Executable, setup
     from calibre.constants import __version__, __appname__
@@ -19,6 +19,9 @@ def freeze():
     from calibre.ebooks.lrf.fonts import FONT_MAP
     import calibre
 
+    is64bit = platform.architecture()[0] == '64bit'
+    arch = 'x86_64' if is64bit else 'i686'
+
 
     QTDIR          = '/usr/lib/qt4'
     QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml',
@@ -47,7 +50,8 @@ def freeze():
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libgthread-2.0.so.0',
-                       '/usr/lib/gcc/i686-pc-linux-gnu/4.3.3/libstdc++.so.6',
+                       '/usr/lib/gcc/***-pc-linux-gnu/4.4.1/libstdc++.so.6'.replace('***',
+                           arch).replace('i686', 'i486'),
                        '/usr/lib/libpng12.so.0',
                        '/usr/lib/libexslt.so.0',
                        '/usr/lib/libMagickWand.so',
@@ -89,7 +93,7 @@ def freeze():
 
     includes = [x[0] for x in executables.values()]
     includes += ['calibre.ebooks.lrf.fonts.prs500.'+x for x in FONT_MAP.values()]
-    includes += ['email.iterators', 'email.generator']
+    includes += ['email.iterators', 'email.generator', 'sqlite3.dump']
 
 
     excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk",
@@ -228,7 +232,8 @@ def freeze():
     open(os.path.join(FREEZE_DIR, 'manifest'), 'wb').write('\n'.join(exes))
 
     print 'Creating archive...'
-    dist = open(os.path.join(DIST_DIR, 'calibre-%s-i686.tar.bz2'%__version__), 'wb')
+    dist = open(os.path.join(DIST_DIR, 'calibre-%s-%s.tar.bz2'%(__version__,
+        arch)), 'wb')
     with closing(tarfile.open(fileobj=dist, mode='w:bz2',
                               format=tarfile.PAX_FORMAT)) as tf:
         for f in walk(FREEZE_DIR):

installer/osx/freeze.py

@@ -353,7 +353,7 @@ def main():
                                        'keyword', 'codeop', 'pydoc', 'readline',
                                        'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
                                        'dateutil', 'email.iterators',
-                                       'email.generator',
+                                       'email.generator', 'sqlite3.dump',
                                        'calibre.ebooks.metadata.amazon',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml', 'dns'],

installer/windows/build_installer.py

@@ -36,7 +36,7 @@ def run_install_jammer(installer_name='<%AppName%>-<%Version%><%Ext%>', build_fo
     compression = 'zlib'
     if build_for_release:
         cmdline += ['--build-for-release']
-        #compression = 'lzma (solid)'
+        compression = 'lzma (solid)'
     cmdline += ['-DCompressionMethod', compression]
     cmdline += ['--build', mpi]
     #print 'Running installjammer with cmdline:'

installer/windows/calibre/calibre.mpi

@@ -232,12 +232,10 @@ test
 }
 FileGroup ::BEF8D398-58BA-1F66-39D6-D4A63D5BEEF9 -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows TarArchive ZipArchive FreeBSD-5-x86 FreeBSD-6-x86 FreeBSD-7-x86 Linux-x86_64 Solaris-x86} -name {Program Files} -parent FileGroups
 File ::8E5D85A4-7608-47A1-CF7C-309060D5FF40 -filemethod {Always overwrite files} -type dir -directory <%InstallDir%> -name /home/kovid/work/calibre/build/py2exe -parent BEF8D398-58BA-1F66-39D6-D4A63D5BEEF9
-File ::FC870EE7-667B-481F-113B-B4504DFCCFA5 -type dir -name bin -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::377C588B-B324-CA09-ED49-4DB5F82A15ED -type dir -name etc -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::55DE4B9F-0881-FF51-E2BA-EC72B5D3425C -type dir -name fonts -parent 377C588B-B324-CA09-ED49-4DB5F82A15ED
 File ::A27B68D9-43A6-B994-3091-E829AFBA340D -type dir -name conf.d -parent 55DE4B9F-0881-FF51-E2BA-EC72B5D3425C
 File ::974ADD48-88E5-BC7A-1963-928A245F133A -type dir -name conf.avail -parent 55DE4B9F-0881-FF51-E2BA-EC72B5D3425C
-File ::5E5273D8-3423-8DC8-83C4-BE000069A803 -name fonts.dtd -parent 55DE4B9F-0881-FF51-E2BA-EC72B5D3425C
 File ::32D7DBE0-E0B1-5BDD-66C5-2A13D8BC8F90 -name fonts.conf -parent 55DE4B9F-0881-FF51-E2BA-EC72B5D3425C
 File ::B95D03D4-EA59-F00E-59E1-BA05758879DA -type dir -name imageformats -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::A624029D-AE0F-49A5-4DAC-7720CDCAB271 -name qmng4.dll -parent B95D03D4-EA59-F00E-59E1-BA05758879DA
@@ -337,7 +335,6 @@ File ::2F90282D-B59F-B6BA-090B-45858AF7F3B2 -name IM_MOD_RL_clipboard_.dll -pare
 File ::B512D139-B295-D7C3-F0B4-43775849CF58 -name numpy.core._sort.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::A2C063AC-2F12-9260-501A-0E8BD0B8A932 -name calibre.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::439B6D05-7DE6-061D-4BCC-3F04F4FA2FA2 -name IM_MOD_RL_png_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::BA464D11-BBCE-DEDA-C354-0C7BE60FAA05 -name IM_MOD_RL_braille_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::8F9FF823-AF6D-A288-8AE6-7D74F55DCE29 -name CORE_RL_bzlib_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::57A3F5D5-BFC8-CB38-5A57-548EE0DB033B -name QtNetwork4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::4DB7E8DE-905A-822A-AF14-17BD5ACEF915 -name IM_MOD_RL_wmf_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -361,7 +358,6 @@ File ::A6419A84-6C22-784E-6D84-D09972770770 -name unicodedata.pyd -parent 8E5D85
 File ::E658FBE0-5860-D041-12D3-76ADD18F804B -name servicemanager.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::C98A6FC4-E341-7FD4-005C-DA2B384E11D8 -name win32api.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::ADA36EEA-7DE1-447C-B1AB-A4908E65E2CD -name IM_MOD_RL_ipl_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::53C2EC15-850F-8F49-6425-C228FB6E6D0E -name libfontconfig-1.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::EDE6F457-C83F-C5FA-9AF4-38FDFF17D929 -name PIL._imagingtk.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::09D0906E-3611-3DB7-32CF-A140585694A7 -name win32pdh.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::4C84F0DC-7157-0C90-2062-180139B03E25 -name IM_MOD_RL_rgb_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -384,17 +380,14 @@ File ::404A98F1-84FD-B6D0-B130-354EECD9253C -name IM_MOD_RL_emf_.dll -parent 8E5
 File ::17034C34-403E-B405-99C1-F80B7F00E27C -name log.xml -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::34E63A2C-65C5-0A84-ACF1-BD6A844D4579 -name pythoncom26.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::2F20484B-53B8-B08E-B691-C5B2D49A9CB4 -name QtWebKit4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::8AF134C8-9189-3F9A-A081-9143FFD44C45 -name freetype6.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::E8A4442D-D0D3-31CD-997A-3CEB641CF5B7 -name IM_MOD_RL_mtv_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::0CA87D0B-5A04-1439-AEE8-C97072D47BA7 -name CORE_RL_tiff_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::AC24F520-88D4-D1CF-5797-27C715CE8ACA -name pyexpat.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::16848F38-71CD-55B8-4D96-1537F6773744 -name IM_MOD_RL_dps_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::33A46CC5-BAC4-5863-C83D-303DCCA0CAA1 -name tk85.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::81116DD3-1715-AA87-472F-544FC616EDAF -name IM_MOD_RL_dcm_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::335A4CFB-5C2D-44E4-C438-7018E8244C3D -name ebook-viewer.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::62A16C3B-ED9C-5187-2807-58857DF3A990 -name calibre-debug.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::55ECA7B7-279A-F51D-81C2-C8DC44CF0E22 -name select.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::A6AF5ECC-A981-4CBD-DBEE-303A9340C603 -name IM_MOD_RL_xps_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::5BCBF71F-18E7-5C52-E3F5-7D7F3028AD46 -name locale.xml -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::5C9FA94C-B8B0-A94B-548D-1D24FDEA5770 -name CORE_RL_wand_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::E39E60BE-DE77-AB8C-42C6-5A7D7DC073E3 -name IM_MOD_RL_ttf_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -443,12 +436,10 @@ File ::396B4F78-FB45-C0B2-ACB3-97769CF5CD5D -name msvcr90.dll -parent 8E5D85A4-7
 File ::1DE767EE-4891-4E54-422D-67A4DFF8C3B5 -name lrfviewer.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::AFFEC28A-615C-E3E6-0026-CCE2594A6D25 -name calibre-server.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::2C64F86B-9366-B52D-F7B2-5BBD51F6982A -name IM_MOD_RL_pwp_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::F4B2EF9C-EB18-B865-6E99-75CFB9B60D87 -name IM_MOD_RL_dds_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::38770D87-6CA9-9E3E-FBA1-A8CCFCD88FB5 -name IM_MOD_RL_fpx_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::BE2D7BC3-D294-AF3F-65E7-3B372DEFDE36 -name PIL._imaging.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::200B9AEC-809F-75B7-DC12-A51BFC2A6F93 -name PyQt4.QtSvg.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::52132A31-D3AE-C617-7568-BF2AF46B5D74 -name IM_MOD_RL_pcl_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::F94472C3-C8D0-950F-5ED9-1611D1CE30E5 -name IM_MOD_RL_inline_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::055ADB4B-20C5-E071-442F-4DA0A8D6F3C5 -name english.xml -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::B10B6B91-0C03-642D-90D8-37B607B164AD -name IM_MOD_RL_wpg_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::BFBB144B-1794-8304-9772-F103A42F2CA4 -name IM_MOD_RL_pdb_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -503,7 +494,6 @@ File ::9BA85EE5-1754-67AF-736D-481CDCC72DD2 -name _imagingft.pyd -parent 8E5D85A
 File ::6254DD0C-8F2C-D4AE-2107-2597D542C181 -name IM_MOD_RL_matte_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::F159D566-88D6-C347-3E3C-55C2DDFC5FD0 -name IM_MOD_RL_mono_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::B873CAA2-011F-94C3-7977-FF344E53C44F -name CORE_RL_jbig_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::7004FCB8-C6F4-C7AF-08E4-B6151B2F7050 -name tcl85.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::6921F62A-4015-4C9F-98A6-BCBBC43B698E -name msvcm90.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::7276E0CA-C205-4B18-19A3-157F1B8523FB -name IM_MOD_RL_xtrn_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::7B9624A9-88B4-C61E-6771-9A34FB6CA3B5 -name PyQt4.QtGui.pyd -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -567,6 +557,15 @@ File ::9E4E5E8F-30C0-E631-9516-2AE01A5CA0E9 -name ebook-device.exe.local -parent
 File ::7BE6B538-70D5-A7EB-5F91-E14CE57B394B -name calibre-complete.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::C4E40030-3EE0-8B05-E6B9-89E81433EE1F -name phonon4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::9E84342F-36ED-7ED3-8F90-1EC55267BCFC -name poppler-qt4.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::86BA442C-90C9-A4E6-1D3E-D144E5F326C1 -name msvcp71.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::11FBAD0B-A2DB-C28A-85B8-D6A22706864F -name mfc71.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::4B9FB3E6-B807-65CC-826D-A398E964D00C -name IM_MOD_RL_hdf_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::3E201C0C-C7CC-5785-74F6-A6CC7F50A15A -name msvcr71.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::2EE42149-1C12-CCA9-9089-AE1809098D0A -name jpeg62.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::B1FD37B4-E91B-DC1C-1C69-FB2E10EB93AE -name libtiff3.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::15E09D95-97D6-92A9-CC4D-120885E4DDAD -name freetype.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::D954BC75-8166-EC1B-D91B-C9779248AA14 -name fontconfig.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::1F3C052A-A5E0-5C65-8D42-EBF44FBE138D -name podofo.dll.manifest -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 Component ::F6829AB7-9F66-4CEE-CA0E-21F54C6D3609 -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows FreeBSD-5-x86 FreeBSD-6-x86 FreeBSD-7-x86 Linux-x86_64 Solaris-x86} -name Main -parent Components
 SetupType ::D9ADE41C-B744-690C-2CED-CF826BF03D2E -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows FreeBSD-5-x86 FreeBSD-6-x86 FreeBSD-7-x86 Linux-x86_64 Solaris-x86} -name Typical -parent SetupTypes
 

installer/windows/freeze.py

@@ -7,15 +7,12 @@ __docformat__ = 'restructuredtext en'
 Freeze app into executable using py2exe.
 '''
 QT_DIR           = 'C:\\Qt\\4.5.2'
-LIBUSB_DIR       = 'C:\\libusb'
+LIBUSB_DIR       = r'C:\cygwin\home\kovid\win32\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
-PDFTOHTML        = 'C:\\cygwin\\home\\kovid\\poppler-0.10.6\\rel\\pdftohtml.exe'
+BINARIES         = r'C:\cygwin\home\kovid\win32\bin'
-POPPLER          = 'C:\\cygwin\\home\\kovid\\poppler'
+IMAGEMAGICK_DIR  = r'C:\cygwin\home\kovid\win32\imagemagick'
-IMAGEMAGICK_DIR  = 'C:\\ImageMagick'
+FONTCONFIG_DIR   = r'C:\cygwin\home\kovid\win32\etc'
-PDFTK            = 'C:\\pdftk.exe'
+VC90             = r'C:\Program Files\Microsoft Visual Studio 9.0\VC\redist\x86\Microsoft.VC90.CRT'
-PODOFO           = 'C:\\podofo'
-FONTCONFIG_DIR   = 'C:\\fontconfig'
-VC90             = r'C:\VC90.CRT'
 
 # ModuleFinder can't handle runtime changes to __path__, but win32com uses them
 import sys
@@ -98,25 +95,17 @@ class BuildEXE(py2exe.build_exe.py2exe):
                 shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
         print '\tAdding unrar'
         shutil.copyfile(LIBUNRAR, os.path.join(PY2EXE_DIR, os.path.basename(LIBUNRAR)))
-        print '\tAdding poppler'
+        print '\tAdding Binaries'
-        for x in ('bin\\pdftohtml.exe', 'bin\\poppler-qt4.dll',
+        for x in glob.glob(os.path.join(BINARIES, '*.dll')) + \
-            'bin\\freetype.dll', 'bin\\jpeg62.dll'):
+            [os.path.join(BINARIES, 'pdftohtml.exe')] + \
-            shutil.copyfile(os.path.join(POPPLER, x),
+            glob.glob(os.path.join(BINARIES, '*.manifest')):
-                    os.path.join(PY2EXE_DIR, os.path.basename(x)))
+            shutil.copyfile(x, os.path.join(PY2EXE_DIR, os.path.basename(x)))
-        print '\tAdding podofo'
-        for f in glob.glob(os.path.join(PODOFO, '*.dll')):
-            shutil.copyfile(f, os.path.join(PY2EXE_DIR, os.path.basename(f)))
-
         print '\tAdding ImageMagick'
         for f in os.listdir(IMAGEMAGICK_DIR):
             shutil.copyfile(os.path.join(IMAGEMAGICK_DIR, f), os.path.join(PY2EXE_DIR, f))
         print '\tCopying fontconfig'
-        for f in glob.glob(os.path.join(FONTCONFIG_DIR, '*')):
+        tgt = os.path.join(PY2EXE_DIR, 'etc')
-            tgt = os.path.join(PY2EXE_DIR, os.path.basename(f))
+        shutil.copytree(FONTCONFIG_DIR, tgt)
-            if os.path.isdir(f):
-                shutil.copytree(f, tgt)
-            else:
-                shutil.copyfile(f, tgt)
 
         print
         print 'Doing DLL redirection' # See http://msdn.microsoft.com/en-us/library/ms682600(VS.85).aspx
@@ -169,8 +158,7 @@ def main(args=sys.argv):
                                              'email.iterators',
                                              'email.generator',
                                              'win32process', 'win32api', 'msvcrt',
-                                             'win32event', 'calibre.ebooks.lrf.any.*',
+                                             'win32event', 'sqlite3.dump',
-                                             'calibre.ebooks.lrf.feeds.*',
                                              'BeautifulSoup', 'pyreadline',
                                              'pydoc', 'IPython.Extensions.*',
                                              'calibre.web.feeds.recipes.*',
@@ -183,7 +171,8 @@ def main(args=sys.argv):
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],
-                                  'dll_excludes' : ['mswsock.dll'],
+                                  'dll_excludes' : ['mswsock.dll', 'tcl85.dll',
+                                      'MSVCP90.dll', 'tk85.dll'],
                                  },
                     },
 

setup.py

@@ -71,7 +71,7 @@ if __name__ == '__main__':
                         tag_release, upload_demo, build_linux, build_windows, \
                         build_osx, upload_installers, upload_user_manual, \
                         upload_to_pypi, stage3, stage2, stage1, upload, \
-                        upload_rss, betas
+                        upload_rss, betas, build_linux32, build_linux64
 
     entry_points['console_scripts'].append(
                             'calibre_postinstall = calibre.linux:post_install')
@@ -94,8 +94,8 @@ if __name__ == '__main__':
                 sources=['src/calibre/utils/windows/winutil.c'],
                 libraries=['shell32', 'setupapi'],
                 include_dirs=os.environ.get('INCLUDE',
-                        'C:/WinDDK/6001.18001/inc/api/;'
+                        'C:/WinDDK/7600.16385.0/inc/api/;'
-                        'C:/WinDDK/6001.18001/inc/crt/').split(';'),
+                        'C:/WinDDK/7600.16385.0/inc/crt/').split(';'),
                 extra_compile_args=['/X']
                 ))
 
@@ -103,8 +103,8 @@ if __name__ == '__main__':
     poppler_lib = '/usr/lib'
     poppler_libs = []
     if iswindows:
-        poppler_inc = r'C:\cygwin\home\kovid\poppler\include\poppler\qt4'
+        poppler_inc = r'C:\cygwin\home\kovid\win32\include\poppler\qt4'
-        poppler_lib = r'C:\cygwin\home\kovid\poppler\lib'
+        poppler_lib = r'C:\cygwin\home\kovid\win32\lib'
         poppler_libs = ['QtCore4', 'QtGui4']
     if isosx:
         poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src'
@@ -124,9 +124,10 @@ if __name__ == '__main__':
         print 'POPPLER_LIB_DIR environment variables.'
 
     podofo_inc = '/usr/include/podofo' if islinux else \
-    'C:\\podofo\\include\\podofo' if iswindows else \
+    r'C:\cygwin\home\kovid\win32\include\podofo' if iswindows else \
     '/usr/local/include/podofo'
-    podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \
+    podofo_lib = '/usr/lib' if islinux else \
+            r'C:\cygwin\home\kovid\win32\lib' if iswindows else \
             '/usr/local/lib'
     podofo_inc = os.environ.get('PODOFO_INC_DIR', podofo_inc)
     if os.path.exists(os.path.join(podofo_inc, 'podofo.h')):
@@ -141,10 +142,10 @@ if __name__ == '__main__':
         print 'PODOFO_LIB_DIR environment variables.'
 
     fc_inc = '/usr/include/fontconfig' if islinux else \
-            r'C:\cygwin\home\kovid\fontconfig\include\fontconfig' if iswindows else \
+            r'C:\cygwin\home\kovid\win32\include\fontconfig' if iswindows else \
             '/Users/kovid/fontconfig/include/fontconfig'
     fc_lib = '/usr/lib' if islinux else \
-            r'C:\cygwin\home\kovid\fontconfig\lib' if iswindows else \
+            r'C:\cygwin\home\kovid\win32\lib' if iswindows else \
             '/Users/kovid/fontconfig/lib'
 
 
@@ -258,6 +259,8 @@ if __name__ == '__main__':
                       'tag_release'   : tag_release,
                       'upload_demo'   : upload_demo,
                       'build_linux'   : build_linux,
+                      'build_linux32'  : build_linux32,
+                      'build_linux64' : build_linux64,
                       'build_windows' : build_windows,
                       'build_osx'     : build_osx,
                       'upload_installers': upload_installers,

src/calibre/__init__.py

@@ -69,7 +69,7 @@ def osx_version():
             return int(m.group(1)), int(m.group(2)), int(m.group(3))
 
 
-_filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+\[\]/]')
+_filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+/]')
 
 def sanitize_file_name(name, substitute='_', as_unicode=False):
     '''

src/calibre/debug.py

@@ -135,6 +135,7 @@ def debug_device_driver():
             print 'failed'
             continue
         success = True
+        if hasattr(dev, '_main_prefix'):
             print 'Main memory:', repr(dev._main_prefix)
         print 'Total space:', dev.total_space()
         break
@@ -144,6 +145,10 @@ def debug_device_driver():
             print dev
             print msg
             print
+    if isosx and os.path.exists('/tmp/ioreg.txt'):
+        print
+        print
+        print "Don't forget to send the file /tmp/ioreg.txt as well"
 
 
 def add_simple_plugin(path_to_plugin):

src/calibre/ebooks/conversion/plumber.py

@@ -46,7 +46,8 @@ class Plumber(object):
         'tags', 'book_producer', 'language'
         ]
 
-    def __init__(self, input, output, log, report_progress=DummyReporter(), dummy=False):
+    def __init__(self, input, output, log, report_progress=DummyReporter(),
+            dummy=False, merge_plugin_recs=True):
         '''
         :param input: Path to input file.
         :param output: Path to output file/directory
@@ -483,6 +484,7 @@ OptionRecommendation(name='language',
             for x in getattr(self, w):
                 temp.add(x.clone())
             setattr(self, w, temp)
+        if merge_plugin_recs:
             self.merge_plugin_recommendations()
 
     @classmethod

src/calibre/ebooks/epub/output.py

@@ -78,7 +78,7 @@ class EPUBOutput(OutputFormatPlugin):
         ),
 
         OptionRecommendation(name='no_default_epub_cover', recommended_value=False,
-            help=_('Normally, if the input file ahs no cover and you don\'t'
+            help=_('Normally, if the input file has no cover and you don\'t'
             ' specify one, a default cover is generated with the title, '
             'authors, etc. This option disables the generation of this cover.')),
 

src/calibre/ebooks/fb2/output.py

@@ -18,7 +18,7 @@ class FB2Output(OutputFormatPlugin):
     options = set([
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to begenning of the book.')),
+            help=_('Add Table of Contents to beginning of the book.')),
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/ebooks/metadata/google_books.py

@@ -6,6 +6,7 @@ __docformat__ = 'restructuredtext en'
 import sys, textwrap
 from urllib import urlencode
 from functools import partial
+from datetime import datetime
 
 from lxml import etree
 from dateutil import parser
@@ -151,7 +152,9 @@ class ResultList(list):
         try:
             d = date(entry)
             if d:
-                d = parser.parse(d[0].text)
+                default = datetime.utcnow()
+                default = datetime(default.year, default.month, 1)
+                d = parser.parse(d[0].text, default=default)
             else:
                 d = None
         except:

src/calibre/ebooks/oeb/stylizer.py

@@ -92,7 +92,10 @@ class CSSSelector(etree.XPath):
 
     def __init__(self, css, namespaces=XPNSMAP):
         css = self.MIN_SPACE_RE.sub(r'\1', css)
+        try:
             path = css_to_xpath(css)
+        except UnicodeEncodeError: # Bug in css_to_xpath
+            path = '/'
         path = self.LOCAL_NAME_RE.sub(r"local-name() = '", path)
         etree.XPath.__init__(self, path, namespaces=namespaces)
         self.css = css

src/calibre/ebooks/pdb/output.py

@@ -29,7 +29,7 @@ class PDBOutput(OutputFormatPlugin):
             'formats.')),
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to begenning of the book.')),
+            help=_('Add Table of Contents to beginning of the book.')),
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/ebooks/pml/output.py

@@ -34,7 +34,7 @@ class PMLOutput(OutputFormatPlugin):
             'The default is cp1252.')),
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to begenning of the book.')),
+            help=_('Add Table of Contents to beginning of the book.')),
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/ebooks/rb/output.py

@@ -18,7 +18,7 @@ class RBOutput(OutputFormatPlugin):
     options = set([
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to begenning of the book.')),
+            help=_('Add Table of Contents to beginning of the book.')),
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/ebooks/txt/output.py

@@ -32,7 +32,7 @@ class TXTOutput(OutputFormatPlugin):
             'formats.')),
         OptionRecommendation(name='inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Add Table of Contents to begenning of the book.')),
+            help=_('Add Table of Contents to beginning of the book.')),
      ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):

src/calibre/ebooks/txt/txtml.py

@@ -91,7 +91,7 @@ class TXTMLizer(object):
 
         # Remove excessive newlines.
         text = re.sub('\n[ ]+\n', '\n\n', text)
-        text = re.sub('\n{5,}', '\n\n\n\n', text)
+        text = re.sub('\n{3,}', '\n\n', text)
 
         # Replace spaces at the beginning and end of lines
         text = re.sub('(?imu)^[ ]+', '', text)

src/calibre/gui2/__init__.py

@@ -450,12 +450,13 @@ class FileDialog(QObject):
                 if os.path.exists(f):
                     self.selected_files.append(f)
             if self.selected_files:
-                self.selected_files = [qstring_to_unicode(q) for q in self.selected_files]
+                self.selected_files = [unicode(q) for q in self.selected_files]
-                dynamic[self.dialog_name] =  os.path.dirname(self.selected_files[0])
+                saved_loc = self.selected_files[0]
+                if os.path.isfile(saved_loc):
+                    saved_loc = os.path.dirname(saved_loc)
+                dynamic[self.dialog_name] = saved_loc
             self.accepted = bool(self.selected_files)
 
-
-
     def get_files(self):
         if islinux and self.fd.result() != self.fd.Accepted:
             return tuple()

src/calibre/gui2/add.py

@@ -5,7 +5,8 @@ import os, shutil, time
 from Queue import Queue, Empty
 from threading import Thread
 
-from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt
+from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt, \
+        QProgressDialog
 
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2 import question_dialog, error_dialog
@@ -13,6 +14,25 @@ from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.metadata import MetaInformation
 from calibre.constants import preferred_encoding
 
+class DuplicatesAdder(QThread):
+
+    def __init__(self, parent, db, duplicates, db_adder):
+        QThread.__init__(self, parent)
+        self.db, self.db_adder = db, db_adder
+        self.duplicates = duplicates
+
+    def run(self):
+        count = 1
+        for mi, cover, formats in self.duplicates:
+            id = self.db.create_book_entry(mi, cover=cover,
+                    add_duplicates=True)
+            self.db_adder.add_formats(id, formats)
+            self.db_adder.number_of_books_added += 1
+            self.emit(SIGNAL('added(PyQt_PyObject)'), count)
+            count += 1
+        self.emit(SIGNAL('adding_done()'))
+
+
 class RecursiveFind(QThread):
 
     def __init__(self, parent, db, root, single):
@@ -196,15 +216,19 @@ class Adder(QObject):
             self.callback(self.paths, self.names, self.infos)
             self.callback_called = True
 
-    def update(self):
+    def duplicates_processed(self):
-        if self.entry_count <= 0:
-            self.timer.stop()
-            self.process_duplicates()
-            self.pd.hide()
         self.db_adder.end = True
         if not self.callback_called:
             self.callback(self.paths, self.names, self.infos)
             self.callback_called = True
+        if hasattr(self, '__p_d'):
+            self.__p_d.hide()
+
+    def update(self):
+        if self.entry_count <= 0:
+            self.timer.stop()
+            self.pd.hide()
+            self.process_duplicates()
             return
 
         try:
@@ -240,18 +264,28 @@ class Adder(QObject):
     def process_duplicates(self):
         duplicates = self.db_adder.duplicates
         if not duplicates:
-            return
+            return self.duplicates_processed()
         self.pd.hide()
         files = [x[0].title for x in duplicates]
         if question_dialog(self._parent, _('Duplicates found!'),
                         _('Books with the same title as the following already '
                         'exist in the database. Add them anyway?'),
                         '\n'.join(files)):
-            for mi, cover, formats in duplicates:
+            pd = QProgressDialog(_('Adding duplicates...'), '', 0, len(duplicates),
-                id = self.db.create_book_entry(mi, cover=cover,
+                    self._parent)
-                        add_duplicates=True)
+            pd.setCancelButton(None)
-                self.db_adder.add_formats(id, formats)
+            pd.setValue(0)
-                self.db_adder.number_of_books_added += 1
+            pd.show()
+            self.__p_d = pd
+            self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates,
+                    self.db_adder)
+            self.connect(self.__d_a, SIGNAL('added(PyQt_PyObject)'),
+                    pd.setValue)
+            self.connect(self.__d_a, SIGNAL('adding_done()'),
+                    self.duplicates_processed)
+            self.__d_a.start()
+        else:
+            return self.duplicates_processed()
 
     def cleanup(self):
         if hasattr(self, 'pd'):

src/calibre/gui2/convert/__init__.py

@@ -34,7 +34,7 @@ class Widget(QWidget):
     def initialize_options(self, get_option, get_help, db=None, book_id=None):
         '''
         :param get_option: A callable that takes one argument: the option name
-        and returns the correspoing OptionRecommendation.
+        and returns the corresponding OptionRecommendation.
         :param get_help: A callable that takes the option name and return a help
         string.
         '''

src/calibre/gui2/convert/structure_detection.py

@@ -28,11 +28,12 @@ class StructureDetectionWidget(Widget, Ui_Form):
                 'remove_footer', 'footer_regex']
                 )
         self.db, self.book_id = db, book_id
+        for x in ('pagebreak', 'rule', 'both', 'none'):
+            self.opt_chapter_mark.addItem(x)
         self.initialize_options(get_option, get_help, db, book_id)
         self.opt_chapter.set_msg(_('Detect chapters at (XPath expression):'))
         self.opt_page_breaks_before.set_msg(_('Insert page breaks before '
             '(XPath expression):'))
-
     def pre_commit_check(self):
         for x in ('header_regex', 'footer_regex'):
             x = getattr(self, 'opt_'+x)

src/calibre/gui2/convert/structure_detection.ui

@@ -29,26 +29,6 @@
    </item>
    <item row="1" column="1">
     <widget class="QComboBox" name="opt_chapter_mark">
-     <item>
-      <property name="text">
-       <string>pagebreak</string>
-      </property>
-     </item>
-     <item>
-      <property name="text">
-       <string>rule</string>
-      </property>
-     </item>
-     <item>
-      <property name="text">
-       <string>both</string>
-      </property>
-     </item>
-     <item>
-      <property name="text">
-       <string>none</string>
-      </property>
-     </item>
     </widget>
    </item>
    <item row="2" column="0">

src/calibre/gui2/dialogs/add_save.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import textwrap
-
-from PyQt4.Qt import QTabWidget
-
-from calibre.gui2.dialogs.add_save_ui import Ui_TabWidget
-from calibre.library.save_to_disk import config, FORMAT_ARG_DESCS
-
-
-class AddSave(QTabWidget, Ui_TabWidget):
-
-    def __init__(self, parent=None):
-        QTabWidget.__init__(self, parent)
-        self.setupUi(self)
-        c = config()
-        opts = c.parse()
-        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf'):
-            g = getattr(self, 'opt_'+x)
-            g.setChecked(getattr(opts, x))
-            help = '\n'.join(textwrap.wrap(c.get_option(x).help, 75))
-            g.setToolTip(help)
-            g.setWhatsThis(help)
-
-        for x in ('formats', 'timefmt'):
-            g = getattr(self, 'opt_'+x)
-            g.setText(getattr(opts, x))
-            help = '\n'.join(textwrap.wrap(c.get_option(x).help, 75))
-            g.setToolTip(help)
-            g.setWhatsThis(help)
-
-        help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
-        self.opt_template.initialize('save_to_disk_template_history',
-                opts.template, help=help)
-
-        variables = sorted(FORMAT_ARG_DESCS.keys())
-        rows = []
-        for var in variables:
-            rows.append(u'<tr><td>%s</td><td>%s</td></tr>'%
-                    (var, FORMAT_ARG_DESCS[var]))
-        table = u'<table>%s</table>'%(u'\n'.join(rows))
-        self.template_variables.setText(table)
-
-    def save_settings(self):
-        c = config()
-        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf'):
-            c.set(x, getattr(self, 'opt_'+x).isChecked())
-        for x in ('formats', 'template', 'timefmt'):
-            c.set(x, unicode(getattr(self, 'opt_'+x).text()).strip())
-
-
-
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    app=QApplication([])
-    a = AddSave()
-    a.show()
-    app.exec_()
-    a.save_settings()
-

src/calibre/gui2/dialogs/config/__init__.py

@@ -41,7 +41,8 @@ class ConfigTabs(QTabWidget):
         log = Log()
         log.outputs = []
 
-        self.plumber = Plumber('dummy.epub', 'dummy.epub', log, dummy=True)
+        self.plumber = Plumber('dummy.epub', 'dummy.epub', log, dummy=True,
+                merge_plugin_recs=False)
 
         def widget_factory(cls):
             return cls(self, self.plumber.get_option_by_name,
@@ -298,7 +299,8 @@ class EmailAccounts(QAbstractTableModel):
         while y in self.accounts:
             c += 1
             y = x + str(c)
-        self.accounts[y] = ['MOBI, EPUB', True,
+        auto_send = len(self.accounts) < 1
+        self.accounts[y] = ['MOBI, EPUB', auto_send,
                                                 len(self.account_order) == 0]
         self.account_order = sorted(self.accounts.keys())
         self.reset()
@@ -756,7 +758,8 @@ class CheckIntegrity(QProgressDialog):
 
     def __init__(self, db, parent=None):
         QProgressDialog.__init__(self, parent)
-        self.setCancelButtonText('')
+        self.db = db
+        self.setCancelButton(None)
         self.setMinimum(0)
         self.setMaximum(100)
         self.setWindowTitle(_('Checking database integrity'))

src/calibre/gui2/dialogs/config/add_save.py

@@ -26,7 +26,8 @@ class AddSave(QTabWidget, Ui_TabWidget):
             self.removeTab(2)
         c = config()
         opts = c.parse()
-        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf'):
+        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
+                'replace_whitespace', 'to_lowercase'):
             g = getattr(self, 'opt_'+x)
             g.setChecked(getattr(opts, x))
             help = '\n'.join(textwrap.wrap(c.get_option(x).help, 75))
@@ -74,7 +75,8 @@ class AddSave(QTabWidget, Ui_TabWidget):
         if not self.validate():
             return False
         c = config()
-        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf'):
+        for x in ('asciiize', 'update_metadata', 'save_cover', 'write_opf',
+                'replace_whitespace', 'to_lowercase'):
             c.set(x, getattr(self, 'opt_'+x).isChecked())
         for x in ('formats', 'template', 'timefmt'):
             c.set(x, unicode(getattr(self, 'opt_'+x).text()).strip())

src/calibre/gui2/dialogs/config/add_save.ui

@@ -77,14 +77,14 @@
       </property>
      </widget>
     </item>
-    <item row="1" column="0" colspan="2">
+    <item row="1" column="0">
      <widget class="QCheckBox" name="opt_save_cover">
       <property name="text">
        <string>Save &amp;cover separately</string>
       </property>
      </widget>
     </item>
-    <item row="2" column="0" colspan="2">
+    <item row="2" column="0">
      <widget class="QCheckBox" name="opt_update_metadata">
       <property name="text">
        <string>Update &amp;metadata in saved copies</string>
@@ -163,6 +163,20 @@
       </layout>
      </widget>
     </item>
+    <item row="1" column="1">
+     <widget class="QCheckBox" name="opt_replace_whitespace">
+      <property name="text">
+       <string>Replace space with &amp;underscores</string>
+      </property>
+     </widget>
+    </item>
+    <item row="2" column="1">
+     <widget class="QCheckBox" name="opt_to_lowercase">
+      <property name="text">
+       <string>Change paths to &amp;lowercase</string>
+      </property>
+     </widget>
+    </item>
    </layout>
   </widget>
  </widget>

src/calibre/gui2/main.py

@@ -14,8 +14,9 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
-from calibre import __version__, __appname__, \
+from calibre import  prints, patheq
-                    iswindows, isosx, prints, patheq
+from calibre.constants import __version__, __appname__, \
+                    iswindows, isosx, filesystem_encoding
 from calibre.utils.filenames import ascii_filename
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import prefs, dynamic
@@ -424,9 +425,18 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             error_dialog(self, _('Bad database location'),
                     _('Bad database location')+':'+self.library_path,
                     det_msg=traceback.format_exc()).exec_()
+            fname = _('Calibre Library')
+            if isinstance(fname, unicode):
+                try:
+                    fname = fname.encode(filesystem_encoding)
+                except:
+                    fname = 'Calibre Library'
+            x = os.path.expanduser('~'+os.sep+fname)
+            if not os.path.exists(x):
+                os.makedirs(x)
             dir = unicode(QFileDialog.getExistingDirectory(self,
                             _('Choose a location for your ebook library.'),
-                            os.path.expanduser('~')))
+                            x))
             if not dir:
                 QCoreApplication.exit(1)
                 raise SystemExit(1)
@@ -1493,6 +1503,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.search.clear_to_help()
         self.status_bar.reset_info()
         self.library_view.sortByColumn(3, Qt.DescendingOrder)
+        self.library_view.model().count_changed()
 
     ############################################################################
 
@@ -1565,14 +1576,23 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             self.device_error_dialog.show()
 
     def job_exception(self, job):
+        if not hasattr(self, '_modeless_dialogs'):
+            self._modeless_dialogs = []
+        if self.isVisible():
+            for x in list(self._modeless_dialogs):
+                if not x.isVisible():
+                    self._modeless_dialogs.remove(x)
         try:
             if 'calibre.ebooks.DRMError' in job.details:
-                error_dialog(self, _('Conversion Error'),
+                d = error_dialog(self, _('Conversion Error'),
                     _('<p>Could not convert: %s<p>It is a '
                       '<a href="%s">DRM</a>ed book. You must first remove the '
                       'DRM using 3rd party tools.')%\
                         (job.description.split(':')[-1],
-                            'http://wiki.mobileread.com/wiki/DRM')).exec_()
+                            'http://wiki.mobileread.com/wiki/DRM'))
+                d.setModal(False)
+                d.show()
+                self._modeless_dialogs.append(d)
                 return
         except:
             pass
@@ -1582,9 +1602,12 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             prints(job.details, file=sys.stderr)
         except:
             pass
-        error_dialog(self, _('Conversion Error'),
+        d = error_dialog(self, _('Conversion Error'),
                 _('<b>Failed</b>')+': '+unicode(job.description),
-                det_msg=job.details).exec_()
+                det_msg=job.details)
+        d.setModal(False)
+        d.show()
+        self._modeless_dialogs.append(d)
 
 
     def initialize_database(self):

src/calibre/gui2/tag_view.py

@@ -93,7 +93,7 @@ class TagsModel(QStandardItemModel):
                 QIcon(':/images/minus.svg')]
         QStandardItemModel.__init__(self)
         self.db = db
-        self.ignore_next_search = False
+        self.ignore_next_search = 0
         self._data = {}
         self.bold_font = QFont()
         self.bold_font.setBold(True)
@@ -129,19 +129,20 @@ class TagsModel(QStandardItemModel):
         self.refresh()
 
     def reinit(self, *args, **kwargs):
-        if not self.ignore_next_search:
+        if self.ignore_next_search == 0:
             for category in self._data.values():
                 for tag in category:
                     tag.state = 0
             self.reset()
-        self.ignore_next_search = False
+        else:
+            self.ignore_next_search -= 1
 
     def toggle(self, index):
         if index.parent().isValid():
             category = self.row_map[index.parent().row()]
             tag = self._data[category][index.row()]
             self.invisibleRootItem().child(index.parent().row()).child(index.row()).toggle()
-            self.ignore_next_search = True
+            self.ignore_next_search = 2
             self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'), index, index)
             return True
         return False

src/calibre/gui2/wizard/__init__.py

@@ -17,6 +17,7 @@ from calibre import __appname__, patheq
 from calibre.library.database2 import LibraryDatabase2
 from calibre.library.move import MoveLibrary
 from calibre.resources import server_resources
+from calibre.constants import filesystem_encoding
 from calibre.gui2.wizard.send_email import smtp_prefs
 from calibre.gui2.wizard.device_ui import Ui_WizardPage as DeviceUI
 from calibre.gui2.wizard.library_ui import Ui_WizardPage as LibraryUI
@@ -423,7 +424,8 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
                         callback)
                 else:
                     rq = Queue()
-                    m = MoveLibrary(oldloc, newloc, db.data.count(), rq)
+                    m = MoveLibrary(oldloc, newloc,
+                            len(db.get_top_level_move_items()[0]), rq)
                     global _mm
                     _mm = MoveMonitor(m, rq, callback, parent)
                     return
@@ -473,6 +475,17 @@ class LibraryPage(QWizardPage, LibraryUI):
     def initializePage(self):
         lp = prefs['library_path']
         if not lp:
+            fname = _('Calibre Library')
+            if isinstance(fname, unicode):
+                try:
+                    fname = fname.encode(filesystem_encoding)
+                except:
+                    fname = 'Calibre Library'
+            lp = os.path.expanduser('~'+os.sep+fname)
+            if not os.path.exists(lp):
+                try:
+                    os.makedirs(lp)
+                except:
                     lp = os.path.expanduser('~')
         self.location.setText(lp)
 

src/calibre/library/cli.py

@@ -183,9 +183,7 @@ def do_list(db, fields, sort_by, ascending, search_text, line_width, separator,
         return template.generate(id="urn:calibre:main", data=data, subtitle=subtitle,
                 sep=os.sep, quote=quote, updated=db.last_modified()).render('xml')
 
-
+def list_option_parser():
-
-def command_list(args, dbpath):
     parser = get_parser(_(
 '''\
 %prog list [options]
@@ -208,6 +206,11 @@ List the books available in the calibre database.
     of = ['text', 'xml', 'stanza']
     parser.add_option('--output-format', choices=of, default='text',
                       help=_('The format in which to output the data. Available choices: %s. Defaults is text.')%of)
+    return parser
+
+
+def command_list(args, dbpath):
+    parser = list_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
     fields = [str(f.strip().lower()) for f in opts.fields.split(',')]
     if 'all' in fields:
@@ -316,9 +319,7 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
     finally:
         sys.stdout = orig
 
-
+def add_option_parser():
-
-def command_add(args, dbpath):
     parser = get_parser(_(
 '''\
 %prog add [options] file1 file2 file3 ...
@@ -333,6 +334,11 @@ the directory related options below.
                       help=_('Process directories recursively'))
     parser.add_option('-d', '--duplicates', action='store_true', default=False,
                       help=_('Add books to database even if they already exist. Comparison is done based on book titles.'))
+    return parser
+
+
+def command_add(args, dbpath):
+    parser = add_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
     if len(args) < 2:
         parser.print_help()
@@ -353,9 +359,8 @@ def do_remove(db, ids):
     if send_message is not None:
         send_message('refreshdb:', 'calibre GUI')
 
-
+def remove_option_parser():
-def command_remove(args, dbpath):
+    return get_parser(_(
-    parser = get_parser(_(
 '''\
 %prog remove ids
 
@@ -363,6 +368,9 @@ Remove the books identified by ids from the database. ids should be a comma sepa
 list of id numbers (you can get id numbers by using the list command). For example, \
 23,34,57-85
 '''))
+
+def command_remove(args, dbpath):
+    parser = remove_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
     if len(args) < 2:
         parser.print_help()
@@ -385,15 +393,18 @@ list of id numbers (you can get id numbers by using the list command). For examp
 def do_add_format(db, id, fmt, path):
     db.add_format_with_hooks(id, fmt.upper(), path, index_is_id=True)
 
-
+def add_format_option_parser():
-def command_add_format(args, dbpath):
+    return get_parser(_(
-    parser = get_parser(_(
 '''\
 %prog add_format [options] id ebook_file
 
 Add the ebook in ebook_file to the available formats for the logical book identified \
 by id. You can get id by using the list command. If the format already exists, it is replaced.
 '''))
+
+
+def command_add_format(args, dbpath):
+    parser = add_format_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
     if len(args) < 3:
         parser.print_help()
@@ -410,8 +421,8 @@ by id. You can get id by using the list command. If the format already exists, i
 def do_remove_format(db, id, fmt):
     db.remove_format(id, fmt, index_is_id=True)
 
-def command_remove_format(args, dbpath):
+def remove_format_option_parser():
-    parser = get_parser(_(
+    return get_parser(_(
 '''
 %prog remove_format [options] id fmt
 
@@ -420,6 +431,10 @@ You can get id by using the list command. fmt should be a file extension \
 like LRF or TXT or EPUB. If the logical book does not have fmt available, \
 do nothing.
 '''))
+
+
+def command_remove_format(args, dbpath):
+    parser = remove_format_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
     if len(args) < 3:
         parser.print_help()
@@ -441,7 +456,7 @@ def do_show_metadata(db, id, as_opf):
     else:
         print unicode(mi).encode(preferred_encoding)
 
-def command_show_metadata(args, dbpath):
+def show_metadata_option_parser():
     parser = get_parser(_(
 '''
 %prog show_metadata [options] id
@@ -451,6 +466,10 @@ id is an id number from the list command.
 '''))
     parser.add_option('--as-opf', default=False, action='store_true',
                       help=_('Print metadata in OPF form (XML)'))
+    return parser
+
+def command_show_metadata(args, dbpath):
+    parser = show_metadata_option_parser()
     opts, args = parser.parse_args(sys.argv[1:]+args)
     if len(args) < 2:
         parser.print_help()
@@ -468,8 +487,8 @@ def do_set_metadata(db, id, stream):
     if send_message is not None:
         send_message('refreshdb:', 'calibre GUI')
 
-def command_set_metadata(args, dbpath):
+def set_metadata_option_parser():
-    parser = get_parser(_(
+    return get_parser(_(
 '''
 %prog set_metadata [options] id /path/to/metadata.opf
 
@@ -478,6 +497,9 @@ from the OPF file metadata.opf. id is an id number from the list command. You
 can get a quick feel for the OPF format by using the --as-opf switch to the
 show_metadata command.
 '''))
+
+def command_set_metadata(args, dbpath):
+    parser = set_metadata_option_parser()
     opts, args = parser.parse_args(sys.argv[1:]+args)
     if len(args) < 3:
         parser.print_help()
@@ -501,7 +523,7 @@ def do_export(db, ids, dir, opts):
             prints('\t'+'\n\t'.join(tb.splitlines()))
             prints(' ')
 
-def command_export(args, dbpath):
+def export_option_parser():
     parser = get_parser(_('''\
 %prog export [options] ids
 
@@ -530,6 +552,16 @@ an opf file). You can get id numbers from the list command.
         parser.add_option(switch, default=opt.default,
                 help=opt.help, dest=pref)
 
+    for pref in ('replace_whitespace', 'to_lowercase'):
+        opt = c.get_option(pref)
+        switch = '--'+pref.replace('_', '-')
+        parser.add_option(switch, default=False, action='store_true',
+                help=opt.help)
+
+    return parser
+
+def command_export(args, dbpath):
+    parser = export_option_parser()
     opts, args = parser.parse_args(sys.argv[1:]+args)
     if (len(args) < 2 and not opts.all):
         parser.print_help()
@@ -541,9 +573,11 @@ an opf file). You can get id numbers from the list command.
     do_export(get_db(dbpath, opts), ids, dir, opts)
     return 0
 
-def main(args=sys.argv):
+COMMANDS = ('list', 'add', 'remove', 'add_format', 'remove_format',
-    commands = ('list', 'add', 'remove', 'add_format', 'remove_format',
                 'show_metadata', 'set_metadata', 'export')
+
+
+def option_parser():
     parser = OptionParser(_(
 '''\
 %%prog command [options] [arguments]
@@ -555,11 +589,16 @@ command is one of:
 
 For help on an individual command: %%prog command --help
 '''
-                          )%'\n  '.join(commands))
+                          )%'\n  '.join(COMMANDS))
+    return parser
+
+
+def main(args=sys.argv):
+    parser = option_parser()
     if len(args) < 2:
         parser.print_help()
         return 1
-    if args[1] not in commands:
+    if args[1] not in COMMANDS:
         if args[1] == '--version':
             parser.print_version()
             return 0

src/calibre/library/database2.py

@@ -1444,14 +1444,33 @@ class LibraryDatabase2(LibraryDatabase):
         if notify:
             self.notify('add', [id])
 
+    def get_top_level_move_items(self):
+        items = set(os.listdir(self.library_path))
+        paths = set([])
+        for x in self.data.universal_set():
+            path = self.path(x, index_is_id=True)
+            path = path.split(os.sep)[0]
+            paths.add(path)
+        paths.add('metadata.db')
+        path_map = {}
+        for x in paths:
+            path_map[x] = x
+        if not self.is_case_sensitive:
+            for x in items:
+                path_map[x.lower()] = x
+            items = set(path_map)
+            paths = set([x.lower() for x in paths])
+        items = items.intersection(paths)
+        return items, path_map
+
     def move_library_to(self, newloc, progress=lambda x: x):
         if not os.path.exists(newloc):
             os.makedirs(newloc)
-        items = os.listdir(self.library_path)
         old_dirs = set([])
-        for i, x in enumerate(items):
+        items, path_map = self.get_top_level_move_items()
+        for x in items:
             src = os.path.join(self.library_path, x)
-            dest = os.path.join(newloc, x)
+            dest = os.path.join(newloc, path_map[x])
             if os.path.isdir(src):
                 if os.path.exists(dest):
                     shutil.rmtree(dest)
@@ -1461,6 +1480,7 @@ class LibraryDatabase2(LibraryDatabase):
                 if os.path.exists(dest):
                     os.remove(dest)
                 shutil.copyfile(src, dest)
+            x = path_map[x]
             if not isinstance(x, unicode):
                 x = x.decode(filesystem_encoding, 'replace')
             progress(x)
@@ -1675,13 +1695,12 @@ books_series_link      feeds
         user_version = self.user_version
         sql = self.conn.dump()
         self.conn.close()
-        dest = self.dbpath+'.old'
+        dest = self.dbpath+'.tmp'
         if os.path.exists(dest):
             os.remove(dest)
-        shutil.copyfile(self.dbpath, dest)
+        conn = None
         try:
-            os.remove(self.dbpath)
+            ndb = DBThread(dest, None)
-            ndb = DBThread(self.dbpath, None)
             ndb.connect()
             conn = ndb.conn
             conn.executescript(sql)
@@ -1690,15 +1709,21 @@ books_series_link      feeds
             conn.commit()
             conn.close()
         except:
-            if os.path.exists(self.dbpath):
+            if conn is not None:
-                os.remove(self.dbpath)
+                try:
-            shutil.copyfile(dest, self.dbpath)
+                    conn.close()
+                except:
+                    pass
+            if os.path.exists(dest):
                 os.remove(dest)
             raise
         else:
-            os.remove(dest)
+            os.remove(self.dbpath)
+            shutil.copyfile(dest, self.dbpath)
             self.connect()
             self.refresh()
+        if os.path.exists(dest):
+            os.remove(dest)
         callback(0.1, _('Checking for missing files.'))
         bad = {}
         us = self.data.universal_set()

src/calibre/library/save_to_disk.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO
+import os, traceback, cStringIO, re
 
 from calibre.utils.config import Config, StringConfig
 from calibre.utils.filenames import shorten_components_to, supports_long_names, \
@@ -17,7 +17,7 @@ from calibre.constants import preferred_encoding, filesystem_encoding
 
 from calibre import strftime
 
-DEFAULT_TEMPLATE = '{author_sort}/{title} - {authors}'
+DEFAULT_TEMPLATE = '{author_sort}/{title}/{title} - {authors}'
 FORMAT_ARG_DESCS = dict(
         title=_('The title'),
         authors=_('The authors'),
@@ -71,6 +71,10 @@ def config(defaults=None):
     x('timefmt', default='%b, %Y',
             help=_('The format in which to display dates. %d - day, %b - month, '
                 '%Y - year. Default is: %b, %Y'))
+    x('to_lowercase', default=False,
+            help=_('Convert paths to lowercase.'))
+    x('replace_whitespace', default=False,
+            help=_('Replace whitespace with underscores.'))
     return c
 
 def preprocess_template(template):
@@ -81,7 +85,9 @@ def preprocess_template(template):
         template = template.decode(preferred_encoding, 'replace')
     return template
 
-def get_components(template, mi, id, timefmt='%b %Y', length=250, sanitize_func=ascii_filename):
+def get_components(template, mi, id, timefmt='%b %Y', length=250,
+        sanitize_func=ascii_filename, replace_whitespace=False,
+        to_lowercase=False):
     format_args = dict(**FORMAT_ARGS)
     if mi.title:
         format_args['title'] = mi.title
@@ -113,6 +119,11 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, sanitize_func=
         components = [str(id)]
     components = [x.encode(filesystem_encoding, 'replace') if isinstance(x,
         unicode) else x for x in components]
+    if to_lowercase:
+        components = [x.lower() for x in components]
+    if replace_whitespace:
+        components = [re.sub(r'\s', '_', x) for x in components]
+
     return shorten_components_to(length, components)
 
 
@@ -134,7 +145,9 @@ def save_book_to_disk(id, db, root, opts, length):
         return True, id, mi.title
 
     components = get_components(opts.template, mi, id, opts.timefmt, length,
-            ascii_filename if opts.asciiize else sanitize_file_name)
+            ascii_filename if opts.asciiize else sanitize_file_name,
+            to_lowercase=opts.to_lowercase,
+            replace_whitespace=opts.replace_whitespace)
     base_path = os.path.join(root, *components)
     base_name = os.path.basename(base_path)
     dirpath = os.path.dirname(base_path)

src/calibre/library/server.py

@@ -110,7 +110,7 @@ class LibraryServer(object):
         <title>${authors}</title>
         <id>urn:calibre:${record[FM['id']]}</id>
         <updated>${timestamp}</updated>
-        <link type="application/atom+xml" href="/?authorid=${record[FM['id']]}" />
+        <link type="application/atom+xml" href="/stanza/?authorid=${record[FM['id']]}" />
     </entry>
     '''))
 
@@ -120,7 +120,7 @@ class LibraryServer(object):
       <title>calibre Library</title>
       <id>$id</id>
       <updated>${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-      <link rel="search" title="Search" type="application/atom+xml" href="/?search={searchTerms}"/>
+      <link rel="search" title="Search" type="application/atom+xml" href="/stanza/?search={searchTerms}"/>
       <author>
         <name>calibre</name>
         <uri>http://calibre.kovidgoyal.net</uri>
@@ -140,7 +140,7 @@ class LibraryServer(object):
       <title>calibre Library</title>
       <id>$id</id>
       <updated>${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-      <link rel="search" title="Search" type="application/atom+xml" href="/?search={searchTerms}"/>
+      <link rel="search" title="Search" type="application/atom+xml" href="/stanza/?search={searchTerms}"/>
       <author>
         <name>calibre</name>
         <uri>http://calibre.kovidgoyal.net</uri>
@@ -152,19 +152,19 @@ class LibraryServer(object):
         <title>By Author</title>
         <id>urn:uuid:fc000fa0-8c23-11de-a31d-0002a5d5c51b</id>
         <updated>${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-        <link type="application/atom+xml" href="/?sortby=byauthor" />
+        <link type="application/atom+xml" href="/stanza/?sortby=byauthor" />
       </entry>
       <entry>
         <title>By Title</title>
         <id>urn:uuid:1df4fe40-8c24-11de-b4c6-0002a5d5c51b</id>
         <updated>${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-        <link type="application/atom+xml" href="/?sortby=bytitle" />
+        <link type="application/atom+xml" href="/stanza/?sortby=bytitle" />
       </entry>
       <entry>
         <title>By Newest</title>
         <id>urn:uuid:3c6d4940-8c24-11de-a4d7-0002a5d5c51b</id>
         <updated>${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-        <link type="application/atom+xml" href="/?sortby=bynewest" />
+        <link type="application/atom+xml" href="/stanza/?sortby=bynewest" />
       </entry>
     </feed>
     '''))
@@ -460,8 +460,11 @@ class LibraryServer(object):
     @expose
     def index(self, **kwargs):
         'The / URL'
-        want_opds = cherrypy.request.headers.get('Stanza-Device-Name', 919) != \
+        ua = cherrypy.request.headers.get('User-Agent', '').strip()
-            919 or cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919
+        want_opds = \
+            cherrypy.request.headers.get('Stanza-Device-Name', 919) != 919 or \
+            cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
+            ua.startswith('Stanza')
         return self.stanza(search=kwargs.get('search', None), sortby=kwargs.get('sortby',None), authorid=kwargs.get('authorid',None)) if want_opds else self.static('index.html')
 
 

src/calibre/library/sqlite.py

@@ -157,6 +157,8 @@ class DatabaseException(Exception):
 def proxy(fn):
     ''' Decorator to call methods on the database connection in the proxy thread '''
     def run(self, *args, **kwargs):
+        if self.closed:
+            raise DatabaseException('Connection closed', '')
         with global_lock:
             if self.proxy.unhandled_error[0] is not None:
                 raise DatabaseException(*self.proxy.unhandled_error)
@@ -174,10 +176,12 @@ class ConnectionProxy(object):
 
     def __init__(self, proxy):
         self.proxy = proxy
+        self.closed = False
 
     def close(self):
-        if self.proxy.unhandled_error is None:
+        if self.proxy.unhandled_error[0] is None:
             self.proxy.requests.put((self.proxy.CLOSE, [], {}))
+            self.closed = True
 
     @proxy
     def get(self, query, all=True): pass

src/calibre/manual/custom.py

@@ -69,6 +69,46 @@ CLI_PREAMBLE='''\
 {usage}
 '''
 
+def generate_calibredb_help(preamble, info):
+    from calibre.library.cli import COMMANDS, get_parser
+    import calibre.library.cli as cli
+    preamble = preamble[:preamble.find('\n\n\n', preamble.find('code-block'))]
+    preamble += textwrap.dedent('''
+
+    :command:`calibredb` is the command line interface to the |app| database. It has
+    several sub-commands, documented below:
+
+    ''')
+
+    global_parser = get_parser('')
+    groups = []
+    for grp in global_parser.option_groups:
+        groups.append((grp.title.capitalize(), grp.description, grp.option_list))
+
+    global_options = '\n'.join(render_options('calibredb', groups, False, False))
+
+
+    lines, toc = [], []
+    for cmd in COMMANDS:
+        parser = getattr(cli, cmd+'_option_parser')()
+        toc.append('  * :ref:`calibredb-%s`'%cmd)
+        lines += ['.. _calibredb-'+cmd+':', '']
+        lines += [cmd, '~'*20, '']
+        usage = parser.usage.strip()
+        usage = [i for i in usage.replace('%prog', 'calibredb').splitlines()]
+        cmdline = '    '+usage[0]
+        usage = usage[1:]
+        usage = [i.replace(cmd, ':command:`%s`'%cmd) for i in usage]
+        lines += ['.. code-block:: none', '', cmdline, '']
+        lines += usage
+        groups = [(None, None, parser.option_list)]
+        lines += ['']
+        lines += render_options('calibredb '+cmd, groups, False)
+        lines += ['']
+
+    toc = '\n'.join(toc)
+    raw = preamble + '\n\n'+toc + '\n\n' + global_options+'\n\n'+'\n'.join(lines)
+    update_cli_doc(os.path.join('cli', 'calibredb.rst'), raw, info)
 
 def generate_ebook_convert_help(preamble, info):
     from calibre.ebooks.conversion.cli import create_option_parser
@@ -125,10 +165,11 @@ def update_cli_doc(path, raw, info):
         info('creating '+os.path.splitext(os.path.basename(path))[0])
         open(path, 'wb').write(raw)
 
-def render_options(cmd, groups, options_header=True):
+def render_options(cmd, groups, options_header=True, add_program=True):
-    lines = []
+    lines = ['']
     if options_header:
         lines = ['[options]', '-'*15, '']
+    if add_program:
         lines += ['.. program:: '+cmd, '']
     for title, desc, options in groups:
         if title:
@@ -153,6 +194,7 @@ def cli_docs(app):
     for script in entry_points['console_scripts']:
         module = script[script.index('=')+1:script.index(':')].strip()
         cmd = script[:script.index('=')].strip()
+        if cmd in ('calibre-complete', 'calibre-parallel'): continue
         module = __import__(module, fromlist=[module.split('.')[-1]])
         if hasattr(module, 'option_parser'):
             documented_cmds.append((cmd, getattr(module, 'option_parser')()))
@@ -180,6 +222,8 @@ def cli_docs(app):
         preamble = CLI_PREAMBLE.format(cmd=cmd, cmdline=cmdline, usage=usage)
         if cmd == 'ebook-convert':
             generate_ebook_convert_help(preamble, info)
+        elif cmd == 'calibredb':
+            generate_calibredb_help(preamble, info)
         else:
             groups = [(None, None, parser.option_list)]
             for grp in parser.option_groups:

src/calibre/manual/faq.rst

@@ -54,11 +54,11 @@ In order to convert a collection of HTML files in a specific oder, you have to c
 
 Then just add this HTML file to the GUI and use the convert button to create your ebook.
 
-How do I convert my file containing non-English characters?
+How do I convert my file containing non-English characters, or smart quotes?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 There are two aspects to this problem: 
   1. Knowing the encoding of the source file: |app| tries to guess what character encoding your source files use, but often, this is impossible, so you need to tell it what encoding to use. This can be done in the GUI via the :guilabel:`Input character encoding` field in the :guilabel:`Look & Feel` section. The command-line tools all have an :option:`--input-encoding` option.
-  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. |app| will then automatically convert the HTML files into the UTF-8 encoding when adding them.
+  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first.
   3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader. 
 
 
@@ -95,13 +95,17 @@ turned into a collection on the reader. Note that the PRS-500 does not support c
 
 How do I use |app| with my iPhone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-First install the Stanza reader on your iPhone from http://www.lexcycle.com . Then,
+First install the Stanza reader on your iPhone using iTunes.
 
-  * Set the output format for calibre to EPUB (The output format can be set next to the big red heart) 
+  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General) 
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
 
-Now you should be able to access your books on your iPhone. 
+Now you should be able to access your books on your iPhone by opening Stanza and going to "Shared Books". Under Shared Books you will see an entry "Book in calibre". If you don't, make sure your iPhone is connected using the WiFi network in your house, not 3G. If the |app| catalog is still not detected in Stanza, you can add it manually in Stanza, by clicking "Online Catalog" and the clicking the plus icon in the lower right corner to add a new catalog. In the Add Catalog screen enter whatever name you like and in the URL field, enter the following::
+
+    http://192.168.1.2:8080/
+
+Replace ``192.168.1.2`` with the local IP address of the computer running |app|. If you have changed the port the |app| content server is running on, you will have to change ``8080`` as well to the new port. The local IP address is the IP address you computer is assigned on your home network. A quick Google search will tell you how to find out your local IP address.  
 
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -136,6 +140,12 @@ Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| is designed to have columns for the most frequently and widely used fields. If it does not have a coulmn for your favorite field, you can always add a tag to the book for that piece of information. |app| also supports a general purpose "comments" fields for longer items.
 
+How do I move my |app| library from one computer to another?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path tot he library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. 
+
+Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to Preferences->Advanced and click the Check database integrity button. It will warn you about missing files, if any, which you should then transfer by hand.
+
 
 Content From The Web
 ---------------------

src/calibre/utils/fonts/__init__.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 import os, sys
 from threading import Thread
 
-from calibre.constants import plugins
+from calibre.constants import plugins, iswindows
 
 _fc, _fc_err = plugins['fontconfig']
 
@@ -31,6 +31,12 @@ class FontConfig(Thread):
             if isinstance(config_dir, unicode):
                 config_dir = config_dir.encode(sys.getfilesystemencoding())
             config = os.path.join(config_dir, 'fonts.conf')
+        if iswindows and getattr(sys, 'frozen', False):
+            config_dir = os.path.join(os.path.dirname(sys.executable),
+                'etc', 'fonts')
+            if isinstance(config_dir, unicode):
+                config_dir = config_dir.encode(sys.getfilesystemencoding())
+            config = os.path.join(config_dir, 'fonts.conf')
         try:
             _fc.initialize(config)
         except:

src/calibre/utils/poppler/__init__.py

@@ -22,6 +22,7 @@ def get_metadata(stream, cover=True):
     raw = stream.read()
     doc = poppler.PDFDoc()
     doc.load(raw)
+    del raw
     title = doc.title
     if not title or not title.strip():
         title = _('Unknown')
@@ -55,7 +56,6 @@ def get_metadata(stream, cover=True):
         if cdata is not None:
             mi.cover_data = ('jpg', cdata)
     del doc
-    del raw
     return mi
 
 

src/calibre/utils/poppler/poppler.cpp

@@ -176,15 +176,16 @@ poppler_PDFDoc_render_page(poppler_PDFDoc *self, PyObject *args, PyObject *kwarg
     int num;
 
     if (!PyArg_ParseTuple(args, "i|ff", &num, &xdpi, &ydpi)) return ans;
-    if ( num < 0 || num >= self->doc->numPages()) {
-        PyErr_SetString(PyExc_ValueError, "Invalid page number");
-        return ans;
-    }
     if ( self->doc->isLocked()) { 
         PyErr_SetString(PyExc_ValueError, "This document is copyrighted.");
         return ans;
     }
 
+    if ( num < 0 || num >= self->doc->numPages()) {
+        PyErr_SetString(PyExc_ValueError, "Invalid page number");
+        return ans;
+    }
+
     page = self->doc->page(num);
     img = page->renderToImage(xdpi, ydpi);
     if (img.isNull()) {

src/calibre/utils/windows/winutil.c

@@ -929,6 +929,7 @@ initwinutil(void) {
     PyModule_AddIntConstant(m, "CSIDL_COOKIES", CSIDL_COOKIES);
     PyModule_AddIntConstant(m, "CSIDL_FLAG_CREATE", CSIDL_FLAG_CREATE);
     PyModule_AddIntConstant(m, "CSIDL_FLAG_DONT_VERIFY", CSIDL_FLAG_DONT_VERIFY);
+    PyModule_AddIntConstant(m, "CSIDL_FONTS", CSIDL_FONTS);
     PyModule_AddIntConstant(m, "CSIDL_HISTORY", CSIDL_HISTORY);
     PyModule_AddIntConstant(m, "CSIDL_INTERNET_CACHE", CSIDL_INTERNET_CACHE);
     PyModule_AddIntConstant(m, "CSIDL_LOCAL_APPDATA", CSIDL_LOCAL_APPDATA);

src/calibre/web/feeds/news.py

@@ -679,7 +679,7 @@ class BasicNewsRecipe(Recipe):
             fetcher.browser_lock = fetcher.DUMMY_LOCK
         res, path, failures = fetcher.start_fetch(url), fetcher.downloaded_paths, fetcher.failed_links
         if not res or not os.path.exists(res):
-            raise Exception(_('Could not fetch article. Run with --debug to see the reason'))
+            raise Exception(_('Could not fetch article. Run with -vv to see the reason'))
         return res, path, failures
 
     def fetch_article(self, url, dir, f, a, num_of_feeds):
@@ -741,6 +741,9 @@ class BasicNewsRecipe(Recipe):
                     url = self.print_version(article.url)
                 except NotImplementedError:
                     url = article.url
+                except:
+                    self.log.exception('Failed to find print version for: '+article.url)
+                    url = None
                 if not url:
                     continue
                 func, arg = (self.fetch_embedded_article, article) if self.use_embedded_content else \

src/calibre/web/feeds/recipes/__init__.py

@@ -42,7 +42,7 @@ recipe_modules = ['recipe_' + r for r in (
            'moneynews', 'der_standard', 'diepresse', 'nzz_ger', 'hna',
            'seattle_times', 'scott_hanselman', 'coding_horror', 'twitchfilms',
            'stackoverflow', 'telepolis_artikel', 'zaobao', 'usnews',
-           'straitstimes', 'index_hu', 'pcworld_hu', 'hrt', 'rts',
+           'straitstimes', 'index_hu', 'pcworld_hu', 'hrt', 'rts', 'axxon_news',
            'h1', 'h2', 'h3', 'phd_comics', 'woz_die', 'elektrolese',
            'climate_progress', 'carta', 'slashdot', 'publico',
            'the_budget_fashionista', 'elperiodico_catalan',
@@ -55,7 +55,7 @@ recipe_modules = ['recipe_' + r for r in (
            'eltiempo_hn', 'slate', 'tnxm', 'bbcvietnamese', 'vnexpress',
            'volksrant', 'theeconomictimes_india', 'ourdailybread',
            'monitor', 'republika', 'beta', 'beta_en', 'glasjavnosti',
-           'esquire',
+           'esquire', 'livemint', 'thedgesingapore', 'darknet',
           )]
 
 

src/calibre/web/feeds/recipes/recipe_axxon_news.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+axxon.com.ar
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
+
+class Axxon_news(BasicNewsRecipe):
+    title                 = 'Axxon noticias'
+    __author__            = 'Darko Miletic'
+    description           = 'Axxon, Ciencia Ficcion en Bits'
+    publisher             = 'Axxon'
+    category              = 'news, SF, Argentina, science, movies'
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = False
+    use_embedded_content  = False
+    language              = _('Spanish')
+    lang                  = 'es-AR'
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : lang
+                        , 'pretty_print'     : True
+                        }
+
+
+    keep_only_tags     = [dict(name='div', attrs={'class':'post'})]
+
+    remove_tags = [dict(name=['object','link','iframe','embed'])]
+
+    feeds          = [(u'Noticias', u'http://axxon.com.ar/noticias/feed/')]
+
+    remove_attributes = ['style','width','height','font','border','align']
+
+
+    def adeify_images2(cls, soup):
+        for item in soup.findAll('img'):
+            for attrib in ['height','width','border','align','style']:
+                if item.has_key(attrib):
+                   del item[attrib]
+            oldParent = item.parent
+            if oldParent.name == 'a':
+               oldParent.name == 'p'
+            myIndex = oldParent.contents.index(item)
+            brtag  = Tag(soup,'br')
+            oldParent.insert(myIndex+1,brtag)
+        return soup
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = self.lang
+        soup.html['lang']     = self.lang
+        mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)])
+        soup.html.insert(0,mlang)
+        return self.adeify_images2(soup)
+

src/calibre/web/feeds/recipes/recipe_darknet.py

@@ -0,0 +1,43 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch darknet.
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class darknet(BasicNewsRecipe):
+
+    title = 'darknet'
+    description = 'Ethical hacking and security news'
+    __author__ = 'Oliver Niesner'
+    language = _('English')
+    use_embedded_content   = False
+    timefmt = ' [%b %d %Y]'
+    max_articles_per_feed = 40
+    no_stylesheets = True
+    oldest_article = 180
+
+    remove_tags = [dict(id='navi_top'),
+		   dict(id='navi_bottom'),
+		   dict(id='logo'),
+		   dict(id='login_suche'),
+		   dict(id='navi_login'),
+		   dict(id='breadcrumb'),
+		   dict(id='subtitle'),
+		   dict(id='bannerzone'),
+		   dict(name='span', attrs={'class':'rsaquo'}),
+		   dict(name='span', attrs={'class':'next'}),
+		   dict(name='span', attrs={'class':'prev'}),
+		   dict(name='div', attrs={'class':'news_logo'}),
+		   dict(name='div', attrs={'class':'nextprev'}),
+		   dict(name='p', attrs={'class':'news_option'}),
+		   dict(name='p', attrs={'class':'news_foren'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'entrybody'})]
+
+    feeds =  [ ('darknet', 'http://feedproxy.google.com/darknethackers') ]
+
+
+

src/calibre/web/feeds/recipes/recipe_elektrolese.py

@@ -32,7 +32,6 @@ class elektrolese(BasicNewsRecipe):
 
 
 
-    feeds =  [ (u'electrolese', u'http://elektrolese.blogspot.com/feeds/posts/default?alt=rss') ]
+    feeds =  [ (u'elektrolese', u'http://elektrolese.blogspot.com/feeds/posts/default?alt=rss') ]
-
 
 

src/calibre/web/feeds/recipes/recipe_hna.py

@@ -19,16 +19,24 @@ class hnaDe(BasicNewsRecipe):
     timefmt = ' [%d %b %Y]'
     max_articles_per_feed = 40
     no_stylesheets = True
+    remove_javascript = True
     encoding = 'iso-8859-1'
 
     remove_tags = [dict(id='topnav'),
 		   dict(id='nav_main'),
+		   dict(id='teaser'),
 		   dict(id='suchen'),
+		   dict(id='superbanner'),
+		   dict(id='navigation'),
+		   dict(id='skyscraper'),
 		   dict(id=''),
                    dict(name='span'),
 		   dict(name='ul', attrs={'class':'linklist'}),
 		   dict(name='a', attrs={'href':'#'}),
+		   dict(name='div', attrs={'class':'hlist'}),
+		   dict(name='div', attrs={'class':'subc noprint'}),
 		   dict(name='p', attrs={'class':'breadcrumb'}),
+		   dict(name='a', attrs={'style':'cursor:hand'}),
 		   dict(name='p', attrs={'class':'h5'})]
     #remove_tags_after = [dict(name='div', attrs={'class':'rahmenbreaking'})]
     remove_tags_after = [dict(name='a', attrs={'href':'#'})]
@@ -38,3 +46,4 @@ class hnaDe(BasicNewsRecipe):
 
 
 
+

src/calibre/web/feeds/recipes/recipe_linuxdevices.py

@@ -8,7 +8,7 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
-class Sueddeutsche(BasicNewsRecipe):
+class LinuxDevices(BasicNewsRecipe):
 
     title = u'Linuxdevices'
     description = 'News about Linux driven Hardware'
@@ -16,22 +16,22 @@ class Sueddeutsche(BasicNewsRecipe):
     use_embedded_content   = False
     timefmt = ' [%a %d %b %Y]'
     max_articles_per_feed = 50
-    language = _('English')
     no_stylesheets = True
-    html2epub_options = 'linearize_tables = True\nbase_font_size2=14'
+    language = _('English')
-    html2lrf_options = ['--ignore-tables']
+    remove_javascript = True
+    conversion_options = { 'linearize_tables' : True}
     encoding = 'latin1'
 
 
-    remove_tags_after = [dict(id='nointelliTXT')]
+    remove_tags_after = [dict(id='intelliTxt')]
     filter_regexps = [r'ad\.doubleclick\.net']
 
     remove_tags = [dict(name='div', attrs={'class':'bannerSuperBanner'}),
                    dict(name='div', attrs={'class':'bannerSky'}),
+                   dict(name='div', attrs={'border':'0'}),
                    dict(name='div', attrs={'class':'footerLinks'}),
                    dict(name='div', attrs={'class':'seitenanfang'}),
                    dict(name='td', attrs={'class':'mar5'}),
-                   dict(name='td', attrs={'class':'mar5'}),
                    dict(name='table', attrs={'class':'pageAktiv'}),
                    dict(name='table', attrs={'class':'xartable'}),
                    dict(name='table', attrs={'class':'wpnavi'}),
@@ -40,24 +40,26 @@ class Sueddeutsche(BasicNewsRecipe):
                    dict(name='table', attrs={'class':'artikelBox'}),
                    dict(name='table', attrs={'class':'kommentare'}),
                    dict(name='table', attrs={'class':'pageBoxBot'}),
+                   dict(name='table', attrs={'td':'height="3"'}),
+                   dict(name='table', attrs={'class':'contentpaneopen'}),
                    dict(name='td', attrs={'nowrap':'nowrap'}),
-                   dict(name='td', attrs={'valign':'middle'}),
                    dict(name='td', attrs={'align':'left'}),
-                   dict(name='td', attrs={'align':'center'}),
                    dict(name='td', attrs={'height':'5'}),
+                   dict(name='td', attrs={'class':'ArticleWidgetsHeadline'}),
                    dict(name='div', attrs={'class':'artikelBox navigatorBox'}),
                    dict(name='div', attrs={'class':'similar-article-box'}),
                    dict(name='div', attrs={'class':'videoBigHack'}),
                    dict(name='td', attrs={'class':'artikelDruckenRight'}),
                    dict(name='td', attrs={'class':'width="200"'}),
+                   dict(name='span', attrs={'class':'content_rating'}),
+                   dict(name='a', attrs={'href':'http://www.addthis.com/bookmark.php'}),
                    dict(name='a', attrs={'href':'/news'}),
-                   dict(name='a', attrs={'href':'/'}),
-                   dict(name='a', attrs={'href':'/articles'}),
                    dict(name='a', attrs={'href':'/cgi-bin/survey/survey.cgi'}),
                    dict(name='a', attrs={'href':'/cgi-bin/board/UltraBoard.pl'}),
                    dict(name='iframe'),
                    dict(name='form'),
                    dict(name='span', attrs={'class':'hidePrint'}),
+                   dict(id='ArticleWidgets'),
                    dict(id='headerLBox'),
                    dict(id='nointelliTXT'),
                    dict(id='rechteSpalte'),
@@ -69,27 +71,18 @@ class Sueddeutsche(BasicNewsRecipe):
                    dict(id='nnav-headerteaser'),
                    dict(id='nnav-head'),
                    dict(id='nnav-top'),
-                   dict(id='nnav-logodiv'),
-                   dict(id='nnav-logo'),
-                   dict(id='nnav-oly'),
                    dict(id='readcomment')]
 
 
 
-    feeds =  [ (u'Linuxdevices', u'http://www.linuxdevices.com/backend/headlines.rss') ]
+    feeds =  [ (u'Linuxdevices', u'http://www.linuxfordevices.com/rss.xml') ]
 
     def preprocess_html(self, soup):
-        for item in soup.findAll(re.compile('^a')):
-            item.extract()
         match = re.compile(r"^Related")
         for item in soup.findAll('b', text=match):
             item.extract()
-        for item in soup.findAll(re.compile('^li')):
-            item.extract()
         for item in soup.findAll(re.compile('^ul')):
             item.extract()
-        for item in soup.find(re.compile('^br')):
-            item.extract()
         for item in soup.findAll('br', limit=10):
             item.extract()
         return soup
@@ -101,4 +94,3 @@ class Sueddeutsche(BasicNewsRecipe):
         return soup
 
 
-

src/calibre/web/feeds/recipes/recipe_livemint.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.livemint.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LiveMint(BasicNewsRecipe):
+    title                 = u'Livemint'
+    __author__            = 'Darko Miletic'
+    description           = 'The Wall Street Journal'
+    publisher             = 'The Wall Street Journal'
+    category              = 'news, games, adventure, technology'
+    language              = _('English')
+    oldest_article        = 15
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    extra_css             = ' #dvArtheadline{font-size: x-large} #dvArtAbstract{font-size: large} '
+
+    keep_only_tags = [dict(name='div', attrs={'class':'innercontent'})]
+
+    remove_tags = [dict(name=['object','link','embed','form','iframe'])]
+
+    feeds = [(u'Articles', u'http://www.livemint.com/SectionRssfeed.aspx?Mid=1')]
+
+    def print_version(self, url):
+        link = url
+        msoup = self.index_to_soup(link)
+        mlink = msoup.find(attrs={'id':'ctl00_bodyplaceholdercontent_cntlArtTool_printUrl'})
+        if mlink:
+           link = 'http://www.livemint.com/Articles/' + mlink['href'].rpartition('/Articles/')[2]
+        return link
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

src/calibre/web/feeds/recipes/recipe_outlook_india.py

@@ -1,87 +1,132 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+__copyright__ = '2009, Kovid Goyal <kovid at kovidgoyal.net>'
-'''
-outlookindia.com
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre import strftime
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class OutlookIndia(BasicNewsRecipe):
 
     title          = 'Outlook India'
-    __author__  = 'Kovid Goyal'
+    __author__     = 'Kovid Goyal and Sujata Raman'
-    description = 'Weekly news magazine focused on India.'
+    description    = 'Weekly news and current affairs in India'
+    no_stylesheets = True
+    encoding       = 'utf-8'
     language = _('English')
     recursions = 1
-    match_regexp = r'full.asp.*&pn=\d+'
+    extra_css = '''
-    
+                 body{font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-    remove_tags = [
+                .fspheading{color:#AF0E25 ; font-family:"Times New Roman",Times,serif; font-weight:bold ; font-size:large; }
-                   dict(name='img', src="images/space.gif"),
+                .fspauthor{color:#AF0E25; font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                   dict(name=lambda tag: tag.name == 'tr' and tag.find('img', src="image/tl.gif") is not None ),
+                .fspintro{color:#666666; font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
-                   dict(name=lambda tag: tag.name == 'table' and tag.find('font', attrs={'class':'fontemailfeed'}) is not None),
+                .fspchannelhome{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+                .fspphotocredit{color:##999999; font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
+                '''
+    keep_only_tags = [
+                      dict(name='div', attrs={'id':["ctl00_cphpagemiddle_reparticle_ctl00_divfullstorytext","ctl00_cphpagemiddle_reparticle_ctl00_divartpic","ctl00_cphpagemiddle_reparticle_ctl00_divfspheading", "ctl00_cphpagemiddle_reparticle_ctl00_divartpiccaption",  "ctl00_cphpagemiddle_reparticle_ctl00_divartpiccredit","ctl00_cphpagemiddle_reparticle_ctl00_divfspintro", "ctl00_cphpagemiddle_reparticle_ctl00_divartbyline", "ctl00_cphpagemiddle_divglitteratiregulars","ctl00_cphpagemiddle_divcartoon","feedbackslatestfirst","ctl00_cphpagemiddle_divregulars","ctl00_cphpagemiddle_divquotes"]}),
                            ]
+    remove_tags = [dict(name=['script','object','hr']),]
 
-    preprocess_regexps = [
+    def get_browser(self):
-                          (re.compile(r'<body.*?<!--Add Banner ends from here-->', re.DOTALL|re.IGNORECASE),
+        br = BasicNewsRecipe.get_browser(self)
-                           lambda match: '<body>'),
+        # This site sends article titles in the cookie which occasionally
-                          
+        # contain non ascii characters causing httplib to fail. Instead just
-                          (re.compile(r'>More Stories:.*', re.DOTALL), 
+        # disable cookies as they're not needed for download. Proper solution
-                           lambda match: '></body></html>'),
+        # would be to implement a unicode aware cookie jar
-                          
+        br.set_cookiejar(None)
-                          (re.compile(r'<!-- Google panel start -->.*', re.DOTALL),
+        return br
-                           lambda match: '</body></html>'), 
-                          ]
 
     def parse_index(self):
-        soup = self.index_to_soup('http://www.outlookindia.com/archivecontents.asp')
+
-        feeds = []
+
-        title = None
+        soup = self.index_to_soup('http://www.outlookindia.com/issues.aspx')
-        bogus = True
+        # find cover pic
-        for table in soup.findAll('table'):
+        div = soup.find('div', attrs={'class':re.compile('cententcellpadding')})
-            if title is None:
+
-                td = table.find('td', background="images/content_band1.jpg")
+        if div is None: return None
-                if td is not None:
+        a = div.find('a')
-                    title = self.tag_to_string(td, False)
+
-                    title = title.replace(u'\xa0', u'').strip()
+        if a is not None:
-                    if 'Cover Story' in title and bogus:
+            href =  'http://www.outlookindia.com/' + a['href']
-                        bogus = False
+
-                        title = None
+        soup = self.index_to_soup(href)
-            else:
+        cover = soup.find('img', attrs={'id':"ctl00_cphpagemiddle_dlissues_ctl00_imgcoverpic"}, src=True)
+        if cover is not None:
+
+            self.cover_url = cover['src']
+
+         # end find cover pic
+         #find current issue
+        div = soup.find('table', attrs={'id':re.compile('ctl00_cphpagemiddle_dlissues')})
+
+        if div is None: return None
+        a = div.find('a')
+
+        if a is not None:
+            href =  'http://www.outlookindia.com/' + a['href']
+
+        soup = self.index_to_soup(href)
+        #find current issue
+
+        #find the articles in the current issue
         articles = []
-                for a in table.findAll('a', href=True):
+
-                    if a.find('img') is not None:
+        for a in soup.findAll('a', attrs={'class':['contentpgsubheadinglink',"contentpgtext6",]}):
-                        continue
+
-                    atitle = self.tag_to_string(a, use_alt=False)
+            if a and a.has_key('href'):
-                    desc = a.findNextSibling('font', attrs={'class':'fontintro'})
+
-                    if desc is not None:
+                url = 'http://www.outlookindia.com/' + a['href']
-                        desc = self.tag_to_string(desc)
+            else:
-                    if not desc:
+                url =''
+
+            title = self.tag_to_string(a)
+
             desc = ''
+            date = ''
             articles.append({
-                            'title':atitle,
+                                 'title':title,
-                            'description': desc,
+                                 'date':date,
-                            'content': '',
+                                 'url':url,
-                            'url':'http://www.outlookindia.com/'+a['href'],
+                                 'description':desc,
-                            'date': '',
+                                })
+        for a in soup.findAll('a', attrs={'id':["ctl00_cphpageleft_hlglitterati","ctl00_cphpageleft_hlposcape",]}):
+
+            if a and a.has_key('href'):
+
+                url = 'http://www.outlookindia.com/' + a['href']
+            else:
+                url =''
+
+            title = self.tag_to_string(a)
+
+            desc = ''
+            date = ''
+            articles.append({
+                                 'title':title,
+                                 'date':date,
+                                 'url':url,
+                                 'description':desc,
                                 })
-                feeds.append((title, articles))
-                title = None 
 
 
-        return feeds
+        return [('Current Issue', articles)]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+
+
+
+    def postrocess_html(self, soup, first):
+
+            for item in soup.findAll(align = "left"):
+                del item['align']
+
+            for tag in soup.findAll(name=['table', 'tr','td','tbody','ul','li','font','span']):
+                tag.name = 'div'
 
-    def postprocess_html(self, soup, first_fetch):
-        bad = []
-        for table in soup.findAll('table'):
-            if table.find(text=re.compile(r'\(\d+ of \d+\)')):
-                bad.append(table)
-        for b in bad:
-            b.extract()
-        soup = soup.findAll('html')[0]
-        for t in soup.findAll(['table', 'tr', 'td']):
-            t.name = 'div' 
             return soup
 

src/calibre/web/feeds/recipes/recipe_pagina12.py

@@ -10,7 +10,7 @@ from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Pagina12(BasicNewsRecipe):
-    title                 = 'Pagina/12'
+    title                 = 'Pagina - 12'
     __author__            = 'Darko Miletic'
     description           = 'Noticias de Argentina y el resto del mundo'
     publisher             = 'La Pagina S.A.'

src/calibre/web/feeds/recipes/recipe_slate.py

@@ -16,7 +16,7 @@ class PeriodicalNameHere(BasicNewsRecipe):
     title                   = 'Slate'
     description             = 'A general-interest publication offering analysis and commentary about politics, news and culture.'
     __author__              = 'GRiker'
-    max_articles_per_feed   = 40
+    max_articles_per_feed   = 20
     oldest_article          = 7.0
     recursions              = 0
     delay                   = 0
@@ -107,9 +107,13 @@ class PeriodicalNameHere(BasicNewsRecipe):
         for older_section in older_section_dates :
             self.section_dates.append(self.tag_to_string(older_section,use_alt=False))
             
+        if soup_top_stories:
             headline_stories = soup_top_stories.find('ul')
+        else:
+            headline_stories = None
         section_lists = soup.findAll('ul')
         # Prepend the headlines to the first section
+        if headline_stories:
             section_lists[0].insert(0,headline_stories)
 
         sections = []
@@ -290,7 +294,8 @@ class PeriodicalNameHere(BasicNewsRecipe):
             excluded = re.compile('|'.join(self.excludedContentKeywords))
             found_excluded = excluded.search(str(soup))
             if found_excluded :
-                return None
+                print "no allowed content found, removing article"
+                raise StringError
 
         # Articles from www.thebigmoney.com use different tagging for byline, dateline and body
         head = soup.find('head')
@@ -423,4 +428,3 @@ class PeriodicalNameHere(BasicNewsRecipe):
                     if article.description is None :
                         article.description = extract_description(article.href)
         
-    

src/calibre/web/feeds/recipes/recipe_thedgesingapore.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.livemint.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Edgesingapore(BasicNewsRecipe):
+    title                 = 'The Edge Singapore'
+    __author__            = 'Darko Miletic'
+    description           = 'Financial news from Singapore'
+    publisher             = 'The Edge Singapore'
+    category              = 'news, finances, singapore'
+    language              = _('English')
+    lang                  = 'en'
+    oldest_article        = 15
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    extra_css             = ' .contentheading{font-size: x-large} .small{font-size: small} .createdate{font-size: small; font-weight: bold} '
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'author'           : publisher
+                        , 'language'         : lang
+                        , 'pretty_print'     : True
+                        , 'linearize_tables' : True
+                        }
+
+
+    remove_tags = [
+                    dict(name=['object','link','embed','form','iframe'])
+                   ,dict(name='div',attrs={'id':'toolbar-article'})
+                   ,dict(name='div',attrs={'class':'backtotop'})
+                   ,dict(name='img',attrs={'alt':'Print'})
+                  ]
+
+    remove_tags_after = dict(name='div',attrs={'class':'backtotop'})
+
+    feeds = [(u'Articles', u'http://feeds.feedburner.com/edgesg')]
+
+    def print_version(self, url):
+        return url + '?tmpl=component&print=1'
+
+    def preprocess_html(self, soup):
+        attribs = [  'style','font','valign'
+                    ,'colspan','width','height'
+                    ,'rowspan','summary','align'
+                    ,'cellspacing','cellpadding'
+                    ,'frames','rules','border'
+                  ]
+        for item in soup.body.findAll(name=['table','td','tr','th','caption','thead','tfoot','tbody','colgroup','col']):
+            item.name = 'div'
+            for attrib in attribs:
+                if item.has_key(attrib):
+                   del item[attrib]
+        return self.adeify_images(soup)

upload.py

@@ -4,7 +4,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import shutil, os, glob, re, cStringIO, sys, tempfile, time, textwrap, socket, \
-       struct, subprocess
+       struct, subprocess, platform
 from datetime import datetime
 from setuptools.command.build_py import build_py as _build_py, convert_path
 from distutils.core import Command
@@ -24,6 +24,7 @@ HTML2LRF = "src/calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 
+is64bit = platform.architecture()[0] == '64bit'
 
 def get_ip_address(ifname):
     import fcntl
@@ -35,7 +36,7 @@ def get_ip_address(ifname):
     )[20:24])
 
 try:
-    HOST=get_ip_address('eth0')
+    HOST=get_ip_address('br0')
 except:
     try:
         HOST=get_ip_address('wlan0')
@@ -481,12 +482,12 @@ class upload_demo(OptionlessCommand):
 def installer_name(ext):
     if ext in ('exe', 'dmg'):
         return 'dist/%s-%s.%s'%(__appname__, __version__, ext)
-    return 'dist/%s-%s-i686.%s'%(__appname__, __version__, ext)
+    ans = 'dist/%s-%s-i686.%s'%(__appname__, __version__, ext)
+    if is64bit:
+        ans = ans.replace('i686', 'x86_64')
+    return ans
 
-
+def _build_linux():
-class build_linux(OptionlessCommand):
-    description = 'Build linux installer'
-    def run(self):
         installer = installer_name('tar.bz2')
         locals = {}
         exec open('installer/linux/freeze.py') in locals
@@ -495,9 +496,15 @@ class build_linux(OptionlessCommand):
             raise Exception('Failed to build installer '+installer)
         return os.path.basename(installer)
 
+class build_linux64(OptionlessCommand):
+    description = 'Build linux 64bit installer'
+
+    def run(self):
+        return _build_linux()
+
 class VMInstaller(OptionlessCommand):
 
-    user_options = [('dont-shutdown', 'd', 'Dont shutdown Vm after build')]
+    user_options = [('dont-shutdown', 'd', 'Dont shutdown VM after build')]
     boolean_options = ['dont-shutdown']
 
     def initialize_options(self):
@@ -521,10 +528,34 @@ class VMInstaller(OptionlessCommand):
     def get_build_script(self, subs):
         return self.BUILD_SCRIPT%subs
 
-    def start_vm(self, ssh_host, build_script, sleep=75):
+    def vmware_started(self):
-        build_script = self.get_build_script(build_script)
+        return 'started' in subprocess.Popen('/etc/init.d/vmware status', shell=True, stdout=subprocess.PIPE).stdout.read()
+
+    def start_vmware(self):
+        if not self.vmware_started():
+            if os.path.exists('/dev/kvm'):
+                check_call('sudo rmmod -w kvm-intel kvm', shell=True)
+            subprocess.Popen('sudo /etc/init.d/vmware start', shell=True)
+
+    def stop_vmware(self):
+            while True:
+                try:
+                    check_call('sudo /etc/init.d/vmware stop', shell=True)
+                    break
+                except:
+                    pass
+            while 'vmblock' in open('/proc/modules').read():
+                check_call('sudo rmmod -f vmblock')
+            check_call('sudo modprobe kvm-intel', shell=True)
+
+
+    def run_vm(self):
         vmware = ('vmware', '-q', '-x', '-n', self.VM)
-        Popen(vmware)
+        self.__p = Popen(vmware)
+
+    def start_vm(self, ssh_host, build_script, sleep=75):
+        self.run_vm()
+        build_script = self.get_build_script(build_script)
         t = tempfile.NamedTemporaryFile(suffix='.sh')
         t.write(build_script)
         t.flush()
@@ -537,26 +568,55 @@ class VMInstaller(OptionlessCommand):
         check_call(('scp', t.name, ssh_host+':build-calibre'))
         check_call('ssh -t %s bash build-calibre'%ssh_host, shell=True)
 
-class build_windows(VMInstaller):
+class KVMInstaller(VMInstaller):
+
+    def run_vm(self):
+        self.stop_vmware()
+        self.__p = Popen(self.VM)
+
+
+
+class build_linux32(KVMInstaller):
+
+    description = 'Build linux 32bit installer'
+    VM = '/vmware/bin/linux_build'
+
+    def run_vm(self):
+        self.__p = Popen('/vmware/bin/linux_build')
+
+    def run(self):
+        if is64bit:
+            installer = installer_name('tar.bz2').replace('x86_64', 'i686')
+            self.start_vm('linux_build', ('python setup.py build_ext',
+                'python', 'setup.py build_linux32'))
+            check_call(('scp', 'linux_build:build/calibre/dist/*.tar.bz2', 'dist'))
+            if not os.path.exists(installer):
+                raise Exception('Failed to build installer '+installer)
+            if not self.dont_shutdown:
+                Popen(('ssh', 'linux_build', 'sudo', '/sbin/poweroff'))
+            return os.path.basename(installer)
+        else:
+            return _build_linux()
+
+
+class build_windows(KVMInstaller):
     description = 'Build windows installer'
-    VM = '/mnt/backup/calibre_windows_xp_home/calibre_windows_xp_home.vmx'
+    VM = '/vmware/bin/win_build'
-    if not os.path.exists(VM):
-        VM = '/home/kovid/calibre_windows_xp_home/calibre_windows_xp_home.vmx'
 
     def run(self):
         installer = installer_name('exe')
-        self.start_vm('windows', ('python setup.py develop',
+        self.start_vm('win_build', ('python setup.py develop',
                                   'python',
                                   r'installer\\windows\\freeze.py'))
         if os.path.exists('build/py2exe'):
             shutil.rmtree('build/py2exe')
-        check_call(('scp', '-rp', 'windows:build/%s/build/py2exe'%__appname__,
+        check_call(('scp', '-rp', 'win_build:build/%s/build/py2exe'%__appname__,
                      'build'))
         if not os.path.exists('build/py2exe'):
             raise Exception('Failed to run py2exe')
-        if not self.dont_shutdown:
-            Popen(('ssh', 'windows', 'shutdown', '-s', '-t', '0'))
         self.run_windows_install_jammer(installer)
+        if not self.dont_shutdown:
+            Popen(('ssh', 'win_build', 'shutdown', '-s', '-t', '0'))
         return os.path.basename(installer)
 
     @classmethod
@@ -573,9 +633,7 @@ class build_windows(VMInstaller):
 
 class build_osx(VMInstaller):
     description = 'Build OS X app bundle'
-    VM = '/mnt/backup/calibre_os_x/Mac OSX.vmx'
+    VM = '/vmware/calibre_os_x/Mac OSX.vmx'
-    if not os.path.exists(VM):
-        VM = '/home/kovid/calibre_os_x/Mac OSX.vmx'
 
     def get_build_script(self, subs):
         return (self.BUILD_SCRIPT%subs).replace('rm ', 'sudo rm ')
@@ -583,17 +641,18 @@ class build_osx(VMInstaller):
     def run(self):
         installer = installer_name('dmg')
         python = '/Library/Frameworks/Python.framework/Versions/Current/bin/python'
-        self.start_vm('osx', ('sudo %s setup.py develop'%python, python,
+        self.start_vmware()
+        self.start_vm('osx_build', ('sudo %s setup.py develop'%python, python,
                               'installer/osx/freeze.py'))
-        check_call(('scp', 'osx:build/calibre/dist/*.dmg', 'dist'))
+        check_call(('scp', 'osx_build:build/calibre/dist/*.dmg', 'dist'))
         if not os.path.exists(installer):
             raise Exception('Failed to build installer '+installer)
         if not self.dont_shutdown:
-            Popen(('ssh', 'osx', 'sudo', '/sbin/shutdown', '-h', 'now'))
+            Popen(('ssh', 'osx_build', 'sudo', '/sbin/shutdown', '-h', 'now'))
+            time.sleep(20)
+            self.stop_vmware()
         return os.path.basename(installer)
 
-
-
 class upload_installers(OptionlessCommand):
     description = 'Upload any installers present in dist/'
     def curl_list_dir(self, url=MOBILEREAD, listonly=1):
@@ -661,8 +720,9 @@ class upload_installers(OptionlessCommand):
 
     def run(self):
         print 'Uploading installers...'
-        for i in ('dmg', 'exe', 'tar.bz2'):
+        installers = list(map(installer_name, ('dmg', 'exe', 'tar.bz2')))
-            self.upload_installer(installer_name(i))
+        installers.append(installers[-1].replace('x86_64', 'i686'))
+        map(self.upload_installer, installers)
 
         check_call('''ssh divok echo %s \\> %s/latest_version'''\
                    %(__version__, DOWNLOADS), shell=True)
@@ -715,6 +775,10 @@ class stage3(OptionlessCommand):
         OptionlessCommand.run(self)
         self.misc()
 
+class build_linux(OptionlessCommand):
+    description = 'Build linux installers'
+    sub_commands = [ ('build_linux64', None), ('build_linux32', None) ]
+
 class stage2(OptionlessCommand):
     description = 'Stage 2 of the build process'
     sub_commands = [