From b2997acb46ec1410fc1f7858938d066131f098cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 May 2014 18:07:08 +0530 Subject: [PATCH 01/21] Update National Geographic. Fixes #1315373 [calibre National Geographic News recipe is not fetching news anymore since few week ago](https://bugs.launchpad.net/calibre/+bug/1315373) --- recipes/nationalgeographic.recipe | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/recipes/nationalgeographic.recipe b/recipes/nationalgeographic.recipe index 44aac314f9..f47f973f49 100644 --- a/recipes/nationalgeographic.recipe +++ b/recipes/nationalgeographic.recipe @@ -15,16 +15,10 @@ class NationalGeographicNews(BasicNewsRecipe): remove_javascript = True no_stylesheets = True use_embedded_content = False + auto_cleanup = True feeds = [(u'news', u'http://feeds.nationalgeographic.com/ng/News/News_Main')] - remove_tags_before = dict(id='page_head') - remove_tags_after = [dict(id='social_buttons'),{'class':'aside'}] - remove_tags = [ - {'class':'hidden'} - - ] - def parse_feeds(self): feeds = BasicNewsRecipe.parse_feeds(self) for curfeed in feeds: From 6a95c08f85c53e5a592445224d20443d8cb0e537 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 4 May 2014 23:12:53 +0530 Subject: [PATCH 02/21] pep8 --- src/calibre/customize/__init__.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index d022c295e2..e2fa03fcca 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -21,7 +21,7 @@ class InvalidPlugin(ValueError): pass -class Plugin(object): # {{{ +class Plugin(object): # {{{ ''' A calibre plugin. Useful members include: @@ -225,7 +225,6 @@ class Plugin(object): # {{{ ans[candidate] = zf.read(candidate) return ans - def customization_help(self, gui=False): ''' Return a string giving help on how to customize this plugin. @@ -284,7 +283,6 @@ class Plugin(object): # {{{ sys.path.insert(0, self.sys_insertion_path) zf.close() - def __exit__(self, *args): ip, it = getattr(self, 'sys_insertion_path', None), getattr(self, '_sys_insertion_tdir', None) @@ -304,7 +302,7 @@ class Plugin(object): # {{{ # }}} -class FileTypePlugin(Plugin): # {{{ +class FileTypePlugin(Plugin): # {{{ ''' A plugin that is associated with a particular set of file types. ''' @@ -356,13 +354,13 @@ class FileTypePlugin(Plugin): # {{{ :param book_id: Database id of the added book. :param book_format: The file type of the book that was added. - :param db: Library database. + :param db: Library database. ''' - pass # Default implementation does nothing + pass # Default implementation does nothing # }}} -class MetadataReaderPlugin(Plugin): # {{{ +class MetadataReaderPlugin(Plugin): # {{{ ''' A plugin that implements reading metadata from a set of file types. ''' @@ -392,7 +390,7 @@ class MetadataReaderPlugin(Plugin): # {{{ return None # }}} -class MetadataWriterPlugin(Plugin): # {{{ +class MetadataWriterPlugin(Plugin): # {{{ ''' A plugin that implements reading metadata from a set of file types. ''' @@ -423,7 +421,7 @@ class MetadataWriterPlugin(Plugin): # {{{ # }}} -class CatalogPlugin(Plugin): # {{{ +class CatalogPlugin(Plugin): # {{{ ''' A plugin that implements a catalog generator. ''' @@ -516,8 +514,7 @@ class CatalogPlugin(Plugin): # {{{ from calibre.customize.ui import config from calibre.ptempfile import PersistentTemporaryDirectory - if not type(self) in builtin_plugins and \ - not self.name in config['disabled_plugins']: + if not type(self) in builtin_plugins and self.name not in config['disabled_plugins']: files_to_copy = ["%s.%s" % (self.name.lower(),ext) for ext in ["ui","py"]] resources = zipfile.ZipFile(self.plugin_path,'r') @@ -553,7 +550,7 @@ class CatalogPlugin(Plugin): # {{{ # }}} -class InterfaceActionBase(Plugin): # {{{ +class InterfaceActionBase(Plugin): # {{{ supported_platforms = ['windows', 'osx', 'linux'] author = 'Kovid Goyal' @@ -580,7 +577,7 @@ class InterfaceActionBase(Plugin): # {{{ # }}} -class PreferencesPlugin(Plugin): # {{{ +class PreferencesPlugin(Plugin): # {{{ ''' A plugin representing a widget displayed in the Preferences dialog. @@ -639,7 +636,7 @@ class PreferencesPlugin(Plugin): # {{{ # }}} -class StoreBase(Plugin): # {{{ +class StoreBase(Plugin): # {{{ supported_platforms = ['windows', 'osx', 'linux'] author = 'John Schember' @@ -688,7 +685,7 @@ class StoreBase(Plugin): # {{{ # }}} -class ViewerPlugin(Plugin): # {{{ +class ViewerPlugin(Plugin): # {{{ ''' These plugins are used to add functionality to the calibre viewer. @@ -733,6 +730,7 @@ class ViewerPlugin(Plugin): # {{{ example, you can modify the toolbars via ui.tool_bar and ui.tool_bar2. ''' pass + def customize_context_menu(self, menu, event, hit_test_result): ''' This method is called every time the context (right-click) menu is @@ -743,4 +741,3 @@ class ViewerPlugin(Plugin): # {{{ pass # }}} - From c33dd1244934b518f35f20f2824904358480f8a7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 May 2014 20:31:25 +0530 Subject: [PATCH 03/21] Edit Book: Fix error when checking spelling if one of the files in the book declares an invalid (empty) language code. Fixes #1316170 [Spellcheck doesn't like ...xml:lang=""](https://bugs.launchpad.net/calibre/+bug/1316170) --- src/calibre/ebooks/oeb/polish/spell.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/spell.py b/src/calibre/ebooks/oeb/polish/spell.py index f599f55579..7fc7542ba7 100644 --- a/src/calibre/ebooks/oeb/polish/spell.py +++ b/src/calibre/ebooks/oeb/polish/spell.py @@ -128,11 +128,17 @@ def read_words_from_html_tag(tag, words, file_name, parent_locale, locale): def locale_from_tag(tag): if 'lang' in tag.attrib: - loc = parse_lang_code(tag.get('lang')) + try: + loc = parse_lang_code(tag.get('lang')) + except ValueError: + loc = None if loc is not None: return loc if '{http://www.w3.org/XML/1998/namespace}lang' in tag.attrib: - loc = parse_lang_code(tag.get('{http://www.w3.org/XML/1998/namespace}lang')) + try: + loc = parse_lang_code(tag.get('{http://www.w3.org/XML/1998/namespace}lang')) + except ValueError: + loc = None if loc is not None: return loc From fb11aa99ac3dd1db598366ba493136e99a8d236f Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Mon, 5 May 2014 18:13:05 +0200 Subject: [PATCH 04/21] New formatter function first_matching_cmp --- manual/template_lang.rst | 5 +++++ src/calibre/utils/formatter_functions.py | 27 +++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/manual/template_lang.rst b/manual/template_lang.rst index 1ff477a5f5..0c2d445c5d 100644 --- a/manual/template_lang.rst +++ b/manual/template_lang.rst @@ -250,6 +250,11 @@ The following functions are available in addition to those described in single-f * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers. * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables. Because the `{` and `}` characters are special, you must use `[[` for the `{` character and `]]` for the '}' character; they are converted automatically. Note also that prefixes and suffixes (the `|prefix|suffix` syntax) cannot be used in the argument to this function when using template program mode. * ``field(name)`` -- returns the metadata field named by ``name``. + * ``first_matching_cmp(val, cmp1, result1, cmp2, r2, ..., else_result)`` -- compares "val < cmpN" in sequence, returning resultN for the first comparison that succeeds. Returns else_result if no comparison succeeds. Example:: + + ``first_matching_cmp(10,5,"small",10,"middle",15,"large","giant")`` + + returns "middle". The same example with a first value of 16 returns "giant". * ``first_non_empty(value, value, ...)`` -- returns the first value that is not empty. If all values are empty, then the empty value is returned. You can have as many values as you want. * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are:: diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 966e51c2ea..3ddecc4169 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -178,6 +178,27 @@ class BuiltinCmp(BuiltinFormatterFunction): return eq return gt +class BuiltinFirstMatchingCmp(BuiltinFormatterFunction): + name = 'first_matching_cmp' + category = 'Relational' + arg_count = -1 + __doc__ = doc = _('first_matching_cmp(val, cmp1, result1, cmp2, r2, ..., else_result) -- ' + 'compares "val < cmpN" in sequence, returning resultN for ' + 'the first comparison that succeeds. Returns else_result ' + 'if no comparison succeeds. Example: ' + 'first_matching_cmp(10,5,"small",10,"middle",15,"large","giant") ' + 'returns "middle". The same example with a first value of 16 returns "giant".') + + def evaluate(self, formatter, kwargs, mi, locals, *args): + if (len(args) % 2) != 0: + raise ValueError(_('first_matching_cmp requires an even number of arguments')) + val = float(args[0] if args[0] and args[0] != 'None' else 0) + for i in range(1, len(args) - 1, 2): + c = float(args[i] if args[i] and args[i] != 'None' else 0) + if val < c: + return args[i+1] + return args[len(args)-1] + class BuiltinStrcat(BuiltinFormatterFunction): name = 'strcat' arg_count = -1 @@ -1310,9 +1331,9 @@ _formatter_builtins = [ BuiltinCapitalize(), BuiltinCmp(), BuiltinContains(), BuiltinCount(), BuiltinCurrentLibraryName(), BuiltinCurrentLibraryPath(), BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(), - BuiltinField(), BuiltinFinishFormatting(), BuiltinFormatDate(), - BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsPaths(), - BuiltinFormatsSizes(), + BuiltinField(), BuiltinFinishFormatting(), BuiltinFirstMatchingCmp(), + BuiltinFormatDate(), BuiltinFormatNumber(), BuiltinFormatsModtimes(), + BuiltinFormatsPaths(), BuiltinFormatsSizes(), BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(), BuiltinIfempty(), BuiltinLanguageCodes(), BuiltinLanguageStrings(), BuiltinInList(), BuiltinListDifference(), BuiltinListEquals(), From d574b8fc7af95c6882e29c340f2ead1f840789c4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 May 2014 21:49:27 +0530 Subject: [PATCH 05/21] ... --- setup/installer/windows/notes.rst | 68 +++++++++++++++++-------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index dacc897266..1bfaa4e616 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -41,7 +41,7 @@ Cygwin This is needed for automation of the build process, and the ease of use of the unix shell (bash). -Install, vim, rsync, openssh, unzip, wget, make at a minimum. +Install, vim, rsync, openssh, unzip, wget, make, zsh, bash-completion, curl at a minimum. After installing python run:: python setup/vcvars.py && echo 'source ~/.vcvars' >> ~/.bash_profile @@ -147,27 +147,34 @@ Put sqlite3*.h from the sqlite windows amalgamation in ~/sw/include APSW ----- -Download source from http://code.google.com/p/apsw/downloads/list and run in visual studio prompt +Download source from http://code.google.com/p/apsw/downloads/list and run python setup.py fetch --all --missing-checksum-ok build --enable-all-extensions install test +Build requirements +------------------- + +Install perl and ruby (needed to build openssl and qt): +Perl: http://www.activestate.com/activeperl +Ruby: http://rubyinstaller.org/ + +Put both perl.exe and ruby.exe in the PATH + OpenSSL -------- -First install ActiveState Perl if you dont already have perl in windows +Get nasm.exe from +http://www.nasm.us/pub/nasm/releasebuilds/2.11/win32/nasm-2.11-win32.zip +and put it somewhere on your PATH (I chose ~/sw/bin) -Then, get nasm.exe from -http://www.nasm.us/pub/nasm/releasebuilds/2.05/nasm-2.05-win32.zip and put it -somewhere on your PATH (I chose ~/sw/bin) - -Download and untar the openssl tarball, follow the instructions in INSTALL.(W32|W64) -to install use prefix q:\openssl +Download and untar the openssl tarball. +To install use a private prefix such as --prefix=C:/cygwin/home/kovid/sw/private/openssl The following *MUST BE RUN* in a Visual Studio Command prompt and not in a cygwin environment. For 32-bit:: - perl Configure VC-WIN32 no-asm enable-static-engine --prefix=Q:/openssl + perl Configure VC-WIN32 no-asm enable-static-engine --prefix=C:/cygwin/home/kovid/sw/private/openssl ms\do_ms.bat nmake -f ms\ntdll.mak nmake -f ms\ntdll.mak test @@ -180,6 +187,27 @@ For 64-bit:: nmake -f ms\ntdll.mak test nmake -f ms\ntdll.mak install +ICU +------- + +Download the win32 source .zip from http://www.icu-project.org/download + +Extract to q:\icu + +Add Q:\icu\bin to PATH and reboot + +In a Visual Studio Command Prompt +cd to \source +Run set PATH=%PATH%;c:\cygwin\bin +Run dos2unix on configure and runConfigureICU + +Run bash ./runConfigureICU Cygwin/MSVC + +Run make (note that you must have GNU make installed in cygwin) + +Optionally run make check + + Qt -------- Download Qt sourcecode (.zip) from: http://download.qt-project.org/official_releases/qt/ @@ -225,26 +253,6 @@ Compiling instructions:: nmake nmake install -ICU -------- - -Download the win32 source .zip from http://www.icu-project.org/download - -Extract to q:\icu - -Add Q:\icu\bin to PATH and reboot - -In a Visual Studio Command Prompt -cd to \source -Run set PATH=%PATH%;c:\cygwin\bin -Run dos2unix on configure and runConfigureICU - -Run bash ./runConfigureICU Cygwin/MSVC - -Run make (note that you must have GNU make installed in cygwin) - -Optionally run make check - zlib ------ From 6f63fc80a2b48e7a6ca65d7d5438ff3c5a76a125 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 May 2014 22:01:04 +0530 Subject: [PATCH 06/21] ... --- setup/installer/windows/notes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 1bfaa4e616..88c17fbdd3 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -168,20 +168,20 @@ http://www.nasm.us/pub/nasm/releasebuilds/2.11/win32/nasm-2.11-win32.zip and put it somewhere on your PATH (I chose ~/sw/bin) Download and untar the openssl tarball. -To install use a private prefix such as --prefix=C:/cygwin/home/kovid/sw/private/openssl +To install use a private prefix such as --prefix=C:/cygwin64/home/kovid/sw/private/openssl The following *MUST BE RUN* in a Visual Studio Command prompt and not in a cygwin environment. For 32-bit:: - perl Configure VC-WIN32 no-asm enable-static-engine --prefix=C:/cygwin/home/kovid/sw/private/openssl + perl Configure VC-WIN32 no-asm enable-static-engine --prefix=C:/cygwin64/home/kovid/sw/private/openssl ms\do_ms.bat nmake -f ms\ntdll.mak nmake -f ms\ntdll.mak test nmake -f ms\ntdll.mak install For 64-bit:: - perl Configure VC-WIN64A no-asm enable-static-engine --prefix=C:/cygwin/home/kovid/sw/private/openssl + perl Configure VC-WIN64A no-asm enable-static-engine --prefix=C:/cygwin64/home/kovid/sw/private/openssl ms\do_win64a.bat nmake -f ms\ntdll.mak nmake -f ms\ntdll.mak test @@ -198,7 +198,7 @@ Add Q:\icu\bin to PATH and reboot In a Visual Studio Command Prompt cd to \source -Run set PATH=%PATH%;c:\cygwin\bin +Run set PATH=%PATH%;c:\cygwin64\bin Run dos2unix on configure and runConfigureICU Run bash ./runConfigureICU Cygwin/MSVC From 3509c7fb6004162a0586e020708374cae49b1708 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 May 2014 22:06:26 +0530 Subject: [PATCH 07/21] SKip contractions ICU test as it is too version dependant --- src/calibre/utils/icu_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/utils/icu_test.py b/src/calibre/utils/icu_test.py index 99441520e9..9069d0acce 100644 --- a/src/calibre/utils/icu_test.py +++ b/src/calibre/utils/icu_test.py @@ -140,6 +140,7 @@ class TestICU(unittest.TestCase): def test_contractions(self): ' Test contractions ' + self.skipTest('Skipping as this depends too much on ICU version') c = icu._icu.Collator('cs') self.ae(icu.contractions(c), frozenset({u'Z\u030c', u'z\u030c', u'Ch', u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH', From 5eaf17fc21b901198b5785d34de4a12f3f8e2005 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 May 2014 22:09:37 +0530 Subject: [PATCH 08/21] ... --- setup/installer/windows/notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 88c17fbdd3..66265e89b2 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -41,7 +41,7 @@ Cygwin This is needed for automation of the build process, and the ease of use of the unix shell (bash). -Install, vim, rsync, openssh, unzip, wget, make, zsh, bash-completion, curl at a minimum. +Install vim, dos2unix, rsync, openssh, unzip, wget, make, zsh, bash-completion, curl at a minimum. After installing python run:: python setup/vcvars.py && echo 'source ~/.vcvars' >> ~/.bash_profile From e3559e9bce86aa283374ac67a9c1fc2be04429c6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 10:25:20 +0530 Subject: [PATCH 09/21] ... --- setup/installer/windows/notes.rst | 156 ++++++++++++++---------------- 1 file changed, 72 insertions(+), 84 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 66265e89b2..6b1ab3222c 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -44,7 +44,7 @@ unix shell (bash). Install vim, dos2unix, rsync, openssh, unzip, wget, make, zsh, bash-completion, curl at a minimum. After installing python run:: - python setup/vcvars.py && echo 'source ~/.vcvars' >> ~/.bash_profile + python setup/vcvars.py && echo 'source ~/.vcvars' >> ~/.zshrc To allow you to use the visual studio tools in the cygwin shell. @@ -160,15 +160,15 @@ Ruby: http://rubyinstaller.org/ Put both perl.exe and ruby.exe in the PATH +Get nasm.exe from (needed for openssl and libjpeg-turbo) +http://www.nasm.us/pub/nasm/releasebuilds/2.11/win32/nasm-2.11-win32.zip +and put it in ~/sw/bin (which must be in PATH) + OpenSSL -------- -Get nasm.exe from -http://www.nasm.us/pub/nasm/releasebuilds/2.11/win32/nasm-2.11-win32.zip -and put it somewhere on your PATH (I chose ~/sw/bin) - Download and untar the openssl tarball. -To install use a private prefix such as --prefix=C:/cygwin64/home/kovid/sw/private/openssl +To install use a private prefix: --prefix=C:/cygwin64/home/kovid/sw/private/openssl The following *MUST BE RUN* in a Visual Studio Command prompt and not in a cygwin environment. @@ -190,79 +190,28 @@ For 64-bit:: ICU ------- -Download the win32 source .zip from http://www.icu-project.org/download +Download the win32 *source* .zip from http://www.icu-project.org/download -Extract to q:\icu +Extract to C:\cygwin64\home\kovid\sw\private\icu -Add Q:\icu\bin to PATH and reboot +The following must be run in the VS Command Prompt, not the cygwin ssh shell -In a Visual Studio Command Prompt -cd to \source -Run set PATH=%PATH%;c:\cygwin64\bin -Run dos2unix on configure and runConfigureICU +cd to \source:: -Run bash ./runConfigureICU Cygwin/MSVC - -Run make (note that you must have GNU make installed in cygwin) - -Optionally run make check - - -Qt --------- -Download Qt sourcecode (.zip) from: http://download.qt-project.org/official_releases/qt/ - -Qt uses its own routine to locate and load "system libraries" including the -openssl libraries needed for "Get Books". This means that we have to apply the -following patch to have Qt load the openssl libraries bundled with calibre: - - ---- src/corelib/plugin/qsystemlibrary.cpp 2011-02-22 05:04:00.000000000 -0700 -+++ src/corelib/plugin/qsystemlibrary.cpp 2011-04-25 20:53:13.635247466 -0600 -@@ -110,7 +110,7 @@ HINSTANCE QSystemLibrary::load(const wch - - #if !defined(QT_BOOTSTRAPPED) - if (!onlySystemDirectory) -- searchOrder << QFileInfo(qAppFileName()).path(); -+ searchOrder << (QFileInfo(qAppFileName()).path().replace(QLatin1Char('/'), QLatin1Char('\\')) + QString::fromLatin1("\\DLLs\\")); - #endif - searchOrder << qSystemDirectory(); - - -Now, run configure and make: - --no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly - - ./configure.exe -ltcg -opensource -release -qt-zlib -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -nomake tools -no-plugin-manifests -openssl -I $OPENSSL_DIR/include -L $OPENSSL_DIR/lib && nmake - -Add the path to the bin folder inside the Qt dir to your system PATH. - -SIP ------ - -Available from: http://www.riverbankcomputing.co.uk/software/sip/download :: - - python configure.py -p win32-msvc2008 && nmake && nmake install - -PyQt4 ----------- - -Compiling instructions:: - - python configure.py -c -j5 -e QtCore -e QtGui -e QtSvg -e QtNetwork -e QtWebKit -e QtXmlPatterns --verbose --confirm-license - nmake - nmake install + set PATH=%PATH%;C:\cygwin64\bin + dos2unix runConfigureICU + bash ./runConfigureICU Cygwin/MSVC + make zlib ------ +http://www.zlib.net/ + Build with:: nmake -f win32/Makefile.msc nmake -f win32/Makefile.msc test - - cp zlib1.dll* ../../bin - cp zlib.lib zdll.* ../../lib - cp zconf.h zlib.h ../../include + cp zlib1.dll* ~/sw/bin && cp zlib.lib zdll.* ~/sw/lib/ && cp zconf.h zlib.h ~/sw/include/ jpeg-8 ------- @@ -270,12 +219,10 @@ jpeg-8 Get the source code from: http://sourceforge.net/projects/libjpeg-turbo/files/ Run:: - chmod +x cmakescripts/* && cd build + chmod +x cmakescripts/* && mkdir -p build && cd build cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DWITH_JPEG8=1 .. nmake - cp sharedlib/jpeg8.dll* ~/sw/bin/ - cp sharedlib/jpeg.lib ~/sw/lib/ - cp jconfig.h ../jerror.h ../jpeglib.h ../jmorecfg.h ~/sw/include + cp sharedlib/jpeg8.dll* ~/sw/bin/ && cp sharedlib/jpeg.lib ~/sw/lib/ && cp jconfig.h ../jerror.h ../jpeglib.h ../jmorecfg.h ~/sw/include libpng --------- @@ -284,12 +231,10 @@ Download the libpng .zip source file from: http://www.libpng.org/pub/png/libpng.html Run:: - mkdir build && cd build - cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DZLIB_INCLUDE_DIR=C:/cygwin/home/kovid/sw/include -DZLIB_LIBRARY=C:/cygwin/home/kovid/sw/lib/zdll.lib .. + mkdir -p build && cd build + cmake -G "NMake Makefiles" -DPNG_SHARED=1 -DCMAKE_BUILD_TYPE=Release -DZLIB_INCLUDE_DIR=C:/cygwin64/home/kovid/sw/include -DZLIB_LIBRARY=C:/cygwin64/home/kovid/sw/lib/zdll.lib .. nmake - cp libpng*.dll ~/sw/bin/ - cp libpng*.lib ~/sw/lib/ - cp pnglibconf.h ../png.h ../pngconf.h ~/sw/include/ + cp libpng*.dll ~/sw/bin/ && cp libpng*.lib ~/sw/lib/ && cp pnglibconf.h ../png.h ../pngconf.h ~/sw/include/ freetype ----------- @@ -302,25 +247,22 @@ and a correct dll #define FT_EXPORT(return_type) __declspec(dllexport) return_type #define FT_EXPORT_DEF(return_type) __declspec(dllexport) return_type - VS 2008 .sln file is present, open it * If you are doing x64 build, click the Win32 dropdown, select Configuration manager->Active solution platform -> New -> x64 - * Change active build type to release mutithreaded + * Change active build type to release multithreaded * Project->Properties->Configuration Properties change configuration type to dll and build solution -cp "`find . -name *.dll`" ~/sw/bin/ -cp "`find . -name freetype.lib`" ~/sw/lib/ +cp "`find . -name freetype.dll`" ~/sw/bin/ && cp "`find . -name freetype.lib`" ~/sw/lib/ Now change configuration back to static for .lib and build solution -cp "`find . -name freetype*MT.lib`" ~/sw/lib/ -cp build/freetype-2.3.9/objs/win32/vc2008/freetype239MT.lib lib/ -cp -rf include/* ~/sw/include/ +cp "`find . -name 'freetype*MT.lib'`" ~/sw/lib/ +cp -rf include ~/sw/include/freetype2 && rm -rf ~/sw/include/freetype2/internal TODO: Test if this bloody thing actually works on 64 bit (apparently freetype assumes sizeof(long) == sizeof(ptr) which is not true in Win64. See for @@ -556,6 +498,52 @@ Then open ChmLib.dsw in Visual Studio, change the configuration to Release (Win32|x64) and build solution, this will generate a static library in Release/ChmLib.lib +Qt +-------- +Download Qt sourcecode (.zip) from: http://download.qt-project.org/official_releases/qt/ + +Qt uses its own routine to locate and load "system libraries" including the +openssl libraries needed for "Get Books". This means that we have to apply the +following patch to have Qt load the openssl libraries bundled with calibre: + + +--- src/corelib/plugin/qsystemlibrary.cpp 2011-02-22 05:04:00.000000000 -0700 ++++ src/corelib/plugin/qsystemlibrary.cpp 2011-04-25 20:53:13.635247466 -0600 +@@ -110,7 +110,7 @@ HINSTANCE QSystemLibrary::load(const wch + + #if !defined(QT_BOOTSTRAPPED) + if (!onlySystemDirectory) +- searchOrder << QFileInfo(qAppFileName()).path(); ++ searchOrder << (QFileInfo(qAppFileName()).path().replace(QLatin1Char('/'), QLatin1Char('\\')) + QString::fromLatin1("\\DLLs\\")); + #endif + searchOrder << qSystemDirectory(); + + +Now, run configure and make: + +-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly + + ./configure.exe -ltcg -opensource -release -qt-zlib -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -nomake tools -no-plugin-manifests -openssl -I $OPENSSL_DIR/include -L $OPENSSL_DIR/lib && nmake + +Add the path to the bin folder inside the Qt dir to your system PATH. + +SIP +----- + +Available from: http://www.riverbankcomputing.co.uk/software/sip/download :: + + python configure.py -p win32-msvc2008 && nmake && nmake install + +PyQt4 +---------- + +Compiling instructions:: + + python configure.py -c -j5 -e QtCore -e QtGui -e QtSvg -e QtNetwork -e QtWebKit -e QtXmlPatterns --verbose --confirm-license + nmake + nmake install + + libimobiledevice ------------------ From 521a8f93c1746725b36ca4f3ee6a4183e0b90742 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Tue, 6 May 2014 16:39:58 +0900 Subject: [PATCH 10/21] update japanese recipes - update: endgaget jp Kahoku Shinpo MSN Sankei News Uni no Himitsu - remove problematic recipes: nikkei_sub* chou chou blog Signed-off-by: Hiroshi Miura --- recipes/chouchoublog.recipe | 37 ---------- recipes/endgadget_ja.recipe | 13 ++-- recipes/kahokushinpo.recipe | 10 +-- recipes/msnsankei.recipe | 5 +- recipes/nikkei_sub_economy.recipe | 110 ----------------------------- recipes/nikkei_sub_industry.recipe | 107 ---------------------------- recipes/nikkei_sub_life.recipe | 104 --------------------------- recipes/nikkei_sub_main.recipe | 103 --------------------------- recipes/nikkei_sub_shakai.recipe | 102 -------------------------- recipes/nikkei_sub_sports.recipe | 108 ---------------------------- recipes/uninohimitu.recipe | 4 +- 11 files changed, 13 insertions(+), 690 deletions(-) delete mode 100644 recipes/chouchoublog.recipe delete mode 100644 recipes/nikkei_sub_economy.recipe delete mode 100644 recipes/nikkei_sub_industry.recipe delete mode 100644 recipes/nikkei_sub_life.recipe delete mode 100644 recipes/nikkei_sub_main.recipe delete mode 100644 recipes/nikkei_sub_shakai.recipe delete mode 100644 recipes/nikkei_sub_sports.recipe diff --git a/recipes/chouchoublog.recipe b/recipes/chouchoublog.recipe deleted file mode 100644 index 8c953deef0..0000000000 --- a/recipes/chouchoublog.recipe +++ /dev/null @@ -1,37 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -http://ameblo.jp/ -''' - -import re -from calibre.web.feeds.news import BasicNewsRecipe - -class SakuraBlog(BasicNewsRecipe): - title = u'chou chou blog' - __author__ = 'Hiroshi Miura' - oldest_article = 4 - publication_type = 'blog' - max_articles_per_feed = 20 - description = 'Japanese popular dog blog' - publisher = '' - category = 'dog, pet, japan' - language = 'ja' - encoding = 'utf-8' - use_embedded_content = True - - feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/chouchou1218/rss20.xml')] - - def parse_feeds(self): - feeds = BasicNewsRecipe.parse_feeds(self) - for curfeed in feeds: - delList = [] - for a,curarticle in enumerate(curfeed.articles): - if re.search(r'rssad.jp', curarticle.url): - delList.append(curarticle) - if len(delList)>0: - for d in delList: - index = curfeed.articles.index(d) - curfeed.articles[index:index+1] = [] - return feeds - diff --git a/recipes/endgadget_ja.recipe b/recipes/endgadget_ja.recipe index 7eca0a6966..5c030caef1 100644 --- a/recipes/endgadget_ja.recipe +++ b/recipes/endgadget_ja.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' japan.engadget.com ''' @@ -20,19 +20,20 @@ class EndgadgetJapan(BasicNewsRecipe): index = 'http://japanese.engadget.com/' remove_javascript = True - remove_tags_before = dict(name="h1", attrs={'class':"post_title"}) - remove_tags_after = dict(name='div', attrs={'class':'post_body'}) + remove_tags_before = dict(name="header", attrs={'class':"header"}) + remove_tags_after = dict(name='div', attrs={'class':'post-meta'}) def parse_index(self): feeds = [] newsarticles = [] soup = self.index_to_soup(self.index) - for topstories in soup.findAll('div',attrs={'class':'post_content'}): - itt = topstories.find('h4') + for topstories in soup.findAll('header',attrs={'class':'post-header'}): + itt = topstories.find('h2') itema = itt.find('a',href=True) + itemtime = topstories.find('span',attrs={'class':'time'}) newsarticles.append({ 'title' :itema.string - ,'date' :'' + ,'date' :itemtime.string ,'url' :itema['href'] ,'description':'' }) diff --git a/recipes/kahokushinpo.recipe b/recipes/kahokushinpo.recipe index 06879a1375..88ac978c6a 100644 --- a/recipes/kahokushinpo.recipe +++ b/recipes/kahokushinpo.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' www.kahoku.co.jp ''' @@ -21,11 +21,5 @@ class KahokuShinpoNews(BasicNewsRecipe): feeds = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')] - keep_only_tags = [ dict(id="page_title"), - dict(id="news_detail"), - dict(id="bt_title"), - {'class':"photoLeft"}, - dict(id="bt_body") - ] - remove_tags = [ {'class':"button"}] + keep_only_tags = [ {'class':"category"},{'class':"ttl"},{'class':'photoimg'},{'class':"txt"},{'class':"data"}] diff --git a/recipes/msnsankei.recipe b/recipes/msnsankei.recipe index 59664d055f..4bd61d295c 100644 --- a/recipes/msnsankei.recipe +++ b/recipes/msnsankei.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' sankei.jp.msn.com ''' @@ -20,5 +20,4 @@ class MSNSankeiNewsProduct(BasicNewsRecipe): feeds = [(u'\u65b0\u5546\u54c1', u'http://sankei.jp.msn.com/rss/news/release.xml')] - remove_tags_before = dict(id="NewsTitle") - remove_tags_after = dict(id="RelatedTitle") + keep_only_tags = [dict(id=['MainContent'])] diff --git a/recipes/nikkei_sub_economy.recipe b/recipes/nikkei_sub_economy.recipe deleted file mode 100644 index 7a256f7553..0000000000 --- a/recipes/nikkei_sub_economy.recipe +++ /dev/null @@ -1,110 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - -class NikkeiNet_sub_economy(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u7d4c\u6e08)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - {'class':"cmn-article_list"}, - dict(id="ABOUT-NIKKEI"), - {'class':"cmn-sub_market"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ (u'\u653f\u6cbb', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=seiji'), - (u'\u8ca1\u52d9', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=zaimu'), - (u'\u7d4c\u6e08', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=keizai'), - (u'\u30de\u30fc\u30b1\u30c3\u30c8', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=market'), - (u'\u96c7\u7528', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=koyou'), - (u'\u6559\u80b2', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kyouiku'), - (u'\u304a\u304f\u3084\u307f', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=okuyami'), - (u'\u4eba\u4e8b', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=zinzi'), - ] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/nikkei_sub_industry.recipe b/recipes/nikkei_sub_industry.recipe deleted file mode 100644 index 11a17b2415..0000000000 --- a/recipes/nikkei_sub_industry.recipe +++ /dev/null @@ -1,107 +0,0 @@ - -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - - -class NikkeiNet_sub_industory(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u7523\u696d)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ (u'\u65e5\u7d4c\u4f01\u696d', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=sangyo'), - (u'\u65e5\u7d4c\u88fd\u54c1', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=newpro'), - (u'internet', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=internet'), - (u'\u56fd\u969b', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kaigai'), - (u'\u79d1\u5b66', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kagaku'), - - ] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/nikkei_sub_life.recipe b/recipes/nikkei_sub_life.recipe deleted file mode 100644 index c2b908ca98..0000000000 --- a/recipes/nikkei_sub_life.recipe +++ /dev/null @@ -1,104 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - - -class NikkeiNet_sub_life(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ (u'\u304f\u3089\u3057', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kurashi'), - (u'\u30a8\u30b3', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=eco'), - (u'\u5065\u5eb7', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=kenkou'), - (u'\u7279\u96c6', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=special') - ] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/nikkei_sub_main.recipe b/recipes/nikkei_sub_main.recipe deleted file mode 100644 index 84503cccf3..0000000000 --- a/recipes/nikkei_sub_main.recipe +++ /dev/null @@ -1,103 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - - -class NikkeiNet_sub_main(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u7dcf\u5408)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - {'class':"cmn-article_list"}, - {'class':"cmn-dashedline"}, - {'class':"cmn-hide"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ (u'NIKKEI', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=main')] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/nikkei_sub_shakai.recipe b/recipes/nikkei_sub_shakai.recipe deleted file mode 100644 index be21b3c43f..0000000000 --- a/recipes/nikkei_sub_shakai.recipe +++ /dev/null @@ -1,102 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - - -class NikkeiNet_sub_shakai(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(Social)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ - (u'\u793e\u4f1a', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=shakai') - ] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/nikkei_sub_sports.recipe b/recipes/nikkei_sub_sports.recipe deleted file mode 100644 index 47e335a4c3..0000000000 --- a/recipes/nikkei_sub_sports.recipe +++ /dev/null @@ -1,108 +0,0 @@ - -__license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' -''' -www.nikkei.com -''' - -import re -from calibre.web.feeds.recipes import BasicNewsRecipe -import mechanize -from calibre.ptempfile import PersistentTemporaryFile - - -class NikkeiNet_sub_sports(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u30b9\u30dd\u30fc\u30c4)' - __author__ = 'Hiroshi Miura' - description = 'News and current market affairs from Japan' - cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - masthead_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' - needs_subscription = True - oldest_article = 2 - max_articles_per_feed = 20 - language = 'ja' - remove_javascript = False - temp_files = [] - - remove_tags_before = {'class':"cmn-section cmn-indent"} - remove_tags = [ - {'class':"JSID_basePageMove JSID_baseAsyncSubmit cmn-form_area JSID_optForm_utoken"}, - {'class':"cmn-article_keyword cmn-clearfix"}, - {'class':"cmn-print_headline cmn-clearfix"}, - ] - remove_tags_after = {'class':"cmn-pr_list"} - - feeds = [ - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u30d7\u30ed\u91ce\u7403', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=baseball'), - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u5927\u30ea\u30fc\u30b0', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=mlb'), - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u30b5\u30c3\u30ab\u30fc', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=soccer'), - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u30b4\u30eb\u30d5', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=golf'), - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u76f8\u64b2', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=sumou'), - (u'\u30b9\u30dd\u30fc\u30c4\uff1a\u7af6\u99ac', u'http://www.zou3.net/php/rss/nikkei2rss.php?head=keiba') - ] - - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - - cj = mechanize.LWPCookieJar() - br.set_cookiejar(cj) - - #br.set_debug_http(True) - #br.set_debug_redirects(True) - #br.set_debug_responses(True) - - if self.username is not None and self.password is not None: - #print "----------------------------get login form--------------------------------------------" - # open login form - br.open('https://id.nikkei.com/lounge/nl/base/LA0010.seam') - response = br.response() - #print "----------------------------get login form---------------------------------------------" - #print "----------------------------set login form---------------------------------------------" - # remove disabled input which brings error on mechanize - response.set_data(response.get_data().replace("", " -->")) - br.set_response(response) - br.select_form(name='LA0010Form01') - br['LA0010Form01:LA0010Email'] = self.username - br['LA0010Form01:LA0010Password'] = self.password - br.form.find_control(id='LA0010Form01:LA0010AutoLoginOn',type="checkbox").get(nr=0).selected = True - br.submit() - br.response() - #print "----------------------------send login form---------------------------------------------" - #print "----------------------------open news main page-----------------------------------------" - # open news site - br.open('http://www.nikkei.com/') - br.response() - #print "----------------------------www.nikkei.com BODY --------------------------------------" - #print response2.get_data() - #print "-------------------------^^-got auto redirect form----^^--------------------------------" - # forced redirect in default - br.select_form(nr=0) - br.submit() - response3 = br.response() - # return some cookie which should be set by Javascript - #print response3.geturl() - raw = response3.get_data() - #print "---------------------------response to form --------------------------------------------" - # grab cookie from JS and set it - redirectflag = re.search(r"var checkValue = '(\d+)';", raw, re.M).group(1) - br.select_form(nr=0) - - self.temp_files.append(PersistentTemporaryFile('_fa.html')) - self.temp_files[-1].write("#LWP-Cookies-2.0\n") - - self.temp_files[-1].write("Set-Cookie3: Cookie-dummy=Cookie-value; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].write("Set-Cookie3: redirectFlag="+redirectflag+"; domain=\".nikkei.com\"; path=\"/\"; path_spec; secure; expires=\"2029-12-21 05:07:59Z\"; version=0\n") - self.temp_files[-1].close() - cj.load(self.temp_files[-1].name) - - br.submit() - - #br.set_debug_http(False) - #br.set_debug_redirects(False) - #br.set_debug_responses(False) - return br - - - - diff --git a/recipes/uninohimitu.recipe b/recipes/uninohimitu.recipe index aac412744c..7740307142 100644 --- a/recipes/uninohimitu.recipe +++ b/recipes/uninohimitu.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' http://ameblo.jp/sauta19/ ''' @@ -18,7 +18,7 @@ class UniNoHimituKichiBlog(BasicNewsRecipe): category = 'cat, pet, japan' language = 'ja' encoding = 'utf-8' - + keep_only_tags = [{'class':'entry_head'},{'class':'subContentsInner'}] feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/sauta19/rss20.xml')] def parse_feeds(self): From 76ba5f06e2b902d023ca7c67e47d7ca861beb167 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 16:50:09 +0530 Subject: [PATCH 11/21] ... --- setup/installer/windows/notes.rst | 64 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 6b1ab3222c..9b1f049e89 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -277,9 +277,8 @@ Apparently expat requires stdint.h which VS 2008 does not have. So we get our own. Run:: - cd lib - wget http://msinttypes.googlecode.com/svn/trunk/stdint.h - mkdir build && cd build + cd lib && wget http://msinttypes.googlecode.com/svn/trunk/stdint.h && cd .. + mkdir -p build && cd build cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release .. nmake cp expat.dll ~/sw/bin/ && cp expat.lib ~/sw/lib/ @@ -294,11 +293,12 @@ Run:: Then follow these instructions: http://www.codeproject.com/Articles/302012/How-to-Build-libiconv-with-Microsoft-Visual-Studio +NOTE: Built as MT rather than MD so no manifest + Change the type to Release and config to x64 or Win32 and Build solution and then:: - cp "`find . -name *.dll`" ~/sw/bin/ - cp "`find . -name *.dll.manifest`" ~/sw/bin/ - cp "`find . -name *.lib`" ~/sw/lib/iconv.lib + cp "`find . -name '*.dll'`" ~/sw/bin/ + cp "`find . -name '*.lib'`" ~/sw/lib/iconv.lib cp "`find . -name iconv.h`" ~/sw/include/ Information for using a static version of libiconv is at the link above. @@ -310,10 +310,10 @@ Get it from: ftp://xmlsoft.org/libxml2/ Run:: cd win32 - cscript.exe configure.js include=C:/cygwin/home/kovid/sw/include lib=C:/cygwin/home/kovid/sw/lib prefix=C:/cygwin/home/kovid/sw zlib=yes iconv=yes + cscript.exe configure.js include=C:/cygwin64/home/kovid/sw/include lib=C:/cygwin64/home/kovid/sw/lib prefix=C:/cygwin64/home/kovid/sw zlib=yes iconv=yes nmake /f Makefile.msvc - mkdir -p ~/sw/include/libxml2/libxml - cp include/libxml/*.h ~/sw/include/libxml2/libxml/ + cd .. + mkdir -p ~/sw/include/libxml2/libxml && cp include/libxml/*.h ~/sw/include/libxml2/libxml/ find . -type f \( -name "*.dll" -o -name "*.dll.manifest" \) -exec cp "{}" ~/sw/bin/ \; find . -name libxml2.lib -exec cp "{}" ~/sw/lib/ \; @@ -324,13 +324,14 @@ Get it from: ftp://xmlsoft.org/libxml2/ Run:: cd win32 - cscript.exe configure.js include=C:/cygwin/home/kovid/sw/include include=C:/cygwin/home/kovid/sw/include/libxml2 lib=C:/cygwin/home/kovid/sw/lib prefix=C:/cygwin/home/kovid/sw zlib=yes iconv=yes + cscript.exe configure.js include=C:/cygwin64/home/kovid/sw/include include=C:/cygwin64/home/kovid/sw/include/libxml2 lib=C:/cygwin64/home/kovid/sw/lib prefix=C:/cygwin64/home/kovid/sw zlib=yes iconv=yes nmake /f Makefile.msvc mkdir -p ~/sw/include/libxslt ~/sw/include/libexslt + cd .. cp libxslt/*.h ~/sw/include/libxslt/ cp libexslt/*.h ~/sw/include/libexslt/ find . -type f \( -name "*.dll" -o -name "*.dll.manifest" \) -exec cp "{}" ~/sw/bin/ \; - find . -name lib*xslt.lib -exec cp "{}" ~/sw/lib/ \; + find . -name 'lib*xslt.lib' -exec cp "{}" ~/sw/lib/ \; lxml ------ @@ -340,46 +341,43 @@ Get the source from: http://pypi.python.org/pypi/lxml Change the include dirs and lib dirs by editing setupinfo.py and changing the library_dirs() function to return:: - return ['C:/cygwin/home/kovid/sw/lib'] + return ['C:/cygwin64/home/kovid/sw/lib'] and the include_dirs() function to return - return ['C:/cygwin/home/kovid/sw/include/libxml2', 'C:/cygwin/home/kovid/sw/include'] + return ['C:/cygwin64/home/kovid/sw/include/libxml2', 'C:/cygwin64/home/kovid/sw/include'] Run:: python setup.py install +lcms2 +------------ + +Use the VS .sln in the Projects/VS2008 folder to build (remeber to set build +type to Release and choose x64 is needed):: + + cp include/*.h ~/sw/include/ + find . -type f \( -name "*.dll" -o -name "*.dll.manifest" \) -exec cp "{}" ~/sw/bin/ \; + find . -type f \( -name "*.lib" \) -exec cp "{}" ~/sw/lib/ \; + + Python Imaging Library ------------------------ -For 32-bit: -Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/ - -For 64-bit: Download from http://pypi.python.org/pypi/Pillow/ Edit setup.py setting the ROOT values, like this:: - SW = r'C:\cygwin\home\kovid\sw' + SW = r'C:\cygwin64\home\kovid\sw' JPEG_ROOT = ZLIB_ROOT = FREETYPE_ROOT = (SW+r'\lib', SW+r'\include') +Set zip_safe=False + +PIL needs lcms 1.x we have lcms 2.x so disable the lcms module. Look for the +line feature.want('lcms') and prefix it with ``False and``. + Build and install with:: - python setup.py build python setup.py install -Note that the lcms module will not be built. PIL requires lcms-1.x but only -lcms-2.x can be compiled as a 64 bit library. - -Pillow >= 2.2 installs itself as a .egg file. calibre needs it to be a PIL -directory. Extract the PIL directory as follows: - cd /cygdrive/c/Python27/Lib/site-packages - mkdir p && cd p - unzip ../Pillow-*.egg - cd .. && rm Pillow-*.egg && mv p/PIL . && chmod +x PIL/*.pyd - -Test it on the target system with - -calibre-debug -c "from PIL import Image; import _imaging, _imagingmath, _imagingft" - kdewin32-msvc ---------------- From 5b009747b7d7c32338069ac917b844eed500b36a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 18:51:44 +0530 Subject: [PATCH 12/21] Book polishing: If the books being polished have an ORIGINAL_EPUB/AZW3 format, warn the user that polishing will use it as the source and ask for confirmation. See #1316158 (Repost Enhancement Request: Polish Tool) --- src/calibre/gui2/actions/polish.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/polish.py b/src/calibre/gui2/actions/polish.py index 69c8d6fc23..b77849d882 100644 --- a/src/calibre/gui2/actions/polish.py +++ b/src/calibre/gui2/actions/polish.py @@ -432,7 +432,20 @@ class PolishAction(InterfaceAction): return None db = self.gui.library_view.model().db ans = (db.id(r) for r in rows) - return self.get_supported_books(ans) + ans = self.get_supported_books(ans) + for fmts in ans.itervalues(): + for x in fmts: + if x.startswith('ORIGINAL_'): + from calibre.gui2.dialogs.confirm_delete import confirm + if not confirm(_( + 'One of the books you are polishing has an {0} format.' + ' Polishing will use this as the source and overwrite' + ' any existing {1} format. Are you sure you want to proceed?').format( + x, x[len('ORIGINAL_'):]), 'confirm_original_polish', title=_('Are you sure?'), + confirm_msg=_('Ask for this confirmation again')): + return {} + break + return ans def get_supported_books(self, book_ids): from calibre.ebooks.oeb.polish.main import SUPPORTED From 4bb290657d7144586fb47f3070f7854b6bacd1c3 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Tue, 6 May 2014 23:16:13 +0900 Subject: [PATCH 13/21] recipe: update yomiuri onlie world Signed-off-by: Hiroshi Miura --- recipes/yomiuri.recipe | 43 +++++++++++++--------------- recipes/yomiuri_world.recipe | 54 ++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/recipes/yomiuri.recipe b/recipes/yomiuri.recipe index fb17bb1210..ef9d86e18a 100644 --- a/recipes/yomiuri.recipe +++ b/recipes/yomiuri.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' www.yomiuri.co.jp ''' @@ -16,16 +16,13 @@ class YOLNews(BasicNewsRecipe): publisher = 'Yomiuri Online News' category = 'news, japan' language = 'ja' - encoding = 'Shift_JIS' + encoding = 'UTF-8' index = 'http://www.yomiuri.co.jp/latestnews/' remove_javascript = True masthead_title = u'YOMIURI ONLINE' - keep_only_tags = [{'class':"article-def"}] - remove_tags = [{'class':"RelatedArticle"}, - {'class':"sbtns"} - ] - remove_tags_after = {'class':"date-def"} + + keep_only_tags = [{'class':"article text-resizeable"}] def parse_feeds(self): feeds = BasicNewsRecipe.parse_feeds(self) @@ -42,22 +39,22 @@ class YOLNews(BasicNewsRecipe): def parse_index(self): feeds = [] + newsarticles = [] soup = self.index_to_soup(self.index) - topstories = soup.find('ul',attrs={'class':'list-def'}) - if topstories: - newsarticles = [] - for itt in topstories.findAll('li'): - itema = itt.find('a',href=True) - if itema: - itd1 = itema.findNextSibling(text = True) - itd2 = itd1.findNextSibling(text = True) - itd3 = itd2.findNextSibling(text = True) - newsarticles.append({ - 'title' :itema.string - ,'date' :''.join([itd1, itd2, itd3]) - ,'url' :'http://www.yomiuri.co.jp' + itema['href'] - ,'description':'' - }) - feeds.append(('latest', newsarticles)) + listlatest = soup.find('ul', attrs={'class':'list-common list-common-latest'}) + if listlatest: + for itt in listlatest.findAll('li'): + itema = itt.find('a',href=True) + if itema: + item_headline = itema.find('span',attrs={'class':'headline'}) + item_date = item_headline.find('span',attrs={'class':'update'}) + newsarticles.append({ + 'title' :item_headline.contents[0] + ,'date' :item_date + ,'url' :itema['href'] + ,'description':'' + }) + feeds.append(('latest', newsarticles)) return feeds + diff --git a/recipes/yomiuri_world.recipe b/recipes/yomiuri_world.recipe index 41ee4fd23d..d7570d4753 100644 --- a/recipes/yomiuri_world.recipe +++ b/recipes/yomiuri_world.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Hiroshi Miura ' +__copyright__ = '2010,2014, Hiroshi Miura ' ''' www.yomiuri.co.jp ''' @@ -16,16 +16,12 @@ class YOLNews(BasicNewsRecipe): publisher = 'Yomiuri Online News' category = 'news, japan' language = 'ja' - encoding = 'Shift_JIS' + encoding = 'UTF-8' index = 'http://www.yomiuri.co.jp/world/' remove_javascript = True masthead_title = u"YOMIURI ONLINE" - keep_only_tags = [{'class':"article-def"}] - remove_tags = [{'class':"RelatedArticle"}, - {'class':"sbtns"} - ] - remove_tags_after = {'class':"date-def"} + keep_only_tags = [{'class':"article text-resizeable"}] def parse_feeds(self): feeds = BasicNewsRecipe.parse_feeds(self) @@ -42,20 +38,36 @@ class YOLNews(BasicNewsRecipe): def parse_index(self): feeds = [] + newsarticles = [] soup = self.index_to_soup(self.index) - topstories = soup.find('ul',attrs={'class':'list-def'}) - if topstories: - newsarticles = [] - for itt in topstories.findAll('li'): - itema = itt.find('a',href=True) - if itema: - itd1 = itema.findNextSibling(text = True) - newsarticles.append({ - 'title' :itema.string - ,'date' :''.join([itd1]) - ,'url' :'http://www.yomiuri.co.jp' + itema['href'] - ,'description':'' - }) - feeds.append(('World', newsarticles)) + mainspan = soup.find('div', attrs={'class':'pbNested span-main-inr'}) + if mainspan: + topstories = mainspan.find('ul',attrs={'class':'list-top'}) + if topstories: + for itt in topstories.findAll('li'): + itema = itt.find('a',href=True) + if itema: + item_headline = itema.find('span',attrs={'class':'headline'}) + item_date = item_headline.find('span',attrs={'class':'update'}) + newsarticles.append({ + 'title' :item_headline.contents[0] + ,'date' :item_date + ,'url' :itema['href'] + ,'description':'' + }) + secondstories = mainspan.find('ul', attrs={'class':'list-common'}) + if secondstories: + for itt in secondstories.findAll('li'): + itema = itt.find('a',href=True) + if itema: + item_headline = itema.find('span',attrs={'class':'headline'}) + item_date = item_headline.find('span',attrs={'class':'update'}) + newsarticles.append({ + 'title' :item_headline.contents[0] + ,'date' :item_date + ,'url' :itema['href'] + ,'description':'' + }) + feeds.append(('World', newsarticles)) return feeds From e02bef764fd4504e8f907770c9b6f4065047e581 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 21:01:08 +0530 Subject: [PATCH 14/21] ... --- setup/installer/windows/notes.rst | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 9b1f049e89..cce6880aff 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -378,26 +378,6 @@ line feature.want('lcms') and prefix it with ``False and``. Build and install with:: python setup.py install -kdewin32-msvc ----------------- - -I dont think this is needed any more, I've left it here just in case I'm wrong. - -Get it from http://www.winkde.org/pub/kde/ports/win32/repository/kdesupport/ -mkdir build -Run cmake - -Set build type to release and configuration to dll - -Build - -cp build/kdewin32-msvc-0.3.9/build/include/* include/ -cp build/kdewin32-msvc-0.3.9/build/bin/Release/*.dll bin/ -cp build/kdewin32-msvc-0.3.9/build/bin/Release/*.lib lib/ -cp build/kdewin32-msvc-0.3.9/build/bin/Release/*.exp lib/ -cp -r build/kdewin32-msvc-0.3.9/include/msvc/ include/ -cp build/kdewin32-msvc-0.3.9/include/*.h include/ - poppler ------------- @@ -408,7 +388,7 @@ On 64 bit cmake might not let you choose Visual Studio 2008, in whcih case leave the source field blank, click configure choose Visual Studio 2008 and then enter the source field. -In Cmake: disable GTK, Qt, OPenjpeg, cpp, lcms, gtk_tests, qt_tests. Enable +In cmake: disable GTK, Qt, openjpeg, cpp, lcms, gtk_tests, qt_tests. Enable jpeg, png and zlib:: cp build/utils/Release/*.exe ../../bin/ From 04b0930625b907f051dcfe8456558aec3d2eaaf1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 21:03:41 +0530 Subject: [PATCH 15/21] Edit book: Allow copying the list of errors from the Check Book tool to the clipboard by right clicking on the list of errors --- src/calibre/gui2/tweak_book/check.py | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/check.py b/src/calibre/gui2/tweak_book/check.py index fba761dbee..22a8afaeaa 100644 --- a/src/calibre/gui2/tweak_book/check.py +++ b/src/calibre/gui2/tweak_book/check.py @@ -9,7 +9,7 @@ __copyright__ = '2013, Kovid Goyal ' import sys from PyQt4.Qt import ( - QIcon, Qt, QSplitter, QListWidget, QTextBrowser, QPalette, + QIcon, Qt, QSplitter, QListWidget, QTextBrowser, QPalette, QMenu, QListWidgetItem, pyqtSignal, QApplication, QStyledItemDelegate) from calibre.ebooks.oeb.polish.check.base import WARN, INFO, DEBUG, ERROR, CRITICAL @@ -28,6 +28,19 @@ def icon_for_level(level): icon = None return QIcon(I(icon)) if icon else QIcon() +def prefix_for_level(level): + if level > WARN: + text = _('ERROR') + elif level == WARN: + text = _('WARNING') + elif level == INFO: + text = _('INFO') + else: + text = '' + if text: + text += ': ' + return text + class Delegate(QStyledItemDelegate): def initStyleOption(self, option, index): @@ -47,6 +60,8 @@ class Check(QSplitter): self.setChildrenCollapsible(False) self.items = i = QListWidget(self) + i.setContextMenuPolicy(Qt.CustomContextMenu) + i.customContextMenuRequested.connect(self.context_menu) self.items.setSpacing(3) self.items.itemDoubleClicked.connect(self.current_item_activated) self.items.currentItemChanged.connect(self.current_item_changed) @@ -66,6 +81,22 @@ class Check(QSplitter): if state is not None: self.restoreState(state) + def context_menu(self, pos): + m = QMenu() + if self.items.count() > 0: + m.addAction(QIcon(I('edit-copy.png')), _('Copy list of errors to clipboard'), self.copy_to_clipboard) + if list(m.actions()): + m.exec_(self.mapToGlobal(pos)) + + def copy_to_clipboard(self): + items = [] + for item in (self.items.item(i) for i in xrange(self.items.count())): + msg = unicode(item.text()) + msg = prefix_for_level(item.data(Qt.UserRole).toPyObject().level) + msg + items.append(msg) + if items: + QApplication.clipboard().setText('\n'.join(items)) + def save_state(self): tprefs.set('check-book-splitter-state', bytearray(self.saveState())) From c436362416074e42c1a85ef63aa2f291374b6f36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 6 May 2014 21:32:13 +0530 Subject: [PATCH 16/21] ... --- setup/installer/windows/notes.rst | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index cce6880aff..a0b49471aa 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -350,16 +350,6 @@ and the include_dirs() function to return Run:: python setup.py install -lcms2 ------------- - -Use the VS .sln in the Projects/VS2008 folder to build (remeber to set build -type to Release and choose x64 is needed):: - - cp include/*.h ~/sw/include/ - find . -type f \( -name "*.dll" -o -name "*.dll.manifest" \) -exec cp "{}" ~/sw/bin/ \; - find . -type f \( -name "*.lib" \) -exec cp "{}" ~/sw/lib/ \; - Python Imaging Library ------------------------ @@ -372,9 +362,6 @@ Edit setup.py setting the ROOT values, like this:: Set zip_safe=False -PIL needs lcms 1.x we have lcms 2.x so disable the lcms module. Look for the -line feature.want('lcms') and prefix it with ``False and``. - Build and install with:: python setup.py install @@ -391,7 +378,7 @@ then enter the source field. In cmake: disable GTK, Qt, openjpeg, cpp, lcms, gtk_tests, qt_tests. Enable jpeg, png and zlib:: - cp build/utils/Release/*.exe ../../bin/ + cp build/utils/Release/*.exe ~/sw/bin podofo ---------- From 12947188459278a86da2f15c4b396852923f028c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 May 2014 06:56:06 +0530 Subject: [PATCH 17/21] Content server: Make .ibook format files available in OPDS feeds --- resources/mime.types | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/mime.types b/resources/mime.types index b7c33e978f..0f0ab41301 100644 --- a/resources/mime.types +++ b/resources/mime.types @@ -1382,3 +1382,4 @@ application/vnd.ms-opentype otf application/font-woff woff application/x-font-truetype ttf text/xml plist +application/x-ibooks+zip ibook From 418cb87f13e272dd047803ffe1b1315a77f326d1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 May 2014 08:27:33 +0530 Subject: [PATCH 18/21] Subsetting embedded fonts: Fix an error when subsetting embedded fonts on windows if the username contains quotes or apostrophes. Fixes #1316533 [subset all embedded fonts > colect_font_stats error](https://bugs.launchpad.net/calibre/+bug/1316533) --- src/calibre/ebooks/oeb/polish/stats.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/oeb/polish/stats.py b/src/calibre/ebooks/oeb/polish/stats.py index 77b99ff9b6..948e6aa912 100644 --- a/src/calibre/ebooks/oeb/polish/stats.py +++ b/src/calibre/ebooks/oeb/polish/stats.py @@ -253,6 +253,10 @@ class StatsCollector(object): src = rule.get('src', None) if not src: continue + if src.startswith('url(') and src.endswith(')') and src[4] not in {'"', "'"}: + # Quote the url otherwise cssutils fails to parse it if it has + # ' or " in it + src = "url('" + src[4:-1].replace("'", "\\'") + "')" style = self.parser.parseStyle('background-image:%s'%src, validate=False) src = style.getProperty('background-image').propertyValue[0].uri name = self.href_to_name(src, '@font-face rule') From 1767c90680afa567d2ae419b10a39e294811fe55 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 May 2014 14:34:11 +0530 Subject: [PATCH 19/21] Edit Book: A new tool to filter style information (Tools->Filter style information). Useful to easily remove some CSS properties from the entire book. --- manual/edit.rst | 11 ++++ src/calibre/ebooks/oeb/polish/css.py | 78 ++++++++++++++++++++++++++ src/calibre/gui2/tweak_book/boss.py | 18 +++++- src/calibre/gui2/tweak_book/ui.py | 3 + src/calibre/gui2/tweak_book/widgets.py | 64 ++++++++++++++++++++- 5 files changed, 170 insertions(+), 4 deletions(-) diff --git a/manual/edit.rst b/manual/edit.rst index d4a2269156..a43992b375 100644 --- a/manual/edit.rst +++ b/manual/edit.rst @@ -400,6 +400,17 @@ If you use this tool multiple times, each invocation will cause the previously created inline Table of Contents to be replaced. The tool can be accessed via :guilabel:`Tools->Table of Contents->Insert inline Table of Contents`. +Filter style information +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This tool can be used to easily remove specified CSS style properties from the +entire book. You can tell it what properties you want removed, for example, +``color, background-color, line-height`` and it will remove them from +everywhere they occur --- stylesheets, ``