From 55a11e5037f7d03723889a4fdc7cd8d4f5260183 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 26 May 2013 20:40:29 +0530 Subject: [PATCH 1/3] Update Read It Later --- recipes/readitlater.recipe | 150 ++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 60 deletions(-) diff --git a/recipes/readitlater.recipe b/recipes/readitlater.recipe index 6f48ac116b..60e77ae558 100644 --- a/recipes/readitlater.recipe +++ b/recipes/readitlater.recipe @@ -1,61 +1,65 @@ """ -Pocket Calibre Recipe v1.3 +Pocket Calibre Recipe v1.4 """ from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe -import urllib2 -import urllib +from string import Template import json import operator -import tempfile import re +import tempfile +import urllib +import urllib2 -__license__ = 'GPL v3' + +__license__ = 'GPL v3' __copyright__ = ''' 2010, Darko Miletic 2011, Przemyslaw Kryger -2012, tBunnyMan +2012-2013, tBunnyMan ''' class Pocket(BasicNewsRecipe): - title = 'Pocket' - __author__ = 'Darko Miletic, Przemyslaw Kryger, Keith Callenberg, tBunnyMan' - description = '''Personalized news feeds. Go to getpocket.com to setup up \ - your news. This version displays pages of articles from \ - oldest to newest, with max & minimum counts, and marks articles \ - read after downloading.''' - publisher = 'getpocket.com' - category = 'news, custom' + title = 'Pocket' + __author__ = 'Darko Miletic, Przemyslaw Kryger, Keith Callenberg, tBunnyMan' + description = '''Personalized news feeds. Go to getpocket.com to setup up + your news. This version displays pages of articles from + oldest to newest, with max & minimum counts, and marks + articles read after downloading.''' + publisher = 'getpocket.com' + category = 'news, custom' + + #Settings people change max_articles_per_feed = 50 minimum_articles = 10 - #Set this to False for testing - mark_as_read_after_dl = False - #MUST be either 'oldest' or 'newest' - sort_method = 'oldest' - #To filter by tag this needs to be a single tag in quotes; IE 'calibre' + mark_as_read_after_dl = True # Set this to False for testing + sort_method = 'oldest' # MUST be either 'oldest' or 'newest' + # To filter by tag this needs to be a single tag in quotes; IE 'calibre' only_pull_tag = None - #You don't want to change anything under here unless you REALLY know what you are doing - no_stylesheets = True - use_embedded_content = False - needs_subscription = True + #You don't want to change anything under + no_stylesheets = True + use_embedded_content = False + needs_subscription = True articles_are_obfuscated = True - apikey = '19eg0e47pbT32z4793Tf021k99Afl889' - index_url = u'http://getpocket.com' - ajax_url = u'http://getpocket.com/a/x/getArticle.php' - read_api_url = index_url + u'/v3/get' - modify_api_url = index_url + u'/v3/send' - legacy_login_url = index_url + u'/l' # We use this to cheat oAuth - articles = [] + apikey = '19eg0e47pbT32z4793Tf021k99Afl889' + index_url = u'http://getpocket.com' + read_api_url = index_url + u'/v3/get' + modify_api_url = index_url + u'/v3/send' + legacy_login_url = index_url + u'/l' # We use this to cheat oAuth + articles = [] def get_browser(self, *args, **kwargs): """ - We need to pretend to be a recent version of safari for the mac to prevent User-Agent checks - Pocket api requires username and password so fail loudly if it's missing from the config. + We need to pretend to be a recent version of safari for the mac to + prevent User-Agent checks Pocket api requires username and password so + fail loudly if it's missing from the config. """ br = BasicNewsRecipe.get_browser(self, - user_agent='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4') + user_agent='Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; \ + en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) \ + Version/5.0.3 Safari/533.19.4') if self.username is not None and self.password is not None: br.open(self.legacy_login_url) br.select_form(nr=0) @@ -63,7 +67,7 @@ class Pocket(BasicNewsRecipe): br['password'] = self.password br.submit() else: - self.user_error("This Recipe requires authentication, please configured user & pass") + self.user_error("This Recipe requires authentication") return br def get_auth_uri(self): @@ -71,21 +75,20 @@ class Pocket(BasicNewsRecipe): uri = "" uri = u'{0}&apikey={1!s}'.format(uri, self.apikey) if self.username is None or self.password is None: - self.user_error("Username or password is blank. Pocket no longer supports blank passwords") + self.user_error("Username or password is blank.") else: uri = u'{0}&username={1!s}'.format(uri, self.username) uri = u'{0}&password={1!s}'.format(uri, self.password) return uri def get_pull_articles_uri(self): - """Return the part of the uri that has all of the get request settings""" uri = "" - uri = u'{0}&state={1}'.format(uri, u'unread') # TODO This could be modded to allow pulling archives - uri = u'{0}&contentType={1}'.format(uri, u'article') # TODO This COULD return images too + uri = u'{0}&state={1}'.format(uri, u'unread') + uri = u'{0}&contentType={1}'.format(uri, u'article') uri = u'{0}&sort={1}'.format(uri, self.sort_method) uri = u'{0}&count={1!s}'.format(uri, self.max_articles_per_feed) if self.only_pull_tag is not None: - uri = u'{0}tag={1}'.format(uri, self.only_pull_tag) + uri = u'{0}&tag={1}'.format(uri, self.only_pull_tag) return uri def parse_index(self): @@ -100,11 +103,12 @@ class Pocket(BasicNewsRecipe): response = urllib2.urlopen(request) pocket_feed = json.load(response)['list'] except urllib2.HTTPError as e: - self.log.exception("Pocket returned an error: {0}\nurl: {1}".format(e, fetch_url)) + self.log.exception("Pocket returned an error: {0}".format(e.info())) return [] except urllib2.URLError as e: self.log.exception("Unable to connect to getpocket.com's api: {0}\nurl: {1}".format(e, fetch_url)) return [] + if len(pocket_feed) < self.minimum_articles: self.mark_as_read_after_dl = False self.user_error("Only {0} articles retrieved, minimum_articles not reached".format(len(pocket_feed))) @@ -120,39 +124,65 @@ class Pocket(BasicNewsRecipe): 'sort': pocket_article[1]['sort_id'] }) self.articles = sorted(self.articles, key=operator.itemgetter('sort')) - print self.articles return [("My Pocket Articles for {0}".format(strftime('[%I:%M %p]')), self.articles)] - def get_obfuscated_article(self, url): + def get_textview(self, url): + """ + Since Pocket's v3 API they removed access to textview. They also + redesigned their page to make it much harder to scrape their textview. + We need to pull the article, retrieve the formcheck id, then use it + to querty for the json version + This function will break when pocket hates us + """ + ajax_url = self.index_url + u'/a/x/getArticle.php' soup = self.index_to_soup(url) - formcheck_script_tag = soup.find('script', text=re.compile("formCheck")) - form_check = formcheck_script_tag.split("=")[1].replace("'", "").replace(";", "").strip() + fc_tag = soup.find('script', text=re.compile("formCheck")) + fc_id = re.search(r"\'([\d\w]+)\'", fc_tag).group(1) article_id = url.split("/")[-1] - data = urllib.urlencode({'itemId': article_id, 'formCheck': form_check}) - response = self.browser.open(self.ajax_url, data) - article_json = json.load(response)['article']['article'] + data = urllib.urlencode({'itemId': article_id, 'formCheck': fc_id}) + try: + response = self.browser.open(ajax_url, data) + except urllib2.HTTPError as e: + self.log.exception("unable to get textview {0}".format(e.info())) + raise e + return json.load(response)['article'] + + def get_obfuscated_article(self, url): + """ + Our get_textview returns parsed json so prettify it to something well + parsed by calibre. + """ + article = self.get_textview(url) + template = Template('

$title

\ + $img\ +
$body
') + try: + image = ''.format(article['images']['1']['src']) + except: + image = '' with tempfile.NamedTemporaryFile(delete=False) as tf: - tf.write(article_json) + tf.write(template.safe_substitute( + title=article['title'], + img=image, + body=article['article'] + )) return tf.name def mark_as_read(self, mark_list): - formatted_list = [] + actions_list = [] for article_id in mark_list: - formatted_list.append({ + actions_list.append({ 'action': 'archive', 'item_id': article_id }) - command = { - 'actions': formatted_list - } - mark_read_url = u'{0}?{1}'.format( + mark_read_url = u'{0}?actions={1}{2}'.format( self.modify_api_url, + json.dumps(actions_list, separators=(',', ':')), self.get_auth_uri() ) try: - request = urllib2.Request(mark_read_url, json.dumps(command)) - response = urllib2.urlopen(request) - print u'response = {0}'.format(response.info()) + request = urllib2.Request(mark_read_url) + urllib2.urlopen(request) except urllib2.HTTPError as e: self.log.exception('Pocket returned an error while archiving articles: {0}'.format(e)) return [] @@ -162,7 +192,7 @@ class Pocket(BasicNewsRecipe): def cleanup(self): if self.mark_as_read_after_dl: - self.mark_as_read([x[1]['item_id'] for x in self.articles]) + self.mark_as_read([x['item_id'] for x in self.articles]) else: pass @@ -174,7 +204,7 @@ class Pocket(BasicNewsRecipe): try: from calibre.ebooks import calibre_cover title = self.title if isinstance(self.title, unicode) else \ - self.title.decode('utf-8', 'replace') + self.title.decode('utf-8', 'replace') date = strftime(self.timefmt) time = strftime('[%I:%M %p]') img_data = calibre_cover(title, date, time) @@ -192,4 +222,4 @@ class Pocket(BasicNewsRecipe): self.log.exception(error_message) raise RuntimeError(error_message) -# vim:ft=python +# vim:ft=python tabstop=8 expandtab shiftwidth=4 softtabstop=4 From 7495fd3dd61ab3d2a7d0855f7899b4626282f4d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 26 May 2013 21:05:17 +0530 Subject: [PATCH 2/3] Update 32 bit windows build to Qt 4.8.4 as well --- setup/installer/windows/freeze.py | 30 +++++++++++++++++++----------- setup/installer/windows/notes.rst | 7 ++----- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 7fe890a305..6237fa0071 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -15,7 +15,7 @@ from setup.installer.windows.wix import WixMixIn ICU_DIR = os.environ.get('ICU_DIR', r'Q:\icu') OPENSSL_DIR = os.environ.get('OPENSSL_DIR', r'Q:\openssl') -QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\4.8.2') +QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\current') QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', @@ -173,7 +173,8 @@ class Win32Freeze(Command, WixMixIn): shutil.copy2(x, self.dll_dir) for x in QT_DLLS: x += '4.dll' - if not x.startswith('phonon'): x = 'Qt'+x + if not x.startswith('phonon'): + x = 'Qt'+x shutil.copy2(os.path.join(QT_DIR, 'bin', x), self.dll_dir) shutil.copy2(r'C:\windows\system32\python%s.dll'%self.py_ver, self.dll_dir) @@ -280,7 +281,8 @@ class Win32Freeze(Command, WixMixIn): for ex in ('expatw', 'testplug'): if ex in f.lower(): ok = False - if not ok: continue + if not ok: + continue dest = self.dll_dir shutil.copy2(f, dest) for x in ('zlib1.dll', 'libxml2.dll', 'libxslt.dll', 'libexslt.dll'): @@ -294,8 +296,10 @@ class Win32Freeze(Command, WixMixIn): for f in glob.glob(self.j(impath, pat)): ok = True for ex in ('magick++', 'x11.dll', 'xext.dll'): - if ex in f.lower(): ok = False - if not ok: continue + if ex in f.lower(): + ok = False + if not ok: + continue shutil.copy2(f, self.dll_dir) def embed_manifests(self): @@ -303,7 +307,8 @@ class Win32Freeze(Command, WixMixIn): for x in os.walk(self.base): for f in x[-1]: base, ext = os.path.splitext(f) - if ext != '.manifest': continue + if ext != '.manifest': + continue dll = self.j(x[0], base) manifest = self.j(x[0], f) res = 2 @@ -337,7 +342,8 @@ class Win32Freeze(Command, WixMixIn): 'An executable program' desc = DESCRIPTIONS.get(internal_name, defdesc) license = 'GNU GPL v3.0' - def e(val): return val.replace('"', r'\"') + def e(val): + return val.replace('"', r'\"') if product_description is None: product_description = __appname__ + ' - E-book management' rc = template.format( @@ -353,7 +359,7 @@ class Win32Freeze(Command, WixMixIn): product_name=e(__appname__), product_description=e(product_description), legal_copyright=e(license), - legal_trademarks=e(__appname__ + \ + legal_trademarks=e(__appname__ + ' is a registered U.S. trademark number 3,666,525') ) if extra_data: @@ -540,7 +546,8 @@ class Win32Freeze(Command, WixMixIn): cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split() cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver] for src, obj in zip(sources, objects): - if not self.newer(obj, headers+[src]): continue + if not self.newer(obj, headers+[src]): + continue cmd = [msvc.cc] + cflags + dflags + ['/Fo'+obj, '/Tc'+src] self.run_builder(cmd, show_output=True) @@ -681,7 +688,7 @@ class Win32Freeze(Command, WixMixIn): if os.path.isdir(abspath): if not os.listdir(abspath): return - zinfo.external_attr = 0700 << 16 + zinfo.external_attr = 0o700 << 16 zf.writestr(zinfo, '') for x in os.listdir(abspath): if x not in exclude: @@ -690,7 +697,7 @@ class Win32Freeze(Command, WixMixIn): ext = os.path.splitext(name)[1].lower() if ext in ('.dll',): raise ValueError('Cannot add %r to zipfile'%abspath) - zinfo.external_attr = 0600 << 16 + zinfo.external_attr = 0o600 << 16 if ext in ('.py', '.pyc', '.pyo', '.pyd'): with open(abspath, 'rb') as f: zf.writestr(zinfo, f.read()) @@ -699,3 +706,4 @@ class Win32Freeze(Command, WixMixIn): + diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 9812fe598a..297279a8b2 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -176,8 +176,7 @@ For 64-bit:: Qt -------- -Download Qt sourcecode (.zip) from: http://qt-project.org/downloads -Extract Qt sourcecode to C:\Qt\current +Download Qt sourcecode (.zip) from: http://download.qt-project.org/official_releases/qt/ Qt uses its own routine to locate and load "system libraries" including the openssl libraries needed for "Get Books". This means that we have to apply the @@ -200,9 +199,7 @@ Now, run configure and make: -no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly --no-accessibility is added because accessibility may be causing crashes on win 8 64 bit - - ./configure.exe -ltcg -opensource -release -qt-zlib -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-accessibility -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -nomake tools -no-plugin-manifests -openssl -I $OPENSSL_DIR/include -L $OPENSSL_DIR/lib && nmake + ./configure.exe -ltcg -opensource -release -qt-zlib -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -nomake tools -no-plugin-manifests -openssl -I $OPENSSL_DIR/include -L $OPENSSL_DIR/lib && nmake Add the path to the bin folder inside the Qt dir to your system PATH. From 00ee35c05a0469f2e96c8068773c21fd86aa8dc6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 26 May 2013 21:06:50 +0530 Subject: [PATCH 3/3] Make the lighter color a little less light --- src/qtcurve/style/qtcurve.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qtcurve/style/qtcurve.cpp b/src/qtcurve/style/qtcurve.cpp index 276e339e62..2c012ef196 100644 --- a/src/qtcurve/style/qtcurve.cpp +++ b/src/qtcurve/style/qtcurve.cpp @@ -5255,7 +5255,7 @@ void Style::drawPrimitive(PrimitiveElement element, const QStyleOption *option, if (color.lightness() > 128) color = color.darker(widget->property("highlight_current_item").toInt()); else - color = color.lighter(); + color = color.lighter(135); } bool square((opts.square&SQUARE_LISTVIEW_SELECTION) &&