From c020a3d12797a57634a700addaaa0caa87bee150 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 23 Oct 2010 19:21:01 -0700 Subject: [PATCH 1/8] STNN by Larry Chan --- resources/recipes/stnn.recipe | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 resources/recipes/stnn.recipe diff --git a/resources/recipes/stnn.recipe b/resources/recipes/stnn.recipe new file mode 100644 index 0000000000..7be357c921 --- /dev/null +++ b/resources/recipes/stnn.recipe @@ -0,0 +1,60 @@ + + +__license__ = 'GPL v3' +__copyright__ = '2010, Larry Chan ' +''' +Singtao STNN +''' +from calibre.web.feeds.recipes import BasicNewsRecipe + +class SingtaoSTNN(BasicNewsRecipe): + title = 'Singtao STNN' + __author__ = 'Larry Chan, larry1chan' + description = 'Chinese News' + oldest_article = 2 + max_articles_per_feed = 100 + simultaneous_downloads = 5 + no_stylesheets = True + #delay = 1 + use_embedded_content = False + encoding = 'gb2312' + publisher = 'Singtao STNN' + category = 'news, China, world' + language = 'zh' + publication_type = 'newsportal' + extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} ' + masthead_url = 'http://www.stnn.cc/images/0806/logo_080728.gif' + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + ,'linearize_tables': True + } + + + remove_tags_before = dict(name='div', attrs={'class':['page_box']}) + remove_tags_after = dict(name='div', attrs={'class':['pagelist']}) + + keep_only_tags = [ + dict(name='div', attrs={'class':['font_title clearfix']}), + dict(name='div', attrs={'id':['content_zoom']}) + + ] + + remove_attributes = ['width','height','href'] + + # for a full list of rss check out [url]http://www.stnn.cc/rss/[/url] + + feeds = [ (u'Headline News', u'http://www.stnn.cc/rss/news/index.xml'), + (u'Breaking News', u'http://www.stnn.cc/rss/tufa/index.xml'), + (u'Finance', u'http://www.stnn.cc/rss/fin/index.xml'), + (u'Entertainment', u'http://www.stnn.cc/rss/ent/index.xml'), + (u'International', u'http://www.stnn.cc/rss/guoji/index.xml'), + (u'China', u'http://www.stnn.cc/rss/china/index.xml'), + (u'Opnion', u'http://www.stnn.cc/rss/fin_op/index.xml'), + (u'Blog', u'http://blog.stnn.cc/uploadfile/rssblogtypehotlog.xml'), + (u'Hong Kong', u'http://www.stnn.cc/rss/hongkong/index.xml') + + ] + From 4110646e8a2f81d84b5c72a9347fb5661811137f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 24 Oct 2010 09:16:16 -0700 Subject: [PATCH 2/8] Fix #7287 (Building calibre fails on PyQt 4.8.0) --- src/calibre/gui2/lrf_renderer/main.ui | 189 +++++++++++--------------- src/calibre/gui2/viewer/main.ui | 4 +- 2 files changed, 81 insertions(+), 112 deletions(-) diff --git a/src/calibre/gui2/lrf_renderer/main.ui b/src/calibre/gui2/lrf_renderer/main.ui index 0e7b54edb8..4143b4f509 100644 --- a/src/calibre/gui2/lrf_renderer/main.ui +++ b/src/calibre/gui2/lrf_renderer/main.ui @@ -1,7 +1,8 @@ - + + MainWindow - - + + 0 0 @@ -9,75 +10,51 @@ 701 - - + + 0 0 - + LRF Viewer - - + + :/images/viewer.png:/images/viewer.png - - - - 0 - 39 - 601 - 662 - - - - + + + 0 - - + + 0 - - - - 0 - 0 - 601 - 662 - - - - + + + 0 - - - + + + true - - - - 0 - 0 - 601 - 701 - - - + + - + Qt::Vertical - + 20 40 @@ -86,34 +63,34 @@ - - + + QFrame::StyledPanel - + QFrame::Raised - + - - + + 0 - + -1 - - + + 11 75 true - + Parsing LRF file @@ -123,10 +100,10 @@ - + Qt::Vertical - + 20 40 @@ -140,93 +117,85 @@ - - - - 0 - 0 - 601 - 39 - - - + + LRF Viewer toolbar - + Qt::AllToolBarAreas - - TopToolBarArea + + Qt::TopToolBarArea - - true + + false - - - - - - - - - + + + + + + + + + - - - + + + :/images/next.png:/images/next.png - + Next Page - - - + + + :/images/previous.png:/images/previous.png - + Previous Page - - - + + + :/images/back.png:/images/back.png - + Back - - - + + + :/images/forward.png:/images/forward.png - + Forward - - + + Next match - - - + + + :/images/document_open.png:/images/document_open.png - + Open ebook - - - + + + :/images/config.png:/images/config.png - + Configure @@ -239,7 +208,7 @@ - + diff --git a/src/calibre/gui2/viewer/main.ui b/src/calibre/gui2/viewer/main.ui index e3b8fb8a61..4cfa1590da 100644 --- a/src/calibre/gui2/viewer/main.ui +++ b/src/calibre/gui2/viewer/main.ui @@ -108,7 +108,7 @@ - LeftToolBarArea + Qt::LeftToolBarArea false @@ -136,7 +136,7 @@ - TopToolBarArea + Qt::TopToolBarArea false From 138d503aba2de23bdbfbe1a0c488c84dbe2b2754 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 24 Oct 2010 14:57:43 -0700 Subject: [PATCH 3/8] Fix #7276 (Viewer crash on main app close. (v0.7.24)) --- src/calibre/gui2/viewer/main.py | 3 +++ src/calibre/utils/ipc/worker.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index e113ef0611..09019af18b 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -716,6 +716,9 @@ View an ebook. def main(args=sys.argv): + # Ensure viewer can continue to function if GUI is closed + os.environ.pop('CALIBRE_WORKER_TEMP_DIR', None) + parser = option_parser() opts, args = parser.parse_args(args) pid = os.fork() if False and (islinux or isfreebsd) else -1 diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index e3584380a1..d8ffad7c53 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -105,7 +105,7 @@ def main(): notifier.start() result = func(*args, **kwargs) - if result is not None: + if result is not None and os.path.exists(os.path.dirname(resultf)): cPickle.dump(result, open(resultf, 'wb'), -1) notifier.queue.put(None) From 7c25c4d149f70c3abc281a52c328a0efada46e36 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Mon, 25 Oct 2010 21:18:06 -0300 Subject: [PATCH 4/8] Changes for kobo WIFI version 1.7 --- src/calibre/devices/kobo/driver.py | 63 ++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 418bfe5e0d..3562da55d2 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,7 +22,9 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' - version = (1, 0, 6) + version = (1, 0, 7) + + dbversion = 0 supported_platforms = ['windows', 'osx', 'linux'] @@ -92,7 +94,7 @@ class KOBO(USBMS): if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] lpath = lpath.replace('\\', '/') -# print "LPATH: " + lpath + # debug_print("LPATH: ", lpath, " - Title: " , title) playlist_map = {} @@ -112,7 +114,7 @@ class KOBO(USBMS): #print "Image name Normalized: " + imagename if imagename is not None: bl[idx].thumbnail = ImageWrapper(imagename) - if ContentType != '6': + if (ContentType != '6'and self.dbversion < 8) or (self.dbversion >= 8): if self.update_metadata_item(bl[idx]): # print 'update_metadata_item returned true' changed = True @@ -120,10 +122,16 @@ class KOBO(USBMS): playlist_map[lpath] not in bl[idx].device_collections: bl[idx].device_collections.append(playlist_map[lpath]) else: - if ContentType == '6': + if ContentType == '6' and self.dbversion < 8: book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) else: - book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) + try: + book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) + except: + debug_print("prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, \ + "mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID) + raise + # print 'Update booklist' book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else [] @@ -143,6 +151,13 @@ class KOBO(USBMS): # numrows = row[0] #cursor.close() + # Determine the database version + # 4 - Bluetooth Kobo Rev 2 (1.4) + # 8 - WIFI KOBO Rev 1 + cursor.execute('select version from dbversion') + result = cursor.fetchone() + self.dbversion = result[0] + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus from content where BookID is Null' @@ -153,7 +168,8 @@ class KOBO(USBMS): # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) path = self.path_from_contentid(row[3], row[5], oncard) - mime = mime_type_ext(path_to_ext(row[3])) + mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip' + # debug_print("mime:", mime) if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7]) @@ -206,7 +222,7 @@ class KOBO(USBMS): cursor.close() cursor = connection.cursor() - if ContentType == 6: + if ContentType == 6 and self.dbversion < 8: # Delete the shortcover_pages first cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) @@ -249,7 +265,7 @@ class KOBO(USBMS): path = self.normalize_path(path) # print "Delete file normalized path: " + path extension = os.path.splitext(path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path) ContentID = self.contentid_from_path(path, ContentType) @@ -332,9 +348,14 @@ class KOBO(USBMS): def contentid_from_path(self, path, ContentType): if ContentType == 6: - ContentID = os.path.splitext(path)[0] - # Remove the prefix on the file. it could be either - ContentID = ContentID.replace(self._main_prefix, '') + if self.dbversion < 8: + ContentID = os.path.splitext(path)[0] + # Remove the prefix on the file. it could be either + ContentID = ContentID.replace(self._main_prefix, '') + else: + ContentID = path + ContentID = ContentID.replace(self._main_prefix + '.kobo/kepub/', '') + if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, '') elif ContentType == 999: # HTML Files @@ -350,6 +371,13 @@ class KOBO(USBMS): ContentID = ContentID.replace("\\", '/') return ContentID + def get_content_type_from_path(self, path): + # Strictly speaking the ContentType could be 6 or 10 + # however newspapers have the same storage format + if path.find('kepub') >= 0: + ContentType = 6 + return ContentType + def get_content_type_from_extension(self, extension): if extension == '.kobo': # Kobo books do not have book files. They do have some images though @@ -369,19 +397,22 @@ class KOBO(USBMS): print 'path from_contentid cardb' elif oncard == 'carda': path = path.replace("file:///mnt/sd/", self._card_a_prefix) - # print "SD Card: " + filename + # print "SD Card: " + path else: - if ContentType == "6": + if ContentType == "6" and self.dbversion < 8: # This is a hack as the kobo files do not exist # but the path is required to make a unique id # for calibre's reference path = self._main_prefix + path + '.kobo' # print "Path: " + path + elif (ContentType == "6" or ContentType == "10") and self.dbversion >= 8: + path = self._main_prefix + '.kobo/kepub/' + path + # print "Internal: " + path else: # if path.startswith("file:///mnt/onboard/"): path = path.replace("file:///mnt/onboard/", self._main_prefix) path = path.replace("/mnt/onboard/", self._main_prefix) - # print "Internal: " + filename + # print "Internal: " + path return path @@ -469,7 +500,7 @@ class KOBO(USBMS): book.device_collections = ['Im_Reading'] extension = os.path.splitext(book.path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) ContentID = self.contentid_from_path(book.path, ContentType) datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) @@ -505,7 +536,7 @@ class KOBO(USBMS): book.device_collections = ['Read'] extension = os.path.splitext(book.path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) ContentID = self.contentid_from_path(book.path, ContentType) # datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) From d7f649ce985334da8a52be56351f1807b2e643f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 25 Oct 2010 18:14:52 -0700 Subject: [PATCH 5/8] Ming Pao by Eddie Lau --- resources/recipes/ming_pao.recipe | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 resources/recipes/ming_pao.recipe diff --git a/resources/recipes/ming_pao.recipe b/resources/recipes/ming_pao.recipe new file mode 100644 index 0000000000..6a61405698 --- /dev/null +++ b/resources/recipes/ming_pao.recipe @@ -0,0 +1,64 @@ +cense__ = 'GPL v3' +__copyright__ = '2010, Eddie Lau' +''' +modified from Singtao Toronto calibre recipe by rty +''' + +import datetime +from calibre.web.feeds.recipes import BasicNewsRecipe + +class AdvancedUserRecipe1278063072(BasicNewsRecipe): + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper' + publisher = 'news.mingpao.com' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + + keep_only_tags = [dict(name='h1'), + dict(attrs={'id':['newscontent01','newscontent02']})] + + def get_fetchdate(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time + dt_local = dt_utc - datetime.timedelta(-8.0/24) + return dt_local.strftime("%Y%m%d") + + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm'), (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), (u'\u793e\u8a55\u2027\u7b46\u9663 Editorial', 'http://news.mingpao.com/' + dateStr + '/mrindex.htm'), (u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm'), ('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm'), (u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + return feeds + + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet']}) + current_articles = [] + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + current_articles.append({'title': title, 'url': url, 'description':''}) + return current_articles + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(width=True): + del item['width'] + return soup + From 284ca8f7a15a6d653761694e125a721a2d68a1bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 26 Oct 2010 08:08:57 -0600 Subject: [PATCH 6/8] ... --- src/calibre/web/feeds/news.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index cb6bf30bcf..869799f6bb 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -842,6 +842,9 @@ class BasicNewsRecipe(Recipe): except NotImplementedError: feeds = self.parse_feeds() + if not feeds: + raise ValueError('No articles found, aborting') + #feeds = FeedCollection(feeds) self.report_progress(0, _('Trying to download cover...')) From 9a93d3fd2dccbf0f1fb9212dc00134ea41a8310a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 26 Oct 2010 08:17:02 -0600 Subject: [PATCH 7/8] /browse: Fix handling of non-ascii saved searches --- src/calibre/library/server/browse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 463fcd6fde..142f40efab 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -509,7 +509,7 @@ class BrowseServer(object): hide_sort = 'true' if dt == 'series' else 'false' if category == 'search': - which = unhexlify(cid) + which = unhexlify(cid).decode('utf-8') try: ids = self.search_cache('search:"%s"'%which) except: From 330a7c989b163e27ca36424c1b680a95c8a47e40 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 26 Oct 2010 08:18:42 -0600 Subject: [PATCH 8/8] Content server: Make /browse the default --- src/calibre/library/server/content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index d95cd1818c..52a08e6175 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -124,7 +124,7 @@ class ContentServer(object): if want_mobile: return self.mobile() - return self.static('index.html') + return self.browse_toplevel() def old(self, **kwargs): return self.static('index.html')