diff --git a/resources/recipes/ming_pao.recipe b/resources/recipes/ming_pao.recipe new file mode 100644 index 0000000000..6a61405698 --- /dev/null +++ b/resources/recipes/ming_pao.recipe @@ -0,0 +1,64 @@ +cense__ = 'GPL v3' +__copyright__ = '2010, Eddie Lau' +''' +modified from Singtao Toronto calibre recipe by rty +''' + +import datetime +from calibre.web.feeds.recipes import BasicNewsRecipe + +class AdvancedUserRecipe1278063072(BasicNewsRecipe): + title = 'Ming Pao - Hong Kong' + oldest_article = 1 + max_articles_per_feed = 100 + __author__ = 'Eddie Lau' + description = 'Hong Kong Chinese Newspaper' + publisher = 'news.mingpao.com' + category = 'Chinese, News, Hong Kong' + remove_javascript = True + use_embedded_content = False + no_stylesheets = True + language = 'zh' + encoding = 'Big5-HKSCS' + recursions = 0 + conversion_options = {'linearize_tables':True} + masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif' + + keep_only_tags = [dict(name='h1'), + dict(attrs={'id':['newscontent01','newscontent02']})] + + def get_fetchdate(self): + dt_utc = datetime.datetime.utcnow() + # convert UTC to local hk time + dt_local = dt_utc - datetime.timedelta(-8.0/24) + return dt_local.strftime("%Y%m%d") + + def parse_index(self): + feeds = [] + dateStr = self.get_fetchdate() + for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'), (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm'), (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'), (u'\u793e\u8a55\u2027\u7b46\u9663 Editorial', 'http://news.mingpao.com/' + dateStr + '/mrindex.htm'), (u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'), (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'), (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm'), ('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'), (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm'), (u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),]: + articles = self.parse_section(url) + if articles: + feeds.append((title, articles)) + return feeds + + def parse_section(self, url): + dateStr = self.get_fetchdate() + soup = self.index_to_soup(url) + divs = soup.findAll(attrs={'class': ['bullet']}) + current_articles = [] + for i in divs: + a = i.find('a', href = True) + title = self.tag_to_string(a) + url = a.get('href', False) + url = 'http://news.mingpao.com/' + dateStr + '/' +url + current_articles.append({'title': title, 'url': url, 'description':''}) + return current_articles + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(width=True): + del item['width'] + return soup + diff --git a/resources/recipes/stnn.recipe b/resources/recipes/stnn.recipe new file mode 100644 index 0000000000..7be357c921 --- /dev/null +++ b/resources/recipes/stnn.recipe @@ -0,0 +1,60 @@ + + +__license__ = 'GPL v3' +__copyright__ = '2010, Larry Chan ' +''' +Singtao STNN +''' +from calibre.web.feeds.recipes import BasicNewsRecipe + +class SingtaoSTNN(BasicNewsRecipe): + title = 'Singtao STNN' + __author__ = 'Larry Chan, larry1chan' + description = 'Chinese News' + oldest_article = 2 + max_articles_per_feed = 100 + simultaneous_downloads = 5 + no_stylesheets = True + #delay = 1 + use_embedded_content = False + encoding = 'gb2312' + publisher = 'Singtao STNN' + category = 'news, China, world' + language = 'zh' + publication_type = 'newsportal' + extra_css = ' body{ font-family: Verdana,Helvetica,Arial,sans-serif } .introduction{font-weight: bold} .story-feature{display: block; padding: 0; border: 1px solid; width: 40%; font-size: small} .story-feature h2{text-align: center; text-transform: uppercase} ' + masthead_url = 'http://www.stnn.cc/images/0806/logo_080728.gif' + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + ,'linearize_tables': True + } + + + remove_tags_before = dict(name='div', attrs={'class':['page_box']}) + remove_tags_after = dict(name='div', attrs={'class':['pagelist']}) + + keep_only_tags = [ + dict(name='div', attrs={'class':['font_title clearfix']}), + dict(name='div', attrs={'id':['content_zoom']}) + + ] + + remove_attributes = ['width','height','href'] + + # for a full list of rss check out [url]http://www.stnn.cc/rss/[/url] + + feeds = [ (u'Headline News', u'http://www.stnn.cc/rss/news/index.xml'), + (u'Breaking News', u'http://www.stnn.cc/rss/tufa/index.xml'), + (u'Finance', u'http://www.stnn.cc/rss/fin/index.xml'), + (u'Entertainment', u'http://www.stnn.cc/rss/ent/index.xml'), + (u'International', u'http://www.stnn.cc/rss/guoji/index.xml'), + (u'China', u'http://www.stnn.cc/rss/china/index.xml'), + (u'Opnion', u'http://www.stnn.cc/rss/fin_op/index.xml'), + (u'Blog', u'http://blog.stnn.cc/uploadfile/rssblogtypehotlog.xml'), + (u'Hong Kong', u'http://www.stnn.cc/rss/hongkong/index.xml') + + ] + diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 418bfe5e0d..3562da55d2 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -22,7 +22,9 @@ class KOBO(USBMS): gui_name = 'Kobo Reader' description = _('Communicate with the Kobo Reader') author = 'Timothy Legge and Kovid Goyal' - version = (1, 0, 6) + version = (1, 0, 7) + + dbversion = 0 supported_platforms = ['windows', 'osx', 'linux'] @@ -92,7 +94,7 @@ class KOBO(USBMS): if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] lpath = lpath.replace('\\', '/') -# print "LPATH: " + lpath + # debug_print("LPATH: ", lpath, " - Title: " , title) playlist_map = {} @@ -112,7 +114,7 @@ class KOBO(USBMS): #print "Image name Normalized: " + imagename if imagename is not None: bl[idx].thumbnail = ImageWrapper(imagename) - if ContentType != '6': + if (ContentType != '6'and self.dbversion < 8) or (self.dbversion >= 8): if self.update_metadata_item(bl[idx]): # print 'update_metadata_item returned true' changed = True @@ -120,10 +122,16 @@ class KOBO(USBMS): playlist_map[lpath] not in bl[idx].device_collections: bl[idx].device_collections.append(playlist_map[lpath]) else: - if ContentType == '6': + if ContentType == '6' and self.dbversion < 8: book = Book(prefix, lpath, title, authors, mime, date, ContentType, ImageID, size=1048576) else: - book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) + try: + book = self.book_from_path(prefix, lpath, title, authors, mime, date, ContentType, ImageID) + except: + debug_print("prefix: ", prefix, "lpath: ", lpath, "title: ", title, "authors: ", authors, \ + "mime: ", mime, "date: ", date, "ContentType: ", ContentType, "ImageID: ", ImageID) + raise + # print 'Update booklist' book.device_collections = [playlist_map[lpath]] if lpath in playlist_map else [] @@ -143,6 +151,13 @@ class KOBO(USBMS): # numrows = row[0] #cursor.close() + # Determine the database version + # 4 - Bluetooth Kobo Rev 2 (1.4) + # 8 - WIFI KOBO Rev 1 + cursor.execute('select version from dbversion') + result = cursor.fetchone() + self.dbversion = result[0] + query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus from content where BookID is Null' @@ -153,7 +168,8 @@ class KOBO(USBMS): # self.report_progress((i+1) / float(numrows), _('Getting list of books on device...')) path = self.path_from_contentid(row[3], row[5], oncard) - mime = mime_type_ext(path_to_ext(row[3])) + mime = mime_type_ext(path_to_ext(path)) if path.find('kepub') == -1 else 'application/epub+zip' + # debug_print("mime:", mime) if oncard != 'carda' and oncard != 'cardb' and not row[3].startswith("file:///mnt/sd/"): changed = update_booklist(self._main_prefix, path, row[0], row[1], mime, row[2], row[5], row[6], row[7]) @@ -206,7 +222,7 @@ class KOBO(USBMS): cursor.close() cursor = connection.cursor() - if ContentType == 6: + if ContentType == 6 and self.dbversion < 8: # Delete the shortcover_pages first cursor.execute('delete from shortcover_page where shortcoverid in (select ContentID from content where BookID = ?)', t) @@ -249,7 +265,7 @@ class KOBO(USBMS): path = self.normalize_path(path) # print "Delete file normalized path: " + path extension = os.path.splitext(path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path) ContentID = self.contentid_from_path(path, ContentType) @@ -332,9 +348,14 @@ class KOBO(USBMS): def contentid_from_path(self, path, ContentType): if ContentType == 6: - ContentID = os.path.splitext(path)[0] - # Remove the prefix on the file. it could be either - ContentID = ContentID.replace(self._main_prefix, '') + if self.dbversion < 8: + ContentID = os.path.splitext(path)[0] + # Remove the prefix on the file. it could be either + ContentID = ContentID.replace(self._main_prefix, '') + else: + ContentID = path + ContentID = ContentID.replace(self._main_prefix + '.kobo/kepub/', '') + if self._card_a_prefix is not None: ContentID = ContentID.replace(self._card_a_prefix, '') elif ContentType == 999: # HTML Files @@ -350,6 +371,13 @@ class KOBO(USBMS): ContentID = ContentID.replace("\\", '/') return ContentID + def get_content_type_from_path(self, path): + # Strictly speaking the ContentType could be 6 or 10 + # however newspapers have the same storage format + if path.find('kepub') >= 0: + ContentType = 6 + return ContentType + def get_content_type_from_extension(self, extension): if extension == '.kobo': # Kobo books do not have book files. They do have some images though @@ -369,19 +397,22 @@ class KOBO(USBMS): print 'path from_contentid cardb' elif oncard == 'carda': path = path.replace("file:///mnt/sd/", self._card_a_prefix) - # print "SD Card: " + filename + # print "SD Card: " + path else: - if ContentType == "6": + if ContentType == "6" and self.dbversion < 8: # This is a hack as the kobo files do not exist # but the path is required to make a unique id # for calibre's reference path = self._main_prefix + path + '.kobo' # print "Path: " + path + elif (ContentType == "6" or ContentType == "10") and self.dbversion >= 8: + path = self._main_prefix + '.kobo/kepub/' + path + # print "Internal: " + path else: # if path.startswith("file:///mnt/onboard/"): path = path.replace("file:///mnt/onboard/", self._main_prefix) path = path.replace("/mnt/onboard/", self._main_prefix) - # print "Internal: " + filename + # print "Internal: " + path return path @@ -469,7 +500,7 @@ class KOBO(USBMS): book.device_collections = ['Im_Reading'] extension = os.path.splitext(book.path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) ContentID = self.contentid_from_path(book.path, ContentType) datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) @@ -505,7 +536,7 @@ class KOBO(USBMS): book.device_collections = ['Read'] extension = os.path.splitext(book.path)[1] - ContentType = self.get_content_type_from_extension(extension) + ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path) ContentID = self.contentid_from_path(book.path, ContentType) # datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) diff --git a/src/calibre/gui2/lrf_renderer/main.ui b/src/calibre/gui2/lrf_renderer/main.ui index 0e7b54edb8..4143b4f509 100644 --- a/src/calibre/gui2/lrf_renderer/main.ui +++ b/src/calibre/gui2/lrf_renderer/main.ui @@ -1,7 +1,8 @@ - + + MainWindow - - + + 0 0 @@ -9,75 +10,51 @@ 701 - - + + 0 0 - + LRF Viewer - - + + :/images/viewer.png:/images/viewer.png - - - - 0 - 39 - 601 - 662 - - - - + + + 0 - - + + 0 - - - - 0 - 0 - 601 - 662 - - - - + + + 0 - - - + + + true - - - - 0 - 0 - 601 - 701 - - - + + - + Qt::Vertical - + 20 40 @@ -86,34 +63,34 @@ - - + + QFrame::StyledPanel - + QFrame::Raised - + - - + + 0 - + -1 - - + + 11 75 true - + Parsing LRF file @@ -123,10 +100,10 @@ - + Qt::Vertical - + 20 40 @@ -140,93 +117,85 @@ - - - - 0 - 0 - 601 - 39 - - - + + LRF Viewer toolbar - + Qt::AllToolBarAreas - - TopToolBarArea + + Qt::TopToolBarArea - - true + + false - - - - - - - - - + + + + + + + + + - - - + + + :/images/next.png:/images/next.png - + Next Page - - - + + + :/images/previous.png:/images/previous.png - + Previous Page - - - + + + :/images/back.png:/images/back.png - + Back - - - + + + :/images/forward.png:/images/forward.png - + Forward - - + + Next match - - - + + + :/images/document_open.png:/images/document_open.png - + Open ebook - - - + + + :/images/config.png:/images/config.png - + Configure @@ -239,7 +208,7 @@ - + diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index e113ef0611..09019af18b 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -716,6 +716,9 @@ View an ebook. def main(args=sys.argv): + # Ensure viewer can continue to function if GUI is closed + os.environ.pop('CALIBRE_WORKER_TEMP_DIR', None) + parser = option_parser() opts, args = parser.parse_args(args) pid = os.fork() if False and (islinux or isfreebsd) else -1 diff --git a/src/calibre/gui2/viewer/main.ui b/src/calibre/gui2/viewer/main.ui index e3b8fb8a61..4cfa1590da 100644 --- a/src/calibre/gui2/viewer/main.ui +++ b/src/calibre/gui2/viewer/main.ui @@ -108,7 +108,7 @@ - LeftToolBarArea + Qt::LeftToolBarArea false @@ -136,7 +136,7 @@ - TopToolBarArea + Qt::TopToolBarArea false diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4943dd9dd5..efe92d3c63 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -653,8 +653,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.dirtied(book_ids) def get_metadata_for_dump(self, idx, remove_from_dirtied=True): + path, mi = (None, None) try: - path, mi = (None, None) # While a book is being created, the path is empty. Don't bother to # try to write the opf, because it will go to the wrong folder. if self.path(idx, index_is_id=True): diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 463fcd6fde..142f40efab 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -509,7 +509,7 @@ class BrowseServer(object): hide_sort = 'true' if dt == 'series' else 'false' if category == 'search': - which = unhexlify(cid) + which = unhexlify(cid).decode('utf-8') try: ids = self.search_cache('search:"%s"'%which) except: diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index d95cd1818c..52a08e6175 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -124,7 +124,7 @@ class ContentServer(object): if want_mobile: return self.mobile() - return self.static('index.html') + return self.browse_toplevel() def old(self, **kwargs): return self.static('index.html') diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py index e3584380a1..d8ffad7c53 100644 --- a/src/calibre/utils/ipc/worker.py +++ b/src/calibre/utils/ipc/worker.py @@ -105,7 +105,7 @@ def main(): notifier.start() result = func(*args, **kwargs) - if result is not None: + if result is not None and os.path.exists(os.path.dirname(resultf)): cPickle.dump(result, open(resultf, 'wb'), -1) notifier.queue.put(None) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index cb6bf30bcf..869799f6bb 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -842,6 +842,9 @@ class BasicNewsRecipe(Recipe): except NotImplementedError: feeds = self.parse_feeds() + if not feeds: + raise ValueError('No articles found, aborting') + #feeds = FeedCollection(feeds) self.report_progress(0, _('Trying to download cover...'))