From 2f60c9e064afd781ae400d7bfb32fd4814929907 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 29 Oct 2011 07:41:21 +0200 Subject: [PATCH 01/37] Add hour/minute/seconds formatting to date format strings. --- src/calibre/manual/template_lang.rst | 8 +++++ src/calibre/utils/date.py | 44 ++++++++++++++++++++---- src/calibre/utils/formatter_functions.py | 8 +++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index 26c03d8491..72e9c9aa27 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -258,6 +258,14 @@ The following functions are available in addition to those described in single-f MMMM : the long localized month name (e.g. "January" to "December"). yy : the year as two digit number (00 to 99). yyyy : the year as four digit number. + h : the hours without a leading 0 (0 to 11 or 0 to 23, depending on am/pm) + hh : the minutes with a leading 0 (00 to 11 or 00 to 23, depending on am/pm) + m : the minutes without a leading 0 (0 to 59) + mm : the minutes with a leading 0 (00 to 59) + s : the seconds without a leading 0 (0 to 59) + ss : the seconds with a leading 0 (00 to 59) + ap : use a 12-hour clock instead of a 24-hour clock, with 'ap' replaced by the localized string for am or pm. + AP : use a 12-hour clock instead of a 24-hour clock, with 'AP' replaced by the localized string for AM or PM. iso : the date with time and timezone. Must be the only format present. * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables. diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 00190ac23d..e9db6e68ad 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -170,11 +170,37 @@ def format_date(dt, format, assume_utc=False, as_utc=False): if format == 'iso': return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc) + ampm = 'ap' in format.lower() + if dt == UNDEFINED_DATE: return '' strf = partial(strftime, t=dt.timetuple()) + def format_hour(hr): + l = len(hr) + h = dt.hour + if ampm: + h = h%12 + if l == 1: return '%d'%h + return '%02d'%h + + def format_minute(min): + l = len(min) + if l == 1: return '%d'%dt.minute + return '%02d'%dt.minute + + def format_second(min): + l = len(min) + if l == 1: return '%d'%dt.second + return '%02d'%dt.second + + def format_ampm(ap): + res = strf('%p') + if ap == 'AP': + return res + return res.lower() + def format_day(dy): l = len(dy) if l == 1: return '%d'%dt.day @@ -193,17 +219,23 @@ def format_date(dt, format, assume_utc=False, as_utc=False): if len(yr) == 2: return '%02d'%(dt.year % 100) return '%04d'%dt.year + function_index = { + 'd': format_day, + 'M': format_month, + 'y': format_year, + 'h': format_hour, + 'm': format_minute, + 's': format_second, + 'a': format_ampm, + 'A': format_ampm, + } def repl_func(mo): s = mo.group(0) if s is None: return '' - if s[0] == 'd': - return format_day(s) - if s[0] == 'M': - return format_month(s) - return format_year(s) + return function_index[s[0]](s) - return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format) + return re.sub('(s{1,2})|(m{1,2})|(h{1,2})|(ap)|(AP)|(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format) def replace_months(datestr, clang): # Replace months by english equivalent for parse_date diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 964120f550..d4958e5d10 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -747,6 +747,14 @@ class BuiltinFormatDate(BuiltinFormatterFunction): 'MMMM : the long localized month name (e.g. "January" to "December"). ' 'yy : the year as two digit number (00 to 99). ' 'yyyy : the year as four digit number. ' + 'h : the hours without a leading 0 (0 to 11 or 0 to 23, depending on am/pm) ' + 'hh : the minutes with a leading 0 (00 to 11 or 00 to 23, depending on am/pm) ' + 'm : the minutes without a leading 0 (0 to 59) ' + 'mm : the minutes with a leading 0 (00 to 59) ' + 's : the seconds without a leading 0 (0 to 59) ' + 'ss : the seconds with a leading 0 (00 to 59) ' + 'ap : use a 12-hour clock instead of a 24-hour clock, with "ap" replaced by the localized string for am or pm ' + 'AP : use a 12-hour clock instead of a 24-hour clock, with "AP" replaced by the localized string for AM or PM ' 'iso : the date with time and timezone. Must be the only format present') def evaluate(self, formatter, kwargs, mi, locals, val, format_string): From 96e4bf02c4524685add1c015f8b67b70c7460c6d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 30 Oct 2011 08:26:23 +0530 Subject: [PATCH 02/37] Fix #875196 (Cover search didn't resume after pressing Back) --- src/calibre/gui2/metadata/single_download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 255abe3999..70b32a78c6 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -587,7 +587,6 @@ class CoversModel(QAbstractListModel): # {{{ return 1 return pmap.width()*pmap.height() - def clear_failed(self): good = [] pmap = {} @@ -729,7 +728,8 @@ class CoversWidget(QWidget): # {{{ except Empty: break - self.covers_view.clear_failed() + if self.continue_processing: + self.covers_view.clear_failed() if self.worker.error is not None: error_dialog(self, _('Download failed'), @@ -759,7 +759,7 @@ class CoversWidget(QWidget): # {{{ self.continue_processing = False def cancel(self): - self.continue_processing = False + self.cleanup() self.abort.set() def cover_pixmap(self): From 228fdb8d94cbf6342f054e1800aa4bc98f8917b2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 30 Oct 2011 22:20:33 +0530 Subject: [PATCH 03/37] Updated Daily Mirror --- recipes/daily_mirror.recipe | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/recipes/daily_mirror.recipe b/recipes/daily_mirror.recipe index 5d4dbe3f4b..f0d28c72e7 100644 --- a/recipes/daily_mirror.recipe +++ b/recipes/daily_mirror.recipe @@ -1,10 +1,11 @@ from calibre.web.feeds.news import BasicNewsRecipe - +import re class AdvancedUserRecipe1306061239(BasicNewsRecipe): title = u'The Daily Mirror' description = 'News as provide by The Daily Mirror -UK' __author__ = 'Dave Asbury' + # last updated 30/10/11 language = 'en_GB' cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' @@ -12,26 +13,30 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): masthead_url = 'http://www.nmauk.co.uk/nma/images/daily_mirror.gif' - oldest_article = 1 - max_articles_per_feed = 100 + oldest_article = 2 + max_articles_per_feed = 30 remove_empty_feeds = True remove_javascript = True no_stylesheets = True + extra_css = ''' + body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} + ''' keep_only_tags = [ - dict(name='h1'), - dict(attrs={'class':['article-attr']}), - dict(name='div', attrs={'class' : [ 'article-body', 'crosshead']}) + dict(name='div',attrs={'id' : 'body-content'}) + ] - - ] + remove_tags_after = [dict (name='div',attrs={'class' : 'related'})] remove_tags = [ - dict(name='div', attrs={'class' : ['caption', 'article-resize']}), - dict( attrs={'class':'append-html'}) - ] - + dict(name='div',attrs={'id' : ['sidebar','menu','search-box','roffers-top']}), + dict(name='div',attrs={'class' :['inline-ad span-16 last','article-resize','related','list teasers']}), + dict(attrs={'class' : ['channellink','article-tags','replace','append-html']}), + dict(name='div',attrs={'class' : 'span-12 last sl-others addthis_toolbox addthis_default_style'}) + ] + preprocess_regexps = [ + (re.compile(r'
.*?
', re.IGNORECASE | re.DOTALL), lambda match: '')] feeds = [ @@ -43,10 +48,10 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): ,(u'Music News','http://www.mirror.co.uk/celebs/music/rss.xml') ,(u'Celebs and Tv Gossip','http://www.mirror.co.uk/celebs/tv/rss.xml') ,(u'Sport','http://www.mirror.co.uk/sport/rss.xml') - ,(u'Life Style','http://www.mirror.co.uk/life-style/rss.xml') - ,(u'Advice','http://www.mirror.co.uk/advice/rss.xml') - ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') + ,(u'Life Style','http://www.mirror.co.uk/life-style/rss.xml') + ,(u'Advice','http://www.mirror.co.uk/advice/rss.xml') + ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') # example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') - ] + ] From 35ec15f7d487de1caeaf2d781f2c8f296f9eaed0 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Sun, 30 Oct 2011 23:59:43 -0300 Subject: [PATCH 04/37] Fix longstanding bug that would prevent re-adding a epub that has been previously deleted from the Kobo using Calibre --- src/calibre/devices/kobo/driver.py | 48 +++++++++++++++++++----------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 479beed089..cbb3cf0181 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -61,18 +61,25 @@ class KOBO(USBMS): ' ebook file itself. With this option, calibre will send a ' 'separate cover image to the reader, useful if you ' 'have modified the cover.'), - _('Upload Black and White Covers') + _('Upload Black and White Covers'), + _('Show expired books') + + ':::'+_('A bug in an earlier version left non kepubs book records' + ' in the datbase. With this option Calibre will show the ' + 'expired records and allow you to delete them with ' + 'the new delete logic.'), ] EXTRA_CUSTOMIZATION_DEFAULT = [ ', '.join(['tags']), True, + True, True ] OPT_COLLECTIONS = 0 OPT_UPLOAD_COVERS = 1 OPT_UPLOAD_GRAYSCALE_COVERS = 2 + OPT_SHOW_EXPIRED_BOOK_RECORDS = 3 def initialize(self): USBMS.initialize(self) @@ -232,18 +239,23 @@ class KOBO(USBMS): self.dbversion = result[0] debug_print("Database Version: ", self.dbversion) + + opts = self.settings() if self.dbversion >= 16: - query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + query= _('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, Accessibility from content where ' \ - 'BookID is Null and ( ___ExpirationStatus <> "3" or ___ExpirationStatus is Null)' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') elif self.dbversion < 16 and self.dbversion >= 14: - query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + query= _('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, FavouritesIndex, "-1" as Accessibility from content where ' \ - 'BookID is Null and ( ___ExpirationStatus <> "3" or ___ExpirationStatus is Null)' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') elif self.dbversion < 14 and self.dbversion >= 8: - query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ + query= _('select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility from content where ' \ - 'BookID is Null and ( ___ExpirationStatus <> "3" or ___ExpirationStatus is Null)' + 'BookID is Null and not ((___ExpirationStatus=3 or ___ExpirationStatus is Null) %(expiry)s') % dict(expiry=' and ContentType = 6)' \ + if opts.extra_customization[self.OPT_SHOW_EXPIRED_BOOK_RECORDS] else ')') else: query= 'select Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ' \ 'ImageID, ReadStatus, "-1" as ___ExpirationStatus, "-1" as FavouritesIndex, "-1" as Accessibility from content where BookID is Null' @@ -343,21 +355,23 @@ class KOBO(USBMS): # Kobo does not delete the Book row (ie the row where the BookID is Null) # The next server sync should remove the row cursor.execute('delete from content where BookID = ?', t) - try: - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \ - 'where BookID is Null and ContentID =?',t) - except Exception as e: - if 'no such column' not in str(e): - raise + if ContentType == 6: try: - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0 ' \ + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0, ___ExpirationStatus=3 ' \ 'where BookID is Null and ContentID =?',t) except Exception as e: if 'no such column' not in str(e): raise - cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\' ' \ - 'where BookID is Null and ContentID =?',t) - + try: + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\', ___PercentRead=0 ' \ + 'where BookID is Null and ContentID =?',t) + except Exception as e: + if 'no such column' not in str(e): + raise + cursor.execute('update content set ReadStatus=0, FirstTimeReading = \'true\' ' \ + 'where BookID is Null and ContentID =?',t) + else: + cursor.execute('delete from content where BookID is Null and ContentID =?',t) connection.commit() From 6d2734de0d4c377dc25b29c18726ca145a97c919 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 31 Oct 2011 17:32:56 +0530 Subject: [PATCH 05/37] Fix #884039 (New Android Device Request) --- src/calibre/devices/android/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 80f3fcbf4a..4c8f39c32b 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -160,7 +160,8 @@ class ANDROID(USBMS): 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK', 'MB525', 'ANDROID2.3', 'SGH-I997', 'GT-I5800_CARD', 'MB612', 'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A', - 'ALPANDIGITAL', 'ANDROID_MID', 'VTAB1008', 'EMX51_BBG_ANDROI'] + 'ALPANDIGITAL', 'ANDROID_MID', 'VTAB1008', 'EMX51_BBG_ANDROI', + 'UMS'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', From 9744bd7400b74bb79fef943f17dd37d6c78ed840 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 31 Oct 2011 18:18:00 +0530 Subject: [PATCH 06/37] Fix #884194 (fixed ScienceNews recipe) --- recipes/science_news.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/science_news.recipe b/recipes/science_news.recipe index b1862e9112..01a01d1787 100644 --- a/recipes/science_news.recipe +++ b/recipes/science_news.recipe @@ -40,7 +40,7 @@ class Sciencenews(BasicNewsRecipe): ,dict(name='div', attrs={'class': 'embiggen'}) ] - feeds = [(u"Science News / News Items", u'http://sciencenews.org/view/feed/type/news/name/news.rss')] + feeds = [(u"Science News / News Items", u'http://sciencenews.org/index.php/feed/type/news/name/news.rss/view/feed/name/all.rss')] def get_cover_url(self): cover_url = None From 70dafe05cc87c723f151e9b403fbb827e9e3aed4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 07:41:52 +0530 Subject: [PATCH 07/37] Various Greek news sources by Stelios --- recipes/capital_gr.recipe | 35 ++++++++++++++++++++++++++++ recipes/in_gr.recipe | 34 +++++++++++++++++++++++++++ recipes/newsbeast.recipe | 48 +++++++++++++++++++++++++++++++++++++++ recipes/skai.recipe | 37 ++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+) create mode 100644 recipes/capital_gr.recipe create mode 100644 recipes/in_gr.recipe create mode 100644 recipes/newsbeast.recipe create mode 100644 recipes/skai.recipe diff --git a/recipes/capital_gr.recipe b/recipes/capital_gr.recipe new file mode 100644 index 0000000000..8dac48c95b --- /dev/null +++ b/recipes/capital_gr.recipe @@ -0,0 +1,35 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class Capital(BasicNewsRecipe): + title = 'Capital.gr' + __author__ ='Stelios' + description = 'Financial News from Greece' + #max_articles_per_feed = 100 + oldest_article = 3 + publisher = 'Capital.gr' + category = 'news, GR' + language = 'el' + encoding = 'windows-1253' + cover_url = 'http://files.capital.gr/images/caplogo.gif' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + keep_only_tags = [ + dict(name='h1'), + dict(name='p'), + dict(name='span', attrs={'id' : ["textbody"]}) + ] + +#3 posts seemed to have utf8 encoding + feeds = [ + (u'\u039F\u039B\u0395\u03A3 \u039F\u0399 \u0395\u0399\u0394\u0397\u03A3\u0395\u0399\u03A3', 'http://www.capital.gr/news/newsrss.asp?s=-1'), + (u'\u0395\u03A0\u0399\u03A7\u0395\u0399\u03A1\u0397\u03A3\u0395\u0399\u03A3', 'http://www.capital.gr/news/newsrss.asp?s=-2'), + (u'\u0391\u0393\u039F\u03A1\u0395\u03A3', 'http://www.capital.gr/news/newsrss.asp?s=-3'), + (u'\u039F\u0399\u039A\u039F\u039D\u039F\u039C\u0399\u0391', 'http://www.capital.gr/news/newsrss.asp?s=-4'), + (u'\u03A7\u03A1\u0397\u039C. \u0391\u039D\u0391\u039A\u039F\u0399\u039D\u03A9\u03A3\u0395\u0399\u03A3', 'http://www.capital.gr/news/newsrss.asp?s=-6'), + (u'\u039C\u03CC\u03BD\u03B9\u03BC\u03B5\u03C2 \u03C3\u03C4\u03AE\u03BB\u03B5\u03C2: \u039C\u0395 \u0391\u03A0\u039F\u03A8\u0397', 'http://www.capital.gr/articles/articlesrss.asp?catid=4'), + (u'\u039C\u03CC\u03BD\u03B9\u03BC\u03B5\u03C2 \u03C3\u03C4\u03AE\u03BB\u03B5\u03C2: \u03A3\u0399\u03A9\u03A0\u0397\u03A4\u0397\u03A1\u0399\u039F', 'http://www.capital.gr/articles/articlesrss.asp?catid=6'), + (u'\u039C\u03CC\u03BD\u03B9\u03BC\u03B5\u03C2 \u03C3\u03C4\u03AE\u03BB\u03B5\u03C2: \u03A0\u0399\u03A3\u03A9 \u0391\u03A0\u039F \u03A4\u0399\u03A3 \u0393\u03A1\u0391\u039C\u039C\u0395\u03A3', 'http://www.capital.gr/articles/articlesrss.asp?catid=8'), + #(u'\u039C\u03CC\u03BD\u03B9\u03BC\u03B5\u03C2 \u03C3\u03C4\u03AE\u03BB\u03B5\u03C2: \u03A4\u0395\u03A7\u039D\u039F\u039B\u039F\u0393\u0399\u0391', 'http://www.capital.gr/news/newsrss.asp?s=-8') not working for now +] + diff --git a/recipes/in_gr.recipe b/recipes/in_gr.recipe new file mode 100644 index 0000000000..68ad523c0f --- /dev/null +++ b/recipes/in_gr.recipe @@ -0,0 +1,34 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class ingr(BasicNewsRecipe): + title = 'in.gr' + __author__ = 'Stelios' + description = 'News from Greece' +# max_articles_per_feed = 100 + oldest_article = 4 + publisher = 'in.gr' + category = 'news, GR' + language = 'el' + encoding = 'utf8' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + encoding = 'utf8' + keep_only_tags = [ + dict(name='h1'), + + dict(name='div', attrs={'id' : ['in-news-article']}) + ] + remove_tags = [ +dict(name='em', attrs={'class' : ['credits']}), +dict(name='div', attrs={'class' : ['article-tools-hor', 'promo-banners gAds', 'main', 'article-listen-player', 'article-tools-hor-bttm', 'tools-sec', 'article-tools', 'article-listen-player-ver']}) +] + + + feeds = [ + (u'\u0395\u03BB\u03BB\u03AC\u03B4\u03B1', 'http://rss.in.gr/feed/news/greece'), + (u'\u0395\u03B9\u03B4\u03AE\u03C3\u03B5\u03B9\u03C2', 'http://rss.in.gr/feed/news'), + (u'\u039A\u03CC\u03C3\u03BC\u03BF\u03C2', 'http://rss.in.gr/feed/news/world'), + (u'\u0395\u03C0\u03B9\u03C3\u03C4\u03AE\u03BC\u03B7', 'http://rss.in.gr/feed/news/science'), + (u'\u03A0\u03BF\u03BB\u03B9\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://rss.in.gr/feed/news/culture') + ] diff --git a/recipes/newsbeast.recipe b/recipes/newsbeast.recipe new file mode 100644 index 0000000000..ed81af8fe3 --- /dev/null +++ b/recipes/newsbeast.recipe @@ -0,0 +1,48 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + + +class newsbeast(BasicNewsRecipe): + title = 'Newsbeast' + __author__ = 'Stelios' + description = 'News from Greece' + oldest_article = 2 + max_articles_per_feed = 100 + publisher = 'newsbeast' + category = 'news, GR' + language = 'el' + encoding = 'utf8' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + encoding = 'utf8' + keep_only_tags = [ + dict(name='div', attrs={'class' : ['article-title']}), + # dict(name='img', attrs={'class' : ['article_photo']}), + #If enabled feeds exceede 15MB + dict(name='div', attrs={'class' : ['txt']}) +] + remove_tags = [ + dict(name='table', attrs={'id':['artFoot']}), + dict(name='img'), + #If removed feeds exceede 15MB + dict(name='p', attrs={'class':['article-details']}) +] + + feeds = [ + (u'\u0395\u03BB\u03BB\u03AC\u03B4\u03B1', 'http://www.newsbeast.gr/feeds/greece'), + (u'\u039A\u03CC\u03C3\u03BC\u03BF\u03C2', 'http://www.newsbeast.gr/feeds/world'), + (u'\u03A0\u03BF\u03BB\u03B9\u03C4\u03B9\u03BA\u03AE', 'http://www.newsbeast.gr/feeds/politiki'), + (u'\u039F\u03B9\u03BA\u03BF\u03BD\u03BF\u03BC\u03AF\u03B1', 'http://www.newsbeast.gr/feeds/financial'), + (u'\u0391\u03B8\u03BB\u03B7\u03C4\u03B9\u03BA\u03AC', 'http://www.newsbeast.gr/feeds/sports'), + (u'\u039A\u03BF\u03B9\u03BD\u03C9\u03BD\u03AF\u03B1', 'http://www.newsbeast.gr/feeds/society'), + (u'\u03A0\u03B5\u03C1\u03B9\u03B2\u03AC\u03BB\u03BB\u03BF\u03BD', 'http://www.newsbeast.gr/feeds/environment'), + (u'Media', 'http://www.newsbeast.gr/feeds/media'), + (u'\u0394\u03B9\u03B1\u03C3\u03BA\u03AD\u03B4\u03B1\u03C3\u03B7', 'http://www.newsbeast.gr/feeds/entertainment'), + (u'Lifestyle', 'http://www.newsbeast.gr/feeds/lifestyle'), + (u'\u03A4\u03B5\u03C7\u03BD\u03BF\u03BB\u03BF\u03B3\u03AF\u03B1', 'http://www.newsbeast.gr/feeds/technology'), + (u'\u0391\u03C5\u03C4\u03BF\u03BA\u03AF\u03BD\u03B7\u03C4\u03BF', 'http://www.newsbeast.gr/feeds/car'), + (u'\u0393\u03C5\u03BD\u03B1\u03AF\u03BA\u03B1', 'http://www.newsbeast.gr/feeds/woman'), + (u'\u03A5\u03B3\u03B5\u03AF\u03B1', 'http://www.newsbeast.gr/feeds/health'), + (u'\u03A0\u03BF\u03BB\u03B9\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://www.newsbeast.gr/feeds/culture'), + (u'\u038C,\u03C4\u03B9 \u03BD\u03B1 \u03BD\u03B1\u03B9', 'http://www.newsbeast.gr/feeds/weird') + ] diff --git a/recipes/skai.recipe b/recipes/skai.recipe new file mode 100644 index 0000000000..e742845a78 --- /dev/null +++ b/recipes/skai.recipe @@ -0,0 +1,37 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + + +class SKAI(BasicNewsRecipe): + title = 'SKAI' + __author__ = 'Stelios' + description = 'News from Greece' + oldest_article = 2 + max_articles_per_feed = 100 + publisher = 'skai.gr' + category = 'news, GR' + language = 'el' + encoding = 'utf8' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + encoding = 'utf8' + keep_only_tags = [ + dict(name='h1'), + dict(name='div', attrs={'class' : ['articleText']}) +] + + + feeds = [ + (u'\u039A\u03C5\u03C1\u03B9\u03CC\u03C4\u03B5\u03C1\u03B5\u03C2 \u0395\u03B9\u03B4\u03AE\u03C3\u03B5\u03B9\u03C2', 'http://feeds.feedburner.com/skai/Uulu'), + (u'\u0395\u03BB\u03BB\u03AC\u03B4\u03B1', 'http://feeds.feedburner.com/skai/PLwa'), + (u'\u039A\u03CC\u03C3\u03BC\u03BF\u03C2', 'http://feeds.feedburner.com/skai/aqOL'), + (u'\u03A0\u03BF\u03BB\u03B9\u03C4\u03B9\u03BA\u03AE','http://feeds.feedburner.com/skai/yinm'), + (u'\u039F\u03B9\u03BA\u03BF\u03BD\u03BF\u03BC\u03AF\u03B1', 'http://feeds.feedburner.com/skai/oPUt'), + (u'\u03A4\u03B5\u03C7\u03BD\u03BF\u03BB\u03BF\u03B3\u03AF\u03B1', 'http://feeds.feedburner.com/skai/fqsg'), + (u'\u0391\u03B8\u03BB\u03B7\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://feeds.feedburner.com/skai/TfmK'), + (u'\u03A5\u03B3\u03B5\u03AF\u03B1', 'http://feeds.feedburner.com/skai/TABn'), + (u'\u03A0\u03BF\u03BB\u03B9\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://feeds.feedburner.com/skai/ppGl'), + (u'\u0391\u03C5\u03C4\u03BF\u03BA\u03AF\u03BD\u03B7\u03C3\u03B7', 'http://feeds.feedburner.com/skai/HCCc'), + (u'\u03A0\u03B5\u03C1\u03B9\u03B2\u03AC\u03BB\u03BB\u03BF\u03BD', 'http://feeds.feedburner.com/skai/jVWs'), + (u'\u03A0\u03B1\u03C1\u03AC\u03BE\u03B5\u03BD\u03B1', 'http://feeds.feedburner.com/skai/bpAR') +] From 8b0b1ca357b3a8a6d91960ea34376c12518b8958 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 08:07:00 +0530 Subject: [PATCH 08/37] ... --- recipes/tovima.recipe | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 recipes/tovima.recipe diff --git a/recipes/tovima.recipe b/recipes/tovima.recipe new file mode 100644 index 0000000000..ba7b50afb8 --- /dev/null +++ b/recipes/tovima.recipe @@ -0,0 +1,39 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class Tovima(BasicNewsRecipe): + title = 'To Vima' + __author__ = 'Stelios' + description = ' News from Greece' + #max_articles_per_feed = 100 + oldest_article = 3 + publisher = 'To Vima' + category = 'news, GR' + language = 'el' + encoding = 'utf8' + cover_url = 'http://www.tovima.gr/Themes/1/Default/Media/Home//small-n-short-logo.jpg' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + extra_css = ''' + .article_title{font-family :Arial,Helvetica,sans-serif; font-weight: bold; font-size:large;} + .article_text{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + ''' + keep_only_tags = [ + + dict(name='div', attrs={'class' : ['article_title']}), + dict(name='div', attrs={'class' : ['article_text']}) + ] + remove_tags = [ + dict(name='div', attrs={'class' : ['article_cat']}) + ] + feeds = [ + (u'\u03C0\u03BF\u03BB\u03B9\u03C4\u03B9\u03BA\u03AE', 'http://www.tovima.gr/feed/politics/'), + (u'\u03BF\u03B9\u03BA\u03BF\u03BD\u03BF\u03BC\u03AF\u03B1', 'http://www.tovima.gr/feed/finance/'), + (u'\u03B3\u03BD\u03CE\u03BC\u03B5\u03C2', 'http://www.tovima.gr/feed/opinions/'), + (u'blogs', 'http://www.tovima.gr/feed/blogs/'), + (u'\u03BA\u03CC\u03C3\u03BC\u03BF\u03C2','http://www.tovima.gr/feed/world/'), + (u'science', 'http://www.tovima.gr/feed/science/'), + (u'\u03BA\u03BF\u03B9\u03BD\u03C9\u03BD\u03AF\u03B1', 'http://www.tovima.gr/feed/society/'), + (u'\u03C0\u03BF\u03BB\u03B9\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://www.tovima.gr/feed/culture/'), + (u'\u03B1\u03B8\u03BB\u03B7\u03C4\u03B9\u03C3\u03BC\u03CC\u03C2', 'http://www.tovima.gr/feed/sports/') +] From a8b3896c674263ea026a8d924c517776728d186b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 08:07:54 +0530 Subject: [PATCH 09/37] Fix #884266 (Updated recipe for NIN online) --- recipes/nin.recipe | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/recipes/nin.recipe b/recipes/nin.recipe index 66dd58330e..33c7a683be 100644 --- a/recipes/nin.recipe +++ b/recipes/nin.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2008-2010, Darko Miletic ' +__copyright__ = '2008-2011, Darko Miletic ' ''' www.nin.co.rs ''' @@ -29,6 +29,7 @@ class Nin(BasicNewsRecipe): use_embedded_content = False language = 'sr' publication_type = 'magazine' + masthead_url = 'http://www.nin.co.rs/img/head/logo.jpg' extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Verdana, Lucida, sans1, sans-serif} @@ -72,9 +73,11 @@ class Nin(BasicNewsRecipe): def get_cover_url(self): cover_url = None soup = self.index_to_soup(self.INDEX) - link_item = soup.find('img',attrs={'width':'100','border':'0'}) - if link_item: - cover_url = self.PREFIX + link_item['src'] + for item in soup.findAll('a', href=True): + if item['href'].startswith('/pages/issue.php?id='): + simg = item.find('img') + if simg: + return self.PREFIX + item.img['src'] return cover_url def parse_index(self): From ab4562c0e6ced333056b98321b92ac0e2f39877d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 08:56:48 +0530 Subject: [PATCH 10/37] T1 driver: Workaround for T1 showing error messages when opening some news downloads on the device --- src/calibre/devices/prst1/driver.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 342a32bb37..737371e245 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -560,14 +560,21 @@ class PRST1(USBMS): cursor = connection.cursor() + periodical_schema = \ + "'http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0'" + # Setting this to the SONY periodical schema apparently causes errors + # with some periodicals, therefore set it to null, since the special + # periodical navigation doesn't work anyway. + periodical_schema = 'null' + query = ''' UPDATE books - SET conforms_to = 'http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0', + SET conforms_to = %s, periodical_name = ?, description = ?, publication_date = ? WHERE _id = ? - ''' + '''%periodical_schema t = (name, None, pubdate, book.bookId,) cursor.execute(query, t) From 07c1176791f856d8b36db22c5e517dd0ed13df4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 09:00:56 +0530 Subject: [PATCH 11/37] ... --- recipes/men24_gr.recipe | 43 +++++++++++++++++++++++++++++++++++++++++ recipes/protagon.recipe | 26 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 recipes/men24_gr.recipe create mode 100644 recipes/protagon.recipe diff --git a/recipes/men24_gr.recipe b/recipes/men24_gr.recipe new file mode 100644 index 0000000000..8ce68e6c86 --- /dev/null +++ b/recipes/men24_gr.recipe @@ -0,0 +1,43 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + +class Men24(BasicNewsRecipe): + title = 'Men24.gr' + __author__ = 'Stelios' + description = 'Greek Mens portal' + oldest_article = 14 + max_articles_per_feed = 100 + language = 'el' + cover_url = 'http://www.men24.gr/ast/img/men24Logo.jpg' + category = 'magazines, GR' + language = 'el' + encoding = 'windows-1253' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + extra_css = ''' + .artPrintTitle{font-family :Arial,Helvetica,sans-serif; font-weight: bold; font-size:large;} + .artPrintSubtitle{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + ''' + remove_tags = [ + dict(name='td', attrs={'class':['artPrintCategory']}), + dict(name='table', attrs={'class':['footer']}), + dict(name='img') +] + feeds = [ + (u'\u038C\u03BB\u03B5\u03C2 \u03BF\u03B9 \u03B5\u03B9\u03B4\u03AE\u03C3\u03B5\u03B9\u03C2', 'http://www.men24.gr/svc/rss/lastNews/'), + (u'\u03A3\u03C4\u03C5\u03BB', 'http://www.men24.gr/svc/rss/categoryNews/?category=style'), + (u'Fitness', 'http://www.men24.gr/svc/rss/categoryNews/?category=fitness'), + (u'Gadgets', 'http://www.men24.gr/svc/rss/categoryNews/?category=gadgets'), + (u'\u0394\u03B9\u03B1\u03C3\u03BA\u03AD\u03B4\u03B1\u03C3\u03B7', 'http://www.men24.gr/svc/rss/categoryNews/?category=fun'), + (u'\u03A7\u03C1\u03AE\u03BC\u03B1 \u03BA\u03B1\u03B9 \u039A\u03B1\u03C1\u03B9\u03AD\u03C1\u03B1', 'http://www.men24.gr/svc/rss/categoryNews/?category=money'), + (u'Special Edition', 'http://www.men24.gr/svc/rss/categoryNews/?category=special'), + (u'\u0388\u03C1\u03C9\u03C4\u03B1\u03C2 \u03BA\u03B1\u03B9 Sex', 'http://www.men24.gr/svc/rss/categoryNews/?category=love'), + (u'\u0386\u03BD\u03C4\u03C1\u03B5\u03C2 \u03C4\u03BF\u03C5 24', 'http://www.men24.gr/svc/rss/categoryNews/?category=men'), + (u'\u0393\u03C5\u03BD\u03B1\u03AF\u03BA\u03B5\u03C2', 'http://www.men24.gr/svc/rss/categoryNews/?category=women'), + (u'\u039F\u03B4\u03B7\u03B3\u03BF\u03AF', 'http://www.men24.gr/svc/rss/categoryNews/?category=guides'), + (u'\u03A4\u03B6\u03CC\u03B3\u03BF\u03C2', 'http://www.men24.gr/svc/rss/categoryNews/?category=gamble') + +] + + def print_version(self, url): + return url.replace('.asp', '.print.asp') diff --git a/recipes/protagon.recipe b/recipes/protagon.recipe new file mode 100644 index 0000000000..6f537890a8 --- /dev/null +++ b/recipes/protagon.recipe @@ -0,0 +1,26 @@ +from calibre.web.feeds.recipes import BasicNewsRecipe + + +class protagon(BasicNewsRecipe): + title = 'Protagon' + __author__ = 'Stelios' + description = 'Opinion articles in Greek' + oldest_article = 7 + max_articles_per_feed = 100 + publisher = 'Various' + category = 'GR' + language = 'el' + encoding = 'utf8' + no_stylesheets = True + use_embedded_content = False + remove_empty_feeds = True + + keep_only_tags = [ + dict(name='h1', attrs={'id' : ['title']}), + dict(name='div', attrs={'class' : ['freetext']}) +] + + feeds = [ + (u'\u0398\u03AD\u03BC\u03B1\u03C4\u03B1', 'http://www.protagon.gr/rss?i=protagon.el.8emata') +] + From b8691e801070ba6637958071bc4d3e81a451a55a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 Nov 2011 09:07:54 +0530 Subject: [PATCH 12/37] ... --- recipes/deutsche_welle_es.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/deutsche_welle_es.recipe b/recipes/deutsche_welle_es.recipe index c68a03b981..7cf58b0a55 100644 --- a/recipes/deutsche_welle_es.recipe +++ b/recipes/deutsche_welle_es.recipe @@ -16,7 +16,7 @@ class DeutscheWelle_es(BasicNewsRecipe): max_articles_per_feed = 100 use_embedded_content = False no_stylesheets = True - language = 'de_ES' + language = 'de' publication_type = 'newsportal' remove_empty_feeds = True masthead_url = 'http://www.dw-world.de/skins/std/channel1/pics/dw_logo1024.gif' From 1e07787d026a4c6799392d53a1a607440bcb149f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 08:54:26 +0530 Subject: [PATCH 13/37] Driver for Kobo Vox. Fixes #884762 (Calibre support for kobo vox) --- src/calibre/devices/android/driver.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 4c8f39c32b..a637eca03e 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -138,8 +138,12 @@ class ANDROID(USBMS): # Advent 0x0955 : { 0x7100 : [0x9999] }, # This is the same as the Notion Ink Adam + # Kobo + 0x2237: { 0x2208 : [0x0226] }, + } - EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books'] + EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books', + 'sdcard/ebooks'] EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' 'send e-books to on the device. The first one that exists will ' 'be used') @@ -149,7 +153,7 @@ class ANDROID(USBMS): 'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS', 'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA', 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', - 'VIZIO', 'GOOGLE', 'FREESCAL'] + 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -161,7 +165,7 @@ class ANDROID(USBMS): 'MB525', 'ANDROID2.3', 'SGH-I997', 'GT-I5800_CARD', 'MB612', 'GT-S5830_CARD', 'GT-S5570_CARD', 'MB870', 'MID7015A', 'ALPANDIGITAL', 'ANDROID_MID', 'VTAB1008', 'EMX51_BBG_ANDROI', - 'UMS'] + 'UMS', '.K080'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', From 0600e911f1dde5b00c195c12917caaadb947cd6f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 09:14:21 +0530 Subject: [PATCH 14/37] HTML Input: Limit emory consumption when converting HTML files that link to lage binary files. Fixes #884821 (python terminated) --- src/calibre/ebooks/html/input.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 3de116c411..3688668bfe 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -109,14 +109,16 @@ class HTMLFile(object): try: with open(self.path, 'rb') as f: - src = f.read() + src = f.read(4096) + self.is_binary = level > 0 and not bool(self.HTML_PAT.search(src)) + if not self.is_binary: + src += f.read() except IOError as err: msg = 'Could not read from file: %s with error: %s'%(self.path, as_unicode(err)) if level == 0: raise IOError(msg) raise IgnoreFile(msg, err.errno) - self.is_binary = level > 0 and not bool(self.HTML_PAT.search(src[:4096])) if not self.is_binary: if not encoding: encoding = xml_to_unicode(src[:4096], verbose=verbose)[-1] From 55794f2d9de05f5eb17e6ee5f1d8a63f32a179f5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 09:15:45 +0530 Subject: [PATCH 15/37] Do not load QtWebkit when running a conversion that does not output to PDF --- src/calibre/ebooks/pdf/output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/output.py b/src/calibre/ebooks/pdf/output.py index 24050abf6a..0ea1c52e2e 100644 --- a/src/calibre/ebooks/pdf/output.py +++ b/src/calibre/ebooks/pdf/output.py @@ -15,7 +15,6 @@ from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation from calibre.ebooks.metadata.opf2 import OPF from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.pdf.writer import PDFWriter, ImagePDFWriter, PDFMetadata from calibre.ebooks.pdf.pageoptions import UNITS, PAPER_SIZES, \ ORIENTATIONS @@ -90,6 +89,7 @@ class PDFOutput(OutputFormatPlugin): self.convert_text(oeb_book) def convert_images(self, images): + from calibre.ebooks.pdf.writer import ImagePDFWriter self.write(ImagePDFWriter, images) def get_cover_data(self): @@ -105,6 +105,7 @@ class PDFOutput(OutputFormatPlugin): self.cover_data = None def convert_text(self, oeb_book): + from calibre.ebooks.pdf.writer import PDFWriter self.log.debug('Serializing oeb input to disk for processing...') self.get_cover_data() @@ -119,6 +120,7 @@ class PDFOutput(OutputFormatPlugin): self.write(PDFWriter, [s.path for s in opf.spine]) def write(self, Writer, items): + from calibre.ebooks.pdf.writer import PDFMetadata writer = Writer(self.opts, self.log, cover_data=self.cover_data) close = False From b44bb37f622ccbf4772501d47060fd568717eff5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 09:33:03 +0530 Subject: [PATCH 16/37] Fix #885058 (Not detecting HTC Incredible S Android phone) --- src/calibre/devices/android/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a637eca03e..5b707a9cd1 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -36,7 +36,8 @@ class ANDROID(USBMS): 0xca2 : [0x100, 0x0227, 0x0226, 0x222], 0xca3 : [0x100, 0x0227, 0x0226, 0x222], 0xca4 : [0x100, 0x0227, 0x0226, 0x222], - 0xca9 : [0x100, 0x0227, 0x0226, 0x222] + 0xca9 : [0x100, 0x0227, 0x0226, 0x222], + 0xcac : [0x100, 0x0227, 0x0226, 0x222], }, # Eken From ad67152628815922a2d8b8491a559b2b453b544c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 11:12:34 +0530 Subject: [PATCH 17/37] Fix #885027 (Private bug) --- src/calibre/devices/linux_mount_helper.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 550510106e..cf22e4112f 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -211,6 +211,15 @@ int main(int argc, char** argv) } action = argv[1]; dev = argv[2]; mp = argv[3]; + /* Ensure that PATH only contains system directories to prevent execution of + arbitrary executables as root */ + if (setenv("PATH", + "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin\0", + 1) != 0) { + fprintf(stderr, "Failed to restrict PATH env var, aborting.\n"); + exit(EXIT_FAILURE); + } + if (strncmp(action, "mount", 5) == 0) { status = do_mount(dev, mp); } else if (strncmp(action, "eject", 5) == 0) { From 4393d91aef25e931ed1f46b61d2e9b63cfebccc5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 16:00:30 +0530 Subject: [PATCH 18/37] Fix #884828 (Wrong country name Czechoslovakia) --- src/calibre/utils/localization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index a4f0aba73b..925e8847be 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -121,7 +121,7 @@ _extra_lang_codes = { 'en_TH' : _('English (Thailand)'), 'en_TR' : _('English (Turkey)'), 'en_CY' : _('English (Cyprus)'), - 'en_CZ' : _('English (Czechoslovakia)'), + 'en_CZ' : _('English (Czech Republic)'), 'en_PK' : _('English (Pakistan)'), 'en_HR' : _('English (Croatia)'), 'en_ID' : _('English (Indonesia)'), From 8b21b4a4ad105047d30e51734f1e28f8e9a0f58c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Nov 2011 18:12:34 +0530 Subject: [PATCH 19/37] calibre-mount-helper: Refuse to mount anythin under /usr, /bin or /sbin --- src/calibre/devices/linux_mount_helper.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index cf22e4112f..79c974f915 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -44,6 +44,12 @@ int do_mount(const char *dev, const char *mp) { fprintf(stderr, "Specified device node does not exist\n"); return EXIT_FAILURE; } + + if (strncmp("/usr", mp, 4) == 0 || strncmp("/bin", mp, 4) == 0 || strncmp("/sbin", mp, 5) == 0) { + fprintf(stderr, "Trying to mount to a mount point under /usr, /bin, /sbin is not allowed\n"); + return EXIT_FAILURE; + } + if (!exists(mp)) { if (mkdir(mp, S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) != 0) { errsv = errno; From eb2a40ffef0876e0618ca7bb39170d6922b6354f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 07:22:44 +0530 Subject: [PATCH 20/37] calibre-mount-helper: Validate dev arg as well as mount --- src/calibre/devices/linux_mount_helper.c | 26 +++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 79c974f915..9fda0e9401 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -33,6 +33,25 @@ void ensure_root() { } } +int check_args(const char *dev, const char *mp) { + if (dev == NULL || strlen(dev) < strlen("/dev/") || mp == NULL || strlen(mp) < strlen("/media/")) { + fprintf(stderr, "Invalid arguments\n"); + return False; + } + + if (strncmp("/media/", mp, 6) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + return False; + } + + if (strncmp("/dev/", dev, 5) != 0) { + fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); + return False; + } + + return True; +} + int do_mount(const char *dev, const char *mp) { char options[1000], marker[2000]; #ifdef __NetBSD__ @@ -45,11 +64,6 @@ int do_mount(const char *dev, const char *mp) { return EXIT_FAILURE; } - if (strncmp("/usr", mp, 4) == 0 || strncmp("/bin", mp, 4) == 0 || strncmp("/sbin", mp, 5) == 0) { - fprintf(stderr, "Trying to mount to a mount point under /usr, /bin, /sbin is not allowed\n"); - return EXIT_FAILURE; - } - if (!exists(mp)) { if (mkdir(mp, S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) != 0) { errsv = errno; @@ -226,6 +240,8 @@ int main(int argc, char** argv) exit(EXIT_FAILURE); } + if (!check_args(dev, mp)) exit(EXIT_FAILURE); + if (strncmp(action, "mount", 5) == 0) { status = do_mount(dev, mp); } else if (strncmp(action, "eject", 5) == 0) { From f64a88cd37aeeb9e63b1e78fb5bd5139bd08dfef Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 07:27:41 +0530 Subject: [PATCH 21/37] ... --- src/calibre/devices/linux_mount_helper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 9fda0e9401..badcf0ac54 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -39,12 +39,12 @@ int check_args(const char *dev, const char *mp) { return False; } - if (strncmp("/media/", mp, 6) != 0) { + if (strncmp("/media/", mp, strlen("/media/")) != 0) { fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); return False; } - if (strncmp("/dev/", dev, 5) != 0) { + if (strncmp("/dev/", dev, strlen("/dev/")) != 0) { fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); return False; } From 646071ecb02ac9a77d2073a7d1168afcbd9ea787 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 07:31:39 +0530 Subject: [PATCH 22/37] ... --- src/calibre/devices/linux_mount_helper.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index badcf0ac54..e470a69ca3 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -9,6 +9,8 @@ #include #define MARKER ".created_by_calibre_mount_helper" +#define DEV "/dev/" +#define MEDIA "/media/" #define False 0 #define True 1 @@ -34,17 +36,17 @@ void ensure_root() { } int check_args(const char *dev, const char *mp) { - if (dev == NULL || strlen(dev) < strlen("/dev/") || mp == NULL || strlen(mp) < strlen("/media/")) { + if (dev == NULL || strlen(dev) < strlen(DEV) || mp == NULL || strlen(mp) < strlen(MEDIA)) { fprintf(stderr, "Invalid arguments\n"); return False; } - if (strncmp("/media/", mp, strlen("/media/")) != 0) { + if (strncmp(MEDIA, mp, strlen("MEDIA")) != 0) { fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); return False; } - if (strncmp("/dev/", dev, strlen("/dev/")) != 0) { + if (strncmp(DEV, dev, strlen(DEV)) != 0) { fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); return False; } From 543c42d9b599acd627b987b8723dd66156aa71cc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 07:54:23 +0530 Subject: [PATCH 23/37] ... --- recipes/sigma_live.recipe | 14 ++++++++++++++ recipes/zougla.recipe | 12 ++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 recipes/sigma_live.recipe create mode 100644 recipes/zougla.recipe diff --git a/recipes/sigma_live.recipe b/recipes/sigma_live.recipe new file mode 100644 index 0000000000..44da1d8180 --- /dev/null +++ b/recipes/sigma_live.recipe @@ -0,0 +1,14 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class sigmalive(BasicNewsRecipe): + title = u'SigmaLive' + __author__ = 'Stelios' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + category = 'news, CY' + description = 'Cypriot News' + language = 'el' + encoding = 'utf8' + feeds = [(u'sigmalive', u'http://sigmalive.com/rss/latest')] + diff --git a/recipes/zougla.recipe b/recipes/zougla.recipe new file mode 100644 index 0000000000..72175f6a98 --- /dev/null +++ b/recipes/zougla.recipe @@ -0,0 +1,12 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1320264153(BasicNewsRecipe): + title = u'zougla' + __author__ = 'Stelios' + language = 'el' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + feeds = [(u'zougla', u'http://www.zougla.gr/ArticleRss.xml')] + From d43d661830ea1422160c2b80b02466dcad70e0cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 08:10:28 +0530 Subject: [PATCH 24/37] Fix #885332 (Incorrent URL with --url-prefix) --- src/calibre/library/server/mobile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index 0cb7a86126..a6b5c389de 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -156,7 +156,7 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS, body.append(HR()) body.append(DIV( A(_('Switch to the full interface (non-mobile interface)'), - href="/browse", + href=prefix+"/browse", style="text-decoration: none; color: blue", title=_('The full interface gives you many more features, ' 'but it may not work well on a small screen')), From 99e686faf5ad15c9b0d51a11e148f7d9571ba722 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 08:33:19 +0530 Subject: [PATCH 25/37] Forgot to use realpath() --- src/calibre/devices/linux_mount_helper.c | 23 +++++++++++++++++++++-- src/calibre/ptempfile.py | 4 ++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index e470a69ca3..bdad985d64 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -36,17 +37,35 @@ void ensure_root() { } int check_args(const char *dev, const char *mp) { + char buffer[PATH_MAX+1]; + if (dev == NULL || strlen(dev) < strlen(DEV) || mp == NULL || strlen(mp) < strlen(MEDIA)) { fprintf(stderr, "Invalid arguments\n"); return False; } - if (strncmp(MEDIA, mp, strlen("MEDIA")) != 0) { + if (exists(mp)) { + if (realpath(mp, buffer) == NULL) { + fprintf(stderr, "Unable to resolve mount path\n"); + return False; + } + if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + return False; + } + } + + if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); return False; } - if (strncmp(DEV, dev, strlen(DEV)) != 0) { + if (realpath(dev, buffer) == NULL) { + fprintf(stderr, "Unable to resolve dev path\n"); + return False; + } + + if (strncmp(DEV, buffer, strlen(DEV)) != 0) { fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); return False; } diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index aca8397f53..9343c88efa 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -194,4 +194,8 @@ class SpooledTemporaryFile(tempfile.SpooledTemporaryFile): tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix, prefix=prefix, dir=dir, mode=mode, bufsize=bufsize) +def better_mktemp(*args, **kwargs): + fd, path = tempfile.mkstemp(*args, **kwargs) + os.close(fd) + return path From ab964a5c70e48fc7577c662b2e5c5bb6889891e8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 3 Nov 2011 10:30:44 +0100 Subject: [PATCH 26/37] Make composite tags-like column names to appear in the look&feel hierarchical fields dropdown list. --- src/calibre/gui2/preferences/look_feel.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index c017fe69c2..06ec5392c9 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -146,9 +146,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('default_author_link', gprefs) choices = set([k for k in db.field_metadata.all_field_keys() - if db.field_metadata[k]['is_category'] and + if (db.field_metadata[k]['is_category'] and (db.field_metadata[k]['datatype'] in ['text', 'series', 'enumeration']) and - not db.field_metadata[k]['display'].get('is_names', False)]) + not db.field_metadata[k]['display'].get('is_names', False)) + or + (db.field_metadata[k]['datatype'] in ['composite'] and + db.field_metadata[k]['display'].get('make_category', False))]) choices -= set(['authors', 'publisher', 'formats', 'news', 'identifiers']) choices |= set(['search']) self.opt_categories_using_hierarchy.update_items_cache(choices) From 965a2b22a47ff615af07b65eb59a12d486b8f008 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 16:13:50 +0530 Subject: [PATCH 27/37] calibre-mount-helper: Stymie timing based attacks using symlinks in /dev --- src/calibre/devices/linux_mount_helper.c | 97 ++++++++++++++---------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index bdad985d64..73aadb54d4 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -36,43 +36,6 @@ void ensure_root() { } } -int check_args(const char *dev, const char *mp) { - char buffer[PATH_MAX+1]; - - if (dev == NULL || strlen(dev) < strlen(DEV) || mp == NULL || strlen(mp) < strlen(MEDIA)) { - fprintf(stderr, "Invalid arguments\n"); - return False; - } - - if (exists(mp)) { - if (realpath(mp, buffer) == NULL) { - fprintf(stderr, "Unable to resolve mount path\n"); - return False; - } - if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { - fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); - return False; - } - } - - if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { - fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); - return False; - } - - if (realpath(dev, buffer) == NULL) { - fprintf(stderr, "Unable to resolve dev path\n"); - return False; - } - - if (strncmp(DEV, buffer, strlen(DEV)) != 0) { - fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); - return False; - } - - return True; -} - int do_mount(const char *dev, const char *mp) { char options[1000], marker[2000]; #ifdef __NetBSD__ @@ -236,6 +199,51 @@ int cleanup(const char *dev, const char *mp) { return cleanup_mount_point(mp); } +void check_dev(const char*dev) { + char buffer[PATH_MAX+1]; + + if (dev == NULL || strlen(dev) < strlen(DEV)) { + fprintf(stderr, "Invalid arguments\n"); + exit(EXIT_FAILURE); + } + + if (realpath(dev, buffer) == NULL) { + fprintf(stderr, "Unable to resolve dev path\n"); + exit(EXIT_FAILURE); + } + + if (strncmp(DEV, buffer, strlen(DEV)) != 0) { + fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); + exit(EXIT_FAILURE); + } +} + +void check_mount_point(const char *mp) { + char buffer[PATH_MAX+1]; + + if (mp == NULL || strlen(mp) < strlen(MEDIA)) { + fprintf(stderr, "Invalid arguments\n"); + exit(EXIT_FAILURE); + } + + if (exists(mp)) { + if (realpath(mp, buffer) == NULL) { + fprintf(stderr, "Unable to resolve mount path\n"); + exit(EXIT_FAILURE); + } + if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + exit(EXIT_FAILURE); + } + } + + if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + exit(EXIT_FAILURE); + } + +} + int main(int argc, char** argv) { char *action, *dev, *mp; @@ -261,13 +269,24 @@ int main(int argc, char** argv) exit(EXIT_FAILURE); } - if (!check_args(dev, mp)) exit(EXIT_FAILURE); - if (strncmp(action, "mount", 5) == 0) { + dev = realpath(argv[2], NULL); + if (dev == NULL) { + fprintf(stderr, "Failed to resolve device node.\n"); + exit(EXIT_FAILURE); + } + check_dev(dev); check_mount_point(mp); status = do_mount(dev, mp); } else if (strncmp(action, "eject", 5) == 0) { + dev = realpath(argv[2], NULL); + if (dev == NULL) { + fprintf(stderr, "Failed to resolve device node.\n"); + exit(EXIT_FAILURE); + } + check_dev(dev); check_mount_point(mp); status = do_eject(dev, mp); } else if (strncmp(action, "cleanup", 7) == 0) { + check_mount_point(mp); status = cleanup(dev, mp); } else { fprintf(stderr, "Unrecognized action: must be mount, eject or cleanup\n"); From 805e968ebccb4362149eb1da0d45abf8a096cf82 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 18:08:00 +0530 Subject: [PATCH 28/37] ... --- src/calibre/devices/linux_mount_helper.c | 79 ++++++++++++++++-------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 73aadb54d4..7829d43fad 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -36,6 +36,32 @@ void ensure_root() { } } +void check_mount_point(const char *mp) { + char buffer[PATH_MAX+1]; + + if (mp == NULL || strlen(mp) < strlen(MEDIA)) { + fprintf(stderr, "Invalid arguments\n"); + exit(EXIT_FAILURE); + } + + if (exists(mp)) { + if (realpath(mp, buffer) == NULL) { + fprintf(stderr, "Unable to resolve mount path\n"); + exit(EXIT_FAILURE); + } + if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + exit(EXIT_FAILURE); + } + } + + if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { + fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); + exit(EXIT_FAILURE); + } + +} + int do_mount(const char *dev, const char *mp) { char options[1000], marker[2000]; #ifdef __NetBSD__ @@ -54,6 +80,17 @@ int do_mount(const char *dev, const char *mp) { fprintf(stderr, "Failed to create mount point with error: %s\n", strerror(errsv)); } } + /* only mount if mp is under /media */ + mp = realpath(mp, NULL); + if (mp == NULL) { + fprintf(stderr, "realpath on mp failed.\n"); + exit(EXIT_FAILURE); + } + if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { + fprintf(stderr, "mount point is not under /media\n"); + exit(EXIT_FAILURE); + } + snprintf(marker, 2000, "%s/%s", mp, MARKER); if (!exists(marker)) { int fd = creat(marker, S_IRUSR|S_IWUSR); @@ -218,35 +255,9 @@ void check_dev(const char*dev) { } } -void check_mount_point(const char *mp) { - char buffer[PATH_MAX+1]; - - if (mp == NULL || strlen(mp) < strlen(MEDIA)) { - fprintf(stderr, "Invalid arguments\n"); - exit(EXIT_FAILURE); - } - - if (exists(mp)) { - if (realpath(mp, buffer) == NULL) { - fprintf(stderr, "Unable to resolve mount path\n"); - exit(EXIT_FAILURE); - } - if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { - fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); - exit(EXIT_FAILURE); - } - } - - if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { - fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); - exit(EXIT_FAILURE); - } - -} - int main(int argc, char** argv) { - char *action, *dev, *mp; + char *action, *dev, *mp, *temp; int status = EXIT_FAILURE; /*printf("Real UID\t= %d\n", getuid()); @@ -275,6 +286,8 @@ int main(int argc, char** argv) fprintf(stderr, "Failed to resolve device node.\n"); exit(EXIT_FAILURE); } + temp = realpath(mp, NULL); + if (temp != NULL) mp = temp; check_dev(dev); check_mount_point(mp); status = do_mount(dev, mp); } else if (strncmp(action, "eject", 5) == 0) { @@ -283,9 +296,21 @@ int main(int argc, char** argv) fprintf(stderr, "Failed to resolve device node.\n"); exit(EXIT_FAILURE); } + temp = realpath(mp, NULL); + if (temp == NULL) { + fprintf(stderr, "Mount point does not exist\n"); + exit(EXIT_FAILURE); + } + mp = temp; check_dev(dev); check_mount_point(mp); status = do_eject(dev, mp); } else if (strncmp(action, "cleanup", 7) == 0) { + temp = realpath(mp, NULL); + if (temp == NULL) { + fprintf(stderr, "Mount point does not exist\n"); + exit(EXIT_FAILURE); + } + mp = temp; check_mount_point(mp); status = cleanup(dev, mp); } else { From b581b47ff1ca74167ef15b20cecb54bf3e6b66ea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 3 Nov 2011 18:27:19 +0530 Subject: [PATCH 29/37] Fix Die Zeit subscription version --- recipes/zeitde_sub.recipe | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/recipes/zeitde_sub.recipe b/recipes/zeitde_sub.recipe index 25fe0f5b23..583e86e7ac 100644 --- a/recipes/zeitde_sub.recipe +++ b/recipes/zeitde_sub.recipe @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 mode: python -*- __license__ = 'GPL v3' @@ -123,6 +123,9 @@ class ZeitEPUBAbo(BasicNewsRecipe): # new login process response = browser.open(url) + # Get rid of nested form + response.set_data(response.get_data().replace('
', '')) + browser.set_response(response) browser.select_form(nr=2) browser.form['name']=self.username browser.form['pass']=self.password @@ -178,7 +181,11 @@ class ZeitEPUBAbo(BasicNewsRecipe): browser = self.get_browser() # new login process - browser.open(url) + response=browser.open(url) + # Get rid of nested form + response.set_data(response.get_data().replace('
', '')) + browser.set_response(response) + browser.select_form(nr=2) browser.form['name']=self.username browser.form['pass']=self.password @@ -211,4 +218,3 @@ class ZeitEPUBAbo(BasicNewsRecipe): self.log.warning('Using static old low-res cover') cover_url = 'http://images.zeit.de/bilder/titelseiten_zeit/1946/001_001.jpg' return cover_url - From c940cb2cb602f924c5a435c671dabc2ddff14f5e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Nov 2011 08:39:02 +0530 Subject: [PATCH 30/37] calibre-mount-helper: Refuse to mount non block devices --- src/calibre/devices/linux_mount_helper.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 7829d43fad..2ab83f3c3c 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -236,8 +236,9 @@ int cleanup(const char *dev, const char *mp) { return cleanup_mount_point(mp); } -void check_dev(const char*dev) { +void check_dev(const char *dev) { char buffer[PATH_MAX+1]; + struct stat file_info; if (dev == NULL || strlen(dev) < strlen(DEV)) { fprintf(stderr, "Invalid arguments\n"); @@ -253,6 +254,16 @@ void check_dev(const char*dev) { fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); exit(EXIT_FAILURE); } + + if (stat(dev, &file_info) != 0) { + fprintf(stderr, "stat call on dev node failed\n"); + exit(EXIT_FAILURE); + } + + if (!S_ISBLK(file_info.st_mode)) { + fprintf(stderr, "dev node is not a block device\n"); + exit(EXIT_FAILURE); + } } int main(int argc, char** argv) From 75341b3c99fb871620def6c872c3ced41dd244d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Nov 2011 18:44:44 +0530 Subject: [PATCH 31/37] Catavencii by Silviu Cotoara --- recipes/catavencii.recipe | 51 +++++++++++++++++++++++++++++++++++ recipes/icons/catavencii.png | Bin 0 -> 600 bytes 2 files changed, 51 insertions(+) create mode 100644 recipes/catavencii.recipe create mode 100644 recipes/icons/catavencii.png diff --git a/recipes/catavencii.recipe b/recipes/catavencii.recipe new file mode 100644 index 0000000000..7dff212d74 --- /dev/null +++ b/recipes/catavencii.recipe @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +catavencii.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Catavencii(BasicNewsRecipe): + title = u'Ca\u0163avencii' + __author__ = u'Silviu Cotoar\u0103' + publisher = u'Ca\u0163avencii' + description = u'Ca\u0163avencii' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Romania' + encoding = 'utf-8' + cover_url = 'http://www.simonatache.ro/wp-content/uploads/2011/06/catavencii-logo.png' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'id':'content'}) + ] + + remove_tags = [ + dict(name='div', attrs={'id':'breadcrumbs'}) + , dict(name='span', attrs={'class':'info'}) + , dict(name='div', attrs={'id':'social-media-article'}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':'social-media-article'}) + ] + feeds = [ + (u'\u0218tiri', u'http://www.catavencii.ro/rss') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/icons/catavencii.png b/recipes/icons/catavencii.png new file mode 100644 index 0000000000000000000000000000000000000000..87d87ba9ddca8a7f1066f05a960dbe2ef806ee5e GIT binary patch literal 600 zcmV-e0;m0nP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!~g&e!~vBn4jTXf0qsddK~y+Tg_GSY zgK-qcAJ%4h3)vcLq(oviZ!<5+1&Rwq`~ynK%}SKGP;!F{v@5xAOG+ukl`G1HLN4TD zSCf~vVc6LCI_KFQd7c>;zn-1v`JK=Cp6}Ta;2#7Yy8v$^@N5G6%nwQ96gjP zh}9WDAD#305Cq}ZqsvxfXR|n>P*OsICd>5hN5SzT26(k$ZM6_P8%4kyef+e;cUy(_){GcP zgFOv+Q7T|#cx(YmmPSRro*k7zrAm-Na&t5Y+*BzPFo65|o-@T%<@(YJtu|2x(2);* zH~AoIffD&Ls1*cB1xrLdmX``*S|}>iBJ82jD@`04(4((A3nzzVxVo@lekKpvBzbi< zxxXhX@~4sks9(ph;omVbV0%4kv=T5rlKU^9x09;#l-MgJU~<&>C!nS>1ztC~&;GFK zB0yTEHyc)*1xQO(#X#~*8caW+0Z(=CxTZu|xl5}YODfbIcW=g#qMJJ2^0000 Date: Fri, 4 Nov 2011 22:49:09 +0530 Subject: [PATCH 32/37] Do not error out if there is an invalid regex for title sort set in tweaks --- src/calibre/ebooks/metadata/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index decba3b780..a6b3c1ad21 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -116,10 +116,14 @@ def title_sort(title, order=None): title = title[1:] match = _title_pat.search(title) if match: - prep = match.group(1) - title = title[len(prep):] + ', ' + prep - if title[0] in _ignore_starts: - title = title[1:] + try: + prep = match.group(1) + except IndexError: + pass + else: + title = title[len(prep):] + ', ' + prep + if title[0] in _ignore_starts: + title = title[1:] return title.strip() coding = zip( From b660c352a4e80056f22f78d55869579d91e652bf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 4 Nov 2011 23:43:28 +0530 Subject: [PATCH 33/37] calibre-mount-helper: disallow paths that contain /shm/ --- src/calibre/devices/linux_mount_helper.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index 2ab83f3c3c..dfeaa7985e 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -260,10 +260,16 @@ void check_dev(const char *dev) { exit(EXIT_FAILURE); } + if (strstr(dev, "/shm/") != NULL) { + fprintf(stderr, "naughty, naughty!\n"); + exit(EXIT_FAILURE); + } + if (!S_ISBLK(file_info.st_mode)) { fprintf(stderr, "dev node is not a block device\n"); exit(EXIT_FAILURE); } + } int main(int argc, char** argv) From e148d614122d038760b1ed90278956a5d577f3df Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Nov 2011 00:24:11 +0530 Subject: [PATCH 34/37] calibre-mount-helper: A nicer mount point location check --- src/calibre/devices/linux_mount_helper.c | 35 ++++++++++++++++++++++-- src/calibre/devices/usbms/device.py | 10 +++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index dfeaa7985e..eefbd1ffcc 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -8,6 +8,7 @@ #include #include #include +#include #define MARKER ".created_by_calibre_mount_helper" #define DEV "/dev/" @@ -272,6 +273,37 @@ void check_dev(const char *dev) { } +char *get_real_mount_point(const char *mp) { + /* Resolve the mount point to a canonical path. Assumes that mp + * either points to an existing directory, or only the leaf node + * of mp does not exist. This is safe for the usage scenario of calibre-mount-helper. */ + char *dirname = NULL, *basename, *p, *buffer; + buffer = calloc(PATH_MAX+1, sizeof(char)); + if (buffer == NULL) exit(EXIT_FAILURE); + if (realpath(mp, buffer) != NULL) return buffer; + + dirname = calloc(PATH_MAX+1, sizeof(char)); + if (dirname == NULL) exit(EXIT_FAILURE); + + strncpy(dirname, mp, PATH_MAX); + p = rindex(dirname, '/'); + if (p == NULL) { + fprintf(stderr, "mountpoint must have atleast one /\n"); + exit(EXIT_FAILURE); + } + basename = p+1; + *p = 0; + dirname = realpath(dirname, NULL); + if (dirname == NULL) { + fprintf(stderr, "parent directory of mount point cannot be resolved, ensure the mountpoint does not have a trailing slash.\n"); + exit(EXIT_FAILURE); + } + snprintf(buffer, PATH_MAX, "%s/%s", dirname, basename); + free(dirname); + return buffer; +} + + int main(int argc, char** argv) { char *action, *dev, *mp, *temp; @@ -303,8 +335,7 @@ int main(int argc, char** argv) fprintf(stderr, "Failed to resolve device node.\n"); exit(EXIT_FAILURE); } - temp = realpath(mp, NULL); - if (temp != NULL) mp = temp; + mp = get_real_mount_point(mp); check_dev(dev); check_mount_point(mp); status = do_mount(dev, mp); } else if (strncmp(action, "eject", 5) == 0) { diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index e6120f337f..0e899517a9 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -622,8 +622,11 @@ class Device(DeviceConfig, DevicePlugin): if getattr(sys, 'frozen', False): cmd = os.path.join(sys.executables_location, 'bin', cmd) cmd = [cmd, 'mount'] + mlabel = label + if mlabel.endswith('/'): + mlabel = mlabel[:-1] try: - p = subprocess.Popen(cmd + [node, '/media/'+label]) + p = subprocess.Popen(cmd + [node, '/media/'+mlabel]) except OSError: raise DeviceError( _('Could not find mount helper: %s.')%cmd[0]) @@ -777,9 +780,12 @@ class Device(DeviceConfig, DevicePlugin): # try all the nodes to see what we can mount for dev in devs[i].split(): mp='/media/'+label+'-'+dev + mmp = mp + if mmp.endswith('/'): + mmp = mmp[:-1] #print "trying ", dev, "on", mp try: - p = subprocess.Popen(cmd + ["/dev/"+dev, mp]) + p = subprocess.Popen(cmd + ["/dev/"+dev, mmp]) except OSError: raise DeviceError(_('Could not find mount helper: %s.')%cmd[0]) while p.poll() is None: From 03b68a52b39ea0fb81b2fd57b67acf81caafc27f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Nov 2011 01:11:15 +0530 Subject: [PATCH 35/37] oops --- src/calibre/devices/linux_mount_helper.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index eefbd1ffcc..c7cd54d6b5 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -238,7 +238,6 @@ int cleanup(const char *dev, const char *mp) { } void check_dev(const char *dev) { - char buffer[PATH_MAX+1]; struct stat file_info; if (dev == NULL || strlen(dev) < strlen(DEV)) { @@ -246,12 +245,7 @@ void check_dev(const char *dev) { exit(EXIT_FAILURE); } - if (realpath(dev, buffer) == NULL) { - fprintf(stderr, "Unable to resolve dev path\n"); - exit(EXIT_FAILURE); - } - - if (strncmp(DEV, buffer, strlen(DEV)) != 0) { + if (strncmp(DEV, dev, strlen(DEV)) != 0) { fprintf(stderr, "Trying to operate on a dev node not under /dev\n"); exit(EXIT_FAILURE); } From 8da4907ccf6992c0de85005df6aa012130ec4a2e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Nov 2011 01:56:12 +0530 Subject: [PATCH 36/37] ... --- src/calibre/devices/linux_mount_helper.c | 45 ++---------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/src/calibre/devices/linux_mount_helper.c b/src/calibre/devices/linux_mount_helper.c index c7cd54d6b5..41ce2c84b2 100644 --- a/src/calibre/devices/linux_mount_helper.c +++ b/src/calibre/devices/linux_mount_helper.c @@ -38,24 +38,12 @@ void ensure_root() { } void check_mount_point(const char *mp) { - char buffer[PATH_MAX+1]; if (mp == NULL || strlen(mp) < strlen(MEDIA)) { fprintf(stderr, "Invalid arguments\n"); exit(EXIT_FAILURE); } - if (exists(mp)) { - if (realpath(mp, buffer) == NULL) { - fprintf(stderr, "Unable to resolve mount path\n"); - exit(EXIT_FAILURE); - } - if (strncmp(MEDIA, buffer, strlen(MEDIA)) != 0) { - fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); - exit(EXIT_FAILURE); - } - } - if (strncmp(MEDIA, mp, strlen(MEDIA)) != 0) { fprintf(stderr, "Trying to operate on a mount point not under /media is not allowed\n"); exit(EXIT_FAILURE); @@ -267,41 +255,12 @@ void check_dev(const char *dev) { } -char *get_real_mount_point(const char *mp) { - /* Resolve the mount point to a canonical path. Assumes that mp - * either points to an existing directory, or only the leaf node - * of mp does not exist. This is safe for the usage scenario of calibre-mount-helper. */ - char *dirname = NULL, *basename, *p, *buffer; - buffer = calloc(PATH_MAX+1, sizeof(char)); - if (buffer == NULL) exit(EXIT_FAILURE); - if (realpath(mp, buffer) != NULL) return buffer; - - dirname = calloc(PATH_MAX+1, sizeof(char)); - if (dirname == NULL) exit(EXIT_FAILURE); - - strncpy(dirname, mp, PATH_MAX); - p = rindex(dirname, '/'); - if (p == NULL) { - fprintf(stderr, "mountpoint must have atleast one /\n"); - exit(EXIT_FAILURE); - } - basename = p+1; - *p = 0; - dirname = realpath(dirname, NULL); - if (dirname == NULL) { - fprintf(stderr, "parent directory of mount point cannot be resolved, ensure the mountpoint does not have a trailing slash.\n"); - exit(EXIT_FAILURE); - } - snprintf(buffer, PATH_MAX, "%s/%s", dirname, basename); - free(dirname); - return buffer; -} - int main(int argc, char** argv) { char *action, *dev, *mp, *temp; int status = EXIT_FAILURE; + exit(EXIT_FAILURE); /*printf("Real UID\t= %d\n", getuid()); printf("Effective UID\t= %d\n", geteuid()); @@ -343,7 +302,7 @@ int main(int argc, char** argv) fprintf(stderr, "Mount point does not exist\n"); exit(EXIT_FAILURE); } - mp = temp; + chdir(temp); check_dev(dev); check_mount_point(mp); status = do_eject(dev, mp); } else if (strncmp(action, "cleanup", 7) == 0) { From 50f3e72b4a360b734c39db3012aa5bd9be57554d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 5 Nov 2011 02:46:10 +0530 Subject: [PATCH 37/37] b365 Realitatea by Silviu Cotoara --- recipes/b365realitatea.recipe | 52 +++++++++++++++++++++++++++++++ recipes/icons/b365realitatea.png | Bin 0 -> 323 bytes 2 files changed, 52 insertions(+) create mode 100644 recipes/b365realitatea.recipe create mode 100644 recipes/icons/b365realitatea.png diff --git a/recipes/b365realitatea.recipe b/recipes/b365realitatea.recipe new file mode 100644 index 0000000000..80a1ee225b --- /dev/null +++ b/recipes/b365realitatea.recipe @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +b365.realitatea.net +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class b365Realitatea(BasicNewsRecipe): + title = u'b365 Realitatea' + __author__ = u'Silviu Cotoar\u0103' + publisher = u'b365 Realitatea' + description = u'b365 Realitatea' + oldest_article = 5 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Romania,Bucuresti' + encoding = 'utf-8' + cover_url = 'http://b365.realitatea.net/wp-content/themes/b/images/b365-logo.png' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'class':'newsArticle'}) + ] + + remove_tags = [ + dict(name='div', attrs={'class':'date'}) + , dict(name='dic', attrs={'class':'addthis_toolbox addthis_default_style'}) + , dict(name='div', attrs={'class':'related_posts'}) + , dict(name='div', attrs={'id':'RelevantiWidget'}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':'RelevantiWidget'}) + ] + feeds = [ + (u'\u0218tiri', u'http://b365.realitatea.net/rss-full/') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) + diff --git a/recipes/icons/b365realitatea.png b/recipes/icons/b365realitatea.png new file mode 100644 index 0000000000000000000000000000000000000000..040169ca8819af5a61fd4cec50cb7861847bcdad GIT binary patch literal 323 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!73?$#)eFPFv3GfMVJ;T5NME`g1e%H})S4rtV z5Zt|c*Tv;-Xz2elXWo^UpAi(iEK?8&RL@uvN^`tnRdQTiaogb`Ga4+NNAdhLU-jXCxV(1PLxxeO{stG*q?3HKHUX zu_V4V4HONz`6-!cl?V+cAw~vP b#uip4Mi33AFDfSkH86O(`njxgN@xNA?q*&H literal 0 HcmV?d00001