diff --git a/resources/recipes/houston_chronicle.recipe b/resources/recipes/houston_chronicle.recipe index 77e35dfc0c..3ec1abbf0f 100644 --- a/resources/recipes/houston_chronicle.recipe +++ b/resources/recipes/houston_chronicle.recipe @@ -1,17 +1,41 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from calibre.web.feeds.news import BasicNewsRecipe class HoustonChronicle(BasicNewsRecipe): title = u'The Houston Chronicle' description = 'News from Houston, Texas' - __author__ = 'Kovid Goyal' + __author__ = 'Kovid Goyal and Sujata Raman' language = 'en' timefmt = ' [%a, %d %b, %Y]' no_stylesheets = True - keep_only_tags = [dict(id=['story-head', 'story'])] - remove_tags = [dict(id=['share-module', 'resource-box', - 'resource-box-header'])] + keep_only_tags = [ + dict(id=['story-head', 'story']) + ] + + remove_tags = [ + dict(id=['share-module', 'resource-box', + 'resource-box-header']) + ] + + extra_css = ''' + h1{font-family :Arial,Helvetica,sans-serif; font-size:large;} + h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#666666;} + h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;} + h4{font-family :Arial,Helvetica,sans-serif; font-size: x-small;} + p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + #story-head h1{font-family :Arial,Helvetica,sans-serif; font-size: xx-large;} + #story-head h2{font-family :Arial,Helvetica,sans-serif; font-size: small; color:#000000;} + #story-head h3{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;} + #story-head h4{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;} + #story{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;} + #Text-TextSubhed BoldCond PoynterAgateZero h3{color:#444444;font-family :Arial,Helvetica,sans-serif; font-size:small;} + .p260x p{font-family :Arial,Helvetica,serif; font-size:x-small;font-style:italic;} + .p260x h6{color:#777777;font-family :Arial,Helvetica,sans-serif; font-size:xx-small;} + ''' + def parse_index(self): soup = self.index_to_soup('http://www.chron.com/news/') @@ -64,3 +88,6 @@ class HoustonChronicle(BasicNewsRecipe): feeds.append((current_section, current_articles)) return feeds + + + diff --git a/resources/recipes/san_fran_chronicle.recipe b/resources/recipes/san_fran_chronicle.recipe index 5649da9bf8..cfdf0b07f0 100644 --- a/resources/recipes/san_fran_chronicle.recipe +++ b/resources/recipes/san_fran_chronicle.recipe @@ -7,10 +7,11 @@ sfgate.com ''' from calibre.web.feeds.news import BasicNewsRecipe +import re class SanFranciscoChronicle(BasicNewsRecipe): title = u'San Francisco Chronicle' - __author__ = u'Darko Miletic' + __author__ = u'Darko Miletic and Sujata Raman' description = u'San Francisco news' language = 'en' @@ -19,13 +20,56 @@ class SanFranciscoChronicle(BasicNewsRecipe): no_stylesheets = True use_embedded_content = False - remove_tags_before = {'class':'articleheadings'} - remove_tags_after = dict(name='div', attrs={'id':'articlecontent' }) - remove_tags = [ - dict(name='div', attrs={'class':'tools tools_top'}) - ,dict(name='div', attrs={'id':'articlebox' }) - ] + + + remove_tags_before = {'id':'printheader'} + + remove_tags = [ + dict(name='div',attrs={'id':'printheader'}) + ,dict(name='a', attrs={'href':re.compile('http://ads\.pheedo\.com.*')}) + ,dict(name='div',attrs={'id':'footer'}) + ] + + extra_css = ''' + h1{font-family :Arial,Helvetica,sans-serif; font-size:large;} + h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#666666;} + h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;} + h4{font-family :Arial,Helvetica,sans-serif; font-size: x-small;} + p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .byline{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;} + .date{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;} + .dtlcomment{font-style:italic;} + .georgia h3{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#000000;} + ''' feeds = [ (u'Top News Stories', u'http://www.sfgate.com/rss/feeds/news.xml') ] + + def print_version(self,url): + url= url +"&type=printable" + return url + + def get_article_url(self, article): + print str(article['title_detail']['value']) + url = article.get('guid',None) + url = "http://www.sfgate.com/cgi-bin/article.cgi?f="+url + if "Presented By:" in str(article['title_detail']['value']): + url = '' + return url + + + + + + + + + + + + + + + + diff --git a/src/calibre/devices/usbms/cli.py b/src/calibre/devices/usbms/cli.py index 67d457efe1..829267a0b8 100644 --- a/src/calibre/devices/usbms/cli.py +++ b/src/calibre/devices/usbms/cli.py @@ -4,8 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import os -import shutil +import os, shutil, time from calibre.devices.errors import PathError @@ -55,6 +54,7 @@ class CLI(object): shutil.copyfileobj(infile, dest) except IOError: print 'WARNING: First attempt to send file to device failed' + time.sleep(0.2) infile.seek(0) dest.seek(0) dest.truncate() diff --git a/src/calibre/ebooks/rtf/preprocess.py b/src/calibre/ebooks/rtf/preprocess.py index 07e6d41fac..a3076651fd 100644 --- a/src/calibre/ebooks/rtf/preprocess.py +++ b/src/calibre/ebooks/rtf/preprocess.py @@ -131,9 +131,9 @@ class RtfTokenParser(): if isString(self.tokens[i].name, "\\'"): i = i + 1 if not isinstance(self.tokens[i], tokenData): - raise BaseException('Error: token8bitChar without data.') + raise Exception('Error: token8bitChar without data.') if len(self.tokens[i].data) < 2: - raise BaseException('Error: token8bitChar without data.') + raise Exception('Error: token8bitChar without data.') newTokens.append(token8bitChar(self.tokens[i].data[0:2])) if len(self.tokens[i].data) > 2: newTokens.append(tokenData(self.tokens[i].data[2:])) @@ -195,7 +195,7 @@ class RtfTokenParser(): i = i + 1 j = j + 1 continue - raise BaseException('Error: incorect utf replacement.') + raise Exception('Error: incorect utf replacement.') #calibre rtf2xml does not support utfreplace replace = [] @@ -248,7 +248,7 @@ class RtfTokenizer(): if isChar(self.rtfData[i], '\\'): if i + 1 >= len(self.rtfData): - raise BaseException('Error: Control character found at the end of the document.') + raise Exception('Error: Control character found at the end of the document.') if lastDataStart > -1: self.tokens.append(tokenData(self.rtfData[lastDataStart : i])) @@ -269,7 +269,7 @@ class RtfTokenizer(): i = i + 1 if not consumed: - raise BaseException('Error (at:%d): Control Word without end.'%(tokenStart)) + raise Exception('Error (at:%d): Control Word without end.'%(tokenStart)) #we have numeric argument before delimiter if isChar(self.rtfData[i], '-') or isDigit(self.rtfData[i]): @@ -283,10 +283,10 @@ class RtfTokenizer(): l = l + 1 i = i + 1 if l > 10 : - raise BaseException('Error (at:%d): Too many digits in control word numeric argument.'%[tokenStart]) + raise Exception('Error (at:%d): Too many digits in control word numeric argument.'%[tokenStart]) if not consumed: - raise BaseException('Error (at:%d): Control Word without numeric argument end.'%[tokenStart]) + raise Exception('Error (at:%d): Control Word without numeric argument end.'%[tokenStart]) separator = '' if isChar(self.rtfData[i], ' '): diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index bf4fd02fa8..4be4156f1b 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -1,10 +1,11 @@ -from calibre.ebooks.metadata import authors_to_string __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' + import os, textwrap, traceback, re, shutil from operator import attrgetter - from math import cos, sin, pi +from contextlib import closing + from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \ QItemDelegate, QPainterPath, QLinearGradient, QBrush, \ QPen, QStyle, QPainter, \ @@ -22,7 +23,8 @@ from calibre.gui2 import NONE, TableView, qstring_to_unicode, config, \ from calibre.gui2.widgets import EnLineEdit, TagsLineEdit from calibre.utils.search_query_parser import SearchQueryParser from calibre.ebooks.metadata.meta import set_metadata as _set_metadata -from calibre.ebooks.metadata import string_to_authors, fmt_sidx +from calibre.ebooks.metadata import string_to_authors, fmt_sidx, \ + authors_to_string from calibre.utils.config import tweaks from calibre.utils.date import dt_factory, qt_to_dt, isoformat @@ -469,9 +471,10 @@ class BooksModel(QAbstractTableModel): break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - src = self.db.format(id, format, index_is_id=True, as_file=True) - shutil.copyfileobj(src, pt) - pt.flush() + with closing(self.db.format(id, format, index_is_id=True, + as_file=True)) as src: + shutil.copyfileobj(src, pt) + pt.flush() pt.seek(0) if set_metadata: _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True), @@ -505,8 +508,10 @@ class BooksModel(QAbstractTableModel): break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - pt.write(self.db.format(row, format)) - pt.flush() + with closing(self.db.format(row, format, as_file=True)) as src: + shutil.copyfileobj(src, pt) + pt.flush() + pt.seek(0) if set_metadata: _set_metadata(pt, self.db.get_metadata(row, get_cover=True), format) diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 425b4c2d49..a92d195bff 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -6,14 +6,14 @@ __docformat__ = 'restructuredtext en' ''' A parser for search queries with a syntax very similar to that used by -the Google search engine. +the Google search engine. -For details on the search query syntax see :class:`SearchQueryParser`. -To use the parser, subclass :class:`SearchQueryParser` and implement the -methods :method:`SearchQueryParser.universal_set` and +For details on the search query syntax see :class:`SearchQueryParser`. +To use the parser, subclass :class:`SearchQueryParser` and implement the +methods :method:`SearchQueryParser.universal_set` and :method:`SearchQueryParser.get_matches`. See for example :class:`Tester`. -If this module is run, it will perform a series of unit tests. +If this module is run, it will perform a series of unit tests. ''' import sys, string, operator @@ -24,26 +24,26 @@ from calibre.utils.pyparsing import Keyword, Group, Forward, CharsNotIn, Suppres class SearchQueryParser(object): ''' - Parses a search query. - + Parses a search query. + A search query consists of tokens. The tokens can be combined using - the `or`, `and` and `not` operators as well as grouped using parentheses. + the `or`, `and` and `not` operators as well as grouped using parentheses. When no operator is specified between two tokens, `and` is assumed. - + Each token is a string of the form `location:query`. `location` is a string - from :member:`LOCATIONS`. It is optional. If it is omitted, it is assumed to - be `all`. `query` is an arbitrary string that must not contain parentheses. + from :member:`LOCATIONS`. It is optional. If it is omitted, it is assumed to + be `all`. `query` is an arbitrary string that must not contain parentheses. If it contains whitespace, it should be quoted by enclosing it in `"` marks. - + Examples:: - + * `Asimov` [search for the string "Asimov" in location `all`] * `comments:"This is a good book"` [search for "This is a good book" in `comments`] * `author:Asimov tag:unread` [search for books by Asimov that have been tagged as unread] * `author:Asimov or author:Hardy` [search for books by Asimov or Hardy] - * `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy] + * `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy] ''' - + LOCATIONS = [ 'tag', 'title', @@ -57,12 +57,12 @@ class SearchQueryParser(object): 'isbn', 'all', ] - + @staticmethod def run_tests(parser, result, tests): failed = [] for test in tests: - print '\tTesting:', test[0], + print '\tTesting:', test[0], res = parser.parseString(test[0]) if list(res.get(result, None)) == test[1]: print 'OK' @@ -70,7 +70,7 @@ class SearchQueryParser(object): print 'FAILED:', 'Expected:', test[1], 'Got:', list(res.get(result, None)) failed.append(test[0]) return failed - + def __init__(self, test=False): self._tests_failed = False # Define a token @@ -95,50 +95,50 @@ class SearchQueryParser(object): ('title:"one two"', ['title', 'one two']), ) ) - + Or = Forward() - + Parenthesis = Group( Suppress('(') + Or + Suppress(')') ).setResultsName('parenthesis') | Token - - + + Not = Forward() Not << (Group( Suppress(Keyword("not", caseless=True)) + Not ).setResultsName("not") | Parenthesis) - + And = Forward() And << (Group( Not + Suppress(Keyword("and", caseless=True)) + And ).setResultsName("and") | Group( - Not + OneOrMore(~oneOf("and or") + And) + Not + OneOrMore(~oneOf("and or", caseless=True) + And) ).setResultsName("and") | Not) - + Or << (Group( And + Suppress(Keyword("or", caseless=True)) + Or ).setResultsName("or") | And) - + if test: Or.validate() self._tests_failed = bool(failed) - + self._parser = Or #self._parser.setDebug(True) #self.parse('(tolstoy)') self._parser.setDebug(False) - - + + def parse(self, query): res = self._parser.parseString(query)[0] return self.evaluate(res) - + def method(self, group_name): return getattr(self, 'evaluate_'+group_name) - + def evaluate(self, parse_result): return self.method(parse_result.getName())(parse_result) - + def evaluate_and(self, argument): return self.evaluate(argument[0]).intersection(self.evaluate(argument[1])) @@ -150,27 +150,27 @@ class SearchQueryParser(object): def evaluate_parenthesis(self, argument): return self.evaluate(argument[0]) - + def evaluate_token(self, argument): return self.get_matches(argument[0], argument[1]) - + def get_matches(self, location, query): ''' - Should return the set of matches for :param:'location` and :param:`query`. - + Should return the set of matches for :param:'location` and :param:`query`. + :param:`location` is one of the items in :member:`SearchQueryParser.LOCATIONS`. - :param:`query` is a string literal. + :param:`query` is a string literal. ''' return set([]) - + def universal_set(self): ''' Should return the set of all matches. ''' return set([]) - + class Tester(SearchQueryParser): - + texts = { 1: [u'Eugenie Grandet', u'Honor\xe9 de Balzac', u'manybooks.net', u'lrf'], 2: [u'Fanny Hill', u'John Cleland', u'manybooks.net', u'lrf'], @@ -459,30 +459,30 @@ class Tester(SearchQueryParser): u'Washington Square Press', u'lrf,rar'] } - + tests = { 'Dysfunction' : set([348]), 'title:Dysfunction' : set([348]), - 'title:Dysfunction or author:Laurie': set([348, 444]), + 'title:Dysfunction OR author:Laurie': set([348, 444]), '(tag:txt or tag:pdf)': set([33, 258, 354, 305, 242, 51, 55, 56, 154]), - '(tag:txt or tag:pdf) and author:Tolstoy': set([55, 56]), + '(tag:txt OR tag:pdf) and author:Tolstoy': set([55, 56]), 'Tolstoy txt': set([55, 56]), 'Hamilton Amsterdam' : set([]), u'Beär' : set([91]), 'dysfunc or tolstoy': set([348, 55, 56]), - 'tag:txt and not tolstoy': set([33, 258, 354, 305, 242, 154]), + 'tag:txt AND NOT tolstoy': set([33, 258, 354, 305, 242, 154]), 'not tag:lrf' : set([305]), 'london:thames': set([13]), 'publisher:london:thames': set([13]), '"(1977)"': set([13]), } fields = {'title':0, 'author':1, 'publisher':2, 'tag':3} - + _universal_set = set(texts.keys()) - + def universal_set(self): return self._universal_set - + def get_matches(self, location, query): location = location.lower() if location in self.fields.keys(): @@ -491,19 +491,19 @@ class Tester(SearchQueryParser): getter = lambda y: ''.join(x if x else '' for x in y) else: getter = lambda x: '' - + if not query: - return set([]) + return set([]) query = query.lower() return set(key for key, val in self.texts.items() \ if query and query in getattr(getter(val), 'lower', lambda : '')()) - - - + + + def run_tests(self): failed = [] for query in self.tests.keys(): - print 'Testing query:', query, + print 'Testing query:', query, res = self.parse(query) if res != self.tests[query]: print 'FAILED', 'Expected:', self.tests[query], 'Got:', res @@ -511,7 +511,7 @@ class Tester(SearchQueryParser): else: print 'OK' return failed - + def main(args=sys.argv): tester = Tester(test=True) @@ -519,7 +519,7 @@ def main(args=sys.argv): if tester._tests_failed or failed: print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<' return 1 - + return 0 if __name__ == '__main__':