mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-08 10:44:09 -04:00
KG revisions
This commit is contained in:
GRiker
commit
10da9fccb6
@ -1,17 +1,41 @@
|
||||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
class HoustonChronicle(BasicNewsRecipe):
|
||||
|
||||
title = u'The Houston Chronicle'
|
||||
description = 'News from Houston, Texas'
|
||||
__author__ = 'Kovid Goyal'
|
||||
__author__ = 'Kovid Goyal and Sujata Raman'
|
||||
language = 'en'
|
||||
timefmt = ' [%a, %d %b, %Y]'
|
||||
no_stylesheets = True
|
||||
|
||||
keep_only_tags = [dict(id=['story-head', 'story'])]
|
||||
remove_tags = [dict(id=['share-module', 'resource-box',
|
||||
'resource-box-header'])]
|
||||
keep_only_tags = [
|
||||
dict(id=['story-head', 'story'])
|
||||
]
|
||||
|
||||
remove_tags = [
|
||||
dict(id=['share-module', 'resource-box',
|
||||
'resource-box-header'])
|
||||
]
|
||||
|
||||
extra_css = '''
|
||||
h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
|
||||
h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#666666;}
|
||||
h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
|
||||
h4{font-family :Arial,Helvetica,sans-serif; font-size: x-small;}
|
||||
p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
|
||||
#story-head h1{font-family :Arial,Helvetica,sans-serif; font-size: xx-large;}
|
||||
#story-head h2{font-family :Arial,Helvetica,sans-serif; font-size: small; color:#000000;}
|
||||
#story-head h3{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;}
|
||||
#story-head h4{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;}
|
||||
#story{font-family :Arial,Helvetica,sans-serif; font-size:xx-small;}
|
||||
#Text-TextSubhed BoldCond PoynterAgateZero h3{color:#444444;font-family :Arial,Helvetica,sans-serif; font-size:small;}
|
||||
.p260x p{font-family :Arial,Helvetica,serif; font-size:x-small;font-style:italic;}
|
||||
.p260x h6{color:#777777;font-family :Arial,Helvetica,sans-serif; font-size:xx-small;}
|
||||
'''
|
||||
|
||||
|
||||
def parse_index(self):
|
||||
soup = self.index_to_soup('http://www.chron.com/news/')
|
||||
@ -64,3 +88,6 @@ class HoustonChronicle(BasicNewsRecipe):
|
||||
feeds.append((current_section, current_articles))
|
||||
return feeds
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -7,10 +7,11 @@ sfgate.com
|
||||
'''
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
import re
|
||||
|
||||
class SanFranciscoChronicle(BasicNewsRecipe):
|
||||
title = u'San Francisco Chronicle'
|
||||
__author__ = u'Darko Miletic'
|
||||
__author__ = u'Darko Miletic and Sujata Raman'
|
||||
description = u'San Francisco news'
|
||||
language = 'en'
|
||||
|
||||
@ -19,13 +20,56 @@ class SanFranciscoChronicle(BasicNewsRecipe):
|
||||
no_stylesheets = True
|
||||
use_embedded_content = False
|
||||
|
||||
remove_tags_before = {'class':'articleheadings'}
|
||||
remove_tags_after = dict(name='div', attrs={'id':'articlecontent' })
|
||||
remove_tags = [
|
||||
dict(name='div', attrs={'class':'tools tools_top'})
|
||||
,dict(name='div', attrs={'id':'articlebox' })
|
||||
]
|
||||
|
||||
|
||||
remove_tags_before = {'id':'printheader'}
|
||||
|
||||
remove_tags = [
|
||||
dict(name='div',attrs={'id':'printheader'})
|
||||
,dict(name='a', attrs={'href':re.compile('http://ads\.pheedo\.com.*')})
|
||||
,dict(name='div',attrs={'id':'footer'})
|
||||
]
|
||||
|
||||
extra_css = '''
|
||||
h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
|
||||
h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#666666;}
|
||||
h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
|
||||
h4{font-family :Arial,Helvetica,sans-serif; font-size: x-small;}
|
||||
p{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
|
||||
.byline{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;}
|
||||
.date{font-family :Arial,Helvetica,sans-serif; font-size: xx-small;}
|
||||
.dtlcomment{font-style:italic;}
|
||||
.georgia h3{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#000000;}
|
||||
'''
|
||||
|
||||
feeds = [
|
||||
(u'Top News Stories', u'http://www.sfgate.com/rss/feeds/news.xml')
|
||||
]
|
||||
|
||||
def print_version(self,url):
|
||||
url= url +"&type=printable"
|
||||
return url
|
||||
|
||||
def get_article_url(self, article):
|
||||
print str(article['title_detail']['value'])
|
||||
url = article.get('guid',None)
|
||||
url = "http://www.sfgate.com/cgi-bin/article.cgi?f="+url
|
||||
if "Presented By:" in str(article['title_detail']['value']):
|
||||
url = ''
|
||||
return url
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -4,8 +4,7 @@ __license__ = 'GPL 3'
|
||||
__copyright__ = '2009, John Schember <john@nachtimwald.com>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import os, shutil, time
|
||||
|
||||
from calibre.devices.errors import PathError
|
||||
|
||||
@ -55,6 +54,7 @@ class CLI(object):
|
||||
shutil.copyfileobj(infile, dest)
|
||||
except IOError:
|
||||
print 'WARNING: First attempt to send file to device failed'
|
||||
time.sleep(0.2)
|
||||
infile.seek(0)
|
||||
dest.seek(0)
|
||||
dest.truncate()
|
||||
|
||||
@ -131,9 +131,9 @@ class RtfTokenParser():
|
||||
if isString(self.tokens[i].name, "\\'"):
|
||||
i = i + 1
|
||||
if not isinstance(self.tokens[i], tokenData):
|
||||
raise BaseException('Error: token8bitChar without data.')
|
||||
raise Exception('Error: token8bitChar without data.')
|
||||
if len(self.tokens[i].data) < 2:
|
||||
raise BaseException('Error: token8bitChar without data.')
|
||||
raise Exception('Error: token8bitChar without data.')
|
||||
newTokens.append(token8bitChar(self.tokens[i].data[0:2]))
|
||||
if len(self.tokens[i].data) > 2:
|
||||
newTokens.append(tokenData(self.tokens[i].data[2:]))
|
||||
@ -195,7 +195,7 @@ class RtfTokenParser():
|
||||
i = i + 1
|
||||
j = j + 1
|
||||
continue
|
||||
raise BaseException('Error: incorect utf replacement.')
|
||||
raise Exception('Error: incorect utf replacement.')
|
||||
|
||||
#calibre rtf2xml does not support utfreplace
|
||||
replace = []
|
||||
@ -248,7 +248,7 @@ class RtfTokenizer():
|
||||
|
||||
if isChar(self.rtfData[i], '\\'):
|
||||
if i + 1 >= len(self.rtfData):
|
||||
raise BaseException('Error: Control character found at the end of the document.')
|
||||
raise Exception('Error: Control character found at the end of the document.')
|
||||
|
||||
if lastDataStart > -1:
|
||||
self.tokens.append(tokenData(self.rtfData[lastDataStart : i]))
|
||||
@ -269,7 +269,7 @@ class RtfTokenizer():
|
||||
i = i + 1
|
||||
|
||||
if not consumed:
|
||||
raise BaseException('Error (at:%d): Control Word without end.'%(tokenStart))
|
||||
raise Exception('Error (at:%d): Control Word without end.'%(tokenStart))
|
||||
|
||||
#we have numeric argument before delimiter
|
||||
if isChar(self.rtfData[i], '-') or isDigit(self.rtfData[i]):
|
||||
@ -283,10 +283,10 @@ class RtfTokenizer():
|
||||
l = l + 1
|
||||
i = i + 1
|
||||
if l > 10 :
|
||||
raise BaseException('Error (at:%d): Too many digits in control word numeric argument.'%[tokenStart])
|
||||
raise Exception('Error (at:%d): Too many digits in control word numeric argument.'%[tokenStart])
|
||||
|
||||
if not consumed:
|
||||
raise BaseException('Error (at:%d): Control Word without numeric argument end.'%[tokenStart])
|
||||
raise Exception('Error (at:%d): Control Word without numeric argument end.'%[tokenStart])
|
||||
|
||||
separator = ''
|
||||
if isChar(self.rtfData[i], ' '):
|
||||
|
||||
@ -1,10 +1,11 @@
|
||||
from calibre.ebooks.metadata import authors_to_string
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||
|
||||
import os, textwrap, traceback, re, shutil
|
||||
from operator import attrgetter
|
||||
|
||||
from math import cos, sin, pi
|
||||
from contextlib import closing
|
||||
|
||||
from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \
|
||||
QItemDelegate, QPainterPath, QLinearGradient, QBrush, \
|
||||
QPen, QStyle, QPainter, \
|
||||
@ -22,7 +23,8 @@ from calibre.gui2 import NONE, TableView, qstring_to_unicode, config, \
|
||||
from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
|
||||
from calibre.utils.search_query_parser import SearchQueryParser
|
||||
from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
|
||||
from calibre.ebooks.metadata import string_to_authors, fmt_sidx
|
||||
from calibre.ebooks.metadata import string_to_authors, fmt_sidx, \
|
||||
authors_to_string
|
||||
from calibre.utils.config import tweaks
|
||||
from calibre.utils.date import dt_factory, qt_to_dt, isoformat
|
||||
|
||||
@ -469,9 +471,10 @@ class BooksModel(QAbstractTableModel):
|
||||
break
|
||||
if format is not None:
|
||||
pt = PersistentTemporaryFile(suffix='.'+format)
|
||||
src = self.db.format(id, format, index_is_id=True, as_file=True)
|
||||
shutil.copyfileobj(src, pt)
|
||||
pt.flush()
|
||||
with closing(self.db.format(id, format, index_is_id=True,
|
||||
as_file=True)) as src:
|
||||
shutil.copyfileobj(src, pt)
|
||||
pt.flush()
|
||||
pt.seek(0)
|
||||
if set_metadata:
|
||||
_set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True),
|
||||
@ -505,8 +508,10 @@ class BooksModel(QAbstractTableModel):
|
||||
break
|
||||
if format is not None:
|
||||
pt = PersistentTemporaryFile(suffix='.'+format)
|
||||
pt.write(self.db.format(row, format))
|
||||
pt.flush()
|
||||
with closing(self.db.format(row, format, as_file=True)) as src:
|
||||
shutil.copyfileobj(src, pt)
|
||||
pt.flush()
|
||||
pt.seek(0)
|
||||
if set_metadata:
|
||||
_set_metadata(pt, self.db.get_metadata(row, get_cover=True),
|
||||
format)
|
||||
|
||||
@ -6,14 +6,14 @@ __docformat__ = 'restructuredtext en'
|
||||
|
||||
'''
|
||||
A parser for search queries with a syntax very similar to that used by
|
||||
the Google search engine.
|
||||
the Google search engine.
|
||||
|
||||
For details on the search query syntax see :class:`SearchQueryParser`.
|
||||
To use the parser, subclass :class:`SearchQueryParser` and implement the
|
||||
methods :method:`SearchQueryParser.universal_set` and
|
||||
For details on the search query syntax see :class:`SearchQueryParser`.
|
||||
To use the parser, subclass :class:`SearchQueryParser` and implement the
|
||||
methods :method:`SearchQueryParser.universal_set` and
|
||||
:method:`SearchQueryParser.get_matches`. See for example :class:`Tester`.
|
||||
|
||||
If this module is run, it will perform a series of unit tests.
|
||||
If this module is run, it will perform a series of unit tests.
|
||||
'''
|
||||
|
||||
import sys, string, operator
|
||||
@ -24,26 +24,26 @@ from calibre.utils.pyparsing import Keyword, Group, Forward, CharsNotIn, Suppres
|
||||
|
||||
class SearchQueryParser(object):
|
||||
'''
|
||||
Parses a search query.
|
||||
|
||||
Parses a search query.
|
||||
|
||||
A search query consists of tokens. The tokens can be combined using
|
||||
the `or`, `and` and `not` operators as well as grouped using parentheses.
|
||||
the `or`, `and` and `not` operators as well as grouped using parentheses.
|
||||
When no operator is specified between two tokens, `and` is assumed.
|
||||
|
||||
|
||||
Each token is a string of the form `location:query`. `location` is a string
|
||||
from :member:`LOCATIONS`. It is optional. If it is omitted, it is assumed to
|
||||
be `all`. `query` is an arbitrary string that must not contain parentheses.
|
||||
from :member:`LOCATIONS`. It is optional. If it is omitted, it is assumed to
|
||||
be `all`. `query` is an arbitrary string that must not contain parentheses.
|
||||
If it contains whitespace, it should be quoted by enclosing it in `"` marks.
|
||||
|
||||
|
||||
Examples::
|
||||
|
||||
|
||||
* `Asimov` [search for the string "Asimov" in location `all`]
|
||||
* `comments:"This is a good book"` [search for "This is a good book" in `comments`]
|
||||
* `author:Asimov tag:unread` [search for books by Asimov that have been tagged as unread]
|
||||
* `author:Asimov or author:Hardy` [search for books by Asimov or Hardy]
|
||||
* `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy]
|
||||
* `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy]
|
||||
'''
|
||||
|
||||
|
||||
LOCATIONS = [
|
||||
'tag',
|
||||
'title',
|
||||
@ -57,12 +57,12 @@ class SearchQueryParser(object):
|
||||
'isbn',
|
||||
'all',
|
||||
]
|
||||
|
||||
|
||||
@staticmethod
|
||||
def run_tests(parser, result, tests):
|
||||
failed = []
|
||||
for test in tests:
|
||||
print '\tTesting:', test[0],
|
||||
print '\tTesting:', test[0],
|
||||
res = parser.parseString(test[0])
|
||||
if list(res.get(result, None)) == test[1]:
|
||||
print 'OK'
|
||||
@ -70,7 +70,7 @@ class SearchQueryParser(object):
|
||||
print 'FAILED:', 'Expected:', test[1], 'Got:', list(res.get(result, None))
|
||||
failed.append(test[0])
|
||||
return failed
|
||||
|
||||
|
||||
def __init__(self, test=False):
|
||||
self._tests_failed = False
|
||||
# Define a token
|
||||
@ -95,50 +95,50 @@ class SearchQueryParser(object):
|
||||
('title:"one two"', ['title', 'one two']),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Or = Forward()
|
||||
|
||||
|
||||
Parenthesis = Group(
|
||||
Suppress('(') + Or + Suppress(')')
|
||||
).setResultsName('parenthesis') | Token
|
||||
|
||||
|
||||
|
||||
|
||||
Not = Forward()
|
||||
Not << (Group(
|
||||
Suppress(Keyword("not", caseless=True)) + Not
|
||||
).setResultsName("not") | Parenthesis)
|
||||
|
||||
|
||||
And = Forward()
|
||||
And << (Group(
|
||||
Not + Suppress(Keyword("and", caseless=True)) + And
|
||||
).setResultsName("and") | Group(
|
||||
Not + OneOrMore(~oneOf("and or") + And)
|
||||
Not + OneOrMore(~oneOf("and or", caseless=True) + And)
|
||||
).setResultsName("and") | Not)
|
||||
|
||||
|
||||
Or << (Group(
|
||||
And + Suppress(Keyword("or", caseless=True)) + Or
|
||||
).setResultsName("or") | And)
|
||||
|
||||
|
||||
if test:
|
||||
Or.validate()
|
||||
self._tests_failed = bool(failed)
|
||||
|
||||
|
||||
self._parser = Or
|
||||
#self._parser.setDebug(True)
|
||||
#self.parse('(tolstoy)')
|
||||
self._parser.setDebug(False)
|
||||
|
||||
|
||||
|
||||
|
||||
def parse(self, query):
|
||||
res = self._parser.parseString(query)[0]
|
||||
return self.evaluate(res)
|
||||
|
||||
|
||||
def method(self, group_name):
|
||||
return getattr(self, 'evaluate_'+group_name)
|
||||
|
||||
|
||||
def evaluate(self, parse_result):
|
||||
return self.method(parse_result.getName())(parse_result)
|
||||
|
||||
|
||||
def evaluate_and(self, argument):
|
||||
return self.evaluate(argument[0]).intersection(self.evaluate(argument[1]))
|
||||
|
||||
@ -150,27 +150,27 @@ class SearchQueryParser(object):
|
||||
|
||||
def evaluate_parenthesis(self, argument):
|
||||
return self.evaluate(argument[0])
|
||||
|
||||
|
||||
def evaluate_token(self, argument):
|
||||
return self.get_matches(argument[0], argument[1])
|
||||
|
||||
|
||||
def get_matches(self, location, query):
|
||||
'''
|
||||
Should return the set of matches for :param:'location` and :param:`query`.
|
||||
|
||||
Should return the set of matches for :param:'location` and :param:`query`.
|
||||
|
||||
:param:`location` is one of the items in :member:`SearchQueryParser.LOCATIONS`.
|
||||
:param:`query` is a string literal.
|
||||
:param:`query` is a string literal.
|
||||
'''
|
||||
return set([])
|
||||
|
||||
|
||||
def universal_set(self):
|
||||
'''
|
||||
Should return the set of all matches.
|
||||
'''
|
||||
return set([])
|
||||
|
||||
|
||||
class Tester(SearchQueryParser):
|
||||
|
||||
|
||||
texts = {
|
||||
1: [u'Eugenie Grandet', u'Honor\xe9 de Balzac', u'manybooks.net', u'lrf'],
|
||||
2: [u'Fanny Hill', u'John Cleland', u'manybooks.net', u'lrf'],
|
||||
@ -459,30 +459,30 @@ class Tester(SearchQueryParser):
|
||||
u'Washington Square Press',
|
||||
u'lrf,rar']
|
||||
}
|
||||
|
||||
|
||||
tests = {
|
||||
'Dysfunction' : set([348]),
|
||||
'title:Dysfunction' : set([348]),
|
||||
'title:Dysfunction or author:Laurie': set([348, 444]),
|
||||
'title:Dysfunction OR author:Laurie': set([348, 444]),
|
||||
'(tag:txt or tag:pdf)': set([33, 258, 354, 305, 242, 51, 55, 56, 154]),
|
||||
'(tag:txt or tag:pdf) and author:Tolstoy': set([55, 56]),
|
||||
'(tag:txt OR tag:pdf) and author:Tolstoy': set([55, 56]),
|
||||
'Tolstoy txt': set([55, 56]),
|
||||
'Hamilton Amsterdam' : set([]),
|
||||
u'Beär' : set([91]),
|
||||
'dysfunc or tolstoy': set([348, 55, 56]),
|
||||
'tag:txt and not tolstoy': set([33, 258, 354, 305, 242, 154]),
|
||||
'tag:txt AND NOT tolstoy': set([33, 258, 354, 305, 242, 154]),
|
||||
'not tag:lrf' : set([305]),
|
||||
'london:thames': set([13]),
|
||||
'publisher:london:thames': set([13]),
|
||||
'"(1977)"': set([13]),
|
||||
}
|
||||
fields = {'title':0, 'author':1, 'publisher':2, 'tag':3}
|
||||
|
||||
|
||||
_universal_set = set(texts.keys())
|
||||
|
||||
|
||||
def universal_set(self):
|
||||
return self._universal_set
|
||||
|
||||
|
||||
def get_matches(self, location, query):
|
||||
location = location.lower()
|
||||
if location in self.fields.keys():
|
||||
@ -491,19 +491,19 @@ class Tester(SearchQueryParser):
|
||||
getter = lambda y: ''.join(x if x else '' for x in y)
|
||||
else:
|
||||
getter = lambda x: ''
|
||||
|
||||
|
||||
if not query:
|
||||
return set([])
|
||||
return set([])
|
||||
query = query.lower()
|
||||
return set(key for key, val in self.texts.items() \
|
||||
if query and query in getattr(getter(val), 'lower', lambda : '')())
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def run_tests(self):
|
||||
failed = []
|
||||
for query in self.tests.keys():
|
||||
print 'Testing query:', query,
|
||||
print 'Testing query:', query,
|
||||
res = self.parse(query)
|
||||
if res != self.tests[query]:
|
||||
print 'FAILED', 'Expected:', self.tests[query], 'Got:', res
|
||||
@ -511,7 +511,7 @@ class Tester(SearchQueryParser):
|
||||
else:
|
||||
print 'OK'
|
||||
return failed
|
||||
|
||||
|
||||
|
||||
def main(args=sys.argv):
|
||||
tester = Tester(test=True)
|
||||
@ -519,7 +519,7 @@ def main(args=sys.argv):
|
||||
if tester._tests_failed or failed:
|
||||
print '>>>>>>>>>>>>>> Tests Failed <<<<<<<<<<<<<<<'
|
||||
return 1
|
||||
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user