From 2e8e0f6daf05b9395d738e2e1836f89158ba8e6b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 22 Feb 2013 18:48:33 +0530 Subject: [PATCH 1/5] Finish writing one-one fields --- src/calibre/db/tests/writing.py | 32 ++++++++++++++++++++++++++++---- src/calibre/db/write.py | 10 +++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 7b7e815587..f6cf4d2d45 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -41,11 +41,11 @@ class WritingTest(BaseTest): self.create_setter(name, setter)) def run_tests(self, tests): - cl = self.cloned_library results = {} for test in tests: results[test] = [] for val in test.vals: + cl = self.cloned_library cache = self.init_cache(cl) cache.set_field(test.name, {1: val}) cached_res = cache.field_for(test.name, 1) @@ -65,11 +65,16 @@ class WritingTest(BaseTest): test.name, old_sqlite_res, sqlite_res)) del db - - def test_one_one(self): 'Test setting of values in one-one fields' - tests = [] + tests = [self.create_test('#yesno', (True, False, 'true', 'false', None))] + for name, getter, setter in ( + ('series_index', 'series_index', 'set_series_index'), + ('#float', None, None), + ): + vals = ['1.5', None, 0, 1.0] + tests.append(self.create_test(name, tuple(vals), getter, setter)) + for name, getter, setter in ( ('pubdate', 'pubdate', 'set_pubdate'), ('timestamp', 'timestamp', 'set_timestamp'), @@ -78,6 +83,25 @@ class WritingTest(BaseTest): tests.append(self.create_test( name, ('2011-1-12', UNDEFINED_DATE, None), getter, setter)) + for name, getter, setter in ( + ('title', 'title', 'set_title'), + ('uuid', 'uuid', 'set_uuid'), + ('author_sort', 'author_sort', 'set_author_sort'), + ('sort', 'title_sort', 'set_title_sort'), + ('#comments', None, None), + ('comments', 'comments', 'set_comment'), + ): + vals = ['something', None] + if name not in {'comments', '#comments'}: + # Setting text column to '' returns None in the new backend + # and '' in the old. I think None is more correct. + vals.append('') + if name == 'comments': + # Again new behavior of deleting comment rather than setting + # empty string is more correct. + vals.remove(None) + tests.append(self.create_test(name, tuple(vals), getter, setter)) + self.run_tests(tests) def tests(): diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 14ddea8dfb..b3449f1387 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -98,10 +98,14 @@ def get_adapter(name, metadata): if name == 'title': return lambda x: ans(x) or _('Unknown') + if name == 'author_sort': + return lambda x: ans(x) or '' if name == 'authors': return lambda x: ans(x) or (_('Unknown'),) if name in {'timestamp', 'last_modified'}: return lambda x: ans(x) or UNDEFINED_DATE + if name == 'series_index': + return lambda x: 1.0 if ans(x) is None else ans(x) return ans # }}} @@ -148,13 +152,17 @@ class Writer(object): if dt == 'composite' or field.name in { 'id', 'cover', 'size', 'path', 'formats', 'news'}: self.set_books_func = dummy + elif self.name[0] == '#' and self.name.endswith('_index'): + # TODO: Implement this + pass elif field.is_many: # TODO: Implement this pass + # TODO: Remember to change commas to | when writing authors to sqlite else: self.set_books_func = (one_one_in_books if field.metadata['table'] == 'books' else one_one_in_other) - if self.name in {'timestamp', 'uuid'}: + if self.name in {'timestamp', 'uuid', 'sort'}: self.accept_vals = bool def set_books(self, book_id_val_map, db): From 836e00211a4a6a24abe80efdc5b3242be49b6b4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 22 Feb 2013 19:45:12 +0530 Subject: [PATCH 2/5] Implement writing custom series index fields --- src/calibre/db/cache.py | 6 ++++++ src/calibre/db/fields.py | 1 + src/calibre/db/tests/writing.py | 33 ++++++++++++++++++++++----------- src/calibre/db/write.py | 18 ++++++++++++++++-- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 236d25c3e9..af22db64d7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -211,6 +211,12 @@ class Cache(object): self.fields['ondevice'] = create_field('ondevice', VirtualTable('ondevice')) + for name, field in self.fields.iteritems(): + if name[0] == '#' and name.endswith('_index'): + field.series_field = self.fields[name[:-len('_index')]] + elif name == 'series_index': + field.series_field = self.fields['series'] + @read_api def field_for(self, name, book_id, default_value=None): ''' diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 46386b9ba5..dee312dbcd 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -46,6 +46,7 @@ class Field(object): elif name == 'languages': self.category_formatter = calibre_langcode_to_name self.writer = Writer(self) + self.series_field = None @property def metadata(self): diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index f6cf4d2d45..314d6fbb7a 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -22,7 +22,11 @@ class WritingTest(BaseTest): def create_getter(self, name, getter=None): if getter is None: - ans = lambda db:partial(db.get_custom, label=name[1:], + if name.endswith('_index'): + ans = lambda db:partial(db.get_custom_extra, index_is_id=True, + label=name[1:].replace('_index', '')) + else: + ans = lambda db:partial(db.get_custom, label=name[1:], index_is_id=True) else: ans = lambda db:partial(getattr(db, getter), index_is_id=True) @@ -53,22 +57,29 @@ class WritingTest(BaseTest): db = self.init_old(cl) getter = test.getter(db) sqlite_res = getter(1) - test.setter(db)(1, val) - old_cached_res = getter(1) - self.assertEqual(old_cached_res, cached_res, - 'Failed setting for %s with value %r, cached value not the same. Old: %r != New: %r'%( - test.name, val, old_cached_res, cached_res)) - db.refresh() - old_sqlite_res = getter(1) - self.assertEqual(old_sqlite_res, sqlite_res, - 'Failed setting for %s, sqlite value not the same: %r != %r'%( - test.name, old_sqlite_res, sqlite_res)) + if test.name.endswith('_index'): + val = float(val) if val is not None else 1.0 + self.assertEqual(sqlite_res, val, + 'Failed setting for %s with value %r, sqlite value not the same. val: %r != sqlite_val: %r'%( + test.name, val, val, sqlite_res)) + else: + test.setter(db)(1, val) + old_cached_res = getter(1) + self.assertEqual(old_cached_res, cached_res, + 'Failed setting for %s with value %r, cached value not the same. Old: %r != New: %r'%( + test.name, val, old_cached_res, cached_res)) + db.refresh() + old_sqlite_res = getter(1) + self.assertEqual(old_sqlite_res, sqlite_res, + 'Failed setting for %s, sqlite value not the same: %r != %r'%( + test.name, old_sqlite_res, sqlite_res)) del db def test_one_one(self): 'Test setting of values in one-one fields' tests = [self.create_test('#yesno', (True, False, 'true', 'false', None))] for name, getter, setter in ( + ('#series_index', None, None), ('series_index', 'series_index', 'set_series_index'), ('#float', None, None), ): diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index b3449f1387..cb710d271f 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -138,6 +138,21 @@ def one_one_in_other(book_id_val_map, db, field, *args): field.table.book_col_map.update(updated) return set(book_id_val_map) +def custom_series_index(book_id_val_map, db, field, *args): + series_field = field.series_field + sequence = [] + for book_id, sidx in book_id_val_map.iteritems(): + if sidx is None: + sidx = 1.0 + ids = series_field.ids_for_book(book_id) + if ids: + sequence.append((sidx, book_id, ids[0])) + field.table.book_col_map[book_id] = sidx + if sequence: + db.conn.executemany('UPDATE %s SET %s=? WHERE book=? AND value=?'%( + field.metadata['table'], field.metadata['column']), sequence) + return {s[0] for s in sequence} + def dummy(book_id_val_map, *args): return set() @@ -153,8 +168,7 @@ class Writer(object): 'id', 'cover', 'size', 'path', 'formats', 'news'}: self.set_books_func = dummy elif self.name[0] == '#' and self.name.endswith('_index'): - # TODO: Implement this - pass + self.set_books_func = custom_series_index elif field.is_many: # TODO: Implement this pass From 364b5135a814fffa28cbafdf7f4298d4b05dc813 Mon Sep 17 00:00:00 2001 From: Ismael Mejia Date: Fri, 22 Feb 2013 15:21:03 +0100 Subject: [PATCH 3/5] Added recipes for colombian media --- recipes/el_malpensante.recipe | 27 +++++++++++++++++++++++++++ recipes/revista_cromos.recipe | 33 +++++++++++++++++++++++++++++++++ recipes/unperiodico.recipe | 22 ++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 recipes/el_malpensante.recipe create mode 100644 recipes/revista_cromos.recipe create mode 100644 recipes/unperiodico.recipe diff --git a/recipes/el_malpensante.recipe b/recipes/el_malpensante.recipe new file mode 100644 index 0000000000..7a014735b6 --- /dev/null +++ b/recipes/el_malpensante.recipe @@ -0,0 +1,27 @@ +# coding=utf-8 +# https://github.com/iemejia/calibrecolombia + +''' +http://www.elmalpensante.com/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ElMalpensante(BasicNewsRecipe): + title = u'El Malpensante' + language = 'es_CO' + __author__ = 'Ismael Mejia ' + cover_url = 'http://elmalpensante.com/img/layout/logo.gif' + description = 'El Malpensante' + oldest_article = 30 + simultaneous_downloads = 20 + #tags = 'news, sport, blog' + use_embedded_content = True + remove_empty_feeds = True + max_articles_per_feed = 100 + feeds = [(u'Artículos', u'http://www.elmalpensante.com/articulosRSS.php'), + (u'Malpensantías', u'http://www.elmalpensante.com/malpensantiasRSS.php'), + (u'Margaritas', u'http://www.elmalpensante.com/margaritasRSS.php'), +# This one is almost the same as articulos so we leave articles +# (u'Noticias', u'http://www.elmalpensante.com/noticiasRSS.php'), + ] diff --git a/recipes/revista_cromos.recipe b/recipes/revista_cromos.recipe new file mode 100644 index 0000000000..29515971dd --- /dev/null +++ b/recipes/revista_cromos.recipe @@ -0,0 +1,33 @@ +# coding=utf-8 +# https://github.com/iemejia/calibrecolombia + +''' +http://www.cromos.com.co/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ElMalpensante(BasicNewsRecipe): + title = u'Revista Cromos' + language = 'es_CO' + __author__ = 'Ismael Mejia ' + cover_url = 'http://www.cromos.com.co/sites/cromos.com.co/themes/cromos_theme/images/logo_morado.gif' + description = 'Revista Cromos' + oldest_article = 7 + simultaneous_downloads = 20 + #tags = 'news, sport, blog' + use_embedded_content = True + remove_empty_feeds = True + max_articles_per_feed = 100 + feeds = [(u'Cromos', u'http://www.cromos.com.co/rss.xml'), + (u'Moda', u'http://www.cromos.com.co/moda/feed'), + (u'Estilo de Vida', u'http://www.cromos.com.co/estilo-de-vida/feed'), + (u'Cuidado Personal', u'http://www.cromos.com.co/estilo-de-vida/cuidado-personal/feed'), + (u'Salud y Alimentación', u'http://www.cromos.com.co/estilo-de-vida/salud-y-alimentacion/feed'), + (u'Personajes', u'http://www.cromos.com.co/personajes/feed'), + (u'Actualidad', u'http://www.cromos.com.co/personajes/actualidad/feed'), + (u'Espectáculo', u'http://www.cromos.com.co/personajes/espectaculo/feed'), + (u'Reportajes', u'http://www.cromos.com.co/reportajes/feed'), + (u'Eventos', u'http://www.cromos.com.co/eventos/feed'), + (u'Modelos', u'http://www.cromos.com.co/modelos/feed'), + ] diff --git a/recipes/unperiodico.recipe b/recipes/unperiodico.recipe new file mode 100644 index 0000000000..d4edb4e5dc --- /dev/null +++ b/recipes/unperiodico.recipe @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# https://github.com/iemejia/calibrecolombia + +''' +http://www.unperiodico.unal.edu.co/ +''' + +from calibre import strftime +from calibre.web.feeds.news import BasicNewsRecipe + +class UNPeriodico(BasicNewsRecipe): + title = u'UN Periodico' + language = 'es_CO' + __author__ = 'Ismael Mejia ' + cover_url = 'http://www.unperiodico.unal.edu.co/fileadmin/templates/periodico/img/logoperiodico.png' + description = 'UN Periodico' + oldest_article = 30 + max_articles_per_feed = 100 + publication_type = 'newspaper' + feeds = [ + (u'UNPeriodico', u'http://www.unperiodico.unal.edu.co/rss/type/rss2/') + ] From f2f8bcfae1346cce48e2ee2f37f0c2e58b3fd411 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 22 Feb 2013 21:51:11 +0530 Subject: [PATCH 4/5] ... --- src/calibre/gui2/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index f722cf226f..d43e618e9a 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -369,7 +369,7 @@ def build_pipe(print_error=True): t.start() t.join(3.0) if t.is_alive(): - if iswindows(): + if iswindows: cant_start() else: f = os.path.expanduser('~/.calibre_calibre GUI.lock') From d95b50d2e006c0de5eb9caec6515388d0d7453af Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 22 Feb 2013 22:08:09 +0530 Subject: [PATCH 5/5] Update wsj recipe for javascript login --- recipes/wsj.recipe | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index f4254ee7cc..a6a7aa634d 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -55,20 +55,14 @@ class WallStreetJournal(BasicNewsRecipe): ] remove_tags_after = [dict(id="article_story_body"), {'class':"article story"},] + use_javascript_to_login = True - def get_browser(self): - br = BasicNewsRecipe.get_browser(self) - if self.username is not None and self.password is not None: - br.open('http://commerce.wsj.com/auth/login') - br.select_form(nr=1) - br['user'] = self.username - br['password'] = self.password - res = br.submit() - raw = res.read() - if 'Welcome,' not in raw and '>Logout<' not in raw and '>Log Out<' not in raw: - raise ValueError('Failed to log in to wsj.com, check your ' - 'username and password') - return br + def javascript_login(self, br, username, password): + br.visit('https://id.wsj.com/access/pages/wsj/us/login_standalone.html?mg=com-wsj', timeout=120) + f = br.select_form(nr=0) + f['username'] = username + f['password'] = password + br.submit(timeout=120) def populate_article_metadata(self, article, soup, first): if first and hasattr(self, 'add_toc_thumbnail'):