diff --git a/recipes/before_we_go.recipe b/recipes/before_we_go.recipe new file mode 100644 index 0000000000..205e833a9f --- /dev/null +++ b/recipes/before_we_go.recipe @@ -0,0 +1,37 @@ +#!/usr/bin/env python + # vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class BeforeWeGo(BasicNewsRecipe): + title = 'Before We Go' + __author__ = 'bugmen00t' + description = 'Before We Go Blog is a collective of Fantasy, Sci-Fi and Graphic Novel fans from around the world, passionate about providing awesome, enjoyable reviews for anyone who loves a good book!' # noqa + publisher = 'BEFOREWEGOBLOG' + category = 'blog' +# cover_url = u'https://i0.wp.com/beforewegoblog.com/wp-content/uploads/2021/09/beth-with-long-hair.jpeg' + cover_url = u'https://i0.wp.com/beforewegoblog.com/wp-content/uploads/2021/01/before-we-go-blog-1.png' + language = 'en' + no_stylesheets = True + remove_javascript = False + auto_cleanup = False + oldest_article = 14 + max_articles_per_feed = 10 + + remove_tags_before = dict(name='h1', attrs={'class': 'entry-title'}) + + remove_tags_after = dict(name='div', attrs={'id': 'author-bio'}) +# remove_tags_after = dict(name='article') + + remove_tags = [ + dict(name='div', attrs={'class': 'nectar-scrolling-text font_size_10vh custom_color has-custom-divider'}), + dict(name='span', attrs={'class': 'meta-comment-count'}), + dict(name='p', attrs={'id': 'breadcrumbs'}) + ] + + feeds = [ + ('Before We Go', 'https://beforewegoblog.com/feed/') + ] diff --git a/recipes/dev_ua.recipe b/recipes/dev_ua.recipe new file mode 100644 index 0000000000..0eb8c482a5 --- /dev/null +++ b/recipes/dev_ua.recipe @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class WiComix(BasicNewsRecipe): + title = 'dev.ua' + __author__ = 'bugmen00t' + publisher = '\u0422\u041E\u0412 \u00AB\u0414\u0435\u0432 \u0423\u043A\u0440\u0430\u0457\u043D\u0430\u00BB' + category = 'news' + cover_url = u'https://jobs.dev.ua/storage/images/34/70/82/79/original/af4c5c155ec48ed68e1c77ca26a8f0b0.jpg' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 3 + max_articles_per_feed = 30 + description = '\u041C\u0435\u0434\u0456\u0430 \u043F\u0440\u043E \u0442\u0435\u0445\u043D\u043E\u043B\u043E\u0433\u0456\u0457, \u0456\u043D\u043D\u043E\u0432\u0430\u0446\u0456\u0457, \u0432\u0438\u043D\u0430\u0445\u043E\u0434\u0438 \u0432 \u0423\u043A\u0440\u0430\u0457\u043D\u0456 \u0442\u0430 \u0437\u0430 \u0457\u0457 \u043C\u0435\u0436\u0430\u043C\u0438. \u041F\u0440\u043E \u0442\u0435, \u044F\u043A \u0432\u043E\u043D\u0438 \u0432\u043F\u043B\u0438\u0432\u0430\u044E\u0442\u044C \u043D\u0430 \u0436\u0438\u0442\u0442\u044F \u043B\u044E\u0434\u0435\u0439.' # noqa + language = 'uk' + + remove_tags_before = dict(name='div', attrs={'class': 'article__header'}) + + remove_tags_after = dict(name='div', attrs={'class': 'article__body'}) + + remove_tags = [ + dict(name='div', attrs={'class': 'article__reference article__reference_header'}), + dict(name='div', attrs={'class': 'my-lg-5'}), + dict(name='div', attrs={'class': 'video '}), + dict(name='figure', attrs={'class': 'incut'}), + dict(name='figure', attrs={'class': 'global-incut'}) + ] + + feeds = [ + ('dev.ua', 'https://dev.ua/rss') + ] diff --git a/recipes/icons/afr.png b/recipes/icons/afr.png index f3deacc513..58f57a288a 100644 Binary files a/recipes/icons/afr.png and b/recipes/icons/afr.png differ diff --git a/recipes/icons/ancient_egypt.png b/recipes/icons/ancient_egypt.png index e74f750c22..8eb9f7f334 100644 Binary files a/recipes/icons/ancient_egypt.png and b/recipes/icons/ancient_egypt.png differ diff --git a/recipes/icons/before_we_go.png b/recipes/icons/before_we_go.png new file mode 100644 index 0000000000..a9b850e9c9 Binary files /dev/null and b/recipes/icons/before_we_go.png differ diff --git a/recipes/icons/dev_ua.png b/recipes/icons/dev_ua.png new file mode 100644 index 0000000000..192bac5e89 Binary files /dev/null and b/recipes/icons/dev_ua.png differ diff --git a/recipes/icons/economist_espresso.png b/recipes/icons/economist_espresso.png index c338e06599..488b1c29ff 100644 Binary files a/recipes/icons/economist_espresso.png and b/recipes/icons/economist_espresso.png differ diff --git a/recipes/icons/economist_world_ahead.png b/recipes/icons/economist_world_ahead.png index aa6c876a77..237069ce34 100644 Binary files a/recipes/icons/economist_world_ahead.png and b/recipes/icons/economist_world_ahead.png differ diff --git a/recipes/icons/firstpost.png b/recipes/icons/firstpost.png index 693d6365d1..ae891ee5f4 100644 Binary files a/recipes/icons/firstpost.png and b/recipes/icons/firstpost.png differ diff --git a/recipes/icons/gates_notes.png b/recipes/icons/gates_notes.png index 17447a828c..3f01530c2f 100644 Binary files a/recipes/icons/gates_notes.png and b/recipes/icons/gates_notes.png differ diff --git a/recipes/icons/good_e-reader.png b/recipes/icons/good_e-reader.png index 2db5a5b694..8ddf68e7b5 100644 Binary files a/recipes/icons/good_e-reader.png and b/recipes/icons/good_e-reader.png differ diff --git a/recipes/icons/harpers.png b/recipes/icons/harpers.png index 268454be1e..2bcd486ad1 100644 Binary files a/recipes/icons/harpers.png and b/recipes/icons/harpers.png differ diff --git a/recipes/icons/hindutamil.png b/recipes/icons/hindutamil.png index 029c72b471..ee59654c86 100644 Binary files a/recipes/icons/hindutamil.png and b/recipes/icons/hindutamil.png differ diff --git a/recipes/icons/ifzm.png b/recipes/icons/ifzm.png index 1e660b7409..becfe9bd31 100644 Binary files a/recipes/icons/ifzm.png and b/recipes/icons/ifzm.png differ diff --git a/recipes/icons/kirkusreviews.png b/recipes/icons/kirkusreviews.png index abca544dbd..7798f66579 100644 Binary files a/recipes/icons/kirkusreviews.png and b/recipes/icons/kirkusreviews.png differ diff --git a/recipes/icons/military_history.png b/recipes/icons/military_history.png index 26f8e2f282..c991353513 100644 Binary files a/recipes/icons/military_history.png and b/recipes/icons/military_history.png differ diff --git a/recipes/icons/minerva_magazine.png b/recipes/icons/minerva_magazine.png index 57be82fff8..1e3585c000 100644 Binary files a/recipes/icons/minerva_magazine.png and b/recipes/icons/minerva_magazine.png differ diff --git a/recipes/icons/nautilus.png b/recipes/icons/nautilus.png index 6cfc351d79..eeeb38d263 100644 Binary files a/recipes/icons/nautilus.png and b/recipes/icons/nautilus.png differ diff --git a/recipes/icons/new_scientist_mag.png b/recipes/icons/new_scientist_mag.png index 16523e4981..844c1e951c 100644 Binary files a/recipes/icons/new_scientist_mag.png and b/recipes/icons/new_scientist_mag.png differ diff --git a/recipes/icons/newslaundry.png b/recipes/icons/newslaundry.png index 342e059d0d..bcea93df6c 100644 Binary files a/recipes/icons/newslaundry.png and b/recipes/icons/newslaundry.png differ diff --git a/recipes/icons/nikkeiasia.png b/recipes/icons/nikkeiasia.png index 966d9fd485..92f1ee523f 100644 Binary files a/recipes/icons/nikkeiasia.png and b/recipes/icons/nikkeiasia.png differ diff --git a/recipes/icons/poetrymagazine.png b/recipes/icons/poetrymagazine.png index 5959ec16ae..c4869ffd9a 100644 Binary files a/recipes/icons/poetrymagazine.png and b/recipes/icons/poetrymagazine.png differ diff --git a/recipes/icons/poligon.png b/recipes/icons/poligon.png new file mode 100644 index 0000000000..0723476946 Binary files /dev/null and b/recipes/icons/poligon.png differ diff --git a/recipes/icons/scroll.png b/recipes/icons/scroll.png index ceabdfe338..f995d76799 100644 Binary files a/recipes/icons/scroll.png and b/recipes/icons/scroll.png differ diff --git a/recipes/icons/t_invariant_en.png b/recipes/icons/t_invariant_en.png new file mode 100644 index 0000000000..c8ff44ae53 Binary files /dev/null and b/recipes/icons/t_invariant_en.png differ diff --git a/recipes/icons/t_invariant_ru.png b/recipes/icons/t_invariant_ru.png new file mode 100644 index 0000000000..c8ff44ae53 Binary files /dev/null and b/recipes/icons/t_invariant_ru.png differ diff --git a/recipes/icons/t_invariant_ua.png b/recipes/icons/t_invariant_ua.png new file mode 100644 index 0000000000..c8ff44ae53 Binary files /dev/null and b/recipes/icons/t_invariant_ua.png differ diff --git a/recipes/icons/the_week_magazine_free.png b/recipes/icons/the_week_magazine_free.png index f8d3c9013f..28afccf9e3 100644 Binary files a/recipes/icons/the_week_magazine_free.png and b/recipes/icons/the_week_magazine_free.png differ diff --git a/recipes/icons/the_week_uk.png b/recipes/icons/the_week_uk.png index f8d3c9013f..28afccf9e3 100644 Binary files a/recipes/icons/the_week_uk.png and b/recipes/icons/the_week_uk.png differ diff --git a/recipes/icons/times_online.png b/recipes/icons/times_online.png index 2e2cdb8ade..61447d8965 100644 Binary files a/recipes/icons/times_online.png and b/recipes/icons/times_online.png differ diff --git a/recipes/icons/tls_mag.png b/recipes/icons/tls_mag.png index de3e20b531..21391b1fec 100644 Binary files a/recipes/icons/tls_mag.png and b/recipes/icons/tls_mag.png differ diff --git a/recipes/icons/toi.png b/recipes/icons/toi.png index 26da33fff6..1c6911c2b7 100644 Binary files a/recipes/icons/toi.png and b/recipes/icons/toi.png differ diff --git a/recipes/icons/toiprint.png b/recipes/icons/toiprint.png index 26da33fff6..1c6911c2b7 100644 Binary files a/recipes/icons/toiprint.png and b/recipes/icons/toiprint.png differ diff --git a/recipes/icons/wicomix.png b/recipes/icons/wicomix.png new file mode 100644 index 0000000000..9ccd38d2f3 Binary files /dev/null and b/recipes/icons/wicomix.png differ diff --git a/recipes/icons/world_archeology.png b/recipes/icons/world_archeology.png index 39a11410c8..363717470a 100644 Binary files a/recipes/icons/world_archeology.png and b/recipes/icons/world_archeology.png differ diff --git a/recipes/poligon.recipe b/recipes/poligon.recipe new file mode 100644 index 0000000000..1a799888fb --- /dev/null +++ b/recipes/poligon.recipe @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class Poligon(BasicNewsRecipe): + title = '\u041F\u043E\u043B\u0438\u0433\u043E\u043D' + __author__ = 'bugmen00t' + description = '\u041D\u0435\u0437\u0430\u0432\u0438\u0441\u0438\u043C\u043E\u0435 \u0438\u043D\u0442\u0435\u0440\u043D\u0435\u0442-\u0438\u0437\u0434\u0430\u043D\u0438\u0435, \u0432\u044B\u043F\u0443\u0441\u043A\u0430\u0435\u043C\u043E\u0435 \u0436\u0443\u0440\u043D\u0430\u043B\u0438\u0441\u0442\u0430\u043C\u0438 \u043D\u0435\u0441\u043A\u043E\u043B\u044C\u043A\u0438\u0445 \u0440\u043E\u0441\u0441\u0438\u0439\u0441\u043A\u0438\u0445 \u0438\u0437\u0434\u0430\u043D\u0438\u0439, \u043F\u043E\u0434\u0432\u0435\u0440\u0433\u0448\u0438\u0445\u0441\u044F \u0434\u0430\u0432\u043B\u0435\u043D\u0438\u044E \u0441\u043E \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u0433\u043E\u0441\u0443\u0434\u0430\u0440\u0441\u0442\u0432\u0430. \u041F\u0438\u0448\u0435\u043C \u043E \u0420\u043E\u0441\u0441\u0438\u0438 \u0438 \u043D\u0435 \u0442\u043E\u043B\u044C\u043A\u043E.' # noqa + publisher = 'poligon.media' + category = 'news' + cover_url = u'https://www.plgnmedia.io/wp-content/uploads/2022/02/poligon-share.png' + language = 'ru' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 14 + max_articles_per_feed = 10 + + remove_tags_after = dict(name='article') + + feeds = [ + ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', 'https://poligonmedia.io/feed/'), + ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', 'https://poligonmedia.io/category/news/feed/'), + ('\u0418\u043D\u0442\u0435\u0440\u0432\u044C\u044E', 'https://poligonmedia.io/category/int/feed/'), + ('\u0420\u0430\u0441\u043A\u043B\u0430\u0434', 'https://poligonmedia.io/category/rasklad/feed/'), + ('\u0420\u0430\u0441\u0441\u043B\u0435\u0434\u043E\u0432\u0430\u043D\u0438\u044F', 'https://poligonmedia.io/category/rassled/feed/'), + ('\u0420\u0435\u043F\u043E\u0440\u0442\u0430\u0436', 'https://poligonmedia.io/category/reportazh/feed/'), + ('\u041C\u043E\u0436\u0435\u043C \u043E\u0431\u044A\u044F\u0441\u043D\u0438\u0442\u044C', 'https://poligonmedia.io/category/mozhem-obyasnit/feed/'), + ('\u0418\u0441\u0441\u043B\u0435\u0434\u043E\u0432\u0430\u043D\u0438\u044F', 'https://poligonmedia.io/category/issledovaniya/feed/'), + ('English', 'https://poligon.media/category/eng/feed/') + ] diff --git a/recipes/t_invariant_en.recipe b/recipes/t_invariant_en.recipe new file mode 100644 index 0000000000..bb09638156 --- /dev/null +++ b/recipes/t_invariant_en.recipe @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class TInvariant(BasicNewsRecipe): + title = 'T-Invariant' + __author__ = 'bugmen00t' + description = 'T-Invariant is a multimedia project of scientists and science journalists. Our task is to be a bridge between the academic community in Russia and outside Russia. Let’s keep in touch!' # noqa + publisher = '\u0422-\u0438\u043D\u0432\u0430\u0440\u0438\u0430\u043D\u0442 / T-invariant' + category = 'news' + cover_url = u'https://t-invariant.org/wp-content/uploads/2023/02/logo-s.png' +# cover_url = u'https://tinyurl.com/t-invariant/wp-content/uploads/2023/02/logo-s.png' +# language = 'ru' + language = 'en_RU' +# language = 'uk' +# language = 'de' +# language = 'he' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 45 + max_articles_per_feed = 15 + + remove_tags_before = dict(name='h1') + + remove_tags_after = dict(name='article') + + remove_tags = [ + dict(name='div', attrs={'class': 'media mg-info-author-block'}), + dict(name='div', attrs={'class': 'mg-blog-category mb-1'}), + dict(name='span', attrs={'class': 'newses-tags'}), + dict(name='div', attrs={'class': 'post-share'}), + dict(name='h4') + ] + + feeds = [ +# Russian version +# +# Direct links +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', ‘https://www.t-invariant.org/feed/'), +# ('\u0410\u0440\u0445\u0438\u0432', ‘https://www.t-invariant.org/category/online/feed/'), +# ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', ‘https://www.t-invariant.org/category/astronomy/feed/'), +# ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/biologiya/feed/'), +# ('\u0412\u043E\u0439\u043D\u0430', ‘https://www.t-invariant.org/category/war/feed/'), +# ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', ‘https://www.t-invariant.org/category/vsyo-slozhno/feed/'), +# ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', ‘https://www.t-invariant.org/category/dva-goda-vojne/feed/'), +# ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/dve-storony-odnoj-nauki/feed/'), +# ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/discussion/feed/'), +# ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', ‘https://www.t-invariant.org/category/dissernet/feed/'), +# ('\u0418\u0418', ‘https://www.t-invariant.org/category/ai/feed/'), +# ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/computer-science/feed/'), +# ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', ‘https://www.t-invariant.org/category/history/feed/'), +# ('\u041A\u043B\u0438\u043C\u0430\u0442', ‘https://www.t-invariant.org/category/climate-ru/feed/'), +# ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/kontrevolyutsiya/feed/'), +# ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/korruptsiya/feed/'), +# ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', ‘https://www.t-invariant.org/category/pseudoscience/feed/'), +# ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/mathematics/feed/'), +# ('\u041C\u043E\u0437\u0433', ‘https://www.t-invariant.org/category/brain/feed/'), +# ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', ‘https://www.t-invariant.org/category/ukrainian-science/feed/'), +# ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/science-policy-ru/feed/'), +# ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', ‘https://www.t-invariant.org/category/news/feed/'), +# ('\u041E\u043F\u0440\u043E\u0441', ‘https://www.t-invariant.org/category/survey-ru/feed/'), +# ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/standpiont/feed/'), +# ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/political-science/feed/'), +# ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/popular-science/feed/'), +# ('\u041F\u0440\u0430\u0432\u043E', ‘https://www.t-invariant.org/category/pravo/feed/'), +# ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', ‘https://www.t-invariant.org/category/progress/feed/'), +# ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/psihologiya/feed/'), +# ('\u0420\u0410\u041D', ‘https://www.t-invariant.org/category/ras/feed/'), +# ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/religion/feed/'), +# ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/relocation/feed/'), +# ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/repression/feed/'), +# ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', ‘https://www.t-invariant.org/category/russian-language/feed/'), +# ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', ‘https://www.t-invariant.org/category/sanctions/feed/'), +# ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', ‘https://www.t-invariant.org/category/creators/feed/'), +# ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/sociology/feed/'), +# ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', ‘https://www.t-invariant.org/category/terrorizm/feed/'), +# ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', ‘https://www.t-invariant.org/category/universities/feed/'), +# ('\u0424\u0438\u0437\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/physics/feed/'), +# ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', ‘https://www.t-invariant.org/category/philosophy/feed/'), +# ('\u0428\u043A\u043E\u043B\u0430', ‘https://www.t-invariant.org/category/shkola/feed/'), +# ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/science-assessment/feed/'), +# ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/energy/feed/'), +# ('\u042D\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/ethics/feed/'), +# +# Censorship bypass +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', 'https://tinyurl.com/t-invariant/feed/'), +# ('\u0410\u0440\u0445\u0438\u0432', 'https://tinyurl.com/t-invariant/category/online/feed/'), +# ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', 'https://tinyurl.com/t-invariant/category/astronomy/feed/'), +# ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/biologiya/feed/'), +# ('\u0412\u043E\u0439\u043D\u0430', 'https://tinyurl.com/t-invariant/category/war/feed/'), +# ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', 'https://tinyurl.com/t-invariant/category/vsyo-slozhno/feed/'), +# ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', 'https://tinyurl.com/t-invariant/category/dva-goda-vojne/feed/'), +# ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/dve-storony-odnoj-nauki/feed/'), +# ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/discussion/feed/'), +# ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', 'https://tinyurl.com/t-invariant/category/dissernet/feed/'), +# ('\u0418\u0418', 'https://tinyurl.com/t-invariant/category/ai/feed/'), +# ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/computer-science/feed/'), +# ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', 'https://tinyurl.com/t-invariant/category/history/feed/'), +# ('\u041A\u043B\u0438\u043C\u0430\u0442', 'https://tinyurl.com/t-invariant/category/climate-ru/feed/'), +# ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/kontrevolyutsiya/feed/'), +# ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/korruptsiya/feed/'), +# ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', 'https://tinyurl.com/t-invariant/category/pseudoscience/feed/'), +# ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/mathematics/feed/'), +# ('\u041C\u043E\u0437\u0433', 'https://tinyurl.com/t-invariant/category/brain/feed/'), +# ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', 'https://tinyurl.com/t-invariant/category/ukrainian-science/feed/'), +# ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/science-policy-ru/feed/'), +# ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', 'https://tinyurl.com/t-invariant/category/news/feed/'), +# ('\u041E\u043F\u0440\u043E\u0441', 'https://tinyurl.com/t-invariant/category/survey-ru/feed/'), +# ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/standpiont/feed/'), +# ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/political-science/feed/'), +# ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/popular-science/feed/'), +# ('\u041F\u0440\u0430\u0432\u043E', 'https://tinyurl.com/t-invariant/category/pravo/feed/'), +# ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', 'https://tinyurl.com/t-invariant/category/progress/feed/'), +# ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/psihologiya/feed/'), +# ('\u0420\u0410\u041D', 'https://tinyurl.com/t-invariant/category/ras/feed/'), +# ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/religion/feed/'), +# ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/relocation/feed/'), +# ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/repression/feed/'), +# ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', 'https://tinyurl.com/t-invariant/category/russian-language/feed/'), +# ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', 'https://tinyurl.com/t-invariant/category/sanctions/feed/'), +# ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', 'https://tinyurl.com/t-invariant/category/creators/feed/'), +# ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/sociology/feed/'), +# ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', 'https://tinyurl.com/t-invariant/category/terrorizm/feed/'), +# ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', 'https://tinyurl.com/t-invariant/category/universities/feed/'), +# ('\u0424\u0438\u0437\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/physics/feed/'), +# ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', 'https://tinyurl.com/t-invariant/category/philosophy/feed/'), +# ('\u0428\u043A\u043E\u043B\u0430', 'https://tinyurl.com/t-invariant/category/shkola/feed/'), +# ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/science-assessment/feed/'), +# ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/energy/feed/'), +# ('\u042D\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/ethics/feed/'), +# +# English version +# +# ('T-invariant (English)', 'https://www.t-invariant.org/en/feed/'), +# +# Censorship bypass +# + ('T-invariant (English)', 'https://tinyurl.com/t-invariant/en/feed/'), +# +# Ukrainian version +# +# ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://www.t-invariant.org/uk/feed/'), +# +# Censorship bypass +# +# ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://tinyurl.com/t-invariant/uk/feed/'), +# +# German version +# +# ('T-invariant (Deutsch)', 'https://www.t-invariant.org/de/feed/'), +# +# Censorship bypass +# +# ('T-invariant (Deutsch)', 'https://tinyurl.com/t-invariant/de/feed/'), +# +# Hebrew version +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://www.t-invariant.org/he/feed/'), +# +# Censorship bypass +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://tinyurl.com/t-invariant/he/feed/'), +# + ] diff --git a/recipes/t_invariant_ru.recipe b/recipes/t_invariant_ru.recipe new file mode 100644 index 0000000000..b005182bfa --- /dev/null +++ b/recipes/t_invariant_ru.recipe @@ -0,0 +1,173 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class TInvariant(BasicNewsRecipe): + title = 'T-Invariant' + __author__ = 'bugmen00t' + description = '\u041C\u0443\u043B\u044C\u0442\u0438\u043C\u0435\u0434\u0438\u0439\u043D\u044B\u0439 \u043F\u0440\u043E\u0435\u043A\u0442 \u0443\u0447\u0435\u043D\u044B\u0445 \u0438 \u043D\u0430\u0443\u0447\u043D\u044B\u0445 \u0436\u0443\u0440\u043D\u0430\u043B\u0438\u0441\u0442\u043E\u0432.' # noqa + publisher = '\u0422-\u0438\u043D\u0432\u0430\u0440\u0438\u0430\u043D\u0442 / T-invariant' + category = 'news' +# cover_url = u'https://t-invariant.org/wp-content/uploads/2023/02/logo-s.png' + cover_url = u'https://tinyurl.com/t-invariant/wp-content/uploads/2023/02/logo-s.png' + language = 'ru' +# language = 'en_RU' +# language = 'uk' +# language = 'de' +# language = 'he' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 45 + max_articles_per_feed = 15 + + remove_tags_before = dict(name='h1') + + remove_tags_after = dict(name='article') + + remove_tags = [ + dict(name='div', attrs={'class': 'media mg-info-author-block'}), + dict(name='div', attrs={'class': 'mg-blog-category mb-1'}), + dict(name='span', attrs={'class': 'newses-tags'}), + dict(name='div', attrs={'class': 'post-share'}), + dict(name='h4') + ] + + feeds = [ +# Russian version +# +# Direct links +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', ‘https://www.t-invariant.org/feed/'), +# ('\u0410\u0440\u0445\u0438\u0432', ‘https://www.t-invariant.org/category/online/feed/'), +# ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', ‘https://www.t-invariant.org/category/astronomy/feed/'), +# ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/biologiya/feed/'), +# ('\u0412\u043E\u0439\u043D\u0430', ‘https://www.t-invariant.org/category/war/feed/'), +# ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', ‘https://www.t-invariant.org/category/vsyo-slozhno/feed/'), +# ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', ‘https://www.t-invariant.org/category/dva-goda-vojne/feed/'), +# ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/dve-storony-odnoj-nauki/feed/'), +# ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/discussion/feed/'), +# ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', ‘https://www.t-invariant.org/category/dissernet/feed/'), +# ('\u0418\u0418', ‘https://www.t-invariant.org/category/ai/feed/'), +# ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/computer-science/feed/'), +# ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', ‘https://www.t-invariant.org/category/history/feed/'), +# ('\u041A\u043B\u0438\u043C\u0430\u0442', ‘https://www.t-invariant.org/category/climate-ru/feed/'), +# ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/kontrevolyutsiya/feed/'), +# ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/korruptsiya/feed/'), +# ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', ‘https://www.t-invariant.org/category/pseudoscience/feed/'), +# ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/mathematics/feed/'), +# ('\u041C\u043E\u0437\u0433', ‘https://www.t-invariant.org/category/brain/feed/'), +# ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', ‘https://www.t-invariant.org/category/ukrainian-science/feed/'), +# ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/science-policy-ru/feed/'), +# ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', ‘https://www.t-invariant.org/category/news/feed/'), +# ('\u041E\u043F\u0440\u043E\u0441', ‘https://www.t-invariant.org/category/survey-ru/feed/'), +# ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/standpiont/feed/'), +# ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/political-science/feed/'), +# ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/popular-science/feed/'), +# ('\u041F\u0440\u0430\u0432\u043E', ‘https://www.t-invariant.org/category/pravo/feed/'), +# ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', ‘https://www.t-invariant.org/category/progress/feed/'), +# ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/psihologiya/feed/'), +# ('\u0420\u0410\u041D', ‘https://www.t-invariant.org/category/ras/feed/'), +# ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/religion/feed/'), +# ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/relocation/feed/'), +# ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/repression/feed/'), +# ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', ‘https://www.t-invariant.org/category/russian-language/feed/'), +# ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', ‘https://www.t-invariant.org/category/sanctions/feed/'), +# ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', ‘https://www.t-invariant.org/category/creators/feed/'), +# ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/sociology/feed/'), +# ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', ‘https://www.t-invariant.org/category/terrorizm/feed/'), +# ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', ‘https://www.t-invariant.org/category/universities/feed/'), +# ('\u0424\u0438\u0437\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/physics/feed/'), +# ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', ‘https://www.t-invariant.org/category/philosophy/feed/'), +# ('\u0428\u043A\u043E\u043B\u0430', ‘https://www.t-invariant.org/category/shkola/feed/'), +# ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/science-assessment/feed/'), +# ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/energy/feed/'), +# ('\u042D\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/ethics/feed/'), +# +# Censorship bypass +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', 'https://tinyurl.com/t-invariant/feed/'), + ('\u0410\u0440\u0445\u0438\u0432', 'https://tinyurl.com/t-invariant/category/online/feed/'), + ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', 'https://tinyurl.com/t-invariant/category/astronomy/feed/'), + ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/biologiya/feed/'), + ('\u0412\u043E\u0439\u043D\u0430', 'https://tinyurl.com/t-invariant/category/war/feed/'), + ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', 'https://tinyurl.com/t-invariant/category/vsyo-slozhno/feed/'), + ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', 'https://tinyurl.com/t-invariant/category/dva-goda-vojne/feed/'), + ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/dve-storony-odnoj-nauki/feed/'), + ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/discussion/feed/'), + ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', 'https://tinyurl.com/t-invariant/category/dissernet/feed/'), + ('\u0418\u0418', 'https://tinyurl.com/t-invariant/category/ai/feed/'), + ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/computer-science/feed/'), + ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', 'https://tinyurl.com/t-invariant/category/history/feed/'), + ('\u041A\u043B\u0438\u043C\u0430\u0442', 'https://tinyurl.com/t-invariant/category/climate-ru/feed/'), + ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/kontrevolyutsiya/feed/'), + ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/korruptsiya/feed/'), + ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', 'https://tinyurl.com/t-invariant/category/pseudoscience/feed/'), + ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/mathematics/feed/'), + ('\u041C\u043E\u0437\u0433', 'https://tinyurl.com/t-invariant/category/brain/feed/'), + ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', 'https://tinyurl.com/t-invariant/category/ukrainian-science/feed/'), + ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/science-policy-ru/feed/'), + ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', 'https://tinyurl.com/t-invariant/category/news/feed/'), + ('\u041E\u043F\u0440\u043E\u0441', 'https://tinyurl.com/t-invariant/category/survey-ru/feed/'), + ('\u041E\u0442\u0435\u0447\u0435\u0441\u0442\u0432\u0435\u043D\u043D\u044B\u0435 \u0437\u0430\u043F\u0438\u0441\u043A\u0438 \u0438\u0437 \u043F\u043E\u0434\u043F\u043E\u043B\u044C\u044F', 'https://tinyurl.com/t-invariant/category/story/feed/'), # noqa + ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/standpiont/feed/'), + ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/political-science/feed/'), + ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/popular-science/feed/'), + ('\u041F\u0440\u0430\u0432\u043E', 'https://tinyurl.com/t-invariant/category/pravo/feed/'), + ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', 'https://tinyurl.com/t-invariant/category/progress/feed/'), + ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/psihologiya/feed/'), + ('\u0420\u0410\u041D', 'https://tinyurl.com/t-invariant/category/ras/feed/'), + ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/religion/feed/'), + ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/relocation/feed/'), + ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/repression/feed/'), + ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', 'https://tinyurl.com/t-invariant/category/russian-language/feed/'), + ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', 'https://tinyurl.com/t-invariant/category/sanctions/feed/'), + ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', 'https://tinyurl.com/t-invariant/category/creators/feed/'), + ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/sociology/feed/'), + ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', 'https://tinyurl.com/t-invariant/category/terrorizm/feed/'), + ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', 'https://tinyurl.com/t-invariant/category/universities/feed/'), + ('\u0424\u0438\u0437\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/physics/feed/'), + ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', 'https://tinyurl.com/t-invariant/category/philosophy/feed/'), + ('\u0425\u0440\u043E\u043D\u0438\u043A\u0438 \u043F\u0440\u0435\u0441\u043B\u0435\u0434\u043E\u0432\u0430\u043D\u0438\u044F \u0443\u0447\u0435\u043D\u044B\u0445', 'https://tinyurl.com/t-invariant/category/timeline/feed/'), # noqa + ('\u0428\u043A\u043E\u043B\u0430', 'https://tinyurl.com/t-invariant/category/shkola/feed/'), + ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/science-assessment/feed/'), + ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/energy/feed/'), + ('\u042D\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/ethics/feed/'), +# +# English version +# +# ('T-invariant (English)', 'https://www.t-invariant.org/en/feed/'), +# +# Censorship bypass +# +# ('T-invariant (English)', 'https://tinyurl.com/t-invariant/en/feed/'), +# +# Ukrainian version +# +# ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://www.t-invariant.org/uk/feed/'), +# +# Censorship bypass +# +# ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://tinyurl.com/t-invariant/uk/feed/'), +# +# German version +# +# ('T-invariant (Deutsch)', 'https://www.t-invariant.org/de/feed/'), +# +# Censorship bypass +# +# ('T-invariant (Deutsch)', 'https://tinyurl.com/t-invariant/de/feed/'), +# +# Hebrew version +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://www.t-invariant.org/he/feed/'), +# +# Censorship bypass +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://tinyurl.com/t-invariant/he/feed/'), +# + ] diff --git a/recipes/t_invariant_ua.recipe b/recipes/t_invariant_ua.recipe new file mode 100644 index 0000000000..af46987cad --- /dev/null +++ b/recipes/t_invariant_ua.recipe @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class TInvariant(BasicNewsRecipe): + title = 'T-Invariant' + __author__ = 'bugmen00t' + description = 'T-Invariant is a multimedia project of scientists and science journalists. Our task is to be a bridge between the academic community in Russia and outside Russia. Let’s keep in touch!' # noqa + publisher = '\u0422-\u0438\u043D\u0432\u0430\u0440\u0438\u0430\u043D\u0442 / T-invariant' + category = 'news' + cover_url = u'https://t-invariant.org/wp-content/uploads/2023/02/logo-s.png' +# cover_url = u'https://tinyurl.com/t-invariant/wp-content/uploads/2023/02/logo-s.png' +# language = 'ru' +# language = 'en_RU' + language = 'uk' +# language = 'de' +# language = 'he' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 45 + max_articles_per_feed = 15 + + remove_tags_before = dict(name='h1') + + remove_tags_after = dict(name='article') + + remove_tags = [ + dict(name='div', attrs={'class': 'media mg-info-author-block'}), + dict(name='div', attrs={'class': 'mg-blog-category mb-1'}), + dict(name='span', attrs={'class': 'newses-tags'}), + dict(name='div', attrs={'class': 'post-share'}), + dict(name='h4') + ] + + feeds = [ +# Russian version +# +# Direct links +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', ‘https://www.t-invariant.org/feed/'), +# ('\u0410\u0440\u0445\u0438\u0432', ‘https://www.t-invariant.org/category/online/feed/'), +# ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', ‘https://www.t-invariant.org/category/astronomy/feed/'), +# ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/biologiya/feed/'), +# ('\u0412\u043E\u0439\u043D\u0430', ‘https://www.t-invariant.org/category/war/feed/'), +# ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', ‘https://www.t-invariant.org/category/vsyo-slozhno/feed/'), +# ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', ‘https://www.t-invariant.org/category/dva-goda-vojne/feed/'), +# ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/dve-storony-odnoj-nauki/feed/'), +# ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/discussion/feed/'), +# ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', ‘https://www.t-invariant.org/category/dissernet/feed/'), +# ('\u0418\u0418', ‘https://www.t-invariant.org/category/ai/feed/'), +# ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/computer-science/feed/'), +# ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', ‘https://www.t-invariant.org/category/history/feed/'), +# ('\u041A\u043B\u0438\u043C\u0430\u0442', ‘https://www.t-invariant.org/category/climate-ru/feed/'), +# ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/kontrevolyutsiya/feed/'), +# ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/korruptsiya/feed/'), +# ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', ‘https://www.t-invariant.org/category/pseudoscience/feed/'), +# ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/mathematics/feed/'), +# ('\u041C\u043E\u0437\u0433', ‘https://www.t-invariant.org/category/brain/feed/'), +# ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', ‘https://www.t-invariant.org/category/ukrainian-science/feed/'), +# ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/science-policy-ru/feed/'), +# ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', ‘https://www.t-invariant.org/category/news/feed/'), +# ('\u041E\u043F\u0440\u043E\u0441', ‘https://www.t-invariant.org/category/survey-ru/feed/'), +# ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/standpiont/feed/'), +# ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/political-science/feed/'), +# ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/popular-science/feed/'), +# ('\u041F\u0440\u0430\u0432\u043E', ‘https://www.t-invariant.org/category/pravo/feed/'), +# ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', ‘https://www.t-invariant.org/category/progress/feed/'), +# ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/psihologiya/feed/'), +# ('\u0420\u0410\u041D', ‘https://www.t-invariant.org/category/ras/feed/'), +# ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/religion/feed/'), +# ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', ‘https://www.t-invariant.org/category/relocation/feed/'), +# ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', ‘https://www.t-invariant.org/category/repression/feed/'), +# ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', ‘https://www.t-invariant.org/category/russian-language/feed/'), +# ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', ‘https://www.t-invariant.org/category/sanctions/feed/'), +# ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', ‘https://www.t-invariant.org/category/creators/feed/'), +# ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', ‘https://www.t-invariant.org/category/sociology/feed/'), +# ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', ‘https://www.t-invariant.org/category/terrorizm/feed/'), +# ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', ‘https://www.t-invariant.org/category/universities/feed/'), +# ('\u0424\u0438\u0437\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/physics/feed/'), +# ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', ‘https://www.t-invariant.org/category/philosophy/feed/'), +# ('\u0428\u043A\u043E\u043B\u0430', ‘https://www.t-invariant.org/category/shkola/feed/'), +# ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', ‘https://www.t-invariant.org/category/science-assessment/feed/'), +# ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/energy/feed/'), +# ('\u042D\u0442\u0438\u043A\u0430', ‘https://www.t-invariant.org/category/ethics/feed/'), +# +# Censorship bypass +# +# ('\u0412\u0441\u0435 \u043C\u0430\u0442\u0435\u0440\u0438\u0430\u043B\u044B', 'https://tinyurl.com/t-invariant/feed/'), +# ('\u0410\u0440\u0445\u0438\u0432', 'https://tinyurl.com/t-invariant/category/online/feed/'), +# ('\u0410\u0441\u0442\u0440\u043E\u043D\u043E\u043C\u0438\u044F', 'https://tinyurl.com/t-invariant/category/astronomy/feed/'), +# ('\u0411\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/biologiya/feed/'), +# ('\u0412\u043E\u0439\u043D\u0430', 'https://tinyurl.com/t-invariant/category/war/feed/'), +# ('\u0412\u0441\u0451 \u0441\u043B\u043E\u0436\u043D\u043E', 'https://tinyurl.com/t-invariant/category/vsyo-slozhno/feed/'), +# ('\u0414\u0432\u0430 \u0433\u043E\u0434\u0430 \u0432\u043E\u0439\u043D\u0435', 'https://tinyurl.com/t-invariant/category/dva-goda-vojne/feed/'), +# ('\u0414\u0432\u0435 \u0441\u0442\u043E\u0440\u043E\u043D\u044B \u043E\u0434\u043D\u043E\u0439 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/dve-storony-odnoj-nauki/feed/'), +# ('\u0414\u0438\u0441\u043A\u0443\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/discussion/feed/'), +# ('\u0414\u0438\u0441\u0441\u0435\u0440\u043D\u0435\u0442', 'https://tinyurl.com/t-invariant/category/dissernet/feed/'), +# ('\u0418\u0418', 'https://tinyurl.com/t-invariant/category/ai/feed/'), +# ('\u0418\u043D\u0444\u043E\u0440\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/computer-science/feed/'), +# ('\u0418\u0441\u0442\u043E\u0440\u0438\u044F', 'https://tinyurl.com/t-invariant/category/history/feed/'), +# ('\u041A\u043B\u0438\u043C\u0430\u0442', 'https://tinyurl.com/t-invariant/category/climate-ru/feed/'), +# ('\u041A\u043E\u043D\u0442\u0440\u044D\u0432\u043E\u043B\u044E\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/kontrevolyutsiya/feed/'), +# ('\u041A\u043E\u0440\u0440\u0443\u043F\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/korruptsiya/feed/'), +# ('\u041B\u0436\u0435\u043D\u0430\u0443\u043A\u0430', 'https://tinyurl.com/t-invariant/category/pseudoscience/feed/'), +# ('\u041C\u0430\u0442\u0435\u043C\u0430\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/mathematics/feed/'), +# ('\u041C\u043E\u0437\u0433', 'https://tinyurl.com/t-invariant/category/brain/feed/'), +# ('\u041D\u0430\u0443\u043A\u0430 \u0432 \u0423\u043A\u0440\u0430\u0438\u043D\u0435', 'https://tinyurl.com/t-invariant/category/ukrainian-science/feed/'), +# ('\u041D\u0430\u0443\u0447\u043D\u0430\u044F \u043F\u043E\u043B\u0438\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/science-policy-ru/feed/'), +# ('\u041D\u043E\u0432\u043E\u0441\u0442\u0438', 'https://tinyurl.com/t-invariant/category/news/feed/'), +# ('\u041E\u043F\u0440\u043E\u0441', 'https://tinyurl.com/t-invariant/category/survey-ru/feed/'), +# ('\u041F\u043E\u0437\u0438\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/standpiont/feed/'), +# ('\u041F\u043E\u043B\u0438\u0442\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/political-science/feed/'), +# ('\u041F\u043E\u043F\u0443\u043B\u044F\u0440\u0438\u0437\u0430\u0446\u0438\u044F \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/popular-science/feed/'), +# ('\u041F\u0440\u0430\u0432\u043E', 'https://tinyurl.com/t-invariant/category/pravo/feed/'), +# ('\u041F\u0440\u043E\u0433\u0440\u0435\u0441\u0441', 'https://tinyurl.com/t-invariant/category/progress/feed/'), +# ('\u041F\u0441\u0438\u0445\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/psihologiya/feed/'), +# ('\u0420\u0410\u041D', 'https://tinyurl.com/t-invariant/category/ras/feed/'), +# ('\u0420\u0435\u043B\u0438\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/religion/feed/'), +# ('\u0420\u0435\u043B\u043E\u043A\u0430\u0446\u0438\u044F', 'https://tinyurl.com/t-invariant/category/relocation/feed/'), +# ('\u0420\u0435\u043F\u0440\u0435\u0441\u0441\u0438\u0438', 'https://tinyurl.com/t-invariant/category/repression/feed/'), +# ('\u0420\u0443\u0441\u0441\u043A\u0438\u0439 \u044F\u0437\u044B\u043A', 'https://tinyurl.com/t-invariant/category/russian-language/feed/'), +# ('\u0421\u0430\u043D\u043A\u0446\u0438\u0438', 'https://tinyurl.com/t-invariant/category/sanctions/feed/'), +# ('\u0421\u043E\u0437\u0434\u0430\u0442\u0435\u043B\u0438', 'https://tinyurl.com/t-invariant/category/creators/feed/'), +# ('\u0421\u043E\u0446\u0438\u043E\u043B\u043E\u0433\u0438\u044F', 'https://tinyurl.com/t-invariant/category/sociology/feed/'), +# ('\u0422\u0435\u0440\u0440\u043E\u0440\u0438\u0437\u043C', 'https://tinyurl.com/t-invariant/category/terrorizm/feed/'), +# ('\u0423\u043D\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u044B', 'https://tinyurl.com/t-invariant/category/universities/feed/'), +# ('\u0424\u0438\u0437\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/physics/feed/'), +# ('\u0424\u0438\u043B\u043E\u0441\u043E\u0444\u0438\u044F', 'https://tinyurl.com/t-invariant/category/philosophy/feed/'), +# ('\u0428\u043A\u043E\u043B\u0430', 'https://tinyurl.com/t-invariant/category/shkola/feed/'), +# ('\u042D\u043A\u0441\u043F\u0435\u0440\u0442\u0438\u0437\u0430 \u043D\u0430\u0443\u043A\u0438', 'https://tinyurl.com/t-invariant/category/science-assessment/feed/'), +# ('\u042D\u043D\u0435\u0440\u0433\u0435\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/energy/feed/'), +# ('\u042D\u0442\u0438\u043A\u0430', 'https://tinyurl.com/t-invariant/category/ethics/feed/'), +# +# English version +# +# ('T-invariant (English)', 'https://www.t-invariant.org/en/feed/'), +# +# Censorship bypass +# +# ('T-invariant (English)', 'https://tinyurl.com/t-invariant/en/feed/'), +# +# Ukrainian version +# +# ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://www.t-invariant.org/uk/feed/'), +# +# Censorship bypass +# + ('T-invariant (\u0423\u043A\u0440\u0430\u0457\u043D\u0441\u044C\u043A\u0430)', 'https://tinyurl.com/t-invariant/uk/feed/'), +# +# German version +# +# ('T-invariant (Deutsch)', 'https://www.t-invariant.org/de/feed/'), +# +# Censorship bypass +# +# ('T-invariant (Deutsch)', 'https://tinyurl.com/t-invariant/de/feed/'), +# +# Hebrew version +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://www.t-invariant.org/he/feed/'), +# +# Censorship bypass +# +# ('T-invariant (\u05E2\u05D1\u05E8\u05D9\u05EA)', 'https://tinyurl.com/t-invariant/he/feed/'), +# + ] diff --git a/recipes/wicomix.recipe b/recipes/wicomix.recipe new file mode 100644 index 0000000000..6c9c23b13c --- /dev/null +++ b/recipes/wicomix.recipe @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre.web.feeds.news import BasicNewsRecipe + + +class WiComix(BasicNewsRecipe): + title = 'Wicomix' + __author__ = 'bugmen00t' + description = '\u0418\u043D\u0442\u0435\u0440\u0435\u0441\u043D\u043E \u043F\u0438\u0448\u0435\u043C \u043F\u0440\u043E \u043A\u043E\u043C\u0438\u043A\u0441\u044B \u0438 \u043C\u0430\u043D\u0433\u0443 \u0432 \u0420\u043E\u0441\u0441\u0438\u0438.' # noqa + publisher = '\u0421\u0435\u0440\u0433\u0435\u0439 \u041E\u0440\u0435\u0448\u043A\u0438\u043D' + category = 'blog' + cover_url = u'https://wicomix.files.wordpress.com/2016/03/10865972_581683525300298_6873875730458476144_o.jpg' + language = 'ru' + no_stylesheets = False + remove_javascript = False + auto_cleanup = False + oldest_article = 14 + max_articles_per_feed = 10 + + remove_tags_before = dict(name='article') + + remove_tags_after = dict(name='article') + + remove_tags = [ +# dict(name='div', attrs={'class': 'author-meta'}), + dict(name='div', attrs={'id': 'jp-post-flair'}), + dict(name='footer', attrs={'class': 'entry-meta'}) + ] + + feeds = [ + ('\u041F\u0443\u0431\u043B\u0438\u043A\u0430\u0446\u0438\u0438', 'https://wicomix.com/feed/') +# ('\u041A\u043E\u043C\u043C\u0435\u043D\u0442\u0430\u0440\u0438\u0438', 'https://wicomix.com/comments/feed/') + ] diff --git a/src/calibre/qt_backend.py b/src/calibre/qt_backend.py new file mode 100644 index 0000000000..b19565d493 --- /dev/null +++ b/src/calibre/qt_backend.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python +# License: GPLv3 Copyright: 2024, Kovid Goyal + + +import json +import os +import sys +from contextlib import suppress +from threading import Thread +from time import monotonic + +from qt.core import QApplication, QNetworkAccessManager, QNetworkCookie, QNetworkReply, QNetworkRequest, QObject, Qt, QTimer, QUrl, pyqtSignal, sip + +default_timeout: float = 60. # seconds + + +def qurl_to_string(url: QUrl | str) -> str: + return bytes(QUrl(url).toEncoded()).decode() + + +def qurl_to_key(url: QUrl | str) -> str: + return qurl_to_string(url).rstrip('/') + + +Headers = list[tuple[str, str]] + + +class DownloadRequest(QNetworkRequest): + + error: str = '' + worth_retry: bool = False + reply: QNetworkReply + + def __init__(self, url: str, filename: str, headers: Headers | None = None, timeout: float = default_timeout, req_id: int = 0): + super().__init__(QUrl(url)) + self.setTransferTimeout(int(timeout * 1000)) + self.url, self.filename = url, filename + self.url_key = qurl_to_key(url) + self.headers: Headers = headers or [] + for (name, val) in self.headers: + self.setRawHeader(name.encode(), val.encode()) + self.req_id: int = req_id + self.error_message = '' + self.created_at = self.last_activity_at = monotonic() + self.timeout = timeout + + def as_result(self) -> dict[str, str]: + result = {'action': 'finished', 'id': self.req_id, 'url': self.url, 'output': os.path.join( + self.webengine_download_request.downloadDirectory(), self.webengine_download_request.downloadFileName()), + 'final_url': qurl_to_string(self.webengine_download_request.url()) + } + if self.error: + result['error'], result['worth_retry'] = self.error, self.worth_retry + return result + + def too_slow_or_timed_out(self, now: float) -> bool: + if self.timeout and self.last_activity_at + self.timeout < now: + return True + time_taken = now - self.created_at + if time_taken > default_timeout and self.webengine_download_request is not None: + downloaded = self.webengine_download_request.receivedBytes() + rate = downloaded / time_taken + return rate < 10 + return False + + +class FetchBackend(QNetworkAccessManager): + + request_download = pyqtSignal(str, str, object, float, int) + input_finished = pyqtSignal(str) + set_cookies = pyqtSignal(object) + set_user_agent_signal = pyqtSignal(str) + download_finished = pyqtSignal(object) + + def __init__(self, output_dir: str = '', cache_name: str = '', parent: QObject = None, user_agent: str = '') -> None: + super().__init__(parent) + self.setTransferTimeout(default_timeout) + self.output_dir = output_dir or os.getcwd() + sys.excepthook = self.excepthook + self.request_download.connect(self.download, type=Qt.ConnectionType.QueuedConnection) + self.set_cookies.connect(self._set_cookies, type=Qt.ConnectionType.QueuedConnection) + self.set_user_agent_signal.connect(self.set_user_agent, type=Qt.ConnectionType.QueuedConnection) + self.input_finished.connect(self.on_input_finished, type=Qt.ConnectionType.QueuedConnection) + self.live_requests: set[DownloadRequest] = set() + self.all_request_cookies: list[QNetworkCookie] = [] + self.timeout_timer = t = QTimer(self) + t.setInterval(50) + t.timeout.connect(self.enforce_timeouts) + + def excepthook(self, cls: type, exc: Exception, tb) -> None: + if not isinstance(exc, KeyboardInterrupt): + sys.__excepthook__(cls, exc, tb) + QApplication.instance().exit(1) + + def on_input_finished(self, error_msg: str) -> None: + if error_msg: + self.send_response({'action': 'input_error', 'error': error_msg}) + QApplication.instance().exit(1) + + def enforce_timeouts(self): + now = monotonic() + timed_out = tuple(dr for dr in self.live_requests if dr.too_slow_or_timed_out(now)) + for dr in timed_out: + if dr.webengine_download_request is None: + dr.cancel_on_start = True + else: + dr.webengine_download_request.cancel() + self.live_requests.discard(dr) + if self.live_requests: + self.timeout_timer.start() + + def download(self, url: str, filename: str, extra_headers: Headers | None = None, timeout: float = default_timeout, req_id: int = 0) -> None: + filename = os.path.basename(filename) + qurl = QUrl(url) + dr = DownloadRequest(url, filename, extra_headers, timeout, req_id) + self.dr_identifier_count += 1 + self.pending_download_requests[self.dr_identifier_count] = dr + self.live_requests.add(dr) + if not self.timeout_timer.isActive(): + self.timeout_timer.start() + cs = self.profile().cookieStore() + for c in self.all_request_cookies: + c = QNetworkCookie(c) + c.normalize(qurl) + cs.setCookie(c) + super().download(qurl, str(self.dr_identifier_count)) + + def _download_requested(self, wdr: QWebEngineDownloadRequest) -> None: + try: + idc = int(wdr.suggestedFileName()) + dr: DownloadRequest = self.pending_download_requests.pop(idc) + except Exception: + return + try: + if dr.cancel_on_start: + dr.error = 'Timed out trying to open URL' + dr.worth_retry = True + self.send_response(dr.as_result()) + return + dr.last_activity_at = monotonic() + if dr.filename: + wdr.setDownloadFileName(dr.filename) + dr.webengine_download_request = wdr + self.download_requests_by_id[wdr.id()] = dr + wdr.isFinishedChanged.connect(self._download_finished) + wdr.receivedBytesChanged.connect(self._bytes_received) + wdr.accept() + except Exception: + import traceback + traceback.print_exc() + self.report_finish(wdr, dr) + + def _bytes_received(self) -> None: + wdr: QWebEngineDownloadRequest = self.sender() + if dr := self.download_requests_by_id.get(wdr.id()): + dr.last_activity_at = monotonic() + + def _download_finished(self) -> None: + wdr: QWebEngineDownloadRequest = self.sender() + if dr := self.download_requests_by_id.get(wdr.id()): + self.report_finish(wdr, dr) + + def report_finish(self, wdr: QWebEngineDownloadRequest, dr: DownloadRequest) -> None: + s = wdr.state() + dr.last_activity_at = monotonic() + self.live_requests.discard(dr) + has_result = False + + if s == QWebEngineDownloadRequest.DownloadState.DownloadRequested: + dr.error = 'Open of URL failed' + has_result = True + elif s == QWebEngineDownloadRequest.DownloadState.DownloadCancelled: + dr.error = 'Timed out waiting for download' + dr.worth_retry = True + has_result = True + elif s == QWebEngineDownloadRequest.DownloadState.DownloadInterrupted: + dr.error = wdr.interruptReasonString() + dr.worth_retry = wdr.interruptReason() in ( + QWebEngineDownloadRequest.DownloadInterruptReason.NetworkTimeout, + QWebEngineDownloadRequest.DownloadInterruptReason.NetworkFailed, + QWebEngineDownloadRequest.DownloadInterruptReason.NetworkDisconnected, + QWebEngineDownloadRequest.DownloadInterruptReason.NetworkServerDown, + QWebEngineDownloadRequest.DownloadInterruptReason.ServerUnreachable, + ) + has_result = True + elif s == QWebEngineDownloadRequest.DownloadState.DownloadCompleted: + has_result = True + + if has_result: + result = dr.as_result() + self.download_finished.emit(result) + self.send_response(result) + + def send_response(self, r: dict[str, str]) -> None: + with suppress(OSError): + print(json.dumps(r), flush=True, file=sys.__stdout__) + + def set_user_agent(self, new_val: str) -> None: + self.profile().setHttpUserAgent(new_val) + + def _set_cookie_from_header(self, cookie_string: str) -> None: + cs = self.profile().cookieStore() + for c in QNetworkCookie.parseCookies(cookie_string.encode()): + cs.setCookie(c) + + def _set_cookies(self, cookies: list[dict[str, str]]) -> None: + for c in cookies: + if 'header' in c: + self._set_cookie_from_header(c['header']) + else: + self.set_simple_cookie(c['name'], c['value'], c.get('domain'), c.get('path')) + + def set_simple_cookie(self, name: str, value: str, domain: str | None = None, path: str | None = '/'): + c = QNetworkCookie() + c.setName(name.encode()) + c.setValue(value.encode()) + if domain is not None: + c.setDomain(domain) + if path is not None: + c.setPath(path) + if c.domain(): + self.profile().cookieStore().setCookie(c) + else: + self.all_request_cookies.append(c) + + +def read_commands(backend: FetchBackend, tdir: str) -> None: + file_counter = 0 + error_msg = '' + try: + for line in sys.stdin: + cmd = json.loads(line) + ac = cmd['action'] + if ac == 'download': + file_counter += 1 + timeout = cmd.get('timeout') + if timeout is None: + timeout = default_timeout + backend.request_download.emit(cmd['url'], os.path.join(tdir, str(file_counter)), cmd.get('headers'), timeout, cmd.get('id', 0)) + elif ac == 'set_cookies': + backend.set_cookies.emit(cmd['cookies']) + elif ac == 'set_user_agent': + backend.set_user_agent_signal.emit(cmd['user_agent']) + elif ac == 'quit': + break + except Exception as err: + import traceback + traceback.print_exc() + error_msg = str(err) + backend.input_finished.emit(error_msg) + + +def worker(tdir: str, user_agent: str) -> None: + app = QApplication.instance() + sys.stdout = sys.stderr + backend = FetchBackend(parent=app, user_agent=user_agent) + try: + read_thread = Thread(target=read_commands, args=(backend, tdir), daemon=True) + read_thread.start() + app.exec() + finally: + sip.delete(backend) + del app + + +def develop(url: str) -> None: + from calibre.gui2 import must_use_qt, setup_unix_signals + must_use_qt() + app = QApplication.instance() + app.signal_received = lambda : app.exit(1) + setup_unix_signals(app) + backend = FetchBackend() + num_left = 0 + + def download_finished(dr: DownloadRequest): + nonlocal num_left + num_left -= 1 + if not num_left: + backend.input_finished.emit('') + + backend.download_finished.connect(download_finished) + for i, url in enumerate(sys.argv[1:]): + backend.download(url, f'test-output-{i}') + num_left += 1 + app.exec() + + +if __name__ == '__main__': + develop(sys.argv[-1])