From 79f15cd0109955e84fc069e1ee7598d784a8ea90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jun 2009 17:58:36 -0700 Subject: [PATCH 1/4] Updated recipes for Harpers Magazine --- .../web/feeds/recipes/recipe_harpers.py | 22 +++++++++----- .../web/feeds/recipes/recipe_harpers_full.py | 30 +++++++++++-------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/calibre/web/feeds/recipes/recipe_harpers.py b/src/calibre/web/feeds/recipes/recipe_harpers.py index 6370f6e0ea..6cdcfe800d 100644 --- a/src/calibre/web/feeds/recipes/recipe_harpers.py +++ b/src/calibre/web/feeds/recipes/recipe_harpers.py @@ -6,6 +6,7 @@ __copyright__ = '2008-2009, Darko Miletic ' harpers.org ''' from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag class Harpers(BasicNewsRecipe): title = u"Harper's Magazine" @@ -18,23 +19,30 @@ class Harpers(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False - remove_javascript = True html2lrf_options = [ '--comment', description , '--category', category , '--publisher', publisher ] - - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' - - + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' + + keep_only_tags = [ dict(name='div', attrs={'id':'cached'}) ] remove_tags = [ - dict(name='table', attrs={'class':'rcnt'}) - ,dict(name='table', attrs={'class':'rcnt topline'}) + dict(name='table', attrs={'class':['rcnt','rcnt topline']}) ,dict(name=['link','object','embed']) ] feeds = [(u"Harper's Magazine", u'http://www.harpers.org/rss/frontpage-rss20.xml')] + def preprocess_html(self, soup): + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")]) + soup.head.insert(1,mcharset) + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(xmlns=True): + del item['xmlns'] + return soup + diff --git a/src/calibre/web/feeds/recipes/recipe_harpers_full.py b/src/calibre/web/feeds/recipes/recipe_harpers_full.py index 69ec9d54f5..bec16cd5c4 100644 --- a/src/calibre/web/feeds/recipes/recipe_harpers_full.py +++ b/src/calibre/web/feeds/recipes/recipe_harpers_full.py @@ -9,40 +9,38 @@ images and pdf's are ignored ''' from calibre import strftime - from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag class Harpers_full(BasicNewsRecipe): title = u"Harper's Magazine - articles from printed edition" __author__ = u'Darko Miletic' description = u"Harper's Magazine: Founded June 1850." publisher = "Harpers's" - category = 'news, politics, USA' + category = 'news, politics, USA' oldest_article = 30 max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False - simultaneous_downloads = 1 delay = 1 language = _('English') needs_subscription = True INDEX = strftime('http://www.harpers.org/archive/%Y/%m') LOGIN = 'http://www.harpers.org' cover_url = strftime('http://www.harpers.org/media/pages/%Y/%m/gif/0001.gif') - remove_javascript = True - + html2lrf_options = [ '--comment', description , '--category', category , '--publisher', publisher ] - - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' + + html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' keep_only_tags = [ dict(name='div', attrs={'id':'cached'}) ] remove_tags = [ - dict(name='table', attrs={'class':'rcnt'}) - ,dict(name='table', attrs={'class':'rcnt topline'}) + dict(name='table', attrs={'class':['rcnt','rcnt topline']}) + ,dict(name='link') ] def get_browser(self): @@ -54,13 +52,13 @@ class Harpers_full(BasicNewsRecipe): br['password'] = self.password br.submit() return br - + def parse_index(self): articles = [] print 'Processing ' + self.INDEX soup = self.index_to_soup(self.INDEX) for item in soup.findAll('div', attrs={'class':'title'}): - text_link = item.parent.find('img',attrs={'alt':'Text'}) + text_link = item.parent.find('img',attrs={'alt':'Text'}) if text_link: url = self.LOGIN + item.a['href'] title = item.a.contents[0] @@ -72,4 +70,12 @@ class Harpers_full(BasicNewsRecipe): ,'description':'' }) return [(soup.head.title.string, articles)] - + + def preprocess_html(self, soup): + mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")]) + soup.head.insert(1,mcharset) + for item in soup.findAll(style=True): + del item['style'] + for item in soup.findAll(xmlns=True): + del item['xmlns'] + return soup From 99993e492e61f2d67dc01fc0db45f1ecd089c5e7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jun 2009 21:57:59 -0700 Subject: [PATCH 2/4] beta 3 --- installer/osx/freeze.py | 2 +- src/calibre/constants.py | 2 +- src/calibre/www/settings.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/installer/osx/freeze.py b/installer/osx/freeze.py index fdfa575daf..3ccdb9bd61 100644 --- a/installer/osx/freeze.py +++ b/installer/osx/freeze.py @@ -19,7 +19,7 @@ from modulegraph.find_modules import find_modules PYTHON = '/Library/Frameworks/Python.framework/Versions/Current/bin/python' class BuildAPP(py2app): - QT_PREFIX = '/Users/kovid/qt' + QT_PREFIX = '/Volumes/sw/qt' LOADER_TEMPLATE = \ r'''#!/usr/bin/env python import os, sys, glob diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 28020933bd..319b01f887 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -2,7 +2,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = 'calibre' -__version__ = '0.6.0b2' +__version__ = '0.6.0b3' __author__ = "Kovid Goyal " import re diff --git a/src/calibre/www/settings.py b/src/calibre/www/settings.py index d20185c943..4197ff57f4 100644 --- a/src/calibre/www/settings.py +++ b/src/calibre/www/settings.py @@ -45,7 +45,7 @@ LANGUAGE_CODE = 'en-us' # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. -USE_I18N = False +USE_I18N = True # List of callables that know how to import templates from various sources. TEMPLATE_LOADERS = ( From 462124c87bd0d013c16770b3caa7b8d08b60b6fa Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jun 2009 09:45:23 -0700 Subject: [PATCH 3/4] Support 505s with a BCD of 0x1000 on windows and linux --- src/calibre/devices/prs505/driver.py | 8 +++--- src/calibre/devices/usbms/device.py | 38 +++++++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 378f15cab1..1ff17e8cbb 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -24,12 +24,12 @@ class PRS505(CLI, Device): VENDOR_ID = [0x054c] #: SONY Vendor Id PRODUCT_ID = [0x031e] #: Product Id for the PRS-505 - BCD = [0x229] #: Needed to disambiguate 505 and 700 on linux + BCD = [0x229, 0x1000] #: Needed to disambiguate 505 and 700 on linux VENDOR_NAME = 'SONY' - WINDOWS_MAIN_MEM = 'PRS-505' - WINDOWS_CARD_A_MEM = 'PRS-505/UC:MS' - WINDOWS_CARD_B_MEM = 'PRS-505/UC:SD' + WINDOWS_MAIN_MEM = 'PRS-505' + WINDOWS_CARD_A_MEM = ['PRS-505/UC:MS', 'PRS-505/CE:MS'] + WINDOWS_CARD_B_MEM = ['PRS-505/UC:SD', 'PRS-505/CE:SD'] OSX_MAIN_MEM = 'Sony PRS-505/UC Media' OSX_CARD_A_MEM = 'Sony PRS-505/UC:MS Media' diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index d3e5bada96..98ba781b0b 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -179,13 +179,19 @@ class Device(DeviceConfig, DevicePlugin): return (msz, casz, cbsz) - def windows_match_device(self, pnp_id, device_id): - pnp_id = pnp_id.upper() + def windows_match_device(self, drive, attr): + pnp_id = str(drive.PNPDeviceID).upper() + device_id = getattr(self, attr) + if device_id is None or \ + 'VEN_' + str(self.VENDOR_NAME).upper() not in pnp_id: + return False + if isinstance(device_id, basestring): + device_id = [device_id] - if device_id and pnp_id is not None: - device_id = device_id.upper() + for x in device_id: + x = x.upper() - if 'VEN_' + self.VENDOR_NAME in pnp_id and 'PROD_' + device_id in pnp_id: + if 'PROD_' + x in pnp_id: return True return False @@ -211,18 +217,32 @@ class Device(DeviceConfig, DevicePlugin): return drives def open_windows(self): + + def matches_q(drive, attr): + q = getattr(self, attr) + if q is None: return False + if isinstance(q, basestring): + q = [q] + pnp = str(drive.PNPDeviceID) + for x in q: + if x in pnp: + return True + return False + + time.sleep(6) drives = {} wmi = __import__('wmi', globals(), locals(), [], -1) c = wmi.WMI(find_classes=False) for drive in c.Win32_DiskDrive(): - if self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_CARD_A_MEM): + if self.windows_match_device(drive, 'WINDOWS_CARD_A_MEM'): drives['carda'] = self.windows_get_drive_prefix(drive) - elif self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_CARD_B_MEM): + elif self.windows_match_device(drive, 'WINDOWS_CARD_B_MEM'): drives['cardb'] = self.windows_get_drive_prefix(drive) - elif self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_MAIN_MEM): + elif self.windows_match_device(drive, 'WINDOWS_MAIN_MEM'): drives['main'] = self.windows_get_drive_prefix(drive) - if 'main' in drives.keys() and 'carda' in drives.keys() and 'cardb' in drives.keys(): + if 'main' in drives.keys() and 'carda' in drives.keys() and \ + 'cardb' in drives.keys(): break if 'main' not in drives: From 9266cfb0a5ab86426302bac5da2d86a2af7e6810 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 8 Jun 2009 12:53:58 -0700 Subject: [PATCH 4/4] Implement #2564 (Various new recipes) --- .../gui2/images/news/elperiodico_catalan.png | Bin 0 -> 693 bytes .../gui2/images/news/elperiodico_spanish.png | Bin 0 -> 693 bytes .../gui2/images/news/expansion_spanish.png | Bin 0 -> 806 bytes src/calibre/web/feeds/recipes/__init__.py | 4 +- .../recipes/recipe_elperiodico_catalan.py | 55 ++++++++++++++ .../recipes/recipe_elperiodico_spanish.py | 55 ++++++++++++++ .../feeds/recipes/recipe_expansion_spanish.py | 58 +++++++++++++++ .../web/feeds/recipes/recipe_lavanguardia.py | 69 ++++++++++++++++++ src/calibre/web/feeds/recipes/recipe_marca.py | 55 ++++++++++++++ 9 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/images/news/elperiodico_catalan.png create mode 100644 src/calibre/gui2/images/news/elperiodico_spanish.png create mode 100644 src/calibre/gui2/images/news/expansion_spanish.png create mode 100644 src/calibre/web/feeds/recipes/recipe_elperiodico_catalan.py create mode 100644 src/calibre/web/feeds/recipes/recipe_elperiodico_spanish.py create mode 100644 src/calibre/web/feeds/recipes/recipe_expansion_spanish.py create mode 100644 src/calibre/web/feeds/recipes/recipe_lavanguardia.py create mode 100644 src/calibre/web/feeds/recipes/recipe_marca.py diff --git a/src/calibre/gui2/images/news/elperiodico_catalan.png b/src/calibre/gui2/images/news/elperiodico_catalan.png new file mode 100644 index 0000000000000000000000000000000000000000..96b1d6d453e7ebd4ce63bcfc618c0d1d39982956 GIT binary patch literal 693 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#Qz)x;TbdoL)Ndptne&%u#!5Q(dzceGAuW zw0W;-SBYtIYRa3(wWM+HZTSZWl_o9nt`6YR;a9t{b~UG4dxr@}6NmU>2lIfY)|}OM zkKC19ezNWF-(4pAW$rXCDF40pe)ap``)ZPdyiV3XNVwvDciYWbyZrBNv%GN0>Fzeq z6kR_4Ih@?7Ec@wD++v2FdydTo8r#Wk5-uWNtyOr7i8 zV;>PRN6vglaiu`+`kVLv_*j}RJkuwzrMK^a>h2jcPcJ^s`(+2?fe!)pwsX@g9{j#y zay2V-Z@XKj)* zmymlEF2#G`Yt6oxz3V*OIfNp)btea2ZGP$aHK4)V+PmoOTBF&^Jdd9>l(VapUp3FK zbmgm8J&Phzp8w>|s$G78@p75R)4YHipDOo!(c4tjFk?#4u6g|zmuO%5TC?tM^rBDC zpZ!~R`shKQ!wkF}Ga2XYyS3oU)-`A5%DPL>at`O%@NMIv#Ydf1{94MfIVw^AVU^v~ zT=B)8>Je-AUhUxd!7HKmqtz~TdMhy6RZCnWN|K9G6H8KcQ-B16k%57wu7R1ZfklX+ zk(G&wm9d4cfw`4|L4j26a+rqP{FKbJN=yxARt6?O1koV&Bp;Nh7(8A5T-G@yGywp7 CBrA&m literal 0 HcmV?d00001 diff --git a/src/calibre/gui2/images/news/elperiodico_spanish.png b/src/calibre/gui2/images/news/elperiodico_spanish.png new file mode 100644 index 0000000000000000000000000000000000000000..96b1d6d453e7ebd4ce63bcfc618c0d1d39982956 GIT binary patch literal 693 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#Qz)x;TbdoL)Ndptne&%u#!5Q(dzceGAuW zw0W;-SBYtIYRa3(wWM+HZTSZWl_o9nt`6YR;a9t{b~UG4dxr@}6NmU>2lIfY)|}OM zkKC19ezNWF-(4pAW$rXCDF40pe)ap``)ZPdyiV3XNVwvDciYWbyZrBNv%GN0>Fzeq z6kR_4Ih@?7Ec@wD++v2FdydTo8r#Wk5-uWNtyOr7i8 zV;>PRN6vglaiu`+`kVLv_*j}RJkuwzrMK^a>h2jcPcJ^s`(+2?fe!)pwsX@g9{j#y zay2V-Z@XKj)* zmymlEF2#G`Yt6oxz3V*OIfNp)btea2ZGP$aHK4)V+PmoOTBF&^Jdd9>l(VapUp3FK zbmgm8J&Phzp8w>|s$G78@p75R)4YHipDOo!(c4tjFk?#4u6g|zmuO%5TC?tM^rBDC zpZ!~R`shKQ!wkF}Ga2XYyS3oU)-`A5%DPL>at`O%@NMIv#Ydf1{94MfIVw^AVU^v~ zT=B)8>Je-AUhUxd!7HKmqtz~TdMhy6RZCnWN|K9G6H8KcQ-B16k%57wu7R1ZfklX+ zk(G&wm9d4cfw`4|L4j26a+rqP{FKbJN=yxARt6?O1koV&Bp;Nh7(8A5T-G@yGywp7 CBrA&m literal 0 HcmV?d00001 diff --git a/src/calibre/gui2/images/news/expansion_spanish.png b/src/calibre/gui2/images/news/expansion_spanish.png new file mode 100644 index 0000000000000000000000000000000000000000..58bea2b94e7b4ef6d14ce296f315a5ec57c9b96f GIT binary patch literal 806 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdzwj^(N7l!{JxM1({$v_d#0*}aI zAngIhZYQ(tK!Rljj_E*J0gT&!&6&%XX9;K0Q&H$> z6%(@dzU-otw=rSm+Onpp{SSg8j--jd%MD74JMdD+M^-0kvWZYc=c3lLcOGquyPmR$ z?SIXLL&nDH_G>h&G+&S{fR1>{8c)qH#t2%{nSVzMNo=|jmP0dvIJZC+{YGg z9zQNm=uM4m+&Ph9TW&GqvB!qis>cgw=$xL&a6Yhp!ri>%ve9e9ESARTv3qH%cs@E; zX*aV~*-_x!EcGs>lGy!r2UJ&uXfZULO)IWuaN@8z|65R)rSZ4qI=&09zix(}}+fX`3Z&&NpYy zdQoEaqHOm^A=j-zr#J0bYw>Dh?(dY@|H9sAO-PlznIpFE{b7@{J2D@=Kd?}`@A%^z z)&-*a0imMXa*v-9mVR)9m%EjTt@+?zhC_EYgsr~Iui*2v=;ua;lPQy4SG5RiiCTM` ziKB_(vI(<#Jj09czke4WJj>8fe|$r7<-GQvHGHmtdHe#)9u{ygXoz)xXJ0Vo-D%@B zMeDAyANun1XL|gsO*-xJDT{ACDY1IG?UCDJ!OHm=X0z>|ZNL5Y^qCW-cQ0)EDW=HM zwP?Y`j5o|gx(*z2r{ES)Y) zC!c8FyLWDP&74cuuXDYBzh9v%DdCO2&6jm6!xkU)0;Vz564!{5!B{QuOQ-hh6p_!F2P%}uwss1u` Ppaup{S3j3^P6