Merge from trunk

.bzrignore

@@ -38,6 +38,8 @@ calibre_plugins/
 recipes/.git
 recipes/.gitignore
 recipes/README.md
+recipes/icon_checker.py
+recipes/readme_updater.py
 recipes/katalog_egazeciarz.recipe
 recipes/tv_axnscifi.recipe
 recipes/tv_comedycentral.recipe
@@ -60,6 +62,7 @@ recipes/tv_tvpkultura.recipe
 recipes/tv_tvppolonia.recipe
 recipes/tv_tvpuls.recipe
 recipes/tv_viasathistory.recipe
+recipes/icons/katalog_egazeciarz.png
 recipes/icons/tv_axnscifi.png
 recipes/icons/tv_comedycentral.png
 recipes/icons/tv_discoveryscience.png

Changelog.yaml

@@ -19,6 +19,62 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.24
+  date: 2013-03-22
+
+  new features:
+    - title: "ToC Editor: Allow auto-generation of Table of Contents entries from headings and/or links in the book"
+
+    - title: "EPUB/MOBI Catalogs: Allow saving used settings as presets which can be loaded easily later."
+      tickets: [1155587]
+
+    - title: "Indicate which columns are custom columns when selecting columns in the Preferences"
+      tickets: [1158066]
+
+    - title: "News download: Add an option recipe authors can set to have calibre automatically reduce the size of downloaded images by lowering their quality"
+ 
+  bug fixes:
+    - title: "News download: Fix a regression in 0.9.23 that prevented oldest_article from working with some RSS feeds."
+
+    - title: "Conversion: handle the :before and :after pseudo CSS selectors correctly"
+
+    - title: "AZW3 Output: Handle the case of the <guide> reference to a ToC containing an anchor correctly."
+      tickets: [1158413]
+
+    - title: "BiBTeX catalogs: Fix ISBN not being output and the library_name field causing catalog generation to fail"
+      tickets: [1156432, 1158127]
+
+    - title: "Conversion: Add support for CSS stylesheets that wrap their rules inside a @media rule."
+      tickets: [1157345]
+
+    - title: "Cover browser: Fix scrolling not working for books after the 32678'th book in a large library."
+      tickets: [1153204]
+
+    - title: "Linux: Update bundled libmtp version"
+
+    - title: "Clear the Book details panel when the current search returns no matches."
+      tickets: [1153026]
+
+    - title: "Fix a regression that broke creation of advanced column coloring rules"
+      tickets: [1156291]
+
+    - title: "Amazon metadata download: Handle cover images loaded via javascript on the amazon.de site"
+
+    - title: "Nicer error message when exporting a generated csv catalog to a file open in another program on windows."
+      tickets: [1155539]
+
+    - title: "Fix ebook-convert -h showing ANSI escape codes in the windows command prompt"
+      tickets: [1158499]
+
+  improved recipes:
+    - Various Polish news sources
+    - kath.net
+    - Il Giornale
+    - Kellog Insight
+
+  new recipes:
+    - title: 
+
 - version: 0.9.23
   date: 2013-03-15
 

manual/conversion.rst

@@ -434,6 +434,18 @@ a number of older formats either do not support a metadata based Table of Conten
 documents do not have one. In these cases, the options in this section can help you automatically
 generate a Table of Contents in the converted ebook, based on the actual content in the input document.
 
+.. note:: Using these options can be a little challenging to get exactly right.
+    If you prefer creating/editing the Table of Contents by hand, convert to
+    the EPUB or AZW3 formats and select the checkbox at the bottom of the
+    screen that says 
+    :guilabel:`Manually fine-tune the Table of Contents after conversion`. 
+    This will launch the ToC Editor tool after the conversion. It allows you to
+    create entries in the Table of Contents by simply clicking the place in the
+    book where you want the entry to point. You can also use the ToC Editor by
+    itself, without doing a conversion. Go to :guilabel:`Preferences->Toolbars`
+    and add the ToC Editor to the main toolbar. Then just select the book you
+    want to edit and click the ToC Editor button.
+
 The first option is :guilabel:`Force use of auto-generated Table of Contents`. By checking this option
 you can have |app| override any Table of Contents found in the metadata of the input document with the
 auto generated one. 
@@ -456,7 +468,7 @@ For example, to remove all entries titles "Next" or "Previous" use::
 
     Next|Previous
 
-Finally, the :guilabel:`Level 1,2,3 TOC` options allow you to create a sophisticated multi-level Table of Contents.
+The :guilabel:`Level 1,2,3 TOC` options allow you to create a sophisticated multi-level Table of Contents.
 They are XPath expressions that match tags in the intermediate XHTML produced by the conversion pipeline. See the 
 :ref:`conversion-introduction` for how to get access to this XHTML. Also read the :ref:`xpath-tutorial`, to learn
 how to construct XPath expressions. Next to each option is a button that launches a wizard to help with the creation

manual/faq.rst

@@ -87,7 +87,9 @@ this bug.
 
 How do I convert a collection of HTML files in a specific order?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
+In order to convert a collection of HTML files in a specific oder, you have to
+create a table of contents file. That is, another HTML file that contains links
+to all the other files in the desired order. Such a file looks like::
 
    <html>
       <body>
@@ -102,18 +104,35 @@ In order to convert a collection of HTML files in a specific oder, you have to c
       </body>
    </html>
 
-Then just add this HTML file to the GUI and use the convert button to create your ebook. 
+Then, just add this HTML file to the GUI and use the convert button to create
+your ebook. You can use the option in the Table of Contents section in the
+conversion dialog to control how the Table of Contents is generated.
 
-.. note:: By default, when adding HTML files, |app| follows links in the files in *depth first* order. This means that if file A.html links to B.html and C.html and D.html, but B.html also links to D.html, then the files will be in the order A.html, B.html, D.html, C.html. If instead you want the order to be A.html, B.html, C.html, D.html then you must tell |app| to add your files in *breadth first* order. Do this by going to Preferences->Plugins and customizing the HTML to ZIP plugin.
+.. note:: By default, when adding HTML files, |app| follows links in the files
+    in *depth first* order. This means that if file A.html links to B.html and
+    C.html and D.html, but B.html also links to D.html, then the files will be
+    in the order A.html, B.html, D.html, C.html. If instead you want the order
+    to be A.html, B.html, C.html, D.html then you must tell |app| to add your
+    files in *breadth first* order. Do this by going to Preferences->Plugins
+    and customizing the HTML to ZIP plugin.
 
 The EPUB I produced with |app| is not valid?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-|app| does not guarantee that an EPUB produced by it is valid. The only guarantee it makes is that if you feed it valid XHTML 1.1 + CSS 2.1 it will output a valid EPUB. |app| is designed for ebook consumers, not producers. It tries hard to ensure that EPUBs it produces actually work as intended on a wide variety of devices, a goal that is incompatible with producing valid EPUBs, and one that is far more important to the vast majority of its users. If you need a tool that always produces valid EPUBs, |app| is not for you.
+|app| does not guarantee that an EPUB produced by it is valid. The only
+guarantee it makes is that if you feed it valid XHTML 1.1 + CSS 2.1 it will
+output a valid EPUB. |app| is designed for ebook consumers, not producers. It
+tries hard to ensure that EPUBs it produces actually work as intended on a wide
+variety of devices, a goal that is incompatible with producing valid EPUBs, and
+one that is far more important to the vast majority of its users. If you need a
+tool that always produces valid EPUBs, |app| is not for you.
 
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features:
+ You can get help on any individual feature of the converters by mousing over
+ it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal.
+ A good place to start is to look at the following demo files that demonstrate
+ some of the advanced features:
   * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_
 
 
@@ -126,11 +145,11 @@ Device Integration
 
 What devices does |app| support?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| can directly connect to all the major (and most of the minor) ebook reading devices,
+|app| can directly connect to all the major (and most of the minor) ebook
-smarthphones, tablets, etc.
+reading devices, smarthphones, tablets, etc.  In addition, using the
-In addition, using the :guilabel:`Connect to folder` function you can use it with any ebook reader that exports itself as a USB disk. 
+:guilabel:`Connect to folder` function you can use it with any ebook reader
-You can even connect to Apple devices (via iTunes), using the :guilabel:`Connect to iTunes`
+that exports itself as a USB disk.  You can even connect to Apple devices (via
-function.
+iTunes), using the :guilabel:`Connect to iTunes` function.
 
 .. _devsupport:
 

recipes/adventure_zone_pl.recipe

@@ -10,46 +10,35 @@ class Adventure_zone(BasicNewsRecipe):
     oldest_article = 20
     max_articles_per_feed = 100
     cover_url = 'http://www.adventure-zone.info/inne/logoaz_2012.png'
-    index='http://www.adventure-zone.info/fusion/'
+    index = 'http://www.adventure-zone.info/fusion/'
     use_embedded_content = False
     preprocess_regexps     = [(re.compile(r"<td class='capmain'>Komentarze</td>", re.IGNORECASE), lambda m: ''),
     (re.compile(r'</?table.*?>'), lambda match: ''),
     (re.compile(r'</?tbody.*?>'), lambda match: '')]
-    remove_tags_before= dict(name='td', attrs={'class':'main-bg'})
+    remove_tags_before = dict(name='td', attrs={'class':'main-bg'})
-    remove_tags= [dict(name='img', attrs={'alt':'Drukuj'})]
+    remove_tags = [dict(name='img', attrs={'alt':'Drukuj'})]
-    remove_tags_after= dict(id='comments')
+    remove_tags_after = dict(id='comments')
-    extra_css              = '.main-bg{text-align: left;}  td.capmain{ font-size: 22px; }'
+    extra_css              = '.main-bg{text-align: left;}  td.capmain{ font-size: 22px; } img.news-category {float: left; margin-right: 5px;}'
     feeds          = [(u'Nowinki', u'http://www.adventure-zone.info/fusion/feeds/news.php')]
 
-    '''def parse_feeds (self): 
-      feeds = BasicNewsRecipe.parse_feeds(self) 
-      soup=self.index_to_soup(u'http://www.adventure-zone.info/fusion/feeds/news.php')
-      tag=soup.find(name='channel')
-      titles=[]
-      for r in tag.findAll(name='image'):
-          r.extract()
-      art=tag.findAll(name='item')
-      for i in art:
-            titles.append(i.title.string)
-      for feed in feeds:
-        for article in feed.articles[:]:
-            article.title=titles[feed.articles.index(article)]
-      return feeds'''
-
-
     '''def get_cover_url(self):
         soup = self.index_to_soup('http://www.adventure-zone.info/fusion/news.php')
         cover=soup.find(id='box_OstatninumerAZ')
         self.cover_url='http://www.adventure-zone.info/fusion/'+ cover.center.a.img['src']
         return getattr(self, 'cover_url', self.cover_url)'''
+
     def populate_article_metadata(self, article, soup, first):
         result = re.search('(.+) - Adventure Zone', soup.title.string)
         if result:
-            article.title = result.group(1)
+            result = result.group(1)
         else:
             result = soup.body.find('strong')
             if result:
-                article.title = result.string
+                result = result.string
+        if result:
+            result = result.replace('&amp;', '&')
+            result = result.replace('&#39;', '’')
+            article.title = result
 
     def skip_ad_pages(self, soup):
         skip_tag = soup.body.find(name='td', attrs={'class':'main-bg'})
@@ -78,4 +67,3 @@ class Adventure_zone(BasicNewsRecipe):
                 a['href']=self.index + a['href']
         return soup
 
-            

recipes/astroflesz.recipe

@@ -18,3 +18,10 @@ class Astroflesz(BasicNewsRecipe):
     remove_tags_after = dict(name='div', attrs={'class':'itemLinks'})
     remove_tags = [dict(name='div', attrs={'class':['itemLinks', 'itemToolbar', 'itemRatingBlock']})]
     feeds          = [(u'Wszystkie', u'http://astroflesz.pl/?format=feed')]
+
+    def postprocess_html(self, soup, first_fetch):
+        t = soup.find(attrs={'class':'itemIntroText'})
+        if t:
+            for i in t.findAll('img'):
+                i['style'] = 'float: left; margin-right: 5px;'
+        return soup

recipes/benchmark_pl.recipe

@@ -1,5 +1,7 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre.ebooks.BeautifulSoup import Comment
+
 class BenchmarkPl(BasicNewsRecipe):
     title          = u'Benchmark.pl'
     __author__        = 'fenuks'
@@ -13,10 +15,10 @@ class BenchmarkPl(BasicNewsRecipe):
     no_stylesheets = True
     remove_attributes = ['style']
     preprocess_regexps = [(re.compile(ur'<h3><span style="font-size: small;">&nbsp;Zobacz poprzednie <a href="http://www.benchmark.pl/news/zestawienie/grupa_id/135">Opinie dnia:</a></span>.*</body>', re.DOTALL|re.IGNORECASE), lambda match: '</body>'), (re.compile(ur'Więcej o .*?</ul>', re.DOTALL|re.IGNORECASE), lambda match: '')]
-    keep_only_tags=[dict(name='div', attrs={'class':['m_zwykly', 'gallery']}), dict(id='article')]
+    keep_only_tags = [dict(name='div', attrs={'class':['m_zwykly', 'gallery']}), dict(id='article')]
-    remove_tags_after=dict(name='div', attrs={'class':'body'})
+    remove_tags_after = dict(id='article')
-    remove_tags=[dict(name='div', attrs={'class':['kategoria', 'socialize', 'thumb', 'panelOcenaObserwowane', 'categoryNextToSocializeGallery', 'breadcrumb', 'footer', 'moreTopics']}), dict(name='table', attrs={'background':'http://www.benchmark.pl/uploads/backend_img/a/fotki_newsy/opinie_dnia/bg.png'}), dict(name='table', attrs={'width':'210', 'cellspacing':'1', 'cellpadding':'4', 'border':'0', 'align':'right'})]
+    remove_tags = [dict(name='div', attrs={'class':['comments', 'body', 'kategoria', 'socialize', 'thumb', 'panelOcenaObserwowane', 'categoryNextToSocializeGallery', 'breadcrumb', 'footer', 'moreTopics']}), dict(name='table', attrs = {'background':'http://www.benchmark.pl/uploads/backend_img/a/fotki_newsy/opinie_dnia/bg.png'}), dict(name='table', attrs={'width':'210', 'cellspacing':'1', 'cellpadding':'4', 'border':'0', 'align':'right'})]
-    INDEX= 'http://www.benchmark.pl'
+    INDEX = 'http://www.benchmark.pl'
     feeds          = [(u'Aktualności', u'http://www.benchmark.pl/rss/aktualnosci-pliki.xml'), 
                           (u'Testy i recenzje', u'http://www.benchmark.pl/rss/testy-recenzje-minirecenzje.xml')]
 
@@ -27,7 +29,12 @@ class BenchmarkPl(BasicNewsRecipe):
             soup2 = self.index_to_soup(nexturl['href'])
             nexturl = soup2.find(attrs={'class':'next'})
             pagetext = soup2.find(name='div', attrs={'class':'body'})
-            appendtag.find('div', attrs={'class':'k_ster'}).extract()
+            tag = appendtag.find('div', attrs={'class':'k_ster'})
+            if tag:
+                tag.extract()
+            comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
             pos = len(appendtag.contents)
             appendtag.insert(pos, pagetext)
         if appendtag.find('div', attrs={'class':'k_ster'}):
@@ -37,40 +44,44 @@ class BenchmarkPl(BasicNewsRecipe):
 
 
     def image_article(self, soup, appendtag):
-        nexturl=soup.find('div', attrs={'class':'preview'})
+        nexturl = soup.find('div', attrs={'class':'preview'})
-        if nexturl is not None:
+        if nexturl:
-            nexturl=nexturl.find('a', attrs={'class':'move_next'})
+            nexturl = nexturl.find('a', attrs={'class':'move_next'})
-            image=appendtag.find('div', attrs={'class':'preview'}).div['style'][16:]
+            image = appendtag.find('div', attrs={'class':'preview'}).div['style'][16:]
-            image=self.INDEX + image[:image.find("')")]
+            image = self.INDEX + image[:image.find("')")]
             appendtag.find(attrs={'class':'preview'}).name='img'
             appendtag.find(attrs={'class':'preview'})['src']=image
             appendtag.find('a', attrs={'class':'move_next'}).extract()
-        while nexturl is not None:
+        while nexturl:
-            nexturl= self.INDEX + nexturl['href']
+            nexturl = self.INDEX + nexturl['href']
             soup2 = self.index_to_soup(nexturl)
-            nexturl=soup2.find('a', attrs={'class':'move_next'})
+            nexturl = soup2.find('a', attrs={'class':'move_next'})
-            image=soup2.find('div', attrs={'class':'preview'}).div['style'][16:]
+            image = soup2.find('div', attrs={'class':'preview'}).div['style'][16:]
-            image=self.INDEX + image[:image.find("')")]
+            image = self.INDEX + image[:image.find("')")]
             soup2.find(attrs={'class':'preview'}).name='img'
             soup2.find(attrs={'class':'preview'})['src']=image
-            pagetext=soup2.find('div', attrs={'class':'gallery'})
+            pagetext = soup2.find('div', attrs={'class':'gallery'})
             pagetext.find('div', attrs={'class':'title'}).extract()
             pagetext.find('div', attrs={'class':'thumb'}).extract()
             pagetext.find('div', attrs={'class':'panelOcenaObserwowane'}).extract()       
-            if nexturl is not None:
+            if nexturl:
                 pagetext.find('a', attrs={'class':'move_next'}).extract()
             pagetext.find('a', attrs={'class':'move_back'}).extract()
+            comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
             pos = len(appendtag.contents)
             appendtag.insert(pos, pagetext)
             
 
-
     def preprocess_html(self, soup):
-        if soup.find('div', attrs={'class':'preview'}) is not None:
+        if soup.find('div', attrs={'class':'preview'}):
             self.image_article(soup, soup.body)
         else:
             self.append_page(soup, soup.body)
         for a in soup('a'):
-            if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
+            if a.has_key('href') and not a['href'].startswith('http'):
-                a['href']=self.INDEX + a['href']
+                a['href'] = self.INDEX + a['href']
+        for r in soup.findAll(attrs={'class':['comments', 'body']}):
+            r.extract()
         return soup

recipes/biweekly.recipe

@@ -14,7 +14,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class biweekly(BasicNewsRecipe):
     __author__ = u'Łukasz Grąbczewski'
     title = 'Biweekly'
-    language = 'en'
+    language = 'en_PL'
     publisher = 'National Audiovisual Institute'
     publication_type = 'magazine'
     description = u'link with culture [English edition of Polish magazine]: literature, theatre, film, art, music, views, talks'

recipes/blog_biszopa.recipe

@@ -0,0 +1,30 @@
+__license__ = 'GPL v3'
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BlogBiszopa(BasicNewsRecipe):
+    title          = u'Blog Biszopa'
+    __author__        = 'fenuks'
+    description   = u'Zapiski z Granitowego Miasta'
+    category       = 'history'
+    #publication_type = ''
+    language       = 'pl'
+    #encoding = ''
+    #extra_css = ''
+    cover_url = 'http://blogbiszopa.pl/wp-content/themes/biszop/images/logo.png'
+    masthead_url = ''
+    use_embedded_content = False
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_empty_feeds = True
+    remove_javascript = True
+    remove_attributes = ['style', 'font']
+    ignore_duplicate_articles = {'title', 'url'}
+
+    keep_only_tags = [dict(id='main-content')]
+    remove_tags = [dict(name='footer')]
+    #remove_tags_after = {}
+    #remove_tags_before = {}
+
+    feeds          = [(u'Artyku\u0142y', u'http://blogbiszopa.pl/feed/')]
+ 

recipes/bwmagazine2.recipe

@@ -25,6 +25,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
 
         #Find date
         mag=soup.find('h2',text='Magazine')
+        self.log(mag)
         dates=self.tag_to_string(mag.findNext('h3'))
         self.timefmt = u' [%s]'%dates
 
@@ -32,7 +33,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
         div0 = soup.find ('div', attrs={'class':'column left'})
         section_title = ''
         feeds = OrderedDict()
-        for div in div0.findAll('h4'):
+        for div in div0.findAll(['h4','h5']):
             articles = []
             section_title = self.tag_to_string(div.findPrevious('h3')).strip()
             title=self.tag_to_string(div.a).strip()
@@ -48,7 +49,7 @@ class BusinessWeekMagazine(BasicNewsRecipe):
                 feeds[section_title] += articles
         div1 = soup.find ('div', attrs={'class':'column center'})
         section_title = ''
-        for div in div1.findAll('h5'):
+        for div in div1.findAll(['h4','h5']):
             articles = []
             desc=self.tag_to_string(div.findNext('p')).strip()
             section_title = self.tag_to_string(div.findPrevious('h3')).strip()

recipes/ciekawostki_historyczne.recipe

@@ -1,5 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
+
 class Ciekawostki_Historyczne(BasicNewsRecipe):
     title          = u'Ciekawostki Historyczne'
     oldest_article = 7
@@ -7,42 +8,31 @@ class Ciekawostki_Historyczne(BasicNewsRecipe):
     description   = u'Serwis popularnonaukowy - odkrycia, kontrowersje, historia, ciekawostki, badania, ciekawostki z przeszłości.'
     category       = 'history'
     language       = 'pl'
-    masthead_url= 'http://ciekawostkihistoryczne.pl/wp-content/themes/Wordpress_Magazine/images/logo-ciekawostki-historyczne-male.jpg'
+    masthead_url = 'http://ciekawostkihistoryczne.pl/wp-content/themes/Wordpress_Magazine/images/logo-ciekawostki-historyczne-male.jpg'
-    cover_url='http://ciekawostkihistoryczne.pl/wp-content/themes/Wordpress_Magazine/images/logo-ciekawostki-historyczne-male.jpg'
+    cover_url = 'http://ciekawostkihistoryczne.pl/wp-content/themes/Wordpress_Magazine/images/logo-ciekawostki-historyczne-male.jpg'
     max_articles_per_feed = 100
+    extra_css = 'img.alignleft {float:left; margin-right:5px;} .alignright {float:right; margin-left:5px;}'
+    oldest_article = 12
     preprocess_regexps = [(re.compile(ur'Ten artykuł ma kilka stron.*?</fb:like>', re.DOTALL), lambda match: ''), (re.compile(ur'<h2>Zobacz też:</h2>.*?</ol>', re.DOTALL), lambda match: '')]
-    no_stylesheets=True
+    no_stylesheets = True
-    remove_empty_feeds=True
+    remove_empty_feeds = True
-    keep_only_tags=[dict(name='div', attrs={'class':'post'})]
+    keep_only_tags = [dict(name='div', attrs={'class':'post'})]
-    remove_tags=[dict(id='singlepostinfo')]
+    recursions = 5
+    remove_tags = [dict(id='singlepostinfo')]
+
     feeds          = [(u'Staro\u017cytno\u015b\u0107', u'http://ciekawostkihistoryczne.pl/tag/starozytnosc/feed/'), (u'\u015aredniowiecze', u'http://ciekawostkihistoryczne.pl/tag/sredniowiecze/feed/'), (u'Nowo\u017cytno\u015b\u0107', u'http://ciekawostkihistoryczne.pl/tag/nowozytnosc/feed/'), (u'XIX wiek', u'http://ciekawostkihistoryczne.pl/tag/xix-wiek/feed/'), (u'1914-1939', u'http://ciekawostkihistoryczne.pl/tag/1914-1939/feed/'), (u'1939-1945', u'http://ciekawostkihistoryczne.pl/tag/1939-1945/feed/'), (u'Powojnie (od 1945)', u'http://ciekawostkihistoryczne.pl/tag/powojnie/feed/'), (u'Recenzje', u'http://ciekawostkihistoryczne.pl/category/recenzje/feed/')]
 
-    def append_page(self, soup, appendtag):
+    def is_link_wanted(self, url, tag):
-        tag=soup.find(name='h7')
+        return 'ciekawostkihistoryczne' in url and url[-2] in {'2', '3', '4', '5', '6'}
-        if tag:
-            if tag.br:
-                pass
-            elif tag.nextSibling.name=='p':
-                tag=tag.nextSibling
-                nexturl = tag.findAll('a')
-                for nextpage in nexturl:
-                    tag.extract()
-                    nextpage= nextpage['href']
-                    soup2 = self.index_to_soup(nextpage)
-                    pagetext = soup2.find(name='div', attrs={'class':'post'})
-                    for r in pagetext.findAll('div', attrs={'id':'singlepostinfo'}):
-                        r.extract()
-                    for r in pagetext.findAll('div', attrs={'class':'wp-caption alignright'}):
-                        r.extract()
-                    for r in pagetext.findAll('h1'):
-                        r.extract()
-                    pagetext.find('h6').nextSibling.extract()
-                    pagetext.find('h7').nextSibling.extract()
-                    pos = len(appendtag.contents)
-                    appendtag.insert(pos, pagetext)
 
-    def preprocess_html(self, soup):
+    def postprocess_html(self, soup, first_fetch):
-        self.append_page(soup, soup.body)
+        tag = soup.find('h7')
+        if tag:
+            tag.nextSibling.extract()
+        if not first_fetch:
+            for r in soup.findAll(['h1']):
+                r.extract()
+            soup.find('h6').nextSibling.extract()
         return soup
 
 

recipes/computerworld_pl.recipe

@@ -1,5 +1,5 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 class Computerworld_pl(BasicNewsRecipe):
     title          = u'Computerworld.pl'
@@ -12,8 +12,16 @@ class Computerworld_pl(BasicNewsRecipe):
     no_stylesheets = True
     oldest_article = 7
     max_articles_per_feed = 100
-    keep_only_tags = [dict(attrs={'class':['tyt_news', 'prawo', 'autor', 'tresc']})]
+    remove_attributes = ['style',]
-    remove_tags_after = dict(name='div', attrs={'class':'rMobi'})
+    preprocess_regexps = [(re.compile(u'Zobacz również:', re.IGNORECASE), lambda m: ''), (re.compile(ur'[*]+reklama[*]+', re.IGNORECASE), lambda m: ''),]
-    remove_tags = [dict(name='div', attrs={'class':['nnav', 'rMobi']}), dict(name='table', attrs={'class':'ramka_slx'})]
+    keep_only_tags = [dict(id=['szpaltaL', 's2011'])]
+    remove_tags_after = dict(name='div', attrs={'class':'tresc'})
+    remove_tags = [dict(attrs={'class':['nnav', 'rMobi', 'tagi', 'rec']}),]
     feeds          = [(u'Wiadomo\u015bci', u'http://rssout.idg.pl/cw/news_iso.xml')]
 
+    def skip_ad_pages(self, soup):
+        if soup.title.string.lower() == 'advertisement':
+          tag = soup.find(name='a')
+          if tag:
+            new_soup = self.index_to_soup(tag['href'], raw=True)
+            return new_soup

recipes/conowego_pl.recipe

@@ -1,5 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Comment
+
 class CoNowegoPl(BasicNewsRecipe):
     title          = u'conowego.pl'
     __author__        = 'fenuks'
@@ -10,6 +11,7 @@ class CoNowegoPl(BasicNewsRecipe):
     oldest_article = 7
     max_articles_per_feed = 100
     INDEX = 'http://www.conowego.pl/'
+    extra_css = '.news-single-img {float:left; margin-right:5px;}'
     no_stylesheets = True
     remove_empty_feeds = True
     use_embedded_content = False
@@ -35,6 +37,9 @@ class CoNowegoPl(BasicNewsRecipe):
                 pos = len(appendtag.contents)
                 appendtag.insert(pos, pagetext)
 
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
             for r in appendtag.findAll(attrs={'class':['pages', 'paginationWrap']}):
                 r.extract()
 

recipes/czas_gentlemanow.recipe

@@ -12,11 +12,13 @@ class CzasGentlemanow(BasicNewsRecipe):
     ignore_duplicate_articles = {'title', 'url'}
     oldest_article = 7
     max_articles_per_feed = 100
+    extra_css = '.gallery-item {float:left; margin-right: 10px; max-width: 20%;} .alignright {text-align: right; float:right; margin-left:5px;}\
+    .wp-caption-text {text-align: left;} img.aligncenter {display: block; margin-left: auto; margin-right: auto;} .alignleft {float: left; margin-right:5px;}'
     no_stylesheets = True
     remove_empty_feeds = True
     preprocess_regexps     = [(re.compile(u'<h3>Może Cię też zainteresować:</h3>'), lambda m: '')]
     use_embedded_content = False
     keep_only_tags = [dict(name='div', attrs={'class':'content'})]
-    remove_tags = [dict(attrs={'class':'meta_comments'}), dict(id=['comments', 'related_posts_thumbnails'])]
+    remove_tags = [dict(attrs={'class':'meta_comments'}), dict(id=['comments', 'related_posts_thumbnails', 'respond'])]
     remove_tags_after = dict(id='comments')
     feeds          = [(u'M\u0119ski \u015awiat', u'http://czasgentlemanow.pl/category/meski-swiat/feed/'), (u'Styl', u'http://czasgentlemanow.pl/category/styl/feed/'), (u'Vademecum Gentlemana', u'http://czasgentlemanow.pl/category/vademecum/feed/'), (u'Dom i rodzina', u'http://czasgentlemanow.pl/category/dom-i-rodzina/feed/'), (u'Honor', u'http://czasgentlemanow.pl/category/honor/feed/'), (u'Gad\u017cety Gentlemana', u'http://czasgentlemanow.pl/category/gadzety-gentlemana/feed/')]

recipes/dobreprogamy.recipe

@@ -16,6 +16,7 @@ class Dobreprogramy_pl(BasicNewsRecipe):
     extra_css      = '.title {font-size:22px;}'
     oldest_article = 8
     max_articles_per_feed = 100
+    remove_attrs = ['style', 'width', 'height']
     preprocess_regexps = [(re.compile(ur'<div id="\S+360pmp4">Twoja przeglądarka nie obsługuje Flasha i HTML5 lub wyłączono obsługę JavaScript...</div>'), lambda match: '') ]
     keep_only_tags=[dict(attrs={'class':['news', 'entry single']})]
     remove_tags = [dict(attrs={'class':['newsOptions', 'noPrint', 'komentarze', 'tags  font-heading-master']}), dict(id='komentarze'), dict(name='iframe')]
@@ -28,4 +29,11 @@ class Dobreprogramy_pl(BasicNewsRecipe):
         for a in soup('a'):
             if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
                 a['href']=self.index + a['href']
+        for r in soup.findAll('iframe'):
+            r.parent.extract()
+        return soup
+    def postprocess_html(self, soup, first_fetch):
+        for r in soup.findAll('span', text=''):
+            if not r.string:
+                r.extract()
         return soup

recipes/drytooling_pl.recipe

@@ -8,6 +8,7 @@ class BasicUserRecipe1337668045(BasicNewsRecipe):
     cover_url = 'http://drytooling.com.pl/images/drytooling-kindle.png'
     description = u'Drytooling.com.pl jest serwisem wspinaczki zimowej, alpinizmu i himalaizmu. Jeśli uwielbiasz zimę, nie możesz doczekać się aż wyciągniesz szpej z szafki i uderzysz w Tatry, Alpy, czy może Himalaje, to znajdziesz tutaj naprawdę dużo interesujących Cię treści! Zapraszamy!'
     __author__ = u'Damian Granowski'
+    language = 'pl'
     oldest_article = 100
     max_articles_per_feed = 20
     auto_cleanup = True

recipes/dzieje_pl.recipe

@@ -1,4 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class Dzieje(BasicNewsRecipe):
     title          = u'dzieje.pl'
@@ -8,11 +9,12 @@ class Dzieje(BasicNewsRecipe):
     category       = 'history'
     language       = 'pl'
     ignore_duplicate_articles = {'title', 'url'}
+    extra_css = '.imagecache-default {float:left; margin-right:20px;}'
     index = 'http://dzieje.pl'
     oldest_article = 8
     max_articles_per_feed = 100
-    remove_javascript=True
+    remove_javascript = True
-    no_stylesheets= True
+    no_stylesheets = True
     keep_only_tags = [dict(name='h1', attrs={'class':'title'}), dict(id='content-area')]
     remove_tags = [dict(attrs={'class':'field field-type-computed field-field-tagi'}), dict(id='dogory')]
     #feeds          = [(u'Dzieje', u'http://dzieje.pl/rss.xml')]
@@ -28,16 +30,19 @@ class Dzieje(BasicNewsRecipe):
                 pagetext = soup2.find(id='content-area').find(attrs={'class':'content'})
                 for r in pagetext.findAll(attrs={'class':['fieldgroup group-groupkul', 'fieldgroup group-zdjeciekult', 'fieldgroup group-zdjecieciekaw', 'fieldgroup group-zdjecieksiazka', 'fieldgroup group-zdjeciedu', 'field field-type-filefield field-field-zdjecieglownawyd']}):
                     r.extract()
-                pos = len(appendtag.contents)
+                comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
-                appendtag.insert(pos, pagetext)
+                # appendtag.insert(pos, pagetext)
                 tag = soup2.find('li', attrs={'class':'pager-next'})
             for r in appendtag.findAll(attrs={'class':['item-list', 'field field-type-computed field-field-tagi', ]}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
 
     def find_articles(self, url):
         articles = []
-        soup=self.index_to_soup(url)
+        soup = self.index_to_soup(url)
-        tag=soup.find(id='content-area').div.div
+        tag = soup.find(id='content-area').div.div
         for i in tag.findAll('div', recursive=False):
             temp = i.find(attrs={'class':'views-field-title'}).span.a
             title = temp.string
@@ -64,7 +69,7 @@ class Dzieje(BasicNewsRecipe):
 
     def preprocess_html(self, soup):
         for a in soup('a'):
-            if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
+            if a.has_key('href') and not a['href'].startswith('http'):
-                a['href']=self.index + a['href']
+                a['href'] = self.index + a['href']
         self.append_page(soup, soup.body)
         return soup

recipes/dziennik_pl.recipe

@@ -2,6 +2,8 @@
 
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre.ebooks.BeautifulSoup import Comment
+
 class Dziennik_pl(BasicNewsRecipe):
     title          = u'Dziennik.pl'
     __author__        = 'fenuks'
@@ -9,17 +11,17 @@ class Dziennik_pl(BasicNewsRecipe):
     category       = 'newspaper'
     language       = 'pl'
     masthead_url= 'http://5.s.dziennik.pl/images/logos.png'
-    cover_url= 'http://5.s.dziennik.pl/images/logos.png'
+    cover_url = 'http://5.s.dziennik.pl/images/logos.png'
     no_stylesheets = True
     oldest_article = 7
     max_articles_per_feed = 100
-    remove_javascript=True
+    remove_javascript = True
-    remove_empty_feeds=True
+    remove_empty_feeds = True
     ignore_duplicate_articles = {'title', 'url'}
-    extra_css= 'ul {list-style: none; padding: 0; margin: 0;} li {float: left;margin: 0 0.15em;}'
+    extra_css = 'ul {list-style: none; padding: 0; margin: 0;} li {float: left;margin: 0 0.15em;}'
     preprocess_regexps     = [(re.compile("Komentarze:"), lambda m: ''), (re.compile('<p><strong><a href=".*?">&gt;&gt;&gt; CZYTAJ TAKŻE: ".*?"</a></strong></p>'), lambda m: '')]
-    keep_only_tags=[dict(id='article')]
+    keep_only_tags = [dict(id='article')]
-    remove_tags=[dict(name='div', attrs={'class':['art_box_dodatki', 'new_facebook_icons2', 'leftArt', 'article_print', 'quiz-widget', 'belka-spol', 'belka-spol belka-spol-bottom', 'art_data_tags', 'cl_right', 'boxRounded gal_inside']}), dict(name='a', attrs={'class':['komentarz', 'article_icon_addcommnent']})]
+    remove_tags = [dict(name='div', attrs={'class':['art_box_dodatki', 'new_facebook_icons2', 'leftArt', 'article_print', 'quiz-widget', 'belka-spol', 'belka-spol belka-spol-bottom', 'art_data_tags', 'cl_right', 'boxRounded gal_inside']}), dict(name='a', attrs={'class':['komentarz', 'article_icon_addcommnent']})]
     feeds          = [(u'Wszystko', u'http://rss.dziennik.pl/Dziennik-PL/'),
 		(u'Wiadomości', u'http://rss.dziennik.pl/Dziennik-Wiadomosci'),
 		(u'Gospodarka', u'http://rss.dziennik.pl/Dziennik-Gospodarka'),
@@ -34,26 +36,29 @@ class Dziennik_pl(BasicNewsRecipe):
 		(u'Nieruchomości', u'http://rss.dziennik.pl/Dziennik-Nieruchomosci')]
 
     def skip_ad_pages(self, soup):
-          tag=soup.find(name='a', attrs={'title':'CZYTAJ DALEJ'})
+          tag = soup.find(name='a', attrs={'title':'CZYTAJ DALEJ'})
           if tag:
-            new_soup=self.index_to_soup(tag['href'], raw=True)
+            new_soup = self.index_to_soup(tag['href'], raw=True)
             return new_soup
 
     def append_page(self, soup, appendtag):
-        tag=soup.find('a', attrs={'class':'page_next'})
+        tag = soup.find('a', attrs={'class':'page_next'})
         if tag:
             appendtag.find('div', attrs={'class':'article_paginator'}).extract()
         while tag:
-            soup2= self.index_to_soup(tag['href'])
+            soup2 = self.index_to_soup(tag['href'])
-            tag=soup2.find('a', attrs={'class':'page_next'})
+            tag = soup2.find('a', attrs={'class':'page_next'})
             if not tag:
                 for r in appendtag.findAll('div', attrs={'class':'art_src'}):
                     r.extract()
             pagetext = soup2.find(name='div', attrs={'class':'article_body'})
             for dictionary in self.remove_tags:
-                 v=pagetext.findAll(name=dictionary['name'], attrs=dictionary['attrs'])
+                 v = pagetext.findAll(name=dictionary['name'], attrs=dictionary['attrs'])
                  for delete in v:
                      delete.extract()
+            comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
             pos = len(appendtag.contents)
             appendtag.insert(pos, pagetext)
             if appendtag.find('div', attrs={'class':'article_paginator'}):

recipes/dziennik_wschodni.recipe

@@ -1,5 +1,7 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
+
 class DziennikWschodni(BasicNewsRecipe):
     title          = u'Dziennik Wschodni'
     __author__        = 'fenuks'
@@ -73,6 +75,10 @@ class DziennikWschodni(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
                     
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/echo_dnia.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class EchoDnia(BasicNewsRecipe):
     title          = u'Echo Dnia'
@@ -69,6 +70,10 @@ class EchoDnia(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
                     
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/eclipseonline.recipe

@@ -0,0 +1,38 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class EclipseOnline(BasicNewsRecipe):
+	
+	#
+	# oldest_article specifies the maximum age, in days, of posts to retrieve.
+	# The default of 32 is intended to work well with a "days of month = 1"
+	# recipe schedule to download "monthly issues" of Eclipse Online.
+	# Increase this value to include additional posts. However, the RSS feed
+	# currently only includes the 10 most recent posts, so that's the max.
+	#
+	oldest_article = 32
+	
+	title = u'Eclipse Online'
+	description = u'"Where strange and wonderful things happen, where reality is eclipsed for a little while with something magical and new." Eclipse Online is edited by Jonathan Strahan and published online by Night Shade Books. http://www.nightshadebooks.com/category/eclipse/'
+	publication_type = 'magazine'
+	language = 'en'
+	
+	__author__ = u'Jim DeVona'
+	__version__ = '1.0'
+	
+	# For now, use this Eclipse Online logo as the ebook cover image.
+	# (Disable the cover_url line to let Calibre generate a default cover, including date.)
+	cover_url = 'http://www.nightshadebooks.com/wp-content/uploads/2012/10/Eclipse-Logo.jpg'
+		
+	# Extract the "post" div containing the story (minus redundant metadata) from each page.
+	keep_only_tags = [dict(name='div', attrs={'class':lambda x: x and 'post' in x})]
+	remove_tags = [dict(name='span', attrs={'class': ['post-author', 'post-category', 'small']})]
+
+	# Nice plain markup (like Eclipse's) works best for most e-readers.
+	# Disregard any special styling rules, but center illustrations.
+	auto_cleanup = False
+	no_stylesheets = True
+	remove_attributes = ['style', 'align']
+	extra_css = '.wp-caption {text-align: center;} .wp-caption-text {font-size: small; font-style: italic;}'
+	
+	# Tell Calibre where to look for article links. It will proceed to retrieve
+	# these posts and format them into an ebook according to the above rules.
+	feeds = ['http://www.nightshadebooks.com/category/eclipse/feed/']

recipes/ekologia_pl.recipe

@@ -9,7 +9,7 @@ class EkologiaPl(BasicNewsRecipe):
     language       = 'pl'
     cover_url = 'http://www.ekologia.pl/assets/images/logo/ekologia_pl_223x69.png'
     ignore_duplicate_articles = {'title', 'url'}
-    extra_css = '.title {font-size: 200%;}'
+    extra_css = '.title {font-size: 200%;} .imagePowiazane, .imgCon {float:left; margin-right:5px;}'
     oldest_article = 7
     max_articles_per_feed = 100
     no_stylesheets = True

recipes/esenja.recipe

@@ -3,29 +3,37 @@
 __license__   = 'GPL v3'
 __copyright__ = '2010, matek09, matek09@gmail.com'
 
-from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Comment
 
 class Esensja(BasicNewsRecipe):
 
     title = u'Esensja'
-	__author__ = 'matek09'
+    __author__ = 'matek09 & fenuks'
-	description = 'Monthly magazine'
+    description = 'Magazyn kultury popularnej'
     encoding = 'utf-8'
     no_stylesheets = True
     language = 'pl'
     remove_javascript = True
+    masthead_url = 'http://esensja.pl/img/wrss.gif'
+    oldest_article = 1
+    URL = 'http://esensja.pl'
     HREF = '0'
-
+    remove_attributes = ['style', 'bgcolor', 'alt', 'color']
-	#keep_only_tags =[]
+    keep_only_tags = [dict(attrs={'class':'sekcja'}), ]
     #keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'article'})
-	remove_tags_before = dict(dict(name = 'div', attrs = {'class' : 't-title'}))
+    #remove_tags_before = dict(dict(name = 'div', attrs = {'class' : 't-title'}))
-	remove_tags_after = dict(dict(name = 'img', attrs = {'src' : '../../../2000/01/img/tab_bot.gif'}))
+    remove_tags_after = dict(id='tekst')
 
-	remove_tags =[]
+    remove_tags = [dict(name = 'img', attrs = {'src' : ['../../../2000/01/img/tab_top.gif', '../../../2000/01/img/tab_bot.gif']}),
-	remove_tags.append(dict(name = 'img', attrs = {'src' : '../../../2000/01/img/tab_top.gif'}))
+                    dict(name = 'div', attrs = {'class' : 't-title2 nextpage'}),
-	remove_tags.append(dict(name = 'img', attrs = {'src' : '../../../2000/01/img/tab_bot.gif'}))
+                    #dict(attrs={'rel':'lightbox[galeria]'})
-	remove_tags.append(dict(name = 'div', attrs = {'class' : 't-title2 nextpage'}))
+                    dict(attrs={'class':['tekst_koniec', 'ref', 'wykop']}),
+                    dict(attrs={'itemprop':['copyrightHolder', 'publisher']}),
+                    dict(id='komentarze')
+                    
+                    ]
 
     extra_css = '''
                                     .t-title {font-size: x-large; font-weight: bold; text-align: left}
@@ -35,8 +43,9 @@ class Esensja(BasicNewsRecipe):
                                     .annot-ref {font-style: italic; text-align: left}
                             '''
 
-	preprocess_regexps = [(re.compile(r'alt="[^"]*"'),
+    preprocess_regexps = [(re.compile(r'alt="[^"]*"'), lambda match: ''), 
-						lambda match: '')]
+                        (re.compile(ur'(title|alt)="[^"]*?"', re.DOTALL), lambda match: ''),
+                        ]
 
     def parse_index(self):
         soup = self.index_to_soup('http://www.esensja.pl/magazyn/')
@@ -47,15 +56,19 @@ class Esensja(BasicNewsRecipe):
         soup = self.index_to_soup(self.HREF + '01.html')
         self.cover_url = 'http://www.esensja.pl/magazyn/' + year + '/' + month + '/img/ilustr/cover_b.jpg'
         feeds = []
+        chapter = ''
+        subchapter = ''
+        articles = []
         intro = soup.find('div', attrs={'class' : 'n-title'})
+        '''
         introduction = {'title' : self.tag_to_string(intro.a),
                                         'url' : self.HREF + intro.a['href'],
                                         'date' : '',
                                         'description' : ''}
         chapter = 'Wprowadzenie'
-		subchapter = ''
-		articles = []
         articles.append(introduction)
+        '''
+        
         for tag in intro.findAllNext(attrs={'class': ['chapter', 'subchapter', 'n-title']}):
             if tag.name in 'td':
                 if len(articles) > 0:
@@ -71,17 +84,72 @@ class Esensja(BasicNewsRecipe):
                     subchapter = self.tag_to_string(tag)
                     subchapter = self.tag_to_string(tag)
                 continue
-			articles.append({'title' : self.tag_to_string(tag.a), 'url' : self.HREF + tag.a['href'], 'date' : '', 'description' : ''})
                 
-			a = self.index_to_soup(self.HREF + tag.a['href'])
+            finalurl = tag.a['href']
+            if not finalurl.startswith('http'):
+                finalurl = self.HREF + finalurl
+            articles.append({'title' : self.tag_to_string(tag.a), 'url' : finalurl, 'date' : '', 'description' : ''})
+            
+            a = self.index_to_soup(finalurl)
             i = 1
+            
             while True:
                 div = a.find('div', attrs={'class' : 't-title2 nextpage'})
                 if div is not None:
-					a = self.index_to_soup(self.HREF + div.a['href'])
+                    link = div.a['href']
-					articles.append({'title' : self.tag_to_string(tag.a) + ' c. d. ' + str(i), 'url' : self.HREF + div.a['href'], 'date' : '', 'description' : ''})
+                    if not link.startswith('http'):
+                        link = self.HREF + link
+                    a = self.index_to_soup(link)
+                    articles.append({'title' : self.tag_to_string(tag.a) + ' c. d. ' + str(i), 'url' : link, 'date' : '', 'description' : ''})
                     i = i + 1
                 else:
                     break
 
         return feeds
+
+    def append_page(self, soup, appendtag):
+        r = appendtag.find(attrs={'class':'wiecej_xxx'})
+        if r:
+            nr = r.findAll(attrs={'class':'tn-link'})[-1]
+            try:
+                nr = int(nr.a.string)
+            except:
+                return
+            baseurl = soup.find(attrs={'property':'og:url'})['content'] + '&strona={0}'
+            for number in range(2, nr+1):
+                soup2 = self.index_to_soup(baseurl.format(number))
+                pagetext = soup2.find(attrs={'class':'tresc'})
+                pos = len(appendtag.contents)
+                appendtag.insert(pos, pagetext)
+            for r in appendtag.findAll(attrs={'class':['wiecej_xxx', 'tekst_koniec']}):
+                r.extract()
+            for r in appendtag.findAll('script'):
+                r.extract()
+
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
+    def preprocess_html(self, soup):
+        self.append_page(soup, soup.body)
+        for tag in soup.findAll(attrs={'class':'img_box_right'}):
+            temp = tag.find('img')
+            src = ''
+            if temp:
+                src = temp.get('src', '')
+            for r in tag.findAll('a', recursive=False):
+                r.extract()
+            info = tag.find(attrs={'class':'img_info'})
+            text = str(tag)
+            if not src:
+                src = re.search('src="[^"]*?"', text)
+                if src:
+                    src = src.group(0)
+                    src = src[5:].replace('//', '/')
+            if src:
+                tag.contents = []
+                tag.insert(0, BeautifulSoup('<img src="{0}{1}" />'.format(self.URL, src)))
+            if info:
+                tag.insert(len(tag.contents), info)
+        return soup
+

recipes/esensja_(rss).recipe

@@ -0,0 +1,109 @@
+__license__ = 'GPL v3'
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, Comment
+
+class EsensjaRSS(BasicNewsRecipe):
+    title          = u'Esensja (RSS)'
+    __author__        = 'fenuks'
+    description   = u'Magazyn kultury popularnej'
+    category       = 'reading, fantasy, reviews, boardgames, culture'
+    #publication_type = ''
+    language       = 'pl'
+    encoding = 'utf-8'
+    INDEX = 'http://www.esensja.pl'
+    extra_css = '''.t-title {font-size: x-large; font-weight: bold; text-align: left}
+                    .t-author {font-size: x-small; text-align: left}
+                    .t-title2 {font-size: x-small; font-style: italic; text-align: left}
+                    .text {font-size: small; text-align: left}
+                    .annot-ref {font-style: italic; text-align: left}
+                    '''
+    cover_url = ''
+    masthead_url = 'http://esensja.pl/img/wrss.gif'
+    use_embedded_content = False
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_empty_feeds = True
+    remove_javascript = True
+    ignore_duplicate_articles = {'title', 'url'}
+    preprocess_regexps = [(re.compile(r'alt="[^"]*"'), lambda match: ''), 
+                        (re.compile(ur'(title|alt)="[^"]*?"', re.DOTALL), lambda match: ''),
+                        ]
+    remove_attributes = ['style', 'bgcolor', 'alt', 'color']
+    keep_only_tags = [dict(attrs={'class':'sekcja'}), ]
+    remove_tags_after = dict(id='tekst')
+
+    remove_tags = [dict(name = 'img', attrs = {'src' : ['../../../2000/01/img/tab_top.gif', '../../../2000/01/img/tab_bot.gif']}),
+                    dict(name = 'div', attrs = {'class' : 't-title2 nextpage'}),
+                    #dict(attrs={'rel':'lightbox[galeria]'})
+                    dict(attrs={'class':['tekst_koniec', 'ref', 'wykop']}),
+                    dict(attrs={'itemprop':['copyrightHolder', 'publisher']}),
+                    dict(id='komentarze')                
+                    ]
+    
+    feeds = [(u'Książka', u'http://esensja.pl/rss/ksiazka.rss'), 
+            (u'Film', u'http://esensja.pl/rss/film.rss'), 
+            (u'Komiks', u'http://esensja.pl/rss/komiks.rss'),
+            (u'Gry', u'http://esensja.pl/rss/gry.rss'),
+            (u'Muzyka', u'http://esensja.pl/rss/muzyka.rss'),
+            (u'Twórczość', u'http://esensja.pl/rss/tworczosc.rss'),
+            (u'Varia', u'http://esensja.pl/rss/varia.rss'),
+            (u'Zgryźliwi Tetrycy', u'http://esensja.pl/rss/tetrycy.rss'),
+            (u'Nowe książki', u'http://esensja.pl/rss/xnowosci.rss'),
+            (u'Ostatnio dodane książki', u'http://esensja.pl/rss/xdodane.rss'),
+            ]
+ 
+    def get_cover_url(self):
+        soup = self.index_to_soup(self.INDEX)
+        cover = soup.find(id='panel_1')
+        self.cover_url = self.INDEX + cover.find('a')['href'].replace('index.html', '') + 'img/ilustr/cover_b.jpg'
+        return getattr(self, 'cover_url', self.cover_url)
+
+
+    def append_page(self, soup, appendtag):
+        r = appendtag.find(attrs={'class':'wiecej_xxx'})
+        if r:
+            nr = r.findAll(attrs={'class':'tn-link'})[-1]
+            try:
+                nr = int(nr.a.string)
+            except:
+                return
+            baseurl = soup.find(attrs={'property':'og:url'})['content'] + '&strona={0}'
+            for number in range(2, nr+1):
+                soup2 = self.index_to_soup(baseurl.format(number))
+                pagetext = soup2.find(attrs={'class':'tresc'})
+                pos = len(appendtag.contents)
+                appendtag.insert(pos, pagetext)
+            for r in appendtag.findAll(attrs={'class':['wiecej_xxx', 'tekst_koniec']}):
+                r.extract()
+            for r in appendtag.findAll('script'):
+                r.extract()
+
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
+
+    def preprocess_html(self, soup):
+        self.append_page(soup, soup.body)
+        for tag in soup.findAll(attrs={'class':'img_box_right'}):
+            temp = tag.find('img')
+            src = ''
+            if temp:
+                src = temp.get('src', '')
+            for r in tag.findAll('a', recursive=False):
+                r.extract()
+            info = tag.find(attrs={'class':'img_info'})
+            text = str(tag)
+            if not src:
+                src = re.search('src="[^"]*?"', text)
+                if src:
+                    src = src.group(0)
+                    src = src[5:].replace('//', '/')
+            if src:
+                tag.contents = []
+                tag.insert(0, BeautifulSoup('<img src="{0}{1}" />'.format(self.INDEX, src)))
+            if info:
+                tag.insert(len(tag.contents), info)
+        return soup

recipes/film_org_pl.recipe

@@ -7,6 +7,7 @@ class FilmOrgPl(BasicNewsRecipe):
     description   = u"Recenzje, analizy, artykuły, rankingi - wszystko o filmie dla miłośników kina. Opisy efektów specjalnych, wersji reżyserskich, remake'ów, sequeli. No i forum filmowe. Jedne z największych w Polsce."
     category       = 'film'
     language       = 'pl'
+    extra_css = '.alignright {float:right; margin-left:5px;} .alignleft {float:left; margin-right:5px;}'
     cover_url = 'http://film.org.pl/wp-content/themes/KMF/images/logo_kmf10.png'
     ignore_duplicate_articles = {'title', 'url'}
     oldest_article = 7

recipes/film_web.recipe

@@ -1,6 +1,7 @@
-from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
+
 class FilmWebPl(BasicNewsRecipe):
     title          = u'FilmWeb'
     __author__        = 'fenuks'
@@ -14,11 +15,12 @@ class FilmWebPl(BasicNewsRecipe):
     no_stylesheets = True
     remove_empty_feeds = True
     ignore_duplicate_articles = {'title', 'url'}
-    preprocess_regexps = [(re.compile(u'\(kliknij\,\ aby powiększyć\)', re.IGNORECASE), lambda m: ''), ]#(re.compile(ur' | ', re.IGNORECASE), lambda m: '')]
+    remove_javascript = True
+    preprocess_regexps = [(re.compile(u'\(kliknij\,\ aby powiększyć\)', re.IGNORECASE), lambda m: ''), (re.compile(ur'(<br ?/?>\s*?<br ?/?>\s*?)+', re.IGNORECASE), lambda m: '<br />')]#(re.compile(ur' | ', re.IGNORECASE), lambda m: '')]
     extra_css      = '.hdrBig {font-size:22px;} ul {list-style-type:none; padding: 0; margin: 0;}'
-    remove_tags = [dict(name='div', attrs={'class':['recommendOthers']}), dict(name='ul', attrs={'class':'fontSizeSet'}), dict(attrs={'class':'userSurname anno'})]
+    #remove_tags = [dict()]
     remove_attributes = ['style',]
-    keep_only_tags = [dict(name='h1', attrs={'class':['hdrBig', 'hdrEntity']}), dict(name='div', attrs={'class':['newsInfo', 'newsInfoSmall', 'reviewContent description']})]
+    keep_only_tags = [dict(attrs={'class':['hdr hdr-super', 'newsContent']})]
     feeds          = [(u'News / Filmy w produkcji', 'http://www.filmweb.pl/feed/news/category/filminproduction'),
                          (u'News / Festiwale, nagrody i przeglądy', u'http://www.filmweb.pl/feed/news/category/festival'),
                          (u'News / Seriale', u'http://www.filmweb.pl/feed/news/category/serials'),
@@ -42,6 +44,11 @@ class FilmWebPl(BasicNewsRecipe):
         if skip_tag is not None:
             return self.index_to_soup(skip_tag['href'], raw=True)
 
+    def postprocess_html(self, soup, first_fetch):
+        for r in soup.findAll(attrs={'class':'singlephoto'}):
+            r['style'] = 'float:left; margin-right: 10px;'
+        return soup
+
     def preprocess_html(self, soup):
         for a in soup('a'):
             if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
@@ -51,9 +58,8 @@ class FilmWebPl(BasicNewsRecipe):
         for i in soup.findAll('sup'):
             if not i.string or i.string.startswith('(kliknij'):
                 i.extract()
-        tag = soup.find(name='ul', attrs={'class':'inline sep-line'})
+        for r in soup.findAll(id=re.compile('photo-\d+')):
-        if tag:
+            r.extract()
-            tag.name = 'div'
+        for r in soup.findAll(style=re.compile('float: ?left')):
-            for t in tag.findAll('li'):
+            r['class'] = 'singlephoto'
-                t.name = 'div'
         return soup

recipes/foreignaffairs.recipe

@@ -1,6 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
-from calibre.ptempfile import PersistentTemporaryFile
 
 class ForeignAffairsRecipe(BasicNewsRecipe):
     ''' there are three modifications:
@@ -45,7 +44,6 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
                           'publisher': publisher}
 
     temp_files = []
-    articles_are_obfuscated = True
 
     def get_cover_url(self):
         soup = self.index_to_soup(self.FRONTPAGE)
@@ -53,20 +51,6 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
         img_url =  div.find('img')['src']
         return self.INDEX + img_url
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        br.open(url)
-
-        response = br.follow_link(url_regex = r'/print/[0-9]+', nr = 0)
-        html = response.read()
-
-        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
-        self.temp_files[-1].write(html)
-        self.temp_files[-1].close()
-
-        return self.temp_files[-1].name
-
-
     def parse_index(self):
 
         answer = []
@@ -89,10 +73,10 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
                         if div.find('a') is not None:
                             originalauthor=self.tag_to_string(div.findNext('div', attrs = {'class':'views-field-field-article-book-nid'}).div.a)
                             title=subsectiontitle+': '+self.tag_to_string(div.span.a)+' by '+originalauthor
-                            url=self.INDEX+div.span.a['href']
+                            url=self.INDEX+self.index_to_soup(self.INDEX+div.span.a['href']).find('a', attrs={'class':'fa_addthis_print'})['href']
                             atr=div.findNext('div', attrs = {'class': 'views-field-field-article-display-authors-value'})
                             if atr is not None:
-                                author=self.tag_to_string(atr.span.a)
+                                author=self.tag_to_string(atr.span)
                             else:
                                 author=''
                             desc=div.findNext('span', attrs = {'class': 'views-field-field-article-summary-value'})
@@ -106,10 +90,10 @@ class ForeignAffairsRecipe(BasicNewsRecipe):
                 for div in sec.findAll('div', attrs = {'class': 'views-field-title'}):
                     if div.find('a') is not None:
                         title=self.tag_to_string(div.span.a)
-                        url=self.INDEX+div.span.a['href']
+                        url=self.INDEX+self.index_to_soup(self.INDEX+div.span.a['href']).find('a', attrs={'class':'fa_addthis_print'})['href']
                         atr=div.findNext('div', attrs = {'class': 'views-field-field-article-display-authors-value'})
                         if atr is not None:
-                            author=self.tag_to_string(atr.span.a)
+                            author=self.tag_to_string(atr.span)
                         else:
                             author=''
                         desc=div.findNext('span', attrs = {'class': 'views-field-field-article-summary-value'})

recipes/fortune_magazine.recipe

@@ -0,0 +1,75 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from collections import OrderedDict
+
+class Fortune(BasicNewsRecipe):
+
+    title       = 'Fortune Magazine'
+    __author__  = 'Rick Shang'
+
+    description = 'FORTUNE is a global business magazine that has been revered in its content and credibility since 1930. FORTUNE covers the entire field of business, including specific companies and business trends, prominent business leaders, and new ideas shaping the global marketplace.'
+    language = 'en'
+    category = 'news'
+    encoding = 'UTF-8'
+    keep_only_tags = [dict(attrs={'id':['storycontent']})]
+    remove_tags = [dict(attrs={'class':['hed_side','socialMediaToolbarContainer']})]
+    no_javascript = True
+    no_stylesheets = True
+    needs_subscription = True
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser(self)
+        br.open('http://money.cnn.com/2013/03/21/smallbusiness/legal-marijuana-startups.pr.fortune/index.html')
+        br.select_form(name="paywall-form")
+        br['email'] = self.username
+        br['password'] = self.password
+        br.submit()
+        return br
+
+    def parse_index(self):
+        articles = []
+        soup0 = self.index_to_soup('http://money.cnn.com/magazines/fortune/')
+
+        #Go to the latestissue
+        soup = self.index_to_soup(soup0.find('div',attrs={'class':'latestissue'}).find('a',href=True)['href'])
+
+        #Find cover & date
+        cover_item = soup.find('div', attrs={'id':'cover-story'})
+        cover = cover_item.find('img',src=True)
+        self.cover_url = cover['src']
+        date = self.tag_to_string(cover_item.find('div', attrs={'class':'tocDate'})).strip()
+        self.timefmt = u' [%s]'%date
+
+
+        feeds = OrderedDict()
+        section_title = ''
+
+        #checkout the cover story
+        articles = []
+        coverstory=soup.find('div', attrs={'class':'cnnHeadline'})
+        title=self.tag_to_string(coverstory.a).strip()
+        url=coverstory.a['href']
+        desc=self.tag_to_string(coverstory.findNext('p', attrs={'class':'cnnBlurbTxt'}))
+        articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
+        feeds['Cover Story'] = []
+        feeds['Cover Story'] += articles
+
+        for post in soup.findAll('div', attrs={'class':'cnnheader'}):
+            section_title = self.tag_to_string(post).strip()
+            articles = []
+
+            ul=post.findNext('ul')
+            for link in ul.findAll('li'):
+                links=link.find('h2')
+                title=self.tag_to_string(links.a).strip()
+                url=links.a['href']
+                desc=self.tag_to_string(link.find('p', attrs={'class':'cnnBlurbTxt'}))
+                articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
+
+            if articles:
+                if section_title not in feeds:
+                            feeds[section_title] = []
+                feeds[section_title] += articles
+
+        ans = [(key, val) for key, val in feeds.iteritems()]
+        return ans
+

recipes/gazeta_lubuska.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class GazetaLubuska(BasicNewsRecipe):
     title          = u'Gazeta Lubuska'
@@ -59,6 +60,10 @@ class GazetaLubuska(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
                     
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/gazeta_pomorska.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class GazetaPomorska(BasicNewsRecipe):
     title          = u'Gazeta Pomorska'
@@ -86,6 +87,10 @@ class GazetaPomorska(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
             
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/gazeta_wspolczesna.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class GazetaWspolczesna(BasicNewsRecipe):
     title          = u'Gazeta Wsp\xf3\u0142czesna'
@@ -58,6 +59,10 @@ class GazetaWspolczesna(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
             
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/gazeta_wyborcza.recipe

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from calibre.web.feeds.news import BasicNewsRecipe
-
+from calibre.ebooks.BeautifulSoup import Comment
 
 class Gazeta_Wyborcza(BasicNewsRecipe):
     title = u'Gazeta.pl'
@@ -16,6 +16,7 @@ class Gazeta_Wyborcza(BasicNewsRecipe):
     max_articles_per_feed = 100
     remove_javascript = True
     no_stylesheets = True
+    ignore_duplicate_articles = {'title', 'url'}
     remove_tags_before = dict(id='k0')
     remove_tags_after = dict(id='banP4')
     remove_tags = [dict(name='div', attrs={'class':'rel_box'}), dict(attrs={'class':['date', 'zdjP', 'zdjM', 'pollCont', 'rel_video', 'brand', 'txt_upl']}), dict(name='div', attrs={'id':'footer'})]
@@ -48,6 +49,9 @@ class Gazeta_Wyborcza(BasicNewsRecipe):
                     url = self.INDEX + link['href']
                     soup2 = self.index_to_soup(url)
                     pagetext = soup2.find(id='artykul')
+                    comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+                    for comment in comments:
+                        comment.extract()
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
                     tag = soup2.find('div', attrs={'id': 'Str'})
@@ -65,6 +69,9 @@ class Gazeta_Wyborcza(BasicNewsRecipe):
             nexturl = pagetext.find(id='gal_btn_next')
             if nexturl:
                 nexturl = nexturl.a['href']
+                comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+                for comment in comments:
+                    comment.extract()
                 pos = len(appendtag.contents)
                 appendtag.insert(pos, pagetext)
             rem = appendtag.find(id='gal_navi')
@@ -105,3 +112,7 @@ class Gazeta_Wyborcza(BasicNewsRecipe):
         soup = self.index_to_soup('http://wyborcza.pl/' + cover.contents[3].a['href'])
         self.cover_url = 'http://wyborcza.pl' + soup.img['src']
         return getattr(self, 'cover_url', self.cover_url)
+
+    '''def image_url_processor(self, baseurl, url):
+        print "@@@@@@@@", url
+        return url.replace('http://wyborcza.pl/ ', '')'''

recipes/gcn.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class GCN(BasicNewsRecipe):
     title          = u'Gazeta Codziennej Nowiny'
@@ -16,7 +17,7 @@ class GCN(BasicNewsRecipe):
     remove_empty_feeds = True
     no_stylesheets = True
     ignore_duplicate_articles = {'title', 'url'}
-
+    remove_attributes = ['style']
     preprocess_regexps = [(re.compile(ur'Czytaj:.*?</a>', re.DOTALL), lambda match: ''), (re.compile(ur'Przeczytaj także:.*?</a>', re.DOTALL|re.IGNORECASE), lambda match: ''),
                           (re.compile(ur'Przeczytaj również:.*?</a>', re.DOTALL|re.IGNORECASE), lambda match: ''), (re.compile(ur'Zobacz też:.*?</a>', re.DOTALL|re.IGNORECASE), lambda match: '')]
 
@@ -78,6 +79,10 @@ class GCN(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
 
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/gildia_pl.recipe

@@ -11,12 +11,13 @@ class Gildia(BasicNewsRecipe):
     language       = 'pl'
     oldest_article = 8
     max_articles_per_feed = 100
-    remove_empty_feeds=True
+    remove_empty_feeds = True
-    no_stylesheets=True
+    no_stylesheets = True
     ignore_duplicate_articles = {'title', 'url'}
     preprocess_regexps = [(re.compile(ur'</?sup>'), lambda match: '') ]
-    remove_tags=[dict(name='div', attrs={'class':'backlink'}), dict(name='div', attrs={'class':'im_img'}), dict(name='div', attrs={'class':'addthis_toolbox addthis_default_style'})]
+    ignore_duplicate_articles = {'title', 'url'}
-    keep_only_tags=dict(name='div', attrs={'class':'widetext'})
+    remove_tags = [dict(name='div', attrs={'class':'backlink'}), dict(name='div', attrs={'class':'im_img'}), dict(name='div', attrs={'class':'addthis_toolbox addthis_default_style'})]
+    keep_only_tags = dict(name='div', attrs={'class':'widetext'})
     feeds          = [(u'Gry', u'http://www.gry.gildia.pl/rss'), (u'Literatura', u'http://www.literatura.gildia.pl/rss'), (u'Film', u'http://www.film.gildia.pl/rss'), (u'Horror', u'http://www.horror.gildia.pl/rss'), (u'Konwenty', u'http://www.konwenty.gildia.pl/rss'), (u'Plansz\xf3wki', u'http://www.planszowki.gildia.pl/rss'), (u'Manga i anime', u'http://www.manga.gildia.pl/rss'), (u'Star Wars', u'http://www.starwars.gildia.pl/rss'), (u'Techno', u'http://www.techno.gildia.pl/rss'), (u'Historia', u'http://www.historia.gildia.pl/rss'), (u'Magia', u'http://www.magia.gildia.pl/rss'), (u'Bitewniaki', u'http://www.bitewniaki.gildia.pl/rss'), (u'RPG', u'http://www.rpg.gildia.pl/rss'), (u'LARP', u'http://www.larp.gildia.pl/rss'), (u'Muzyka', u'http://www.muzyka.gildia.pl/rss'), (u'Nauka', u'http://www.nauka.gildia.pl/rss')]
 
 
@@ -34,7 +35,7 @@ class Gildia(BasicNewsRecipe):
 
     def preprocess_html(self, soup):
         for a in soup('a'):
-            if a.has_key('href') and 'http://' not in a['href'] and 'https://' not in a['href']:
+            if a.has_key('href') and not a['href'].startswith('http'):
                 if '/gry/' in a['href']:
                     a['href']='http://www.gry.gildia.pl' + a['href']
                 elif u'książk' in soup.title.string.lower() or u'komiks' in soup.title.string.lower():

recipes/gofin_pl.recipe

@@ -0,0 +1,26 @@
+#!/usr/bin/env  python
+
+__license__ = 'GPL v3'
+__author__ = 'teepel <teepel44@gmail.com>'
+
+'''
+gofin.pl
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class gofin(AutomaticNewsRecipe):
+    title          = u'Gofin'
+    __author__ = 'teepel <teepel44@gmail.com>'
+    language       = 'pl'
+    description =u'Portal Podatkowo-Księgowy'
+    INDEX='http://gofin.pl'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    remove_empty_feeds= True
+    simultaneous_downloads = 5
+    remove_javascript=True
+    no_stylesheets=True
+    auto_cleanup = True
+
+    feeds          = [(u'Podatki', u'http://www.rss.gofin.pl/podatki.xml'), (u'Prawo Pracy', u'http://www.rss.gofin.pl/prawopracy.xml'), (u'Rachunkowo\u015b\u0107', u'http://www.rss.gofin.pl/rachunkowosc.xml'), (u'Sk\u0142adki, zasi\u0142ki, emerytury', u'http://www.rss.gofin.pl/zasilki.xml'),(u'Firma', u'http://www.rss.gofin.pl/firma.xml'), (u'Prawnik radzi', u'http://www.rss.gofin.pl/prawnikradzi.xml')]

recipes/gram_pl.recipe

@@ -1,5 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
+
 class Gram_pl(BasicNewsRecipe):
     title          = u'Gram.pl'
     __author__        = 'fenuks'

recipes/gry_online_pl.recipe

@@ -1,5 +1,6 @@
 import time
 from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class GryOnlinePl(BasicNewsRecipe):
     title          = u'Gry-Online.pl'
@@ -40,10 +41,14 @@ class GryOnlinePl(BasicNewsRecipe):
                     r.extract()
                 for r in pagetext.findAll(attrs={'itemprop':'description'}):
                     r.extract()
+                    
                 pos = len(appendtag.contents)
                 appendtag.insert(pos, pagetext)
             for r in appendtag.findAll(attrs={'class':['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry']}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
         else:
             tag = appendtag.find('div', attrs={'class':'S018stronyr'})
             if tag:
@@ -70,10 +75,16 @@ class GryOnlinePl(BasicNewsRecipe):
                         r.extract()
                     for r in pagetext.findAll(attrs={'itemprop':'description'}):
                         r.extract()
+                        
+                    comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+                    [comment.extract() for comment in comments]
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
                 for r in appendtag.findAll(attrs={'class':['n5p', 'add-info', 'twitter-share-button', 'lista lista3 lista-gry', 'S018strony']}):
                     r.extract()
+                comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+                for comment in comments:
+                    comment.extract()
 
     def image_url_processor(self, baseurl, url):
         if url.startswith('..'):

recipes/harpers_full.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008-2013, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 harpers.org - paid subscription/ printed issue articles
 This recipe only get's article's published in text format
@@ -14,7 +14,7 @@ from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Harpers_full(BasicNewsRecipe):
-    title                 = "Harper's Magazine - articles from printed edition"
+    title                 = "Harper's Magazine - Printed Edition"
     __author__            = 'Darko Miletic'
     description           = "Harper's Magazine, the oldest general-interest monthly in America, explores the issues that drive our national conversation, through long-form narrative journalism and essays, and such celebrated features as the iconic Harper's Index."
     publisher             = "Harpers's"
@@ -29,6 +29,7 @@ class Harpers_full(BasicNewsRecipe):
     needs_subscription    = 'optional'
     masthead_url          = 'http://harpers.org/wp-content/themes/harpers/images/pheader.gif'
     publication_type      = 'magazine'
+    INDEX                 = ''
     LOGIN                 = 'http://harpers.org/wp-content/themes/harpers/ajax_login.php'
     extra_css             = """
                                 body{font-family: adobe-caslon-pro,serif}
@@ -65,33 +66,29 @@ class Harpers_full(BasicNewsRecipe):
 
     def parse_index(self):
         #find current issue
+
         soup = self.index_to_soup('http://harpers.org/')
         currentIssue=soup.find('div',attrs={'class':'mainNavi'}).find('li',attrs={'class':'curentIssue'})
         currentIssue_url=self.tag_to_string(currentIssue.a['href'])
-        self.log(currentIssue_url)
 
         #go to the current issue
         soup1 = self.index_to_soup(currentIssue_url)
-        currentIssue_title = self.tag_to_string(soup1.head.title.string)
+        date = re.split('\s\|\s',self.tag_to_string(soup1.head.title.string))[0]
-        date = re.split('\s\|\s',currentIssue_title)[0]
         self.timefmt =  u' [%s]'%date
 
         #get cover
-        coverurl='http://harpers.org/wp-content/themes/harpers/ajax_microfiche.php?img=harpers-'+re.split('harpers.org/',currentIssue_url)[1]+'gif/0001.gif'
+        self.cover_url = soup1.find('div', attrs = {'class':'picture_hp'}).find('img', src=True)['src']
-        soup2 = self.index_to_soup(coverurl)
+
-        self.cover_url = self.tag_to_string(soup2.find('img')['src'])
-        self.log(self.cover_url)
         articles = []
         count = 0
         for item in soup1.findAll('div', attrs={'class':'articleData'}):
             text_links = item.findAll('h2')
-            if text_links:
             for text_link in text_links:
                 if count == 0:
                    count = 1
                 else:
                    url   = text_link.a['href']
-                       title = self.tag_to_string(text_link.a)
+                   title = text_link.a.contents[0]
                    date  = strftime(' %B %Y')
                    articles.append({
                                       'title'      :title
@@ -99,9 +96,13 @@ class Harpers_full(BasicNewsRecipe):
                                      ,'url'        :url
                                      ,'description':''
                                     })
-        return [(currentIssue_title, articles)]
+        return [(soup1.head.title.string, articles)]
 
     def print_version(self, url):
         return url + '?single=1'
 
-
+    def cleanup(self):
+        soup = self.index_to_soup('http://harpers.org/')
+        signouturl=self.tag_to_string(soup.find('li', attrs={'class':'subLogOut'}).findNext('li').a['href'])
+        self.log(signouturl)
+        self.browser.open(signouturl)

recipes/hindu.recipe

@@ -2,7 +2,6 @@ from __future__ import with_statement
 __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 
-import time
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class TheHindu(BasicNewsRecipe):
@@ -14,44 +13,42 @@ class TheHindu(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets = True
 
-    keep_only_tags = [dict(id='content')]
+    auto_cleanup = True
-    remove_tags = [dict(attrs={'class':['article-links', 'breadcr']}),
+
-            dict(id=['email-section', 'right-column', 'printfooter', 'topover',
-                     'slidebox', 'th_footer'])]
 
     extra_css = '.photo-caption { font-size: smaller }'
 
-    def preprocess_raw_html(self, raw, url):
-        return raw.replace('<body><p>', '<p>').replace('</p></body>', '</p>')
-
-    def postprocess_html(self, soup, first_fetch):
-        for t in soup.findAll(['table', 'tr', 'td','center']):
-            t.name = 'div'
-        return soup
-
     def parse_index(self):
-        today = time.strftime('%Y-%m-%d')
+        soup = self.index_to_soup('http://www.thehindu.com/todays-paper/')
-        soup = self.index_to_soup(
+        div = soup.find('div', attrs={'id':'left-column'})
-                'http://www.thehindu.com/todays-paper/tp-index/?date=' + today)
+        soup.find(id='subnav-tpbar').extract()
-        div = soup.find(id='left-column')
+
-        feeds = []
+
+
         current_section = None
         current_articles = []
-        for x in div.findAll(['h3', 'div']):
+        feeds = []
-            if current_section and x.get('class', '') == 'tpaper':
+        for x in div.findAll(['a', 'span']):
-                a = x.find('a', href=True)
+            if x.name == 'span' and x['class'] == 's-link':
-                if a is not None:
+                # Section heading found
-                    title = self.tag_to_string(a)
+                if current_articles and current_section:
-                    self.log('\tFound article:', title)
-                    current_articles.append({'url':a['href']+'?css=print',
-                        'title':title, 'date': '',
-                        'description':''})
-            if x.name == 'h3':
-                if current_section and current_articles:
                     feeds.append((current_section, current_articles))
                 current_section = self.tag_to_string(x)
-                self.log('Found section:', current_section)
                 current_articles = []
+                self.log('\tFound section:', current_section)
+            elif x.name == 'a':
+
+                        title = self.tag_to_string(x)
+                        url = x.get('href', False)
+                        if not url or not title:
+                            continue
+                        self.log('\t\tFound article:', title)
+                        self.log('\t\t\t', url)
+                        current_articles.append({'title': title, 'url':url,
+                            'description':'', 'date':''})
+
+        if current_articles and current_section:
+             feeds.append((current_section, current_articles))
+
         return feeds
 
-

recipes/history_today.recipe

@@ -1,6 +1,6 @@
-import re
-from calibre.web.feeds.recipes import BasicNewsRecipe
 from collections import OrderedDict
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
 
 class HistoryToday(BasicNewsRecipe):
 
@@ -19,7 +19,6 @@ class HistoryToday(BasicNewsRecipe):
 
 
     needs_subscription = True
-
     def get_browser(self):
         br = BasicNewsRecipe.get_browser(self)
         if self.username is not None and self.password is not None:
@@ -46,8 +45,9 @@ class HistoryToday(BasicNewsRecipe):
 
         #Go to issue
         soup = self.index_to_soup('http://www.historytoday.com/contents')
-        cover = soup.find('div',attrs={'id':'content-area'}).find('img')['src']
+        cover = soup.find('div',attrs={'id':'content-area'}).find('img', attrs={'src':re.compile('.*cover.*')})['src']
         self.cover_url=cover
+        self.log(self.cover_url)
 
         #Go to the main body
 
@@ -84,4 +84,3 @@ class HistoryToday(BasicNewsRecipe):
 
     def cleanup(self):
         self.browser.open('http://www.historytoday.com/logout')
-

recipes/in4_pl.recipe

@@ -1,5 +1,7 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
+from calibre.ebooks.BeautifulSoup import Comment
+
 class in4(BasicNewsRecipe):
     title          = u'IN4.pl'
     oldest_article = 7
@@ -8,14 +10,14 @@ class in4(BasicNewsRecipe):
     description   = u'Serwis Informacyjny - Aktualnosci, recenzje'
     category       = 'IT'
     language       = 'pl'
-    index='http://www.in4.pl/'
+    index = 'http://www.in4.pl/'
     #cover_url= 'http://www.in4.pl/recenzje/337/in4pl.jpg'
     no_stylesheets = True
     remove_empty_feeds = True
     preprocess_regexps = [(re.compile(ur'<a title="translate into.*?</a>', re.DOTALL), lambda match: '') ]
-    keep_only_tags=[dict(name='div', attrs={'class':'left_alone'})]
+    keep_only_tags = [dict(name='div', attrs={'class':'left_alone'})]
-    remove_tags_after=dict(name='img', attrs={'title':'komentarze'})
+    remove_tags_after = dict(name='img', attrs={'title':'komentarze'})
-    remove_tags=[dict(name='img', attrs={'title':'komentarze'})]
+    remove_tags = [dict(name='img', attrs={'title':'komentarze'})]
     feeds          = [(u'Wiadomo\u015bci', u'http://www.in4.pl/rss.php'), (u'Recenzje', u'http://www.in4.pl/rss_recenzje.php'), (u'Mini recenzje', u'http://www.in4.pl/rss_mini.php')]
 
     def append_page(self, soup, appendtag):
@@ -28,10 +30,13 @@ class in4(BasicNewsRecipe):
         while nexturl:
                     soup2 = self.index_to_soup(nexturl)
                     pagetext = soup2.find(id='news')
+                    comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+                    for comment in comments:
+                        comment.extract()
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
-                    nexturl=None
+                    nexturl = None
-                    tag=soup2.findAll('a')
+                    tag = soup2.findAll('a')
                     for z in tag:
                         if z.string and u'następna str' in z.string:
                             nexturl='http://www.in4.pl/' + z['href']

recipes/infra_pl.recipe

@@ -15,7 +15,7 @@ class INFRA(BasicNewsRecipe):
     remove_tags_before=dict(name='h2', attrs={'class':'contentheading'})
     remove_tags_after=dict(attrs={'class':'pagenav'})
     remove_tags=[dict(attrs={'class':'pagenav'})]
-    feeds          = [(u'Najnowsze wiadomo\u015bci', u'http://www.infra.org.pl/index.php?option=com_rd_rss&id=1')]
+    feeds          = [(u'Najnowsze wiadomo\u015bci', u'http://www.infra.org.pl/rss')]
 
     def preprocess_html(self, soup):
         for item in soup.findAll(style=True):

recipes/interia_fakty.recipe

@@ -1,23 +1,24 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = u'2010, Tomasz Dlugosz <tomek3d@gmail.com>'
+__copyright__ = u'2010-2013, Tomasz Dlugosz <tomek3d@gmail.com>'
 '''
 fakty.interia.pl
 '''
-
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class InteriaFakty(BasicNewsRecipe):
     title          = u'Interia.pl - Fakty'
     description    = u'Fakty ze strony interia.pl'
     language = 'pl'
-    oldest_article = 7
+    oldest_article = 1
     __author__ = u'Tomasz D\u0142ugosz'
-    simultaneous_downloads = 2
     no_stylesheets = True
     remove_javascript = True
-    max_articles_per_feed = 100
+    remove_empty_feeds= True
+    use_embedded_content = False
+    ignore_duplicate_articles = {'title', 'url'}
 
     feeds          = [(u'Kraj', u'http://kanaly.rss.interia.pl/kraj.xml'),
                       (u'\u015awiat', u'http://kanaly.rss.interia.pl/swiat.xml'),
@@ -26,14 +27,36 @@ class InteriaFakty(BasicNewsRecipe):
                       (u'Wywiady', u'http://kanaly.rss.interia.pl/wywiady.xml'),
                       (u'Ciekawostki', u'http://kanaly.rss.interia.pl/ciekawostki.xml')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':'article'})]
+    keep_only_tags = [
+        dict(name='h1'),
+        dict(name='div', attrs={'class': ['lead textContent', 'text textContent', 'source']})]
 
-    remove_tags = [
+    remove_tags = [dict(name='div', attrs={'class':['embed embedAd', 'REMOVE', 'boxHeader']})]
-        dict(name='div', attrs={'class':'box fontSizeSwitch'}),
+
-        dict(name='div', attrs={'class':'clear'}),
+    preprocess_regexps = [
-        dict(name='div', attrs={'class':'embed embedLeft articleEmbedArticleList articleEmbedArticleListTitle'}),
+        (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
-        dict(name='span', attrs={'class':'keywords'})]
+        [
+            (r'embed embed(Left|Right|Center) articleEmbed(Audio|Wideo articleEmbedVideo|ArticleFull|ArticleTitle|ArticleListTitle|AlbumHorizontal)">', lambda match: 'REMOVE">'),
+            (r'</div>    <div class="source">', lambda match: ''),
+            (r'<p><a href="http://forum.interia.pl.*?</a></p>', lambda match: '')
+        ]
+    ]
+
+    def get_article_url(self, article):
+        link = article.get('link', None)
+        if link and 'galerie' not in link and link.split('/')[-1]=="story01.htm":
+            link=link.split('/')[-2]
+            encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&',
+                '0D': '?', '0E': '-', '0H': ',', '0I': '_', '0N': '.com', '0L': 'http://'}
+            for k, v in encoding.iteritems():
+                link = link.replace(k, v)
+            return link
+
+    def print_version(self, url):
+        chunks = url.split(',')
+        return chunks[0] + '/podglad-wydruku'+ ',' + ','.join(chunks[1:])
 
     extra_css = '''
-            h2 { font-size: 1.2em; }
+        h1 { font-size:130% }
+        div.info { font-style:italic; font-size:70%}
     '''

recipes/interia_sport.recipe

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = u'2010, Tomasz Dlugosz <tomek3d@gmail.com>'
+__copyright__ = u'2010-2013, Tomasz Dlugosz <tomek3d@gmail.com>'
 '''
 sport.interia.pl
 '''
@@ -13,61 +13,51 @@ class InteriaSport(BasicNewsRecipe):
     title          = u'Interia.pl - Sport'
     description    = u'Sport ze strony interia.pl'
     language = 'pl'
-    oldest_article = 7
+    oldest_article = 1
     __author__ = u'Tomasz D\u0142ugosz'
-    simultaneous_downloads = 3
     no_stylesheets = True
     remove_javascript = True
-    max_articles_per_feed = 100
+    remove_empty_feeds= True
+    use_embedded_content = False
+    ignore_duplicate_articles = {'title', 'url'}
 
     feeds          = [(u'Wydarzenia sportowe', u'http://kanaly.rss.interia.pl/sport.xml'),
                       (u'Pi\u0142ka no\u017cna', u'http://kanaly.rss.interia.pl/pilka_nozna.xml'),
-                      (u'Siatk\xf3wka', u'http://kanaly.rss.interia.pl/siatkowka.xml'),
                       (u'Koszyk\xf3wka', u'http://kanaly.rss.interia.pl/koszykowka.xml'),
-                      (u'NBA', u'http://kanaly.rss.interia.pl/nba.xml'),
-                      (u'Kolarstwo', u'http://kanaly.rss.interia.pl/kolarstwo.xml'),
-                      (u'\u017bu\u017cel', u'http://kanaly.rss.interia.pl/zuzel.xml'),
                       (u'Tenis', u'http://kanaly.rss.interia.pl/tenis.xml')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':'article'})]
+    keep_only_tags = [
+        dict(name='h1'),
+        dict(name='div', attrs={'class': ['lead textContent', 'text textContent', 'source']})]
 
-    remove_tags = [dict(name='div', attrs={'class':'object gallery'}),
+    remove_tags = [dict(name='div', attrs={'class':['embed embedAd', 'REMOVE', 'boxHeader']})]
-                   dict(name='div', attrs={'class':'box fontSizeSwitch'})]
-
-    extra_css = '''
-        .articleDate {
-        font-size: 0.5em;
-        color: black;
-        }
-
-        .articleFoto {
-        display: block;
-        font-family: sans;
-        font-size: 0.5em;
-        text-indent: 0
-        color: black;
-        }
-
-        .articleText {
-        display: block;
-        margin-bottom: 1em;
-        margin-left: 0;
-        margin-right: 0;
-        margin-top: 1em
-        color: black;
-        }
-
-        .articleLead {
-        font-size: 1.2em;
-        }
-        '''
 
     preprocess_regexps = [
         (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
         [
             (r'<p><a href.*?</a></p>', lambda match: ''),
-           # FIXME
+            (r'<p>(<i>)?<b>(ZOBACZ|CZYTAJ) T.*?</div>', lambda match: '</div>'),
-           #(r'(<div id="newsAddContent">)(.*?)(<a href=".*">)(.*?)(</a>)', lambda match: '\1\2\4'),
+            (r'embed embed(Left|Right|Center) articleEmbed(Audio|Wideo articleEmbedVideo|ArticleFull|ArticleTitle|ArticleListTitle|AlbumHorizontal)">', lambda match: 'REMOVE">'),
-            (r'<p>(<i>)?<b>(ZOBACZ|CZYTAJ) T.*?</div>', lambda match: '</div>')
+            (r'</div>    <div class="source">', lambda match: ''),
+            (r'<p><a href="http://forum.interia.pl.*?</a></p>', lambda match: '')
         ]
     ]
+
+    def get_article_url(self, article):
+        link = article.get('link', None)
+        if link and 'galerie' not in link and link.split('/')[-1]=="story01.htm":
+            link=link.split('/')[-2]
+            encoding = {'0B': '.', '0C': '/', '0A': '0', '0F': '=', '0G': '&',
+                '0D': '?', '0E': '-', '0H': ',', '0I': '_', '0N': '.com', '0L': 'http://'}
+            for k, v in encoding.iteritems():
+                link = link.replace(k, v)
+            return link
+
+    def print_version(self, url):
+        chunks = url.split(',')
+        return chunks[0] + '/podglad-wydruku'+ ',' + ','.join(chunks[1:])
+
+    extra_css = '''
+        h1 { font-size:130% }
+        div.info { font-style:italic; font-size:70%}
+    '''

recipes/irish_times.recipe

@@ -1,65 +1,62 @@
 __license__  = 'GPL v3'
-__copyright__ = "2008, Derry FitzGerald. 2009 Modified by Ray Kinsella and David O'Callaghan, 2011 Modified by Phil Burns"
+__copyright__ = "2008, Derry FitzGerald. 2009 Modified by Ray Kinsella and David O'Callaghan, 2011 Modified by Phil Burns, 2013 Tom Scholl"
 '''
 irishtimes.com
 '''
-import re
+import urlparse, re
 
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ptempfile import PersistentTemporaryFile
+
 
 class IrishTimes(BasicNewsRecipe):
     title          = u'The Irish Times'
-    encoding  = 'ISO-8859-15'
+    __author__    = "Derry FitzGerald, Ray Kinsella, David O'Callaghan and Phil Burns, Tom Scholl"
-    __author__    = "Derry FitzGerald, Ray Kinsella, David O'Callaghan and Phil Burns"
     language = 'en_IE'
-    timefmt = ' (%A, %B %d, %Y)'
 
+    masthead_url = 'http://www.irishtimes.com/assets/images/generic/website/logo_theirishtimes.png'
 
+    encoding = 'utf-8'
     oldest_article = 1.0
     max_articles_per_feed = 100
+    remove_empty_feeds = True
     no_stylesheets = True
-    simultaneous_downloads= 5
+    temp_files = []
-
+    articles_are_obfuscated = True
-    r = re.compile('.*(?P<url>http:\/\/(www.irishtimes.com)|(rss.feedsportal.com\/c)\/.*\.html?).*')
-    remove_tags    = [dict(name='div', attrs={'class':'footer'})]
-    extra_css      = 'p, div { margin: 0pt; border: 0pt; text-indent: 0.5em } .headline {font-size: large;} \n .fact { padding-top: 10pt  }'
 
     feeds          = [
-                      ('Frontpage', 'http://www.irishtimes.com/feeds/rss/newspaper/index.rss'),
+                      ('News', 'http://www.irishtimes.com/cmlink/the-irish-times-news-1.1319192'),
-                      ('Ireland', 'http://www.irishtimes.com/feeds/rss/newspaper/ireland.rss'),
+                      ('World', 'http://www.irishtimes.com/cmlink/irishtimesworldfeed-1.1321046'),
-                      ('World', 'http://www.irishtimes.com/feeds/rss/newspaper/world.rss'),
+                      ('Politics', 'http://www.irishtimes.com/cmlink/irish-times-politics-rss-1.1315953'),
-                      ('Finance', 'http://www.irishtimes.com/feeds/rss/newspaper/finance.rss'),
+                      ('Business', 'http://www.irishtimes.com/cmlink/the-irish-times-business-1.1319195'),
-                      ('Features', 'http://www.irishtimes.com/feeds/rss/newspaper/features.rss'),
+                      ('Culture', 'http://www.irishtimes.com/cmlink/the-irish-times-culture-1.1319213'),
-                      ('Sport', 'http://www.irishtimes.com/feeds/rss/newspaper/sport.rss'),
+                      ('Sport', 'http://www.irishtimes.com/cmlink/the-irish-times-sport-1.1319194'),
-                      ('Opinion', 'http://www.irishtimes.com/feeds/rss/newspaper/opinion.rss'),
+                      ('Debate', 'http://www.irishtimes.com/cmlink/debate-1.1319211'),
-                      ('Letters', 'http://www.irishtimes.com/feeds/rss/newspaper/letters.rss'),
+                      ('Life & Style', 'http://www.irishtimes.com/cmlink/the-irish-times-life-style-1.1319214'),
-                      ('Magazine', 'http://www.irishtimes.com/feeds/rss/newspaper/magazine.rss'),
-                      ('Health', 'http://www.irishtimes.com/feeds/rss/newspaper/health.rss'),
-                      ('Education & Parenting', 'http://www.irishtimes.com/feeds/rss/newspaper/education.rss'),
-                      ('Motors', 'http://www.irishtimes.com/feeds/rss/newspaper/motors.rss'),
-                      ('An Teanga Bheo', 'http://www.irishtimes.com/feeds/rss/newspaper/anteangabheo.rss'),
-                      ('Commercial Property', 'http://www.irishtimes.com/feeds/rss/newspaper/commercialproperty.rss'),
-                      ('Science Today', 'http://www.irishtimes.com/feeds/rss/newspaper/sciencetoday.rss'),
-                      ('Property', 'http://www.irishtimes.com/feeds/rss/newspaper/property.rss'),
-                      ('The Tickets', 'http://www.irishtimes.com/feeds/rss/newspaper/theticket.rss'),
-                      ('Weekend', 'http://www.irishtimes.com/feeds/rss/newspaper/weekend.rss'),
-                      ('News features', 'http://www.irishtimes.com/feeds/rss/newspaper/newsfeatures.rss'),
-                      ('Obituaries', 'http://www.irishtimes.com/feeds/rss/newspaper/obituaries.rss'),
                     ]
 
 
-    def print_version(self, url):
+    def get_obfuscated_article(self, url):
-        if url.count('rss.feedsportal.com'):
+        # Insert a pic from the original url, but use content from the print url
-            #u = url.replace('0Bhtml/story01.htm','_pf0Bhtml/story01.htm')
+        pic = None
-            u = url.find('irishtimes')
+        pics = self.index_to_soup(url)
-            u = 'http://www.irishtimes.com' + url[u + 12:]
+        div = pics.find('div', {'class' : re.compile('image-carousel')})
-            u = u.replace('0C', '/')
+        if div:
-            u = u.replace('A', '')
+            pic = div.img
-            u = u.replace('0Bhtml/story01.htm', '_pf.html')
+            if pic:
-        else:
+                try:
-            u = url.replace('.html','_pf.html')
+                    pic['src'] = urlparse.urljoin(url, pic['src'])
-        return u
+                    pic.extract()
+                except:
+                    pic = None
+
+        content = self.index_to_soup(url + '?mode=print&ot=example.AjaxPageLayout.ot')
+        if pic:
+            content.p.insert(0, pic)
+
+        self.temp_files.append(PersistentTemporaryFile('_fa.html'))
+        self.temp_files[-1].write(content.prettify())
+        self.temp_files[-1].close()
+        return self.temp_files[-1].name
 
-    def get_article_url(self, article):
-        return article.link
 

recipes/kath_net.recipe

@@ -11,12 +11,10 @@ class AdvancedUserRecipe1295262156(BasicNewsRecipe):
     auto_cleanup = True
     encoding='iso-8859-1'
 
-
     feeds          = [(u'kath.net', u'http://www.kath.net/2005/xml/index.xml')]
 
-
     def print_version(self, url):
-        return url+"&print=yes"
+        return url+"/print/yes"
 
     extra_css = 'td.textb {font-size: medium;}'
 

recipes/km_blog.recipe

@@ -0,0 +1,36 @@
+
+__license__ = 'GPL v3'
+__author__ = 'teepel <teepel44@gmail.com>, Artur Stachecki <artur.stachecki@gmail.com>'
+
+'''
+korwin-mikke.pl/blog
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class km_blog(BasicNewsRecipe):
+    title          = u'Korwin-Mikke Blog'
+    __author__ = 'teepel <teepel44@gmail.com>'
+    language       = 'pl'
+    description ='Wiadomości z bloga korwin-mikke.pl/blog'
+    INDEX='http://korwin-mikke.pl/blog'
+    remove_empty_feeds= True
+    oldest_article = 7
+    max_articles_per_feed = 100
+    remove_javascript=True
+    no_stylesheets=True
+    remove_empty_feeds = True
+
+    feeds          = [(u'blog', u'http://korwin-mikke.pl/blog/rss')]
+
+    keep_only_tags =[]
+        #this line should show title of the article, but it doesnt work
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'posts view'}))
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'text'}))
+    keep_only_tags.append(dict(name = 'h1'))
+
+    remove_tags =[]
+    remove_tags.append(dict(name = 'p', attrs = {'class' : 'float_right'}))
+    remove_tags.append(dict(name = 'p', attrs = {'class' : 'date'}))
+
+    remove_tags_after=[(dict(name = 'div', attrs = {'class': 'text'}))]

recipes/kosmonauta_pl.recipe

@@ -12,6 +12,8 @@ class Kosmonauta(BasicNewsRecipe):
     INDEX = 'http://www.kosmonauta.net'
     oldest_article = 7
     no_stylesheets = True
+    remove_javascript = True
+    remove_attributes = ['style']
     max_articles_per_feed = 100
     keep_only_tags = [dict(name='div', attrs={'class':'item-page'})]
     remove_tags = [dict(attrs={'class':['article-tools clearfix', 'cedtag', 'nav clearfix', 'jwDisqusForm']})]

recipes/kurier_galicyjski.recipe

@@ -1,5 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup as bs
+from calibre.ebooks.BeautifulSoup import BeautifulSoup as bs, Comment
+
 class KurierGalicyjski(BasicNewsRecipe):
     title          = u'Kurier Galicyjski'
     __author__        = 'fenuks'
@@ -42,6 +43,9 @@ class KurierGalicyjski(BasicNewsRecipe):
                 r.extract()
             for r in appendtag.findAll(attrs={'style':'border-top-width: thin; border-top-style: dashed; border-top-color: #CCC; border-bottom-width: thin; border-bottom-style: dashed; border-bottom-color: #CCC; padding-top:5px; padding-bottom:5px; text-align:right; margin-top:10px; height:20px;'}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
 
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)

recipes/kurier_poranny.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class KurierPoranny(BasicNewsRecipe):
     title          = u'Kurier Poranny'
@@ -73,6 +74,11 @@ class KurierPoranny(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
             
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/ledevoir.recipe

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __author__    = 'Lorenzo Vigentini and Olivier Daigle'
 __copyright__ = '2012, Lorenzo Vigentini <l.vigentini at gmail.com>, Olivier Daigle <odaigle _at nuvucameras __dot__ com>'
 __version__     = 'v1.01'
-__date__        = '22, December 2012'
+__date__        = '17, March 2013'
 __description__   = 'Canadian Paper '
 
 '''
@@ -12,6 +12,7 @@ http://www.ledevoir.com/
 import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.utils.magick import Image
 
 class ledevoir(BasicNewsRecipe):
     author        = 'Lorenzo Vigentini'
@@ -28,10 +29,14 @@ class ledevoir(BasicNewsRecipe):
 
     oldest_article = 1
     max_articles_per_feed = 200
+    min_articles_per_feed = 0
     use_embedded_content  = False
     recursion             = 10
     needs_subscription    = 'optional'
 
+    compress_news_images = True
+    compress_news_images_auto_size = 4
+
     filterDuplicates = False
     url_list = []
 
@@ -66,16 +71,16 @@ class ledevoir(BasicNewsRecipe):
 
     feeds          = [
                        (u'A la une', 'http://www.ledevoir.com/rss/manchettes.xml'),
-#                       (u'Édition complete', 'http://feeds2.feedburner.com/fluxdudevoir'),
+                       (u'Édition complete', 'http://feeds2.feedburner.com/fluxdudevoir'),
-#                       (u'Opinions', 'http://www.ledevoir.com/rss/opinions.xml'),
+                       (u'Opinions', 'http://www.ledevoir.com/rss/opinions.xml'),
-#                       (u'Chroniques', 'http://www.ledevoir.com/rss/chroniques.xml'),
+                       (u'Chroniques', 'http://www.ledevoir.com/rss/chroniques.xml'),
-#                       (u'Politique', 'http://www.ledevoir.com/rss/section/politique.xml?id=51'),
+                       (u'Politique', 'http://www.ledevoir.com/rss/section/politique.xml?id=51'),
-#                       (u'International', 'http://www.ledevoir.com/rss/section/international.xml?id=76'),
+                       (u'International', 'http://www.ledevoir.com/rss/section/international.xml?id=76'),
-#                       (u'Culture', 'http://www.ledevoir.com/rss/section/culture.xml?id=48'),
+                       (u'Culture', 'http://www.ledevoir.com/rss/section/culture.xml?id=48'),
-#                       (u'Environnement', 'http://www.ledevoir.com/rss/section/environnement.xml?id=78'),
+                       (u'Environnement', 'http://www.ledevoir.com/rss/section/environnement.xml?id=78'),
-#                       (u'Societe', 'http://www.ledevoir.com/rss/section/societe.xml?id=52'),
+                       (u'Societe', 'http://www.ledevoir.com/rss/section/societe.xml?id=52'),
-#                       (u'Economie', 'http://www.ledevoir.com/rss/section/economie.xml?id=49'),
+                       (u'Economie', 'http://www.ledevoir.com/rss/section/economie.xml?id=49'),
-#                       (u'Sports', 'http://www.ledevoir.com/rss/section/sports.xml?id=85'),
+                       (u'Sports', 'http://www.ledevoir.com/rss/section/sports.xml?id=85'),
                        (u'Art de vivre', 'http://www.ledevoir.com/rss/section/art-de-vivre.xml?id=50')
                      ]
 
@@ -113,3 +118,23 @@ class ledevoir(BasicNewsRecipe):
         self.url_list.append(url)
         return url
 
+'''
+    def postprocess_html(self, soup, first):
+        #process all the images. assumes that the new html has the correct path
+        if first == 0:
+          return soup
+
+        for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
+             iurl = tag['src']
+             img = Image()
+             img.open(iurl)
+        #     width, height = img.size
+        #     print 'img is: ', iurl, 'width is: ', width, 'height is: ', height 
+             if img < 0:
+                raise RuntimeError('Out of memory')
+             img.set_compression_quality(30)
+             img.save(iurl)
+        return soup
+'''
+        
+           

recipes/linux_journal.recipe

@@ -1,4 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class LinuxJournal(BasicNewsRecipe):
     title          = u'Linux Journal'
@@ -25,6 +26,9 @@ class LinuxJournal(BasicNewsRecipe):
             soup2 = self.index_to_soup('http://www.linuxjournal.com'+ nexturl)
             pagetext = soup2.find(attrs={'class':'node-inner'}).find(attrs={'class':'content'})
             next = appendtag.find('li', attrs={'class':'pager-next'})
+            comments = pagetext.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
             pos = len(appendtag.contents)
             appendtag.insert(pos, pagetext)
         tag = appendtag.find('div', attrs={'class':'links'})

recipes/midday.recipe

@@ -1,13 +0,0 @@
-from calibre.web.feeds.news import CalibrePeriodical
-
-class MiDDay(CalibrePeriodical):
-
-    title = 'MiDDay'
-    calibre_periodicals_slug = 'midday'
-
-    description = '''Get your dose of the latest news, views and fun - from the
-        world of politics, sports and Bollywood to the cartoons, comics and games of
-        the entertainment section - India’s leading tabloid has it all. To subscribe
-        visit <a href="http://news.calibre-ebook.com/periodical/midday">calibre
-        Periodicals</a>.'''
-    language = 'en_IN'

recipes/niebezpiecznik.recipe

@@ -9,6 +9,7 @@ class Niebezpiecznik_pl(BasicNewsRecipe):
     oldest_article = 8
     max_articles_per_feed = 100
     no_stylesheets = True
+    remove_empty_feeds = True
     cover_url = u'http://userlogos.org/files/logos/Karmody/niebezpiecznik_01.png'
     remove_tags = [dict(name='div', attrs={'class':['sociable']}), dict(name='h4'), dict(attrs={'class':'similar-posts'})]
     keep_only_tags = [dict(name='div', attrs={'class':['title', 'entry']})]

recipes/nto.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
 
 class NTO(BasicNewsRecipe):
     title          = u'Nowa Trybuna Opolska'
@@ -58,6 +59,10 @@ class NTO(BasicNewsRecipe):
                     pos = len(appendtag.contents)
                     appendtag.insert(pos, pagetext)
             
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/oclab_pl.recipe

@@ -1,4 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
+
 class OCLab(BasicNewsRecipe):
     title          = u'OCLab.pl'
     oldest_article = 7
@@ -26,6 +28,10 @@ class OCLab(BasicNewsRecipe):
                appendtag.insert(pos, pagetext)
             for r in appendtag.findAll(attrs={'class':'post-nav-bottom-list'}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
+                
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)
         return soup

recipes/optyczne_pl.recipe

@@ -0,0 +1,41 @@
+#!/usr/bin/env  python
+__license__ = 'GPL v3'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class OptyczneRecipe(BasicNewsRecipe):
+    __author__ = u'Artur Stachecki <artur.stachecki@gmail.com>'
+    language = 'pl'
+
+    title = u'optyczne.pl'
+    category = u'News'
+    description = u'Najlepsze testy obiektywów, testy aparatów cyfrowych i testy lornetek w sieci!'
+    cover_url=''
+    remove_empty_feeds= True
+    no_stylesheets=True
+    oldest_article = 7
+    max_articles_per_feed = 100000
+    recursions = 0
+
+    no_stylesheets = True
+    remove_javascript = True
+
+    keep_only_tags =[]
+    keep_only_tags.append(dict(name = 'div', attrs = {'class' : 'news'}))
+
+    remove_tags =[]
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'center'}))
+    remove_tags.append(dict(name = 'div', attrs = {'class' : 'news_foto'}))
+    remove_tags.append(dict(name = 'div', attrs = {'align' : 'right'}))
+
+    extra_css = '''
+                    body {font-family: Arial,Helvetica,sans-serif;}
+                    h1{text-align: left;}
+                    h2{font-size: medium; font-weight: bold;}
+                    p.lead {font-weight: bold; text-align: left;}
+                    .authordate {font-size: small; color: #696969;}
+                    .fot{font-size: x-small; color: #666666;}
+                    '''
+    feeds          = [
+                            ('Aktualnosci', 'http://www.optyczne.pl/rss.xml'),
+                           ]

recipes/overclock_pl.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+
 class Overclock_pl(BasicNewsRecipe):
     title          = u'Overclock.pl'
     oldest_article = 7

recipes/pc_foster.recipe

@@ -1,4 +1,8 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
+
+#currently recipe is not working
+
 class PC_Foster(BasicNewsRecipe):
     title          = u'PC Foster'
     oldest_article = 7
@@ -29,6 +33,9 @@ class PC_Foster(BasicNewsRecipe):
                 appendtag.insert(pos, pagetext)
             for r in appendtag.findAll(attrs={'class':'review_content double'}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
 
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)

recipes/psych.recipe

@@ -67,6 +67,7 @@ class PsychologyToday(BasicNewsRecipe):
             title = title + u' (%s)'%author
             article_page= self.index_to_soup('http://www.psychologytoday.com'+post.find('a', href=True)['href'])
             print_page=article_page.find('li', attrs={'class':'print_html first'})
+            if print_page is not None:
                 url='http://www.psychologytoday.com'+print_page.find('a',href=True)['href']
                 desc = self.tag_to_string(post.find('div', attrs={'class':'collection-node-description'})).strip()
                 self.log('Found article:', title)

recipes/pure_pc.recipe

@@ -1,4 +1,6 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Comment
+
 class PurePC(BasicNewsRecipe):
     title          = u'PurePC'
     oldest_article = 7
@@ -27,6 +29,9 @@ class PurePC(BasicNewsRecipe):
                 appendtag.insert(pos, pagetext)
             for r in appendtag.findAll(attrs={'class':['PageMenuList', 'pager', 'fivestar-widget']}):
                 r.extract()
+            comments = appendtag.findAll(text=lambda text:isinstance(text, Comment))
+            for comment in comments:
+                comment.extract()
 
     def preprocess_html(self, soup):
         self.append_page(soup, soup.body)

recipes/rte.recipe

@@ -6,10 +6,12 @@ class RTE(BasicNewsRecipe):
     max_articles_per_feed = 100
     __author__ = u'Robin Phillips'
     language = 'en_IE'
+    auto_cleanup=True
+    auto_cleanup_keep = '//figure[@class="photography gal642 single"]'
 
     remove_tags = [dict(attrs={'class':['topAd','botad','previousNextItem','headline','footerLinks','footernav']})]
 
     feeds          = [(u'News', u'http://www.rte.ie/rss/news.xml'), (u'Sport', u'http://www.rte.ie/rss/sport.xml'), (u'Soccer', u'http://www.rte.ie/rss/soccer.xml'), (u'GAA', u'http://www.rte.ie/rss/gaa.xml'), (u'Rugby', u'http://www.rte.ie/rss/rugby.xml'), (u'Racing', u'http://www.rte.ie/rss/racing.xml'), (u'Business', u'http://www.rte.ie/rss/business.xml'), (u'Entertainment', u'http://www.rte.ie/rss/entertainment.xml')]
 
-    def print_version(self, url):
+    #def print_version(self, url):
-        return url.replace('http://www', 'http://m')
+        #return url.replace('http://www', 'http://m')

recipes/trystero.recipe

@@ -0,0 +1,26 @@
+#!/usr/bin/env  python
+
+__license__ = 'GPL v3'
+__copyright__ = u'2013, Tomasz Dlugosz <tomek3d@gmail.com>'
+
+'''
+trystero.pl
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class trystero(BasicNewsRecipe):
+    title          = 'Trystero'
+    __author__ = u'Tomasz D\u0142ugosz'
+    language       = 'pl'
+    description =u'Trystero.pl jest niezależnym blogiem finansowym. Publikowane na nim teksty dotyczą rynku kapitałowego, ekonomii, gospodarki i życia społecznego – w takiej mniej więcej kolejności.'
+    oldest_article = 7
+    remove_javascript=True
+    no_stylesheets=True
+
+    feeds          = [(u'Newsy', u'http://www.trystero.pl/feed')]
+
+    keep_only_tags = [
+        dict(name='h1'),
+        dict(name='div', attrs={'class': ['post-content']})]
+

recipes/ubuntu_pomoc_org.recipe

@@ -1,5 +1,6 @@
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+
 class UbuntuPomoc(BasicNewsRecipe):
     title          = u'Ubuntu-pomoc.org'
     __author__        = 'fenuks'
@@ -15,8 +16,8 @@ class UbuntuPomoc(BasicNewsRecipe):
     remove_empty_feeds = True
     use_embedded_content = False
     remove_attrs = ['style']
-    keep_only_tags = [dict(attrs={'class':'post'})]
+    keep_only_tags = [dict(name='article')]
-    remove_tags_after = dict(attrs={'class':'underEntry'})
+    #remove_tags_after = dict(attrs={'class':'underEntry'})
-    remove_tags = [dict(attrs={'class':['underPostTitle', 'yarpp-related', 'underEntry', 'social', 'tags', 'commentlist', 'youtube_sc']}), dict(id=['wp_rp_first', 'commentReply'])]
+    remove_tags = [dict(attrs={'class':['yarpp-related', 'youtube_sc', 'share']}), dict(name='footer')]
     feeds          = [(u'Ca\u0142o\u015b\u0107', u'http://feeds.feedburner.com/Ubuntu-Pomoc'),
-                     (u'Gry', u'http://feeds.feedburner.com/GryUbuntu-pomoc')]
+                    ]

recipes/websecurity_pl.recipe

@@ -0,0 +1,28 @@
+__license__ = 'GPL v3'
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class WebSecurity(BasicNewsRecipe):
+    title          = u'WebSecurity'
+    __author__        = 'fenuks'
+    description   = u'WebSecurity.pl to największy w Polsce portal o bezpieczeństwie sieciowym.'
+    category       = ''
+    #publication_type = ''
+    language       = 'pl'
+    #encoding = ''
+    #extra_css = ''
+    cover_url = 'http://websecurity.pl/images/websecurity-logo.png'
+    masthead_url = ''
+    use_embedded_content = False
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_empty_feeds = True
+    remove_javascript = True
+    remove_attributes = ['style', 'font']
+    ignore_duplicate_articles = {'title', 'url'}
+
+    keep_only_tags = [dict(attrs={'class':'article single'}), dict(id='content')]
+    remove_tags = [dict(attrs={'class':['sociable', 'no-comments']})]
+    remove_tags_after = dict(attrs={'class':'sociable'})
+    feeds          = [(u'Wszystkie', u'http://websecurity.pl/feed/'), (u'Aktualno\u015bci', u'http://websecurity.pl/aktualnosci/feed/'), (u'Artyku\u0142y', u'http://websecurity.pl/artykuly/feed/'), (u'Blogosfera', u'http://websecurity.pl/blogosfera/wpisy/feed/')]
+ 

recipes/wirtualnemedia_pl.recipe

@@ -8,6 +8,7 @@ class WirtualneMedia(BasicNewsRecipe):
     use_embedded_content = False
     remove_empty_feeds = True
     __author__        = 'fenuks'
+    extra_css = '.thumbnail {float:left; max-width:150px; margin-right:5px;}'
     description   = u'Portal o mediach, reklamie, internecie, PR, telekomunikacji - nr 1 w Polsce - WirtualneMedia.pl - wiadomości z pierwszej ręki.'
     category       = 'internet'
     language       = 'pl'

recipes/zaufana_trzecia_strona.recipe

@@ -1,5 +1,6 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
 from calibre.web.feeds.news import BasicNewsRecipe
+
 class ZTS(BasicNewsRecipe):
     title          = u'Zaufana Trzecia Strona'
     __author__        = 'fenuks'
@@ -7,6 +8,7 @@ class ZTS(BasicNewsRecipe):
     category       = 'IT, security'
     language       = 'pl'
     cover_url = 'http://www.zaufanatrzeciastrona.pl/wp-content/uploads/2012/08/z3s_h100.png'
+    extra_css = '.thumbnail {float: left; margin-right:5px;}'
     oldest_article = 7
     max_articles_per_feed = 100
     no_stylesheets = True

setup/iso_639/es.po

@@ -9,14 +9,14 @@ msgstr ""
 "Project-Id-Version: calibre\n"
 "Report-Msgid-Bugs-To: FULL NAME <EMAIL@ADDRESS>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2013-02-26 12:21+0000\n"
+"PO-Revision-Date: 2013-03-19 21:03+0000\n"
-"Last-Translator: Miguel Angel del Olmo <silinio45@gmail.com>\n"
+"Last-Translator: Jorge Luis Granda <costeelation@hotmail.com>\n"
 "Language-Team: Español; Castellano <>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2013-02-27 04:37+0000\n"
+"X-Launchpad-Export-Date: 2013-03-20 04:42+0000\n"
-"X-Generator: Launchpad (build 16506)\n"
+"X-Generator: Launchpad (build 16532)\n"
 
 #. name for aaa
 msgid "Ghotuo"
@@ -9808,7 +9808,7 @@ msgstr "Huave; San Mateo Del Mar"
 
 #. name for huw
 msgid "Hukumina"
-msgstr ""
+msgstr "Hukumina"
 
 #. name for hux
 msgid "Huitoto; Nüpode"
@@ -9816,15 +9816,15 @@ msgstr "Huitoto; Nipode"
 
 #. name for huy
 msgid "Hulaulá"
-msgstr ""
+msgstr "Hulaulá"
 
 #. name for huz
 msgid "Hunzib"
-msgstr ""
+msgstr "Hunzib"
 
 #. name for hvc
 msgid "Haitian Vodoun Culture Language"
-msgstr ""
+msgstr "Idioma de la cultura haitiana vodoun"
 
 #. name for hve
 msgid "Huave; San Dionisio Del Mar"
@@ -9832,11 +9832,11 @@ msgstr "Huave; San Dionisio Del Mar"
 
 #. name for hvk
 msgid "Haveke"
-msgstr ""
+msgstr "Haveke"
 
 #. name for hvn
 msgid "Sabu"
-msgstr ""
+msgstr "Sabu"
 
 #. name for hvv
 msgid "Huave; Santa María Del Mar"
@@ -9844,7 +9844,7 @@ msgstr "Huave; Santa María Del Mar"
 
 #. name for hwa
 msgid "Wané"
-msgstr ""
+msgstr "Wané"
 
 #. name for hwc
 msgid "Creole English; Hawai'i"
@@ -9856,7 +9856,7 @@ msgstr ""
 
 #. name for hya
 msgid "Hya"
-msgstr ""
+msgstr "Hya"
 
 #. name for hye
 msgid "Armenian"
@@ -9864,7 +9864,7 @@ msgstr "Armenio"
 
 #. name for iai
 msgid "Iaai"
-msgstr ""
+msgstr "Iaai"
 
 #. name for ian
 msgid "Iatmul"
@@ -30664,31 +30664,31 @@ msgstr ""
 
 #. name for zpu
 msgid "Zapotec; Yalálag"
-msgstr ""
+msgstr "Zapotec; Yalálag"
 
 #. name for zpv
 msgid "Zapotec; Chichicapan"
-msgstr ""
+msgstr "Zapotec; Chichicapan"
 
 #. name for zpw
 msgid "Zapotec; Zaniza"
-msgstr ""
+msgstr "Zapotec; Zaniza"
 
 #. name for zpx
 msgid "Zapotec; San Baltazar Loxicha"
-msgstr ""
+msgstr "Zapotec; San Baltazar Loxicha"
 
 #. name for zpy
 msgid "Zapotec; Mazaltepec"
-msgstr ""
+msgstr "Zapotec; Mazaltepec"
 
 #. name for zpz
 msgid "Zapotec; Texmelucan"
-msgstr ""
+msgstr "Zapotec; Texmelucan"
 
 #. name for zqe
 msgid "Zhuang; Qiubei"
-msgstr ""
+msgstr "Zhuang; Qiubei"
 
 #. name for zra
 msgid "Kara (Korea)"
@@ -30732,7 +30732,7 @@ msgstr "Malayo estándar"
 
 #. name for zsr
 msgid "Zapotec; Southern Rincon"
-msgstr ""
+msgstr "Zapotec; Southern Rincon"
 
 #. name for zsu
 msgid "Sukurum"
@@ -30760,11 +30760,11 @@ msgstr "Zapoteco de Santa Catarina Albarradas"
 
 #. name for ztp
 msgid "Zapotec; Loxicha"
-msgstr ""
+msgstr "Zapotec; Loxicha"
 
 #. name for ztq
 msgid "Zapotec; Quioquitani-Quierí"
-msgstr ""
+msgstr "Zapotec; Quioquitani-Quierí"
 
 #. name for zts
 msgid "Zapotec; Tilquiapan"

setup/iso_639/nl.po

@@ -12,14 +12,14 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2013-02-04 07:01+0000\n"
+"PO-Revision-Date: 2013-03-16 14:32+0000\n"
 "Last-Translator: drMerry <Unknown>\n"
 "Language-Team: Dutch <vertaling@vrijschrift.org>\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2013-02-05 04:44+0000\n"
+"X-Launchpad-Export-Date: 2013-03-17 04:58+0000\n"
-"X-Generator: Launchpad (build 16468)\n"
+"X-Generator: Launchpad (build 16532)\n"
 "Language: nl\n"
 
 #. name for aaa
@@ -340,7 +340,7 @@ msgstr "Adi"
 
 #. name for adj
 msgid "Adioukrou"
-msgstr ""
+msgstr "Adiokrou"
 
 #. name for adl
 msgid "Galo"
@@ -352,11 +352,11 @@ msgstr "Adang"
 
 #. name for ado
 msgid "Abu"
-msgstr ""
+msgstr "Abu"
 
 #. name for adp
 msgid "Adap"
-msgstr ""
+msgstr "Adap"
 
 #. name for adq
 msgid "Adangbe"
@@ -372,7 +372,7 @@ msgstr "Adamorobe gebarentaal"
 
 #. name for adt
 msgid "Adnyamathanha"
-msgstr ""
+msgstr "Adnyamathanha"
 
 #. name for adu
 msgid "Aduge"
@@ -392,7 +392,7 @@ msgstr "Adyghe"
 
 #. name for adz
 msgid "Adzera"
-msgstr ""
+msgstr "Adzera"
 
 #. name for aea
 msgid "Areba"
@@ -416,11 +416,11 @@ msgstr "Pashai; noordoost"
 
 #. name for aek
 msgid "Haeke"
-msgstr ""
+msgstr "Haeke"
 
 #. name for ael
 msgid "Ambele"
-msgstr ""
+msgstr "Ambele"
 
 #. name for aem
 msgid "Arem"
@@ -432,7 +432,7 @@ msgstr "Armeense gebarentaal"
 
 #. name for aeq
 msgid "Aer"
-msgstr ""
+msgstr "Aer"
 
 #. name for aer
 msgid "Arrernte; Eastern"
@@ -440,7 +440,7 @@ msgstr "Arrernte; oostelijk"
 
 #. name for aes
 msgid "Alsea"
-msgstr ""
+msgstr "Alsea"
 
 #. name for aeu
 msgid "Akeu"
@@ -468,7 +468,7 @@ msgstr "Andai"
 
 #. name for afe
 msgid "Putukwam"
-msgstr ""
+msgstr "Putukwam"
 
 #. name for afg
 msgid "Afghan Sign Language"

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 23)
+numeric_version = (0, 9, 24)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -1666,15 +1666,6 @@ class XinXiiStore(StoreBase):
     headquarters = 'DE'
     formats = ['EPUB', 'PDF']
 
-class StoreZixoStore(StoreBase):
-    name = 'Zixo'
-    author = u'Tomasz Długosz'
-    description = u'Księgarnia z ebookami oraz książkami audio. Aby otwierać książki w formacie Zixo należy zainstalować program dostępny na stronie księgarni. Umożliwia on m.in. dodawanie zakładek i dostosowywanie rozmiaru czcionki.'
-    actual_plugin = 'calibre.gui2.store.stores.zixo_plugin:ZixoStore'
-
-    headquarters = 'PL'
-    formats = ['PDF, ZIXO']
-
 plugins += [
     StoreArchiveOrgStore,
     StoreAmazonKindleStore,
@@ -1724,8 +1715,7 @@ plugins += [
     StoreWeightlessBooksStore,
     StoreWHSmithUKStore,
     StoreWoblinkStore,
-    XinXiiStore,
+    XinXiiStore
-    StoreZixoStore
 ]
 
 # }}}

src/calibre/db/backend.py

@@ -23,9 +23,11 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.icu import sort_key
 from calibre.utils.config import to_json, from_json, prefs, tweaks
 from calibre.utils.date import utcfromtimestamp, parse_date
-from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file)
+from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename,
+                                     WindowsAtomicFolderMove)
+from calibre.utils.recycle_bin import delete_tree
 from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
-        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable,
+        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable,
         CompositeTable, LanguagesTable)
 # }}}
 
@@ -672,7 +674,7 @@ class DB(object):
                         if col == 'cover' else col)
             if not metadata['column']:
                 metadata['column'] = col
-            tables[col] = OneToOneTable(col, metadata)
+            tables[col] = (PathTable if col == 'path' else OneToOneTable)(col, metadata)
 
         for col in ('series', 'publisher', 'rating'):
             tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
@@ -778,6 +780,44 @@ class DB(object):
             self.user_version = 1
     # }}}
 
+    def normpath(self, path):
+        path = os.path.abspath(os.path.realpath(path))
+        if not self.is_case_sensitive:
+            path = os.path.normcase(path).lower()
+        return path
+
+    def rmtree(self, path, permanent=False):
+        if not self.normpath(self.library_path).startswith(self.normpath(path)):
+            delete_tree(path, permanent=permanent)
+
+    def construct_path_name(self, book_id, title, author):
+        '''
+        Construct the directory name for this book based on its metadata.
+        '''
+        author = ascii_filename(author
+                    )[:self.PATH_LIMIT].decode('ascii', 'replace')
+        title  = ascii_filename(title
+                    )[:self.PATH_LIMIT].decode('ascii', 'replace')
+        while author[-1] in (' ', '.'):
+            author = author[:-1]
+        if not author:
+            author = ascii_filename(_('Unknown')).decode(
+                    'ascii', 'replace')
+        return '%s/%s (%d)'%(author, title, book_id)
+
+    def construct_file_name(self, book_id, title, author):
+        '''
+        Construct the file name for this book based on its metadata.
+        '''
+        author = ascii_filename(author
+                    )[:self.PATH_LIMIT].decode('ascii', 'replace')
+        title  = ascii_filename(title
+                    )[:self.PATH_LIMIT].decode('ascii', 'replace')
+        name   = title + ' - ' + author
+        while name.endswith('.'):
+            name = name[:-1]
+        return name
+
     # Database layer API {{{
 
     def custom_table_names(self, num):
@@ -865,7 +905,7 @@ class DB(object):
         return self.format_abspath(book_id, fmt, fname, path) is not None
 
     def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False):
-        path = os.path.join(self.library_path, path, 'cover.jpg')
+        path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg'))
         if windows_atomic_move is not None:
             if not isinstance(dest, basestring):
                 raise Exception("Error, you must pass the dest as a path when"
@@ -926,5 +966,90 @@ class DB(object):
                     shutil.copyfileobj(f, d)
         return True
 
+    def windows_check_if_files_in_use(self, paths):
+        '''
+        Raises an EACCES IOError if any of the files in the folder of book_id
+        are opened in another program on windows.
+        '''
+        if iswindows:
+            for path in paths:
+                spath = os.path.join(self.library_path, *path.split('/'))
+                wam = None
+                if os.path.exists(spath):
+                    try:
+                        wam = WindowsAtomicFolderMove(spath)
+                    finally:
+                        if wam is not None:
+                            wam.close_handles()
+
+    def update_path(self, book_id, title, author, path_field, formats_field):
+        path = self.construct_path_name(book_id, title, author)
+        current_path = path_field.for_book(book_id)
+        formats = formats_field.for_book(book_id, default_value=())
+        fname = self.construct_file_name(book_id, title, author)
+        # Check if the metadata used to construct paths has changed
+        changed = False
+        for fmt in formats:
+            name = formats_field.format_fname(book_id, fmt)
+            if name and name != fname:
+                changed = True
+                break
+        if path == current_path and not changed:
+            return
+        spath = os.path.join(self.library_path, *current_path.split('/'))
+        tpath = os.path.join(self.library_path, *path.split('/'))
+
+        source_ok = current_path and os.path.exists(spath)
+        wam = WindowsAtomicFolderMove(spath) if iswindows and source_ok else None
+        try:
+            if not os.path.exists(tpath):
+                os.makedirs(tpath)
+
+            if source_ok: # Migrate existing files
+                dest = os.path.join(tpath, 'cover.jpg')
+                self.copy_cover_to(current_path, dest,
+                        windows_atomic_move=wam, use_hardlink=True)
+                for fmt in formats:
+                    dest = os.path.join(tpath, fname+'.'+fmt.lower())
+                    self.copy_format_to(book_id, fmt, formats_field.format_fname(book_id, fmt), current_path,
+                                        dest, windows_atomic_move=wam, use_hardlink=True)
+            # Update db to reflect new file locations
+            for fmt in formats:
+                formats_field.table.set_fname(book_id, fmt, fname, self)
+            path_field.table.set_path(book_id, path, self)
+
+            # Delete not needed directories
+            if source_ok:
+                if os.path.exists(spath) and not samefile(spath, tpath):
+                    if wam is not None:
+                        wam.delete_originals()
+                    self.rmtree(spath, permanent=True)
+                    parent = os.path.dirname(spath)
+                    if len(os.listdir(parent)) == 0:
+                        self.rmtree(parent, permanent=True)
+        finally:
+            if wam is not None:
+                wam.close_handles()
+
+        curpath = self.library_path
+        c1, c2 = current_path.split('/'), path.split('/')
+        if not self.is_case_sensitive and len(c1) == len(c2):
+            # On case-insensitive systems, title and author renames that only
+            # change case don't cause any changes to the directories in the file
+            # system. This can lead to having the directory names not match the
+            # title/author, which leads to trouble when libraries are copied to
+            # a case-sensitive system. The following code attempts to fix this
+            # by checking each segment. If they are different because of case,
+            # then rename the segment. Note that the code above correctly
+            # handles files in the directories, so no need to do them here.
+            for oldseg, newseg in zip(c1, c2):
+                if oldseg.lower() == newseg.lower() and oldseg != newseg:
+                    try:
+                        os.rename(os.path.join(curpath, oldseg),
+                                os.path.join(curpath, newseg))
+                    except:
+                        break # Fail silently since nothing catastrophic has happened
+                curpath = os.path.join(curpath, newseg)
+
    # }}}
 

src/calibre/db/cache.py

@@ -12,6 +12,7 @@ from io import BytesIO
 from collections import defaultdict
 from functools import wraps, partial
 
+from calibre.constants import iswindows
 from calibre.db import SPOOL_SIZE
 from calibre.db.categories import get_categories
 from calibre.db.locking import create_locks, RecordLock
@@ -219,6 +220,8 @@ class Cache(object):
                     field.series_field = self.fields['series']
                 elif name == 'authors':
                     field.author_sort_field = self.fields['author_sort']
+                elif name == 'title':
+                    field.title_sort_field = self.fields['sort']
 
     @read_api
     def field_for(self, name, book_id, default_value=None):
@@ -619,11 +622,12 @@ class Cache(object):
 
     @write_api
     def set_field(self, name, book_id_to_val_map, allow_case_change=True):
-        # TODO: Specialize title/authors to also update path
-        # TODO: Handle updating caches used by composite fields
-        # TODO: Ensure the sort fields are updated for title/author/series?
         f = self.fields[name]
         is_series = f.metadata['datatype'] == 'series'
+        update_path = name in {'title', 'authors'}
+        if update_path and iswindows:
+            paths = (x for x in (self._field_for('path', book_id) for book_id in book_id_to_val_map) if x)
+            self.backend.windows_check_if_files_in_use(paths)
 
         if is_series:
             bimap, simap = {}, {}
@@ -646,8 +650,28 @@ class Cache(object):
             sf = self.fields[f.name+'_index']
             dirtied |= sf.writer.set_books(simap, self.backend, allow_case_change=False)
 
+        if dirtied and self.composites:
+            for name in self.composites:
+                self.fields[name].pop_cache(dirtied)
+
+        if dirtied and update_path:
+            self._update_path(dirtied, mark_as_dirtied=False)
+
+        # TODO: Mark these as dirtied so that the opf is regenerated
+
         return dirtied
 
+    @write_api
+    def update_path(self, book_ids, mark_as_dirtied=True):
+        for book_id in book_ids:
+            title = self._field_for('title', book_id, default_value=_('Unknown'))
+            author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0]
+            self.backend.update_path(book_id, title, author, self.fields['path'], self.fields['formats'])
+            if mark_as_dirtied:
+                pass
+            # TODO: Mark these books as dirtied so that metadata.opf is
+            # re-created
+
     # }}}
 
 class SortKey(object):

src/calibre/db/fields.py

@@ -167,8 +167,9 @@ class CompositeField(OneToOneField):
         with self._lock:
             self._render_cache = {}
 
-    def pop_cache(self, book_id):
+    def pop_cache(self, book_ids):
         with self._lock:
+            for book_id in book_ids:
                 self._render_cache.pop(book_id, None)
 
     def get_value_with_cache(self, book_id, get_metadata):
@@ -177,6 +178,8 @@ class CompositeField(OneToOneField):
         if ans is None:
             mi = get_metadata(book_id)
             ans = mi.get('#'+self.metadata['label'])
+            with self._lock:
+                self._render_cache[book_id] = ans
         return ans
 
     def sort_keys_for_books(self, get_metadata, lang_map, all_book_ids):

src/calibre/db/tables.py

@@ -13,7 +13,6 @@ from dateutil.tz import tzoffset
 
 from calibre.constants import plugins
 from calibre.utils.date import parse_date, local_tz, UNDEFINED_DATE
-from calibre.utils.localization import lang_map
 from calibre.ebooks.metadata import author_to_author_sort
 
 _c_speedup = plugins['speedup'][0]
@@ -83,6 +82,13 @@ class OneToOneTable(Table):
             self.metadata['column'], self.metadata['table'])):
             self.book_col_map[row[0]] = self.unserialize(row[1])
 
+class  PathTable(OneToOneTable):
+
+    def set_path(self, book_id, path, db):
+        self.book_col_map[book_id] = path
+        db.conn.execute('UPDATE books SET path=? WHERE id=?',
+                        (path, book_id))
+
 class SizeTable(OneToOneTable):
 
     def read(self, db):
@@ -144,7 +150,7 @@ class ManyToManyTable(ManyToOneTable):
     '''
 
     table_type = MANY_MANY
-    selectq = 'SELECT book, {0} FROM {1}'
+    selectq = 'SELECT book, {0} FROM {1} ORDER BY id'
 
     def read_maps(self, db):
         for row in db.conn.execute(
@@ -161,8 +167,6 @@ class ManyToManyTable(ManyToOneTable):
 
 class AuthorsTable(ManyToManyTable):
 
-    selectq = 'SELECT book, {0} FROM {1} ORDER BY id'
-
     def read_id_maps(self, db):
         self.alink_map = {}
         self.asort_map  = {}
@@ -196,6 +200,11 @@ class FormatsTable(ManyToManyTable):
         for key in tuple(self.book_col_map.iterkeys()):
             self.book_col_map[key] = tuple(sorted(self.book_col_map[key]))
 
+    def set_fname(self, book_id, fmt, fname, db):
+        self.fname_map[book_id][fmt] = fname
+        db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?',
+                        (fname, book_id, fmt))
+
 class IdentifiersTable(ManyToManyTable):
 
     def read_id_maps(self, db):
@@ -215,6 +224,3 @@ class LanguagesTable(ManyToManyTable):
 
     def read_id_maps(self, db):
         ManyToManyTable.read_id_maps(self, db)
-        lm = lang_map()
-        self.lang_name_map = {x:lm.get(x, x) for x in self.id_map.itervalues()}
-

src/calibre/db/tests/base.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import unittest, os, shutil, tempfile, atexit
+import unittest, os, shutil, tempfile, atexit, gc
 from functools import partial
 from io import BytesIO
 from future_builtins import map
@@ -21,6 +21,7 @@ class BaseTest(unittest.TestCase):
         self.create_db(self.library_path)
 
     def tearDown(self):
+        gc.collect(), gc.collect()
         shutil.rmtree(self.library_path)
 
     def create_db(self, library_path):
@@ -36,6 +37,7 @@ class BaseTest(unittest.TestCase):
         db.add_format(1, 'FMT1', BytesIO(b'book1fmt1'), index_is_id=True)
         db.add_format(1, 'FMT2', BytesIO(b'book1fmt2'), index_is_id=True)
         db.add_format(2, 'FMT1', BytesIO(b'book2fmt1'), index_is_id=True)
+        db.conn.close()
         return dest
 
     def init_cache(self, library_path):
@@ -65,6 +67,10 @@ class BaseTest(unittest.TestCase):
         shutil.copytree(library_path, dest)
         return dest
 
+    @property
+    def cloned_library(self):
+        return self.clone_library(self.library_path)
+
     def compare_metadata(self, mi1, mi2):
         allfk1 = mi1.all_field_keys()
         allfk2 = mi2.all_field_keys()
@@ -79,6 +85,8 @@ class BaseTest(unittest.TestCase):
             attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
             if attr == 'formats':
                 attr1, attr2 = map(lambda x:tuple(x) if x else (), (attr1, attr2))
+            if isinstance(attr1, (tuple, list)) and 'authors' not in attr and 'languages' not in attr:
+                attr1, attr2 = set(attr1), set(attr2)
             self.assertEqual(attr1, attr2,
                     '%s not the same: %r != %r'%(attr, attr1, attr2))
             if attr.startswith('#'):

src/calibre/db/tests/filesystem.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import unittest, os
+from io import BytesIO
+
+from calibre.constants import iswindows
+from calibre.db.tests.base import BaseTest
+
+class FilesystemTest(BaseTest):
+
+    def get_filesystem_data(self, cache, book_id):
+        fmts = cache.field_for('formats', book_id)
+        ans = {}
+        for fmt in fmts:
+            buf = BytesIO()
+            if cache.copy_format_to(book_id, fmt, buf):
+                ans[fmt] = buf.getvalue()
+        buf = BytesIO()
+        if cache.copy_cover_to(book_id, buf):
+            ans['cover'] = buf.getvalue()
+        return ans
+
+    def test_metadata_move(self):
+        'Test the moving of files when title/author change'
+        cl = self.cloned_library
+        cache = self.init_cache(cl)
+        ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field
+
+        # Test that changing metadata on a book with no formats/cover works
+        ae(sf('title', {3:'moved1'}), set([3]))
+        ae(sf('authors', {3:'moved1'}), set([3]))
+        ae(sf('title', {3:'Moved1'}), set([3]))
+        ae(sf('authors', {3:'Moved1'}), set([3]))
+        ae(cache.field_for('title', 3), 'Moved1')
+        ae(cache.field_for('authors', 3), ('Moved1',))
+
+        # Now try with a book that has covers and formats
+        orig_data = self.get_filesystem_data(cache, 1)
+        orig_fpath = cache.format_abspath(1, 'FMT1')
+        ae(sf('title', {1:'moved'}), set([1]))
+        ae(sf('authors', {1:'moved'}), set([1]))
+        ae(sf('title', {1:'Moved'}), set([1]))
+        ae(sf('authors', {1:'Moved'}), set([1]))
+        ae(cache.field_for('title', 1), 'Moved')
+        ae(cache.field_for('authors', 1), ('Moved',))
+        cache2 = self.init_cache(cl)
+        for c in (cache, cache2):
+            data = self.get_filesystem_data(c, 1)
+            ae(set(orig_data.iterkeys()), set(data.iterkeys()))
+            ae(orig_data, data, 'Filesystem data does not match')
+            ae(c.field_for('path', 1), 'Moved/Moved (1)')
+            ae(c.field_for('path', 3), 'Moved1/Moved1 (3)')
+        fpath = c.format_abspath(1, 'FMT1').replace(os.sep, '/').split('/')
+        ae(fpath[-3:], ['Moved', 'Moved (1)', 'Moved - Moved.fmt1'])
+        af(os.path.exists(os.path.dirname(orig_fpath)), 'Original book folder still exists')
+
+    @unittest.skipUnless(iswindows, 'Windows only')
+    def test_windows_atomic_move(self):
+        'Test book file open in another process when changing metadata'
+        cl = self.cloned_library
+        cache = self.init_cache(cl)
+        fpath = cache.format_abspath(1, 'FMT1')
+        f = open(fpath, 'rb')
+        with self.assertRaises(IOError):
+            cache.set_field('title', {1:'Moved'})
+        f.close()
+        self.assertNotEqual(cache.field_for('title', 1), 'Moved', 'Title was changed despite file lock')
+
+

src/calibre/db/tests/main.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import unittest, os, argparse
+
+def find_tests():
+    return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('name', nargs='?', default=None, help='The name of the test to run, for e.g. writing.WritingTest.many_many_basic')
+    args = parser.parse_args()
+    if args.name:
+        unittest.TextTestRunner(verbosity=4).run(unittest.defaultTestLoader.loadTestsFromName(args.name))
+    else:
+        unittest.TextTestRunner(verbosity=4).run(find_tests())
+

src/calibre/db/tests/reading.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import unittest, datetime
+import datetime
 
 from calibre.utils.date import utc_tz
 from calibre.db.tests.base import BaseTest
@@ -115,6 +115,8 @@ class ReadingTest(BaseTest):
         for book_id, test in tests.iteritems():
             for field, expected_val in test.iteritems():
                 val = cache.field_for(field, book_id)
+                if isinstance(val, tuple) and 'authors' not in field and 'languages' not in field:
+                    val, expected_val = set(val), set(expected_val)
                 self.assertEqual(expected_val, val,
                         'Book id: %d Field: %s failed: %r != %r'%(
                             book_id, field, expected_val, val))
@@ -173,6 +175,7 @@ class ReadingTest(BaseTest):
             mi.format_metadata = dict(mi.format_metadata)
             if mi.formats:
                 mi.formats = tuple(mi.formats)
+        old.conn.close()
         old = None
 
         cache = self.init_cache(self.library_path)
@@ -189,6 +192,7 @@ class ReadingTest(BaseTest):
         from calibre.library.database2 import LibraryDatabase2
         old = LibraryDatabase2(self.library_path)
         covers = {i: old.cover(i, index_is_id=True) for i in old.all_ids()}
+        old.conn.close()
         old = None
         cache = self.init_cache(self.library_path)
         for book_id, cdata in covers.iteritems():
@@ -247,6 +251,7 @@ class ReadingTest(BaseTest):
             '#formats:fmt1', '#formats:fmt2', '#formats:fmt1 and #formats:fmt2',
 
         )}
+        old.conn.close()
         old = None
 
         cache = self.init_cache(self.library_path)
@@ -263,6 +268,7 @@ class ReadingTest(BaseTest):
         from calibre.library.database2 import LibraryDatabase2
         old = LibraryDatabase2(self.library_path)
         old_categories = old.get_categories()
+        old.conn.close()
         cache = self.init_cache(self.library_path)
         new_categories = cache.get_categories()
         self.assertEqual(set(old_categories), set(new_categories),
@@ -305,6 +311,7 @@ class ReadingTest(BaseTest):
             i, index_is_id=True) else set() for i in ids}
         formats = {i:{f:old.format(i, f, index_is_id=True) for f in fmts} for
                    i, fmts in lf.iteritems()}
+        old.conn.close()
         old = None
         cache = self.init_cache(self.library_path)
         for book_id, fmts in lf.iteritems():
@@ -328,12 +335,3 @@ class ReadingTest(BaseTest):
 
     # }}}
 
-def tests():
-    return unittest.TestLoader().loadTestsFromTestCase(ReadingTest)
-
-def run():
-    unittest.TextTestRunner(verbosity=2).run(tests())
-
-if __name__ == '__main__':
-    run()
-

src/calibre/db/tests/writing.py

@@ -7,19 +7,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import unittest
 from collections import namedtuple
 from functools import partial
 
+from calibre.ebooks.metadata import author_to_author_sort
 from calibre.utils.date import UNDEFINED_DATE
 from calibre.db.tests.base import BaseTest
 
 class WritingTest(BaseTest):
 
-    @property
-    def cloned_library(self):
-        return self.clone_library(self.library_path)
-
     def create_getter(self, name, getter=None):
         if getter is None:
             if name.endswith('_index'):
@@ -214,7 +210,7 @@ class WritingTest(BaseTest):
             {1, 2})
         for name in ('tags', '#tags'):
             f = cache.fields[name]
-            af(sf(name, {1:('tag one', 'News')}, allow_case_change=False))
+            af(sf(name, {1:('News', 'tag one')}, allow_case_change=False))
             ae(sf(name, {1:'tag one, News'}), {1, 2})
             ae(sf(name, {3:('tag two', 'sep,sep2')}), {2, 3})
             ae(len(f.table.id_map), 4)
@@ -225,7 +221,7 @@ class WritingTest(BaseTest):
                 ae(len(c.fields[name].table.id_map), 3)
                 ae(len(c.fields[name].table.id_map), 3)
                 ae(c.field_for(name, 1), ())
-                ae(c.field_for(name, 2), ('tag one', 'tag two'))
+                ae(c.field_for(name, 2), ('tag two', 'tag one'))
             del cache2
 
         # Authors
@@ -244,27 +240,55 @@ class WritingTest(BaseTest):
                 ae(c.field_for(name, 3), ('Kovid Goyal', 'Divok Layog'))
                 ae(c.field_for(name, 2), ('An, Author',))
                 ae(c.field_for(name, 1), ('Unknown',) if name=='authors' else ())
-                ae(c.field_for('author_sort', 1), 'Unknown')
+                if name == 'authors':
-                ae(c.field_for('author_sort', 2), 'An, Author')
+                    ae(c.field_for('author_sort', 1), author_to_author_sort('Unknown'))
-                ae(c.field_for('author_sort', 3), 'Goyal, Kovid & Layog, Divok')
+                    ae(c.field_for('author_sort', 2), author_to_author_sort('An, Author'))
+                    ae(c.field_for('author_sort', 3), author_to_author_sort('Kovid Goyal') + ' & ' + author_to_author_sort('Divok Layog'))
             del cache2
         ae(cache.set_field('authors', {1:'KoviD GoyaL'}), {1, 3})
         ae(cache.field_for('author_sort', 1), 'GoyaL, KoviD')
         ae(cache.field_for('author_sort', 3), 'GoyaL, KoviD & Layog, Divok')
 
-        # TODO: identifiers, languages
+        # Languages
+        f = cache.fields['languages']
+        ae(f.table.id_map, {1: 'eng', 2: 'deu'})
+        ae(sf('languages', {1:''}), set([1]))
+        ae(cache.field_for('languages', 1), ())
+        ae(sf('languages', {2:('und',)}), set([2]))
+        af(f.table.id_map)
+        ae(sf('languages', {1:'eng,fra,deu', 2:'es,Dutch', 3:'English'}), {1, 2, 3})
+        ae(cache.field_for('languages', 1), ('eng', 'fra', 'deu'))
+        ae(cache.field_for('languages', 2), ('spa', 'nld'))
+        ae(cache.field_for('languages', 3), ('eng',))
+        ae(sf('languages', {3:None}), set([3]))
+        ae(cache.field_for('languages', 3), ())
+        ae(sf('languages', {1:'deu,fra,eng'}), set([1]), 'Changing order failed')
+        ae(sf('languages', {2:'deu,eng,eng'}), set([2]))
+        cache2 = self.init_cache(cl)
+        for c in (cache, cache2):
+            ae(cache.field_for('languages', 1), ('deu', 'fra', 'eng'))
+            ae(cache.field_for('languages', 2), ('deu', 'eng'))
+        del cache2
+
+        # Identifiers
+        f = cache.fields['identifiers']
+        ae(sf('identifiers', {3: 'one:1,two:2'}), set([3]))
+        ae(sf('identifiers', {2:None}), set([2]))
+        ae(sf('identifiers', {1: {'test':'1', 'two':'2'}}), set([1]))
+        cache2 = self.init_cache(cl)
+        for c in (cache, cache2):
+            ae(c.field_for('identifiers', 3), {'one':'1', 'two':'2'})
+            ae(c.field_for('identifiers', 2), {})
+            ae(c.field_for('identifiers', 1), {'test':'1', 'two':'2'})
+        del cache2
+
+        # Test setting of title sort
+        ae(sf('title', {1:'The Moose', 2:'Cat'}), {1, 2})
+        cache2 = self.init_cache(cl)
+        for c in (cache, cache2):
+            ae(c.field_for('sort', 1), 'Moose, The')
+            ae(c.field_for('sort', 2), 'Cat')
+
 
     # }}}
 
-def tests():
-    tl = unittest.TestLoader()
-    # return tl.loadTestsFromName('writing.WritingTest.test_many_many_basic')
-    return tl.loadTestsFromTestCase(WritingTest)
-
-def run():
-    unittest.TextTestRunner(verbosity=2).run(tests())
-
-if __name__ == '__main__':
-    run()
-
-

src/calibre/db/write.py

@@ -12,9 +12,10 @@ from functools import partial
 from datetime import datetime
 
 from calibre.constants import preferred_encoding, ispy3
-from calibre.ebooks.metadata import author_to_author_sort
+from calibre.ebooks.metadata import author_to_author_sort, title_sort
 from calibre.utils.date import (parse_only_date, parse_date, UNDEFINED_DATE,
                                 isoformat)
+from calibre.utils.localization import canonicalize_lang
 from calibre.utils.icu import strcmp
 
 if ispy3:
@@ -96,6 +97,30 @@ def adapt_bool(x):
             x = bool(int(x))
     return x if x is None else bool(x)
 
+def adapt_languages(to_tuple, x):
+    ans = []
+    for lang in to_tuple(x):
+        lc = canonicalize_lang(lang)
+        if not lc or lc in ans or lc in ('und', 'zxx', 'mis', 'mul'):
+            continue
+        ans.append(lc)
+    return tuple(ans)
+
+def clean_identifier(typ, val):
+    typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
+    val = val.strip().replace(',', '|').replace(':', '|')
+    return typ, val
+
+def adapt_identifiers(to_tuple, x):
+    if not isinstance(x, dict):
+        x = {k:v for k, v in (y.partition(':')[0::2] for y in to_tuple(x))}
+    ans = {}
+    for k, v in x.iteritems():
+        k, v = clean_identifier(k, v)
+        if k and v:
+            ans[k] = v
+    return ans
+
 def get_adapter(name, metadata):
     dt = metadata['datatype']
     if dt == 'text':
@@ -133,6 +158,10 @@ def get_adapter(name, metadata):
         return lambda x: ans(x) or UNDEFINED_DATE
     if name == 'series_index':
         return lambda x: 1.0 if ans(x) is None else ans(x)
+    if name == 'languages':
+        return partial(adapt_languages, ans)
+    if name == 'identifiers':
+        return partial(adapt_identifiers, ans)
 
     return ans
 # }}}
@@ -145,6 +174,10 @@ def one_one_in_books(book_id_val_map, db, field, *args):
         db.conn.executemany(
             'UPDATE books SET %s=? WHERE id=?'%field.metadata['column'], sequence)
         field.table.book_col_map.update(book_id_val_map)
+    if field.name == 'title':
+        # Set the title sort field
+        field.title_sort_field.writer.set_books(
+            {k:title_sort(v) for k, v in book_id_val_map.iteritems()}, db)
     return set(book_id_val_map)
 
 def one_one_in_other(book_id_val_map, db, field, *args):
@@ -384,6 +417,31 @@ def many_many(book_id_val_map, db, field, allow_case_change, *args):
 
 # }}}
 
+def identifiers(book_id_val_map, db, field, *args): # {{{
+    table = field.table
+    updates = set()
+    for book_id, identifiers in book_id_val_map.iteritems():
+        if book_id not in table.book_col_map:
+            table.book_col_map[book_id] = {}
+        current_ids = table.book_col_map[book_id]
+        remove_keys = set(current_ids) - set(identifiers)
+        for key in remove_keys:
+            table.col_book_map.get(key, set()).discard(book_id)
+            current_ids.pop(key, None)
+        current_ids.update(identifiers)
+        for key, val in identifiers.iteritems():
+            if key not in table.col_book_map:
+                table.col_book_map[key] = set()
+            table.col_book_map[key].add(book_id)
+            updates.add((book_id, key, val))
+    db.conn.executemany('DELETE FROM identifiers WHERE book=?',
+                        ((x,) for x in book_id_val_map))
+    if updates:
+        db.conn.executemany('INSERT OR REPLACE INTO identifiers (book, type, val) VALUES (?, ?, ?)',
+                            tuple(updates))
+    return set(book_id_val_map)
+# }}}
+
 def dummy(book_id_val_map, *args):
     return set()
 
@@ -400,6 +458,8 @@ class Writer(object):
             self.set_books_func = dummy
         elif self.name[0] == '#' and self.name.endswith('_index'):
             self.set_books_func = custom_series_index
+        elif self.name == 'identifiers':
+            self.set_books_func = identifiers
         elif field.is_many_many:
             self.set_books_func = many_many
         elif field.is_many:

src/calibre/devices/misc.py

@@ -413,15 +413,15 @@ class WAYTEQ(USBMS):
 
     name           = 'WayteQ device interface'
     gui_name       = 'WayteQ xBook'
-    description    = _('Communicate with the WayteQ Reader')
+    description    = _('Communicate with the WayteQ and SPC Dickens Readers')
     author         = 'Kovid Goyal'
     supported_platforms = ['windows', 'osx', 'linux']
 
     # Ordered list of supported formats
     FORMATS     = ['epub', 'mobi', 'prc', 'fb2', 'txt', 'pdf', 'html', 'rtf', 'chm', 'djvu', 'doc']
 
-    VENDOR_ID   = [0x05e3]
+    VENDOR_ID   = [0x05e3, 0x05e8]
-    PRODUCT_ID  = [0x0726]
+    PRODUCT_ID  = [0x0726, 0x0728]
     BCD         = [0x0222]
 
     EBOOK_DIR_MAIN = 'Documents'
@@ -431,6 +431,14 @@ class WAYTEQ(USBMS):
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'RK28_SDK_DEMO'
     SUPPORTS_SUB_DIRS = True
 
+    def get_gui_name(self):
+        try:
+            if self.detected_device.idVendor == 0x05e8:
+                return 'SPC Dickens'
+        except Exception:
+            pass
+        return self.gui_name
+
     def get_carda_ebook_dir(self, for_upload=False):
         if for_upload:
             return 'Documents'

src/calibre/devices/mtp/unix/devices.c

@@ -11,18 +11,6 @@
 const calibre_device_entry_t calibre_mtp_device_table[] = {
 #include "upstream/music-players.h"
 
-    // Amazon Kindle Fire HD
-    , { "Amazon", 0x1949, "Fire HD", 0x0007, DEVICE_FLAGS_ANDROID_BUGS}
-    , { "Amazon", 0x1949, "Fire HD", 0x0008, DEVICE_FLAGS_ANDROID_BUGS}
-    , { "Amazon", 0x1949, "Fire HD", 0x000a, DEVICE_FLAGS_ANDROID_BUGS}
-
-    // Nexus 10
-    , { "Google", 0x18d1, "Nexus 10", 0x4ee2, DEVICE_FLAGS_ANDROID_BUGS}
-    , { "Google", 0x18d1, "Nexus 10", 0x4ee1, DEVICE_FLAGS_ANDROID_BUGS}
-
-    // Kobo Arc
-    , { "Kobo", 0x2237, "Arc", 0xd108, DEVICE_FLAGS_ANDROID_BUGS}
-
     , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
 };
 

src/calibre/devices/mtp/unix/upstream/music-players.h

@@ -294,6 +294,13 @@
       DEVICE_FLAG_UNIQUE_FILENAMES |
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
   // The "YP-R2" (0x04e8/0x512d) is NOT MTP, it is UMS only.
+  // Guessing on device flags for the MTP mode...
+ { "Samsung", 0x04e8, "YP-R2", 0x512e,
+     DEVICE_FLAG_UNLOAD_DRIVER |
+     DEVICE_FLAG_OGG_IS_UNKNOWN |
+     DEVICE_FLAG_UNIQUE_FILENAMES |
+     DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
+     DEVICE_FLAG_PLAYLIST_SPL_V1 },
   // From Manuel Carro
   // Copied from Q2
  { "Samsung", 0x04e8, "YP-Q3", 0x5130,
@@ -309,6 +316,7 @@
      DEVICE_FLAG_OGG_IS_UNKNOWN |
      DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
      DEVICE_FLAG_PLAYLIST_SPL_V1 },
+  // YP-F3 is NOT MTP - USB mass storage
   // From a rouge .INF file
   // this device ID seems to have been recycled for:
   // the Samsung SGH-A707 Cingular cellphone
@@ -393,7 +401,9 @@
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER |
       DEVICE_FLAG_LONG_TIMEOUT |
-      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	},
+      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	|
+      DEVICE_FLAG_OGG_IS_UNKNOWN |
+      DEVICE_FLAG_FLAC_IS_UNKNOWN },
   // Reported by David Goodenough <dfgdga@users.sourceforge.net>
   // Guessing on flags.
   { "Samsung", 0x04e8, "Galaxy Y", 0x685e,
@@ -401,14 +411,18 @@
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER |
       DEVICE_FLAG_LONG_TIMEOUT |
-      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	},
+      DEVICE_FLAG_PROPLIST_OVERRIDES_OI |
+      DEVICE_FLAG_OGG_IS_UNKNOWN |
+      DEVICE_FLAG_FLAC_IS_UNKNOWN },
   { "Samsung", 0x04e8,
       "Galaxy models (MTP)", 0x6860,
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL |
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER |
       DEVICE_FLAG_LONG_TIMEOUT |
-      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	},
+      DEVICE_FLAG_PROPLIST_OVERRIDES_OI |
+      DEVICE_FLAG_OGG_IS_UNKNOWN |
+      DEVICE_FLAG_FLAC_IS_UNKNOWN },
   // From: Erik Berglund <erikjber@users.sourceforge.net>
   // Logs indicate this needs DEVICE_FLAG_NO_ZERO_READS
   // No Samsung platlists on this device.
@@ -419,7 +433,9 @@
   { "Samsung", 0x04e8, "Galaxy models Kies mode", 0x6877,
       DEVICE_FLAG_UNLOAD_DRIVER |
       DEVICE_FLAG_LONG_TIMEOUT |
-      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	},
+      DEVICE_FLAG_PROPLIST_OVERRIDES_OI	|
+      DEVICE_FLAG_OGG_IS_UNKNOWN |
+      DEVICE_FLAG_FLAC_IS_UNKNOWN },
   // From: John Gorkos <ab0oo@users.sourceforge.net> and
   // Akos Maroy <darkeye@users.sourceforge.net>
   { "Samsung", 0x04e8, "Vibrant SGH-T959/Captivate/Media player mode", 0x68a9,
@@ -439,7 +455,6 @@
    */
   { "Microsoft/Intel", 0x045e, "Bandon Portable Media Center", 0x00c9,
       DEVICE_FLAG_NONE },
-  // Reported by anonymous sourceforge user
   // HTC Mozart is using the PID, as is Nokia Lumia 800
   // May need MTPZ to work
   { "Microsoft", 0x045e, "Windows Phone", 0x04ec, DEVICE_FLAG_NONE },
@@ -450,12 +465,12 @@
   { "Microsoft", 0x045e, "Windows MTP Simulator", 0x0622, DEVICE_FLAG_NONE },
   // Reported by Edward Hutchins (used for Zune HDs)
   { "Microsoft", 0x045e, "Zune HD", 0x063e, DEVICE_FLAG_NONE },
-  // Reported by anonymous sourceforge user
   { "Microsoft", 0x045e, "Kin 1", 0x0640, DEVICE_FLAG_NONE },
-  // Reported by anonymous sourceforge user
   { "Microsoft/Sharp/nVidia", 0x045e, "Kin TwoM", 0x0641, DEVICE_FLAG_NONE },
   // Reported by Farooq Zaman (used for all Zunes)
   { "Microsoft", 0x045e, "Zune", 0x0710, DEVICE_FLAG_NONE },
+  // Reported by Olegs Jeremejevs
+  { "Microsoft/HTC", 0x045e, "HTC 8S", 0xf0ca, DEVICE_FLAG_NONE },
 
   /*
    * JVC
@@ -517,33 +532,52 @@
   // From Anonymous SourceForge User
   { "Philips", 0x0471, "GoGear Vibe/02", 0x20e5,
       DEVICE_FLAG_UNLOAD_DRIVER },
+  // Reported by Philip Rhoades
+  { "Philips", 0x0471, "GoGear Ariaz/97", 0x2138,
+      DEVICE_FLAG_UNLOAD_DRIVER },
   // from XNJB user
   { "Philips", 0x0471, "PSA235", 0x7e01, DEVICE_FLAG_NONE },
 
   /*
    * Acer
+   * Reporters:
+   * Franck VDL <franckv@users.sourceforge.net>
+   * Matthias Arndt <simonsunnyboy@users.sourceforge.net>
+   * Arvin Schnell <arvins@users.sourceforge.net>
+   * Philippe Marzouk <philm@users.sourceforge.net>
+   * nE0sIghT <ne0sight@users.sourceforge.net>
+   * Maxime de Roucy <maxime1986@users.sourceforge.net>
    */
-  // Reported by anonymous sourceforge user
+  { "Acer", 0x0502, "Iconia TAB A500 (ID1)", 0x3325,
-  { "Acer", 0x0502, "Iconia TAB A500 (ID1)", 0x3325, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by: Franck VDL <franckv@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A500 (ID2)", 0x3341,
-  { "Acer", 0x0502, "Iconia TAB A500 (ID2)", 0x3341, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by: Matthias Arndt <simonsunnyboy@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A501 (ID1)", 0x3344,
-  { "Acer", 0x0502, "Iconia TAB A501", 0x3344, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by: anonymous sourceforge user
+  { "Acer", 0x0502, "Iconia TAB A501 (ID2)", 0x3345,
-  { "Acer", 0x0502, "Iconia TAB A100 (ID1)", 0x3348, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by: Arvin Schnell <arvins@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A100 (ID1)", 0x3348,
-  { "Acer", 0x0502, "Iconia TAB A100 (ID2)", 0x3349, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Philippe Marzouk <philm@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A100 (ID2)", 0x3349,
-  { "Acer", 0x0502, "Iconia TAB A700", 0x3378, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
+  { "Acer", 0x0502, "Iconia TAB A700", 0x3378,
-  { "Acer", 0x0502, "Iconia TAB A200 (ID1)", 0x337c, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
+  { "Acer", 0x0502, "Iconia TAB A200 (ID1)", 0x337c,
-  { "Acer", 0x0502, "Iconia TAB A200 (ID2)", 0x337d, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by nE0sIghT <ne0sight@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A200 (ID2)", 0x337d,
-  { "Acer", 0x0502, "Iconia TAB A510", 0x338a, DEVICE_FLAGS_ANDROID_BUGS },
+      DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Maxime de Roucy <maxime1986@users.sourceforge.net>
+  { "Acer", 0x0502, "Iconia TAB A510 (ID1)", 0x3389,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Acer", 0x0502, "Iconia TAB A510 (ID2)", 0x338a,
+      DEVICE_FLAGS_ANDROID_BUGS },
   { "Acer", 0x0502, "E350 Liquid Gallant Duo", 0x33c3,
       DEVICE_FLAGS_ANDROID_BUGS },
+  { "Acer", 0x0502, "Iconia TAB A210", 0x33cb,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Acer", 0x0502, "Iconia TAB A110", 0x33d8,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
+
 
   /*
    * SanDisk
@@ -743,6 +777,10 @@
   { "iRiver", 0x4102, "E50", 0x1151,
     DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | DEVICE_FLAG_NO_ZERO_READS |
     DEVICE_FLAG_OGG_IS_UNKNOWN },
+  // Reported by anonymous SourceForge user, guessing on flags
+  { "iRiver", 0x4102, "E150", 0x1152,
+    DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | DEVICE_FLAG_NO_ZERO_READS |
+    DEVICE_FLAG_OGG_IS_UNKNOWN },
   // Reported by Jakub Matraszek <jakub.matraszek@gmail.com>
   { "iRiver", 0x4102, "T5", 0x1153,
     DEVICE_FLAG_UNLOAD_DRIVER | DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
@@ -877,8 +915,10 @@
   { "Archos", 0x0e79, "SPOD (MTP mode)", 0x1341, DEVICE_FLAG_UNLOAD_DRIVER },
   { "Archos", 0x0e79, "5S IT (MTP mode)", 0x1351, DEVICE_FLAG_UNLOAD_DRIVER },
   { "Archos", 0x0e79, "5H IT (MTP mode)", 0x1357, DEVICE_FLAG_UNLOAD_DRIVER },
-  { "Archos", 0x0e79, "Arnova Childpad", 0x1458, DEVICE_FLAG_UNLOAD_DRIVER },
+  { "Archos", 0x0e79, "Arnova Childpad", 0x1458, DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous Sourceforge user
+  { "Archos", 0x0e79, "Arnova 8c G3", 0x145e, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Archos", 0x0e79, "Arnova 10bG3 Tablet", 0x146b, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Archos", 0x0e79, "97 Xenon", 0x149a, DEVICE_FLAGS_ANDROID_BUGS },
   { "Archos", 0x0e79, "8o G9 (MTP mode)", 0x1508, DEVICE_FLAG_UNLOAD_DRIVER },
   // Reported by Clément <clemvangelis@users.sourceforge.net>
   { "Archos", 0x0e79, "8o G9 Turbo (MTP mode)", 0x1509,
@@ -887,14 +927,10 @@
   { "Archos", 0x0e79, "80G9", 0x1518, DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Till <Till@users.sourceforge.net>
   { "Archos", 0x0e79, "101 G9", 0x1528, DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
   { "Archos", 0x0e79, "101 G9 (v2)", 0x1529, DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
   { "Archos", 0x0e79, "101 G9 Turbo 250 HD", 0x1538,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
   { "Archos", 0x0e79, "101 G9 Turbo", 0x1539, DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous sourceforge user
   { "Archos", 0x0e79, "70it2 (mode 1)", 0x1568, DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Sebastien ROHAUT
   { "Archos", 0x0e79, "70it2 (mode 2)", 0x1569, DEVICE_FLAGS_ANDROID_BUGS },
@@ -1075,6 +1111,10 @@
       DEVICE_FLAG_UNLOAD_DRIVER },
   // From: Maxin B. John <maxin.john@gmail.com>
   { "Nokia", 0x0421, "N9", 0x051a, DEVICE_FLAG_NONE },
+  { "Nokia", 0x0421, "C5-00", 0x0592, DEVICE_FLAG_NONE },
+  // Reported by Sampo Savola
+  // Covers Lumia 920, 820 and probably any WP8 device.
+  { "Nokia", 0x0421, "Nokia Lumia WP8", 0x0661, DEVICE_FLAG_NONE },
   // Reported by Richard Wall <richard@the-moon.net>
   { "Nokia", 0x05c6, "5530 Xpressmusic", 0x0229, DEVICE_FLAG_NONE },
   // Reported by anonymous SourceForge user
@@ -1109,9 +1149,12 @@
   { "Thomson / RCA", 0x069b, "Lyra HC308A", 0x3035, DEVICE_FLAG_NONE },
 
   /*
-   * NTT DoCoMo
+   * Fujitsu devices
    */
-  { "FOMA", 0x04c5, "F903iX HIGH-SPEED", 0x1140, DEVICE_FLAG_NONE },
+  { "Fujitsu, Ltd", 0x04c5, "F903iX HIGH-SPEED", 0x1140, DEVICE_FLAG_NONE },
+  // Reported by Thomas Bretthauer
+  { "Fujitsu, Ltd", 0x04c5, "STYLISTIC M532", 0x133b,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * Palm device userland program named Pocket Tunes
@@ -1254,7 +1297,6 @@
   /*
    * LG Electronics
    */
-  // From anonymous SourceForge user
   // Uncertain if this is really the MTP mode device ID...
   { "LG Electronics Inc.", 0x043e, "T54", 0x7040,
       DEVICE_FLAG_UNLOAD_DRIVER },
@@ -1271,20 +1313,20 @@
   { "LG Electronics Inc.", 0x1004, "GR-500 Music Player", 0x611b,
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_ALWAYS_PROBE_DESCRIPTOR },
-  // Reported by anonymous sourceforge user
   { "LG Electronics Inc.", 0x1004, "KM900", 0x6132,
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER },
-  // Reported by anonymous sourceforge user
   { "LG Electronics Inc.", 0x1004, "LG8575", 0x619a,
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER },
-  // Reported by anonymous sourceforge user
   { "LG Electronics Inc.", 0x1004, "V909 G-Slate", 0x61f9,
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
       DEVICE_FLAG_UNLOAD_DRIVER },
+  { "LG Electronics Inc.", 0x1004, "LG2 Optimus", 0x6225,
+      DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
+      DEVICE_FLAG_UNLOAD_DRIVER },
   // Reported by Brian J. Murrell
-  { "LG Electronics Inc.", 0x1004, "LG-E617G/P700", 0x631c,
+  { "LG Electronics Inc.", 0x1004, "LG-E610/E612/E617G/E970/P700", 0x631c,
       DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
@@ -1294,69 +1336,69 @@
    * reported to see a pattern here.
    */
   // Reported by Alessandro Radaelli <alessandro.radaelli@aruba.it>
-  { "Sony", 0x054c, "Walkman NWZ-A815/NWZ-A818", 0x0325,
+  { "Sony", 0x054c, "NWZ-A815/NWZ-A818", 0x0325,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by anonymous Sourceforge user.
-  { "Sony", 0x054c, "Walkman NWZ-S516", 0x0326,
+  { "Sony", 0x054c, "NWZ-S516", 0x0326,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Endre Oma <endre.88.oma@gmail.com>
-  { "Sony", 0x054c, "Walkman NWZ-S615F/NWZ-S616F/NWZ-S618F", 0x0327,
+  { "Sony", 0x054c, "NWZ-S615F/NWZ-S616F/NWZ-S618F", 0x0327,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Jean-Marc Bourguet <jm@bourguet.org>
-  { "Sony", 0x054c, "Walkman NWZ-S716F", 0x035a,
+  { "Sony", 0x054c, "NWZ-S716F", 0x035a,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Anon SF User / Anthon van der Neut <avanderneut@avid.com>
-  { "Sony", 0x054c, "Walkman NWZ-A826/NWZ-A828/NWZ-A829", 0x035b,
+  { "Sony", 0x054c, "NWZ-A826/NWZ-A828/NWZ-A829", 0x035b,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Niek Klaverstijn <niekez@users.sourceforge.net>
-  { "Sony", 0x054c, "Walkman NWZ-A726/NWZ-A728/NWZ-A768", 0x035c,
+  { "Sony", 0x054c, "NWZ-A726/NWZ-A728/NWZ-A768", 0x035c,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Mehdi AMINI <mehdi.amini - at - ulp.u-strasbg.fr>
-  { "Sony", 0x054c, "Walkman NWZ-B135", 0x036e,
+  { "Sony", 0x054c, "NWZ-B135", 0x036e,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by <tiagoboldt@users.sourceforge.net>
-  { "Sony", 0x054c, "Walkman NWZ-E436F", 0x0385,
+  { "Sony", 0x054c, "NWZ-E436F", 0x0385,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Michael Wilkinson
-  { "Sony", 0x054c, "Walkman NWZ-W202", 0x0388,
+  { "Sony", 0x054c, "NWZ-W202", 0x0388,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Ondrej Sury <ondrej@sury.org>
-  { "Sony", 0x054c, "Walkman NWZ-S739F", 0x038c,
+  { "Sony", 0x054c, "NWZ-S739F", 0x038c,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Marco Filipe Nunes Soares Abrantes Pereira <marcopereira@ua.pt>
-  { "Sony", 0x054c, "Walkman NWZ-S638F", 0x038e,
+  { "Sony", 0x054c, "NWZ-S638F", 0x038e,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Elliot <orwells@users.sourceforge.net>
-  { "Sony", 0x054c, "Walkman NWZ-X1050B/NWZ-X1060B",
+  { "Sony", 0x054c, "NWZ-X1050B/NWZ-X1060B",
     0x0397, DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Silvio J. Gutierrez <silviogutierrez@users.sourceforge.net>
-  { "Sony", 0x054c, "Walkman NWZ-X1051/NWZ-X1061", 0x0398,
+  { "Sony", 0x054c, "NWZ-X1051/NWZ-X1061", 0x0398,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Gregory Boddin <gregory@siwhine.net>
-  { "Sony", 0x054c, "Walkman NWZ-B142F", 0x03d8,
+  { "Sony", 0x054c, "NWZ-B142F", 0x03d8,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Rick Warner <rick@reptileroom.net>
-  { "Sony", 0x054c, "Walkman NWZ-E344", 0x03fc,
+  { "Sony", 0x054c, "NWZ-E344/E345", 0x03fc,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Jonathan Stowe <gellyfish@users.sourceforge.net>
-  { "Sony", 0x054c, "Walkman NWZ-E445", 0x03fd,
+  { "Sony", 0x054c, "NWZ-E445", 0x03fd,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Anonymous SourceForge user
-  { "Sony", 0x054c, "Walkman NWZ-S545", 0x03fe,
+  { "Sony", 0x054c, "NWZ-S545", 0x03fe,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
-  { "Sony", 0x054c, "Walkman NWZ-A845", 0x0404,
+  { "Sony", 0x054c, "NWZ-A845", 0x0404,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by anonymous SourceForge user
-  { "Sony", 0x054c, "Walkman NWZ-W252B", 0x04bb,
+  { "Sony", 0x054c, "NWZ-W252B", 0x04bb,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Suspect this device has strong DRM features
   // See https://answers.launchpad.net/ubuntu/+source/libmtp/+question/149587
-  { "Sony", 0x054c, "Walkman NWZ-B153F", 0x04be,
+  { "Sony", 0x054c, "NWZ-B153F", 0x04be,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
-  { "Sony", 0x054c, "Walkman NWZ-E354", 0x04cb,
+  { "Sony", 0x054c, "NWZ-E354", 0x04cb,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Toni Burgarello
-  { "Sony", 0x054c, "Walkman NWZ-S754", 0x04cc,
+  { "Sony", 0x054c, "NWZ-S754", 0x04cc,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Hideki Yamane <henrich@debian.org>
   { "Sony", 0x054c, "Sony Tablet P1", 0x04d1,
@@ -1364,9 +1406,7 @@
   // Reported by dmiceman
   { "Sony", 0x054c, "NWZ-B163F", 0x059a,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
-  // Reported by anonymous Sourceforge user
+  { "Sony", 0x054c, "NWZ-E464", 0x05a6,
-  // guessing on device flags...
-  { "Sony", 0x054c, "Walkman NWZ-E464", 0x05a6,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
   // Reported by Jan Rheinlaender <jrheinlaender@users.sourceforge.net>
   { "Sony", 0x054c, "NWZ-S765", 0x05a8,
@@ -1377,7 +1417,8 @@
   // Reported by ghalambaz <ghalambaz@users.sourceforge.net>
   { "Sony", 0x054c, "Sony Tablet S1", 0x05b4,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Anonymous SourceForge user
+  { "Sony", 0x054c, "NWZ-B173F", 0x0689,
+      DEVICE_FLAGS_SONY_NWZ_BUGS },
   { "Sony", 0x054c, "DCR-SR75", 0x1294,
       DEVICE_FLAGS_SONY_NWZ_BUGS },
 
@@ -1496,6 +1537,7 @@
    * Jean-François  B. <changi67@users.sourceforge.net>
    * Eduard Bloch <blade@debian.org>
    * Ah Hong <hongster@users.sourceforge.net>
+   * Eowyn Carter
    */
   { "SonyEricsson", 0x0fce, "LT15i Xperia arc S MTP", 0x014f,
       DEVICE_FLAG_NONE },
@@ -1503,6 +1545,8 @@
       DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "MK16i Xperia MTP", 0x015a,
       DEVICE_FLAG_NONE },
+  { "SonyEricsson", 0x0fce, "R800/R88i Xperia Play MTP", 0x015d,
+      DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "ST18a Xperia Ray MTP", 0x0161,
       DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro MTP", 0x0166,
@@ -1533,12 +1577,26 @@
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "ST27i/ST27a Xperia go MTP", 0x017e,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "ST23i Xperia Miro MTP", 0x0180,
+      DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "SO-05D Xperia SX MTP", 0x0181,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT30p Xperia T MTP", 0x0182,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT25i Xperia V MTP", 0x0186,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia J MTP", 0x0188,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia ZL MTP", 0x0189,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia E MTP", 0x018c,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x018D,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Z MTP", 0x0193,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x0194,
+      DEVICE_FLAG_NONE },
 
   /*
    * MTP+UMS personalities of MTP devices (see above)
@@ -1565,12 +1623,26 @@
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "ST27i/ST27a Xperia go MTP+CDROM", 0x417e,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "ST23i Xperia Miro MTP+CDROM", 0x4180,
+      DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "SO-05D Xperia SX MTP+CDROM", 0x4181,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT30p Xperia T MTP+CDROM", 0x4182,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT25i Xperia V MTP+CDROM", 0x4186,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia J MTP+CDROM", 0x4188,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia ZL MTP", 0x4189,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia E MTP+CDROM", 0x418c,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x418d,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Z MTP", 0x4193,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x4194,
+      DEVICE_FLAG_NONE },
 
   /*
    * MTP+ADB personalities of MTP devices (see above)
@@ -1579,20 +1651,20 @@
       DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "MT11i Xperia Neo MTP+ADB", 0x5156,
       DEVICE_FLAG_NONE },
-  { "SonyEricsson", 0x0fce, "ST17i Xperia Active MTP+ADB", 0x5168,
-      DEVICE_FLAG_NONE },
-  { "SONY", 0x0fce, "LT26i Xperia S MTP+ADB", 0x5169,
-      DEVICE_FLAG_NO_ZERO_READS },
   { "SonyEricsson", 0x0fce, "MK16i Xperia MTP+ADB", 0x515a,
       DEVICE_FLAG_NONE },
+  { "SonyEricsson", 0x0fce, "R800/R88i Xperia Play MTP+ADB", 0x515d,
+      DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "ST18i Xperia Ray MTP+ADB", 0x5161,
       DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro MTP+ADB", 0x5166,
       DEVICE_FLAG_NONE },
   { "SonyEricsson", 0x0fce, "ST15i Xperia Mini MTP+ADB", 0x5167,
       DEVICE_FLAG_NONE },
-  { "SonyEricsson", 0x0fce, "LT26i Xperia S MTP+ADB", 0x5169,
+  { "SonyEricsson", 0x0fce, "ST17i Xperia Active MTP+ADB", 0x5168,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "LT26i Xperia S MTP+ADB", 0x5169,
+      DEVICE_FLAG_NO_ZERO_READS },
   { "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro MTP+ADB", 0x516d,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "ST21i Xperia Tipo MTP+ADB", 0x5170,
@@ -1613,12 +1685,26 @@
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "ST27i/ST27a Xperia go MTP+ADB", 0x517e,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "ST23i Xperia Miro MTP+ADB", 0x5180,
+      DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "SO-05D Xperia SX MTP+ADB", 0x5181,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT30p Xperia T MTP+ADB", 0x5182,
       DEVICE_FLAG_NONE },
   { "SONY", 0x0fce, "LT25i Xperia V MTP+ADB", 0x5186,
       DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia J MTP+ADB", 0x5188,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia ZL MTP", 0x5189,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia E MTP+ADB", 0x518c,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x518d,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Z MTP", 0x5193,
+      DEVICE_FLAG_NONE },
+  { "SONY", 0x0fce, "Xperia Tablet Z MTP", 0x5194,
+      DEVICE_FLAG_NONE },
 
   /*
    * MTP+UMS modes
@@ -1661,17 +1747,23 @@
   { "Motorola", 0x22b8, "V3m/V750 verizon", 0x2a65,
       DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST |
       DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL },
+  { "Motorola", 0x22b8, "Atrix/Razr HD (MTP)", 0x2e32,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "Atrix/Razr HD (MTP+ADB)", 0x2e33,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "RAZR M (XT907)", 0x2e51,
+      DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Jader Rodrigues Simoes <jadersimoes@users.sourceforge.net>
   { "Motorola", 0x22b8, "Xoom 2 Media Edition (ID2)", 0x41cf,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Steven Roemen <sdroemen@users.sourceforge.net>
   { "Motorola", 0x22b8, "Droid X/MB525 (Defy)", 0x41d6,
       DEVICE_FLAG_NONE },
-  // Reported by anonymous user
+  { "Motorola", 0x22b8, "DROID2 (ID1)", 0x41da,
+      DEVICE_FLAG_NONE },
   { "Motorola", 0x22b8, "Milestone / Verizon Droid", 0x41dc,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous user
+  { "Motorola", 0x22b8, "DROID2 (ID2)", 0x42a7,
-  { "Motorola", 0x22b8, "DROID2", 0x42a7,
       DEVICE_FLAGS_ANDROID_BUGS },
   { "Motorola", 0x22b8, "Xoom 2 Media Edition", 0x4311,
       DEVICE_FLAGS_ANDROID_BUGS },
@@ -1693,6 +1785,9 @@
   // Reported by anonymous user
   { "Motorola", 0x22b8, "RAZR2 V8/U9/Z6", 0x6415,
       DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST },
+  // Reported by Brian Dolbec <dol-sen@users.sourceforge.net>
+  { "Motorola", 0x22b8, "Atrix MB860 (MTP)", 0x7088,
+      DEVICE_FLAGS_ANDROID_BUGS },
   /*
    * Motorola Xoom (Wingray) variants
    *
@@ -1719,9 +1814,15 @@
       DEVICE_FLAGS_ANDROID_BUGS },
   { "Motorola", 0x22b8, "Xoom (MTP+ADB)", 0x70a9,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous Sourceforge user
   // "carried by C Spire and other CDMA US carriers"
-  { "Motorola", 0x22b8, "Milestone X2", 0x70ca, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "Milestone X2", 0x70ca,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "XT890/907 (MTP)", 0x710d,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "XT890/907 (MTP+ADB)", 0x710e,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Motorola", 0x22b8, "XT890/907 (MTP+?)", 0x710f,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * Google
@@ -1729,23 +1830,32 @@
    * road to produce an Android tablet it seems... The Vendor ID
    * was originally used for Nexus phones
    */
+  { "Google Inc (for Ainol Novo)", 0x18d1, "Fire/Flame", 0x0007,
+      DEVICE_FLAGS_ANDROID_BUGS },
   { "Google Inc (for Sony)", 0x18d1, "S1", 0x05b3,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by anonymous Sourceforge user
   { "Google Inc (for Barnes & Noble)", 0x18d1, "Nook Color", 0x2d02,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by anonymous Sourceforge user
+  { "Google Inc (for Asus)", 0x18d1, "TF201 Transformer", 0x4d00,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  // Reported by anonymous Sourceforge user
   { "Google Inc (for Asus)", 0x18d1, "TF101 Transformer", 0x4e0f,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Laurent Artaud <laurenta@users.sourceforge.net>
+  // 0x4e21 (Nexus S) is a USB Mass Storage device.
-  { "Google Inc (for Samsung)", 0x18d1, "Nexus S", 0x4e21,
-      DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Chris Smith <tcgsmythe@users.sourceforge.net>
   { "Google Inc (for Asus)", 0x18d1, "Nexus 7 (MTP)", 0x4e41,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Michael Hess <mhess126@gmail.com>
   { "Google Inc (for Asus)", 0x18d1, "Nexus 7 (MTP+ADB)", 0x4e42,
       DEVICE_FLAGS_ANDROID_BUGS },
+  { "Google Inc (for LG Electronics/Samsung)", 0x18d1,
+      "Nexus 4/10 (MTP)", 0x4ee1,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Google Inc (for LG Electronics/Samsung)", 0x18d1,
+      "Nexus 4/10 (MTP+ADB)", 0x4ee2,
+      DEVICE_FLAGS_ANDROID_BUGS },
   // WiFi-only version of Xoom
   // See: http://bugzilla.gnome.org/show_bug.cgi?id=647506
   { "Google Inc (for Motorola)", 0x18d1, "Xoom (MZ604)", 0x70a8,
@@ -1845,9 +1955,16 @@
    */
   { "Coby", 0x1e74, "COBY MP705", 0x6512, DEVICE_FLAG_NONE },
 
+#if 0
   /*
    * Apple devices, which are not MTP natively but can be made to speak MTP
    * using PwnTunes (http://www.pwntunes.net/)
+   * CURRENTLY COMMENTED OUT:
+   * These will make the UDEV rules flag these as MTP devices even if
+   * PwnTunes is NOT installed. That is unacceptable, so a better solution
+   * that actually inspects if the device has PwnTunes/MTP support needs
+   * to be found, see:
+   * https://sourceforge.net/p/libmtp/bugs/759/
    */
   { "Apple", 0x05ac, "iPhone", 0x1290, DEVICE_FLAG_NONE },
   { "Apple", 0x05ac, "iPod Touch 1st Gen", 0x1291, DEVICE_FLAG_NONE },
@@ -1859,6 +1976,7 @@
   { "Apple", 0x05ac, "0x1298", 0x1298, DEVICE_FLAG_NONE },
   { "Apple", 0x05ac, "iPod Touch 3rd Gen", 0x1299, DEVICE_FLAG_NONE },
   { "Apple", 0x05ac, "iPad", 0x129a, DEVICE_FLAG_NONE },
+#endif
 
   // Reported by anonymous SourceForge user, also reported as
   // Pantech Crux, claming to be:
@@ -1872,30 +1990,48 @@
 
   /*
    * Asus
+   * Pattern of PIDs on Android devices seem to be:
+   * n+0 = MTP
+   * n+1 = MTP+ADB
+   * n+2 = ?
+   * n+3 = ?
+   * n+4 = PTP
    */
   // Reported by Glen Overby
-  { "Asus", 0x0b05, "TF300 Transformer", 0x4c80,
+  { "Asus", 0x0b05, "TF300 Transformer (MTP)", 0x4c80,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by jaile <jaile@users.sourceforge.net>
-  { "Asus", 0x0b05, "TF300 Transformer (USB debug mode)", 0x4c81,
+  { "Asus", 0x0b05, "TF300 Transformer (MTP+ADB)", 0x4c81,
       DEVICE_FLAGS_ANDROID_BUGS },
   // Repored by Florian Apolloner <f-apolloner@users.sourceforge.net>
-  { "Asus", 0x0b05, "TF700 Transformer", 0x4c90,
+  { "Asus", 0x0b05, "TF700 Transformer (MTP)", 0x4c90,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "TF700 Transformer (MTP+ADB)", 0x4c91,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "MeMo Pad Smart 10", 0x4cd0,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous Sourceforge user
   { "Asus", 0x0b05, "TF201 Transformer Prime (keyboard dock)", 0x4d00,
       DEVICE_FLAGS_ANDROID_BUGS },
   { "Asus", 0x0b05, "TF201 Transformer Prime (tablet only)", 0x4d01,
       DEVICE_FLAGS_ANDROID_BUGS },
-  { "Asus", 0x0b05, "TFXXX Transformer Prime (unknown version)", 0x4d04,
+  // 4d04 is the PTP mode, don't add it
+  { "Asus", 0x0b05, "SL101 (MTP)", 0x4e00,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous Sourceforge user
+  { "Asus", 0x0b05, "SL101 (MTP+ADB)", 0x4e01,
-  { "Asus", 0x0b05, "TF101 Eeepad Slider", 0x4e01,
       DEVICE_FLAGS_ANDROID_BUGS },
-  { "Asus", 0x0b05, "TF101 Eeepad Transformer", 0x4e0f,
+  { "Asus", 0x0b05, "TF101 Eeepad Transformer (MTP)", 0x4e0f,
       DEVICE_FLAGS_ANDROID_BUGS },
-  { "Asus", 0x0b05, "TF101 Eeepad Transformer (debug mode)", 0x4e1f,
+  { "Asus", 0x0b05, "TF101 Eeepad Transformer (MTP+ADB)", 0x4e1f,
       DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "PadFone (MTP)", 0x5200,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "PadFone (MTP+ADB)", 0x5201,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "PadFone 2 (MTP+?)", 0x5210,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Asus", 0x0b05, "PadFone 2 (MTP)", 0x5211,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
 
 
   /*
@@ -1914,17 +2050,26 @@
   // Reported by: anonymous sourceforge user
   { "Lenovo", 0x17ef, "Lifetab S9512", 0x74cc,
       DEVICE_FLAGS_ANDROID_BUGS },
+  // Reported by Brian J. Murrell
+  { "Lenovo", 0x17ef, "IdeaTab A2109A", 0x7542,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
 
   /*
    * Huawei
    */
   // Reported by anonymous SourceForge user
-  { "Huawei", 0x12d1, "Honor U8860", 0x1051, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Huawei", 0x12d1, "Honor U8860", 0x1051,
+      DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by anonymous SourceForge user
-  { "Huawei", 0x12d1, "Mediapad (mode 0)", 0x360f, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Huawei", 0x12d1, "U8815/U9200", 0x1052,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  // Reported by anonymous SourceForge user
+  { "Huawei", 0x12d1, "Mediapad (mode 0)", 0x360f,
+      DEVICE_FLAGS_ANDROID_BUGS },
   // Reported by Bearsh <bearsh@users.sourceforge.net>
-  { "Huawei", 0x12d1, "Mediapad (mode 1)", 0x361f, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Huawei", 0x12d1, "Mediapad (mode 1)", 0x361f,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * ZTE
@@ -1932,27 +2077,61 @@
    */
   { "ZTE", 0x19d2, "V55 ID 1", 0x0244, DEVICE_FLAGS_ANDROID_BUGS },
   { "ZTE", 0x19d2, "V55 ID 2", 0x0245, DEVICE_FLAGS_ANDROID_BUGS },
+  { "ZTE", 0x19d2, "v790/Blade 3", 0x0306, DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * HTC (High Tech Computer Corp)
+   * Reporters:
+   * Steven Eastland <grassmonk@users.sourceforge.net>
+   * Kevin Cheng <kache@users.sf.net>
    */
-  { "HTC", 0x0bb4, "Zopo ZP100 (ID1)", 0x0c02,
+#if 0
+  /*
+   * This had to be commented out - the same VID+PID is used also for
+   * other modes than MTP, so we need to let mtp-probe do its job on this
+   * device instead of adding it to the database.
+   */
+  { "HTC", 0x0bb4, "Android Device ID1 (Zopo, HD2, Bird...)", 0x0c02,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Steven Eastland <grassmonk@users.sourceforge.net>
+#endif
-  { "HTC", 0x0bb4, "EVO 4G LTE", 0x0c93,
+  { "HTC", 0x0bb4, "EVO 4G LTE/One V (ID1)", 0x0c93,
       DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by Steven Eastland <grassmonk@users.sourceforge.net>
+  { "HTC", 0x0bb4, "EVO 4G LTE/One V (ID2)", 0x0ca8,
-  { "HTC", 0x0bb4, "EVO 4G LTE (second ID)", 0x0ca8,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One S (ID1)", 0x0cec,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC Evo 4G LTE (ID1)", 0x0df5,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One S (ID2)", 0x0df9,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One X (ID1)", 0x0dfb,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One X (ID2)", 0x0dfc,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One X (ID3)", 0x0dfd,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC Butterfly (ID1)", 0x0dfe,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "Droid DNA (MTP+UMS+ADB)", 0x0dff,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC Droid Incredible 4G LTE (MTP)", 0x0e31,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC Droid Incredible 4G LTE (MTP+ADB)", 0x0e32,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "Droid DNA (MTP+UMS)", 0x0ebd,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "HTC", 0x0bb4, "HTC One X (ID2)", 0x0f91,
       DEVICE_FLAGS_ANDROID_BUGS },
   // These identify themselves as "cm_tenderloin", fun...
   // Done by HTC for HP I guess.
-  { "Hewlett-Packard", 0x0bb4, "HP Touchpad", 0x685c,
+  { "Hewlett-Packard", 0x0bb4, "HP Touchpad (MTP)", 0x685c,
       DEVICE_FLAGS_ANDROID_BUGS },
-  { "Hewlett-Packard", 0x0bb4, "HP Touchpad (debug mode)",
+  { "Hewlett-Packard", 0x0bb4, "HP Touchpad (MTP+ADB)", 0x6860,
-      0x6860, DEVICE_FLAGS_ANDROID_BUGS },
-  // Reported by anonymous SourceForge user
-  { "HTC", 0x0bb4, "Zopo ZP100 (ID2)", 0x2008,
       DEVICE_FLAGS_ANDROID_BUGS },
+#if 0
+  { "HTC", 0x0bb4, "Android Device ID2 (Zopo, HD2...)", 0x2008,
+      DEVICE_FLAGS_ANDROID_BUGS },
+#endif
 
   /*
    * NEC
@@ -1963,7 +2142,12 @@
    * nVidia
    */
   // Found on Internet forum
-  { "nVidia", 0x0955, "CM9-Adam", 0x70a9, DEVICE_FLAGS_ANDROID_BUGS },
+  { "nVidia", 0x0955, "CM9-Adam", 0x70a9,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "nVidia", 0x0955, "Nabi2 Tablet (ID1)", 0x7100,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "nVidia", 0x0955, "Nabi2 Tablet (ID2)", 0x7102,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * Vizio
@@ -1972,10 +2156,60 @@
   { "Vizio", 0x0489, "VTAB1008", 0xe040, DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
-   * Viewpia, bq...
+   * Amazon
-   * Seems like some multi-branded OEM product.
    */
-  { "Various", 0x2207, "Viewpia DR/bq Kepler", 0x0001, DEVICE_FLAGS_ANDROID_BUGS },
+  { "Amazon", 0x1949, "Kindle Fire 2G (ID1)", 0x0005,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Amazon", 0x1949, "Kindle Fire (ID1)", 0x0007,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Amazon", 0x1949, "Kindle Fire (ID2)", 0x0008,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Amazon", 0x1949, "Kindle Fire (ID3)", 0x000a,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Barnes&Noble
+   */
+  { "Barnes&Noble", 0x2080, "Nook HD+", 0x0005,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Viewpia, bq, YiFang
+   * Seems like some multi-branded OEM product line.
+   */
+  { "Various", 0x2207, "Viewpia DR/bq Kepler", 0x0001,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "YiFang", 0x2207, "BQ Tesla", 0x0006,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Kobo
+   */
+  // Reported by George Talusan
+  { "Kobo", 0x2237, "Arc (ID1)", 0xd108,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Kobo", 0x2237, "Arc (ID2)", 0xd109,
+      DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Hisense
+   */
+  // Reported by Anonymous SourceForge user
+  { "Hisense", 0x109b, "E860", 0x9109, DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Intel
+   * Also sold rebranded as Orange products
+   */
+  { "Intel", 0x8087, "Xolo 900/AZ210A", 0x09fb, DEVICE_FLAGS_ANDROID_BUGS },
+
+  /*
+   * Xiaomi
+   */
+  { "Xiaomi", 0x2717, "Mi-2 (MTP+ADB)", 0x9039,
+      DEVICE_FLAGS_ANDROID_BUGS },
+  { "Xiaomi", 0x2717, "Mi-2 (MTP)", 0xf003,
+      DEVICE_FLAGS_ANDROID_BUGS },
 
   /*
    * Other strange stuff.

src/calibre/devices/mtp/unix/upstream/update.py

@@ -7,14 +7,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-MP = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/music-players.h;hb=HEAD'
+import os, shutil, subprocess
-DF = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/device-flags.h;hb=HEAD'
-
-import urllib, os, shutil
 
 base = os.path.dirname(os.path.abspath(__file__))
 
-for url, fname in [(MP, 'music-players.h'), (DF, 'device-flags.h')]:
+os.chdir('/tmp')
-    with open(os.path.join(base, fname), 'wb') as f:
+if os.path.exists('libmtp'):
-        shutil.copyfileobj(urllib.urlopen(url), f)
+    shutil.rmtree('libmtp')
+subprocess.check_call(['git', 'clone', 'git://git.code.sf.net/p/libmtp/code',
+                       'libmtp'])
+for x in ('src/music-players.h', 'src/device-flags.h'):
+    with open(os.path.join(base, os.path.basename(x)), 'wb') as f:
+        shutil.copyfileobj(open('libmtp/'+x), f)