Sync to trunk.

resources/recipes/national_post.recipe

@@ -1,20 +1,81 @@
-from calibre.web.feeds.news import BasicNewsRecipe
+import string, re
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
 
-class AdvancedUserRecipe1261379503(BasicNewsRecipe):
-    title = u'National Post'
-    language = 'en_CA'
-    __author__ = 'Nick Redding'
-    description = u"News from Canada"
-    oldest_article = 2
-    max_articles_per_feed = 25
+class NYTimes(BasicNewsRecipe):
 
-    keep_only_tags = [dict(name='div', attrs={'id':'content'})]
-    remove_tags = [dict(name='div', attrs={'class':'story-tools'}),dict(name='div', attrs={'class':'newsblock'}),dict(name='p', attrs={'class':'border-top'}),dict(name='div', attrs={'id':'footer'})]
+    title       = 'National Post'
+    __author__  = 'Krittika Goyal'
+    description = 'Canadian national newspaper'
+    timefmt = ' [%d %b, %Y]'
+    needs_subscription = False
+    
+    no_stylesheets = True
+    #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
+    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags = [
+       dict(name='iframe'),
+       dict(name='div', attrs={'class':'story-tools'}),
+       #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
+       #dict(name='form', attrs={'onsubmit':''}),
+       #dict(name='table', attrs={'cellspacing':'0'}),	
+    ]    
 
-    feeds = [(u'News Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| news'),
-    (u'FP Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| financial%20post|storytype|business'),
-    (u'Arts & Life Headlines', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| arts%20%26%20life|storytype|news'),
-    (u'Canada News', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=storytyp e|webcanada&feed=rss'),
-    (u'World News', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=storytyp e|webworld&feed=rss'),(u'Editorial', u'http://www.nationalpost.com/scripts/sp6query.aspx?catalog=ntnp&type=stry&tags=section| editorial'),
-    (u'FP Opinion', u'http://www.nationalpost.com/scripts/columnists.aspx?publication=national+post&columnty pe=fp')]
+   # def preprocess_html(self, soup):
+        # table = soup.find('table')
+        # if table is not None:
+            # table.extract()
+        # return soup
 
+
+
+    #TO GET ARTICLE TOC
+    def nejm_get_index(self):
+            return self.index_to_soup('http://www.nationalpost.com/todays-paper/index.html')
+    
+    # To parse artice toc
+    def parse_index(self):
+            soup = self.nejm_get_index()
+
+            div = soup.find(id='LegoText4')
+
+            current_section = None
+            current_articles = []
+            feeds = []
+            for x in div.findAll(True):
+                if x.name == 'h4':
+                    # Section found
+                    if current_articles and current_section:
+                        feeds.append((current_section, current_articles))
+                    current_section = self.tag_to_string(x)
+                    current_articles = []
+                    self.log('\tFound section:', current_section)
+                if current_section is not None and x.name == 'h3':
+                    # Article found
+                    title = self.tag_to_string(x)
+                    a = x.find('a', href=lambda x: x and 'story' in x)
+                    if a is None:
+                        continue
+                    url = a.get('href', False)
+                    if not url or not title:
+                        continue
+                    if url.startswith('story'):
+                         url = 'http://www.nationalpost.com/todays-paper/'+url
+                    self.log('\t\tFound article:', title)
+                    self.log('\t\t\t', url)
+                    current_articles.append({'title': title, 'url':url,
+                        'description':'', 'date':''})
+
+            if current_articles and current_section:
+                feeds.append((current_section, current_articles))
+
+            return feeds
+    def preprocess_html(self, soup):
+        story = soup.find(name='div', attrs={'class':'triline'})
+        #td = heading.findParent(name='td')
+        #td.extract()
+        soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
+        body = soup.find(name='body')
+        body.insert(0, story)
+        return soup

resources/recipes/nejm.recipe

@@ -0,0 +1,86 @@
+import string, re
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+
+class NYTimes(BasicNewsRecipe):
+
+    title       = 'New England Journal of Medicine'
+    __author__  = 'Krittika Goyal'
+    description = 'Medical news'
+    timefmt = ' [%d %b, %Y]'
+    needs_subscription = True
+    
+    no_stylesheets = True
+    #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
+    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags = [
+       dict(name='iframe'),
+       #dict(name='div', attrs={'class':'related-articles'}),
+       #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
+       dict(name='form', attrs={'onsubmit':"return verifySearch(this.w,'Keyword, citation, or author')"}),
+       dict(name='table', attrs={'cellspacing':'0'}),	
+    ]    
+
+    def preprocess_html(self, soup):
+        table = soup.find('table')
+        if table is not None:
+            table.extract()
+        return soup
+
+    #TO LOGIN 
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open('http://content.nejm.org/cgi/login?uri=/')
+        br.select_form(nr=0)
+        br['username'] = self.username
+        br['code'] = self.password
+        response = br.submit()
+        raw = response.read()
+        if '<strong>Welcome' not in raw:
+            raise Exception('Login failed. Check your username and password')
+        return br
+
+    #TO GET ARTICLE TOC
+    def nejm_get_index(self):
+            return self.index_to_soup('http://content.nejm.org/current.dtl')
+    
+    # To parse artice toc
+    def parse_index(self):
+            soup = self.nejm_get_index()
+
+            div = soup.find(id='centerTOC')
+
+            current_section = None
+            current_articles = []
+            feeds = []
+            for x in div.findAll(True):
+                if x.name == 'img' and '/toc/' in x.get('src', '') and 'uarrow.gif' not in x.get('src', ''):
+                    # Section heading found
+                    if current_articles and current_section and 'Week in the' not in current_section:
+                        feeds.append((current_section, current_articles))
+                    current_section = x.get('alt')
+                    current_articles = []
+                    self.log('\tFound section:', current_section)
+                if current_section is not None and x.name == 'strong':
+                    title = self.tag_to_string(x)
+                    a = x.parent.find('a', href=lambda x: x and '/full/' in x)
+                    if a is None:
+                        continue
+                    url = a.get('href', False)
+                    if not url or not title:
+                        continue
+                    if url.startswith('/'):
+                         url = 'http://content.nejm.org'+url
+                    self.log('\t\tFound article:', title)
+                    self.log('\t\t\t', url)
+                    if url.startswith('/'):
+                        url = 'http://online.wsj.com'+url
+                    current_articles.append({'title': title, 'url':url,
+                        'description':'', 'date':''})
+
+            if current_articles and current_section:
+                feeds.append((current_section, current_articles))
+
+            return feeds
+

src/calibre/devices/usbms/device.py

@@ -260,7 +260,8 @@ class Device(DeviceConfig, DevicePlugin):
         # if the device is connected without a card, the above
         # will incorrectly identify the main mem as carda
         # See for example the driver for the Nook
-        if 'main' not in drives and 'carda' in drives:
+        if drives.get('carda', None) is not None and \
+                drives.get('main', None) is None:
             drives['main'] = drives.pop('carda')
 
         drives = self.windows_open_callback(drives)

src/calibre/ebooks/oeb/base.py

@@ -1806,7 +1806,7 @@ class OEBBook(object):
         return {OPF_MIME: ('content.opf', package)}
 
     def _update_playorder(self, ncx):
-        hrefs = set(xpath(ncx, '//ncx:content/@src'))
+        hrefs = set(map(urlnormalize, xpath(ncx, '//ncx:content/@src')))
         playorder = {}
         next = 1
         selector = XPath('h:body//*[@id or @name]')
@@ -1828,9 +1828,9 @@ class OEBBook(object):
                 if added:
                     next += 1
         selector = XPath('ncx:content/@src')
-        for elem in xpath(ncx, '//*[@playOrder and ./ncx:content[@src]]'):
-            href = selector(elem)[0]
-            order = playorder.get(href, 0)
+        for i, elem in enumerate(xpath(ncx, '//*[@playOrder and ./ncx:content[@src]]')):
+            href = urlnormalize(selector(elem)[0])
+            order = playorder.get(href, i)
             elem.attrib['playOrder'] = str(order)
         return
 

src/calibre/gui2/library.py

@@ -580,7 +580,7 @@ class BooksModel(QAbstractTableModel):
         def tags(r):
             tags = self.db.data[r][tgdx]
             if tags:
-                return ', '.join(tags.split(','))
+                return ', '.join(sorted(tags.split(',')))
 
         def series(r):
             series = self.db.data[r][srdx]

src/calibre/manual/faq.rst

@@ -86,7 +86,7 @@ At the moment |app| has full support for the SONY PRS 300/500/505/600/700/900, B
 How can I help get my device supported in |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If your device appears as a USB disk to the operating system. Adding support for it to |app| is very easy.
+If your device appears as a USB disk to the operating system, adding support for it to |app| is very easy.
 We just need some information from you:
 
   * What e-book formats does your device support?

src/calibre/web/feeds/recipes/collection.py

@@ -20,7 +20,8 @@ NS = 'http://calibre-ebook.com/recipe_collection'
 E = ElementMaker(namespace=NS, nsmap={None:NS})
 
 def iterate_over_builtin_recipe_files():
-    exclude = ['craigslist', 'iht', 'outlook_india', 'toronto_sun']
+    exclude = ['craigslist', 'iht', 'outlook_india', 'toronto_sun',
+            'indian_express', 'india_today', 'toi']
     d = os.path.dirname
     base = os.path.join(d(d(d(d(d(d(os.path.abspath(__file__))))))), 'resources', 'recipes')
     for x in os.walk(base):