Add category of Recently added books when generating catalog in e-book format

resources/catalog/stylesheet.css

@@ -8,7 +8,7 @@ p.title  {
 	font-size:xx-large;
     border-bottom: solid black 4px;
 	}
-	
+
 p.author {
 	margin-top:0em;
 	margin-bottom:0em;
@@ -31,25 +31,33 @@ p.description {
 	margin-top: 0em;
 	}
 
+p.date_index {
+	font-size:x-large;
+	text-align:center;
+	font-weight:bold;
+	margin-top:1em;
+	margin-bottom:0px;
+	}
+
 p.letter_index {
 	font-size:x-large;
 	text-align:left;
 	margin-top:0px;
-	margin-bottom:0px;	
+	margin-bottom:0px;
 	}
 
 p.author_index {
 	font-size:large;
 	text-align:left;
 	margin-top:0px;
-	margin-bottom:0px;	
+	margin-bottom:0px;
 	text-indent: 0em;
 	}
 
 p.read_book {
 	text-align:left;
 	margin-top:0px;
-	margin-bottom:0px;	
+	margin-bottom:0px;
 	margin-left:2em;
 	text-indent:-2em;
 	}
@@ -57,8 +65,8 @@ p.read_book {
 p.unread_book {
 	text-align:left;
 	margin-top:0px;
-	margin-bottom:0px;	
+	margin-bottom:0px;
 	margin-left:2em;
 	text-indent:-2em;
 	}
-	
+

src/calibre/library/catalog.py

@@ -1,9 +1,10 @@
 import os, re, shutil, htmlentitydefs
 
 from collections import namedtuple
+from datetime import date
 from xml.sax.saxutils import escape
 
-from calibre import filesystem_encoding, prints
+from calibre import filesystem_encoding, prints, strftime
 from calibre.customize import CatalogPlugin
 from calibre.customize.conversion import OptionRecommendation, DummyReporter
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
@@ -94,6 +95,10 @@ class CSV_XML(CatalogPlugin):
                         item = ', '.join(fmt_list)
                     elif field in ['authors','tags']:
                         item = ', '.join(item)
+                    elif field == 'isbn':
+                        # Could be 9, 10 or 13 digits
+                        field = u'%s' % re.sub(r'[\D]','',field)
+
                     if x < len(fields) - 1:
                         if item is not None:
                             outstr += u'"%s",' % unicode(item).replace('"','""')
@@ -483,9 +488,6 @@ class EPUB_MOBI(CatalogPlugin):
         current_step = 0.0
         total_steps = 13.0
 
-        # Used to xlate pubdate to friendly format
-        MONTHS = ['January', 'February','March','April','May','June',
-                      'July','August','September','October','November','December']
         THUMB_WIDTH = 75
         THUMB_HEIGHT = 100
 
@@ -763,11 +765,14 @@ class EPUB_MOBI(CatalogPlugin):
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateHTMLDescriptions()
 
+            if getattr(self.reporter, 'cancel_requested', False): return 1
+            self.generateHTMLByAuthor()
+
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateHTMLByTitle()
 
             if getattr(self.reporter, 'cancel_requested', False): return 1
-            self.generateHTMLByAuthor()
+            self.generateHTMLByDateAdded()
 
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateHTMLByTags()
@@ -786,11 +791,14 @@ class EPUB_MOBI(CatalogPlugin):
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateNCXDescriptions("Descriptions")
 
+            if getattr(self.reporter, 'cancel_requested', False): return 1
+            self.generateNCXByAuthor("Authors")
+
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateNCXByTitle("Titles")
 
             if getattr(self.reporter, 'cancel_requested', False): return 1
-            self.generateNCXByAuthor("Authors")
+            self.generateNCXByDateAdded("Recently Added")
 
             if getattr(self.reporter, 'cancel_requested', False): return 1
             self.generateNCXByTags("Genres")
@@ -867,10 +875,8 @@ class EPUB_MOBI(CatalogPlugin):
                     this_title['publisher'] = re.sub('&', '&amp;', record['publisher'])
 
                 this_title['rating'] = record['rating'] if record['rating'] else 0
-                # <pubdate>2009-11-05 09:29:37</pubdate>
+                this_title['date'] = strftime(u'%b %Y', record['pubdate'].timetuple())
-                date_strings = str(record['pubdate']).split("-")
+                this_title['timestamp'] = record['timestamp']
-                this_title['date'] = '%s %s' % (self.MONTHS[int(date_strings[1])-1], date_strings[0])
-
                 if record['comments']:
                     this_title['description'] = re.sub('&', '&amp;', record['comments'])
                     this_title['short_description'] = self.generateShortDescription(this_title['description'])
@@ -1317,6 +1323,142 @@ class EPUB_MOBI(CatalogPlugin):
             outfile.close()
             self.htmlFileList.append("content/ByAlphaAuthor.html")
 
+        def generateHTMLByDateAdded(self):
+
+            def add_books_to_HTML(this_months_list, dtc):
+                if len(this_months_list):
+                    date_string = strftime(u'%b %Y', current_date.timetuple())
+                    this_months_list = sorted(this_months_list,
+                                        key=lambda x:(x['title_sort'], x['title_sort']))
+                    this_months_list = sorted(this_months_list,
+                                        key=lambda x:(x['author_sort'], x['author_sort']))
+                    prints("Books added in", date_string)
+
+                    # Create a new month anchor
+                    pIndexTag = Tag(soup, "p")
+                    pIndexTag['class'] = "date_index"
+                    aTag = Tag(soup, "a")
+                    aTag['name'] = "%s-%s" % (current_date.year, current_date.month)
+                    pIndexTag.insert(0,aTag)
+                    pIndexTag.insert(1,NavigableString('Books added in '+date_string))
+                    divTag.insert(dtc,pIndexTag)
+                    dtc += 1
+                    current_author = None
+
+                    for purchase in this_months_list:
+                        prints(u" %-40s \t %-20s \t %s" % (purchase['title'],
+                            purchase['author'], purchase['timestamp']))
+
+
+                        if purchase['author'] != current_author:
+                            # Start a new author
+                            current_author = purchase['author']
+                            pAuthorTag = Tag(soup, "p")
+                            pAuthorTag['class'] = "author_index"
+                            emTag = Tag(soup, "em")
+                            aTag = Tag(soup, "a")
+                            aTag['name'] = "%s" % self.generateAuthorAnchor(current_author)
+                            aTag.insert(0,NavigableString(current_author))
+                            emTag.insert(0,aTag)
+                            pAuthorTag.insert(0,emTag)
+                            divTag.insert(dtc,pAuthorTag)
+                            dtc += 1
+
+                        # Add books
+                        pBookTag = Tag(soup, "p")
+                        ptc = 0
+
+                        # Prefix book with read/unread symbol
+                        if purchase['read']:
+                            # check mark
+                            pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
+                            pBookTag['class'] = "read_book"
+                            ptc += 1
+                        else:
+                            # hidden check mark
+                            pBookTag['class'] = "unread_book"
+                            pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
+                            ptc += 1
+
+                        aTag = Tag(soup, "a")
+                        aTag['href'] = "book_%d.html" % (int(float(purchase['id'])))
+                        aTag.insert(0,escape(purchase['title']))
+                        pBookTag.insert(ptc, aTag)
+                        ptc += 1
+
+                        divTag.insert(dtc, pBookTag)
+                        dtc += 1
+                return dtc
+
+            # Write books by reverse chronological order
+            self.opts.log.info(self.updateProgressFullStep("generateHTMLByDateAdded()"))
+
+            # Sort titles case-insensitive
+            self.booksByDate = sorted(self.booksByTitle,
+                                 key=lambda x:(x['timestamp'], x['timestamp']),reverse=True)
+
+            friendly_name = "Recently Added"
+
+            soup = self.generateHTMLEmptyHeader(friendly_name)
+            body = soup.find('body')
+
+            btc = 0
+
+            # Insert section tag
+            aTag = Tag(soup,'a')
+            aTag['name'] = 'section_start'
+            body.insert(btc, aTag)
+            btc += 1
+
+            # Insert the anchor
+            aTag = Tag(soup, "a")
+            anchor_name = friendly_name.lower()
+            aTag['name'] = anchor_name.replace(" ","")
+            body.insert(btc, aTag)
+            btc += 1
+            '''
+            # We don't need this because the kindle inserts section titles
+            #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2>
+            h2Tag = Tag(soup, "h2")
+            aTag = Tag(soup, "a")
+            anchor_name = friendly_name.lower()
+            aTag['name'] = anchor_name.replace(" ","")
+            h2Tag.insert(0,aTag)
+            h2Tag.insert(1,NavigableString('%s' % friendly_name))
+            body.insert(btc,h2Tag)
+            btc += 1
+            '''
+
+            # <p class="letter_index">
+            # <p class="author_index">
+            divTag = Tag(soup, "div")
+            dtc = 0
+
+            current_date = date.fromordinal(1)
+
+            # Loop through books by date
+            this_months_list = []
+            for book in self.booksByDate:
+                if book['timestamp'].month != current_date.month or \
+                   book['timestamp'].year != current_date.year:
+                    dtc = add_books_to_HTML(this_months_list, dtc)
+                    this_months_list = []
+                    current_date = book['timestamp'].date()
+                this_months_list.append(book)
+
+            # Add the last month's list
+            add_books_to_HTML(this_months_list, dtc)
+
+            # Add the divTag to the body
+            body.insert(btc, divTag)
+
+            # Write the generated file to contentdir
+            outfile_spec = "%s/ByDateAdded.html" % (self.contentDir)
+            outfile = open(outfile_spec, 'w')
+            outfile.write(soup.prettify())
+            outfile.close()
+            self.htmlFileList.append("content/ByDateAdded.html")
+
         def generateHTMLByTags(self):
             # Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ...
             # Note that special tags - ~+*[] -  have already been filtered from books[]
@@ -1894,6 +2036,97 @@ class EPUB_MOBI(CatalogPlugin):
 
             self.ncxSoup = soup
 
+        def generateNCXByDateAdded(self, tocTitle):
+
+            self.opts.log.info(self.updateProgressFullStep("generateNCXByDateAdded()"))
+
+            soup = self.ncxSoup
+            HTML_file = "content/ByDateAdded.html"
+            body = soup.find("navPoint")
+            btc = len(body.contents)
+
+            # --- Construct the 'Recently Added' *section* ---
+            navPointTag = Tag(soup, 'navPoint')
+            navPointTag['class'] = "section"
+            file_ID = "%s" % tocTitle.lower()
+            file_ID = file_ID.replace(" ","")
+            navPointTag['id'] = "%s-ID" % file_ID
+            navPointTag['playOrder'] = self.playOrder
+            self.playOrder += 1
+            navLabelTag = Tag(soup, 'navLabel')
+            textTag = Tag(soup, 'text')
+            textTag.insert(0, NavigableString('%s' % tocTitle))
+            navLabelTag.insert(0, textTag)
+            nptc = 0
+            navPointTag.insert(nptc, navLabelTag)
+            nptc += 1
+            contentTag = Tag(soup,"content")
+            contentTag['src'] = "%s#section_start" % HTML_file
+            navPointTag.insert(nptc, contentTag)
+            nptc += 1
+
+            # Create an NCX article entry for each populated month
+            # Loop over the booksByDate list, find start of each month,
+            # add description_preview_count titles
+            # self.authors[0]:friendly [1]:author_sort [2]:book_count
+            current_titles_list = []
+            master_month_list = []
+            current_date = self.booksByDate[0]['timestamp']
+
+            for book in self.booksByDate:
+                if book['timestamp'].month != current_date.month or \
+                   book['timestamp'].year != current_date.year:
+                    # Save the old lists
+                    current_titles_list = " &bull; ".join(current_titles_list)
+
+                    current_titles_list = self.formatNCXText(current_titles_list)
+                    master_month_list.append((current_titles_list, current_date))
+
+                    # Start the new list
+                    current_date = book['timestamp'].date()
+                    current_titles_list = [book['title']]
+                else:
+                    if len(current_titles_list) < self.descriptionClip:
+                        current_titles_list.append(book['title'])
+
+            # Add the last author list
+            current_titles_list = " &bull; ".join(current_titles_list)
+            master_month_list.append((current_titles_list, current_date))
+
+            # Add *article* entries for each populated author initial letter
+            # master_months_list{}: [0]:titles list [1]:date
+            for books_by_month in master_month_list:
+                datestr = strftime(u'%b %Y', books_by_month[1].timetuple())
+                navPointByLetterTag = Tag(soup, 'navPoint')
+                navPointByLetterTag['class'] = "article"
+                navPointByLetterTag['id'] = "%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month )
+                navPointTag['playOrder'] = self.playOrder
+                self.playOrder += 1
+                navLabelTag = Tag(soup, 'navLabel')
+                textTag = Tag(soup, 'text')
+                textTag.insert(0, NavigableString("Books added in " + datestr))
+                navLabelTag.insert(0, textTag)
+                navPointByLetterTag.insert(0,navLabelTag)
+                contentTag = Tag(soup, 'content')
+                contentTag['src'] = "%s#%s-%s" % (HTML_file,
+                    books_by_month[1].year,books_by_month[1].month)
+
+                navPointByLetterTag.insert(1,contentTag)
+
+                if self.generateForKindle:
+                    cmTag = Tag(soup, '%s' % 'calibre:meta')
+                    cmTag['name'] = "description"
+                    cmTag.insert(0, NavigableString(books_by_month[0]))
+                    navPointByLetterTag.insert(2, cmTag)
+
+                navPointTag.insert(nptc, navPointByLetterTag)
+                nptc += 1
+
+            # Add this section to the body
+            body.insert(btc, navPointTag)
+            btc += 1
+            self.ncxSoup = soup
+
         def generateNCXByTags(self, tocTitle):
             # Create an NCX section for 'By Genre'
             # Add each genre as an article

src/calibre/web/feeds/news.py

@@ -760,6 +760,7 @@ class BasicNewsRecipe(Recipe):
         self.report_progress(0, _('Trying to download cover...'))
         self.download_cover()
         self.report_progress(0, _('Generating masthead...'))
+        self.masthead_path = None
         try:
             murl = self.get_masthead_url()
         except:
@@ -767,7 +768,7 @@ class BasicNewsRecipe(Recipe):
             murl = None
         if murl is not None:
             self.download_masthead(murl)
-        else:
+        if self.masthead_path is None:
             self.masthead_path = os.path.join(self.output_dir, 'mastheadImage.jpg')
             self.default_masthead_image(self.masthead_path)
 
@@ -911,8 +912,7 @@ class BasicNewsRecipe(Recipe):
         try:
             self._download_masthead(url)
         except:
-            self.log.exception('Failed to download masthead')
+            self.log.exception("Failed to download supplied masthead_url, synthesizing")
-
 
     def default_cover(self, cover_file):
         '''
@@ -981,6 +981,9 @@ class BasicNewsRecipe(Recipe):
         'Override in subclass to use something other than the recipe title'
         return self.title
 
+    MI_WIDTH = 600
+    MI_HEIGHT = 60
+
     def default_masthead_image(self, out_path):
         try:
             from PIL import Image, ImageDraw, ImageFont
@@ -988,14 +991,13 @@ class BasicNewsRecipe(Recipe):
         except ImportError:
             import Image, ImageDraw, ImageFont
 
-
+        img = Image.new('RGB', (self.MI_WIDTH, self.MI_HEIGHT), 'white')
-        img = Image.new('RGB', (600, 100), 'white')
         draw = ImageDraw.Draw(img)
         font = ImageFont.truetype(P('fonts/liberation/LiberationSerif-Bold.ttf'), 48)
         text = self.get_masthead_title().encode('utf-8')
         width, height = draw.textsize(text, font=font)
-        left = max(int((600 - width)/2.), 0)
+        left = max(int((self.MI_WIDTH - width)/2.), 0)
-        top = max(int((100 - height)/2.), 0)
+        top = max(int((self.MI_HEIGHT - height)/2.), 0)
         draw.text((left, top), text, fill=(0,0,0), font=font)
         img.save(open(out_path, 'wb'), 'JPEG')
 
@@ -1018,10 +1020,10 @@ class BasicNewsRecipe(Recipe):
                         %(path_to_image, msg))
             pw.PixelSetColor(p, 'white')
             width, height = pw.MagickGetImageWidth(img),pw.MagickGetImageHeight(img)
-            scaled, nwidth, nheight = fit_image(width, height, 600, 100)
+            scaled, nwidth, nheight = fit_image(width, height, self.MI_WIDTH, self.MI_HEIGHT)
             if not pw.MagickNewImage(img2, width, height, p):
                 raise RuntimeError('Out of memory')
-            if not pw.MagickNewImage(frame, 600, 100, p):
+            if not pw.MagickNewImage(frame,  self.MI_WIDTH, self.MI_HEIGHT, p):
                 raise RuntimeError('Out of memory')
             if not pw.MagickCompositeImage(img2, img, pw.OverCompositeOp, 0, 0):
                 raise RuntimeError('Out of memory')
@@ -1029,8 +1031,8 @@ class BasicNewsRecipe(Recipe):
                 if not pw.MagickResizeImage(img2, nwidth, nheight, pw.LanczosFilter,
                         0.5):
                     raise RuntimeError('Out of memory')
-            left = int((600 - nwidth)/2.0)
+            left = int((self.MI_WIDTH - nwidth)/2.0)
-            top = int((100 - nheight)/2.0)
+            top = int((self.MI_HEIGHT - nheight)/2.0)
             if not pw.MagickCompositeImage(frame, img2, pw.OverCompositeOp,
                     left, top):
                 raise RuntimeError('Out of memory')