Merge from trunk

recipes/calibre_blog.recipe

@@ -0,0 +1,18 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class CalibreBlog(BasicNewsRecipe):
+    title          = u'Calibre Blog'
+    language       = 'en'
+    __author__     = 'Krittika Goyal'
+    oldest_article = 1000 #days
+    max_articles_per_feed = 5
+    use_embedded_content = False
+
+    no_stylesheets = True
+    auto_cleanup = True
+
+
+    feeds          = [
+('Article',
+ 'http://blog.calibre-ebook.com/feeds/posts/default'),
+]

recipes/new_london_day.recipe

@@ -8,7 +8,7 @@ class AdvancedUserRecipe1294342201(BasicNewsRecipe):
     title          = u'New London Day'
     __author__  = 'Being'
     description = 'State, local and business news from New London, CT'
-    language = 'en_GB'
+    language = 'en'
     oldest_article = 1
     max_articles_per_feed = 200
 

recipes/silicon_republic.recipe

@@ -0,0 +1,22 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011 Neil Grogan'
+#
+# Silicon Republic Recipe
+#
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SiliconRepublic(BasicNewsRecipe):
+    title          = u'Silicon Republic'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    __author__ = u'Neil Grogan'
+    language = 'en_IE'
+
+    remove_tags = [dict(attrs={'class':['thumb','txt','compactbox','icons','catlist','catlistinner','taglist','taglistinner','social','also-in','also-in-inner','also-in-footer','zonek-dfp','paneladvert','rcadvert','panel','h2b']}),
+                dict(id=['header','logo','header-right','sitesearch','rsslinks','topnav','topvideos','topvideos-list','topnews','topnews-list','slideshow','slides','compactheader','compactnews','compactfeatures','article-type','contactlinks-header','banner-zone-k-dfp','footer-related','directory-services','also-in-section','featuredrelated1','featuredrelated2','featuredrelated3','featuredrelated4','advert2-dfp']),
+                dict(name=['script', 'style'])]
+
+
+    feeds = [(u'News', u'http://www.siliconrepublic.com/feeds/')]
+

src/calibre/devices/kobo/bookmark.py

@@ -0,0 +1,112 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Timothy Legge <timlegge@gmail.com> and Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from contextlib import closing
+
+import sqlite3 as sqlite
+
+class Bookmark(): # {{{
+    '''
+    A simple class fetching bookmark data
+    kobo-specific
+    '''
+    def __init__(self, db_path, contentid, path, id, book_format, bookmark_extension):
+        self.book_format = book_format
+        self.bookmark_extension = bookmark_extension
+        self.book_length = 0            # Not Used
+        self.id = id
+        self.last_read = 0
+        self.last_read_location = 0     # Not Used
+        self.path = path
+        self.timestamp = 0
+        self.user_notes = None
+        self.db_path = db_path
+        self.contentid = contentid
+        self.percent_read = 0
+        self.get_bookmark_data()
+        self.get_book_length()          # Not Used
+
+    def get_bookmark_data(self):
+        ''' Return the timestamp and last_read_location '''
+
+        user_notes = {}
+        self.timestamp = os.path.getmtime(self.path)
+        with closing(sqlite.connect(self.db_path)) as connection:
+            # return bytestrings if the content cannot the decoded as unicode
+            connection.text_factory = lambda x: unicode(x, "utf-8", "ignore")
+
+            cursor = connection.cursor()
+            t = (self.contentid,)
+
+            cursor.execute('select bm.bookmarkid, bm.contentid, bm.volumeid, '
+                                'bm.text, bm.annotation, bm.ChapterProgress, '
+                                'bm.StartContainerChildIndex, bm.StartOffset, c.BookTitle, '
+                                'c.TITLE, c.volumeIndex, c.___NumPages '
+                            'from Bookmark bm inner join Content c on '
+                                'bm.contentid = c.contentid and '
+                                'bm.volumeid = ? order by bm.volumeid, bm.chapterprogress', t)
+
+            previous_chapter = 0
+            bm_count = 0
+            for row in cursor:
+                current_chapter = row[10]
+                if previous_chapter == current_chapter:
+                    bm_count = bm_count + 1
+                else:
+                    bm_count = 0
+
+                text = row[3]
+                annotation = row[4]
+
+                # A dog ear (bent upper right corner) is a bookmark
+                if row[6] == row[7] == 0:   # StartContainerChildIndex = StartOffset = 0
+                    e_type = 'Bookmark'
+                    text = row[9]
+                # highlight is text with no annotation
+                elif text is not None and (annotation is None or annotation == ""):
+                    e_type = 'Highlight'
+                elif text and annotation:
+                    e_type = 'Annotation'
+                else:
+                    e_type = 'Unknown annotation type'
+
+                note_id = row[10] + bm_count
+                chapter_title = row[9]
+                # book_title = row[8]
+                chapter_progress = min(round(float(100*row[5]),2),100)
+                user_notes[note_id] = dict(id=self.id,
+                                        displayed_location=note_id,
+                                        type=e_type,
+                                        text=text,
+                                        annotation=annotation,
+                                        chapter=row[10],
+                                        chapter_title=chapter_title,
+                                        chapter_progress=chapter_progress)
+                previous_chapter = row[10]
+                # debug_print("e_type:" , e_type, '\t', 'loc: ', note_id, 'text: ', text,
+                        # 'annotation: ', annotation, 'chapter_title: ', chapter_title,
+                        # 'chapter_progress: ', chapter_progress, 'date: ')
+
+            cursor.execute('select datelastread, ___PercentRead from content '
+                                'where bookid is Null and '
+                                'contentid = ?', t)
+            for row in cursor:
+                self.last_read = row[0]
+                self.percent_read = row[1]
+                # print row[1]
+            cursor.close()
+
+#                self.last_read_location = self.last_read - self.pdf_page_offset
+        self.user_notes = user_notes
+
+
+    def get_book_length(self):
+#TL        self.book_length = 0
+#TL        self.book_length = int(unpack('>I', record0[0x04:0x08])[0])
+        pass
+
+# }}}

src/calibre/devices/kobo/driver.py

@@ -2,15 +2,16 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 
 __license__   = 'GPL v3'
-__copyright__ = '2010, Timothy Legge <timlegge at gmail.com> and Kovid Goyal <kovid@kovidgoyal.net>'
+__copyright__ = '2010, Timothy Legge <timlegge@gmail.com> and Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
+import os, time, calendar
 import sqlite3 as sqlite
 from contextlib import closing
 from calibre.devices.usbms.books import BookList
 from calibre.devices.kobo.books import Book
 from calibre.devices.kobo.books import ImageWrapper
+from calibre.devices.kobo.bookmark import Bookmark
 from calibre.devices.mime import mime_type_ext
 from calibre.devices.usbms.driver import USBMS, debug_print
 from calibre import prints
@@ -24,7 +25,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge'
-    version = (1, 0, 10)
+    version = (1, 0, 11)
 
     dbversion = 0
     fwversion = 0
@@ -47,6 +48,7 @@ class KOBO(USBMS):
 
     EBOOK_DIR_MAIN = ''
     SUPPORTS_SUB_DIRS = True
+    SUPPORTS_ANNOTATIONS = True
 
     VIRTUAL_BOOK_EXTENSIONS = frozenset(['kobo'])
 
@@ -77,11 +79,6 @@ class KOBO(USBMS):
         self.book_class = Book
         self.dbversion = 7
 
-    def create_annotations_path(self, mdata, device_path=None):
-        if device_path:
-            return device_path
-        return USBMS.create_annotations_path(self, mdata)
-
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext
 
@@ -111,6 +108,7 @@ class KOBO(USBMS):
 
         if self.fwversion != '1.0' and self.fwversion != '1.4':
             self.has_kepubs = True
+        debug_print('Version of driver: ', self.version, 'Has kepubs:', self.has_kepubs)
         debug_print('Version of firmware: ', self.fwversion, 'Has kepubs:', self.has_kepubs)
 
         self.booklist_class.rebuild_collections = self.rebuild_collections
@@ -893,3 +891,198 @@ class KOBO(USBMS):
                     tf.write(r.read())
                     paths[idx] = tf.name
         return paths
+
+    def create_annotations_path(self, mdata, device_path=None):
+        if device_path:
+            return device_path
+        return USBMS.create_annotations_path(self, mdata)
+
+    def get_annotations(self, path_map):
+        EPUB_FORMATS = [u'epub']
+        epub_formats = set(EPUB_FORMATS)
+
+        def get_storage():
+            storage = []
+            if self._main_prefix:
+                storage.append(os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN))
+            if self._card_a_prefix:
+                storage.append(os.path.join(self._card_a_prefix, self.EBOOK_DIR_CARD_A))
+            if self._card_b_prefix:
+                storage.append(os.path.join(self._card_b_prefix, self.EBOOK_DIR_CARD_B))
+            return storage
+
+        def resolve_bookmark_paths(storage, path_map):
+            pop_list = []
+            book_ext = {}
+            for id in path_map:
+                file_fmts = set()
+                for fmt in path_map[id]['fmts']:
+                    file_fmts.add(fmt)
+                bookmark_extension = None
+                if file_fmts.intersection(epub_formats):
+                    book_extension = list(file_fmts.intersection(epub_formats))[0]
+                    bookmark_extension = 'epub'
+
+                if bookmark_extension:
+                    for vol in storage:
+                        bkmk_path = path_map[id]['path']
+                        bkmk_path = bkmk_path
+                        if os.path.exists(bkmk_path):
+                            path_map[id] = bkmk_path
+                            book_ext[id] = book_extension
+                            break
+                    else:
+                        pop_list.append(id)
+                else:
+                    pop_list.append(id)
+
+            # Remove non-existent bookmark templates
+            for id in pop_list:
+                path_map.pop(id)
+            return path_map, book_ext
+
+        storage = get_storage()
+        path_map, book_ext = resolve_bookmark_paths(storage, path_map)
+
+        bookmarked_books = {}
+        for id in path_map:
+            extension =  os.path.splitext(path_map[id])[1]
+            ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(path_map[id])
+            ContentID = self.contentid_from_path(path_map[id], ContentType)
+
+            bookmark_ext = extension
+
+            db_path = self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite')
+            myBookmark = Bookmark(db_path, ContentID, path_map[id], id, book_ext[id], bookmark_ext)
+            bookmarked_books[id] = self.UserAnnotation(type='kobo_bookmark', value=myBookmark)
+
+        # This returns as job.result in gui2.ui.annotations_fetched(self,job)
+        return bookmarked_books
+
+    def generate_annotation_html(self, bookmark):
+        from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag, NavigableString
+        # Returns <div class="user_annotations"> ... </div>
+        #last_read_location = bookmark.last_read_location
+        #timestamp = bookmark.timestamp
+        percent_read = bookmark.percent_read
+        debug_print("Date: ",  bookmark.last_read)
+        if bookmark.last_read is not None:
+            try:
+                last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%S"))))
+            except:
+                last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(calendar.timegm(time.strptime(bookmark.last_read, "%Y-%m-%dT%H:%M:%S.%f"))))
+        else:
+            #self.datetime = time.gmtime()
+            last_read = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
+
+        # debug_print("Percent read: ", percent_read)
+        ka_soup = BeautifulSoup()
+        dtc = 0
+        divTag = Tag(ka_soup,'div')
+        divTag['class'] = 'user_annotations'
+
+        # Add the last-read location
+        spanTag = Tag(ka_soup, 'span')
+        spanTag['style'] = 'font-weight:normal'
+        if bookmark.book_format == 'epub':
+            spanTag.insert(0,NavigableString(
+                _("<hr /><b>Book Last Read:</b> %(time)s<br /><b>Percentage Read:</b> %(pr)d%%<hr />") % \
+                            dict(time=last_read,
+                            #loc=last_read_location,
+                            pr=percent_read)))
+        else:
+            spanTag.insert(0,NavigableString(
+                _("<hr /><b>Book Last Read:</b> %(time)s<br /><b>Percentage Read:</b> %(pr)d%%<hr />") % \
+                            dict(time=last_read,
+                            #loc=last_read_location,
+                            pr=percent_read)))
+
+        divTag.insert(dtc, spanTag)
+        dtc += 1
+        divTag.insert(dtc, Tag(ka_soup,'br'))
+        dtc += 1
+
+        if bookmark.user_notes:
+            user_notes = bookmark.user_notes
+            annotations = []
+
+            # Add the annotations sorted by location
+            for location in sorted(user_notes):
+                if user_notes[location]['type'] == 'Bookmark':
+                    annotations.append(
+                        _('<b>Chapter %(chapter)d:</b> %(chapter_title)s<br /><b>%(typ)s</b><br /><b>Chapter Progress:</b> %(chapter_progress)s%%<br />%(annotation)s<br /><hr />') % \
+                            dict(chapter=user_notes[location]['chapter'],
+                                dl=user_notes[location]['displayed_location'],
+                                typ=user_notes[location]['type'],
+                                chapter_title=user_notes[location]['chapter_title'],
+                                chapter_progress=user_notes[location]['chapter_progress'],
+                                annotation=user_notes[location]['annotation'] if user_notes[location]['annotation'] is not None else ""))
+                elif user_notes[location]['type'] == 'Highlight':
+                    annotations.append(
+                        _('<b>Chapter %(chapter)d:</b> %(chapter_title)s<br /><b>%(typ)s</b><br /><b>Chapter Progress:</b> %(chapter_progress)s%%<br /><b>Highlight:</b> %(text)s<br /><hr />') % \
+                            dict(chapter=user_notes[location]['chapter'],
+                                dl=user_notes[location]['displayed_location'],
+                                typ=user_notes[location]['type'],
+                                chapter_title=user_notes[location]['chapter_title'],
+                                chapter_progress=user_notes[location]['chapter_progress'],
+                                text=user_notes[location]['text']))
+                elif user_notes[location]['type'] == 'Annotation':
+                    annotations.append(
+                        _('<b>Chapter %(chapter)d:</b> %(chapter_title)s<br /><b>%(typ)s</b><br /><b>Chapter Progress:</b> %(chapter_progress)s%%<br /><b>Highlight:</b> %(text)s<br /><b>Notes:</b> %(annotation)s<br /><hr />') % \
+                            dict(chapter=user_notes[location]['chapter'],
+                                dl=user_notes[location]['displayed_location'],
+                                typ=user_notes[location]['type'],
+                                chapter_title=user_notes[location]['chapter_title'],
+                                chapter_progress=user_notes[location]['chapter_progress'],
+                                text=user_notes[location]['text'],
+                                annotation=user_notes[location]['annotation']))
+                else:
+                    annotations.append(
+                        _('<b>Chapter %(chapter)d:</b> %(chapter_title)s<br /><b>%(typ)s</b><br /><b>Chapter Progress:</b> %(chapter_progress)s%%<br /><b>Highlight:</b> %(text)s<br /><b>Notes:</b> %(annotation)s<br /><hr />') % \
+                            dict(chapter=user_notes[location]['chapter'],
+                                dl=user_notes[location]['displayed_location'],
+                                typ=user_notes[location]['type'],
+                                chapter_title=user_notes[location]['chapter_title'],
+                                chapter_progress=user_notes[location]['chapter_progress'],
+                                text=user_notes[location]['text'], \
+                                annotation=user_notes[location]['annotation']))
+
+            for annotation in annotations:
+                divTag.insert(dtc, annotation)
+                dtc += 1
+
+        ka_soup.insert(0,divTag)
+        return ka_soup
+
+    def add_annotation_to_library(self, db, db_id, annotation):
+        from calibre.ebooks.BeautifulSoup import Tag
+        bm = annotation
+        ignore_tags = set(['Catalog', 'Clippings'])
+
+        if bm.type == 'kobo_bookmark':
+            mi = db.get_metadata(db_id, index_is_id=True)
+            user_notes_soup = self.generate_annotation_html(bm.value)
+            if mi.comments:
+                a_offset = mi.comments.find('<div class="user_annotations">')
+                ad_offset = mi.comments.find('<hr class="annotations_divider" />')
+
+                if a_offset >= 0:
+                    mi.comments = mi.comments[:a_offset]
+                if ad_offset >= 0:
+                    mi.comments = mi.comments[:ad_offset]
+                if set(mi.tags).intersection(ignore_tags):
+                    return
+                if mi.comments:
+                    hrTag = Tag(user_notes_soup,'hr')
+                    hrTag['class'] = 'annotations_divider'
+                    user_notes_soup.insert(0, hrTag)
+
+                mi.comments += unicode(user_notes_soup.prettify())
+            else:
+                mi.comments = unicode(user_notes_soup.prettify())
+            # Update library comments
+            db.set_comment(db_id, mi.comments)
+
+            # Add bookmark file to db_id
+            db.add_format_with_hooks(db_id, bm.value.bookmark_extension,
+                                            bm.value.path, index_is_id=True)

src/calibre/ebooks/fb2/input.py

@@ -127,7 +127,7 @@ class FB2Input(InputFormatPlugin):
     def extract_embedded_content(self, doc):
         self.binary_map = {}
         for elem in doc.xpath('./*'):
-            if 'binary' in elem.tag and elem.attrib.has_key('id'):
+            if elem.text and 'binary' in elem.tag and elem.attrib.has_key('id'):
                 ct = elem.get('content-type', '')
                 fname = elem.attrib['id']
                 ext = ct.rpartition('/')[-1].lower()

src/calibre/ebooks/mobi/mobiml.py

@@ -138,6 +138,7 @@ class MobiMLizer(object):
             self.mobimlize_elem(body, stylizer, BlockState(nbody),
                                 [FormatState()])
             item.data = nroot
+            #print etree.tostring(nroot)
 
     def mobimlize_font(self, ptsize):
         return self.fnums[self.fmap[ptsize]]
@@ -233,9 +234,19 @@ class MobiMLizer(object):
         elif tag in TABLE_TAGS:
             para.attrib['valign'] = 'top'
         if istate.ids:
-            last = bstate.body[-1]
-            for id in istate.ids:
-                last.addprevious(etree.Element(XHTML('a'), attrib={'id': id}))
+            for id_ in istate.ids:
+                anchor = etree.Element(XHTML('a'), attrib={'id': id_})
+                if tag == 'li':
+                    try:
+                        last = bstate.body[-1][-1]
+                    except:
+                        break
+                    last.insert(0, anchor)
+                    anchor.tail = last.text
+                    last.text = None
+                else:
+                    last = bstate.body[-1]
+                    last.addprevious(anchor)
             istate.ids.clear()
         if not text:
             return

src/calibre/ebooks/mobi/writer2/main.py

@@ -601,7 +601,7 @@ class MobiWriter(object):
         Write the PalmDB header
         '''
         title = ascii_filename(unicode(self.oeb.metadata.title[0])).replace(
-                ' ', '_')[:32]
+                ' ', '_')[:31]
         title = title + (b'\0' * (32 - len(title)))
         now = int(time.time())
         nrecords = len(self.records)

src/calibre/ebooks/odt/input.py

@@ -74,7 +74,10 @@ class Extract(ODF2XHTML):
             style = style[0]
             css = style.text
             if css:
-                style.text, sel_map = self.do_filter_css(css)
+                css, sel_map = self.do_filter_css(css)
+                if not isinstance(css, unicode):
+                    css = css.decode('utf-8', 'ignore')
+                style.text = css
                 for x in root.xpath('//*[@class]'):
                     extra = []
                     orig = x.get('class')

src/calibre/ebooks/rb/writer.py

@@ -104,8 +104,9 @@ class RBWriter(object):
         size = len(text)
 
         pages = []
-        for i in range(0, (len(text) / TEXT_RECORD_SIZE) + 1):
-            pages.append(zlib.compress(text[i * TEXT_RECORD_SIZE : (i * TEXT_RECORD_SIZE) + TEXT_RECORD_SIZE], 9))
+        for i in range(0, (len(text) + TEXT_RECORD_SIZE-1) / TEXT_RECORD_SIZE):
+            zobj = zlib.compressobj(9, zlib.DEFLATED, 13, 8, 0)
+            pages.append(zobj.compress(text[i * TEXT_RECORD_SIZE : (i * TEXT_RECORD_SIZE) + TEXT_RECORD_SIZE]) + zobj.flush())
 
         return (size, pages)
 

src/calibre/gui2/book_details.py

@@ -326,6 +326,18 @@ class CoverView(QWidget): # {{{
         if id_ is not None:
             self.cover_removed.emit(id_)
 
+    def update_tooltip(self, current_path):
+        try:
+            sz = self.pixmap.size()
+        except:
+            sz = QSize(0, 0)
+        self.setToolTip(
+            '<p>'+_('Double-click to open Book Details window') +
+            '<br><br>' + _('Path') + ': ' + current_path +
+            '<br><br>' + _('Cover size: %(width)d x %(height)d')%dict(
+                width=sz.width(), height=sz.height())
+        )
+
     # }}}
 
 # Book Info {{{
@@ -561,16 +573,7 @@ class BookDetails(QWidget): # {{{
 
     def update_layout(self):
         self._layout.do_layout(self.rect())
-        try:
-            sz = self.cover_view.pixmap.size()
-        except:
-            sz = QSize(0, 0)
-        self.setToolTip(
-            '<p>'+_('Double-click to open Book Details window') +
-            '<br><br>' + _('Path') + ': ' + self.current_path +
-            '<br><br>' + _('Cover size: %(width)d x %(height)d')%dict(
-                width=sz.width(), height=sz.height())
-        )
+        self.cover_view.update_tooltip(self.current_path)
 
     def reset_info(self):
         self.show_data(Metadata(_('Unknown')))