Pull from trunk

src/calibre/devices/prs505/driver.py

@@ -112,7 +112,7 @@ class PRS505(Device):
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
             raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
-                                   stdout=subprocess.PIPE).stdout.read()
+                                   stdout=subprocess.PIPE).communicate()[0]
         lines = raw.splitlines()
         names = {}
         for i, line in enumerate(lines):

src/calibre/devices/usbms/device.py

@@ -200,7 +200,7 @@ class Device(_Device):
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
             raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
-                                   stdout=subprocess.PIPE).stdout.read()
+                                   stdout=subprocess.PIPE).communicate()[0]
         lines = raw.splitlines()
         names = {}
 

src/calibre/ebooks/metadata/opf2.py

@@ -9,7 +9,7 @@ lxml based OPF parser.
 
 import sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO
 from urllib import unquote
-from urlparse import urlparse, urldefrag
+from urlparse import urlparse
 
 from lxml import etree
 from dateutil import parser
@@ -258,6 +258,11 @@ class Manifest(ResourceCollection):
             if i.id == id:
                 return i.path
 
+    def type_for_id(self, id):
+        for i in self:
+            if i.id == id:
+                return i.mime_type
+
 class Spine(ResourceCollection):
 
     class Item(Resource):
@@ -487,7 +492,10 @@ class OPF(object):
 
             if toc is None: return
             self.toc = TOC(base_path=self.base_dir)
-            if toc.lower() in ('ncx', 'ncxtoc'):
+            is_ncx = getattr(self, 'manifest', None) is not None and \
+                     self.manifest.type_for_id(toc) is not None and \
+                     'dtbncx' in self.manifest.type_for_id(toc)
+            if is_ncx or toc.lower() in ('ncx', 'ncxtoc'):
                 path = self.manifest.path_for_id(toc)
                 if path:
                     self.toc.read_ncx_toc(path)

src/calibre/ebooks/mobi/reader.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import struct, os, cStringIO, re, functools
+import struct, os, cStringIO, re, functools, datetime
 
 try:
     from PIL import Image as PILImage
@@ -53,6 +53,12 @@ class EXTHHeader(object):
                     self.cover_offset = co
             elif id == 202:
                 self.thumbnail_offset, = struct.unpack('>L', content)
+            elif id == 501:
+                # cdetype
+                pass
+            elif id == 502:
+                # last update time
+                pass
             elif id == 503 and (not title or title == _('Unknown')):
                 title = content
             #else:
@@ -75,8 +81,14 @@ class EXTHHeader(object):
             if not self.mi.tags:
                 self.mi.tags = []
             self.mi.tags.append(content.decode(codec, 'ignore'))
+        elif id == 106:
+            try:
+                self.mi.publish_date = datetime.datetime.strptime(
+                        content, '%Y-%m-%d',).date()
+            except:
+                pass
         #else:
-        #    print 'unhandled metadata record', id, repr(content), codec
+        #    print 'unhandled metadata record', id, repr(content)
 
 
 class BookHeader(object):
@@ -327,8 +339,8 @@ class MobiReader(object):
         mobi_version = self.book_header.mobi_version
         for i, tag in enumerate(root.iter(etree.Element)):
             if tag.tag in ('country-region', 'place', 'placetype', 'placename',
-                           'state', 'city', 'street', 'address'):
+                           'state', 'city', 'street', 'address', 'content'):
-                tag.tag = 'span'
+                tag.tag = 'div' if tag.tag == 'content' else 'span'
                 for key in tag.attrib.keys():
                     tag.attrib.pop(key)
                 continue

src/calibre/ebooks/mobi/writer.py

@@ -211,12 +211,14 @@ class Serializer(object):
 
     def serialize_item(self, item):
         buffer = self.buffer
+        #buffer.write('<mbp:section>')
         if not item.linear:
             self.breaks.append(buffer.tell() - 1)
         self.id_offsets[item.href] = buffer.tell()
         for elem in item.data.find(XHTML('body')):
             self.serialize_elem(elem, item)
-        buffer.write('<mbp:pagebreak/>')
+        #buffer.write('</mbp:section>')
+        buffer.write('</mbp:pagebreak>')
 
     def serialize_elem(self, elem, item, nsrmap=NSRMAP):
         buffer = self.buffer

src/calibre/gui2/dialogs/config.py

@@ -1,6 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, re, time, textwrap
+import os, re, time, textwrap, sys, cStringIO
 from binascii import hexlify, unhexlify
 
 from PyQt4.Qt import    QDialog, QMessageBox, QListWidgetItem, QIcon, \
@@ -11,6 +11,7 @@ from PyQt4.Qt import    QDialog, QMessageBox, QListWidgetItem, QIcon, \
 
 from calibre.constants import islinux, iswindows
 from calibre.gui2.dialogs.config_ui import Ui_Dialog
+from calibre.gui2.dialogs.test_email_ui import Ui_Dialog as TE_Dialog
 from calibre.gui2 import qstring_to_unicode, choose_dir, error_dialog, config, \
                          ALL_COLUMNS, NONE, info_dialog, choose_files
 from calibre.utils.config import prefs
@@ -134,6 +135,33 @@ class CategoryModel(QStringListModel):
             return self.icons[index.row()]
         return QStringListModel.data(self, index, role)
 
+class TestEmail(QDialog, TE_Dialog):
+
+    def __init__(self, accounts, parent):
+        QDialog.__init__(self, parent)
+        TE_Dialog.__init__(self)
+        self.setupUi(self)
+        opts = smtp_prefs().parse()
+        self.test_func = parent.test_email_settings
+        self.connect(self.test_button, SIGNAL('clicked(bool)'), self.test)
+        self.from_.setText(unicode(self.from_.text())%opts.from_)
+        if accounts:
+            self.to.setText(list(accounts.keys())[0])
+        if opts.relay_host:
+            self.label.setText(_('Using: %s:%s@%s:%s and %s encryption')%
+                    (opts.relay_username, unhexlify(opts.relay_password),
+                        opts.relay_host, opts.relay_port, opts.encryption))
+
+    def test(self):
+        self.log.setPlainText(_('Sending...'))
+        self.test_button.setEnabled(False)
+        try:
+            tb = self.test_func(unicode(self.to.text()))
+            if not tb:
+                tb = _('Mail successfully sent')
+            self.log.setPlainText(tb)
+        finally:
+            self.test_button.setEnabled(True)
 
 class EmailAccounts(QAbstractTableModel):
 
@@ -395,6 +423,8 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.connect(self.email_make_default, SIGNAL('clicked(bool)'),
              lambda c: self._email_accounts.make_default(self.email_view.currentIndex()))
         self.email_view.resizeColumnsToContents()
+        self.connect(self.test_email_button, SIGNAL('clicked(bool)'),
+                self.test_email)
 
     def add_email_account(self, checked):
         index = self._email_accounts.add()
@@ -438,6 +468,33 @@ class ConfigDialog(QDialog, Ui_Dialog):
         conf.set('encryption', 'TLS' if self.relay_tls.isChecked() else 'SSL')
         return True
 
+    def test_email(self, *args):
+        if self.set_email_settings():
+          TestEmail(self._email_accounts.accounts, self).exec_()
+
+    def test_email_settings(self, to):
+        opts = smtp_prefs().parse()
+        from calibre.utils.smtp import sendmail, create_mail
+        buf = cStringIO.StringIO()
+        oout, oerr = sys.stdout, sys.stderr
+        sys.stdout = sys.stderr = buf
+        tb = None
+        try:
+            msg = create_mail(opts.from_, to, 'Test mail from calibre',
+                    'Test mail from calibre')
+            sendmail(msg, from_=opts.from_, to=[to],
+                verbose=3, timeout=30, relay=opts.relay_host,
+                username=opts.relay_username,
+                password=unhexlify(opts.relay_password),
+                encryption=opts.encryption, port=opts.relay_port)
+        except:
+            import traceback
+            tb = traceback.format_exc()
+            tb += '\n\nLog:\n' + buf.getvalue()
+        finally:
+            sys.stdout, sys.stderr = oout, oerr
+        return tb
+
     def add_plugin(self):
         path = unicode(self.plugin_path.text())
         if path and os.access(path, os.R_OK) and path.lower().endswith('.zip'):

src/calibre/gui2/dialogs/search.py

@@ -20,7 +20,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         return [t.strip() for t in phrases + raw.split()]
 
     def search_string(self):
-        all, any, phrase, none = map(lambda x: unicode(x.text()), (self.all, self.any, self.phrase, self.none))
+        all, any, phrase, none = map(lambda x: unicode(x.text()),
+                (self.all, self.any, self.phrase, self.none))
         all, any, none = map(self.tokens, (all, any, none))
         phrase = phrase.strip()
         all = ' and '.join(all)
@@ -32,7 +33,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         if all:
             ans += (' and ' if ans else '') + all
         if none:
-            ans += (' and not ' if ans else '') + none
+            ans += (' and not ' if ans else 'not ') + none
         if any:
             ans += (' or ' if ans else '') + any
         return ans

src/calibre/gui2/dialogs/test_email.ui

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>542</width>
+    <height>418</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Test email settings</string>
+  </property>
+  <property name="windowIcon">
+   <iconset resource="../images.qrc">
+    <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QLabel" name="from_">
+     <property name="text">
+      <string>Send test mail from %s to:</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QLineEdit" name="to"/>
+   </item>
+   <item>
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPushButton" name="test_button">
+     <property name="text">
+      <string>&amp;Test</string>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPlainTextEdit" name="log"/>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/library.py

@@ -93,7 +93,7 @@ class DateDelegate(QStyledItemDelegate):
 
     def createEditor(self, parent, option, index):
         qde = QStyledItemDelegate.createEditor(self, parent, option, index)
-        qde.setDisplayFormat('MM/dd/yyyy')
+        qde.setDisplayFormat(unicode(qde.displayFormat()).replace('yy', 'yyyy'))
         qde.setMinimumDate(QDate(101,1,1))
         qde.setCalendarPopup(True)
         return qde
@@ -637,7 +637,8 @@ class BooksView(TableView):
 
     def columns_sorted(self, rating_col, timestamp_col):
         for i in range(self.model().columnCount(None)):
-            if self.itemDelegateForColumn(i) == self.rating_delegate:
+            if self.itemDelegateForColumn(i) in (self.rating_delegate,
+                    self.timestamp_delegate):
                 self.setItemDelegateForColumn(i, self.itemDelegate())
         if rating_col > -1:
             self.setItemDelegateForColumn(rating_col, self.rating_delegate)

src/calibre/library/database2.py

@@ -226,7 +226,11 @@ class ResultCache(SearchQueryParser):
         Returns a list of affected rows or None if the rows are filtered.
         '''
         for id in ids:
-            self._data[id] = conn.get('SELECT * from meta WHERE id=?', (id,))[0]
+            try:
+                self._data[id] = conn.get('SELECT * from meta WHERE id=?',
+                        (id,))[0]
+            except IndexError:
+                return None
         try:
             return map(self.row, ids)
         except ValueError:
@@ -1568,3 +1572,4 @@ books_series_link      feeds
 
         return duplicates
 
+

src/calibre/utils/smtp.py

@@ -45,15 +45,17 @@ def create_mail(from_, to, subject, text=None, attachment_data=None,
 
     return outer.as_string()
 
-def get_mx(host):
+def get_mx(host, verbose=0):
     import dns.resolver
+    if verbose:
+        print 'Find mail exchanger for', host
     answers = list(dns.resolver.query(host, 'MX'))
     answers.sort(cmp=lambda x, y: cmp(int(x.preference), int(y.preference)))
     return [str(x.exchange) for x in answers]
 
 def sendmail_direct(from_, to, msg, timeout, localhost, verbose):
     import smtplib
-    hosts = get_mx(to.split('@')[-1].strip())
+    hosts = get_mx(to.split('@')[-1].strip(), verbose)
     timeout=None # Non blocking sockets sometimes don't work
     s = smtplib.SMTP(timeout=timeout, local_hostname=localhost)
     s.set_debuglevel(verbose)

src/calibre/web/feeds/recipes/__init__.py

@@ -37,6 +37,7 @@ recipe_modules = ['recipe_' + r for r in (
            'new_york_review_of_books_no_sub', 'politico', 'adventuregamers',
            'mondedurable', 'instapaper', 'dnevnik_cro', 'vecernji_list',
            'nacional_cro', '24sata', 'dnevni_avaz', 'glas_srpske', '24sata_rs',
+           'krstarica', 'krstarica_en', 'tanjug',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_krstarica.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+vesti.krstarica.com
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Krstarica(BasicNewsRecipe):
+    title                 = 'Krstarica - Vesti'
+    __author__            = 'Darko Miletic'
+    description           = 'Dnevne vesti iz Srbije i sveta'    
+    publisher             = 'Krstarica'
+    category              = 'news, politics, Serbia'
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    encoding              = 'utf-8'
+    language              = _('Serbian')
+    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em}"' 
+    
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    
+    feeds          = [
+                        (u'Vesti dana'         , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=aktuelno&lang=0'     )
+                       ,(u'Srbija'             , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=scg&lang=0'          )
+                       ,(u'Svet'               , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=svet&lang=0'         )
+                       ,(u'Politika'           , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=politika&lang=0'     )
+                       ,(u'Ekonomija'          , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=ekonomija&lang=0'    )
+                       ,(u'Drustvo'            , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=drustvo&lang=0'      )
+                       ,(u'Kultura'            , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=kultura&lang=0'      )
+                       ,(u'Nauka i Tehnologija', u'http://vesti.krstarica.com/index.php?rss=1&rubrika=nauka&lang=0'        )
+                       ,(u'Medicina'           , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=medicina&lang=0'     )
+                       ,(u'Sport'              , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=sport&lang=0'        )
+                       ,(u'Zanimljivosti'      , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=zanimljivosti&lang=0')
+                     ]
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>'
+        soup.head.insert(0,mtag)
+        titletag = soup.find('h4')
+        if titletag:
+           realtag = titletag.parent.parent
+           realtag.extract()
+           for item in soup.findAll(['table','center']):
+               item.extract()
+           soup.body.insert(1,realtag)            
+           realtag.name = 'div'
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(align=True):
+            del item['align']
+        return soup

src/calibre/web/feeds/recipes/recipe_krstarica_en.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+vesti.krstarica.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Krstarica_en(BasicNewsRecipe):
+    title                 = 'Krstarica - news in english'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Serbia and world'    
+    publisher             = 'Krstarica'
+    category              = 'news, politics, Serbia'
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    encoding              = 'utf-8'
+    language              = _('English')
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em}"' 
+        
+    feeds          = [
+                        (u'Daily news', u'http://vesti.krstarica.com/index.php?rss=1&rubrika=aktuelno&lang=1'     )
+                       ,(u'Serbia'    , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=scg&lang=1'          )
+                       ,(u'Politics'  , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=politika&lang=1'     )
+                       ,(u'Economy'   , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=ekonomija&lang=1'    )
+                       ,(u'Culture'   , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=kultura&lang=1'      )
+                       ,(u'Sports'    , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=sport&lang=1'        )
+                     ]
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>'
+        soup.head.insert(0,mtag)
+        titletag = soup.find('h4')
+        if titletag:
+           realtag = titletag.parent.parent
+           realtag.extract()
+           for item in soup.findAll(['table','center']):
+               item.extract()
+           soup.body.insert(1,realtag)            
+           realtag.name = 'div'
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(align=True):
+            del item['align']
+        return soup

src/calibre/web/feeds/recipes/recipe_tanjug.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+tanjug.rs
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Tanjug(BasicNewsRecipe):
+    title                 = 'Tanjug'
+    __author__            = 'Darko Miletic'
+    description           = 'Novinska agencija TANJUG - Dnevne vesti iz Srbije i sveta'
+    publisher             = 'Tanjug'
+    category              = 'news, politics, Serbia'
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    use_embedded_content  = True
+    encoding              = 'utf-8'
+    lang                  = 'sr-Latn-RS'
+    language              = _('Serbian')
+    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em}"' 
+    
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    
+    feeds          = [(u'Vesti', u'http://www.tanjug.rs/StaticPages/RssTanjug.aspx')]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = self.lang
+        soup.html['lang'    ] = self.lang
+        soup.html['dir'     ] = "ltr"
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>'
+        soup.head.insert(0,mtag)
+        return soup

src/calibre/www/apps/feedjack/fjlib.py

@@ -128,7 +128,7 @@ def get_extra_content(site, sfeeds_ids, ctx):
 def get_posts_tags(object_list, sfeeds_obj, user_id, tag_name):
     """ Adds a qtags property in every post object in a page.
 
-    Use "qtags" instead of "tags" in templates to avoid innecesary DB hits.
+    Use "qtags" instead of "tags" in templates to avoid unnecessary DB hits.
     """
     tagd = {}
     user_obj = None