pre 0.6.20 changes

resources/recipes/daily_telegraph.recipe

@@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class DailyTelegraph(BasicNewsRecipe):
     title          = u'Daily Telegraph'
     __author__     = u'AprilHare'
-    language = 'en'
+    language = 'en_AU'
 
     description    = u'News from down under'
     oldest_article = 2

resources/recipes/outlook_india.recipe

@@ -14,7 +14,6 @@ class OutlookIndia(BasicNewsRecipe):
     encoding       = 'utf-8'
     language = 'en_IN'
 
-    recursions = 1
     extra_css = '''
                  body{font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
                 .fspheading{color:#AF0E25 ; font-family:"Times New Roman",Times,serif; font-weight:bold ; font-size:large; }

resources/recipes/variety.recipe

@@ -0,0 +1,46 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.variety.com
+'''
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class Variety(BasicNewsRecipe):
+    title                  = 'Variety'
+    __author__             = 'Darko Miletic'
+    description            = 'Breaking entertainment movie news, movie reviews, entertainment industry events, news and reviews from Cannes, Oscars, and Hollywood awards.  Featuring box office charts, archives and more.'
+    oldest_article         = 2
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    encoding               = 'cp1252'
+    publisher              = 'Red Business Information'
+    category               = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood'
+    language               = 'en'
+
+    conversion_options = {  
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    remove_tags = [dict(name=['object','link','map'])]
+
+    keep_only_tags = [dict(name='div', attrs={'id':'article'})]
+                  
+    feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )]
+
+    def print_version(self, url):
+        rpt = url.rpartition('?')[0]
+        artid = rpt.rpartition('/')[2]
+        catidr = url.rpartition('categoryid=')[2]
+        catid = catidr.partition('&')[0]
+        return 'http://www.variety.com/index.asp?layout=print_story&articleid=' + artid + '&categoryid=' + catid
+
+    def get_article_url(self, article):
+        return article.get('feedburner_origlink',  None)
+

setup/installer/linux/__init__.py

@@ -6,10 +6,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
 
 from setup.installer import VMInstaller
-from setup import Command, installer_name
+from setup import Command
 
 class Linux32(VMInstaller):
 
@@ -21,17 +20,12 @@ class Linux32(VMInstaller):
     FREEZE_COMMAND = 'linux_freeze'
 
 
-class Linux64(Command):
+class Linux64(Linux32):
 
     description = 'Build 64bit linux binary installer'
-
+    VM_NAME = 'gentoo64_build'
-    sub_commands = ['linux_freeze']
+    VM = '/vmware/bin/gentoo64_build'
-
+    IS_64_BIT = True
-    def run(self, opts):
-        installer = installer_name('tar.bz2', True)
-        if not os.path.exists(installer):
-            raise Exception('Failed to build installer '+installer)
-        return os.path.basename(installer)
 
 class Linux(Command):
 

src/calibre/debug.py

@@ -157,6 +157,16 @@ def add_simple_plugin(path_to_plugin):
 def main(args=sys.argv):
     from calibre.constants import debug
     debug()
+    if len(args) > 2 and args[1] in ('-e', '--exec-file'):
+        sys.argv = [args[2]] + args[3:]
+        ef = os.path.abspath(args[2])
+        base = os.path.dirname(ef)
+        sys.path.insert(0, base)
+        g = globals()
+        g['__name__'] = '__main__'
+        execfile(ef, g)
+        return
+
     opts, args = option_parser().parse_args(args)
     if opts.gui:
         from calibre.gui2.main import main

src/calibre/ebooks/__init__.py

@@ -102,7 +102,6 @@ def render_html(path_to_html, width=590, height=750):
         page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
         loop = QEventLoop()
         renderer = HTMLRenderer(page, loop)
-
         page.connect(page, SIGNAL('loadFinished(bool)'), renderer,
                 Qt.QueuedConnection)
         page.mainFrame().load(QUrl.fromLocalFile(path_to_html))

src/calibre/ebooks/conversion/preprocess.py

@@ -228,17 +228,16 @@ class HTMLPreProcessor(object):
         else:
             rules = []
 
-        pre_rules = []
+        end_rules = []
         if getattr(self.extra_opts, 'remove_header', None):
-            pre_rules.append(
+            end_rules.append(
                 (re.compile(getattr(self.extra_opts, 'header_regex')), lambda match : '')
             )
         if getattr(self.extra_opts, 'remove_footer', None):
-            pre_rules.append(
+            end_rules.append(
                 (re.compile(getattr(self.extra_opts, 'footer_regex')), lambda match : '')
             )
         
-        end_rules = []
         if getattr(self.extra_opts, 'unwrap_factor', 0.0) > 0.01:
             length = line_length(html, getattr(self.extra_opts, 'unwrap_factor'))
             if length:
@@ -247,7 +246,7 @@ class HTMLPreProcessor(object):
                     (re.compile(r'(?<=.{%i}[a-z\.,;:)-IA])\s*(?P<ital></(i|b|u)>)?\s*(<p.*?>)\s*(?=(<(i|b|u)>)?\s*[\w\d(])' % length, re.UNICODE), wrap_lines),
                 )
 
-        for rule in self.PREPROCESS + pre_rules + rules + end_rules:
+        for rule in self.PREPROCESS + rules + end_rules:
             html = rule[0].sub(rule[1], html)
 
         # Handle broken XHTML w/ SVG (ugh)

src/calibre/ebooks/html/input.py

@@ -320,8 +320,8 @@ class HTMLInput(InputFormatPlugin):
             oeb.logger.warn('Title not specified')
             metadata.add('title', self.oeb.translate(__('Unknown')))
 
-        bookid = "urn:uuid:%s" % str(uuid.uuid4())
+        bookid = str(uuid.uuid4())
-        metadata.add('identifier', bookid, id='calibre-uuid')
+        metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')
         for ident in metadata.identifier:
             if 'id' in ident.attrib:
                 self.oeb.uid = metadata.identifier[0]
@@ -409,6 +409,9 @@ class HTMLInput(InputFormatPlugin):
         link = os.path.abspath(link)
         if not os.access(link, os.R_OK):
             return link_
+        if os.path.isdir(link):
+            self.log.warn(link_, 'is a link to a directory. Ignoring.')
+            return link_
         if not islinux:
             link = link.lower()
         if link not in self.added_resources:

src/calibre/ebooks/metadata/__init__.py

@@ -218,7 +218,7 @@ class MetaInformation(object):
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
                      'manifest', 'spine', 'toc', 'cover', 'language',
                      'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc',
-                     'pubdate', 'rights', 'publication_type'):
+                     'pubdate', 'rights', 'publication_type', 'uuid'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
 
@@ -244,7 +244,7 @@ class MetaInformation(object):
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
                   'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate',
-                  'rights', 'publication_type',
+                  'rights', 'publication_type', 'uuid',
                   ):
             setattr(self, x, getattr(mi, x, None))
 
@@ -264,7 +264,7 @@ class MetaInformation(object):
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
                      'cover', 'language', 'guide', 'book_producer',
                      'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
-                     'publication_type'):
+                     'publication_type', 'uuid',):
             if hasattr(mi, attr):
                 val = getattr(mi, attr)
                 if val is not None:

src/calibre/ebooks/metadata/opf2.py

@@ -432,6 +432,9 @@ class OPF(object):
     identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]')
     application_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]')
+    uuid_id_path    = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
+                            '(re:match(@opf:scheme, "uuid", "i") or re:match(@scheme, "uuid", "i"))]')
+
     manifest_path   = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]')
     manifest_ppath  = XPath('descendant::*[re:match(name(), "manifest", "i")]')
     spine_path      = XPath('descendant::*[re:match(name(), "spine", "i")]/*[re:match(name(), "itemref", "i")]')
@@ -747,6 +750,25 @@ class OPF(object):
 
         return property(fget=fget, fset=fset)
 
+    @dynamic_property
+    def uuid(self):
+
+        def fget(self):
+            for match in self.uuid_id_path(self.metadata):
+                return self.get_text(match) or None
+
+        def fset(self, val):
+            matches = self.uuid_id_path(self.metadata)
+            if not matches:
+                attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'uuid'}
+                matches = [self.create_metadata_element('identifier',
+                                                        attrib=attrib)]
+            self.set_text(matches[0], unicode(val))
+
+        return property(fget=fget, fset=fset)
+
+
+
     @dynamic_property
     def book_producer(self):
 
@@ -977,6 +999,9 @@ def metadata_to_opf(mi, as_string=True):
     if not mi.application_id:
         mi.application_id = str(uuid.uuid4())
 
+    if not mi.uuid:
+        mi.uuid = str(uuid.uuid4())
+
     if not mi.book_producer:
         mi.book_producer = __appname__ + ' (%s) '%__version__ + \
             '[http://calibre-ebook.com]'
@@ -986,13 +1011,14 @@ def metadata_to_opf(mi, as_string=True):
 
     root = etree.fromstring(textwrap.dedent(
     '''
-    <package xmlns="http://www.idpf.org/2007/opf" unique-identifier="%(a)s_id">
+    <package xmlns="http://www.idpf.org/2007/opf" unique-identifier="uuid_id">
         <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
             <dc:identifier opf:scheme="%(a)s" id="%(a)s_id">%(id)s</dc:identifier>
+            <dc:identifier opf:scheme="uuid" id="uuid_id">%(uuid)s</dc:identifier>
         </metadata>
         <guide/>
     </package>
-    '''%dict(a=__appname__, id=mi.application_id)))
+    '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid)))
     metadata = root[0]
     guide = root[1]
     metadata[0].tail = '\n'+(' '*8)

src/calibre/ebooks/oeb/iterator.py

@@ -123,7 +123,7 @@ class EbookIterator(object):
                                 else:
                                     print 'Loaded embedded font:', repr(family)
 
-    def __enter__(self, raw_only=False):
+    def __enter__(self, processed=False):
         self.delete_on_exit = []
         self._tdir = TemporaryDirectory('_ebook_iter')
         self.base  = self._tdir.__enter__()
@@ -140,7 +140,7 @@ class EbookIterator(object):
                 plumber.opts, plumber.input_fmt, self.log,
                 {}, self.base)
 
-        if not raw_only and plumber.input_fmt.lower() in ('pdf', 'rb'):
+        if processed or plumber.input_fmt.lower() in ('pdf', 'rb'):
             self.pathtoopf = create_oebbook(self.log, self.pathtoopf, plumber.opts,
                     plumber.input_plugin)
         if hasattr(self.pathtoopf, 'manifest'):

src/calibre/ebooks/oeb/reader.py

@@ -139,10 +139,9 @@ class OEBReader(object):
             mi.book_producer = '%(a)s (%(v)s) [http://%(a)s.kovidgoyal.net]'%\
                 dict(a=__appname__, v=__version__)
         meta_info_to_oeb_metadata(mi, self.oeb.metadata, self.logger)
-        bookid = "urn:uuid:%s" % str(uuid.uuid4()) if mi.application_id is None \
+        self.oeb.metadata.add('identifier', str(uuid.uuid4()), id='uuid_id',
-                else mi.application_id
+                                scheme='uuid')
-        self.oeb.metadata.add('identifier', bookid, id='calibre-uuid')
+        self.oeb.uid = self.oeb.metadata.identifier[-1]
-        self.oeb.uid = self.oeb.metadata.identifier[0]
 
     def _manifest_prune_invalid(self):
         '''

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -80,12 +80,19 @@ class MergeMetadata(object):
     def __call__(self, oeb, mi, opts):
         self.oeb, self.log = oeb, oeb.log
         m = self.oeb.metadata
-        meta_info_to_oeb_metadata(mi, m, oeb.log)
         self.log('Merging user specified metadata...')
+        meta_info_to_oeb_metadata(mi, m, oeb.log)
         cover_id = self.set_cover(mi, opts.prefer_metadata_cover)
         m.clear('cover')
         if cover_id is not None:
             m.add('cover', cover_id)
+        if mi.uuid is not None:
+            m.filter('identifier', lambda x:x.id=='uuid_id')
+            self.oeb.metadata.add('identifier', mi.uuid, id='uuid_id',
+                                    scheme='uuid')
+            self.oeb.uid = self.oeb.metadata.identifier[-1]
+
+
 
 
     def set_cover(self, mi, prefer_metadata_cover):

src/calibre/ebooks/pml/pmlml.py

@@ -153,6 +153,10 @@ class PMLMLizer(object):
         for unused in anchors.difference(links):
             text = text.replace('\\Q="%s"' % unused, '')
 
+        # Replace bad characters.
+        text = text.replace(u'\xc2', '')
+        text = text.replace(u'\xa0', ' ')
+
         # Turn all html entities into unicode. This should not be necessary as
         # lxml should have already done this but we want to be sure it happens.
         for entity in set(re.findall('&.+?;', text)):

src/calibre/gui2/__init__.py

@@ -130,7 +130,7 @@ class CopyButton(QPushButton):
                 return
         except:
             pass
-        return QPushButton.event(self, ev)
+        QPushButton.keyPressEvent(self, ev)
 
 
     def keyReleaseEvent(self, ev):
@@ -139,7 +139,7 @@ class CopyButton(QPushButton):
                 return
         except:
             pass
-        return QPushButton.event(self, ev)
+        QPushButton.keyReleaseEvent(self, ev)
 
     def mouseReleaseEvent(self, ev):
         ev.accept()

src/calibre/gui2/convert/regex_builder.py

@@ -87,12 +87,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
 
     def open_book(self, pathtoebook):
         self.iterator = EbookIterator(pathtoebook)
-        self.iterator.__enter__(raw_only=True)
+        self.iterator.__enter__(processed=True)
         text = [u'']
         for path in self.iterator.spine:
-            html = open(path, 'rb').read().decode(path.encoding, 'replace')
+            html = open(path, 'rb').read().decode('utf-8', 'replace')
             text.append(html)
-        self.preview.setPlainText('\n\n'.join(text))
+        self.preview.setPlainText('\n---\n'.join(text))
 
     def button_clicked(self, button):
         if button == self.button_box.button(QDialogButtonBox.Open):

src/calibre/gui2/jobs.py

@@ -11,7 +11,7 @@ from Queue import Empty, Queue
 
 from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \
     QTimer, SIGNAL, QIcon, QDialog, QAbstractItemDelegate, QApplication, \
-    QSize, QStyleOptionProgressBarV2, QString, QStyle
+    QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -57,6 +57,28 @@ class JobManager(QAbstractTableModel):
         else:
             return QVariant(section+1)
 
+    def show_tooltip(self, arg):
+        widget, pos = arg
+        QToolTip.showText(pos, self.get_tooltip())
+
+    def get_tooltip(self):
+        running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING]
+        waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING]
+        lines = [_('There are %d running jobs:')%len(running_jobs)]
+        for job in running_jobs:
+            desc = job.description
+            if not desc:
+                desc = _('Unknown job')
+            p = 100. if job.is_finished else job.percent
+            lines.append('%s:  %.0f%% done'%(desc, p))
+        lines.extend(['', _('There are %d waiting jobs:')%len(waiting_jobs)])
+        for job in waiting_jobs:
+            desc = job.description
+            if not desc:
+                desc = _('Unknown job')
+            lines.append(desc)
+        return '\n'.join(['calibre', '']+ lines)
+
     def data(self, index, role):
         try:
             if role not in (Qt.DisplayRole, Qt.DecorationRole):

src/calibre/gui2/main.py

@@ -11,7 +11,7 @@ from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
                      QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
                      QToolButton, QDialog, QDesktopServices, QFileDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
-                     QMessageBox, QStackedLayout
+                     QMessageBox, QStackedLayout, QHelpEvent
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre import  prints, patheq
@@ -89,6 +89,18 @@ class Listener(Thread):
         except:
             pass
 
+class SystemTrayIcon(QSystemTrayIcon):
+
+    def __init__(self, icon, parent):
+        QSystemTrayIcon.__init__(self, icon, parent)
+
+    def event(self, ev):
+        if ev.type() == ev.ToolTip:
+            evh = QHelpEvent(ev)
+            self.emit(SIGNAL('tooltip_requested(PyQt_PyObject)'),
+                    (self, evh.globalPos()))
+            return True
+        return QSystemTrayIcon.event(self, ev)
 
 class Main(MainWindow, Ui_MainWindow, DeviceGUI):
     'The main GUI'
@@ -144,8 +156,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.device_connected = False
         self.viewers = collections.deque()
         self.content_server = None
-        self.system_tray_icon = QSystemTrayIcon(QIcon(I('library.png')), self)
+        self.system_tray_icon = SystemTrayIcon(QIcon(I('library.png')), self)
         self.system_tray_icon.setToolTip('calibre')
+        self.connect(self.system_tray_icon,
+                SIGNAL('tooltip_requested(PyQt_PyObject)'),
+                self.job_manager.show_tooltip)
         if not config['systray_icon']:
             self.system_tray_icon.hide()
         else:

src/calibre/library/cli.py

@@ -18,7 +18,9 @@ from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.genshi.template import MarkupTemplate
 
-FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'cover'])
+FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
+    'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
+    'formats', 'isbn', 'uuid', 'cover'])
 
 XML_TEMPLATE = '''\
 <?xml version="1.0"  encoding="UTF-8"?>
@@ -26,6 +28,7 @@ XML_TEMPLATE = '''\
 <py:for each="record in data">
     <record>
         <id>${record['id']}</id>
+        <uuid>${record['uuid']}</uuid>
         <title>${record['title']}</title>
         <authors sort="${record['author_sort']}">
         <py:for each="author in record['authors']">
@@ -71,7 +74,7 @@ STANZA_TEMPLATE='''\
   <py:for each="record in data">
   <entry>
       <title>${record['title']}</title>
-      <id>urn:calibre:${record['id']}</id>
+      <id>urn:calibre:${record['uuid']}</id>
       <author><name>${record['author_sort']}</name></author>
       <updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
       <link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/')).replace('http%3A', 'http:')}" />
@@ -227,7 +230,7 @@ def command_list(args, dbpath):
     if not set(fields).issubset(FIELDS):
         parser.print_help()
         print
-        print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(FIELDS)
+        print >>sys.stderr, _('Invalid fields. Available fields:'), ','.join(sorted(FIELDS))
         return 1
 
     db = get_db(dbpath, opts)

src/calibre/library/database2.py

@@ -59,7 +59,7 @@ copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5,
              'size':6, 'tags':7, 'comments':8, 'series':9, 'series_index':10,
              'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
-             'lccn':16, 'pubdate':17, 'flags':18, 'cover':19}
+             'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
 INDEX_MAP = dict(zip(FIELD_MAP.values(), FIELD_MAP.keys()))
 
 
@@ -447,7 +447,7 @@ class LibraryDatabase2(LibraryDatabase):
 
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
                      'publisher', 'rating', 'series', 'series_index', 'tags',
-                     'title', 'timestamp'):
+                     'title', 'timestamp', 'uuid'):
             setattr(self, prop, functools.partial(get_property,
                     loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
 
@@ -622,6 +622,50 @@ class LibraryDatabase2(LibraryDatabase):
         END TRANSACTION;
         ''')
 
+    def upgrade_version_7(self):
+        'Add uuid column'
+        self.conn.executescript('''
+        BEGIN TRANSACTION;
+        ALTER TABLE books ADD COLUMN uuid TEXT;
+        DROP TRIGGER IF EXISTS books_insert_trg;
+        DROP TRIGGER IF EXISTS books_update_trg;
+        UPDATE books SET uuid=uuid4();
+
+        CREATE TRIGGER books_insert_trg AFTER INSERT ON books
+        BEGIN
+            UPDATE books SET sort=title_sort(NEW.title),uuid=uuid4() WHERE id=NEW.id;
+        END;
+
+        CREATE TRIGGER books_update_trg AFTER UPDATE ON books
+        BEGIN
+            UPDATE books SET sort=title_sort(NEW.title) WHERE id=NEW.id;
+        END;
+
+        DROP VIEW meta;
+        CREATE VIEW meta AS
+        SELECT id, title,
+               (SELECT sortconcat(bal.id, name) FROM books_authors_link AS bal JOIN authors ON(author = authors.id) WHERE book = books.id) authors,
+               (SELECT name FROM publishers WHERE publishers.id IN (SELECT publisher from books_publishers_link WHERE book=books.id)) publisher,
+               (SELECT rating FROM ratings WHERE ratings.id IN (SELECT rating from books_ratings_link WHERE book=books.id)) rating,
+               timestamp,
+               (SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size,
+               (SELECT concat(name) FROM tags WHERE tags.id IN (SELECT tag from books_tags_link WHERE book=books.id)) tags,
+               (SELECT text FROM comments WHERE book=books.id) comments,
+               (SELECT name FROM series WHERE series.id IN (SELECT series FROM books_series_link WHERE book=books.id)) series,
+               series_index,
+               sort,
+               author_sort,
+               (SELECT concat(format) FROM data WHERE data.book=books.id) formats,
+               isbn,
+               path,
+               lccn,
+               pubdate,
+               flags,
+               uuid
+        FROM books;
+
+        END TRANSACTION;
+        ''')
 
 
     def last_modified(self):
@@ -785,6 +829,7 @@ class LibraryDatabase2(LibraryDatabase):
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
         mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
         mi.pubdate     = self.pubdate(idx, index_is_id=index_is_id)
+        mi.uuid        = self.uuid(idx, index_is_id=index_is_id)
         tags = self.tags(idx, index_is_id=index_is_id)
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
@@ -1530,7 +1575,9 @@ class LibraryDatabase2(LibraryDatabase):
         '''
         if prefix is None:
             prefix = self.library_path
-        FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size', 'tags', 'comments', 'series', 'series_index', 'isbn'])
+        FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
+            'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
+            'isbn', 'uuid'])
         data = []
         for record in self.data:
             if record is None: continue

src/calibre/library/server.py

@@ -24,7 +24,7 @@ except ImportError:
 from calibre.constants import __version__, __appname__
 from calibre.utils.genshi.template import MarkupTemplate
 from calibre import fit_image, guess_type, prepare_string_for_xml, \
-        strftime as _strftime
+        strftime as _strftime, prints
 from calibre.library import server_config as config
 from calibre.library.database2 import LibraryDatabase2, FIELD_MAP
 from calibre.utils.config import config_dir
@@ -77,6 +77,159 @@ class LibraryServer(object):
             </book>
         ''')
 
+    MOBILE_UA = re.compile('(?i)(?:iPhone|Opera Mini|NetFront|webOS|Mobile|Android|imode|DoCoMo|Minimo|Blackberry|MIDP|Symbian)')
+
+    MOBILE_BOOK = textwrap.dedent('''\
+    <tr xmlns:py="http://genshi.edgewall.org/">
+    <td class="thumbnail">
+        <img type="image/jpeg" src="/get/thumb/${r[0]}" border="0"/>
+    </td>
+    <td>
+        <py:for each="format in r[13].split(',')">
+            <span class="button"><a href="/get/${format}/${authors}-${r[1]}_${r[0]}.${format}">${format.lower()}</a></span>&nbsp;
+        </py:for>
+       ${r[1]} by ${authors} - ${r[6]/1024}k - ${r[3] if r[3] else ''} ${pubdate} ${'['+r[7]+']' if r[7] else ''}
+    </td>
+    </tr>
+    ''')
+
+    MOBILE = MarkupTemplate(textwrap.dedent('''\
+    <html xmlns:py="http://genshi.edgewall.org/">
+    <head>
+    <style>
+    .navigation table.buttons {
+        width: 100%;
+    }
+    .navigation .button {
+        width: 50%;
+    }
+    .button a, .button:visited a {
+        padding: 0.5em;
+        font-size: 1.25em;
+        border: 1px solid black;
+        text-color: black;
+        background-color: #ddd;
+        border-top: 1px solid ThreeDLightShadow;
+        border-right: 1px solid ButtonShadow;
+        border-bottom: 1px solid ButtonShadow;
+        border-left: 1 px solid ThreeDLightShadow;
+        -moz-border-radius: 0.25em;
+        -webkit-border-radius: 0.25em;
+    }
+
+    .button:hover a {
+        border-top: 1px solid #666;
+        border-right: 1px solid #CCC;
+        border-bottom: 1 px solid #CCC;
+        border-left: 1 px solid #666;
+
+
+    }
+    div.navigation {
+        padding-bottom: 1em;
+        clear: both;
+    }
+
+    #search_box {
+        border: 1px solid #393;
+        -moz-border-radius: 0.5em;
+        -webkit-border-radius: 0.5em;
+        padding: 1em;
+        margin-bottom: 0.5em;
+        float: right;
+    }
+
+    #listing {
+        width: 100%;
+        border-collapse: collapse;
+    }
+    #listing td {
+        padding: 0.25em;
+    }
+
+    #listing td.thumbnail {
+        height: 60px;
+        width: 60px;
+    }
+
+    #listing tr:nth-child(even) {
+
+        background: #eee;
+    }
+
+    #listing .button a{
+        display: inline-block;
+        width: 2.5em;
+        padding-left: 0em;
+        padding-right: 0em;
+        overflow: hidden;
+        text-align: center;
+    }
+
+    #logo {
+        float: left;
+    }
+    #spacer {
+        clear: both;
+    }
+
+    </style>
+    <link rel="icon" href="http://calibre.kovidgoyal.net/chrome/site/favicon.ico" type="image/x-icon" />
+    </head>
+    <body>
+        <div id="logo">
+        <img src="/static/calibre.png" alt="Calibre" />
+        </div>
+        <div id="search_box">
+        <form method="get" action="/mobile">
+        Show <select name="num">
+            <py:for each="option in [5,10,25,100]">
+                 <option py:if="option == num" value="${option}" SELECTED="SELECTED">${option}</option>
+                 <option py:if="option != num" value="${option}">${option}</option>
+            </py:for>
+            </select>
+        books matching <input name="search" id="s" value="${search}" /> sorted by
+
+        <select name="sort">
+            <py:for each="option in ['date','author','title','rating','size','tags','series']">
+                 <option py:if="option == sort" value="${option}" SELECTED="SELECTED">${option}</option>
+                 <option py:if="option != sort" value="${option}">${option}</option>
+            </py:for>
+        </select>
+        <select name="order">
+            <py:for each="option in ['ascending','descending']">
+                 <option py:if="option == order" value="${option}" SELECTED="SELECTED">${option}</option>
+                 <option py:if="option != order" value="${option}">${option}</option>
+            </py:for>
+        </select>
+        <input id="go" type="submit" value="Search"/>
+        </form>
+        </div>
+        <div class="navigation">
+        <span style="display: block; text-align: center;">Books ${start} to ${ min((start+num-1) , total) } of ${total}</span>
+        <table class="buttons">
+        <tr>
+        <td class="button" style="text-align:left;">
+			<a py:if="start > 1" href="${url_base};start=1">First</a>
+			<a py:if="start > 1" href="${url_base};start=${max(start-(num+1),1)}">Previous</a>
+		</td>
+        <td class="button" style="text-align: right;">
+            <a py:if=" total > (start + num) " href="${url_base};start=${start+num}">Next</a>
+            <a py:if=" total > (start + num) " href="${url_base};start=${total-num+1}">Last</a>
+        </td>
+        </tr>
+        </table>
+        </div>
+        <hr class="spacer" />
+        <table id="listing">
+            <py:for each="book in books">
+                ${Markup(book)}
+            </py:for>
+        </table>
+    </body>
+    </html>
+    '''))
+
     LIBRARY = MarkupTemplate(textwrap.dedent('''\
     <?xml version="1.0" encoding="utf-8"?>
     <library xmlns:py="http://genshi.edgewall.org/" start="$start" num="${len(books)}" total="$total" updated="${updated.strftime('%Y-%m-%dT%H:%M:%S+00:00')}">
@@ -89,7 +242,7 @@ class LibraryServer(object):
     STANZA_ENTRY=MarkupTemplate(textwrap.dedent('''\
     <entry xmlns:py="http://genshi.edgewall.org/">
         <title>${record[FM['title']]}</title>
-        <id>urn:calibre:${record[FM['id']]}</id>
+        <id>urn:calibre:${urn}</id>
         <author><name>${authors}</name></author>
         <updated>${timestamp}</updated>
         <link type="${mimetype}" href="/get/${fmt}/${record[FM['id']]}" />
@@ -525,6 +678,7 @@ class LibraryServer(object):
                     extra='\n'.join(extra),
                     mimetype=mimetype,
                     fmt=fmt,
+                    urn=record[FIELD_MAP['uuid']],
                     timestamp=strftime('%Y-%m-%dT%H:%M:%S+00:00', record[5])
                     )
             books.append(self.STANZA_ENTRY.generate(**data)\
@@ -534,6 +688,52 @@ class LibraryServer(object):
                 next_link=next_link, updated=updated, id='urn:calibre:main').render('xml')
 
 
+    @expose
+    def mobile(self, start='1', num='25', sort='date', search='',
+                _=None, order='descending'):
+        '''
+        Serves metadata from the calibre database as XML.
+
+        :param sort: Sort results by ``sort``. Can be one of `title,author,rating`.
+        :param search: Filter results by ``search`` query. See :class:`SearchQueryParser` for query syntax
+        :param start,num: Return the slice `[start:start+num]` of the sorted and filtered results
+        :param _: Firefox seems to sometimes send this when using XMLHttpRequest with no caching
+        '''
+        try:
+            start = int(start)
+        except ValueError:
+            raise cherrypy.HTTPError(400, 'start: %s is not an integer'%start)
+        try:
+            num = int(num)
+        except ValueError:
+            raise cherrypy.HTTPError(400, 'num: %s is not an integer'%num)
+        ids = self.db.data.parse(search) if search and search.strip() else self.db.data.universal_set()
+        ids = sorted(ids)
+        items = [r for r in iter(self.db) if r[0] in ids]
+        if sort is not None:
+            self.sort(items, sort, (order.lower().strip() == 'ascending'))
+
+        book, books = MarkupTemplate(self.MOBILE_BOOK), []
+        for record in items[(start-1):(start-1)+num]:
+            aus = record[2] if record[2] else __builtin__._('Unknown')
+            authors = '|'.join([i.replace('|', ',') for i in aus.split(',')])
+            record[10] = fmt_sidx(float(record[10]))
+            ts, pd = strftime('%Y/%m/%d %H:%M:%S', record[5]), \
+                strftime('%Y/%m/%d %H:%M:%S', record[FIELD_MAP['pubdate']])
+            books.append(book.generate(r=record, authors=authors, timestamp=ts,
+                pubdate=pd).render('xml').decode('utf-8'))
+        updated = self.db.last_modified()
+
+        cherrypy.response.headers['Content-Type'] = 'text/html; charset=utf-8'
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+
+
+        url_base = "/mobile?search=" + search+";order="+order+";sort="+sort+";num="+str(num)
+
+        return self.MOBILE.generate(books=books, start=start, updated=updated, search=search, sort=sort, order=order, num=num,
+                                     total=len(ids), url_base=url_base).render('html')
+
+
     @expose
     def library(self, start='0', num='50', sort=None, search=None,
                 _=None, order='ascending'):
@@ -584,10 +784,22 @@ class LibraryServer(object):
             cherrypy.request.headers.get('Stanza-Device-Name', 919) != 919 or \
             cherrypy.request.headers.get('Want-OPDS-Catalog', 919) != 919 or \
             ua.startswith('Stanza')
+
+        # A better search would be great
+        want_mobile = self.MOBILE_UA.search(ua) is not None
+        if self.opts.develop and not want_mobile:
+            prints('User agent:', ua)
+
+        if want_opds:
             return self.stanza(search=kwargs.get('search', None), sortby=kwargs.get('sortby',None), authorid=kwargs.get('authorid',None),
                            tagid=kwargs.get('tagid',None),
                            seriesid=kwargs.get('seriesid',None),
-                           offset=kwargs.get('offset', 0)) if want_opds else self.static('index.html')
+                           offset=kwargs.get('offset', 0))
+
+        if want_mobile:
+            return self.mobile()
+
+        return self.static('index.html')
 
 
     @expose

src/calibre/library/sqlite.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Wrapper for multi-threaded access to a single sqlite database connection. Serializes
 all calls.
 '''
-import sqlite3 as sqlite, traceback, time
+import sqlite3 as sqlite, traceback, time, uuid
 from sqlite3 import IntegrityError
 from threading import Thread
 from Queue import Queue
@@ -121,6 +121,7 @@ class DBThread(Thread):
         self.conn.create_aggregate('concat', 1, Concatenate)
         self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
         self.conn.create_function('title_sort', 1, title_sort)
+        self.conn.create_function('uuid4', 0, lambda : str(uuid.uuid4()))
 
     def run(self):
         try:

src/calibre/manual/conversion.rst

@@ -301,11 +301,17 @@ Removing headers and footers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 These options are useful primarily for conversion of PDF documents. Often, the conversion leaves
-behing page headers and footers in the text. These options use regular expressions to try and detect
+behind page headers and footers in the text. These options use regular expressions to try and detect
 the headers and footers and remove them. Remember that they operate on the intermediate XHTML produced
 by the conversion pipeline. There is also a wizard to help you customize the regular expressions for
 your document.
 
+The header and footer regular expressions are used in conjunction with the remove header and footer options.
+If the remove option is not enabled the regular expression will not be applied to remove the matched text.
+The removal works by using a python regular expression. All matched text is simply removed from
+the document. You can learn more about regular expressions and their syntax at
+http://docs.python.org/library/re.html.
+
 Miscellaneous
 ~~~~~~~~~~~~~~
 
@@ -403,23 +409,76 @@ This will result in an automatically generated two level Table of Contents that
 Format specific tips
 ----------------------
 
-Here you will find tips specific to the conversion of particular formats.
+Here you will find tips specific to the conversion of particular formats. Options specific to particular
+format, whether input or output are available in the conversion dialog under their own section, for example
+`TXT Input` or `EPUB Output`.
 
 Convert Microsoft Word documents
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 |app| does not directly convert .doc files from Microsoft Word. However, in Word, you can save the document
 as HTML and then convert the resulting HTML file with |app|. When saving as HTML, be sure to use the
-"Save as filtered HTML" option as this will produce clean HTML that will convert well.
+"Save as Web Page, Filtered" option as this will produce clean HTML that will convert well.
 
 There is a Word macro package that can automate the conversion of Word documents using |app|. It also makes
 generating the Table of Contents much simpler. It is called BookCreator and is available for free
-`here <http://www.mobileread.com/forums/showthread.php?t=36098>`_.
+`here <http://www.mobileread.com/forums/showthread.php?t=28313>`_.
 
 Convert TXT documents
 ~~~~~~~~~~~~~~~~~~~~~~
 
+TXT documents have no well defined way to specify formatting like bold, italics, etc, or document structure like paragraphs, headings, sections and so on.
+Since TXT documents provide no way to explicitly mark parts of
+the text, by default |app| only groups lines in the input document into paragraphs. The default is to assume one or
+more blank lines are a paragraph boundary::
+
+    This is the first.
+    
+    This is the
+    second paragraph.
+
+TXT input supports a number of options to differentiate how paragraphs are detected.
+
+    :guilabel:`Treat each line as a paragraph`
+        Assumes that every line is a paragraph::
+
+            This is the first.
+            This is the second.
+            This is the third.
+        
+    :guilabel:`Assume print formatting`
+        Assumes that every paragraph starts with an indent (either a tab or 2+ spaces). Paragraphs end when
+        the next line that starts with an indent is reached::
+
+            This is the
+            first.
+            This is the second.
+            
+            This is the
+            third.
+
+    :guilabel:`Process using markdown`
+        |app| also supports running TXT input though a transformation preprocessor known as markdown. Markdown
+        allows for basic formatting to be added to TXT documents, such as bold, italics, section headings, tables,
+        loists, a Table of Contents, etc. Marking chapter headings with a leading # and setting the chapter XPath detection
+        expression to "//h:h1" is the easiest way to have a proper table of contents generated from a TXT document.
+        You can learn more about the markdown syntax at http://daringfireball.net/projects/markdown/syntax.
+
+
 Convert PDF documents
-~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+PDF documents are one of the worst formats to convert from. They are a fixed page size and text placement format.
+Meaning, it is very difficult to determine where one paragraph ends and another begins. |app| will try to unwrap
+paragraphs using a configurable, :guilabel:`Line Un-Wrapping Factor`. This is a scale used to determine the length
+at which a line should be unwrapped. Valid values are a decimal
+between 0 and 1. The default is 0.5, this is the median line length. Lower this value to include more
+text in the unwrapping. Increase to include less.
+
+Also, they often have headers and footers as part of the document that will become included with the text.
+Use the options to remove headers and footers to mitigate this issue. If the headers and footers are not
+removed from the text it can throw off the paragraph unwrapping.
+
+Some limitations of PDF input is complex, multi-column, and image based documents are not supported.
+Extraction of vector images and tables from within the document is also not supported.
 

src/calibre/utils/complete.py

@@ -65,6 +65,7 @@ def split(src):
 
 
 def files_and_dirs(prefix, allowed_exts=[]):
+    prefix = os.path.expanduser(prefix)
     for i in glob.iglob(prefix+'*'):
         _, ext = os.path.splitext(i)
         ext = ext.lower().replace('.', '')