Sync to trunk

installer/linux/freeze.py

@@ -93,7 +93,7 @@ def freeze():
                 'dateutil', 'dns', 'email']
 
     includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
-    includes += [x.split('/')[-1].rpartition('.')[0] for x in \
+    includes += ['calibre.gui2.convert.'+x.split('/')[-1].rpartition('.')[0] for x in \
             glob.glob('src/calibre/gui2/convert/*.py')]
 
     LOADER = '/tmp/loader.py'

src/calibre/debug.py

@@ -87,6 +87,7 @@ def migrate(old, new):
     print 'Database migrated to', os.path.abspath(new)
 
 def debug_device_driver():
+    from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner
     s = DeviceScanner()
     s.scan()
@@ -113,17 +114,15 @@ def debug_device_driver():
         raw = Device.run_ioreg()
         open('/tmp/ioreg.txt', 'wb').write(raw)
         print 'ioreg output saved to /tmp/ioreg.txt'
-    from calibre.devices import devices
+    for dev in device_plugins():
-    for dev in devices():
+        print 'Looking for', dev.__class__.__name__
-        print 'Looking for', dev.__name__
         connected = s.is_device_connected(dev)
         if connected:
             print 'Device Connected:', dev
             print 'Trying to open device...'
-            d = dev()
+            dev.open()
-            d.open()
+            print 'Main memory:', repr(dev._main_prefix)
-            print 'Main memory:', repr(d._main_prefix)
+            print 'Total space:', dev.total_space()
-            print 'Total space:', d.total_space()
             break
 
 def add_simple_plugin(path_to_plugin):

src/calibre/devices/prs505/driver.py

@@ -60,9 +60,8 @@ class PRS505(CLI, Device):
                             <cache xmlns="http://www.kinoma.com/FskCache/1">
                             </cache>
                             '''.encode('utf8'))
-                    return True
+                return True
             except:
-                self._card_prefix = None
                 import traceback
                 traceback.print_exc()
             return False

src/calibre/devices/usbms/device.py

@@ -6,7 +6,8 @@ intended to be subclassed with the relevant parts implemented for a particular
 device. This class handles device detection.
 '''
 
-import os, subprocess, time, re
+import os, subprocess, time, re, sys, glob
+from itertools import repeat
 
 from calibre.devices.interface import DevicePlugin
 from calibre.devices.errors import DeviceError
@@ -35,6 +36,7 @@ class Device(DeviceConfig, DevicePlugin):
 
     MAIN_MEMORY_VOLUME_LABEL  = ''
     STORAGE_CARD_VOLUME_LABEL = ''
+    STORAGE_CARD2_VOLUME_LABEL = None
 
     FDI_TEMPLATE = \
 '''
@@ -290,51 +292,172 @@ class Device(DeviceConfig, DevicePlugin):
         self._card_a_prefix = get_card_prefix(card_a_pat)
         self._card_b_prefix = get_card_prefix(card_b_pat)
 
-    def open_linux(self):
+    def find_device_nodes(self):
-        import dbus
-        bus = dbus.SystemBus()
-        hm  = dbus.Interface(bus.get_object("org.freedesktop.Hal", "/org/freedesktop/Hal/Manager"), "org.freedesktop.Hal.Manager")
 
-        def conditional_mount(dev):
+        def walk(base):
-            mmo = bus.get_object("org.freedesktop.Hal", dev)
+            base = os.path.abspath(os.path.realpath(base))
-            label = mmo.GetPropertyString('volume.label', dbus_interface='org.freedesktop.Hal.Device')
+            for x in os.listdir(base):
-            is_mounted = mmo.GetPropertyString('volume.is_mounted', dbus_interface='org.freedesktop.Hal.Device')
+                p = os.path.join(base, x)
-            mount_point = mmo.GetPropertyString('volume.mount_point', dbus_interface='org.freedesktop.Hal.Device')
+                if os.path.islink(p) or not os.access(p, os.R_OK):
-            fstype = mmo.GetPropertyString('volume.fstype', dbus_interface='org.freedesktop.Hal.Device')
+                    continue
-            if is_mounted:
+                isfile = os.path.isfile(p)
-                return str(mount_point)
+                yield p, isfile
-            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'],
+                if not isfile:
-                          dbus_interface='org.freedesktop.Hal.Device.Volume')
+                    for y, q in walk(p):
-            return os.path.normpath('/media/'+label)+'/'
+                        yield y, q
 
-        mm = hm.FindDeviceStringMatch(__appname__+'.mainvolume', self.__class__.__name__)
+        def raw2num(raw):
-        if not mm:
+            raw = raw.lower()
-            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%(self.__class__.__name__,))
+            if not raw.startswith('0x'):
-        self._main_prefix = None
+                raw = '0x' + raw
-        for dev in mm:
+            return int(raw, 16)
-            try:
+
-                self._main_prefix = conditional_mount(dev)+os.sep
+        # Find device node based on vendor, product and bcd
-                break
+        d, j = os.path.dirname, os.path.join
-            except dbus.exceptions.DBusException:
+        usb_dir = None
+
+        def test(val, attr):
+            q = getattr(self, attr)
+            if q is None: return True
+            return q == val or val in q
+
+        for x, isfile in walk('/sys/devices'):
+            if isfile and x.endswith('idVendor'):
+                usb_dir = d(x)
+                for y in ('idProduct',):
+                    if not os.access(j(usb_dir, y), os.R_OK):
+                        usb_dir = None
+                        continue
+                e = lambda q : raw2num(open(j(usb_dir, q)).read())
+                ven, prod = map(e, ('idVendor', 'idProduct'))
+                if not (test(ven, 'VENDOR_ID') and test(prod, 'PRODUCT_ID')):
+                    usb_dir = None
+                    continue
+                if self.BCD is not None:
+                    if not os.access(j(usb_dir, 'bcdDevice'), os.R_OK) or \
+                            not test(e('bcdDevice'), 'BCD'):
+                        usb_dir = None
+                        continue
+                    else:
+                        break
+                else:
+                    break
+
+        if usb_dir is None:
+            raise DeviceError(_('Unable to detect the %s disk drive.')
+                    %self.__class__.__name__)
+
+        devnodes, ok = [], {}
+        for x, isfile in walk(usb_dir):
+            if not isfile and '/block/' in x:
+                parts = x.split('/')
+                idx = parts.index('block')
+                if idx == len(parts)-2:
+                    sz = j(x, 'size')
+                    node = parts[idx+1]
+                    try:
+                        exists = int(open(sz).read()) > 0
+                        if exists:
+                            node = self.find_largest_partition(x)
+                            ok[node] = True
+                        else:
+                            ok[node] = False
+                    except:
+                        ok[node] = False
+                    if ok[node]:
+                        devnodes.append(node)
+        devnodes += list(repeat(None, 3))
+        return tuple(['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]])
+
+    def node_mountpoint(self, node):
+        for line in open('/proc/mounts').readlines():
+            line = line.split()
+            if line[0] == node:
+                return line[1]
+        return None
+
+    def find_largest_partition(self, path):
+        node = path.split('/')[-1]
+        nodes = []
+        for x in glob.glob(path+'/'+node+'*'):
+            sz = x + '/size'
+
+            if not os.access(sz, os.R_OK):
                 continue
-
-        if not self._main_prefix:
-            raise DeviceError('Could not open device for reading. Try a reboot.')
-
-        self._card_a_prefix = self._card_b_prefix = None
-        cards = hm.FindDeviceStringMatch(__appname__+'.cardvolume', self.__class__.__name__)
-
-        def mount_card(dev):
             try:
-                return conditional_mount(dev)+os.sep
+                sz = int(open(sz).read())
             except:
-                import traceback
+                continue
-                print traceback
+            if sz > 0:
+                nodes.append((x.split('/')[-1], sz))
 
-        if len(cards) >= 1:
+        nodes.sort(cmp=lambda x, y: cmp(x[1], y[1]))
-            self._card_a_prefix = mount_card(cards[0])
+        if not nodes:
-        if len(cards) >=2:
+            return node
-            self._card_b_prefix = mount_card(cards[1])
+        return nodes[-1][0]
+
+
+    def open_linux(self):
+
+        def mount(node, type):
+            mp = self.node_mountpoint(node)
+            if mp is not None:
+                return mp, 0
+            if type == 'main':
+                label = self.MAIN_MEMORY_VOLUME_LABEL
+            if type == 'carda':
+                label = self.STORAGE_CARD_VOLUME_LABEL
+            if type == 'cardb':
+                label = self.STORAGE_CARD2_VOLUME_LABEL
+                if label is None:
+                    label = self.STORAGE_CARD_VOLUME_LABEL + ' 2'
+            extra = 0
+            label = label.replace(' ', '_')
+            while True:
+                q = '_(%d)'%extra if extra else ''
+                if not os.path.exists('/media/'+label+q):
+                    break
+                extra += 1
+            if extra:
+                label += '_(%d)'%extra
+
+            def do_mount(node, label):
+                cmd = ['pmount', '-w', '-s']
+                try:
+                    p = subprocess.Popen(cmd + [node, label])
+                except OSError:
+                    raise DeviceError(_('You must install the pmount package.'))
+                while p.poll() is None:
+                    time.sleep(0.1)
+                return p.returncode
+
+            ret = do_mount(node, label)
+            if ret != 0:
+                return None, ret
+            return self.node_mountpoint(node)+'/', 0
+
+
+        main, carda, cardb = self.find_device_nodes()
+        if main is None:
+            raise DeviceError(_('Unable to detect the %s disk drive.')
+                    %self.__class__.__name__)
+
+        mp, ret = mount(main, 'main')
+        if mp is None:
+            raise DeviceError(
+            _('Unable to mount main memory (Error code: %d)')%ret)
+        if not mp.endswith('/'): mp += '/'
+        self._main_prefix = mp
+        cards = [x for x in (carda, cardb) if x is not None]
+        prefix, typ = '_card_a_prefix', 'carda'
+        for card in cards:
+            mp, ret = mount(card, typ)
+            if mp is None:
+                print >>sys.stderr, 'Unable to mount card (Error code: %d)'%ret
+            else:
+                if not mp.endswith('/'): mp += '/'
+                setattr(self, prefix, mp)
+                prefix, typ = '_card_b_prefix', 'cardb'
 
     def open(self):
         time.sleep(5)

src/calibre/ebooks/__init__.py

@@ -23,7 +23,7 @@ class DRMError(ValueError):
     pass
 
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm',
-                   'html', 'xhtml', 'pdf', 'prc', 'mobi', 'azw',
+                   'html', 'xhtml', 'pdf', 'prc', 'mobi', 'azw', 'doc',
                    'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'oebzip',
                    'rb', 'imp', 'odt']
 

src/calibre/ebooks/lit/output.py

@@ -7,8 +7,7 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from calibre.customize.conversion import OutputFormatPlugin, \
+from calibre.customize.conversion import OutputFormatPlugin
-        OptionRecommendation
 
 class LITOutput(OutputFormatPlugin):
 
@@ -16,10 +15,6 @@ class LITOutput(OutputFormatPlugin):
     author = 'Marshall T. Vandegrift'
     file_type = 'lit'
 
-    recommendations = set([
-        ('dont_split_on_page_breaks', False, OptionRecommendation.HIGH),
-        ])
-
     def convert(self, oeb, output_path, input_plugin, opts, log):
         self.log, self.opts, self.oeb = log, opts, oeb
         from calibre.ebooks.oeb.transforms.manglecase import CaseMangler
@@ -27,9 +22,7 @@ class LITOutput(OutputFormatPlugin):
         from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder
         from calibre.ebooks.lit.writer import LitWriter
         from calibre.ebooks.oeb.transforms.split import Split
-        split = Split(not self.opts.dont_split_on_page_breaks,
+        split = Split(split_on_page_breaks=True, max_flow_size=0)
-                max_flow_size=0
-                )
         split(self.oeb, self.opts)
 
 

src/calibre/linux.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Post installation script for linux '''
 import sys, os, shutil
-from subprocess import check_call, call
+from subprocess import check_call
 
 from calibre import __version__, __appname__
 from calibre.customize.ui import device_plugins
@@ -263,49 +263,6 @@ def setup_udev_rules(group_file, reload, fatal_errors):
                '''BUS=="usb", SYSFS{idProduct}=="029b", SYSFS{idVendor}=="054c", MODE="660", GROUP="%s"\n'''%(group,)
              )
     udev.close()
-    fdi = open_file('/usr/share/hal/fdi/policy/20thirdparty/10-calibre.fdi')
-    manifest.append(fdi.name)
-    fdi.write('<?xml version="1.0" encoding="UTF-8"?>\n\n<deviceinfo version="0.2">\n')
-    for cls in DEVICES:
-        fdi.write(\
-'''
-  <device>
-      <match key="usb_device.vendor_id" int="%(vendor_id)s">
-          <match key="usb_device.product_id" int="%(product_id)s">
-              <match key="usb_device.device_revision_bcd" int="%(bcd)s">
-                  <merge key="calibre.deviceclass" type="string">%(cls)s</merge>
-              </match>
-          </match>
-      </match>
-  </device>
-'''%dict(cls=cls.__class__.__name__, vendor_id=cls.VENDOR_ID, product_id=cls.PRODUCT_ID,
-         prog=__appname__, bcd=cls.BCD))
-        fdi.write('\n'+cls.get_fdi())
-    fdi.write('\n</deviceinfo>\n')
-    fdi.close()
-    if reload:
-        called = False
-        for hal in ('hald', 'hal', 'haldaemon'):
-            hal = os.path.join('/etc/init.d', hal)
-            if os.access(hal, os.X_OK):
-                call((hal, 'restart'))
-                called = True
-                break
-        if not called and os.access('/etc/rc.d/rc.hald', os.X_OK):
-            call(('/etc/rc.d/rc.hald', 'restart'))
-
-        try:
-            check_call('udevadm control --reload_rules', shell=True)
-        except:
-            try:
-                check_call('udevcontrol reload_rules', shell=True)
-            except:
-                try:
-                    check_call('/etc/init.d/udev reload', shell=True)
-                except:
-                    if fatal_errors:
-                        raise Exception("Couldn't reload udev, you may have to reboot")
-                    print >>sys.stderr, "Couldn't reload udev, you may have to reboot"
     return manifest
 
 def option_parser():
@@ -314,7 +271,7 @@ def option_parser():
     parser.add_option('--use-destdir', action='store_true', default=False, dest='destdir',
                       help='If set, respect the environment variable DESTDIR when installing files')
     parser.add_option('--do-not-reload-udev-hal', action='store_true', dest='dont_reload', default=False,
-                      help='If set, do not try to reload udev rules and HAL FDI files')
+                      help='Does nothing. Present for legacy reasons.')
     parser.add_option('--group-file', default='/etc/group', dest='group_file',
                       help='File from which to read group information. Default: %default')
     parser.add_option('--dont-check-root', action='store_true', default=False, dest='no_root',

src/calibre/trac/plugins/download.py

@@ -21,6 +21,7 @@ DEPENDENCIES = [
             ('dnspython', '1.6.0', 'dnspython', 'dnspython', 'dnspython', 'dnspython'),
             ('poppler', '0.10.5', 'poppler', 'poppler', 'poppler', 'poppler'),
             ('podofo', '0.7', 'podofo', 'podofo', 'podofo', 'podofo'),
+            ('pmount', '0.9.19', 'pmount', 'pmount', 'pmount', 'pmount'),
             ]
 
 

src/calibre/web/feeds/recipes/__init__.py

@@ -44,7 +44,8 @@ recipe_modules = ['recipe_' + r for r in (
            'stackoverflow', 'telepolis_artikel', 'zaobao', 'usnews',
            'straitstimes', 'index_hu', 'pcworld_hu', 'hrt', 'rts',
            'h1', 'h2', 'h3', 'phd_comics', 'woz_die', 'elektrolese',
-           'climate_progress', 'carta', 'slashdot',
+           'climate_progress', 'carta', 'slashdot', 'publico',
+           'the_budget_fashionista'
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_publico.py

@@ -0,0 +1,40 @@
+"""
+publico.py - v1.0
+
+Copyright (c) 2009, David Rodrigues - http://sixhat.net
+All rights reserved.
+"""
+
+__license__ = 'GPL 3'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class Publico(BasicNewsRecipe):
+    title          = u'P\xc3\xbablico'
+    __author__     = 'David Rodrigues'
+    oldest_article = 1
+    max_articles_per_feed = 30
+    encoding='utf-8'
+    no_stylesheets = True
+    language = _('Portuguese')
+    preprocess_regexps = [(re.compile(u"\uFFFD", re.DOTALL|re.IGNORECASE),  lambda match: ''),]
+
+    feeds          = [
+                        (u'Geral', u'http://feeds.feedburner.com/PublicoUltimaHora'),
+                        (u'Internacional', u'http://www.publico.clix.pt/rss.ashx?idCanal=11'),
+                        (u'Pol\xc3\xadtica', u'http://www.publico.clix.pt/rss.ashx?idCanal=12'),
+                        (u'Ci\xc3\xaancias', u'http://www.publico.clix.pt/rss.ashx?idCanal=13'),
+                        (u'Desporto', u'http://desporto.publico.pt/rss.ashx'),
+                        (u'Economia', u'http://www.publico.clix.pt/rss.ashx?idCanal=57'),
+                        (u'Educa\xc3\xa7\xc3\xa3o', u'http://www.publico.clix.pt/rss.ashx?idCanal=58'),
+                        (u'Local', u'http://www.publico.clix.pt/rss.ashx?idCanal=59'),
+                        (u'Media e Tecnologia', u'http://www.publico.clix.pt/rss.ashx?idCanal=61'),
+                        (u'Sociedade', u'http://www.publico.clix.pt/rss.ashx?idCanal=62')
+                    ]
+    remove_tags    = [dict(name='script'), dict(id='linhaTitulosHeader')]
+    keep_only_tags = [dict(name='div')]
+
+    def print_version(self,url):
+        s=re.findall("id=[0-9]+",url);
+        return "http://ww2.publico.clix.pt/print.aspx?"+s[0]

src/calibre/web/feeds/recipes/recipe_the_budget_fashionista.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.thebudgetfashionista.com
+'''
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
+
+class TheBudgetFashionista(BasicNewsRecipe):
+    title                  = 'The Budget Fashionista'
+    __author__             = 'Darko Miletic'
+    description            = 'Saving your money since 2003'
+    oldest_article         = 7
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    encoding               = 'utf-8'
+    publisher              = 'TBF GROUP, LLC.'
+    category               = 'news, fashion, comsetics, women'
+    lang                   = 'en-US'
+    language               = _('English')
+
+    preprocess_regexps = [(re.compile(r"</head>{0,1}", re.DOTALL|re.IGNORECASE),lambda match: '')]
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"'
+
+    keep_only_tags = [dict(name='div', attrs={'id':'singlepost'})]
+    remove_tags_after = dict(name='div', attrs={'id':'postnav'})
+    remove_tags = [
+                     dict(name=['object','link','script','iframe','form'])
+                    ,dict(name='div', attrs={'id':'postnav'})
+                  ]
+
+    feeds = [(u'Articles', u'http://www.thebudgetfashionista.com/feeds/atom/')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup
+
+    def postprocess_html(self, soup, x):
+        body = soup.find('body')
+        post = soup.find('div', attrs={'id':'singlepost'})
+        if post and body:
+          post.extract()
+          body.extract()
+          soup.html.append(body)
+          body.insert(1,post)
+        mlang = Tag(soup,'meta',[("http-equiv","Content-Language"),("content",self.lang)])
+        mcharset = Tag(soup,'meta',[("http-equiv","Content-Type"),("content","text/html; charset=utf-8")])
+        soup.head.insert(0,mlang)
+        soup.head.insert(1,mcharset)
+        return self.adeify_images(soup)