Pull from trunk

src/calibre/__init__.py

@@ -26,12 +26,16 @@ mimetypes.add_type('text/x-sony-bbeb+xml',                '.lrs')
 mimetypes.add_type('application/xhtml+xml',               '.xhtml')
 mimetypes.add_type('image/svg+xml',                       '.svg')
 mimetypes.add_type('application/x-sony-bbeb',             '.lrf')
+mimetypes.add_type('application/x-sony-bbeb',             '.lrx')
 mimetypes.add_type('application/x-dtbncx+xml',            '.ncx')
 mimetypes.add_type('application/adobe-page-template+xml', '.xpgt')
 mimetypes.add_type('application/x-font-opentype',         '.otf')
 mimetypes.add_type('application/x-font-truetype',         '.ttf')
 mimetypes.add_type('application/oebps-package+xml',       '.opf')
 mimetypes.add_type('application/ereader',                 '.pdb')
+mimetypes.add_type('application/mobi',                    '.mobi')
+mimetypes.add_type('application/mobi',                    '.prc')
+mimetypes.add_type('application/mobi',                    '.azw')
 guess_type = mimetypes.guess_type
 import cssutils
 cssutils.log.setLevel(logging.WARN)

src/calibre/customize/builtins.py

@@ -132,13 +132,24 @@ class HTMLMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read MOBI metadata'
-    file_types  = set(['mobi', 'prc', '.azw'])
+    file_types  = set(['mobi', 'prc', 'azw'])
     description = _('Read metadata from %s files')%'MOBI'
     
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.mobi.reader import get_metadata
         return get_metadata(stream)
 
+
+class TOPAZMetadataReader(MetadataReaderPlugin):
+    
+    name        = 'Read Topaz metadata'
+    file_types  = set(['tpz', 'azw1'])
+    description = _('Read metadata from %s files')%'MOBI'
+    
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.topaz import get_metadata
+        return get_metadata(stream)
+
 class ODTMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read ODT metadata'

src/calibre/devices/kindle/driver.py

@@ -4,9 +4,9 @@ __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 Device driver for Amazon's Kindle
 '''
 
-import os
+import os, re
 
-from calibre.devices.usbms.driver import USBMS
+from calibre.devices.usbms.driver import USBMS, metadata_from_formats
 
 class KINDLE(USBMS):
     # Ordered list of supported formats
@@ -30,6 +30,9 @@ class KINDLE(USBMS):
     EBOOK_DIR_CARD = "documents"
     SUPPORTS_SUB_DIRS = True
     
+    WIRELESS_FILE_NAME_PATTERN = re.compile(
+    r'(?P<title>[^-]+)-asin_(?P<asin>[a-zA-Z\d]{10,})-type_(?P<type>\w{4})-v_(?P<index>\d+).*')
+
     def delete_books(self, paths, end_session=True):
         for path in paths:
             if os.path.exists(path):
@@ -41,6 +44,16 @@ class KINDLE(USBMS):
                 if os.path.exists(filepath + '.mbp'):
                     os.unlink(filepath + '.mbp')
                     
+    @classmethod
+    def metadata_from_path(cls, path):
+        mi = metadata_from_formats([path])
+        if mi.title == _('Unknown') or ('-asin' in mi.title and '-type' in mi.title):
+            match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path))
+            if match is not None:
+                mi.title = match.group('title')
+        return mi
+
+
 class KINDLE2(KINDLE):
     
     PRODUCT_ID = [0x0002]

src/calibre/devices/mime.py

@@ -1,19 +1,20 @@
-__license__   = 'GPL v3'
+from __future__ import with_statement
-__copyright__ = '2009, John Schember <john at nachtimwald.com>'
+__license__ = 'GPL 3'
-'''
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-Global Mime mapping of ebook types.
+__docformat__ = 'restructuredtext en'
-'''
 
-MIME_MAP   = {
+from calibre import guess_type
-                'azw'  : 'application/azw',
-                'epub' : 'application/epub+zip',
-                'html' : 'text/html',
-                'lrf'  : 'application/x-sony-bbeb',
-                'lrx'  : 'application/x-sony-bbeb',
-                'mobi' : 'application/mobi',
-                'pdf'  : 'application/pdf',
-                'prc'  : 'application/prc',
-                'rtf'  : 'application/rtf',
-                'txt'  : 'text/plain',
-              }
 
+def _mt(path):
+    mt = guess_type(path)[0]
+    if not mt:
+        mt = 'application/octet-stream'
+    return mt
+
+def mime_type_ext(ext):
+    if not ext.startswith('.'):
+        ext = '.'+ext
+    return _mt('a'+ext)
+
+def mime_type_path(path):
+    return _mt(path)

src/calibre/devices/usbms/driver.py

@@ -15,7 +15,7 @@ from calibre.ebooks.metadata import authors_to_string
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
 from calibre.devices.errors import FreeSpaceError, PathError
-from calibre.devices.mime import MIME_MAP
+from calibre.devices.mime import mime_type_ext
 
 class File(object):
     def __init__(self, path):
@@ -226,14 +226,17 @@ class USBMS(Device):
         if not os.path.isdir(path):
             os.utime(path, None)
 
+    @classmethod
+    def metadata_from_path(cls, path):
+        return metadata_from_formats([path])
+    
     @classmethod
     def book_from_path(cls, path):
         fileext = path_to_ext(path)
-    
+        mi = cls.metadata_from_path(path)
-        mi = metadata_from_formats([path])
+        mime = mime_type_ext(fileext)
-        mime = MIME_MAP[fileext] if fileext in MIME_MAP.keys() else 'Unknown'
-        
         authors = authors_to_string(mi.authors)
         
-        return Book(path, mi.title, authors, mime)
+        book = Book(path, mi.title, authors, mime)
+        return book
 

src/calibre/ebooks/epub/iterator.py

@@ -118,11 +118,17 @@ class EbookIterator(object):
         self.spine = [SpineItem(i.path) for i in self.opf.spine]
         
         cover = self.opf.cover
-        if os.path.splitext(self.pathtoebook)[1].lower() in ('.lit', '.mobi', '.prc') and cover:
+        if os.path.splitext(self.pathtoebook)[1].lower() in \
+                                    ('.lit', '.mobi', '.prc') and cover:
             cfile = os.path.join(os.path.dirname(self.spine[0]), 'calibre_ei_cover.html')
             open(cfile, 'wb').write(TITLEPAGE%cover)
             self.spine[0:0] = [SpineItem(cfile)]
             
+        if self.opf.path_to_html_toc is not None and \
+           self.opf.path_to_html_toc not in self.spine:
+            self.spine.append(SpineItem(self.opf.path_to_html_toc))
+            
+        
         sizes = [i.character_count for i in self.spine]
         self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
         for p, s in zip(self.pages, self.spine):

src/calibre/ebooks/metadata/meta.py

@@ -15,7 +15,7 @@ _METADATA_PRIORITIES = [
                        'html', 'htm', 'xhtml', 'xhtm',
                        'rtf', 'fb2', 'pdf', 'prc', 'odt',
                        'epub', 'lit', 'lrx', 'lrf', 'mobi',
-                       'rb', 'imp'
+                       'rb', 'imp', 'azw'
                       ]
 
 # The priorities for loading metadata from different file types
@@ -41,7 +41,9 @@ def metadata_from_formats(formats):
     for path, ext in zip(formats, extensions):
         with open(path, 'rb') as stream:
             try:
-                mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True))
+                newmi = get_metadata(stream, stream_type=ext, 
+                                     use_libprs_metadata=True)
+                mi.smart_update(newmi)
             except:
                 continue
             if getattr(mi, 'application_id', None) is not None:
@@ -58,7 +60,7 @@ def get_metadata(stream, stream_type='lrf', use_libprs_metadata=False):
     if stream_type: stream_type = stream_type.lower()
     if stream_type in ('html', 'html', 'xhtml', 'xhtm', 'xml'):
         stream_type = 'html'
-    if stream_type in ('mobi', 'prc'):
+    if stream_type in ('mobi', 'prc', 'azw'):
         stream_type = 'mobi'
     if stream_type in ('odt', 'ods', 'odp', 'odg', 'odf'):
         stream_type = 'odt'

src/calibre/ebooks/metadata/opf2.py

@@ -444,6 +444,7 @@ class OPF(object):
         if not hasattr(stream, 'read'):
             stream = open(stream, 'rb')
         self.basedir  = self.base_dir = basedir
+        self.path_to_html_toc = None
         raw, self.encoding = xml_to_unicode(stream.read(), strip_encoding_pats=True, resolve_entities=True)
         raw = raw[raw.find('<'):]
         self.root     = etree.fromstring(raw, self.PARSER)
@@ -495,6 +496,7 @@ class OPF(object):
                     if f:
                         self.toc.read_ncx_toc(f[0])
             else:
+                self.path_to_html_toc = toc
                 self.toc.read_html_toc(toc)
         except:
             pass

src/calibre/ebooks/metadata/topaz.py

@@ -0,0 +1,40 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+''' Read metadata from Amazon's topaz format '''
+
+def read_record(raw, name):
+    idx = raw.find(name)
+    if idx > -1:
+        length = ord(raw[idx+len(name)])
+        return raw[idx+len(name)+1:idx+len(name)+1+length]
+
+def get_metadata(stream):
+    raw = stream.read(8*1024)
+    if not raw.startswith('TPZ'):
+        raise ValueError('Not a Topaz file')
+    first = raw.find('metadata')
+    if first < 0:
+        raise ValueError('Invalid Topaz file')
+    second = raw.find('metadata', first+10)
+    if second < 0:
+        raise ValueError('Invalid Topaz file')
+    raw = raw[second:second+1000]
+    authors = read_record(raw, 'Authors')
+    if authors:
+        authors = authors.decode('utf-8', 'replace').split(';')
+    else:
+        authors = [_('Unknown')]
+    title = read_record(raw, 'Title')
+    if title:
+        title = title.decode('utf-8', 'replace')
+    else:
+        raise ValueError('No metadata in file')
+    from calibre.ebooks.metadata import MetaInformation
+    return MetaInformation(title, authors)
+
+if __name__ == '__main__':
+    import sys
+    print get_metadata(open(sys.argv[1], 'rb'))

src/calibre/ebooks/mobi/reader.py

@@ -92,6 +92,8 @@ class BookHeader(object):
             self.sublanguage = 'NEUTRAL'
             self.exth_flag, self.exth = 0, None
             self.ancient = True
+            self.first_image_index = -1
+            self.mobi_version = 1
         else:
             self.ancient = False
             self.doctype = raw[16:20]
@@ -537,7 +539,7 @@ class MobiReader(object):
             os.makedirs(output_dir)
         image_index = 0
         self.image_names = []
-        start = self.book_header.first_image_index
+        start = getattr(self.book_header, 'first_image_index', -1)
         if start > self.num_sections or start < 0:
             # BAEN PRC files have bad headers 
             start=0

src/calibre/gui2/add.py

@@ -224,7 +224,10 @@ class AddRecursive(Add):
             files = _('<p>Books with the same title as the following already '
                       'exist in the database. Add them anyway?<ul>')
             for mi in self.duplicates:
-                files += '<li>'+mi[0].title+'</li>\n'
+                title = mi[0].title
+                if not isinstance(title, unicode):
+                    title = title.decode(preferred_encoding, 'replace')
+                files += '<li>'+title+'</li>\n'
             d = WarningDialog (_('Duplicates found!'), 
                               _('Duplicates found!'), 
                               files+'</ul></p>', parent=self._parent)

src/calibre/gui2/dialogs/metadata_single.ui

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <ui version="4.0">
  <class>MetadataSingleDialog</class>
  <widget class="QDialog" name="MetadataSingleDialog">
@@ -10,7 +11,7 @@
    </rect>
   </property>
   <property name="sizePolicy">
-   <sizepolicy vsizetype="MinimumExpanding" hsizetype="MinimumExpanding" >
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
@@ -51,7 +52,7 @@
         <number>0</number>
        </property>
        <item>
-        <widget class="QWidget" native="1" name="central_widget" >
+        <widget class="QWidget" name="central_widget" native="true">
          <property name="minimumSize">
           <size>
            <width>800</width>
@@ -92,7 +93,7 @@
                    </property>
                   </widget>
                  </item>
-                 <item rowspan="2" row="0" column="2" >
+                 <item row="0" column="2" rowspan="2">
                   <widget class="QToolButton" name="swap_button">
                    <property name="toolTip">
                     <string>Swap the author and title</string>
@@ -226,7 +227,7 @@
                    <item>
                     <widget class="QLineEdit" name="tags">
                      <property name="toolTip">
-                      <string>Tags categorize the book. This is particularly useful while searching. &lt;br>&lt;br>They can be any words or phrases, separated by commas.</string>
+                      <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
                      </property>
                     </widget>
                    </item>
@@ -270,7 +271,7 @@
                    <item>
                     <widget class="QComboBox" name="series">
                      <property name="sizePolicy">
-                      <sizepolicy vsizetype="Fixed" hsizetype="MinimumExpanding" >
+                      <sizepolicy hsizetype="MinimumExpanding" vsizetype="Fixed">
                        <horstretch>0</horstretch>
                        <verstretch>0</verstretch>
                       </sizepolicy>
@@ -374,7 +375,7 @@
               <item>
                <widget class="QPushButton" name="fetch_metadata_button">
                 <property name="text">
-                 <string>Fetch metadata from server</string>
+                 <string>&amp;Fetch metadata from server</string>
                 </property>
                </widget>
               </item>
@@ -385,7 +386,7 @@
               <item>
                <widget class="QGroupBox" name="af_group_box">
                 <property name="sizePolicy">
-                 <sizepolicy vsizetype="Minimum" hsizetype="Preferred" >
+                 <sizepolicy hsizetype="Preferred" vsizetype="Minimum">
                   <horstretch>0</horstretch>
                   <verstretch>0</verstretch>
                  </sizepolicy>
@@ -396,10 +397,10 @@
                 <layout class="QVBoxLayout" name="verticalLayout">
                  <item>
                   <layout class="QGridLayout" name="gridLayout">
-                   <item rowspan="3" row="0" column="0" >
+                   <item row="0" column="0" rowspan="3">
                     <widget class="QListWidget" name="formats">
                      <property name="sizePolicy">
-                      <sizepolicy vsizetype="Minimum" hsizetype="Minimum" >
+                      <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
                        <horstretch>0</horstretch>
                        <verstretch>0</verstretch>
                       </sizepolicy>
@@ -487,7 +488,7 @@
               <item>
                <widget class="QGroupBox" name="bc_box">
                 <property name="sizePolicy">
-                 <sizepolicy vsizetype="Expanding" hsizetype="Preferred" >
+                 <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
                   <horstretch>0</horstretch>
                   <verstretch>10</verstretch>
                  </sizepolicy>
@@ -499,7 +500,7 @@
                  <item>
                   <widget class="ImageView" name="cover">
                    <property name="sizePolicy">
-                    <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+                    <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
                      <horstretch>0</horstretch>
                      <verstretch>0</verstretch>
                     </sizepolicy>
@@ -588,7 +589,7 @@
                    <item>
                     <widget class="QPushButton" name="fetch_cover_button">
                      <property name="text">
-                      <string>Fetch cover image from server</string>
+                      <string>Fetch &amp;cover image from server</string>
                      </property>
                     </widget>
                    </item>
@@ -598,7 +599,7 @@
                       <string>Change the username and/or password for your account at LibraryThing.com</string>
                      </property>
                      <property name="text">
-                      <string>Change password</string>
+                      <string>Change &amp;password</string>
                      </property>
                     </widget>
                    </item>

src/calibre/gui2/viewer/documentview.py

@@ -450,7 +450,9 @@ class DocumentView(QWebView):
                 self.manager.scrolled(self.scroll_fraction)
             
     def wheel_event(self, down=True):
-        QWebView.wheelEvent(self, QWheelEvent(QPoint(100, 100), (-120 if down else 120), Qt.NoButton, Qt.NoModifier))
+        QWebView.wheelEvent(self, 
+            QWheelEvent(QPoint(100, 100), (-120 if down else 120), 
+                        Qt.NoButton, Qt.NoModifier))
     
     def next_page(self):
         if self.document.at_bottom:
@@ -538,6 +540,26 @@ class DocumentView(QWebView):
             self.next_page()
         elif key in [Qt.Key_PageUp, Qt.Key_Backspace, Qt.Key_Up]:
             self.previous_page()
+        elif key in [Qt.Key_Home]:
+            if event.modifiers() & Qt.ControlModifier:
+                if self.manager is not None:
+                    self.manager.goto_start()
+            else:
+                self.scroll_to(0)
+        elif key in [Qt.Key_End]:
+            if event.modifiers() & Qt.ControlModifier:
+                if self.manager is not None:
+                    self.manager.goto_end()
+            else:
+                self.scroll_to(1)
+        elif key in [Qt.Key_J]:
+            self.wheel_event()
+        elif key in [Qt.Key_K]:
+            self.wheel_event(down=False)
+        elif key in [Qt.Key_H]:
+            self.scroll_by(x=-15)
+        elif key in [Qt.Key_L]:
+            self.scroll_by(x=15)
         else:
             return QWebView.keyPressEvent(self, event)
         

src/calibre/gui2/viewer/main.py

@@ -342,6 +342,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if pos is not None:
             self.goto_page(pos)
    
+    def goto_start(self):
+        self.goto_page(1)
+        
+    def goto_end(self):
+        self.goto_page(self.pos.maximum())
+    
     def goto_page(self, new_page):
         if self.current_page is not None:
             for page in self.iterator.spine:

src/calibre/library/server.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 HTTP server for remote access to the calibre database.
 '''
 
-import sys, textwrap, operator, os, re, logging, subprocess
+import sys, textwrap, operator, os, re, logging
 from itertools import repeat
 from logging.handlers import RotatingFileHandler
 from datetime import datetime

src/calibre/manual/faq.rst

@@ -123,15 +123,12 @@ turned into a collection on the reader. Note that the PRS-500 does not support c
 How do I use |app| with my iPhone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 First install the Stanza reader on your iPhone from http://www.lexcycle.com . Then,
+
   * Set the output format for calibre to EPUB (The output format can be set next to the big red heart) 
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
-  * Now you should be able to access your books on your iPhone. If not, try the following:
-    In the Stanza reader on your iPhone, add a new catalog. The URL of the catalog is of the form 
-    ``http://10.34.56.89:8080/stanza``,  where you should replace the IP address ``10.34.56.89`` 
-    with the IP address of your computer. Stanza will the use the |app| content server to access all the
-    EPUB books in your |app| database.
 
+Now you should be able to access your books on your iPhone. 
 
 Library Management
 ------------------

src/calibre/trac/plugins/download.py

@@ -177,11 +177,11 @@ else:
                 compatibility='%s works on OS X Tiger and above.'%(__appname__,),
                 path=MOBILEREAD+file, app=__appname__,
                 note=Markup(\
-    '''
+    u'''
     <ol>
     <li>Before trying to use the command line tools, you must run the app at least once. This will ask you for you password and then setup the symbolic links for the command line tools.</li>
     <li>The app cannot be run from within the dmg. You must drag it to a folder on your filesystem (The Desktop, Applications, wherever).</li> 
-    <li>In order for localization of the user interface in your language, select your language in the configuration dialog (by clicking the hammer icon next to the search bar) and select your language.</li>
+    <li>In order for localization of the user interface in your language, select your language in the preferences (by pressing u\2318+P) and select your language.</li>
     </ol>
     '''))
             return 'binary.html', data, None

src/calibre/utils/Zeroconf.py

@@ -933,7 +933,11 @@ class Reaper(threading.Thread):
 
     def run(self):
         while 1:
+            try:
                 self.zeroconf.wait(10 * 1000)
+            except TypeError: # By Kovid
+                globals()['_GLOBAL_DONE'] = 1
+                return
             if globals()['_GLOBAL_DONE']:
                 return
             now = currentTimeMillis()

src/calibre/utils/mdns.py

@@ -41,7 +41,7 @@ def publish(desc, type, port, properties=None, add_hostname=True):
     '''
     port = int(port)
     server = start_server()
-    hostname = socket.gethostname()
+    hostname = socket.gethostname().partition('.')[0]
     if add_hostname:
         desc += ' (on %s)'%hostname
     local_ip = get_external_ip()
@@ -58,40 +58,3 @@ def stop_server():
     global _server
     if _server is not None:
         _server.close()
-
-'''
-class Publish(object):
-    
-    def __init__(self, desc, name, port, txt=''):
-        self.desc = desc
-        self.name = name
-        self.port = port
-        self.txt  = txt
-        
-    def start(self):
-        if iswindows:
-            return
-        if isosx:
-            args = ['dns-sd', self.desc, self.name, '.', self.port]
-        else:
-            args = ['avahi-publish-service', self.desc, self.name, self.port]
-        if self.txt:
-            args.append(self.txt)
-            
-        self.process = subprocess.Popen(args)
-    
-    def stop(self):
-        if iswindows:
-            pass
-        else:
-            process = getattr(self, 'process', None)
-            if process is not None:
-                process.poll()
-                if process.returncode is not None:
-                    process.terminate()
-                    process.poll()
-                    if process.returncode is not None:
-                        process.kill()
-
-def publish(desc, name, port, txt):
-''' 

src/calibre/web/feeds/recipes/recipe_new_scientist.py

@@ -1,14 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
-'''
-newscientist.com
-'''
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>, AprilHare'
 '''
 newscientist.com
 '''
@@ -16,23 +9,34 @@ newscientist.com
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
-    title                 = u'New Scientist - Online News'
+    title                 = 'New Scientist - Online News'
     __author__            = 'Darko Miletic'
-    description           = 'News from Science'
+    description           = 'Science news and science articles from New Scientist.'
     language              = _('English')
+    publisher             = 'New Scientist'
+    category              = 'science news, science articles, science jobs, drugs, cancer, depression, computer software, sex'    
+    delay                 = 3    
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    remove_javascript     = True
+    encoding              = 'utf-8'
     
-    keep_only_tags = [
+    html2lrf_options = [
-                        dict(name='div'  , attrs={'id':'pgtop'   })
+                          '--comment', description
-                       ,dict(name='div'  , attrs={'id':'maincol' })
+                        , '--category', category
+                        , '--publisher', publisher
                         ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol']})]
+
     remove_tags = [
-                     dict(name='div'  , attrs={'class':'hldBd' })
+                     dict(name='div', attrs={'class':['hldBd','adline','pnl','infotext' ]})
-                    ,dict(name='div'  , attrs={'id':'compnl' })
+                    ,dict(name='div', attrs={'id'   :['compnl','artIssueInfo','artTools']})
-                    ,dict(name='div'  , attrs={'id':'artIssueInfo' })
+                    ,dict(name='p'  , attrs={'class':['marker','infotext'               ]})
                   ]
 
     feeds          = [
@@ -46,3 +50,20 @@ class NewScientist(BasicNewsRecipe):
                        ,(u'Science in Society'      , u'http://www.newscientist.com/feed/view?id=5&type=channel' )
                        ,(u'Tech'                    , u'http://www.newscientist.com/feed/view?id=7&type=channel' )
                      ]
+
+    def get_article_url(self, article):
+        url = article.get('link',  None)
+        raw = article.get('description',  None)
+        rsoup = self.index_to_soup(raw)
+        atags = rsoup.findAll('a',href=True)
+        for atag in atags:
+            if atag['href'].startswith('http://res.feedsportal.com/viral/sendemail2.html?'):
+               st, sep, rest = atag['href'].partition('&link=')
+               real_url, sep2, drest = rest.partition('" target=')
+               return real_url
+        return url
+
+    def print_version(self, url):
+        rawurl, sep, params = url.partition('?')
+        return rawurl + '?full=true&print=true'
+