Merge from trunk

Changelog.yaml

@@ -60,6 +60,8 @@
     - title: "TXT Input: New paragraph-type option (off) to disable modifying the paragraph structure."
 
     - title: "Device driver for the Kendo/Yifang M7 and the Wolder Mibuk Life"
+    
+    - title: "For people building calibre from source, note that calibre now requires SIP >= 4.12 to build" 
   
   bug fixes:
     - title: "Fix main memory and storage card for Cybook Orizon being swapped with some firmwares"

resources/recipes/roger_ebert_blog.recipe

@@ -0,0 +1,136 @@
+import re
+import urllib2
+import time
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup, SoupStrainer
+from calibre import strftime
+
+'''
+      Help Needed:
+       Still can't figure out why I'm getting strange characters.  Esp. the Great Movies descriptions in the TOC.
+       Anyone help me figure that out?
+
+      Change Log:
+       2011-02-19:  Version 2:  Added "Oscars" section and fixed date problem
+'''
+
+class Ebert(BasicNewsRecipe):
+    title                 = 'Roger Ebert'
+    __author__            = 'Shane Erstad'
+    version               = 2
+    description           = 'Roger Ebert Movie Reviews'
+    publisher             = 'Chicago Sun Times'
+    category              = 'movies'
+    oldest_article        = 8
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'UTF-8'
+    masthead_url          = 'http://rogerebert.suntimes.com/graphics/global/roger.jpg'
+    language              = 'en'
+    remove_empty_feeds    = False
+    PREFIX                  = 'http://rogerebert.suntimes.com'
+    patternReviews                = r'<span class="*?movietitle"*?>(.*?)</span>.*?<div class="*?headline"*?>(.*?)</div>(.*?)</div>'
+    patternCommentary       = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?COMMENTARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternPeople           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?PEOPLE.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternOscars           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?OSCARS.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+    patternGlossary           = r'<div class="*?headline"*?>.*?(<a href="/apps/pbcs.dll/article\?AID=.*?GLOSSARY.*?" id="ltred">.*?</a>).*?<div class="blurb clear">(.*?)</div>'
+
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+
+    feeds          = [
+                        (u'Reviews'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=reviews' )
+                        ,(u'Commentary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=COMMENTARY')
+                        ,(u'Great Movies'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=REVIEWS08')
+                        ,(u'People'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=PEOPLE')
+                        ,(u'Oscars'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=OSCARS')
+                        ,(u'Glossary'   , u'http://rogerebert.suntimes.com/apps/pbcs.dll/section?category=GLOSSARY')
+
+                     ]
+
+    preprocess_regexps = [
+        (re.compile(r'<font.*?>.*?This is a printer friendly.*?</font>.*?<hr>', re.DOTALL|re.IGNORECASE),
+            lambda m: '')
+    ]
+
+
+
+    def print_version(self, url):
+        return url + '&template=printart'
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.log('\tFeedurl: ', feedurl)
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            page = urllib2.urlopen(feedurl).read()
+
+            if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    pattern = self.patternReviews
+            elif feedtitle == 'Commentary':
+                    pattern = self.patternCommentary
+            elif feedtitle == 'People':
+                    pattern = self.patternPeople
+            elif feedtitle == 'Glossary':
+                    pattern = self.patternGlossary
+            elif feedtitle == 'Oscars':
+                    pattern = self.patternOscars
+
+
+            regex = re.compile(pattern, re.IGNORECASE|re.DOTALL)
+
+            for match in regex.finditer(page):
+                if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                    movietitle = match.group(1)
+                    thislink = match.group(2)
+                    description = match.group(3)
+                elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary' or feedtitle == 'Oscars':
+                    thislink = match.group(1)
+                    description = match.group(2)
+
+                self.log(thislink)
+
+                for link in BeautifulSoup(thislink, parseOnlyThese=SoupStrainer('a')):
+                    thisurl = self.PREFIX + link['href']
+                    thislinktext = self.tag_to_string(link)
+
+                    if feedtitle == 'Reviews' or feedtitle == 'Great Movies':
+                        thistitle = movietitle
+                    elif feedtitle == 'Commentary' or feedtitle == 'People' or feedtitle == 'Glossary' or feedtitle == 'Oscars':
+                        thistitle = thislinktext
+
+                    if thistitle == '':
+                        continue
+
+
+                    pattern2 = r'AID=\/(.*?)\/'
+                    reg2 = re.compile(pattern2, re.IGNORECASE|re.DOTALL)
+                    match2 = reg2.search(thisurl)
+                    if match2:
+                        c = time.strptime(match2.group(1),"%Y%m%d")
+                        mydate=strftime("%A, %B %d, %Y", c)
+                    else:
+                        mydate = strftime("%A, %B %d, %Y")
+                    self.log(mydate)
+
+                    articles.append({
+                                      'title'      :thistitle
+                                     ,'date'       :'  [' + mydate + ']'
+                                     ,'url'        :thisurl
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+
+        return totalfeeds

setup/commands.py

@@ -17,7 +17,7 @@ __all__ = [
         'manual', 'tag_release',
         'pypi_register', 'pypi_upload', 'upload_to_server',
         'upload_user_manual', 'upload_to_mobileread', 'upload_demo',
-        'upload_to_sourceforge', 'upload_to_google_code',
+        'upload_to_sourceforge', 'upload_to_google_code', 'reupload',
         'linux32', 'linux64', 'linux', 'linux_freeze',
         'osx32_freeze', 'osx', 'rsync', 'push',
         'win32_freeze', 'win32', 'win',
@@ -63,13 +63,14 @@ stage4 = Stage4()
 publish = Publish()
 
 from setup.upload import UploadUserManual, UploadInstallers, UploadDemo, \
-        UploadToServer, UploadToSourceForge, UploadToGoogleCode
+        UploadToServer, UploadToSourceForge, UploadToGoogleCode, ReUpload
 upload_user_manual = UploadUserManual()
 upload_to_mobileread = UploadInstallers()
 upload_demo = UploadDemo()
 upload_to_server = UploadToServer()
 upload_to_sourceforge = UploadToSourceForge()
 upload_to_google_code = UploadToGoogleCode()
+reupload = ReUpload()
 
 from setup.installer import Rsync, Push
 rsync = Rsync()

setup/upload.py

@@ -39,8 +39,22 @@ def installer_description(fname):
         return 'OS X dmg'
     return 'Unknown file'
 
+class ReUpload(Command): # {{{
 
-class UploadToGoogleCode(Command):
+    description = 'Re-uplaod any installers present in dist/'
+
+    sub_commands = ['upload_to_google_code', 'upload_to_sourceforge']
+
+    def pre_sub_commands(self, opts):
+        opts.re_upload = True
+
+    def run(self, opts):
+        for x in installers():
+            if os.path.exists(x):
+                os.remove(x)
+# }}}
+
+class UploadToGoogleCode(Command): # {{{
 
     USERNAME = 'kovidgoyal'
     # Password can be gotten by going to
@@ -52,21 +66,49 @@ class UploadToGoogleCode(Command):
     UPLOAD_HOST = 'calibre-ebook.googlecode.com'
     FILES_LIST = 'http://code.google.com/p/calibre-ebook/downloads/list'
 
+    def add_options(self, parser):
+        parser.add_option('--re-upload', default=False, action='store_true',
+                help='Re-upload all installers currently in dist/')
+
+    def re_upload(self):
+        fnames = set([os.path.basename(x) for x in installers() if not
+                x.endswith('.tar.gz') and os.path.exists(x)])
+        existing = set(self.old_files.keys()).intersection(fnames)
+        br = self.login_to_gmail()
+        for x in fnames:
+            src = os.path.join('dist', x)
+            if not os.access(src, os.R_OK):
+                continue
+            if x in existing:
+                self.info('Deleting', x)
+                br.open('http://code.google.com/p/calibre-ebook/downloads/delete?name=%s'%x)
+                br.select_form(predicate=lambda y: 'delete.do' in y.action)
+                br.form.find_control(name='delete')
+                br.submit(name='delete')
+            self.upload_one(src)
+
+    def upload_one(self, fname):
+        self.info('Uploading', fname)
+        typ = 'Type-Source' if fname.endswith('.gz') else 'Type-Installer'
+        ext = os.path.splitext(fname)[1][1:]
+        op  = 'OpSys-'+{'msi':'Windows','dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
+        desc = installer_description(fname)
+        path = self.upload(os.path.abspath(fname), desc,
+                labels=[typ, op, 'Featured'])
+        self.info('\tUploaded to:', path)
+        return path
+
     def run(self, opts):
         self.opts = opts
         self.password = open(self.PASSWORD_FILE).read().strip()
         self.paths = {}
         self.old_files = self.get_files_hosted_by_google_code()
 
+        if opts.re_upload:
+            return self.re_upload()
+
         for fname in installers():
-            self.info('Uploading', fname)
-            typ = 'Type-Source' if fname.endswith('.gz') else 'Type-Installer'
-            ext = os.path.splitext(fname)[1][1:]
-            op  = 'OpSys-'+{'msi':'Windows','dmg':'OSX','bz2':'Linux','gz':'All'}[ext]
-            desc = installer_description(fname)
-            path = self.upload(os.path.abspath(fname), desc,
-                    labels=[typ, op, 'Featured'])
-            self.info('\tUploaded to:', path)
+            path = self.upload_one(fname)
             self.paths[os.path.basename(fname)] = path
         self.info('Updating path map')
         self.info(repr(self.paths))
@@ -189,11 +231,9 @@ class UploadToGoogleCode(Command):
             return self.upload(fname, desc, labels=labels, retry=retry+1)
         raise Exception('Failed to upload '+fname)
 
+# }}}
 
-
-
-
-class UploadToSourceForge(Command):
+class UploadToSourceForge(Command): # {{{
 
     description = 'Upload release files to sourceforge'
 
@@ -217,9 +257,10 @@ class UploadToSourceForge(Command):
         self.opts = opts
         self.upload_installers()
 
+# }}}
 
-class UploadInstallers(Command):
-    description = 'Upload any installers present in dist/'
+class UploadInstallers(Command): # {{{
+    description = 'Upload any installers present in dist/ to mobileread'
     def curl_list_dir(self, url=MOBILEREAD, listonly=1):
         import pycurl
         c = pycurl.Curl()
@@ -289,17 +330,18 @@ class UploadInstallers(Command):
         installers = list(map(installer_name, ('dmg', 'msi', 'tar.bz2')))
         installers.append(installer_name('tar.bz2', is64bit=True))
         map(self.upload_installer, installers)
+# }}}
 
-class UploadUserManual(Command):
+class UploadUserManual(Command): # {{{
     description = 'Build and upload the User Manual'
     sub_commands = ['manual']
 
     def run(self, opts):
         check_call(' '.join(['scp', '-r', 'src/calibre/manual/.build/html/*',
                     'divok:%s'%USER_MANUAL]), shell=True)
+# }}}
 
-
-class UploadDemo(Command):
+class UploadDemo(Command): # {{{
 
     description = 'Rebuild and upload various demos'
 
@@ -317,8 +359,9 @@ class UploadDemo(Command):
             'zip -j /tmp/html-demo.zip * /tmp/html2lrf.lrf', shell=True)
 
         check_call('scp /tmp/html-demo.zip divok:%s/'%(DOWNLOADS,), shell=True)
+# }}}
 
-class UploadToServer(Command):
+class UploadToServer(Command): # {{{
 
     description = 'Upload miscellaneous data to calibre server'
 
@@ -348,6 +391,6 @@ class UploadToServer(Command):
         check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS),
                 shell=True)
         shutil.rmtree(tdir)
-
+# }}}
 
 

src/calibre/ebooks/metadata/amazon.py

@@ -50,7 +50,7 @@ def to_asin(br, isbn):
     else:
         asin = isbn
     with cache_lock:
-        asin_cache[isbn] = ans if ans else False
+        asin_cache[isbn] = asin if asin else False
     return asin
 
 

src/calibre/ebooks/pdf/images.cpp

@@ -301,7 +301,7 @@ void PNGWriter::write_splash_bitmap(SplashBitmap *bitmap) {
 
 void calibre_png_mem_write(png_structp png_ptr, png_bytep data, png_size_t length) {
     if (!png_ptr || length < 1) return;
-    vector<char> *buf = static_cast< vector<char>* >(png_ptr->io_ptr);
+    vector<char> *buf = static_cast< vector<char>* >(png_get_io_ptr(png_ptr));
     buf->reserve(buf->capacity() + length); 
     do {
         buf->push_back(static_cast<char>(*data));

src/calibre/gui2/preferences/create_custom_column.py

@@ -153,17 +153,17 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         display_dict = {}
 
         if col_type == 'datetime':
-            if self.date_format_box.text().strip():
+            if unicode(self.date_format_box.text()).strip():
                 display_dict = {'date_format':unicode(self.date_format_box.text()).strip()}
             else:
                 display_dict = {'date_format': None}
         elif col_type == 'composite':
-            if not self.composite_box.text().strip():
+            if not unicode(self.composite_box.text()).strip():
                 return self.simple_error('', _('You must enter a template for'
                     ' composite columns'))
             display_dict = {'composite_template':unicode(self.composite_box.text()).strip()}
         elif col_type == 'enumeration':
-            if not self.enum_box.text():
+            if not unicode(self.enum_box.text()).strip():
                 return self.simple_error('', _('You must enter at least one'
                     ' value for enumeration columns'))
             l = [v.strip() for v in unicode(self.enum_box.text()).split(',')]

src/calibre/utils/chm/swig_chm.c

@@ -164,36 +164,6 @@ SWIG_TypeCast(swig_type_info *ty, void *ptr)
   return (*ty->converter)(ptr);
 }
 
-/* Dynamic pointer casting. Down an inheritance hierarchy */
-SWIGRUNTIME(swig_type_info *) 
-SWIG_TypeDynamicCast(swig_type_info *ty, void **ptr) 
-{
-  swig_type_info *lastty = ty;
-  if (!ty || !ty->dcast) return ty;
-  while (ty && (ty->dcast)) {
-     ty = (*ty->dcast)(ptr);
-     if (ty) lastty = ty;
-  }
-  return lastty;
-}
-
-/* Return the name associated with this type */
-SWIGRUNTIME(const char *)
-SWIG_TypeName(const swig_type_info *ty) {
-  return ty->name;
-}
-
-/* Search for a swig_type_info structure */
-SWIGRUNTIME(swig_type_info *)
-SWIG_TypeQuery(const char *name) {
-  swig_type_info *ty = swig_type_list;
-  while (ty) {
-    if (ty->str && (strcmp(name,ty->str) == 0)) return ty;
-    if (ty->name && (strcmp(name,ty->name) == 0)) return ty;
-    ty = ty->prev;
-  }
-  return 0;
-}
 
 /* Set the clientdata field for a type */
 SWIGRUNTIME(void)
@@ -365,21 +335,6 @@ SWIG_newvarlink(void) {
   return ((PyObject*) result);
 }
 
-SWIGRUNTIME(void)
-SWIG_addvarlink(PyObject *p, char *name,
-	   PyObject *(*get_attr)(void), int (*set_attr)(PyObject *p)) {
-  swig_varlinkobject *v;
-  swig_globalvar *gv;
-  v= (swig_varlinkobject *) p;
-  gv = (swig_globalvar *) malloc(sizeof(swig_globalvar));
-  gv->name = (char *) malloc(strlen(name)+1);
-  strcpy(gv->name,name);
-  gv->get_attr = get_attr;
-  gv->set_attr = set_attr;
-  gv->next = v->vars;
-  v->vars = gv;
-}
-
 /* Pack binary data into a string */
 SWIGRUNTIME(char *)
 SWIG_PackData(char *c, void *ptr, int sz) {
@@ -395,29 +350,6 @@ SWIG_PackData(char *c, void *ptr, int sz) {
   return c;
 }
 
-/* Unpack binary data from a string */
-SWIGRUNTIME(char *)
-SWIG_UnpackData(char *c, void *ptr, int sz) {
-  register unsigned char uu = 0;
-  register int d;
-  unsigned char *u = (unsigned char *) ptr;
-  int i;
-  for (i = 0; i < sz; i++, u++) {
-    d = *(c++);
-    if ((d >= '0') && (d <= '9'))
-      uu = ((d - '0') << 4);
-    else if ((d >= 'a') && (d <= 'f'))
-      uu = ((d - ('a'-10)) << 4);
-    d = *(c++);
-    if ((d >= '0') && (d <= '9'))
-      uu |= (d - '0');
-    else if ((d >= 'a') && (d <= 'f'))
-      uu |= (d - ('a'-10));
-    *u = uu;
-  }
-  return c;
-}
-
 /* Convert a pointer value */
 SWIGRUNTIME(int)
 SWIG_ConvertPtr(PyObject *obj, void **ptr, swig_type_info *ty, int flags) {
@@ -510,39 +442,6 @@ type_error:
   return -1;
 }
 
-/* Convert a packed value value */
-SWIGRUNTIME(int)
-SWIG_ConvertPacked(PyObject *obj, void *ptr, int sz, swig_type_info *ty, int flags) {
-  swig_type_info *tc;
-  char  *c;
-
-  if ((!obj) || (!PyString_Check(obj))) goto type_error;
-  c = PyString_AsString(obj);
-  /* Pointer values must start with leading underscore */
-  if (*c != '_') goto type_error;
-  c++;
-  c = SWIG_UnpackData(c,ptr,sz);
-  if (ty) {
-    tc = SWIG_TypeCheck(c,ty);
-    if (!tc) goto type_error;
-  }
-  return 0;
-
-type_error:
-
-  if (flags) {
-    if (ty) {
-      char *temp = (char *) malloc(64+strlen(ty->name));
-      sprintf(temp,"Type error. Expected %s", ty->name);
-      PyErr_SetString(PyExc_TypeError, temp);
-      free((char *) temp);
-    } else {
-      PyErr_SetString(PyExc_TypeError,"Expected a pointer");
-    }
-  }
-  return -1;
-}
-
 /* Create a new pointer object */
 SWIGRUNTIME(PyObject *)
 SWIG_NewPointerObj(void *ptr, swig_type_info *type, int own) {
@@ -1071,7 +970,7 @@ static PyObject *_wrap_chm_retrieve_object(PyObject *self, PyObject *args) {
     resultobj = PyLong_FromLongLong(result);
     {
         PyObject *o;
-        o = PyString_FromStringAndSize(arg3, arg5);
+        o = PyString_FromStringAndSize((const char *)arg3, arg5);
         resultobj = t_output_helper(resultobj,o);