Modification of the bibtex catalog plugin + his ui interface.

src/calibre/gui2/catalog/catalog_bibtex.py

@@ -15,6 +15,12 @@ class PluginWidget(QWidget, Ui_Form):
 
     TITLE = _('BibTeX Options')
     HELP  = _('Options specific to')+' BibTeX '+_('output')
+    OPTION_FIELDS = [('bib_cit','{authors}{id}'),
+                     ('bib_entry', 0), #mixed
+                     ('bibfile_enc', 0), #utf-8
+                     ('bibfile_enctag', 0), #strict
+                     ('impcit', True) ] 
+
     sync_enabled = False
     formats = set(['bib'])
 
@@ -23,20 +29,30 @@ class PluginWidget(QWidget, Ui_Form):
         self.setupUi(self)
         from calibre.library.catalog import FIELDS
         self.all_fields = []
-        for x in FIELDS:
+        for x in FIELDS :
             if x != 'all':
                 self.all_fields.append(x)
                 QListWidgetItem(x, self.db_fields)
 
-    def initialize(self, name):
+    def initialize(self, name): #not working properly to update
         self.name = name
         fields = gprefs.get(name+'_db_fields', self.all_fields)
-        # Restore the activated fields from last use
+        # Restore the activated db_fields from last use
         for x in xrange(self.db_fields.count()):
             item = self.db_fields.item(x)
             item.setSelected(unicode(item.text()) in fields)
+        # Update dialog fields from stored options
+        for opt in self.OPTION_FIELDS:
+            opt_value = gprefs.get(self.name + '_' + opt[0], opt[1])
+            if opt[0] in ['bibfile_enc', 'bibfile_enctag', 'bib_entry']:
+                getattr(self, opt[0]).setCurrentIndex(opt_value)
+            elif opt[0] == 'impcit' :
+                getattr(self, opt[0]).setChecked(opt_value)
+            else:
+                getattr(self, opt[0]).setText(opt_value)
 
     def options(self):
+        
         # Save the currently activated fields
         fields = []
         for x in xrange(self.db_fields.count()):
@@ -45,8 +61,24 @@ class PluginWidget(QWidget, Ui_Form):
                 fields.append(unicode(item.text()))
         gprefs.set(self.name+'_db_fields', fields)
 
-        # Return a dictionary with current options for this widget
+        # Dictionary currently activated fields
         if len(self.db_fields.selectedItems()):
-            return {'fields':[unicode(item.text()) for item in self.db_fields.selectedItems()]}
+            opts_dict = {'fields':[unicode(item.text()) for item in self.db_fields.selectedItems()]}
         else:
-            return {'fields':['all']}
+            opts_dict = {'fields':['all']}
+        
+        # Save/return the current options
+        # bib_cit stores as text
+        # 'bibfile_enc','bibfile_enctag' stores as int (Indexes)
+        for opt in self.OPTION_FIELDS:
+            if opt[0] in ['bibfile_enc', 'bibfile_enctag', 'bib_entry']:
+                opt_value = getattr(self,opt[0]).currentIndex()
+            elif opt[0] == 'impcit' :
+                opt_value = getattr(self, opt[0]).isChecked()
+            else :
+                opt_value = unicode(getattr(self, opt[0]).text())
+            gprefs.set(self.name + '_' + opt[0], opt_value)
+            
+            opts_dict[opt[0]] = opt_value
+
+        return opts_dict

src/calibre/gui2/catalog/catalog_bibtex.ui

@@ -1,47 +1,173 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
+<ui version="4.0">
- <class>Form</class>
+ <class>Form</class>
- <widget class="QWidget" name="Form">
+ <widget class="QWidget" name="Form">
-  <property name="geometry">
+  <property name="geometry">
-   <rect>
+   <rect>
-    <x>0</x>
+    <x>0</x>
-    <y>0</y>
+    <y>0</y>
-    <width>579</width>
+    <width>579</width>
-    <height>411</height>
+    <height>411</height>
-   </rect>
+   </rect>
-  </property>
+  </property>
-  <property name="windowTitle">
+  <property name="windowTitle">
-   <string>Form</string>
+   <string>Form</string>
-  </property>
+  </property>
-  <layout class="QGridLayout" name="gridLayout">
+  <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0">
+   <item row="0" column="0">
-    <widget class="QLabel" name="label_6">
+    <widget class="QLabel" name="label">
-     <property name="text">
+     <property name="text">
-      <string>Fields to include in output:</string>
+      <string>Bib file encoding:</string>
-     </property>
+     </property>
-     <property name="alignment">
+    </widget>
-      <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+   </item>
-     </property>
+   <item row="0" column="1">
-    </widget>
+    <widget class="QLabel" name="label_5">
-   </item>
+     <property name="text">
-   <item row="0" column="1">
+      <string>Fields to include in output:</string>
-    <widget class="QListWidget" name="db_fields">
+     </property>
-     <property name="sizePolicy">
+    </widget>
-      <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+   </item>
-       <horstretch>0</horstretch>
+   <item row="1" column="0">
-       <verstretch>0</verstretch>
+    <widget class="QComboBox" name="bibfile_enc">
-      </sizepolicy>
+     <item>
-     </property>
+      <property name="text">
-     <property name="toolTip">
+       <string notr="true">utf-8</string>
-      <string extracomment="Select all fields to be exported"/>
+      </property>
-     </property>
+     </item>
-     <property name="selectionMode">
+     <item>
-      <enum>QAbstractItemView::MultiSelection</enum>
+      <property name="text">
-     </property>
+       <string notr="true">cp1252</string>
-    </widget>
+      </property>
-   </item>
+     </item>
-  </layout>
+     <item>
- </widget>
+      <property name="text">
- <resources/>
+       <string>ascii/LaTeX</string>
- <connections/>
+      </property>
-</ui>
+     </item>
+    </widget>
+   </item>
+   <item row="1" column="1" rowspan="12">
+    <widget class="QListWidget" name="db_fields">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
+     <property name="toolTip">
+      <string extracomment="Select all fields to be exported"/>
+     </property>
+     <property name="selectionMode">
+      <enum>QAbstractItemView::MultiSelection</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Encoding configuration (change if you have errors) :</string>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="0">
+    <widget class="QComboBox" name="bibfile_enctag">
+     <item>
+      <property name="text">
+       <string>strict</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>replace</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>ignore</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>backslashreplace</string>
+      </property>
+     </item>
+    </widget>
+   </item>
+   <item row="4" column="0">
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>60</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item row="5" column="0">
+    <widget class="QLabel" name="label_6">
+     <property name="text">
+      <string>BibTeX entry type:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="0">
+    <widget class="QComboBox" name="bib_entry">
+     <item>
+      <property name="text">
+       <string>mixed</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>misc</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>book</string>
+      </property>
+     </item>
+    </widget>
+   </item>
+   <item row="7" column="0">
+    <widget class="QCheckBox" name="impcit">
+     <property name="text">
+      <string>Create a citation tag?</string>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="0">
+    <widget class="QLabel" name="label_3">
+     <property name="text">
+      <string>Expression to form the BibTeX citation tag:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="10" column="0">
+    <widget class="QLineEdit" name="bib_cit"/>
+   </item>
+   <item row="11" column="0">
+    <widget class="QLabel" name="label_4">
+     <property name="text">
+      <string>Some explanation about this template:
+ -The fields availables are 'author_sort', 'authors', 'id',
+    'isbn', 'pubdate', 'publisher', 'series_index', 'series',
+   'tags', 'timestamp', 'title', 'uuid'
+ -For list types ie authors and tags, only the first element
+   wil be selected.
+ -For time field, only the date will be used. </string>
+     </property>
+     <property name="scaledContents">
+      <bool>false</bool>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/library/catalog.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 __license__   = 'GPL v3'
 __copyright__ = '2010, Greg Riker <griker at hotmail.com>'
 
@@ -17,14 +19,15 @@ from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.date import isoformat, now as nowf
 from calibre.utils.logging import default_log as log
 
-#Bibtex functions
-from calibre.utils.bibtex import bibtex_author_format, utf8ToBibtex, ValidateCitationKey
-
 FIELDS = ['all', 'author_sort', 'authors', 'comments',
           'cover', 'formats', 'id', 'isbn', 'pubdate', 'publisher', 'rating',
           'series_index', 'series', 'size', 'tags', 'timestamp', 'title',
           'uuid']
           
+#Allowed fields for template
+TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate',
+    'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ]
+
 class CSV_XML(CatalogPlugin):
     'CSV/XML catalog generator'
 
@@ -115,7 +118,8 @@ class CSV_XML(CatalogPlugin):
                         item = u'%s' % re.sub(r'[\D]', '', item)
                     elif field in ['pubdate', 'timestamp']:
                         item = isoformat(item)
-
+                    
+                    #Format the line
                     if x < len(fields) - 1:
                         if item is not None:
                             outstr += u'"%s",' % unicode(item).replace('"','""')
@@ -207,21 +211,74 @@ class BIBTEX(CatalogPlugin):
                     'Available fields: %s.\n'
                     "Default: '%%default'\n"
                     "Applies to: BIBTEX output format")%', '.join(FIELDS)),
+                    
+            Option('--sort-by',
+                default = 'id',
+                dest = 'sort_by',
+                action = None,
+                help = _('Output field to sort on.\n'
+                'Available fields: author_sort, id, rating, size, timestamp, title.\n'
+                "Default: '%default'\n"
+                "Applies to: BIBTEX output format")),
+                
+            Option('--create-citation',
+                default = 'True',
+                dest = 'impcit',
+                action = None,
+                help = _('Create a citation for BibTeX entries.\n'
+                'Boolean value: True, False\n'
+                "Default: '%default'\n"
+                "Applies to: BIBTEX output format")),
+                
+            Option('--citation-template',
+                default = '{authors}{id}',
+                dest = 'bib_cit',
+                action = None,
+                help = _('The template for citation creation from database fields.\n'
+                    ' Should be a template with {} enclosed fields.\n'
+                    'Available fields: %s.\n'
+                    "Default: '%%default'\n"
+                    "Applies to: BIBTEX output format")%', '.join(TEMPLATE_ALLOWED_FIELDS)),
+            
+            Option('--choose-encoding',
+                default = 'utf8',
+                dest = 'bibfile_enc',
+                action = None,
+                help = _('BibTeX file encoding output.\n'
+                'Available types: utf8, cp1252, ascii.\n'
+                "Default: '%default'\n"
+                "Applies to: BIBTEX output format")),
+            
+            Option('--choose-encoding-configuration',
+                default = 'strict',
+                dest = 'bibfile_enctag',
+                action = None,
+                help = _('BibTeX file encoding flag.\n'
+                'Available types: strict, replace, ignore, backslashreplace.\n'
+                "Default: '%default'\n"
+                "Applies to: BIBTEX output format")),
 
             Option('--entry-type',
                 default = 'book',
-                dest = 'entry_type',
+                dest = 'bib_entry',
                 action = None,
-                help = _('Entry type for BIBTEX catalog.\n'
+                help = _('Entry type for BibTeX catalog.\n'
                 'Available types: book, misc, mixed.\n'
                 "Default: '%default'\n"
                 "Applies to: BIBTEX output format"))]
-                
+                     
     def run(self, path_to_output, opts, db, notification=DummyReporter()):
     
         import codecs
+        from types import StringType, UnicodeType
         
-        def create_bibtex_entry(entry, fields, mode = "mixed"):
+        from calibre.library.save_to_disk import preprocess_template
+        #Bibtex functions
+        from calibre.utils.bibtex import bibtex_author_format, utf8ToBibtex, ValidateCitationKey
+        
+        def create_bibtex_entry(entry, fields, mode, template_citation, 
+            asccii_bibtex = True, citation_bibtex = True):
+            
             #Bibtex doesn't like UTF-8 but keep unicode until writing
             #Define starting chain or if book valid strict and not book return a Fail string
             
@@ -234,14 +291,10 @@ class BIBTEX(CatalogPlugin):
                 #case strict book
                 return ''
                 
-            # Citation tag (not the best should be a user defined thing with regexp)
+            if citation_bibtex :
-            if not len(entry["isbn"]) == 0 :
+                # Citation tag
-                bibtex_entry.append(u'%s' % utf8ToBibtex(ValidateCitationKey(re.sub(u'[\D]',
+                bibtex_entry.append(make_bibtex_citation(entry, template_citation, asccii_bibtex))
-                    u'', entry["isbn"]))))
+                bibtex_entry = [u' '.join(bibtex_entry)]
-            else :
-                bibtex_entry.append(u'%s' % utf8ToBibtex(ValidateCitationKey(str(entry["id"]))))
-            
-            bibtex_entry = [u' '.join(bibtex_entry)]
             
             for field in fields:
                 item = entry[field]
@@ -259,7 +312,7 @@ class BIBTEX(CatalogPlugin):
                     
                 elif field in ['title', 'publisher', 'cover', 'uuid',
                         'author_sort', 'series'] :
-                    bibtex_entry.append(u'%s = "%s"' % (field, utf8ToBibtex(item)))
+                    bibtex_entry.append(u'%s = "%s"' % (field, utf8ToBibtex(item, asccii_bibtex)))
                     
                 elif field == 'id' :
                     bibtex_entry.append(u'calibreid = "%s"' % int(item))
@@ -272,11 +325,13 @@ class BIBTEX(CatalogPlugin):
                     
                 elif field == 'tags' : 
                     #A list to flatten
-                    bibtex_entry.append(u'tags = "%s"' % utf8ToBibtex(u', '.join(item)))
+                    bibtex_entry.append(u'tags = "%s"' % utf8ToBibtex(u', '.join(item), asccii_bibtex))
                     
                 elif field == 'comments' : 
                     #\n removal
-                    bibtex_entry.append(u'note = "%s"' % utf8ToBibtex(item.replace(u'\n',u' ')))
+                    item = item.replace(u'\r\n',u' ')
+                    item = item.replace(u'\n',u' ')
+                    bibtex_entry.append(u'note = "%s"' % utf8ToBibtex(item, asccii_bibtex))
                     
                 elif field == 'isbn' :
                     # Could be 9, 10 or 13 digits
@@ -295,8 +350,8 @@ class BIBTEX(CatalogPlugin):
                 elif field == 'pubdate' :
                     bibtex_entry.append(u'year = "%s"' % item.year)
                     #Messing locale in date string formatting
-                    bibtex_entry.append(u'month = "%s"' % utf8ToBibtex(item.strftime("%b").decode(preferred_encoding)))
+                    bibtex_entry.append(u'month = "%s"' % utf8ToBibtex(item.strftime("%b").decode(preferred_encoding),
-                    #bibtex_entry.append('month = "%s"' % utf8ToBibtex(item.strftime("%B").decode(getlocale()[1])))
+                        asccii_bibtex))
             
             bibtex_entry = u',\n    '.join(bibtex_entry)
             bibtex_entry += u' }\n\n'
@@ -313,8 +368,73 @@ class BIBTEX(CatalogPlugin):
             else :
                 return True
 
+        def make_bibtex_citation(entry, template_citation, asccii_bibtex):
+            
+            #define a function to replace the template entry by its value
+            def tpl_replace(objtplname) :
+            
+                tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
+                
+                if tpl_field in TEMPLATE_ALLOWED_FIELDS :
+                    if tpl_field in ['pubdate', 'timestamp'] :
+                        tpl_field = isoformat(entry[tpl_field]).partition('T')[0]
+                    elif tpl_field in ['tags', 'authors'] :
+                        tpl_field =entry[tpl_field][0]
+                    elif tpl_field in ['id', 'series_index'] :
+                        tpl_field = str(entry[tpl_field])
+                    else :
+                        tpl_field = entry[tpl_field]
+                    return tpl_field
+                else:
+                    return u''            
+
+            if len(template_citation) >0 :
+                tpl_citation = utf8ToBibtex(ValidateCitationKey(re.sub(u'\{[^{}]*\}', 
+                    tpl_replace, template_citation)), asccii_bibtex)
+
+                if len(tpl_citation) >0 :
+                    return tpl_citation
+                
+            if len(entry["isbn"]) > 0 :
+                template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"])
+                
+            else :
+                template_citation = u'%s' % str(entry["id"])
+            
+            if asccii_bibtex :
+                return ValidateCitationKey(template_citation.encode('ascii', 'replace'))
+            else :
+                return ValidateCitationKey(template_citation)
+        
         self.fmt = path_to_output.rpartition('.')[2]
         self.notification = notification
+        
+        # Combobox options
+        bibfile_enc = ['utf8', 'cp1252', 'ascii']
+        bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace']
+        bib_entry = ['mixed', 'misc', 'book']
+        
+        # Needed beacause CLI return str vs int by widget
+        try:
+            bibfile_enc = bibfile_enc[opts.bibfile_enc]
+            bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
+            bib_entry = bib_entry[opts.bib_entry]
+        except:
+            if opts.bibfile_enc in bibfile_enc :
+                bibfile_enc = opts.bibfile_enc
+            else :
+                log(" WARNING: incorrect --choose-encoding flag, revert to default")
+                bibfile_enc = bibfile_enc[0]
+            if opts.bibfile_enctag in bibfile_enctag :
+                bibfile_enctag = opts.bibfile_enctag
+            else :
+                log(" WARNING: incorrect --choose-encoding-configuration flag, revert to default")
+                bibfile_enctag = bibfile_enctag[0]
+            if opts.bib_entry in bib_entry :
+                bib_entry = opts.bib_entry
+            else :
+                log(" WARNING: incorrect --entry-type flag, revert to default")
+                bib_entry = bib_entry[0]
 
         if opts.verbose:
             opts_dict = vars(opts)
@@ -332,6 +452,10 @@ class BIBTEX(CatalogPlugin):
                     log(" Fields: %s" % ', '.join(FIELDS[1:]))
                 else:
                     log(" Fields: %s" % opts_dict['fields'])
+                    
+            log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag))
+            
+            log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit']))
 
         # If a list of ids are provided, don't use search_text
         if opts.ids:
@@ -347,20 +471,47 @@ class BIBTEX(CatalogPlugin):
         
         if not len(data):
             log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
-            
+        
+        #Entries writing after Bibtex formating (or not)
+        if bibfile_enc != 'ascii' :
+            asccii_bibtex = False
+        else :
+            asccii_bibtex = True
+        
+        #Check and go to default in case of bad CLI
+        if isinstance(opts.impcit, (StringType, UnicodeType)) :
+            if opts.impcit == 'False' :
+                citation_bibtex= False
+            elif opts.impcit == 'True' :
+                citation_bibtex= True
+            else :
+                log(" WARNING: incorrect --create-citation, revert to default")
+                citation_bibtex= True
+        else :
+            citation_bibtex= opts.impcit
+        
+        template_citation = preprocess_template(opts.bib_cit)
+        
         #Open output and write entries
-        #replace should be an option and not the default to generate errors for improving
+        outfile = codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)
-        outfile = codecs.open(path_to_output, 'w', 'ascii','replace')
         
         #File header
-        nb_entries = len(opts.ids)
+        nb_entries = len(data)
+        
+        #check in book strict if all is ok else throw a warning into log
+        if bib_entry == 'book' :
+            nb_books = len(filter(check_entry_book_valid, data))
+            if nb_books < nb_entries :
+                log(" WARNING: only %d entries in %d are book compatible" % (nb_books, nb_entries))
+                nb_entries = nb_books
+        
         outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries))
         outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n'
             % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding)))
         
-        #Entries wrintng after Bibtex formating
         for entry in data:
-            outfile.write(create_bibtex_entry(entry, fields))
+            outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, 
+                asccii_bibtex, citation_bibtex))
         
         outfile.close()
 

src/calibre/utils/bibtex.py

@@ -2502,14 +2502,14 @@ def BraceUppercase(text):
     return text
 
 def resolveEntities(text):
-    for entity in entity_mapping.keys():
+    for entity, entity_map in entity_mapping.iteritems():
-        text = text.replace(entity, entity_mapping[entity])
+        text = text.replace(entity, entity_map)
     return text
 
 def resolveUnicode(text):
     #UTF-8 text as entry
-    for unichar in utf8enc2latex_mapping.keys():
+    for unichar, latexenc in utf8enc2latex_mapping.iteritems() :
-        text = text.replace(unichar, utf8enc2latex_mapping[unichar])
+        text = text.replace(unichar, latexenc)
     return text.replace(u'$}{$', u'')
 
 def escapeSpecialCharacters(text):
@@ -2524,13 +2524,10 @@ def escapeSpecialCharacters(text):
     
 #Calibre functions
 #Go from an unicode entry to ASCII Bibtex format without encoding
+#Option to go to official ASCII Bibtex or unofficial UTF-8
 def utf8ToBibtex(text, asccii_bibtex = True):
     if len(text) == 0:
         return ''
-    '''try :
-        text = text.decode('cp1252')
-    except (TypeError, UnicodeDecodeError, ValueError):
-        pass '''
     text.replace('\\', '\\\\')
     text = resolveEntities(text)
     if asccii_bibtex :
@@ -2539,6 +2536,4 @@ def utf8ToBibtex(text, asccii_bibtex = True):
     
 def bibtex_author_format(item):
     #Format authors for Bibtex compliance (get a list as input)
-    item = u' and'.join([author for author in item])
+    return utf8ToBibtex(u' and'.join([author for author in item]))
-    return utf8ToBibtex(item)
-