From fbc61d4f07246af8b94a2718ae4598bba4d8dbf4 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 12 Jul 2012 12:14:40 +0200
Subject: [PATCH 01/18] Add Mills & Boon store

---
 src/calibre/customize/builtins.py             | 11 +++
 .../gui2/store/stores/foyles_uk_plugin.py     |  2 +-
 .../gui2/store/stores/mills_boon_uk_plugin.py | 78 +++++++++++++++++++
 3 files changed, 90 insertions(+), 1 deletion(-)
 create mode 100644 src/calibre/gui2/store/stores/mills_boon_uk_plugin.py

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 71cce109dd..8761c8e589 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -1484,6 +1484,16 @@ class StoreManyBooksStore(StoreBase):
     headquarters = 'US'
     formats = ['EPUB', 'FB2', 'JAR', 'LIT', 'LRF', 'MOBI', 'PDB', 'PDF', 'RB', 'RTF', 'TCR', 'TXT', 'ZIP']
 
+class StoreMillsBoonUKStore(StoreBase):
+    name = 'Mills and Boon UK'
+    author = 'Charles Haley'
+    description = u'"Bring Romance to Life" "[A] hallmark for romantic fiction, recognised around the world."'
+    actual_plugin = 'calibre.gui2.store.stores.mills_boon_uk_plugin:MillsBoonUKStore'
+
+    headquarters = 'UK'
+    formats = ['EPUB']
+    affiliate = True
+
 class StoreMobileReadStore(StoreBase):
     name = 'MobileRead'
     description = u'Ebooks handcrafted with the utmost care.'
@@ -1656,6 +1666,7 @@ plugins += [
     StoreLibreDEStore,
     StoreLitResStore,
     StoreManyBooksStore,
+    StoreMillsBoonUKStore,
     StoreMobileReadStore,
     StoreNextoStore,
     StoreOpenBooksStore,
diff --git a/src/calibre/gui2/store/stores/foyles_uk_plugin.py b/src/calibre/gui2/store/stores/foyles_uk_plugin.py
index b684dd533d..819c412758 100644
--- a/src/calibre/gui2/store/stores/foyles_uk_plugin.py
+++ b/src/calibre/gui2/store/stores/foyles_uk_plugin.py
@@ -6,7 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import urllib2, re
+import urllib2
 from contextlib import closing
 
 from lxml import html
diff --git a/src/calibre/gui2/store/stores/mills_boon_uk_plugin.py b/src/calibre/gui2/store/stores/mills_boon_uk_plugin.py
new file mode 100644
index 0000000000..6aa4b4b0b7
--- /dev/null
+++ b/src/calibre/gui2/store/stores/mills_boon_uk_plugin.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class MillsBoonUKStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://www.awin1.com/awclick.php?mid=1150&id=120917'
+        detail_url = 'http://www.awin1.com/cread.php?awinmid=1150&awinaffid=120917&clickref=&p='
+
+        if external or self.config.get('open_external', False):
+            if detail_item:
+                url = detail_url + detail_item
+            open_url(QUrl(url_slash_cleaner(url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        base_url = 'http://millsandboon.co.uk'
+        url = base_url + '/pages/searchres.htm?search=true&booktypesearch=ebook&first=yes&inputsearch=' + urllib2.quote(query)
+        br = browser()
+
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="catProdDiv"]'):
+                if counter <= 0:
+                    break
+                id_ = ''.join(data.xpath('.//div[@class="catProdImage"]/div/a/@href')).strip()
+                id_ = base_url + id_[2:]
+                if not id_:
+                    continue
+
+                cover_url = ''.join(data.xpath('.//div[@class="catProdImage"]/div/a/img/@src'))
+                cover_url = base_url + cover_url[2:]
+                title =  ''.join(data.xpath('.//div[@class="catProdImage"]/div/a/img/@alt')).strip()
+                title = title[23:]
+                author = ''.join(data.xpath('.//div[@class="catProdDetails"]/div[@class="catProdDetails-top"]/p[1]/a/text()'))
+                price = ''.join(data.xpath('.//span[@class="priceBold"]/text()'))
+                format_ = ''.join(data.xpath('.//p[@class="doc-meta-format"]/span[last()]/text()'))
+                drm = SearchResult.DRM_LOCKED
+
+                counter -= 1
+
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id_
+                s.drm = drm
+                s.formats = format_
+
+                yield s

From a0256f9705b71a317bf1b92d9ff15e7bb4b14fa7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 09:32:34 +0530
Subject: [PATCH 02/18] ...

---
 src/calibre/gui2/complete2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py
index 43fcf59298..8e0718c78e 100644
--- a/src/calibre/gui2/complete2.py
+++ b/src/calibre/gui2/complete2.py
@@ -284,6 +284,8 @@ class LineEdit(QLineEdit, LineEditECM):
         if self.no_popup: return
         self.update_completions()
         select_first = len(self.mcompleter.model().current_prefix) > 0
+        if not select_first:
+            self.mcompleter.setCurrentIndex(QModelIndex())
         self.complete(select_first=select_first)
 
     def update_completions(self):

From 177e20f3d49efc05ac61e57533e17042ceccea26 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 09:39:27 +0530
Subject: [PATCH 03/18] Remove the completion performance tweak as we no longer
 use is

---
 resources/default_tweaks.py  | 9 ---------
 src/calibre/gui2/complete.py | 3 +--
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 5642b00190..420df116ce 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -98,15 +98,6 @@ authors_split_regex = r'(?i),?\s+(and|with)\s+'
 #   categories_use_field_for_author_name = 'author_sort'
 categories_use_field_for_author_name = 'author'
 
-#: Completion sort order: choose when to change from lexicographic to ASCII-like
-# Calibre normally uses locale-dependent lexicographic ordering when showing
-# completion values. This means that the sort order is correct for the user's
-# language. However, this can be slow. Performance is improved by switching to
-# ascii ordering. This tweak controls when that switch happens. Set it to zero
-# to always use ascii ordering. Set it to something larger than zero to switch
-# to ascii ordering for performance reasons.
-completion_change_to_ascii_sorting = 2500
-
 #: Control partitioning of Tag Browser
 # When partitioning the tags browser, the format of the subcategory label is
 # controlled by a template: categories_collapsed_name_template if sorting by
diff --git a/src/calibre/gui2/complete.py b/src/calibre/gui2/complete.py
index ec2720b534..1c2d567130 100644
--- a/src/calibre/gui2/complete.py
+++ b/src/calibre/gui2/complete.py
@@ -16,7 +16,6 @@ from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt,
 from calibre.utils.icu import sort_key
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
-from calibre.utils.config_base import tweaks
 
 class CompleteModel(QAbstractListModel):
 
@@ -27,7 +26,7 @@ class CompleteModel(QAbstractListModel):
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
-        if len(items) < tweaks['completion_change_to_ascii_sorting']:
+        if len(items) < 2500:
             self.items = sorted(items, key=sort_key)
             self.sorting = QCompleter.UnsortedModel
         else:

From e49bc057e85e19df4776f66c3c5a75330cc5c464 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 10:07:21 +0530
Subject: [PATCH 04/18] MOBI: Add support for the new language EXTH header
 field in MOBI files generated by kindlegen 2.5

---
 src/calibre/ebooks/mobi/debug/headers.py  |  1 +
 src/calibre/ebooks/mobi/reader/headers.py | 18 ++++++++++++++----
 src/calibre/ebooks/mobi/writer8/exth.py   |  6 ++++++
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/mobi/debug/headers.py b/src/calibre/ebooks/mobi/debug/headers.py
index 6aaafbffa3..5b80a46f1b 100644
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@@ -163,6 +163,7 @@ class EXTHRecord(object):
                 501 : 'cdetype', # 4 chars (PDOC or EBOK)
                 502 : 'lastupdatetime',
                 503 : 'updatedtitle',
+                524 : 'language',
         }.get(self.type, repr(self.type))
 
         if (self.name in {'coveroffset', 'thumboffset', 'hasfakecover',
diff --git a/src/calibre/ebooks/mobi/reader/headers.py b/src/calibre/ebooks/mobi/reader/headers.py
index 51da05ea83..bfbffe546e 100644
--- a/src/calibre/ebooks/mobi/reader/headers.py
+++ b/src/calibre/ebooks/mobi/reader/headers.py
@@ -13,6 +13,7 @@ from calibre.utils.date import parse_date
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.mobi.langcodes import main_language, sub_language, mobi2iana
+from calibre.utils.localization import canonicalize_lang
 
 NULL_INDEX = 0xffffffff
 
@@ -68,6 +69,14 @@ class EXTHHeader(object): # {{{
                     title = content.decode(codec)
                 except:
                     pass
+            elif idx == 524: # Lang code
+                try:
+                    lang = content.decode(codec)
+                    lang = canonicalize_lang(lang)
+                    if lang:
+                        self.mi.language = lang
+                except:
+                    pass
             #else:
             #    print 'unknown record', idx, repr(content)
         if title:
@@ -201,10 +210,11 @@ class BookHeader(object):
                     self.exth = EXTHHeader(raw[16 + self.length:], self.codec,
                             self.title)
                     self.exth.mi.uid = self.unique_id
-                    try:
-                        self.exth.mi.language = mobi2iana(langid, sublangid)
-                    except:
-                        self.log.exception('Unknown language code')
+                    if self.exth.mi.is_null('language'):
+                        try:
+                            self.exth.mi.language = mobi2iana(langid, sublangid)
+                        except:
+                            self.log.exception('Unknown language code')
                 except:
                     self.log.exception('Invalid EXTH header')
                     self.exth_flag = 0
diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index daf6da62d6..72d1ce5a67 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -12,6 +12,7 @@ from struct import pack
 from io import BytesIO
 
 from calibre.ebooks.mobi.utils import utf8_text
+from calibre.utils.localization import lang_as_iso639_1
 
 EXTH_CODES = {
     'creator': 100,
@@ -35,6 +36,7 @@ EXTH_CODES = {
     'hasfakecover': 203,
     'lastupdatetime': 502,
     'title': 503,
+    'language': 524,
 }
 
 COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+')
@@ -68,6 +70,10 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
                     pass
                 else:
                     continue
+            if term == 'language':
+                d2 = lang_as_iso639_1(data)
+                if d2:
+                    data = d2
             data = utf8_text(data)
             exth.write(pack(b'>II', code, len(data) + 8))
             exth.write(data)

From 56fb3001632387c87d784a7ee2ece0a9168a10da Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 10:12:15 +0530
Subject: [PATCH 05/18] ...

---
 src/calibre/ebooks/mobi/writer8/exth.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index 72d1ce5a67..df952a8eb4 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -59,6 +59,16 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
             else:
                 creators = [unicode(c) for c in items]
             items = creators
+        elif term == 'rights':
+            try:
+                rights = utf8_text(unicode(metadata.rights[0]))
+            except:
+                rights = b'Unknown'
+            exth.write(pack(b'>II', EXTH_CODES['rights'], len(rights) + 8))
+            exth.write(rights)
+            nrecs += 1
+            continue
+
         for item in items:
             data = unicode(item)
             if term != 'description':
@@ -78,14 +88,6 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
             exth.write(pack(b'>II', code, len(data) + 8))
             exth.write(data)
             nrecs += 1
-        if term == 'rights' :
-            try:
-                rights = utf8_text(unicode(metadata.rights[0]))
-            except:
-                rights = b'Unknown'
-            exth.write(pack(b'>II', EXTH_CODES['rights'], len(rights) + 8))
-            exth.write(rights)
-            nrecs += 1
 
     # Write UUID as ASIN
     uuid = None

From f45341702f2cf539deea1da905256df6a88d3864 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 11:08:15 +0530
Subject: [PATCH 06/18] KF8 Output: Fix calibre produced KF8 files not showing
 the 'Use publisher font' option on the Kindle Touch when they have embedded
 fonts

---
 src/calibre/ebooks/mobi/writer2/main.py      | 8 +++++++-
 src/calibre/ebooks/mobi/writer2/resources.py | 2 ++
 src/calibre/ebooks/mobi/writer8/exth.py      | 2 +-
 src/calibre/ebooks/mobi/writer8/mobi.py      | 2 ++
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/mobi/writer2/main.py b/src/calibre/ebooks/mobi/writer2/main.py
index e9f10a605e..b0146daebc 100644
--- a/src/calibre/ebooks/mobi/writer2/main.py
+++ b/src/calibre/ebooks/mobi/writer2/main.py
@@ -297,10 +297,13 @@ class MobiWriter(object):
 
         # 0x70 - 0x73 : EXTH flags
         # Bit 6 (0b1000000) being set indicates the presence of an EXTH header
+        # Bit 12 being set indicates the presence of embedded fonts
         # The purpose of the other bits is unknown
         exth_flags = 0b1010000
         if self.is_periodical:
             exth_flags |= 0b1000
+        if self.resources.has_fonts:
+            exth_flags |= 0b1000000000000
         record0.write(pack(b'>I', exth_flags))
 
         # 0x74 - 0x93 : Unknown
@@ -406,7 +409,10 @@ class MobiWriter(object):
         # Now change the header fields that need to be different in the MOBI 6
         # header
         header_fields['first_resource_record'] = first_image_record
-        header_fields['exth_flags'] = 0b100001010000 # Kinglegen uses this
+        ef = 0b100001010000 # Kinglegen uses this
+        if self.resources.has_fonts:
+            ef |= 0b1000000000000
+        header_fields['exth_flags'] = ef
         header_fields['fdst_record'] = pack(b'>HH', 1, last_content_record)
         header_fields['fdst_count'] = 1 # Why not 0? Kindlegen uses 1
         header_fields['flis_record'] = flis_number
diff --git a/src/calibre/ebooks/mobi/writer2/resources.py b/src/calibre/ebooks/mobi/writer2/resources.py
index 2f12793b03..bdf20a6f2c 100644
--- a/src/calibre/ebooks/mobi/writer2/resources.py
+++ b/src/calibre/ebooks/mobi/writer2/resources.py
@@ -32,6 +32,7 @@ class Resources(object):
         self.used_image_indices = set()
         self.image_indices = set()
         self.cover_offset = self.thumbnail_offset = None
+        self.has_fonts = False
 
         self.add_resources(add_fonts)
 
@@ -109,6 +110,7 @@ class Resources(object):
                         'ttf', 'otf'} and isinstance(item.data, bytes):
                     self.records.append(write_font_record(item.data))
                     self.item_map[item.href] = len(self.records)
+                    self.has_fonts = True
 
     def add_extra_images(self):
         '''
diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index df952a8eb4..508b77ce5b 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -140,7 +140,7 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
         nrecs += 1
 
     if be_kindlegen2:
-        vals = {204:201, 205:2, 206:2, 207:35621}
+        vals = {204:201, 205:2, 206:5, 207:0}
     elif is_periodical:
         # Pretend to be amazon's super secret periodical generator
         vals = {204:201, 205:2, 206:0, 207:101}
diff --git a/src/calibre/ebooks/mobi/writer8/mobi.py b/src/calibre/ebooks/mobi/writer8/mobi.py
index 48e0833ddb..769b301a50 100644
--- a/src/calibre/ebooks/mobi/writer8/mobi.py
+++ b/src/calibre/ebooks/mobi/writer8/mobi.py
@@ -277,6 +277,8 @@ class KF8Book(object):
         self.exth_flags = 0b1010000
         if writer.opts.mobi_periodical:
             self.exth_flags |= 0b1000
+        if resources.has_fonts:
+            self.exth_flags |= 0b1000000000000
 
         self.opts = writer.opts
         self.start_offset = writer.start_offset

From 314a45ae7eed904af94c9c5fa530dd38af0a86e5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 12:25:33 +0530
Subject: [PATCH 07/18] KF8 Output: Fix a couple of bugs in unused EXTH fields

---
 src/calibre/ebooks/mobi/debug/headers.py |  6 +++---
 src/calibre/ebooks/mobi/writer8/exth.py  | 10 +++++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/calibre/ebooks/mobi/debug/headers.py b/src/calibre/ebooks/mobi/debug/headers.py
index 5b80a46f1b..2b5d6920a5 100644
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@@ -144,9 +144,9 @@ class EXTHRecord(object):
                 118 : 'retailprice',
                 119 : 'retailpricecurrency',
                 121 : 'KF8 header section index',
-                125 : 'KF8 resources (images/fonts) count',
-                129 : 'KF8 cover URI',
-                131 : 'KF8 unknown count',
+                125 : 'KF8 unknown count',
+                129 : 'KF8 thumbnail URI',
+                131 : 'KF8 resources (images/fonts) count',
                 201 : 'coveroffset',
                 202 : 'thumboffset',
                 203 : 'hasfakecover',
diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index 508b77ce5b..603767b3d7 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -29,8 +29,9 @@ EXTH_CODES = {
     'versionnumber': 114,
     'startreading': 116,
     'kf8_header_index': 121,
-    'num_of_resources': 125,
-    'kf8_unknown_count': 131,
+    'kf8_unknown_count': 125,
+    'kf8_thumbnail_uri': 129,
+    'num_of_resources': 131,
     'coveroffset': 201,
     'thumboffset': 202,
     'hasfakecover': 203,
@@ -159,7 +160,10 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
     if thumbnail_offset is not None:
         exth.write(pack(b'>III', EXTH_CODES['thumboffset'], 12,
             thumbnail_offset))
-        nrecs += 1
+        cover_uri_str = bytes('kindle:embed:%04X' %(thumbnail_offset))
+        exth.write(pack(b'>II', EXTH_CODES['kf8_cover_uri'], len(cover_uri_str) + 8))
+        exth.write(cover_uri_str)
+        nrecs += 2
 
     if start_offset is not None:
         try:

From ae461c5370354773eac847b3c3c56931e32c70d6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 12:26:47 +0530
Subject: [PATCH 08/18] ...

---
 src/calibre/ebooks/mobi/writer8/exth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index 603767b3d7..0af54946cb 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -161,7 +161,7 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
         exth.write(pack(b'>III', EXTH_CODES['thumboffset'], 12,
             thumbnail_offset))
         cover_uri_str = bytes('kindle:embed:%04X' %(thumbnail_offset))
-        exth.write(pack(b'>II', EXTH_CODES['kf8_cover_uri'], len(cover_uri_str) + 8))
+        exth.write(pack(b'>II', EXTH_CODES['kf8_thumbnail_uri'], len(cover_uri_str) + 8))
         exth.write(cover_uri_str)
         nrecs += 2
 

From 3ff4708ee99825ac8d83ae0a872a723cc7f1f2e9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 13:48:06 +0530
Subject: [PATCH 09/18] Add env var to not use native file dialogs

---
 manual/customize.rst         |  1 +
 src/calibre/gui2/__init__.py | 16 ++++++++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/manual/customize.rst b/manual/customize.rst
index e2e2825de6..ceee4ece62 100644
--- a/manual/customize.rst
+++ b/manual/customize.rst
@@ -30,6 +30,7 @@ Environment variables
     * ``CALIBRE_OVERRIDE_DATABASE_PATH`` - allows you to specify the full path to metadata.db. Using this variable you can have metadata.db be in a location other than the library folder. Useful if your library folder is on a networked drive that does not support file locking.
     * ``CALIBRE_DEVELOP_FROM`` - Used to run from a calibre development environment. See :ref:`develop`.
     * ``CALIBRE_OVERRIDE_LANG`` - Used to force the language used by the interface (ISO 639 language code)
+    * ``CALIBRE_NO_NATIVE_FILEDIALOGS`` - Causes calibre to not use native file dialogs for selecting files/directories.
     * ``SYSFS_PATH`` - Use if sysfs is mounted somewhere other than /sys
     * ``http_proxy`` - Used on linux to specify an HTTP proxy
 
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 370e2dc993..00f5bef03d 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -573,17 +573,24 @@ class FileDialog(QObject):
         if not isinstance(initial_dir, basestring):
             initial_dir = os.path.expanduser(default_dir)
         self.selected_files = []
+        use_native_dialog = not os.environ.has_key('CALIBRE_NO_NATIVE_FILEDIALOGS')
         with SanitizeLibraryPath():
+            opts = QFileDialog.Option()
+            if not use_native_dialog:
+                opts |= QFileDialog.DontUseNativeDialog
             if mode == QFileDialog.AnyFile:
-                f = unicode(QFileDialog.getSaveFileName(parent, title, initial_dir, ftext, ""))
+                f = unicode(QFileDialog.getSaveFileName(parent, title,
+                    initial_dir, ftext, "", opts))
                 if f:
                     self.selected_files.append(f)
             elif mode == QFileDialog.ExistingFile:
-                f = unicode(QFileDialog.getOpenFileName(parent, title, initial_dir, ftext, ""))
+                f = unicode(QFileDialog.getOpenFileName(parent, title,
+                    initial_dir, ftext, "", opts))
                 if f and os.path.exists(f):
                     self.selected_files.append(f)
             elif mode == QFileDialog.ExistingFiles:
-                fs = QFileDialog.getOpenFileNames(parent, title, initial_dir, ftext, "")
+                fs = QFileDialog.getOpenFileNames(parent, title, initial_dir,
+                        ftext, "", opts)
                 for f in fs:
                     f = unicode(f)
                     if not f: continue
@@ -594,7 +601,8 @@ class FileDialog(QObject):
                     if f and os.path.exists(f):
                         self.selected_files.append(f)
             else:
-                opts = QFileDialog.ShowDirsOnly if mode == QFileDialog.Directory else QFileDialog.Option()
+                if mode == QFileDialog.Directory:
+                    opts |= QFileDialog.ShowDirsOnly
                 f = unicode(QFileDialog.getExistingDirectory(parent, title, initial_dir, opts))
                 if os.path.exists(f):
                     self.selected_files.append(f)

From f1e3d2b03d89f2ee290302168fccb1e4f817e978 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 22:02:53 +0530
Subject: [PATCH 10/18] Fix American Prospect

---
 recipes/aprospect.recipe | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/recipes/aprospect.recipe b/recipes/aprospect.recipe
index 1e07ce5f52..d0680f75c6 100644
--- a/recipes/aprospect.recipe
+++ b/recipes/aprospect.recipe
@@ -12,7 +12,11 @@ class AmericanProspect(BasicNewsRecipe):
     no_stylesheets        = True
     remove_javascript     = True
 
-    keep_only_tags = [dict(name='div', attrs={'class':'pad_10L10R'})]
-    remove_tags    = [dict(name='form'), dict(name='div', attrs={'class':['bkt_caption','sharebox noprint','badgebox']})]
+    #keep_only_tags = [dict(name='div', attrs={'class':'pad_10L10R'})]
+    #remove_tags    = [dict(name='form'), dict(name='div', attrs={'class':['bkt_caption','sharebox noprint','badgebox']})]
+    use_embedded_content = False
 
+    no_stylesheets = True
+    auto_cleanup = True
     feeds       = [(u'Articles', u'feed://www.prospect.org/articles_rss.jsp')]
+ 

From 6de2b807915c378b7bd36b875bff174d51654089 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sat, 14 Jul 2012 22:35:20 +0530
Subject: [PATCH 11/18] Fix find_identical_books() choking on books with too
 many authors

---
 src/calibre/library/database2.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index ba660be49f..9e6a0a3951 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1096,8 +1096,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         identical_book_ids = set([])
         if mi.authors:
             try:
+                quathors = mi.authors[:10] # Too many authors causes parsing of
+                                           # the search expression to fail
                 query = u' and '.join([u'author:"=%s"'%(a.replace('"', '')) for a in
-                    mi.authors])
+                    quathors])
+                qauthors = mi.authors[10:]
             except ValueError:
                 return identical_book_ids
             try:
@@ -1105,6 +1108,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except:
                 traceback.print_exc()
                 return identical_book_ids
+            if qauthors and book_ids:
+                matches = set()
+                qauthors = {lower(x) for x in qauthors}
+                for book_id in book_ids:
+                    aut = self.authors(book_id, index_is_id=True)
+                    if aut:
+                        aut = {lower(x.replace('|', ',')) for x in
+                                aut.split(',')}
+                        if aut.issuperset(qauthors):
+                            matches.add(book_id)
+                book_ids = matches
+
             for book_id in book_ids:
                 fbook_title = self.title(book_id, index_is_id=True)
                 fbook_title = fuzzy_title(fbook_title)

From e8e109a25a95fee76eabfa379f5968ae590f5f35 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 15 Jul 2012 11:13:40 +0530
Subject: [PATCH 12/18] Linux binaries: Bundle libglib to avoid
 incompatibilities with glib on various distros. See #1022019

---
 setup/installer/linux/freeze2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py
index 2aa5504bc6..6ecb21768f 100644
--- a/setup/installer/linux/freeze2.py
+++ b/setup/installer/linux/freeze2.py
@@ -23,6 +23,7 @@ MAGICK_PREFIX = '/usr'
 binary_includes = [
                 '/usr/bin/pdftohtml',
                 '/usr/bin/pdfinfo',
+                '/usr/lib/libglib-2.0.so.0',
                 '/usr/bin/pdftoppm',
                 '/usr/lib/libwmflite-0.2.so.7',
                 '/usr/lib/liblcms.so.1',

From 77208efb4bc08be9645a5dfe41e49ed58c677723 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 15 Jul 2012 11:15:44 +0530
Subject: [PATCH 13/18] Revert KF8 EXTH field changes

---
 src/calibre/ebooks/mobi/debug/headers.py |  6 +++---
 src/calibre/ebooks/mobi/writer8/exth.py  | 10 +++-------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/calibre/ebooks/mobi/debug/headers.py b/src/calibre/ebooks/mobi/debug/headers.py
index 2b5d6920a5..5b80a46f1b 100644
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@@ -144,9 +144,9 @@ class EXTHRecord(object):
                 118 : 'retailprice',
                 119 : 'retailpricecurrency',
                 121 : 'KF8 header section index',
-                125 : 'KF8 unknown count',
-                129 : 'KF8 thumbnail URI',
-                131 : 'KF8 resources (images/fonts) count',
+                125 : 'KF8 resources (images/fonts) count',
+                129 : 'KF8 cover URI',
+                131 : 'KF8 unknown count',
                 201 : 'coveroffset',
                 202 : 'thumboffset',
                 203 : 'hasfakecover',
diff --git a/src/calibre/ebooks/mobi/writer8/exth.py b/src/calibre/ebooks/mobi/writer8/exth.py
index 0af54946cb..508b77ce5b 100644
--- a/src/calibre/ebooks/mobi/writer8/exth.py
+++ b/src/calibre/ebooks/mobi/writer8/exth.py
@@ -29,9 +29,8 @@ EXTH_CODES = {
     'versionnumber': 114,
     'startreading': 116,
     'kf8_header_index': 121,
-    'kf8_unknown_count': 125,
-    'kf8_thumbnail_uri': 129,
-    'num_of_resources': 131,
+    'num_of_resources': 125,
+    'kf8_unknown_count': 131,
     'coveroffset': 201,
     'thumboffset': 202,
     'hasfakecover': 203,
@@ -160,10 +159,7 @@ def build_exth(metadata, prefer_author_sort=False, is_periodical=False,
     if thumbnail_offset is not None:
         exth.write(pack(b'>III', EXTH_CODES['thumboffset'], 12,
             thumbnail_offset))
-        cover_uri_str = bytes('kindle:embed:%04X' %(thumbnail_offset))
-        exth.write(pack(b'>II', EXTH_CODES['kf8_thumbnail_uri'], len(cover_uri_str) + 8))
-        exth.write(cover_uri_str)
-        nrecs += 2
+        nrecs += 1
 
     if start_offset is not None:
         try:

From caba1b15d654fe19a8737678ae01eea75e28eda1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 15 Jul 2012 14:31:53 +0530
Subject: [PATCH 14/18] PDF Output: Fix rendering of cover as first age of PDF
 (ignore margins so that the image covers the entire page)

---
 src/calibre/ebooks/pdf/writer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py
index ac371b74a6..51b03f96c5 100644
--- a/src/calibre/ebooks/pdf/writer.py
+++ b/src/calibre/ebooks/pdf/writer.py
@@ -223,7 +223,8 @@ class PDFWriter(QObject): # {{{
         if self.cover_data is None:
             return
         item_path = os.path.join(self.tmp_path, 'cover.pdf')
-        printer = get_pdf_printer(self.opts, output_file_name=item_path)
+        printer = get_pdf_printer(self.opts, output_file_name=item_path,
+                for_comic=True)
         self.combine_queue.insert(0, item_path)
         p = QPixmap()
         p.loadFromData(self.cover_data)

From ea2b35c53110dc3f5b97d5f0ac984fcdd8732793 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Sun, 15 Jul 2012 18:35:30 +0530
Subject: [PATCH 15/18] Fix #1024934 (No connection to new Android devices)

---
 src/calibre/devices/android/driver.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 9302854ad6..f17eba522b 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -194,7 +194,7 @@ class ANDROID(USBMS):
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
             'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
             'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
-            'PMP5097C']
+            'PMP5097C', 'MASS', 'NOVO7']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -211,7 +211,8 @@ class ANDROID(USBMS):
             'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW',
             'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
             'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
-            'THINKPAD_TABLET', 'SGH-T989', 'YP-G70']
+            'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
+            'ADVANCED']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

From bbc5811735509f73a3a09b6701543ac36aa6df2b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 16 Jul 2012 09:23:19 +0530
Subject: [PATCH 16/18] ...

---
 recipes/the_sun.recipe | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/recipes/the_sun.recipe b/recipes/the_sun.recipe
index 11500430ff..ae7c599328 100644
--- a/recipes/the_sun.recipe
+++ b/recipes/the_sun.recipe
@@ -6,16 +6,15 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1325006965(BasicNewsRecipe):
 
     title          = u'The Sun UK'
-    description = 'A Recipe for The Sun tabloid UK'
+    description = 'Articles from The Sun tabloid UK'
     __author__ = 'Dave Asbury'
-    # last updated 29/4/12
+    # last updated 15/7/12
     language = 'en_GB'
     oldest_article = 1
     max_articles_per_feed = 15
     remove_empty_feeds = True
     no_stylesheets = True
-    #auto_cleanup = True
-    #articles_are_obfuscated = True
+
 
     masthead_url = 'http://www.thesun.co.uk/sol/img/global/Sun-logo.gif'
     encoding = 'UTF-8'
@@ -34,7 +33,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
 
 
     keep_only_tags = [
-                               dict(name='h1'),dict(name='h2',attrs={'class' : 'medium centered'}),
+                               dict(name='h1'),dict(name='h2',attrs={'class' : ['large','large centered','medium centered','medium']}),dict(name='h3'),
                dict(name='div',attrs={'class' : 'text-center'}),
                dict(name='div',attrs={'id' : 'bodyText'})
                # dict(name='p')
@@ -72,22 +71,18 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
         cov2 = str(cov)
         cov2=cov2[27:-18]
         #cov2 now is pic url, now  go back to original function
-
         br = browser()
         br.set_handle_redirect(False)
         try:
             br.open_novisit(cov2)
             cover_url = cov2
         except:
-            cover_url = random.choice((
+            cover_url = random.choice([
                 'http://img.thesun.co.uk/multimedia/archive/00905/errorpage6_677961a_905507a.jpg'
                 ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage7_677962a_905505a.jpg'
                 ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage5_677960a_905512a.jpg'
                 ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage2_677957a_905502a.jpg'
                 ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage3_677958a_905503a.jpg'
-                ))
+                ])
 
         return cover_url
-
-
-

From e6a4163623ae427146d11f7876cf2ea20cbb4b40 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 16 Jul 2012 20:19:43 +0530
Subject: [PATCH 17/18] ...

---
 src/calibre/gui2/complete2.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py
index 8e0718c78e..59f1070577 100644
--- a/src/calibre/gui2/complete2.py
+++ b/src/calibre/gui2/complete2.py
@@ -189,12 +189,11 @@ class Completer(QListView): # {{{
                     e.accept()
                     return True
                 return False
-            if key in (Qt.Key_End, Qt.Key_Home, Qt.Key_Up, Qt.Key_Down,
-                    Qt.Key_PageUp, Qt.Key_PageDown):
+            if key in (Qt.Key_PageUp, Qt.Key_PageDown):
                 # Let the list view handle these keys
                 return False
-            if key in (Qt.Key_Tab, Qt.Key_Backtab):
-                self.next_match(previous=key == Qt.Key_Backtab)
+            if key in (Qt.Key_Tab, Qt.Key_Backtab, Qt.Key_Up, Qt.Key_Down):
+                self.next_match(previous=key in (Qt.Key_Backtab, Qt.Key_Up))
                 e.accept()
                 return True
             # Send to widget

From a317af75aaefbc3af8e379e885a0e462fc925726 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 17 Jul 2012 09:08:23 +0530
Subject: [PATCH 18/18] Add a tweak to not preselect the first completion in
 the new combobox

---
 resources/default_tweaks.py   | 8 ++++++++
 src/calibre/gui2/complete2.py | 5 +++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 420df116ce..3bba5ecca5 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -516,3 +516,11 @@ default_tweak_format = None
 #    enable_multicharacters_in_tag_browser = False
 enable_multicharacters_in_tag_browser = True
 
+#: Do not preselect a completion when editing authors/tags/series/etc.
+# This means that you can make changes and press Enter and your changes will
+# not be overwritten by a matching completion. However, if you wish to use the
+# completions you will now have to press Tab to select one before pressing
+# Enter. Which technique you prefer will depend on the state of metadata in
+# your library and your personal editing style.
+preselect_first_completion = False
+
diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py
index 59f1070577..603177498d 100644
--- a/src/calibre/gui2/complete2.py
+++ b/src/calibre/gui2/complete2.py
@@ -16,6 +16,7 @@ from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt, pyqtSignal, QObject,
 from calibre.utils.icu import sort_key, primary_startswith
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
+from calibre.utils.config import tweaks
 
 class CompleteModel(QAbstractListModel): # {{{
 
@@ -157,8 +158,8 @@ class Completer(QListView): # {{{
 
         p.setGeometry(pos.x(), pos.y(), w, h)
 
-        if (select_first and not self.currentIndex().isValid() and
-                self.model().rowCount() > 0):
+        if (tweaks['preselect_first_completion'] and select_first and not
+                self.currentIndex().isValid() and self.model().rowCount() > 0):
             self.setCurrentIndex(self.model().index(0))
 
         if not p.isVisible():