Ensure kakasi databases are always rebuilt on windows

setup/commands.py

@@ -11,7 +11,7 @@ __all__ = [
         'build', 'build_pdf2xml', 'server',
         'gui',
         'develop', 'install',
-        'resources',
+        'kakasi', 'resources',
         'check',
         'sdist',
         'manual', 'tag_release',
@@ -49,8 +49,9 @@ gui = GUI()
 from setup.check import Check
 check = Check()
 
-from setup.resources import Resources
+from setup.resources import Resources, Kakasi
 resources = Resources()
+kakasi = Kakasi()
 
 from setup.publish import Manual, TagRelease, Stage1, Stage2, \
         Stage3, Stage4, Publish

setup/installer/windows/__init__.py

@@ -32,6 +32,7 @@ class Win32(VMInstaller):
     FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command} --no-ice'
     INSTALLER_EXT = 'msi'
     SHUTDOWN_CMD = ['shutdown.exe', '-s', '-f', '-t', '0']
+    BUILD_BUILD  =  ['python setup.py kakasi',] + VMInstaller.BUILD_BUILD
 
     def download_installer(self):
         installer = self.installer()

setup/installer/windows/wix-template.xml

@@ -11,9 +11,6 @@
 						SummaryCodepage='1252' />
 
                 <Media Id="1" Cabinet="{app}.cab" CompressionLevel="{compression}" EmbedCab="yes" />
-                <!-- The following line is needed because of the patch to QtCore4.dll. You can remove this line
-                     after you update Qt beyond 4.7.2. 'emus' means re-install even if version is the same not just if it is older. -->
-                <Property Id='REINSTALLMODE' Value='emus'/>
 
                 <Upgrade Id="{upgrade_code}">
                     <UpgradeVersion Maximum="{version}"

setup/resources.py

@@ -23,13 +23,119 @@ def get_opts_from_parser(parser):
         for o in g.option_list:
             for x in do_opt(o): yield x
 
-class Resources(Command):
+class Kakasi(Command):
 
-    description = 'Compile various needed calibre resources'
+    description = 'Compile resources for unihandecode'
 
     KAKASI_PATH = os.path.join(Command.SRC,  __appname__,
             'ebooks', 'unihandecode', 'pykakasi')
 
+    def run(self, opts):
+        self.records = {}
+        src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','kanwadict2.db')
+        base = os.path.dirname(dest)
+        if not os.path.exists(base):
+            os.makedirs(base)
+
+        if self.newer(dest, src) or iswindows:
+            self.info('\tGenerating Kanwadict')
+
+            for line in open(src, "r"):
+                self.parsekdict(line)
+            self.kanwaout(dest)
+
+        src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','itaijidict2.pickle')
+
+        if self.newer(dest, src) or iswindows:
+            self.info('\tGenerating Itaijidict')
+            self.mkitaiji(src, dest)
+
+        src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','kanadict2.pickle')
+
+        if self.newer(dest, src) or iswindows:
+            self.info('\tGenerating kanadict')
+            self.mkkanadict(src, dest)
+
+        return
+
+
+    def mkitaiji(self, src, dst):
+        dic = {}
+        for line in open(src, "r"):
+            line = line.decode("utf-8").strip()
+            if line.startswith(';;'): # skip comment
+                continue
+            if re.match(r"^$",line):
+                continue
+            pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
+            dic[pair[0]] = pair[1]
+        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
+
+    def mkkanadict(self, src, dst):
+        dic = {}
+        for line in open(src, "r"):
+            line = line.decode("utf-8").strip()
+            if line.startswith(';;'): # skip comment
+                continue
+            if re.match(r"^$",line):
+                continue
+            (alpha, kana) = line.split(' ')
+            dic[kana] = alpha
+        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
+
+    def parsekdict(self, line):
+        line = line.decode("utf-8").strip()
+        if line.startswith(';;'): # skip comment
+            return
+        (yomi, kanji) = line.split(' ')
+        if ord(yomi[-1:]) <= ord('z'):
+            tail = yomi[-1:]
+            yomi = yomi[:-1]
+        else:
+            tail = ''
+        self.updaterec(kanji, yomi, tail)
+
+    def updaterec(self, kanji, yomi, tail):
+        key = "%04x"%ord(kanji[0])
+        if key in self.records:
+            if kanji in self.records[key]:
+                rec = self.records[key][kanji]
+                rec.append((yomi,tail))
+                self.records[key].update( {kanji: rec} )
+            else:
+                self.records[key][kanji]=[(yomi, tail)]
+        else:
+            self.records[key] = {}
+            self.records[key][kanji]=[(yomi, tail)]
+
+    def kanwaout(self, out):
+        try:
+            # Needed as otherwise anydbm tries to create a gdbm db when the db
+            # created on Unix is found
+            os.remove(out)
+        except:
+            pass
+        dic = anydbm.open(out, 'n')
+        for (k, v) in self.records.iteritems():
+            dic[k] = compress(marshal.dumps(v))
+        dic.close()
+
+    def clean(self):
+        kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')
+        if os.path.exists(kakasi):
+            shutil.rmtree(kakasi)
+
+class Resources(Command):
+
+    description = 'Compile various needed calibre resources'
+    sub_commands = ['kakasi']
+
     def run(self, opts):
         scripts = {}
         for x in ('console', 'gui'):
@@ -117,108 +223,13 @@ class Resources(Command):
         import json
         json.dump(function_dict, open(dest, 'wb'), indent=4)
 
-        self.run_kakasi(opts)
-
-    def run_kakasi(self, opts):
-        self.records = {}
-        src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanwadict2.db')
-        base = os.path.dirname(dest)
-        if not os.path.exists(base):
-            os.makedirs(base)
-
-        if self.newer(dest, src) or iswindows:
-            self.info('\tGenerating Kanwadict')
-
-            for line in open(src, "r"):
-                self.parsekdict(line)
-            self.kanwaout(dest)
-
-        src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','itaijidict2.pickle')
-
-        if self.newer(dest, src) or iswindows:
-            self.info('\tGenerating Itaijidict')
-            self.mkitaiji(src, dest)
-
-        src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanadict2.pickle')
-
-        if self.newer(dest, src) or iswindows:
-            self.info('\tGenerating kanadict')
-            self.mkkanadict(src, dest)
-
-        return
-
-
-    def mkitaiji(self, src, dst):
-        dic = {}
-        for line in open(src, "r"):
-            line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
-                continue
-            if re.match(r"^$",line):
-                continue
-            pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
-            dic[pair[0]] = pair[1]
-        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
-
-    def mkkanadict(self, src, dst):
-        dic = {}
-        for line in open(src, "r"):
-            line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
-                continue
-            if re.match(r"^$",line):
-                continue
-            (alpha, kana) = line.split(' ')
-            dic[kana] = alpha
-        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
-
-    def parsekdict(self, line):
-        line = line.decode("utf-8").strip()
-        if line.startswith(';;'): # skip comment
-            return
-        (yomi, kanji) = line.split(' ')
-        if ord(yomi[-1:]) <= ord('z'):
-            tail = yomi[-1:]
-            yomi = yomi[:-1]
-        else:
-            tail = ''
-        self.updaterec(kanji, yomi, tail)
-
-    def updaterec(self, kanji, yomi, tail):
-            key = "%04x"%ord(kanji[0])
-            if key in self.records:
-                if kanji in self.records[key]:
-                    rec = self.records[key][kanji]
-                    rec.append((yomi,tail))
-                    self.records[key].update( {kanji: rec} )
-                else:
-                    self.records[key][kanji]=[(yomi, tail)]
-            else:
-                self.records[key] = {}
-                self.records[key][kanji]=[(yomi, tail)]
-
-    def kanwaout(self, out):
-        dic = anydbm.open(out, 'c')
-        for (k, v) in self.records.iteritems():
-            dic[k] = compress(marshal.dumps(v))
-        dic.close()
-
-
     def clean(self):
         for x in ('scripts', 'recipes', 'ebook-convert-complete'):
             x = self.j(self.RESOURCES, x+'.pickle')
             if os.path.exists(x):
                 os.remove(x)
-        kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')
+        from setup.commands import kakasi
-        if os.path.exists(kakasi):
+        kakasi.clean()
-            shutil.rmtree(kakasi)
-