diff --git a/installer/linux/freeze.py b/installer/linux/freeze.py index 8524172a72..31f645dff1 100644 --- a/installer/linux/freeze.py +++ b/installer/linux/freeze.py @@ -31,6 +31,7 @@ def freeze(): '/usr/lib/libsqlite3.so.0', '/usr/lib/libsqlite3.so.0', '/usr/lib/libmng.so.1', + '/usr/lib/libpodofo.so.0.6.99', '/lib/libz.so.1', '/lib/libbz2.so.1', '/lib/libbz2.so.1', diff --git a/installer/osx/freeze.py b/installer/osx/freeze.py index 1861596f61..75621da017 100644 --- a/installer/osx/freeze.py +++ b/installer/osx/freeze.py @@ -229,6 +229,11 @@ _check_symlinks_prescript() all_modules = main_modules['console'] + main_modules['gui'] all_functions = main_functions['console'] + main_functions['gui'] print + print 'Adding PoDoFo' + pdf = glob.glob(os.path.expanduser('~/podofo/*.dylib'))[0] + shutil.copyfile(pdf, os.path.join(frameworks_dir, os.path.basename(pdf))) + + loader_path = os.path.join(resource_dir, 'loaders') if not os.path.exists(loader_path): os.mkdir(loader_path) diff --git a/installer/windows/freeze.py b/installer/windows/freeze.py index a4988c6703..f545f7e534 100644 --- a/installer/windows/freeze.py +++ b/installer/windows/freeze.py @@ -12,6 +12,7 @@ LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' PDFTOHTML = 'C:\\cygwin\\home\\kovid\\poppler-0.10.6\\rel\\pdftohtml.exe' IMAGEMAGICK_DIR = 'C:\\ImageMagick' PDFTK = 'C:\\pdftk.exe' +PODOFO = 'C:\\podofo' FONTCONFIG_DIR = 'C:\\fontconfig' VC90 = r'C:\VC90.CRT' @@ -101,8 +102,11 @@ class BuildEXE(py2exe.build_exe.py2exe): shutil.copyfile(PDFTOHTML, os.path.join(PY2EXE_DIR, os.path.basename(PDFTOHTML))) shutil.copyfile(PDFTOHTML+'.manifest', os.path.join(PY2EXE_DIR, os.path.basename(PDFTOHTML)+'.manifest')) - print '\tAdding pdftk' - shutil.copyfile(PDFTK, os.path.join(PY2EXE_DIR, os.path.basename(PDFTK))) + #print '\tAdding pdftk' + #shutil.copyfile(PDFTK, os.path.join(PY2EXE_DIR, os.path.basename(PDFTK))) + print 'Adding podofo' + for f in glob.glob(os.path.join(PODOFO, '*.dll')): + shutil.copyfile(f, os.path.join(PY2EXE_DIR, os.path.basename(f))) print '\tAdding ImageMagick' for f in os.listdir(IMAGEMAGICK_DIR): diff --git a/pyqtdistutils.py b/pyqtdistutils.py index 0e53aaabfe..b91b011bc1 100644 --- a/pyqtdistutils.py +++ b/pyqtdistutils.py @@ -80,13 +80,16 @@ CONFIG += x86 ppc os.chdir(cwd) def build_sbf(self, sip, sbf, bdir): - print '\tBuilding spf...' + print '\tBuilding sbf...' sip_bin = self.sipcfg.sip_bin + pyqt_sip_flags = [] + if hasattr(self, 'pyqtcfg'): + pyqt_sip_flags += ['-I', self.pyqtcfg.pyqt_sip_dir] + pyqt_sip_flags += self.pyqtcfg.pyqt_sip_flags.split() self.spawn([sip_bin, "-c", bdir, "-b", sbf, - '-I', self.pyqtcfg.pyqt_sip_dir, - ] + self.pyqtcfg.pyqt_sip_flags.split()+ + ] + pyqt_sip_flags + [sip]) def build_pyqt(self, bdir, sbf, ext, qtobjs, headers): @@ -94,9 +97,14 @@ CONFIG += x86 ppc build_file=sbf, dir=bdir, makefile='Makefile.pyqt', universal=OSX_SDK, qt=1) + makefile.extra_libs = ext.libraries + makefile.extra_lib_dirs = ext.library_dirs + makefile.extra_cxxflags = ext.extra_compile_args + if 'win32' in sys.platform: makefile.extra_lib_dirs += WINDOWS_PYTHON makefile.extra_include_dirs = list(set(map(os.path.dirname, headers))) + makefile.extra_include_dirs += ext.include_dirs makefile.extra_lflags += qtobjs makefile.generate() cwd = os.getcwd() @@ -110,7 +118,7 @@ CONFIG += x86 ppc def build_extension(self, ext): self.inplace = True # Causes extensions to be built in the source tree - + fullname = self.get_ext_fullname(ext.name) if self.inplace: # ignore build-lib -- put the compiled extension into @@ -127,14 +135,14 @@ CONFIG += x86 ppc else: ext_filename = os.path.join(self.build_lib, self.get_ext_filename(fullname)) - bdir = os.path.abspath(os.path.join(self.build_temp, fullname)) + bdir = os.path.abspath(os.path.join(self.build_temp, fullname)) if not os.path.exists(bdir): os.makedirs(bdir) - + if not isinstance(ext, PyQtExtension): if not iswindows: return _build_ext.build_extension(self, ext) - + c_sources = [f for f in ext.sources if os.path.splitext(f)[1].lower() in ('.c', '.cpp', '.cxx')] compile_args = '/c /nologo /Ox /MD /W3 /GX /DNDEBUG'.split() compile_args += ext.extra_compile_args @@ -147,7 +155,7 @@ CONFIG += x86 ppc objects.append(o) compiler = cc + ['/Tc'+f, '/Fo'+o] self.spawn(compiler) - out = os.path.join(bdir, base+'.pyd') + out = os.path.join(bdir, base+'.pyd') linker = [msvc.linker] + '/DLL /nologo /INCREMENTAL:NO'.split() linker += ['/LIBPATH:'+x for x in self.library_dirs] linker += [x+'.lib' for x in ext.libraries] @@ -156,9 +164,9 @@ CONFIG += x86 ppc for src in (out, out+'.manifest'): shutil.copyfile(src, os.path.join('src', 'calibre', 'plugins', os.path.basename(src))) return - - - + + + if not os.path.exists(bdir): os.makedirs(bdir) ext.sources2 = map(os.path.abspath, ext.sources) @@ -200,6 +208,14 @@ CONFIG += x86 ppc shutil.copyfile(mod, ext_filename) shutil.copymode(mod, ext_filename) + + if self.force or newer_group([mod], ext_filename, 'newer'): + if os.path.exists(ext_filename): + os.unlink(ext_filename) + shutil.copyfile(mod, ext_filename) + shutil.copymode(mod, ext_filename) + + def get_sip_output_list(self, sbf, bdir): """ Parse the sbf file specified to extract the name of the generated source diff --git a/setup.py b/setup.py index 003067b34f..5e018ebd20 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,25 @@ if __name__ == '__main__': entry_points['console_scripts'].append( 'calibre_postinstall = calibre.linux:post_install') - ext_modules = [ + optional = [] + + + podofo_inc = '/usr/include/podofo' if islinux else \ + 'C:\\podofo\\include\\podofo' if iswindows else \ + '/Users/kovid/podofo/include/podofo' + podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \ + '/Users/kovid/podofo/lib' + if os.path.exists(os.path.join(podofo_inc, 'PdfString.h')): + eca = ['/EHsc'] if iswindows else [] + optional.append(PyQtExtension('calibre.plugins.podofo', [], + ['src/calibre/utils/podofo/podofo.sip'], + libraries=['podofo'], extra_compile_args=eca, + library_dirs=[os.environ.get('PODOFO_LIB_DIR', podofo_lib)], + include_dirs=\ + [os.environ.get('PODOFO_INC_DIR', podofo_inc)])) + + ext_modules = optional + [ + Extension('calibre.plugins.lzx', sources=['src/calibre/utils/lzx/lzxmodule.c', 'src/calibre/utils/lzx/compressor.c', diff --git a/src/calibre/constants.py b/src/calibre/constants.py index a99f2421ce..4f1d83046a 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -53,7 +53,7 @@ if plugins is None: plugin_path = getattr(pkg_resources, 'resource_filename')('calibre', 'plugins') sys.path.insert(0, plugin_path) - for plugin in ['pictureflow', 'lzx', 'msdes', 'cPalmdoc'] + \ + for plugin in ['pictureflow', 'lzx', 'msdes', 'podofo', 'cPalmdoc'] + \ (['winutil'] if iswindows else []) + \ (['usbobserver'] if isosx else []): try: diff --git a/src/calibre/devices/bebook/driver.py b/src/calibre/devices/bebook/driver.py index 639a9fdd13..0980554387 100644 --- a/src/calibre/devices/bebook/driver.py +++ b/src/calibre/devices/bebook/driver.py @@ -20,9 +20,9 @@ class BEBOOK(USBMS): PRODUCT_ID = [0x8803, 0x6803] BCD = [0x312] - VENDOR_NAME = 'BEBOOK' - WINDOWS_MAIN_MEM = 'BEBOOK_INTERNAL_MEMORY' - WINDOWS_CARD_A_MEM = 'BEBOOK_STORAGE_CARD' + VENDOR_NAME = 'LINUX' + WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET' + WINDOWS_CARD_MEM = 'FILE-STOR_GADGET' OSX_MAIN_MEM = 'BeBook Internal Memory' OSX_CARD_A_MEM = 'BeBook Storage Card' @@ -34,17 +34,25 @@ class BEBOOK(USBMS): FDI_LUNS = {'lun0':1, 'lun1':0, 'lun2':2} + def windows_sort_drives(self, drives): + main = drives.get('main', None) + card = drives.get('carda', None) + if card and main and card < main: + drives['main'] = card + drives['carda'] = main + + return drives + + class BEBOOK_MINI(BEBOOK): name = 'BeBook Mini driver' description = _('Communicate with the BeBook Mini eBook reader.') + VENDOR_ID = [0x0492] PRODUCT_ID = [0x8813] BCD = [0x319] - WINDOWS_MAIN_MEM = 'BEBOOKMINI_INTERNAL_MEMORY' - WINDOWS_CARD_MEM = 'BEBOOKMINI_STORAGE_CARD' - OSX_MAIN_MEM = 'BeBook Mini Internal Memory' OSX_CARD_MEM = 'BeBook Mini Storage Card' diff --git a/src/calibre/devices/eb600/driver.py b/src/calibre/devices/eb600/driver.py index c390ce582f..b42c77f172 100644 --- a/src/calibre/devices/eb600/driver.py +++ b/src/calibre/devices/eb600/driver.py @@ -43,7 +43,7 @@ class EB600(USBMS): def windows_sort_drives(self, drives): main = drives.get('main', None) - card = drives.get('card', None) + card = drives.get('carda', None) if card and main and card < main: drives['main'] = card drives['carda'] = main diff --git a/src/calibre/devices/jetbook/driver.py b/src/calibre/devices/jetbook/driver.py index 05da539dd8..199566357b 100644 --- a/src/calibre/devices/jetbook/driver.py +++ b/src/calibre/devices/jetbook/driver.py @@ -76,7 +76,7 @@ class JETBOOK(USBMS): if newpath == path: newpath = os.path.join(newpath, author, title) - + if not os.path.exists(newpath): os.makedirs(newpath) @@ -97,7 +97,7 @@ class JETBOOK(USBMS): self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) self.report_progress(1.0, _('Transferring books to device...')) - + return zip(paths, cycle([on_card])) @classmethod @@ -109,7 +109,7 @@ class JETBOOK(USBMS): return txt.decode(sys.getfilesystemencoding(), 'replace') return txt - + from calibre.devices.usbms.driver import metadata_from_formats mi = metadata_from_formats([path]) @@ -126,10 +126,10 @@ class JETBOOK(USBMS): def windows_sort_drives(self, drives): main = drives.get('main', None) - card = drives.get('card', None) + card = drives.get('carda', None) if card and main and card < main: drives['main'] = card - drives['card'] = main + drives['carda'] = main return drives diff --git a/src/calibre/ebooks/metadata/pdf.py b/src/calibre/ebooks/metadata/pdf.py index e05e95012d..038418cd41 100644 --- a/src/calibre/ebooks/metadata/pdf.py +++ b/src/calibre/ebooks/metadata/pdf.py @@ -7,9 +7,7 @@ import sys, os, cStringIO from threading import Thread from calibre import StreamReadWrapper -from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.ptempfile import TemporaryDirectory -from pyPdf import PdfFileReader, PdfFileWriter try: from calibre.utils.PythonMagickWand import \ NewMagickWand, MagickReadImage, MagickSetImageFormat, \ @@ -17,11 +15,17 @@ try: _imagemagick_loaded = True except: _imagemagick_loaded = False +from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.utils.pdftk import set_metadata as pdftk_set_metadata +from calibre.utils.podofo import get_metadata as podofo_get_metadata, \ + set_metadata as podofo_set_metadata + def get_metadata(stream, extract_cover=True): - """ Return metadata as a L{MetaInfo} object """ - mi = MetaInformation(_('Unknown'), [_('Unknown')]) + try: + mi = podofo_get_metadata(stream) + except: + mi = get_metadata_pypdf(stream) stream.seek(0) if extract_cover and _imagemagick_loaded: @@ -32,7 +36,26 @@ def get_metadata(stream, extract_cover=True): except: import traceback traceback.print_exc() + return mi + +def set_metadata(stream, mi): + stream.seek(0) + try: + return podofo_set_metadata(stream, mi) + except: + pass + try: + return pdftk_set_metadata(stream, mi) + except: + pass + set_metadata_pypdf(stream, mi) + + +def get_metadata_pypdf(stream): + """ Return metadata as a L{MetaInfo} object """ + from pyPdf import PdfFileReader + mi = MetaInformation(_('Unknown'), [_('Unknown')]) try: with StreamReadWrapper(stream) as stream: info = PdfFileReader(stream).getDocumentInfo() @@ -66,18 +89,12 @@ class MetadataWriter(Thread): except RuntimeError: pass -def set_metadata(stream, mi): - stream.seek(0) - try: - pdftk_set_metadata(stream, mi) - except: - pass - else: - return - +def set_metadata_pypdf(stream, mi): # Use a StringIO object for the pdf because we will want to over # write it later and if we are working on the stream directly it # could cause some issues. + + from pyPdf import PdfFileReader, PdfFileWriter raw = cStringIO.StringIO(stream.read()) orig_pdf = PdfFileReader(raw) title = mi.title if mi.title else orig_pdf.documentInfo.title @@ -88,7 +105,7 @@ def set_metadata(stream, mi): for page in orig_pdf.pages: out_pdf.addPage(page) writer.start() - writer.join(15) # Wait 15 secs for writing to complete + writer.join(10) # Wait 10 secs for writing to complete out_pdf.killed = True writer.join() if out_pdf.killed: @@ -102,6 +119,8 @@ def set_metadata(stream, mi): stream.seek(0) def get_cover(stream): + from pyPdf import PdfFileReader, PdfFileWriter + try: with StreamReadWrapper(stream) as stream: pdf = PdfFileReader(stream) diff --git a/src/calibre/gui2/convert/gui_conversion.py b/src/calibre/gui2/convert/gui_conversion.py index 8f25acb7be..f8da2aa824 100644 --- a/src/calibre/gui2/convert/gui_conversion.py +++ b/src/calibre/gui2/convert/gui_conversion.py @@ -4,12 +4,15 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -from calibre.ebooks.conversion.plumber import Plumber +from calibre.ebooks.conversion.plumber import Plumber, DummyReporter from calibre.utils.logging import Log +from calibre.customize.conversion import OptionRecommendation -def gui_convert(input, output, recommendations, notification): - plumber = Plumber(input, output, Log(), notification) +def gui_convert(input, output, recommendations, notification=DummyReporter()): + recommendations = list(recommendations) + recommendations.append(('verbose', 2, OptionRecommendation.HIGH)) + plumber = Plumber(input, output, Log(), report_progress=notification) plumber.merge_ui_recommendations(recommendations) - + plumber.run() diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 33e985a0d1..47276d2519 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -179,6 +179,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): md.addSeparator() md.addAction(_('Download metadata and covers')) md.addAction(_('Download only metadata')) + md.addAction(_('Download only covers')) self.metadata_menu = md self.add_menu = QMenu() self.add_menu.addAction(_('Add books from a single directory')) @@ -209,6 +210,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): partial(self.download_metadata, covers=True)) QObject.connect(md.actions()[5], SIGNAL('triggered(bool)'), partial(self.download_metadata, covers=False)) + QObject.connect(md.actions()[6], SIGNAL('triggered(bool)'), + partial(self.download_metadata, covers=True, + set_metadata=False)) + self.save_menu = QMenu() @@ -834,7 +839,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ############################### Edit metadata ############################## - def download_metadata(self, checked, covers=True): + def download_metadata(self, checked, covers=True, set_metadata=True): rows = self.library_view.selectionModel().selectedRows() previous = self.library_view.currentIndex() if not rows or len(rows) == 0: @@ -845,10 +850,12 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): db = self.library_view.model().db ids = [db.id(row.row()) for row in rows] from calibre.gui2.metadata import DownloadMetadata - self._download_book_metadata = DownloadMetadata(db, ids, get_covers=covers) + self._download_book_metadata = DownloadMetadata(db, ids, + get_covers=covers, set_metadata=set_metadata) self._download_book_metadata.start() + x = _('covers') if covers and not set_metadata else _('metadata') self.progress_indicator.start( - _('Downloading metadata for %d book(s)')%len(ids)) + _('Downloading %s for %d book(s)')%(x, len(ids))) self._book_metadata_download_check = QTimer(self) self.connect(self._book_metadata_download_check, SIGNAL('timeout()'), self.book_metadata_download_check) diff --git a/src/calibre/gui2/metadata.py b/src/calibre/gui2/metadata.py index 607fd6a41b..247880d7b0 100644 --- a/src/calibre/gui2/metadata.py +++ b/src/calibre/gui2/metadata.py @@ -41,11 +41,12 @@ class Worker(Thread): class DownloadMetadata(Thread): - def __init__(self, db, ids, get_covers): + def __init__(self, db, ids, get_covers, set_metadata=True): Thread.__init__(self) self.setDaemon(True) self.metadata = {} self.covers = {} + self.set_metadata = set_metadata self.db = db self.updated = set([]) self.get_covers = get_covers @@ -96,8 +97,9 @@ class DownloadMetadata(Thread): self.commit_covers() self.commit_covers(True) - for id in self.fetched_metadata: - self.db.set_metadata(id, self.metadata[id]) + if self.set_metadata: + for id in self.fetched_metadata: + self.db.set_metadata(id, self.metadata[id]) self.updated = set(self.fetched_metadata) diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 4f3f837679..711c10943b 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -16,6 +16,7 @@ from calibre.gui2 import warning_dialog from calibre.gui2.convert.single import NoSupportedInputFormats from calibre.gui2.convert.single import Config as SingleConfig from calibre.gui2.convert.bulk import BulkConfig +from calibre.customize.conversion import OptionRecommendation from calibre.utils.config import prefs def convert_single_ebook(parent, db, book_ids, auto_conversion=False, out_format=None): @@ -131,13 +132,16 @@ def fetch_scheduled_recipe(recipe, script): fmt = prefs['output_format'].lower() pt = PersistentTemporaryFile(suffix='_recipe_out.%s'%fmt.lower()) pt.close() - args = ['ebook-convert', script, pt.name, '-vv'] + recs = [] + args = [script, pt.name, recs] if recipe.needs_subscription: x = config.get('recipe_account_info_%s'%recipe.id, False) if not x: raise ValueError(_('You must set a username and password for %s')%recipe.title) - args.extend(['--username', x[0], '--password', x[1]]) - - return 'ebook-convert', [args], _('Fetch news from ')+recipe.title, fmt.upper(), [pt] + recs.append(('username', x[0], OptionRecommendation.HIGH)) + recs.append(('password', x[1], OptionRecommendation.HIGH)) + + + return 'gui_convert', args, _('Fetch news from ')+recipe.title, fmt.upper(), [pt] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 85d2f73e85..c18de5b336 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -402,7 +402,8 @@ class LibraryDatabase2(LibraryDatabase): def get_property(idx, index_is_id=False, loc=-1): row = self.data._data[idx] if index_is_id else self.data[idx] - return row[loc] + if row is not None: + return row[loc] for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', 'publisher', 'rating', 'series', 'series_index', 'tags', diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 0c02ef2925..717effd455 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -220,7 +220,7 @@ Post any output you see in a help message on the `Forum '%\ (self.email, self.country, self.amount, self.date.isoformat(), 'name="%s"'%self.name if self.name else '') @@ -93,7 +98,7 @@ class Country(list): def __str__(self): return self.name + ': %.2f%%'%self.percent - + def __cmp__(self, other): return cmp(self.total, other.total) @@ -118,7 +123,7 @@ class Stats: if r.date not in self.days.keys(): self.days[r.date] = [] self.days[r.date].append(r) - + self.min, self.max = start, end self.period = (self.max - self.min) + timedelta(days=1) daily_totals = [] @@ -139,7 +144,24 @@ class Stats: self.countries[r.country].append(r) for country in self.countries.values(): country.percent = (100 * country.total/self.total) if self.total else 0. - + + def get_daily_averages(self): + month_buckets, month_order = {}, [] + x = self.min + for t in self.daily_totals: + month = (x.year, x.month) + if month not in month_buckets: + month_buckets[month] = 0. + month_order.append(month) + month_buckets[month] += t + x += timedelta(days=1) + c = calendar.Calendar() + month_days = [days_in_month(*x) for x in month_order] + month_averages = [month_buckets[x]/float(y) for x, y in zip(month_order, month_days)] + return month_order, month_averages + + + def __str__(self): buf = cStringIO.StringIO() print >>buf, '\tTotal: %.2f'%self.total @@ -149,7 +171,7 @@ class Stats: for c in self.countries.values(): print >>buf, '\t\t', c return buf.getvalue() - + def to_html(self, num_of_countries=sys.maxint): countries = sorted(self.countries.values(), cmp=cmp, reverse=True)[:num_of_countries] crows = ['%s%.2f %%'%(c.name, c.percent) for c in countries] @@ -168,32 +190,48 @@ class Stats:
%(ctable)s - ''')%dict(total=self.total, da=self.daily_average, ac=self.average, + ''')%dict(total=self.total, da=self.daily_average, ac=self.average, ctable=ctable, period=self.period.days, num=len(self.totals), - dd=self.daily_deviation, ad=self.average_deviation, - dpd=len(self.totals)/float(self.period.days), + dd=self.daily_deviation, ad=self.average_deviation, + dpd=len(self.totals)/float(self.period.days), min=self.min.isoformat(), max=self.max.isoformat()) - + def expose(func): - + def do(self, *args, **kwargs): dict.update(cherrypy.response.headers, {'Server':'Donations_server/1.0'}) return func(self, *args, **kwargs) - + return cherrypy.expose(do) class Server(object): - + TRENDS = '/tmp/donations_trend.png' MONTH_TRENDS = '/tmp/donations_month_trend.png' - + AVERAGES = '/tmp/donations_averages.png' + def __init__(self, apache=False, root='/', data_file='/tmp/donations.xml'): self.apache = apache self.document_root = root self.data_file = data_file self.read_records() - + + def calculate_daily_averages(self): + stats = self.get_slice(self.earliest, self.latest) + fig = plt.figure(2, (10, 4), 96)#, facecolor, edgecolor, frameon, FigureClass) + fig.clear() + ax = fig.add_subplot(111) + month_order, month_averages = stats.get_daily_averages() + x = [date(y, m, 1) for y, m in month_order[:-1]] + ax.plot(x, month_averages[:-1]) + ax.set_xlabel('Month') + ax.set_ylabel('Daily average ($)') + ax.xaxis.set_major_locator(mdates.MonthLocator(interval=2)) + ax.xaxis.set_major_formatter(mdates.DateFormatter('%m/%y')) + fig.savefig(self.AVERAGES) + + def calculate_month_trend(self, days=31): stats = self.get_slice(date.today()-timedelta(days=days-1), date.today()) fig = plt.figure(2, (10, 4), 96)#, facecolor, edgecolor, frameon, FigureClass) @@ -214,14 +252,14 @@ Total: $%(total).2f Daily average: $%(da).2f \u00b1 %(dd).2f Average contribution: $%(ac).2f \u00b1 %(ad).2f Donors per day: %(dpd).2f - '''%dict(total=stats.total, da=stats.daily_average, + '''%dict(total=stats.total, da=stats.daily_average, dd=stats.daily_deviation, ac=stats.average, ad=stats.average_deviation, dpd=len(stats.totals)/float(stats.period.days), ) text = ax.annotate(text, (0.5, 0.65), textcoords='axes fraction') fig.savefig(self.MONTH_TRENDS) - + def calculate_trend(self): def months(start, end): pos = range_for_month(start.year, start.month)[0] @@ -253,7 +291,7 @@ Donors per day: %(dpd).2f fig.savefig(self.TRENDS) #plt.show() - + def read_records(self): self.tree = etree.parse(self.data_file) self.last_read_time = time.time() @@ -269,21 +307,22 @@ Donors per day: %(dpd).2f self.earliest, self.latest = min_date, max_date self.calculate_trend() self.calculate_month_trend() - + self.calculate_daily_averages() + def get_slice(self, start_date, end_date): stats = Stats([r for r in self.records if r.date >= start_date and r.date <= end_date], start_date, end_date) return stats - + def month(self, year, month): return self.get_slice(*range_for_month(year, month)) - + def year(self, year): return self.get_slice(*range_for_year(year)) - + def range_to_date(self, raw): return date(*map(int, raw.split('-'))) - + def build_page(self, period_type, data): if os.stat(self.data_file).st_mtime >= self.last_read_time: self.read_records() @@ -294,7 +333,7 @@ Donors per day: %(dpd).2f yy = data if period_type == 'year' else year rl = data[0] if period_type == 'range' else '' rr = data[1] if period_type == 'range' else '' - + def build_month_list(current): months = [] for i in range(1, 13): @@ -302,7 +341,7 @@ Donors per day: %(dpd).2f sel = 'selected="selected"' if i == current else '' months.append(''%(i, sel, month)) return months - + def build_year_list(current): all_years = sorted(range(self.earliest.year, self.latest.year+1, 1)) if current not in all_years: @@ -312,11 +351,11 @@ Donors per day: %(dpd).2f sel = 'selected="selected"' if year == current else '' years.append(''%(year, sel, year)) return years - + mmlist = ''%('\n'.join(build_month_list(mm))) mylist = ''%('\n'.join(build_year_list(my))) yylist = ''%('\n'.join(build_year_list(yy))) - + if period_type == 'month': range_stats = range_for_month(my, mm) elif period_type == 'year': @@ -332,9 +371,9 @@ Donors per day: %(dpd).2f range_stats = '
Invalid input:\n%s
'%err else: range_stats = self.get_slice(*range_stats).to_html(num_of_countries=10) - + today = self.get_slice(date.today(), date.today()) - + return textwrap.dedent('''\ @@ -365,7 +404,7 @@ Donors per day: %(dpd).2f if ((dayobj.getMonth()+1!=monthfield)||(dayobj.getDate()!=dayfield)||(dayobj.getFullYear()!=yearfield)) return false; return true; } - + function check_period_form(form) { if (form.period_type[2].checked) { if (!test_date(form.range_left.value)) { @@ -381,11 +420,11 @@ Donors per day: %(dpd).2f } return true; } - + function is_empty(val) { return val.trim().length == 0 } - + function check_add_form(form) { var test_amount = /[\.0-9]+/; if (is_empty(form.email.value)) { @@ -409,8 +448,8 @@ Donors per day: %(dpd).2f return false; } return true; - } - + } + function rationalize_periods() { var form = document.forms[0]; var disabled = !form.period_type[0].checked; @@ -438,7 +477,7 @@ Donors per day: %(dpd).2f

Donations to date

%(todate)s - +

Donations in period

@@ -463,10 +502,13 @@ Donors per day: %(dpd).2f
-

Income trends for the last year

Income trends -

Income trends for the last 31 days

+

Income trends for the last year

Month income trend +

Income trends for the last 31 days

+ Daily average + income trend +

Income trends since records started

@@ -479,26 +521,31 @@ Donors per day: %(dpd).2f rl=rl, rr=rr, range_stats=range_stats, root=self.document_root, today=today.total ) - + @expose def index(self): month = date.today().month year = date.today().year cherrypy.response.headers['Content-Type'] = 'application/xhtml+xml' return self.build_page('month', (year, month)) - + @expose def trend_png(self): cherrypy.response.headers['Content-Type'] = 'image/png' return open(self.TRENDS, 'rb').read() - + @expose def month_trend_png(self): cherrypy.response.headers['Content-Type'] = 'image/png' return open(self.MONTH_TRENDS, 'rb').read() - + @expose - def show(self, period_type='month', month_month='', month_year='', + def average_trend_png(self): + cherrypy.response.headers['Content-Type'] = 'image/png' + return open(self.AVERAGES, 'rb').read() + + @expose + def show(self, period_type='month', month_month='', month_year='', year_year='', range_left='', range_right=''): if period_type == 'month': mm = int(month_month) if month_month else date.today().month @@ -510,7 +557,7 @@ Donors per day: %(dpd).2f data = (range_left, range_right) cherrypy.response.headers['Content-Type'] = 'application/xhtml+xml' return self.build_page(period_type, data) - + def config(): config = { 'global': { @@ -527,9 +574,9 @@ def apache_start(): 'environment' : 'production', 'show_tracebacks' : False, }) - cherrypy.tree.mount(Server(apache=True, root='/donations/', data_file='/var/www/calibre.kovidgoyal.net/donations.xml'), + cherrypy.tree.mount(Server(apache=True, root='/donations/', data_file='/var/www/calibre.kovidgoyal.net/donations.xml'), '/donations', config=config()) - + def main(args=sys.argv): server = Server() diff --git a/src/calibre/utils/ipc/__init__.py b/src/calibre/utils/ipc/__init__.py new file mode 100644 index 0000000000..3d1a86922e --- /dev/null +++ b/src/calibre/utils/ipc/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py new file mode 100644 index 0000000000..6c0ba46885 --- /dev/null +++ b/src/calibre/utils/ipc/launch.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import subprocess, os, sys, time + +from calibre.constants import iswindows, isosx, isfrozen +from calibre.utils.config import prefs +from calibre.ptempfile import PersistentTemporaryFile + +if iswindows: + import win32process + +class Worker(object): + ''' + Platform independent object for launching child processes. All processes + have the environment variable :envvar:`CALIBRE_WORKER` set. + + Useful attributes: ``is_alive``, ``returncode`` + usefule methods: ``kill`` + + To launch child simply call the Worker object. By default, the child's + output is redirected to an on disk file, the path to which is returned by + the call. + ''' + + @property + def osx_interpreter(self): + exe = os.path.basename(sys.executable) + return exe if 'python' in exe else 'python' + + @property + def osx_contents_dir(self): + fd = os.path.realpath(getattr(sys, 'frameworks_dir')) + return os.path.dirname(fd) + + @property + def executable(self): + if iswindows: + return os.path.join(os.path.dirname(sys.executable), + 'calibre-parallel.exe' if isfrozen else \ + 'Scripts\\calibre-parallel.exe') + if isosx: + if not isfrozen: return 'calibre-parallel' + contents = os.path.join(self.osx_contents_dir, + 'console.app', 'Contents') + return os.path.join(contents, 'MacOS', self.osx_interpreter) + + return os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \ + if isfrozen else 'calibre-parallel' + + @property + def gui_executable(self): + if isfrozen and isosx: + return os.path.join(self.osx_contents_dir, + 'MacOS', self.osx_interpreter) + + return self.executable + + @property + def env(self): + env = dict(os.environ) + env['CALIBRE_WORKER'] = '1' + env.update(self._env) + return env + + @property + def is_alive(self): + return hasattr(self, 'child') and self.child.poll() is not None + + @property + def returncode(self): + if not hasattr(self, 'child'): return None + self.child.poll() + return self.child.returncode + + def kill(self): + try: + if self.is_alive: + if iswindows: + return self.child.kill() + try: + self.child.terminate() + st = time.time() + while self.is_alive and time.time()-st < 2: + time.sleep(0.2) + finally: + if self.is_alive: + self.child.kill() + except: + pass + + def __init__(self, env, gui=False): + self._env = {} + self.gui = gui + if isosx and isfrozen: + contents = os.path.join(self.osx_contents_dir, 'console.app', 'Contents') + resources = os.path.join(contents, 'Resources') + fd = os.path.join(contents, 'Frameworks') + self._env['PYTHONHOME'] = resources + self._env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick') + self._env['DYLD_LIBRARY_PATH'] = os.path.join(fd, 'ImageMagick', 'lib') + if isfrozen and not (iswindows or isosx): + self._env['LD_LIBRARY_PATH'] = getattr(sys, 'frozen_path') + ':'\ + + os.environ.get('LD_LIBRARY_PATH', '') + self._env.update(env) + + def __call__(self, redirect_output=True, cwd=None, priority=None): + ''' + If redirect_output is True, output from the child is redirected + to a file on disk and this method returns the path to that file. + ''' + exe = self.gui_executable if self.gui else self.executable + env = self.env + env['ORIGWD'] = cwd or os.path.abspath(os.getcwd()) + _cwd = cwd + if isfrozen and not iswindows and not isosx: + _cwd = getattr(sys, 'frozen_path', None) + if priority is None: + priority = prefs['worker_process_priority'] + cmd = [exe] + if isosx: + cmd += ['-c', 'from calibre.utils.worker import main; main()'] + args = { + 'env' : env, + 'cwd' : _cwd, + } + if iswindows: + priority = { + 'high' : win32process.HIGH_PRIORITY_CLASS, + 'normal' : win32process.NORMAL_PRIORITY_CLASS, + 'low' : win32process.IDLE_PRIORITY_CLASS}[priority] + args['creationflags'] = win32process.CREATE_NO_WINDOW|priority + ret = None + if redirect_output: + self._file = PersistentTemporaryFile('_worker_redirect.log') + args['stdout'] = self._file._fd + args['stderr'] = subprocess.STDOUT + ret = self._file.name + + self.child = subprocess.Popen(cmd, **args) + + return ret + + + diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py new file mode 100644 index 0000000000..3d1a86922e --- /dev/null +++ b/src/calibre/utils/ipc/server.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/utils/ipc/worker.py b/src/calibre/utils/ipc/worker.py new file mode 100644 index 0000000000..75b42c9a25 --- /dev/null +++ b/src/calibre/utils/ipc/worker.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, cPickle +from multiprocessing.connection import Client +from threading import Thread +from queue import Queue +from contextlib import closing + +PARALLEL_FUNCS = { + 'lrfviewer' : + ('calibre.gui2.lrf_renderer.main', 'main', None), + + 'ebook-viewer' : + ('calibre.gui2.viewer.main', 'main', None), + + 'render_pages' : + ('calibre.ebooks.comic.input', 'render_pages', 'notification'), + + 'gui_convert' : + ('calibre.gui2.convert.gui_conversion', 'gui_convert', 'notification'), +} + +class Progress(Thread): + + def __init__(self, conn): + self.daemon = True + Thread.__init__(self) + self.conn = conn + self.queue = Queue() + + def __call__(self, percent, msg=''): + self.queue.put((percent, msg)) + + def run(self): + while True: + x = self.queue.get() + if x is None: + break + try: + self.conn.send(x) + except: + break + + + +def get_func(name): + module, func, notification = PARALLEL_FUNCS[name] + module = __import__(module, fromlist=[1]) + func = getattr(module, func) + return func, notification + +def main(): + address = cPickle.loads(os.environ['CALIBRE_WORKER_ADDRESS']) + key = os.environ['CALIBRE_WORKER_KEY'] + with closing(Client(address, authkey=key)) as conn: + name, args, kwargs = conn.recv() + func, notification = get_func(name) + notifier = Progress(conn) + if notification: + kwargs[notification] = notifier + notifier.start() + + func(*args, **kwargs) + + notifier.queue.put(None) + + return 0 + + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py new file mode 100644 index 0000000000..6c2846cee2 --- /dev/null +++ b/src/calibre/utils/podofo/__init__.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre.constants import plugins, preferred_encoding +from calibre.ebooks.metadata import MetaInformation, string_to_authors, \ + authors_to_string + +podofo, podofo_err = plugins['podofo'] + +class Unavailable(Exception): pass + +def get_metadata(stream): + if not podofo: + raise Unavailable(podofo_err) + raw = stream.read() + stream.seek(0) + p = podofo.PdfMemDocument() + p.Load(raw, len(raw)) + info = p.GetInfo() + title = info.GetTitle().decode('utf-8').strip() + if not title: + title = getattr(stream, 'name', _('Unknown')) + title = os.path.splitext(os.path.basename(title))[0] + author = info.GetAuthor().decode('utf-8').strip() + authors = string_to_authors(author) if author else [_('Unknown')] + mi = MetaInformation(title, authors) + creator = info.GetCreator().decode('utf-8').strip() + if creator: + mi.book_producer = creator + return mi + +def prep(val): + if not val: + return u'' + if not isinstance(val, unicode): + val = val.decode(preferred_encoding, 'replace') + return val.strip() + +def set_metadata(stream, mi): + if not podofo: + raise Unavailable(podofo_err) + raw = stream.read() + p = podofo.PdfMemDocument() + p.Load(raw, len(raw)) + info = p.GetInfo() + title = prep(mi.title) + touched = False + if title: + info.SetTitle(title) + touched = True + + author = prep(authors_to_string(mi.authors)) + if author: + print repr(author) + info.SetAuthor(author) + touched = True + + bkp = prep(mi.book_producer) + if bkp: + info.SetCreator(bkp) + touched = True + + if touched: + p.SetInfo(info) + from calibre.ptempfile import TemporaryFile + with TemporaryFile('_pdf_set_metadata.pdf') as f: + p.Write(f) + raw = open(f, 'rb').read() + stream.seek(0) + stream.truncate() + stream.write(raw) + stream.flush() + stream.seek(0) + +if __name__ == '__main__': + f = '/tmp/t.pdf' + import StringIO + stream = StringIO.StringIO(open(f).read()) + mi = get_metadata(open(f)) + print + print 'Original metadata:' + print mi + mi.title = 'Test title' + mi.authors = ['Test author', 'author2'] + mi.book_producer = 'calibre' + set_metadata(stream, mi) + open('/tmp/x.pdf', 'wb').write(stream.getvalue()) + print + print 'New pdf written to /tmp/x.pdf' + + diff --git a/src/calibre/utils/podofo/podofo.sip b/src/calibre/utils/podofo/podofo.sip new file mode 100644 index 0000000000..0964e3a7e6 --- /dev/null +++ b/src/calibre/utils/podofo/podofo.sip @@ -0,0 +1,128 @@ +%Module podofo 0 + +%MappedType PdfString +{ +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; +%End +%ConvertFromTypeCode + if (sipCpp -> IsValid()) { + std::string raw = sipCpp->GetStringUtf8(); + return PyString_FromStringAndSize(raw.c_str(), raw.length()); + } else return PyString_FromString(""); +%End +%ConvertToTypeCode + if (sipIsErr == NULL) { + if (sipIsErr == NULL) + return (PyUnicode_Check(sipPy) || PyString_Check(sipPy)); + } + if (sipPy == Py_None) { + *sipCppPtr = NULL; + return 0; + } + if (PyString_Check(sipPy)) { + *sipCppPtr = new PdfString((pdf_utf8 *)PyString_AS_STRING(sipPy)); + return sipGetState(sipTransferObj); + } + if (PyUnicode_Check(sipPy)) { + Py_UNICODE* u = PyUnicode_AS_UNICODE(sipPy); + PyObject *u8 = PyUnicode_EncodeUTF8(u, PyUnicode_GET_SIZE(sipPy), "replace"); + pdf_utf8 *s8 = (pdf_utf8 *)PyString_AS_STRING(u8); + *sipCppPtr = new PdfString(s8); + return sipGetState(sipTransferObj); + } + *sipCppPtr = (PdfString *)sipForceConvertTo_PdfString(sipPy,sipIsErr); + return 1; +%End +}; + +class PdfObject { +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; +%End + public: + PdfObject(); + +}; + +class PdfInfo { +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; +%End + public: + PdfInfo(PdfObject *); + + PdfString GetAuthor() const; + PdfString GetSubject() const; + PdfString GetTitle() const; + PdfString GetKeywords() const; + PdfString GetCreator() const; + PdfString GetProducer() const; + + void SetAuthor(PdfString &); + void SetSubject(PdfString &); + void SetTitle(PdfString &); + void SetKeywords(PdfString &); + void SetCreator(PdfString &); + void SetProducer(PdfString &); + +}; + +class PdfOutputDevice { +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; +%End + public: + PdfOutputDevice(char *, long); + unsigned long GetLength(); + unsigned long Tell(); + void Flush(); +}; + + +class PdfMemDocument { +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +using namespace PoDoFo; +%End + public: + PdfMemDocument(); + + void Load(const char *filename); + void Load(const char *buffer, long size); + void Write(const char *filename); + PdfInfo *GetInfo() const; + + protected: + void SetInfo(PdfInfo * /TransferThis/); + + private: + PdfMemDocument(PdfMemDocument &); + +}; + + +%Exception PoDoFo::PdfError /PyName=PdfError/ +{ +%TypeHeaderCode +#define USING_SHARED_PODOFO +#include +%End +%RaiseCode + const char *detail = sipExceptionRef.what(); + + SIP_BLOCK_THREADS + PyErr_SetString(sipException_PoDoFo_PdfError, detail); + SIP_UNBLOCK_THREADS +%End +}; +