From be0f3b096a019d35ff35f390472cb254da8c1e23 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 13:29:07 -0700 Subject: [PATCH 1/6] Fix a regression in 0.7.33 that broke updating covers in ebook files when saving to disk. Fixes #7886 (Possible bug in setting new cover) --- src/calibre/library/save_to_disk.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 62a2e28e27..7090a2afa8 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -253,6 +253,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, if not os.path.exists(dirpath): raise + ocover = mi.cover if opts.save_cover and cover and os.access(cover, os.R_OK): with open(base_path+'.jpg', 'wb') as f: with open(cover, 'rb') as s: @@ -266,6 +267,8 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, with open(base_path+'.opf', 'wb') as f: f.write(opf) + mi.cover = ocover + written = False for fmt in formats: global plugboard_save_to_disk_value, plugboard_any_format_value From 3a1fd7af5665ddcfb8523ae460359703e7779ac8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 15:06:34 -0700 Subject: [PATCH 2/6] Bulk metadata edit: Add options to delete cover/generate default cover. Fixes #7885 (Bulk remove covers) --- src/calibre/gui2/dialogs/metadata_bulk.py | 26 +++++++++++++++-- src/calibre/gui2/dialogs/metadata_bulk.ui | 35 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index a640c50fb8..e0f1f83c73 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -102,7 +102,7 @@ class MyBlockingBusy(QDialog): remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, \ do_autonumber, do_remove_format, remove_format, do_swap_ta, \ do_remove_conv, do_auto_author, series, do_series_restart, \ - series_start_value, do_title_case, clear_series = self.args + series_start_value, do_title_case, cover_action, clear_series = self.args # first loop: do author and title. These will commit at the end of each @@ -129,6 +129,23 @@ class MyBlockingBusy(QDialog): self.db.set_title(id, titlecase(title), notify=False) if au: self.db.set_authors(id, string_to_authors(au), notify=False) + if cover_action == 'remove': + self.db.remove_cover(id) + elif cover_action == 'generate': + from calibre.ebooks import calibre_cover + from calibre.ebooks.metadata import fmt_sidx + from calibre.gui2 import config + mi = self.db.get_metadata(id, index_is_id=True) + series_string = None + if mi.series: + series_string = _('Book %s of %s')%( + fmt_sidx(mi.series_index, + use_roman=config['use_roman_numerals_for_series_number']), + mi.series) + + cdata = calibre_cover(mi.title, mi.format_field('authors')[-1], + series_string=series_string) + self.db.set_cover(id, cdata) elif self.current_phase == 2: # All of these just affect the DB, so we can tolerate a total rollback if do_auto_author: @@ -678,11 +695,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): do_remove_conv = self.remove_conversion_settings.isChecked() do_auto_author = self.auto_author_sort.isChecked() do_title_case = self.change_title_to_title_case.isChecked() + cover_action = None + if self.cover_remove.isChecked(): + cover_action = 'remove' + elif self.cover_generate.isChecked(): + cover_action = 'generate' args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, do_autonumber, do_remove_format, remove_format, do_swap_ta, do_remove_conv, do_auto_author, series, do_series_restart, - series_start_value, do_title_case, clear_series) + series_start_value, do_title_case, cover_action, clear_series) bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') %len(self.ids), args, self.db, self.ids, diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 344bde0fa0..cd644f88ba 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -381,7 +381,7 @@ Future conversion of these books will use the default settings. - + Qt::Vertical @@ -394,6 +394,39 @@ Future conversion of these books will use the default settings. + + + + Change &cover + + + + + + &No change + + + true + + + + + + + &Remove cover + + + + + + + &Generate default cover + + + + + + From 97536e397fc138b581e04ebb1f26c8e41efdf38f Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 13 Dec 2010 18:38:37 -0500 Subject: [PATCH 3/6] FB2 Output: add blank line after paragraphs when insert-blank-line option used. Use instead of CSS because not many readers support CSS in FB2 files. --- src/calibre/ebooks/fb2/fb2ml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 51bfaa7293..89c12db103 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -73,6 +73,10 @@ class FB2MLizer(object): text = re.sub(r'(?miu)

\s*

', '', text) text = re.sub(r'(?miu)\s+

', '

', text) text = re.sub(r'(?miu)

', '

\n\n

', text) + + if self.opts.insert_blank_line: + text = re.sub(r'(?miu)

', '

', text) + return text def fb2_header(self): From 3df1780251d802d7de47bbf0484a932e0bf945bf Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 13 Dec 2010 18:57:05 -0500 Subject: [PATCH 4/6] FB2 Output: Add support for some 2.1 style tags. --- src/calibre/ebooks/fb2/fb2ml.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 89c12db103..5efc360f1f 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -297,6 +297,18 @@ class FB2MLizer(object): s_out, s_tags = self.handle_simple_tag('emphasis', tag_stack+tags) fb2_out += s_out tags += s_tags + elif tag in ('del', 'strike'): + s_out, s_tags = self.handle_simple_tag('strikethrough', tag_stack+tags) + fb2_out += s_out + tags += s_tags + elif tag == 'sub': + s_out, s_tags = self.handle_simple_tag('sub', tag_stack+tags) + fb2_out += s_out + tags += s_tags + elif tag == 'sup': + s_out, s_tags = self.handle_simple_tag('sup', tag_stack+tags) + fb2_out += s_out + tags += s_tags # Processes style information. if style['font-style'] == 'italic': @@ -307,6 +319,10 @@ class FB2MLizer(object): s_out, s_tags = self.handle_simple_tag('strong', tag_stack+tags) fb2_out += s_out tags += s_tags + elif style['text-decoration'] == 'line-through': + s_out, s_tags = self.handle_simple_tag('strikethrough', tag_stack+tags) + fb2_out += s_out + tags += s_tags # Process element text. if hasattr(elem_tree, 'text') and elem_tree.text: From acbbc16bf3af727e317efd87d9d4b146950747ea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 17:07:00 -0700 Subject: [PATCH 5/6] Infrastructure to detect mem leaks in the edit metadata dialog --- src/calibre/gui2/actions/edit_metadata.py | 8 ++- src/calibre/gui2/dialogs/metadata_single.py | 69 ++++++++++++++++++++- src/calibre/utils/mem.py | 55 ++++++++++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 src/calibre/utils/mem.py diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 60a943ccb9..11949632e9 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -154,15 +154,17 @@ class EditMetadataAction(InterfaceAction): d.view_format.connect(lambda fmt:self.gui.iactions['View'].view_format(row_list[current_row], fmt)) - if d.exec_() != d.Accepted: - d.view_format.disconnect() + ret = d.exec_() + d.break_cycles() + if ret != d.Accepted: break - d.view_format.disconnect() + changed.add(d.id) if d.row_delta == 0: break current_row += d.row_delta + if changed: self.gui.library_view.model().refresh_ids(list(changed)) current = self.gui.library_view.currentIndex() diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index d9bb1c2a33..9e3a8c7eda 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -293,7 +293,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): finally: self.fetch_cover_button.setEnabled(True) self.unsetCursor() - self.pi.stop() + if self.pi is not None: + self.pi.stop() # }}} @@ -442,7 +443,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): ResizableDialog.__init__(self, window) self.cover_fetcher = None self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) - self.cancel_all = False base = unicode(self.author_sort.toolTip()) self.ok_aus_tooltip = '

' + textwrap.fill(base+'

'+ _(' The green color indicates that the current ' @@ -573,7 +573,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): QObject.connect(self.series, SIGNAL('editTextChanged(QString)'), self.enable_series_index) self.series.lineEdit().editingFinished.connect(self.increment_series_index) - self.show() pm = QPixmap() if cover: pm.loadFromData(cover) @@ -593,6 +592,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.original_author = unicode(self.authors.text()).strip() self.original_title = unicode(self.title.text()).strip() + self.show() + def create_custom_column_editors(self): w = self.central_widget.widget(1) layout = w.layout() @@ -907,3 +908,65 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): dynamic.set('metasingle_window_geometry', bytes(self.saveGeometry())) dynamic.set('metasingle_splitter_state', bytes(self.splitter.saveState())) + + def break_cycles(self): + try: + self.view_format.disconnect() + except: + pass # Fails if view format was never connected + self.view_format = None + self.db = None + self.pi = None + self.cover_data = self.cpixmap = None + +if __name__ == '__main__': + from calibre.library import db + from PyQt4.Qt import QApplication + from calibre.utils.mem import memory + import gc + + + app = QApplication([]) + db = db() + + # Initialize all Qt Objects once + d = MetadataSingleDialog(None, 4, db) + d.break_cycles() + d.reject() + del d + + for i in range(3): + gc.collect() + before = memory() + + gc.collect() + d = MetadataSingleDialog(None, 4, db) + d.break_cycles() + d.reject() + del d + + for i in range(3): + gc.collect() + print 'Used memory:', memory(before)/1024.**2, 'MB' + gc.collect() + + ''' + nmap, omap = {}, {} + for x in objects: + omap[id(x)] = x + for x in nobjects: + nmap[id(x)] = x + + new_ids = set(nmap.keys()) - set(omap.keys()) + print "New ids:", len(new_ids) + for i in new_ids: + o = nmap[i] + if o is objects: + continue + print repr(o)[:1050] + refs = gc.get_referrers(o) + for r in refs: + if r is objects or r is nobjects: + continue + print '\t', r + ''' diff --git a/src/calibre/utils/mem.py b/src/calibre/utils/mem.py new file mode 100644 index 0000000000..f48aec34c6 --- /dev/null +++ b/src/calibre/utils/mem.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +## {{{ http://code.activestate.com/recipes/286222/ (r1) +import os + +_proc_status = '/proc/%d/status' % os.getpid() + +_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, + 'KB': 1024.0, 'MB': 1024.0*1024.0} + +def _VmB(VmKey): + '''Private. + ''' + global _proc_status, _scale + # get pseudo file /proc//status + try: + t = open(_proc_status) + v = t.read() + t.close() + except: + return 0.0 # non-Linux? + # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index(VmKey) + v = v[i:].split(None, 3) # whitespace + if len(v) < 3: + return 0.0 # invalid format? + # convert Vm value to bytes + return float(v[1]) * _scale[v[2]] + + +def memory(since=0.0): + '''Return memory usage in bytes. + ''' + return _VmB('VmSize:') - since + + +def resident(since=0.0): + '''Return resident memory usage in bytes. + ''' + return _VmB('VmRSS:') - since + + +def stacksize(since=0.0): + '''Return stack size in bytes. + ''' + return _VmB('VmStk:') - since +## end of http://code.activestate.com/recipes/286222/ }}} + + + From f71f60ab0cb84364a876dc7ee3950474e115d338 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 19:02:14 -0700 Subject: [PATCH 6/6] Edit metadata dialog: Fix memory leak caused by Next/Previous buttons --- src/calibre/gui2/dialogs/metadata_single.py | 47 +++++++-------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 9e3a8c7eda..4a9bb784c8 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -910,14 +910,18 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): bytes(self.splitter.saveState())) def break_cycles(self): - try: - self.view_format.disconnect() - except: - pass # Fails if view format was never connected - self.view_format = None - self.db = None - self.pi = None - self.cover_data = self.cpixmap = None + # Break any reference cycles that could prevent python + # from garbage collecting this dialog + def disconnect(signal): + try: + signal.disconnect() + except: + pass # Fails if view format was never connected + disconnect(self.view_format) + for b in ('next_button', 'prev_button'): + x = getattr(self, b, None) + if x is not None: + disconnect(x.clicked) if __name__ == '__main__': from calibre.library import db @@ -935,38 +939,17 @@ if __name__ == '__main__': d.reject() del d - for i in range(3): + for i in range(5): gc.collect() before = memory() - gc.collect() d = MetadataSingleDialog(None, 4, db) - d.break_cycles() d.reject() + d.break_cycles() del d - for i in range(3): + for i in range(5): gc.collect() print 'Used memory:', memory(before)/1024.**2, 'MB' - gc.collect() - ''' - nmap, omap = {}, {} - for x in objects: - omap[id(x)] = x - for x in nobjects: - nmap[id(x)] = x - new_ids = set(nmap.keys()) - set(omap.keys()) - print "New ids:", len(new_ids) - for i in new_ids: - o = nmap[i] - if o is objects: - continue - print repr(o)[:1050] - refs = gc.get_referrers(o) - for r in refs: - if r is objects or r is nobjects: - continue - print '\t', r - '''