Improve the memory() function on linux and explicitly break a few more cycles

src/calibre/gui2/custom_column_widgets.py

@@ -25,7 +25,7 @@ class Base(object):
     def __init__(self, db, col_id, parent=None):
         self.db, self.col_id = db, col_id
         self.col_metadata = db.custom_column_num_map[col_id]
-        self.initial_val = None
+        self.initial_val = self.widgets = None
         self.setup_ui(parent)
 
     def initialize(self, book_id):
@@ -54,6 +54,9 @@ class Base(object):
     def normalize_ui_val(self, val):
         return val
 
+    def break_cycles(self):
+        self.db = self.widgets = self.initial_val = None
+
 class Bool(Base):
 
     def setup_ui(self, parent):

src/calibre/gui2/metadata/basic_widgets.py

@@ -126,6 +126,9 @@ class TitleEdit(EnLineEdit):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.dialog = None
+
 class TitleSortEdit(TitleEdit):
 
     TITLE_ATTR = 'title_sort'
@@ -151,6 +154,7 @@ class TitleSortEdit(TitleEdit):
         self.title_edit.textChanged.connect(self.update_state)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
         autogen_button.clicked.connect(self.auto_generate)
         self.update_state()
 
@@ -169,6 +173,9 @@ class TitleSortEdit(TitleEdit):
 
     def auto_generate(self, *args):
         self.current_val = title_sort(self.title_edit.current_val)
+        self.title_edit.textChanged.disconnect()
+        self.textChanged.disconnect()
+        self.autogen_button.clicked.disconnect()
 
 # }}}
 
@@ -186,6 +193,7 @@ class AuthorsEdit(MultiCompleteComboBox):
         self.setWhatsThis(self.TOOLTIP)
         self.setEditable(True)
         self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
+        self.manage_authors_signal = manage_authors
         manage_authors.triggered.connect(self.manage_authors)
 
     def manage_authors(self):
@@ -270,6 +278,10 @@ class AuthorsEdit(MultiCompleteComboBox):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.db = self.dialog = None
+        self.manage_authors_signal.triggered.disconnect()
+
 class AuthorSortEdit(EnLineEdit):
 
     TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
@@ -298,6 +310,10 @@ class AuthorSortEdit(EnLineEdit):
         self.authors_edit.editTextChanged.connect(self.update_state_and_val)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
+        self.copy_a_to_as_action = copy_a_to_as_action
+        self.copy_as_to_a_action = copy_as_to_a_action
+
         autogen_button.clicked.connect(self.auto_generate)
         copy_a_to_as_action.triggered.connect(self.auto_generate)
         copy_as_to_a_action.triggered.connect(self.copy_to_authors)
@@ -369,6 +385,15 @@ class AuthorSortEdit(EnLineEdit):
         db.set_author_sort(id_, aus, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        self.db = None
+        self.authors_edit.editTextChanged.disconnect()
+        self.textChanged.disconnect()
+        self.autogen_button.clicked.disconnect()
+        self.copy_a_to_as_action.triggered.disconnect()
+        self.copy_as_to_a_action.triggered.disconnect()
+        self.authors_edit = None
+
 # }}}
 
 # Series {{{
@@ -428,6 +453,10 @@ class SeriesEdit(MultiCompleteComboBox):
                                             commit=True, allow_case_change=True)
         return True
 
+    def break_cycles(self):
+        self.dialog = None
+
+
 class SeriesIndexEdit(QDoubleSpinBox):
 
     TOOLTIP = ''
@@ -489,6 +518,11 @@ class SeriesIndexEdit(QDoubleSpinBox):
                 import traceback
                 traceback.print_exc()
 
+    def break_cycles(self):
+        self.series_edit.currentIndexChanged.disconnect()
+        self.series_edit.editTextChanged.disconnect()
+        self.series_edit.lineEdit().editingFinished.disconnect()
+        self.db = self.series_edit = self.dialog = None
 
 # }}}
 
@@ -700,6 +734,8 @@ class FormatsManager(QWidget): # {{{
             if old != prefs['read_file_metadata']:
                 prefs['read_file_metadata'] = old
 
+    def break_cycles(self):
+        self.dialog = None
 # }}}
 
 class Cover(ImageView): # {{{
@@ -861,6 +897,10 @@ class Cover(ImageView): # {{{
                 db.remove_cover(id_, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        self.cover_changed.disconnect()
+        self.dialog = self._cdata = self.current_val = self.original_val = None
+
 # }}}
 
 class CommentsEdit(Editor): # {{{

src/calibre/gui2/metadata/single.py

@@ -481,6 +481,13 @@ class MetadataSingleDialogBase(ResizableDialog):
             x = getattr(self, b, None)
             if x is not None:
                 disconnect(x.clicked)
+        for widget in self.basic_metadata_widgets:
+            bc = getattr(widget, 'break_cycles', None)
+            if bc is not None and callable(bc):
+                bc()
+        for widget in getattr(self, 'custom_metadata_widgets', []):
+            widget.break_cycles()
+
     # }}}
 
 class Splitter(QSplitter):

src/calibre/gui2/tag_view.py

@@ -627,7 +627,8 @@ class TagTreeItem(object): # {{{
             except:
                 pass
         self.parent = self.icon_state_map = self.bold_font = self.tag = \
-                self.icon = self.children = None
+                      self.icon = self.children = self.tooltip = \
+                      self.py_name = self.id_set = self.category_key = None
 
     def __str__(self):
         if self.type == self.ROOT:
@@ -1121,7 +1122,7 @@ class TagsModel(QAbstractItemModel): # {{{
         self.search_restriction = s
 
     def get_node_tree(self, sort):
-        old_row_map = self.row_map[:]
+        old_row_map_len = len(self.row_map)
         self.row_map = []
         self.categories = {}
 
@@ -1176,7 +1177,7 @@ class TagsModel(QAbstractItemModel): # {{{
                 self.row_map.append(category)
                 self.categories[category] = tb_categories[category]['name']
 
-        if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
+        if old_row_map_len != 0 and old_row_map_len != len(self.row_map):
             # A category has been added or removed. We must force a rebuild of
             # the model
             return None
@@ -1367,6 +1368,9 @@ class TagsModel(QAbstractItemModel): # {{{
                 self.beginRemoveRows(self.createIndex(category.row(), 0, category),
                                      start, len(child_map)-1)
                 category.children = ctags
+                for i in range(start, len(child_map)):
+                    child_map[i].break_cycles()
+                child_map = None
                 self.endRemoveRows()
             else:
                 state_map = {}

src/calibre/utils/mem.py

@@ -8,61 +8,157 @@ __docformat__ = 'restructuredtext en'
 '''
 Measure memory usage of the current process.
 
-The key function is memory() which returns the current memory usage in bytes.
+The key function is memory() which returns the current memory usage in MB.
 You can pass a number to memory and it will be subtracted from the returned
 value.
 '''
 
-import gc, os
+import gc, os, re
 
 from calibre.constants import iswindows, islinux
 
 if islinux:
-    ## {{{ http://code.activestate.com/recipes/286222/ (r1)
+    # Taken, with thanks, from:
+    # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
 
-    _proc_status = '/proc/%d/status' % os.getpid()
+    def permute(args):
+        ret = []
+        if args:
+            first = args.pop(0)
+            for y in permute(args):
+                for x in first:
+                    ret.append(x + y)
+        else:
+            ret.append('')
+        return ret
 
-    _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
+    def parsed_groups(match, *types):
-            'KB': 1024.0, 'MB': 1024.0*1024.0}
+        groups = match.groups()
+        assert len(groups) == len(types)
+        return tuple([type(group) for group, type in zip(groups, types)])
 
-    def _VmB(VmKey):
+    class VMA(dict):
-        '''Private.
+        def __init__(self, *args):
-        '''
+            (self.start, self.end, self.perms, self.offset,
-        global _proc_status, _scale
+            self.major, self.minor, self.inode, self.filename) = args
-        # get pseudo file  /proc/<pid>/status
-        try:
-            t = open(_proc_status)
-            v = t.read()
-            t.close()
-        except:
-            return 0.0  # non-Linux?
-        # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
-        i = v.index(VmKey)
-        v = v[i:].split(None, 3)  # whitespace
-        if len(v) < 3:
-            return 0.0  # invalid format?
-        # convert Vm value to bytes
-        return float(v[1]) * _scale[v[2]]
 
+    def parse_smaps(pid):
+        with open('/proc/%s/smaps'%pid, 'r') as maps:
+            hex = lambda s: int(s, 16)
+
+            ret = []
+            header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
+                                r'(..):(..) (\d+) *(.*)$')
+            detail = re.compile(r'^(.*): +(\d+) kB')
+            for line in maps:
+                m = header.match(line)
+                if m:
+                    vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
+                    ret.append(vma)
+                else:
+                    m = detail.match(line)
+                    if m:
+                        k, v = parsed_groups(m, str, int)
+                        assert k not in vma
+                        vma[k] = v
+                    else:
+                        print 'unparseable line:', line
+            return ret
+
+    perms = permute(['r-', 'w-', 'x-', 'ps'])
+
+    def make_summary_dicts(vmas):
+        mapped = {}
+        anon = {}
+        for d in mapped, anon:
+            # per-perm
+            for k in perms:
+                d[k] = {}
+                d[k]['Size'] = 0
+                for y in 'Shared', 'Private':
+                    d[k][y] = {}
+                    for z in 'Clean', 'Dirty':
+                        d[k][y][z] = 0
+            # totals
+            for y in 'Shared', 'Private':
+                d[y] = {}
+                for z in 'Clean', 'Dirty':
+                    d[y][z] = 0
+
+        for vma in vmas:
+            if vma.major == '00' and vma.minor == '00':
+                d = anon
+            else:
+                d = mapped
+            for y in 'Shared', 'Private':
+                for z in 'Clean', 'Dirty':
+                    d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
+                    d[y][z] += vma.get(y + '_' + z, 0)
+            d[vma.perms]['Size'] += vma.get('Size', 0)
+        return mapped, anon
+
+    def values(d, args):
+        if args:
+            ret = ()
+            first = args[0]
+            for k in first:
+                ret += values(d[k], args[1:])
+            return ret
+        else:
+            return (d,)
+
+    def print_summary(dicts_and_titles):
+        def desc(title, perms):
+            ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
+                ('Anonymous', 'rwxp'): 'Writable code (stack)',
+                ('Mapped', 'r-xp'): 'Code',
+                ('Mapped', 'rwxp'): 'Writable code (jump tables)',
+                ('Mapped', 'r--p'): 'Read-only data',
+                ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
+            if ret:
+                return '  -- ' + ret
+            else:
+                return ''
+
+        for d, title in dicts_and_titles:
+            print title, 'memory:'
+            print '               Shared            Private'
+            print '           Clean    Dirty    Clean    Dirty'
+            for k in perms:
+                if d[k]['Size']:
+                    print ('    %s %7d  %7d  %7d  %7d%s'
+                        % ((k,)
+                            + values(d[k], (('Shared', 'Private'),
+                                            ('Clean', 'Dirty')))
+                            + (desc(title, k),)))
+            print ('   total %7d  %7d  %7d  %7d'
+                % values(d, (('Shared', 'Private'),
+                                ('Clean', 'Dirty'))))
+
+        print '   ' + '-' * 40
+        print ('   total %7d  %7d  %7d  %7d'
+            % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles]))))
+
+    def print_stats(pid=None):
+        if pid is None:
+            pid = os.getpid()
+        vmas = parse_smaps(pid)
+        mapped, anon = make_summary_dicts(vmas)
+        print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
 
     def linux_memory(since=0.0):
-        '''Return memory usage in bytes.
+        vmas = parse_smaps(os.getpid())
-        '''
+        mapped, anon = make_summary_dicts(vmas)
-        return _VmB('VmSize:') - since
+        dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
+        totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles])))
+        return (totals[-1]/1024.) - since
 
-
-    def resident(since=0.0):
-        '''Return resident memory usage in bytes.
-        '''
-        return _VmB('VmRSS:') - since
-
-
-    def stacksize(since=0.0):
-        '''Return stack size in bytes.
-        '''
-        return _VmB('VmStk:') - since
-    ## end of http://code.activestate.com/recipes/286222/ }}}
     memory = linux_memory
+
 elif iswindows:
     import win32process
     import win32con
@@ -95,7 +191,7 @@ elif iswindows:
 
     def win_memory(since=0.0):
         info = meminfo(get_handle(os.getpid()))
-        return info['WorkingSetSize'] - since
+        return (info['WorkingSetSize']/1024.**2) - since
 
     memory = win_memory