mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Improve the memory() function on linux and explicitly break a few more cycles
This commit is contained in:
Kovid Goyal
commit
de246e1cd0
@ -25,7 +25,7 @@ class Base(object):
|
||||
def __init__(self, db, col_id, parent=None):
|
||||
self.db, self.col_id = db, col_id
|
||||
self.col_metadata = db.custom_column_num_map[col_id]
|
||||
self.initial_val = None
|
||||
self.initial_val = self.widgets = None
|
||||
self.setup_ui(parent)
|
||||
|
||||
def initialize(self, book_id):
|
||||
@ -54,6 +54,9 @@ class Base(object):
|
||||
def normalize_ui_val(self, val):
|
||||
return val
|
||||
|
||||
def break_cycles(self):
|
||||
self.db = self.widgets = self.initial_val = None
|
||||
|
||||
class Bool(Base):
|
||||
|
||||
def setup_ui(self, parent):
|
||||
|
||||
@ -126,6 +126,9 @@ class TitleEdit(EnLineEdit):
|
||||
|
||||
return property(fget=fget, fset=fset)
|
||||
|
||||
def break_cycles(self):
|
||||
self.dialog = None
|
||||
|
||||
class TitleSortEdit(TitleEdit):
|
||||
|
||||
TITLE_ATTR = 'title_sort'
|
||||
@ -151,6 +154,7 @@ class TitleSortEdit(TitleEdit):
|
||||
self.title_edit.textChanged.connect(self.update_state)
|
||||
self.textChanged.connect(self.update_state)
|
||||
|
||||
self.autogen_button = autogen_button
|
||||
autogen_button.clicked.connect(self.auto_generate)
|
||||
self.update_state()
|
||||
|
||||
@ -169,6 +173,9 @@ class TitleSortEdit(TitleEdit):
|
||||
|
||||
def auto_generate(self, *args):
|
||||
self.current_val = title_sort(self.title_edit.current_val)
|
||||
self.title_edit.textChanged.disconnect()
|
||||
self.textChanged.disconnect()
|
||||
self.autogen_button.clicked.disconnect()
|
||||
|
||||
# }}}
|
||||
|
||||
@ -186,6 +193,7 @@ class AuthorsEdit(MultiCompleteComboBox):
|
||||
self.setWhatsThis(self.TOOLTIP)
|
||||
self.setEditable(True)
|
||||
self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
|
||||
self.manage_authors_signal = manage_authors
|
||||
manage_authors.triggered.connect(self.manage_authors)
|
||||
|
||||
def manage_authors(self):
|
||||
@ -270,6 +278,10 @@ class AuthorsEdit(MultiCompleteComboBox):
|
||||
|
||||
return property(fget=fget, fset=fset)
|
||||
|
||||
def break_cycles(self):
|
||||
self.db = self.dialog = None
|
||||
self.manage_authors_signal.triggered.disconnect()
|
||||
|
||||
class AuthorSortEdit(EnLineEdit):
|
||||
|
||||
TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
|
||||
@ -298,6 +310,10 @@ class AuthorSortEdit(EnLineEdit):
|
||||
self.authors_edit.editTextChanged.connect(self.update_state_and_val)
|
||||
self.textChanged.connect(self.update_state)
|
||||
|
||||
self.autogen_button = autogen_button
|
||||
self.copy_a_to_as_action = copy_a_to_as_action
|
||||
self.copy_as_to_a_action = copy_as_to_a_action
|
||||
|
||||
autogen_button.clicked.connect(self.auto_generate)
|
||||
copy_a_to_as_action.triggered.connect(self.auto_generate)
|
||||
copy_as_to_a_action.triggered.connect(self.copy_to_authors)
|
||||
@ -369,6 +385,15 @@ class AuthorSortEdit(EnLineEdit):
|
||||
db.set_author_sort(id_, aus, notify=False, commit=False)
|
||||
return True
|
||||
|
||||
def break_cycles(self):
|
||||
self.db = None
|
||||
self.authors_edit.editTextChanged.disconnect()
|
||||
self.textChanged.disconnect()
|
||||
self.autogen_button.clicked.disconnect()
|
||||
self.copy_a_to_as_action.triggered.disconnect()
|
||||
self.copy_as_to_a_action.triggered.disconnect()
|
||||
self.authors_edit = None
|
||||
|
||||
# }}}
|
||||
|
||||
# Series {{{
|
||||
@ -428,6 +453,10 @@ class SeriesEdit(MultiCompleteComboBox):
|
||||
commit=True, allow_case_change=True)
|
||||
return True
|
||||
|
||||
def break_cycles(self):
|
||||
self.dialog = None
|
||||
|
||||
|
||||
class SeriesIndexEdit(QDoubleSpinBox):
|
||||
|
||||
TOOLTIP = ''
|
||||
@ -489,6 +518,11 @@ class SeriesIndexEdit(QDoubleSpinBox):
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
def break_cycles(self):
|
||||
self.series_edit.currentIndexChanged.disconnect()
|
||||
self.series_edit.editTextChanged.disconnect()
|
||||
self.series_edit.lineEdit().editingFinished.disconnect()
|
||||
self.db = self.series_edit = self.dialog = None
|
||||
|
||||
# }}}
|
||||
|
||||
@ -700,6 +734,8 @@ class FormatsManager(QWidget): # {{{
|
||||
if old != prefs['read_file_metadata']:
|
||||
prefs['read_file_metadata'] = old
|
||||
|
||||
def break_cycles(self):
|
||||
self.dialog = None
|
||||
# }}}
|
||||
|
||||
class Cover(ImageView): # {{{
|
||||
@ -861,6 +897,10 @@ class Cover(ImageView): # {{{
|
||||
db.remove_cover(id_, notify=False, commit=False)
|
||||
return True
|
||||
|
||||
def break_cycles(self):
|
||||
self.cover_changed.disconnect()
|
||||
self.dialog = self._cdata = self.current_val = self.original_val = None
|
||||
|
||||
# }}}
|
||||
|
||||
class CommentsEdit(Editor): # {{{
|
||||
|
||||
@ -481,6 +481,13 @@ class MetadataSingleDialogBase(ResizableDialog):
|
||||
x = getattr(self, b, None)
|
||||
if x is not None:
|
||||
disconnect(x.clicked)
|
||||
for widget in self.basic_metadata_widgets:
|
||||
bc = getattr(widget, 'break_cycles', None)
|
||||
if bc is not None and callable(bc):
|
||||
bc()
|
||||
for widget in getattr(self, 'custom_metadata_widgets', []):
|
||||
widget.break_cycles()
|
||||
|
||||
# }}}
|
||||
|
||||
class Splitter(QSplitter):
|
||||
|
||||
@ -627,7 +627,8 @@ class TagTreeItem(object): # {{{
|
||||
except:
|
||||
pass
|
||||
self.parent = self.icon_state_map = self.bold_font = self.tag = \
|
||||
self.icon = self.children = None
|
||||
self.icon = self.children = self.tooltip = \
|
||||
self.py_name = self.id_set = self.category_key = None
|
||||
|
||||
def __str__(self):
|
||||
if self.type == self.ROOT:
|
||||
@ -1121,7 +1122,7 @@ class TagsModel(QAbstractItemModel): # {{{
|
||||
self.search_restriction = s
|
||||
|
||||
def get_node_tree(self, sort):
|
||||
old_row_map = self.row_map[:]
|
||||
old_row_map_len = len(self.row_map)
|
||||
self.row_map = []
|
||||
self.categories = {}
|
||||
|
||||
@ -1176,7 +1177,7 @@ class TagsModel(QAbstractItemModel): # {{{
|
||||
self.row_map.append(category)
|
||||
self.categories[category] = tb_categories[category]['name']
|
||||
|
||||
if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
|
||||
if old_row_map_len != 0 and old_row_map_len != len(self.row_map):
|
||||
# A category has been added or removed. We must force a rebuild of
|
||||
# the model
|
||||
return None
|
||||
@ -1367,6 +1368,9 @@ class TagsModel(QAbstractItemModel): # {{{
|
||||
self.beginRemoveRows(self.createIndex(category.row(), 0, category),
|
||||
start, len(child_map)-1)
|
||||
category.children = ctags
|
||||
for i in range(start, len(child_map)):
|
||||
child_map[i].break_cycles()
|
||||
child_map = None
|
||||
self.endRemoveRows()
|
||||
else:
|
||||
state_map = {}
|
||||
|
||||
@ -8,61 +8,157 @@ __docformat__ = 'restructuredtext en'
|
||||
'''
|
||||
Measure memory usage of the current process.
|
||||
|
||||
The key function is memory() which returns the current memory usage in bytes.
|
||||
The key function is memory() which returns the current memory usage in MB.
|
||||
You can pass a number to memory and it will be subtracted from the returned
|
||||
value.
|
||||
'''
|
||||
|
||||
import gc, os
|
||||
import gc, os, re
|
||||
|
||||
from calibre.constants import iswindows, islinux
|
||||
|
||||
if islinux:
|
||||
## {{{ http://code.activestate.com/recipes/286222/ (r1)
|
||||
# Taken, with thanks, from:
|
||||
# http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
|
||||
|
||||
_proc_status = '/proc/%d/status' % os.getpid()
|
||||
def permute(args):
|
||||
ret = []
|
||||
if args:
|
||||
first = args.pop(0)
|
||||
for y in permute(args):
|
||||
for x in first:
|
||||
ret.append(x + y)
|
||||
else:
|
||||
ret.append('')
|
||||
return ret
|
||||
|
||||
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
|
||||
'KB': 1024.0, 'MB': 1024.0*1024.0}
|
||||
def parsed_groups(match, *types):
|
||||
groups = match.groups()
|
||||
assert len(groups) == len(types)
|
||||
return tuple([type(group) for group, type in zip(groups, types)])
|
||||
|
||||
def _VmB(VmKey):
|
||||
'''Private.
|
||||
'''
|
||||
global _proc_status, _scale
|
||||
# get pseudo file /proc/<pid>/status
|
||||
try:
|
||||
t = open(_proc_status)
|
||||
v = t.read()
|
||||
t.close()
|
||||
except:
|
||||
return 0.0 # non-Linux?
|
||||
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
|
||||
i = v.index(VmKey)
|
||||
v = v[i:].split(None, 3) # whitespace
|
||||
if len(v) < 3:
|
||||
return 0.0 # invalid format?
|
||||
# convert Vm value to bytes
|
||||
return float(v[1]) * _scale[v[2]]
|
||||
class VMA(dict):
|
||||
def __init__(self, *args):
|
||||
(self.start, self.end, self.perms, self.offset,
|
||||
self.major, self.minor, self.inode, self.filename) = args
|
||||
|
||||
def parse_smaps(pid):
|
||||
with open('/proc/%s/smaps'%pid, 'r') as maps:
|
||||
hex = lambda s: int(s, 16)
|
||||
|
||||
ret = []
|
||||
header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
|
||||
r'(..):(..) (\d+) *(.*)$')
|
||||
detail = re.compile(r'^(.*): +(\d+) kB')
|
||||
for line in maps:
|
||||
m = header.match(line)
|
||||
if m:
|
||||
vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
|
||||
ret.append(vma)
|
||||
else:
|
||||
m = detail.match(line)
|
||||
if m:
|
||||
k, v = parsed_groups(m, str, int)
|
||||
assert k not in vma
|
||||
vma[k] = v
|
||||
else:
|
||||
print 'unparseable line:', line
|
||||
return ret
|
||||
|
||||
perms = permute(['r-', 'w-', 'x-', 'ps'])
|
||||
|
||||
def make_summary_dicts(vmas):
|
||||
mapped = {}
|
||||
anon = {}
|
||||
for d in mapped, anon:
|
||||
# per-perm
|
||||
for k in perms:
|
||||
d[k] = {}
|
||||
d[k]['Size'] = 0
|
||||
for y in 'Shared', 'Private':
|
||||
d[k][y] = {}
|
||||
for z in 'Clean', 'Dirty':
|
||||
d[k][y][z] = 0
|
||||
# totals
|
||||
for y in 'Shared', 'Private':
|
||||
d[y] = {}
|
||||
for z in 'Clean', 'Dirty':
|
||||
d[y][z] = 0
|
||||
|
||||
for vma in vmas:
|
||||
if vma.major == '00' and vma.minor == '00':
|
||||
d = anon
|
||||
else:
|
||||
d = mapped
|
||||
for y in 'Shared', 'Private':
|
||||
for z in 'Clean', 'Dirty':
|
||||
d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
|
||||
d[y][z] += vma.get(y + '_' + z, 0)
|
||||
d[vma.perms]['Size'] += vma.get('Size', 0)
|
||||
return mapped, anon
|
||||
|
||||
def values(d, args):
|
||||
if args:
|
||||
ret = ()
|
||||
first = args[0]
|
||||
for k in first:
|
||||
ret += values(d[k], args[1:])
|
||||
return ret
|
||||
else:
|
||||
return (d,)
|
||||
|
||||
def print_summary(dicts_and_titles):
|
||||
def desc(title, perms):
|
||||
ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
|
||||
('Anonymous', 'rwxp'): 'Writable code (stack)',
|
||||
('Mapped', 'r-xp'): 'Code',
|
||||
('Mapped', 'rwxp'): 'Writable code (jump tables)',
|
||||
('Mapped', 'r--p'): 'Read-only data',
|
||||
('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
|
||||
if ret:
|
||||
return ' -- ' + ret
|
||||
else:
|
||||
return ''
|
||||
|
||||
for d, title in dicts_and_titles:
|
||||
print title, 'memory:'
|
||||
print ' Shared Private'
|
||||
print ' Clean Dirty Clean Dirty'
|
||||
for k in perms:
|
||||
if d[k]['Size']:
|
||||
print (' %s %7d %7d %7d %7d%s'
|
||||
% ((k,)
|
||||
+ values(d[k], (('Shared', 'Private'),
|
||||
('Clean', 'Dirty')))
|
||||
+ (desc(title, k),)))
|
||||
print (' total %7d %7d %7d %7d'
|
||||
% values(d, (('Shared', 'Private'),
|
||||
('Clean', 'Dirty'))))
|
||||
|
||||
print ' ' + '-' * 40
|
||||
print (' total %7d %7d %7d %7d'
|
||||
% tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
|
||||
('Clean', 'Dirty')))
|
||||
for d, title in dicts_and_titles]))))
|
||||
|
||||
def print_stats(pid=None):
|
||||
if pid is None:
|
||||
pid = os.getpid()
|
||||
vmas = parse_smaps(pid)
|
||||
mapped, anon = make_summary_dicts(vmas)
|
||||
print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
|
||||
|
||||
def linux_memory(since=0.0):
|
||||
'''Return memory usage in bytes.
|
||||
'''
|
||||
return _VmB('VmSize:') - since
|
||||
vmas = parse_smaps(os.getpid())
|
||||
mapped, anon = make_summary_dicts(vmas)
|
||||
dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
|
||||
totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
|
||||
('Clean', 'Dirty')))
|
||||
for d, title in dicts_and_titles])))
|
||||
return (totals[-1]/1024.) - since
|
||||
|
||||
|
||||
def resident(since=0.0):
|
||||
'''Return resident memory usage in bytes.
|
||||
'''
|
||||
return _VmB('VmRSS:') - since
|
||||
|
||||
|
||||
def stacksize(since=0.0):
|
||||
'''Return stack size in bytes.
|
||||
'''
|
||||
return _VmB('VmStk:') - since
|
||||
## end of http://code.activestate.com/recipes/286222/ }}}
|
||||
memory = linux_memory
|
||||
|
||||
elif iswindows:
|
||||
import win32process
|
||||
import win32con
|
||||
@ -95,7 +191,7 @@ elif iswindows:
|
||||
|
||||
def win_memory(since=0.0):
|
||||
info = meminfo(get_handle(os.getpid()))
|
||||
return info['WorkingSetSize'] - since
|
||||
return (info['WorkingSetSize']/1024.**2) - since
|
||||
|
||||
memory = win_memory
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user