mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-11-04 03:27:00 -05:00
Start work on writing many-many fields
This commit is contained in:
Kovid Goyal
parent
5a490baca1
commit
1fbbbcede9
@ -53,7 +53,7 @@ def multiple_text(sep, x):
|
||||
if isinstance(x, unicode):
|
||||
x = x.split(sep)
|
||||
x = (y.strip() for y in x if y.strip())
|
||||
return (' '.join(y.split()) for y in x if y)
|
||||
return tuple(' '.join(y.split()) for y in x if y)
|
||||
|
||||
def adapt_datetime(x):
|
||||
if isinstance(x, (unicode, bytes)):
|
||||
@ -178,6 +178,30 @@ def safe_lower(x):
|
||||
except (TypeError, ValueError, KeyError, AttributeError):
|
||||
return x
|
||||
|
||||
def get_db_id(val, db, m, table, kmap, rid_map, allow_case_changes,
|
||||
case_changes, val_map, sql_val_map=lambda x:x):
|
||||
''' Get the db id for the value val. If val does not exist in the db it is
|
||||
inserted into the db. '''
|
||||
kval = kmap(val)
|
||||
item_id = rid_map.get(kval, None)
|
||||
if item_id is None:
|
||||
db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
|
||||
m['table'], m['column']), (sql_val_map(val),))
|
||||
item_id = rid_map[kval] = db.conn.last_insert_rowid()
|
||||
table.id_map[item_id] = val
|
||||
table.col_book_map[item_id] = set()
|
||||
elif allow_case_changes and val != table.id_map[item_id]:
|
||||
case_changes[item_id] = val
|
||||
val_map[val] = item_id
|
||||
|
||||
def change_case(case_changes, dirtied, db, table, m, sql_val_map=lambda x:x):
|
||||
db.conn.executemany(
|
||||
'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']),
|
||||
tuple((sql_val_map(val), item_id) for item_id, val in case_changes.iteritems()))
|
||||
for item_id, val in case_changes.iteritems():
|
||||
table.id_map[item_id] = val
|
||||
dirtied.update(table.col_book_map[item_id])
|
||||
|
||||
def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
||||
dirtied = set()
|
||||
m = field.metadata
|
||||
@ -185,101 +209,54 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
||||
dt = m['datatype']
|
||||
is_custom_series = dt == 'series' and table.name.startswith('#')
|
||||
|
||||
# Map values to their canonical form for later comparison
|
||||
# Map values to db ids, including any new values
|
||||
kmap = safe_lower if dt in {'text', 'series'} else lambda x:x
|
||||
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
|
||||
val_map = {None:None}
|
||||
case_changes = {}
|
||||
for val in book_id_val_map.itervalues():
|
||||
if val is not None:
|
||||
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
|
||||
case_changes, val_map)
|
||||
|
||||
if case_changes:
|
||||
change_case(case_changes, dirtied, db, table, m)
|
||||
|
||||
book_id_item_id_map = {k:val_map[v] for k, v in book_id_val_map.iteritems()}
|
||||
|
||||
# Ignore those items whose value is the same as the current value
|
||||
no_changes = {k:nval for k, nval in book_id_val_map.iteritems() if
|
||||
kmap(nval) == kmap(field.for_book(k, default_value=None))}
|
||||
for book_id in no_changes:
|
||||
del book_id_val_map[book_id]
|
||||
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
|
||||
if v != table.book_col_map.get(k, None)}
|
||||
dirtied |= set(book_id_item_id_map)
|
||||
|
||||
# If we are allowed case changes check that none of the ignored items are
|
||||
# case changes. If they are, update the item's case in the db.
|
||||
if allow_case_change:
|
||||
for book_id, nval in no_changes.iteritems():
|
||||
if nval is not None and nval != field.for_book(
|
||||
book_id, default_value=None):
|
||||
# Change of case
|
||||
item_id = table.book_col_map[book_id]
|
||||
db.conn.execute('UPDATE %s SET %s=? WHERE id=?'%(
|
||||
m['table'], m['column']), (nval, item_id))
|
||||
table.id_map[item_id] = nval
|
||||
dirtied |= table.col_book_map[item_id]
|
||||
|
||||
deleted = {k:v for k, v in book_id_val_map.iteritems() if v is None}
|
||||
updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None}
|
||||
link_table = table.link_table
|
||||
# Update the book->col and col->book maps
|
||||
deleted = set()
|
||||
updated = {}
|
||||
for book_id, item_id in book_id_item_id_map.iteritems():
|
||||
old_item_id = table.book_col_map.get(book_id, None)
|
||||
if old_item_id is not None:
|
||||
table.col_book_map[old_item_id].discard(book_id)
|
||||
if item_id is None:
|
||||
table.book_col_map.pop(book_id, None)
|
||||
deleted.add(book_id)
|
||||
else:
|
||||
table.book_col_map[book_id] = item_id
|
||||
table.col_book_map[item_id].add(book_id)
|
||||
updated[book_id] = item_id
|
||||
|
||||
# Update the db link table
|
||||
if deleted:
|
||||
db.conn.executemany('DELETE FROM %s WHERE book=?'%link_table,
|
||||
tuple((book_id,) for book_id in deleted))
|
||||
for book_id in deleted:
|
||||
item_id = table.book_col_map.pop(book_id, None)
|
||||
if item_id is not None:
|
||||
table.col_book_map[item_id].discard(book_id)
|
||||
dirtied |= set(deleted)
|
||||
|
||||
db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
|
||||
tuple((k,) for k in deleted))
|
||||
if updated:
|
||||
rid_map = {kmap(v):k for k, v in table.id_map.iteritems()}
|
||||
book_id_item_id_map = {k:rid_map.get(kmap(v), None) for k, v in
|
||||
book_id_val_map.iteritems()}
|
||||
|
||||
# items that dont yet exist
|
||||
new_items = {k:v for k, v in updated.iteritems() if
|
||||
book_id_item_id_map[k] is None}
|
||||
# items that already exist
|
||||
changed_items = {k:book_id_item_id_map[k] for k in updated if
|
||||
book_id_item_id_map[k] is not None}
|
||||
def sql_update(imap):
|
||||
sql = (
|
||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
|
||||
if is_custom_series else
|
||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
||||
)
|
||||
db.conn.executemany(sql.format(link_table, m['link_column']),
|
||||
tuple((book_id, book_id, item_id) for book_id, item_id in
|
||||
imap.iteritems()))
|
||||
|
||||
if new_items:
|
||||
item_ids = {}
|
||||
val_map = {}
|
||||
for val in set(new_items.itervalues()):
|
||||
lval = kmap(val)
|
||||
if lval in val_map:
|
||||
item_id = val_map[lval]
|
||||
else:
|
||||
db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
|
||||
m['table'], m['column']), (val,))
|
||||
item_id = val_map[lval] = db.conn.last_insert_rowid()
|
||||
item_ids[val] = item_id
|
||||
table.id_map[item_id] = val
|
||||
imap = {}
|
||||
for book_id, val in new_items.iteritems():
|
||||
item_id = item_ids[val]
|
||||
old_item_id = table.book_col_map.get(book_id, None)
|
||||
if old_item_id is not None:
|
||||
table.col_book_map[old_item_id].discard(book_id)
|
||||
if item_id not in table.col_book_map:
|
||||
table.col_book_map[item_id] = set()
|
||||
table.col_book_map[item_id].add(book_id)
|
||||
table.book_col_map[book_id] = imap[book_id] = item_id
|
||||
sql_update(imap)
|
||||
dirtied |= set(imap)
|
||||
|
||||
if changed_items:
|
||||
imap = {}
|
||||
sql_update(changed_items)
|
||||
for book_id, item_id in changed_items.iteritems():
|
||||
old_item_id = table.book_col_map.get(book_id, None)
|
||||
if old_item_id != item_id:
|
||||
table.book_col_map[book_id] = item_id
|
||||
table.col_book_map[item_id].add(book_id)
|
||||
if old_item_id is not None:
|
||||
table.col_book_map[old_item_id].discard(book_id)
|
||||
imap[book_id] = item_id
|
||||
sql_update(imap)
|
||||
dirtied |= set(imap)
|
||||
sql = (
|
||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
|
||||
if is_custom_series else
|
||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
||||
)
|
||||
db.conn.executemany(sql.format(table.link_table, m['link_column']),
|
||||
tuple((book_id, book_id, item_id) for book_id, item_id in
|
||||
updated.iteritems()))
|
||||
|
||||
# Remove no longer used items
|
||||
remove = {item_id for item_id in table.id_map if not
|
||||
@ -294,6 +271,83 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
||||
return dirtied
|
||||
# }}}
|
||||
|
||||
# Many-Many fields {{{
|
||||
def many_many(book_id_val_map, db, field, allow_case_change, *args):
|
||||
dirtied = set()
|
||||
m = field.metadata
|
||||
table = field.table
|
||||
dt = m['datatype']
|
||||
is_authors = field.name == 'authors'
|
||||
|
||||
# Map values to db ids, including any new values
|
||||
kmap = safe_lower if dt == 'text' else lambda x:x
|
||||
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
|
||||
sql_val_map = lambda x:x.replace(',', '|') if is_authors else lambda x:x
|
||||
val_map = {}
|
||||
case_changes = {}
|
||||
for vals in book_id_val_map.itervalues():
|
||||
for val in vals:
|
||||
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
|
||||
case_changes, val_map, sql_val_map=sql_val_map)
|
||||
|
||||
if case_changes:
|
||||
change_case(case_changes, dirtied, db, table, m,
|
||||
sql_val_map=sql_val_map)
|
||||
|
||||
book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
|
||||
for k, vals in book_id_val_map.iteritems()}
|
||||
|
||||
# Ignore those items whose value is the same as the current value
|
||||
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
|
||||
if v != table.book_col_map.get(k, None)}
|
||||
dirtied |= set(book_id_item_id_map)
|
||||
|
||||
# Update the book->col and col->book maps
|
||||
deleted = set()
|
||||
updated = {}
|
||||
for book_id, item_ids in book_id_item_id_map.iteritems():
|
||||
old_item_ids = table.book_col_map.get(book_id, None)
|
||||
if old_item_ids:
|
||||
for old_item_id in old_item_ids:
|
||||
table.col_book_map[old_item_id].discard(book_id)
|
||||
if item_ids:
|
||||
table.book_col_map[book_id] = item_ids
|
||||
for item_id in item_ids:
|
||||
table.col_book_map[item_id].add(book_id)
|
||||
updated[book_id] = item_ids
|
||||
else:
|
||||
table.book_col_map.pop(book_id, None)
|
||||
deleted.add(book_id)
|
||||
|
||||
# Update the db link table
|
||||
if deleted:
|
||||
db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
|
||||
tuple((k,) for k in deleted))
|
||||
if updated:
|
||||
vals = tuple(
|
||||
(book_id, book_id, val) for book_id, vals in updated.iteritems()
|
||||
for val in vals
|
||||
)
|
||||
sql = (
|
||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
||||
)
|
||||
db.conn.executemany(sql.format(table.link_table, m['link_column']), vals)
|
||||
|
||||
# Remove no longer used items
|
||||
remove = {item_id for item_id in table.id_map if not
|
||||
table.col_book_map.get(item_id, False)}
|
||||
if remove:
|
||||
db.conn.executemany('DELETE FROM %s WHERE id=?'%m['table'],
|
||||
tuple((item_id,) for item_id in remove))
|
||||
for item_id in remove:
|
||||
del table.id_map[item_id]
|
||||
table.col_book_map.pop(item_id, None)
|
||||
|
||||
return dirtied
|
||||
|
||||
|
||||
# }}}
|
||||
|
||||
def dummy(book_id_val_map, *args):
|
||||
return set()
|
||||
|
||||
@ -311,9 +365,7 @@ class Writer(object):
|
||||
elif self.name[0] == '#' and self.name.endswith('_index'):
|
||||
self.set_books_func = custom_series_index
|
||||
elif field.is_many_many:
|
||||
# TODO: Implement this
|
||||
pass
|
||||
# TODO: Remember to change commas to | when writing authors to sqlite
|
||||
self.set_books_func = many_many
|
||||
elif field.is_many:
|
||||
self.set_books_func = (self.set_books_for_enum if dt ==
|
||||
'enumeration' else many_one)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user