mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
Start work on writing many-many fields
This commit is contained in:
Kovid Goyal
parent
5a490baca1
commit
1fbbbcede9
@ -53,7 +53,7 @@ def multiple_text(sep, x):
|
|||||||
if isinstance(x, unicode):
|
if isinstance(x, unicode):
|
||||||
x = x.split(sep)
|
x = x.split(sep)
|
||||||
x = (y.strip() for y in x if y.strip())
|
x = (y.strip() for y in x if y.strip())
|
||||||
return (' '.join(y.split()) for y in x if y)
|
return tuple(' '.join(y.split()) for y in x if y)
|
||||||
|
|
||||||
def adapt_datetime(x):
|
def adapt_datetime(x):
|
||||||
if isinstance(x, (unicode, bytes)):
|
if isinstance(x, (unicode, bytes)):
|
||||||
@ -178,6 +178,30 @@ def safe_lower(x):
|
|||||||
except (TypeError, ValueError, KeyError, AttributeError):
|
except (TypeError, ValueError, KeyError, AttributeError):
|
||||||
return x
|
return x
|
||||||
|
|
||||||
|
def get_db_id(val, db, m, table, kmap, rid_map, allow_case_changes,
|
||||||
|
case_changes, val_map, sql_val_map=lambda x:x):
|
||||||
|
''' Get the db id for the value val. If val does not exist in the db it is
|
||||||
|
inserted into the db. '''
|
||||||
|
kval = kmap(val)
|
||||||
|
item_id = rid_map.get(kval, None)
|
||||||
|
if item_id is None:
|
||||||
|
db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
|
||||||
|
m['table'], m['column']), (sql_val_map(val),))
|
||||||
|
item_id = rid_map[kval] = db.conn.last_insert_rowid()
|
||||||
|
table.id_map[item_id] = val
|
||||||
|
table.col_book_map[item_id] = set()
|
||||||
|
elif allow_case_changes and val != table.id_map[item_id]:
|
||||||
|
case_changes[item_id] = val
|
||||||
|
val_map[val] = item_id
|
||||||
|
|
||||||
|
def change_case(case_changes, dirtied, db, table, m, sql_val_map=lambda x:x):
|
||||||
|
db.conn.executemany(
|
||||||
|
'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']),
|
||||||
|
tuple((sql_val_map(val), item_id) for item_id, val in case_changes.iteritems()))
|
||||||
|
for item_id, val in case_changes.iteritems():
|
||||||
|
table.id_map[item_id] = val
|
||||||
|
dirtied.update(table.col_book_map[item_id])
|
||||||
|
|
||||||
def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
||||||
dirtied = set()
|
dirtied = set()
|
||||||
m = field.metadata
|
m = field.metadata
|
||||||
@ -185,101 +209,54 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
|||||||
dt = m['datatype']
|
dt = m['datatype']
|
||||||
is_custom_series = dt == 'series' and table.name.startswith('#')
|
is_custom_series = dt == 'series' and table.name.startswith('#')
|
||||||
|
|
||||||
# Map values to their canonical form for later comparison
|
# Map values to db ids, including any new values
|
||||||
kmap = safe_lower if dt in {'text', 'series'} else lambda x:x
|
kmap = safe_lower if dt in {'text', 'series'} else lambda x:x
|
||||||
|
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
|
||||||
|
val_map = {None:None}
|
||||||
|
case_changes = {}
|
||||||
|
for val in book_id_val_map.itervalues():
|
||||||
|
if val is not None:
|
||||||
|
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
|
||||||
|
case_changes, val_map)
|
||||||
|
|
||||||
|
if case_changes:
|
||||||
|
change_case(case_changes, dirtied, db, table, m)
|
||||||
|
|
||||||
|
book_id_item_id_map = {k:val_map[v] for k, v in book_id_val_map.iteritems()}
|
||||||
|
|
||||||
# Ignore those items whose value is the same as the current value
|
# Ignore those items whose value is the same as the current value
|
||||||
no_changes = {k:nval for k, nval in book_id_val_map.iteritems() if
|
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
|
||||||
kmap(nval) == kmap(field.for_book(k, default_value=None))}
|
if v != table.book_col_map.get(k, None)}
|
||||||
for book_id in no_changes:
|
dirtied |= set(book_id_item_id_map)
|
||||||
del book_id_val_map[book_id]
|
|
||||||
|
|
||||||
# If we are allowed case changes check that none of the ignored items are
|
# Update the book->col and col->book maps
|
||||||
# case changes. If they are, update the item's case in the db.
|
deleted = set()
|
||||||
if allow_case_change:
|
updated = {}
|
||||||
for book_id, nval in no_changes.iteritems():
|
for book_id, item_id in book_id_item_id_map.iteritems():
|
||||||
if nval is not None and nval != field.for_book(
|
old_item_id = table.book_col_map.get(book_id, None)
|
||||||
book_id, default_value=None):
|
if old_item_id is not None:
|
||||||
# Change of case
|
table.col_book_map[old_item_id].discard(book_id)
|
||||||
item_id = table.book_col_map[book_id]
|
if item_id is None:
|
||||||
db.conn.execute('UPDATE %s SET %s=? WHERE id=?'%(
|
table.book_col_map.pop(book_id, None)
|
||||||
m['table'], m['column']), (nval, item_id))
|
deleted.add(book_id)
|
||||||
table.id_map[item_id] = nval
|
else:
|
||||||
dirtied |= table.col_book_map[item_id]
|
table.book_col_map[book_id] = item_id
|
||||||
|
table.col_book_map[item_id].add(book_id)
|
||||||
deleted = {k:v for k, v in book_id_val_map.iteritems() if v is None}
|
updated[book_id] = item_id
|
||||||
updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None}
|
|
||||||
link_table = table.link_table
|
|
||||||
|
|
||||||
|
# Update the db link table
|
||||||
if deleted:
|
if deleted:
|
||||||
db.conn.executemany('DELETE FROM %s WHERE book=?'%link_table,
|
db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
|
||||||
tuple((book_id,) for book_id in deleted))
|
tuple((k,) for k in deleted))
|
||||||
for book_id in deleted:
|
|
||||||
item_id = table.book_col_map.pop(book_id, None)
|
|
||||||
if item_id is not None:
|
|
||||||
table.col_book_map[item_id].discard(book_id)
|
|
||||||
dirtied |= set(deleted)
|
|
||||||
|
|
||||||
if updated:
|
if updated:
|
||||||
rid_map = {kmap(v):k for k, v in table.id_map.iteritems()}
|
sql = (
|
||||||
book_id_item_id_map = {k:rid_map.get(kmap(v), None) for k, v in
|
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
|
||||||
book_id_val_map.iteritems()}
|
if is_custom_series else
|
||||||
|
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
||||||
# items that dont yet exist
|
)
|
||||||
new_items = {k:v for k, v in updated.iteritems() if
|
db.conn.executemany(sql.format(table.link_table, m['link_column']),
|
||||||
book_id_item_id_map[k] is None}
|
tuple((book_id, book_id, item_id) for book_id, item_id in
|
||||||
# items that already exist
|
updated.iteritems()))
|
||||||
changed_items = {k:book_id_item_id_map[k] for k in updated if
|
|
||||||
book_id_item_id_map[k] is not None}
|
|
||||||
def sql_update(imap):
|
|
||||||
sql = (
|
|
||||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
|
|
||||||
if is_custom_series else
|
|
||||||
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
|
||||||
)
|
|
||||||
db.conn.executemany(sql.format(link_table, m['link_column']),
|
|
||||||
tuple((book_id, book_id, item_id) for book_id, item_id in
|
|
||||||
imap.iteritems()))
|
|
||||||
|
|
||||||
if new_items:
|
|
||||||
item_ids = {}
|
|
||||||
val_map = {}
|
|
||||||
for val in set(new_items.itervalues()):
|
|
||||||
lval = kmap(val)
|
|
||||||
if lval in val_map:
|
|
||||||
item_id = val_map[lval]
|
|
||||||
else:
|
|
||||||
db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
|
|
||||||
m['table'], m['column']), (val,))
|
|
||||||
item_id = val_map[lval] = db.conn.last_insert_rowid()
|
|
||||||
item_ids[val] = item_id
|
|
||||||
table.id_map[item_id] = val
|
|
||||||
imap = {}
|
|
||||||
for book_id, val in new_items.iteritems():
|
|
||||||
item_id = item_ids[val]
|
|
||||||
old_item_id = table.book_col_map.get(book_id, None)
|
|
||||||
if old_item_id is not None:
|
|
||||||
table.col_book_map[old_item_id].discard(book_id)
|
|
||||||
if item_id not in table.col_book_map:
|
|
||||||
table.col_book_map[item_id] = set()
|
|
||||||
table.col_book_map[item_id].add(book_id)
|
|
||||||
table.book_col_map[book_id] = imap[book_id] = item_id
|
|
||||||
sql_update(imap)
|
|
||||||
dirtied |= set(imap)
|
|
||||||
|
|
||||||
if changed_items:
|
|
||||||
imap = {}
|
|
||||||
sql_update(changed_items)
|
|
||||||
for book_id, item_id in changed_items.iteritems():
|
|
||||||
old_item_id = table.book_col_map.get(book_id, None)
|
|
||||||
if old_item_id != item_id:
|
|
||||||
table.book_col_map[book_id] = item_id
|
|
||||||
table.col_book_map[item_id].add(book_id)
|
|
||||||
if old_item_id is not None:
|
|
||||||
table.col_book_map[old_item_id].discard(book_id)
|
|
||||||
imap[book_id] = item_id
|
|
||||||
sql_update(imap)
|
|
||||||
dirtied |= set(imap)
|
|
||||||
|
|
||||||
# Remove no longer used items
|
# Remove no longer used items
|
||||||
remove = {item_id for item_id in table.id_map if not
|
remove = {item_id for item_id in table.id_map if not
|
||||||
@ -294,6 +271,83 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
|
|||||||
return dirtied
|
return dirtied
|
||||||
# }}}
|
# }}}
|
||||||
|
|
||||||
|
# Many-Many fields {{{
|
||||||
|
def many_many(book_id_val_map, db, field, allow_case_change, *args):
|
||||||
|
dirtied = set()
|
||||||
|
m = field.metadata
|
||||||
|
table = field.table
|
||||||
|
dt = m['datatype']
|
||||||
|
is_authors = field.name == 'authors'
|
||||||
|
|
||||||
|
# Map values to db ids, including any new values
|
||||||
|
kmap = safe_lower if dt == 'text' else lambda x:x
|
||||||
|
rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
|
||||||
|
sql_val_map = lambda x:x.replace(',', '|') if is_authors else lambda x:x
|
||||||
|
val_map = {}
|
||||||
|
case_changes = {}
|
||||||
|
for vals in book_id_val_map.itervalues():
|
||||||
|
for val in vals:
|
||||||
|
get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
|
||||||
|
case_changes, val_map, sql_val_map=sql_val_map)
|
||||||
|
|
||||||
|
if case_changes:
|
||||||
|
change_case(case_changes, dirtied, db, table, m,
|
||||||
|
sql_val_map=sql_val_map)
|
||||||
|
|
||||||
|
book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
|
||||||
|
for k, vals in book_id_val_map.iteritems()}
|
||||||
|
|
||||||
|
# Ignore those items whose value is the same as the current value
|
||||||
|
book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
|
||||||
|
if v != table.book_col_map.get(k, None)}
|
||||||
|
dirtied |= set(book_id_item_id_map)
|
||||||
|
|
||||||
|
# Update the book->col and col->book maps
|
||||||
|
deleted = set()
|
||||||
|
updated = {}
|
||||||
|
for book_id, item_ids in book_id_item_id_map.iteritems():
|
||||||
|
old_item_ids = table.book_col_map.get(book_id, None)
|
||||||
|
if old_item_ids:
|
||||||
|
for old_item_id in old_item_ids:
|
||||||
|
table.col_book_map[old_item_id].discard(book_id)
|
||||||
|
if item_ids:
|
||||||
|
table.book_col_map[book_id] = item_ids
|
||||||
|
for item_id in item_ids:
|
||||||
|
table.col_book_map[item_id].add(book_id)
|
||||||
|
updated[book_id] = item_ids
|
||||||
|
else:
|
||||||
|
table.book_col_map.pop(book_id, None)
|
||||||
|
deleted.add(book_id)
|
||||||
|
|
||||||
|
# Update the db link table
|
||||||
|
if deleted:
|
||||||
|
db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
|
||||||
|
tuple((k,) for k in deleted))
|
||||||
|
if updated:
|
||||||
|
vals = tuple(
|
||||||
|
(book_id, book_id, val) for book_id, vals in updated.iteritems()
|
||||||
|
for val in vals
|
||||||
|
)
|
||||||
|
sql = (
|
||||||
|
'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
|
||||||
|
)
|
||||||
|
db.conn.executemany(sql.format(table.link_table, m['link_column']), vals)
|
||||||
|
|
||||||
|
# Remove no longer used items
|
||||||
|
remove = {item_id for item_id in table.id_map if not
|
||||||
|
table.col_book_map.get(item_id, False)}
|
||||||
|
if remove:
|
||||||
|
db.conn.executemany('DELETE FROM %s WHERE id=?'%m['table'],
|
||||||
|
tuple((item_id,) for item_id in remove))
|
||||||
|
for item_id in remove:
|
||||||
|
del table.id_map[item_id]
|
||||||
|
table.col_book_map.pop(item_id, None)
|
||||||
|
|
||||||
|
return dirtied
|
||||||
|
|
||||||
|
|
||||||
|
# }}}
|
||||||
|
|
||||||
def dummy(book_id_val_map, *args):
|
def dummy(book_id_val_map, *args):
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
@ -311,9 +365,7 @@ class Writer(object):
|
|||||||
elif self.name[0] == '#' and self.name.endswith('_index'):
|
elif self.name[0] == '#' and self.name.endswith('_index'):
|
||||||
self.set_books_func = custom_series_index
|
self.set_books_func = custom_series_index
|
||||||
elif field.is_many_many:
|
elif field.is_many_many:
|
||||||
# TODO: Implement this
|
self.set_books_func = many_many
|
||||||
pass
|
|
||||||
# TODO: Remember to change commas to | when writing authors to sqlite
|
|
||||||
elif field.is_many:
|
elif field.is_many:
|
||||||
self.set_books_func = (self.set_books_for_enum if dt ==
|
self.set_books_func = (self.set_books_for_enum if dt ==
|
||||||
'enumeration' else many_one)
|
'enumeration' else many_one)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user