diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py index 7f2ba5baee..cb74848f4d 100644 --- a/src/calibre/db/write.py +++ b/src/calibre/db/write.py @@ -53,7 +53,7 @@ def multiple_text(sep, x): if isinstance(x, unicode): x = x.split(sep) x = (y.strip() for y in x if y.strip()) - return (' '.join(y.split()) for y in x if y) + return tuple(' '.join(y.split()) for y in x if y) def adapt_datetime(x): if isinstance(x, (unicode, bytes)): @@ -178,6 +178,30 @@ def safe_lower(x): except (TypeError, ValueError, KeyError, AttributeError): return x +def get_db_id(val, db, m, table, kmap, rid_map, allow_case_changes, + case_changes, val_map, sql_val_map=lambda x:x): + ''' Get the db id for the value val. If val does not exist in the db it is + inserted into the db. ''' + kval = kmap(val) + item_id = rid_map.get(kval, None) + if item_id is None: + db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%( + m['table'], m['column']), (sql_val_map(val),)) + item_id = rid_map[kval] = db.conn.last_insert_rowid() + table.id_map[item_id] = val + table.col_book_map[item_id] = set() + elif allow_case_changes and val != table.id_map[item_id]: + case_changes[item_id] = val + val_map[val] = item_id + +def change_case(case_changes, dirtied, db, table, m, sql_val_map=lambda x:x): + db.conn.executemany( + 'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']), + tuple((sql_val_map(val), item_id) for item_id, val in case_changes.iteritems())) + for item_id, val in case_changes.iteritems(): + table.id_map[item_id] = val + dirtied.update(table.col_book_map[item_id]) + def many_one(book_id_val_map, db, field, allow_case_change, *args): dirtied = set() m = field.metadata @@ -185,101 +209,54 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args): dt = m['datatype'] is_custom_series = dt == 'series' and table.name.startswith('#') - # Map values to their canonical form for later comparison + # Map values to db ids, including any new values kmap = safe_lower if dt in {'text', 'series'} else lambda x:x + rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()} + val_map = {None:None} + case_changes = {} + for val in book_id_val_map.itervalues(): + if val is not None: + get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, + case_changes, val_map) + + if case_changes: + change_case(case_changes, dirtied, db, table, m) + + book_id_item_id_map = {k:val_map[v] for k, v in book_id_val_map.iteritems()} # Ignore those items whose value is the same as the current value - no_changes = {k:nval for k, nval in book_id_val_map.iteritems() if - kmap(nval) == kmap(field.for_book(k, default_value=None))} - for book_id in no_changes: - del book_id_val_map[book_id] + book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems() + if v != table.book_col_map.get(k, None)} + dirtied |= set(book_id_item_id_map) - # If we are allowed case changes check that none of the ignored items are - # case changes. If they are, update the item's case in the db. - if allow_case_change: - for book_id, nval in no_changes.iteritems(): - if nval is not None and nval != field.for_book( - book_id, default_value=None): - # Change of case - item_id = table.book_col_map[book_id] - db.conn.execute('UPDATE %s SET %s=? WHERE id=?'%( - m['table'], m['column']), (nval, item_id)) - table.id_map[item_id] = nval - dirtied |= table.col_book_map[item_id] - - deleted = {k:v for k, v in book_id_val_map.iteritems() if v is None} - updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None} - link_table = table.link_table + # Update the book->col and col->book maps + deleted = set() + updated = {} + for book_id, item_id in book_id_item_id_map.iteritems(): + old_item_id = table.book_col_map.get(book_id, None) + if old_item_id is not None: + table.col_book_map[old_item_id].discard(book_id) + if item_id is None: + table.book_col_map.pop(book_id, None) + deleted.add(book_id) + else: + table.book_col_map[book_id] = item_id + table.col_book_map[item_id].add(book_id) + updated[book_id] = item_id + # Update the db link table if deleted: - db.conn.executemany('DELETE FROM %s WHERE book=?'%link_table, - tuple((book_id,) for book_id in deleted)) - for book_id in deleted: - item_id = table.book_col_map.pop(book_id, None) - if item_id is not None: - table.col_book_map[item_id].discard(book_id) - dirtied |= set(deleted) - + db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table, + tuple((k,) for k in deleted)) if updated: - rid_map = {kmap(v):k for k, v in table.id_map.iteritems()} - book_id_item_id_map = {k:rid_map.get(kmap(v), None) for k, v in - book_id_val_map.iteritems()} - - # items that dont yet exist - new_items = {k:v for k, v in updated.iteritems() if - book_id_item_id_map[k] is None} - # items that already exist - changed_items = {k:book_id_item_id_map[k] for k in updated if - book_id_item_id_map[k] is not None} - def sql_update(imap): - sql = ( - 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)' - if is_custom_series else - 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)' - ) - db.conn.executemany(sql.format(link_table, m['link_column']), - tuple((book_id, book_id, item_id) for book_id, item_id in - imap.iteritems())) - - if new_items: - item_ids = {} - val_map = {} - for val in set(new_items.itervalues()): - lval = kmap(val) - if lval in val_map: - item_id = val_map[lval] - else: - db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%( - m['table'], m['column']), (val,)) - item_id = val_map[lval] = db.conn.last_insert_rowid() - item_ids[val] = item_id - table.id_map[item_id] = val - imap = {} - for book_id, val in new_items.iteritems(): - item_id = item_ids[val] - old_item_id = table.book_col_map.get(book_id, None) - if old_item_id is not None: - table.col_book_map[old_item_id].discard(book_id) - if item_id not in table.col_book_map: - table.col_book_map[item_id] = set() - table.col_book_map[item_id].add(book_id) - table.book_col_map[book_id] = imap[book_id] = item_id - sql_update(imap) - dirtied |= set(imap) - - if changed_items: - imap = {} - sql_update(changed_items) - for book_id, item_id in changed_items.iteritems(): - old_item_id = table.book_col_map.get(book_id, None) - if old_item_id != item_id: - table.book_col_map[book_id] = item_id - table.col_book_map[item_id].add(book_id) - if old_item_id is not None: - table.col_book_map[old_item_id].discard(book_id) - imap[book_id] = item_id - sql_update(imap) - dirtied |= set(imap) + sql = ( + 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)' + if is_custom_series else + 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)' + ) + db.conn.executemany(sql.format(table.link_table, m['link_column']), + tuple((book_id, book_id, item_id) for book_id, item_id in + updated.iteritems())) # Remove no longer used items remove = {item_id for item_id in table.id_map if not @@ -294,6 +271,83 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args): return dirtied # }}} +# Many-Many fields {{{ +def many_many(book_id_val_map, db, field, allow_case_change, *args): + dirtied = set() + m = field.metadata + table = field.table + dt = m['datatype'] + is_authors = field.name == 'authors' + + # Map values to db ids, including any new values + kmap = safe_lower if dt == 'text' else lambda x:x + rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()} + sql_val_map = lambda x:x.replace(',', '|') if is_authors else lambda x:x + val_map = {} + case_changes = {} + for vals in book_id_val_map.itervalues(): + for val in vals: + get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, + case_changes, val_map, sql_val_map=sql_val_map) + + if case_changes: + change_case(case_changes, dirtied, db, table, m, + sql_val_map=sql_val_map) + + book_id_item_id_map = {k:tuple(val_map[v] for v in vals) + for k, vals in book_id_val_map.iteritems()} + + # Ignore those items whose value is the same as the current value + book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems() + if v != table.book_col_map.get(k, None)} + dirtied |= set(book_id_item_id_map) + + # Update the book->col and col->book maps + deleted = set() + updated = {} + for book_id, item_ids in book_id_item_id_map.iteritems(): + old_item_ids = table.book_col_map.get(book_id, None) + if old_item_ids: + for old_item_id in old_item_ids: + table.col_book_map[old_item_id].discard(book_id) + if item_ids: + table.book_col_map[book_id] = item_ids + for item_id in item_ids: + table.col_book_map[item_id].add(book_id) + updated[book_id] = item_ids + else: + table.book_col_map.pop(book_id, None) + deleted.add(book_id) + + # Update the db link table + if deleted: + db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table, + tuple((k,) for k in deleted)) + if updated: + vals = tuple( + (book_id, book_id, val) for book_id, vals in updated.iteritems() + for val in vals + ) + sql = ( + 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)' + ) + db.conn.executemany(sql.format(table.link_table, m['link_column']), vals) + + # Remove no longer used items + remove = {item_id for item_id in table.id_map if not + table.col_book_map.get(item_id, False)} + if remove: + db.conn.executemany('DELETE FROM %s WHERE id=?'%m['table'], + tuple((item_id,) for item_id in remove)) + for item_id in remove: + del table.id_map[item_id] + table.col_book_map.pop(item_id, None) + + return dirtied + + +# }}} + def dummy(book_id_val_map, *args): return set() @@ -311,9 +365,7 @@ class Writer(object): elif self.name[0] == '#' and self.name.endswith('_index'): self.set_books_func = custom_series_index elif field.is_many_many: - # TODO: Implement this - pass - # TODO: Remember to change commas to | when writing authors to sqlite + self.set_books_func = many_many elif field.is_many: self.set_books_func = (self.set_books_for_enum if dt == 'enumeration' else many_one)