Start work on writing many-many fields

src/calibre/db/write.py

@@ -53,7 +53,7 @@ def multiple_text(sep, x):
     if isinstance(x, unicode):
         x = x.split(sep)
     x = (y.strip() for y in x if y.strip())
-    return (' '.join(y.split()) for y in x if y)
+    return tuple(' '.join(y.split()) for y in x if y)
 
 def adapt_datetime(x):
     if isinstance(x, (unicode, bytes)):
@@ -178,6 +178,30 @@ def safe_lower(x):
     except (TypeError, ValueError, KeyError, AttributeError):
         return x
 
+def get_db_id(val, db, m, table, kmap, rid_map, allow_case_changes,
+              case_changes, val_map, sql_val_map=lambda x:x):
+    ''' Get the db id for the value val. If val does not exist in the db it is
+    inserted into the db. '''
+    kval = kmap(val)
+    item_id = rid_map.get(kval, None)
+    if item_id is None:
+        db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
+            m['table'], m['column']), (sql_val_map(val),))
+        item_id = rid_map[kval] = db.conn.last_insert_rowid()
+        table.id_map[item_id] = val
+        table.col_book_map[item_id] = set()
+    elif allow_case_changes and val != table.id_map[item_id]:
+        case_changes[item_id] = val
+    val_map[val] = item_id
+
+def change_case(case_changes, dirtied, db, table, m, sql_val_map=lambda x:x):
+    db.conn.executemany(
+        'UPDATE %s SET %s=? WHERE id=?'%(m['table'], m['column']),
+        tuple((sql_val_map(val), item_id) for item_id, val in case_changes.iteritems()))
+    for item_id, val in case_changes.iteritems():
+        table.id_map[item_id] = val
+        dirtied.update(table.col_book_map[item_id])
+
 def many_one(book_id_val_map, db, field, allow_case_change, *args):
     dirtied = set()
     m = field.metadata
@@ -185,101 +209,54 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
     dt = m['datatype']
     is_custom_series = dt == 'series' and table.name.startswith('#')
 
-    # Map values to their canonical form for later comparison
+    # Map values to db ids, including any new values
     kmap = safe_lower if dt in {'text', 'series'} else lambda x:x
+    rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
+    val_map = {None:None}
+    case_changes = {}
+    for val in book_id_val_map.itervalues():
+        if val is not None:
+            get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
+                    case_changes, val_map)
+
+    if case_changes:
+        change_case(case_changes, dirtied, db, table, m)
+
+    book_id_item_id_map = {k:val_map[v] for k, v in book_id_val_map.iteritems()}
 
     # Ignore those items whose value is the same as the current value
-    no_changes = {k:nval for k, nval in book_id_val_map.iteritems() if
+    book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
-                  kmap(nval) == kmap(field.for_book(k, default_value=None))}
+        if v != table.book_col_map.get(k, None)}
-    for book_id in no_changes:
+    dirtied |= set(book_id_item_id_map)
-        del book_id_val_map[book_id]
 
-    # If we are allowed case changes check that none of the ignored items are
+    # Update the book->col and col->book maps
-    # case changes. If they are, update the item's case in the db.
+    deleted = set()
-    if allow_case_change:
+    updated = {}
-        for book_id, nval in no_changes.iteritems():
+    for book_id, item_id in book_id_item_id_map.iteritems():
-            if nval is not None and nval != field.for_book(
+        old_item_id = table.book_col_map.get(book_id, None)
-                book_id, default_value=None):
+        if old_item_id is not None:
-                # Change of case
+            table.col_book_map[old_item_id].discard(book_id)
-                item_id = table.book_col_map[book_id]
+        if item_id is None:
-                db.conn.execute('UPDATE %s SET %s=? WHERE id=?'%(
+            table.book_col_map.pop(book_id, None)
-                    m['table'], m['column']), (nval, item_id))
+            deleted.add(book_id)
-                table.id_map[item_id] = nval
+        else:
-                dirtied |= table.col_book_map[item_id]
+            table.book_col_map[book_id] = item_id
-
+            table.col_book_map[item_id].add(book_id)
-    deleted = {k:v for k, v in book_id_val_map.iteritems() if v is None}
+            updated[book_id] = item_id
-    updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None}
-    link_table = table.link_table
 
+    # Update the db link table
     if deleted:
-        db.conn.executemany('DELETE FROM %s WHERE book=?'%link_table,
+        db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
-            tuple((book_id,) for book_id in deleted))
+                            tuple((k,) for k in deleted))
-        for book_id in deleted:
-            item_id = table.book_col_map.pop(book_id, None)
-            if item_id is not None:
-                table.col_book_map[item_id].discard(book_id)
-        dirtied |= set(deleted)
-
     if updated:
-        rid_map = {kmap(v):k for k, v in table.id_map.iteritems()}
-        book_id_item_id_map = {k:rid_map.get(kmap(v), None) for k, v in
-                               book_id_val_map.iteritems()}
-
-        # items that dont yet exist
-        new_items = {k:v for k, v in updated.iteritems() if
-                     book_id_item_id_map[k] is None}
-        # items that already exist
-        changed_items = {k:book_id_item_id_map[k] for k in updated if
-                         book_id_item_id_map[k] is not None}
-        def sql_update(imap):
         sql = (
             'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
             if is_custom_series else
             'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
         )
-            db.conn.executemany(sql.format(link_table, m['link_column']),
+        db.conn.executemany(sql.format(table.link_table, m['link_column']),
             tuple((book_id, book_id, item_id) for book_id, item_id in
-                       imap.iteritems()))
+                    updated.iteritems()))
-
-        if new_items:
-            item_ids = {}
-            val_map = {}
-            for val in set(new_items.itervalues()):
-                lval = kmap(val)
-                if lval in val_map:
-                    item_id = val_map[lval]
-                else:
-                    db.conn.execute('INSERT INTO %s(%s) VALUES (?)'%(
-                        m['table'], m['column']), (val,))
-                    item_id = val_map[lval] = db.conn.last_insert_rowid()
-                item_ids[val] = item_id
-                table.id_map[item_id] = val
-            imap = {}
-            for book_id, val in new_items.iteritems():
-                item_id = item_ids[val]
-                old_item_id = table.book_col_map.get(book_id, None)
-                if old_item_id is not None:
-                    table.col_book_map[old_item_id].discard(book_id)
-                if item_id not in table.col_book_map:
-                    table.col_book_map[item_id] = set()
-                table.col_book_map[item_id].add(book_id)
-                table.book_col_map[book_id] = imap[book_id] = item_id
-            sql_update(imap)
-            dirtied |= set(imap)
-
-        if changed_items:
-            imap = {}
-            sql_update(changed_items)
-            for book_id, item_id in changed_items.iteritems():
-                old_item_id = table.book_col_map.get(book_id, None)
-                if old_item_id != item_id:
-                    table.book_col_map[book_id] = item_id
-                    table.col_book_map[item_id].add(book_id)
-                    if old_item_id is not None:
-                        table.col_book_map[old_item_id].discard(book_id)
-                    imap[book_id] = item_id
-            sql_update(imap)
-            dirtied |= set(imap)
 
     # Remove no longer used items
     remove = {item_id for item_id in table.id_map if not
@@ -294,6 +271,83 @@ def many_one(book_id_val_map, db, field, allow_case_change, *args):
     return dirtied
 # }}}
 
+# Many-Many fields {{{
+def many_many(book_id_val_map, db, field, allow_case_change, *args):
+    dirtied = set()
+    m = field.metadata
+    table = field.table
+    dt = m['datatype']
+    is_authors = field.name == 'authors'
+
+    # Map values to db ids, including any new values
+    kmap = safe_lower if dt == 'text' else lambda x:x
+    rid_map = {kmap(item):item_id for item_id, item in table.id_map.iteritems()}
+    sql_val_map = lambda x:x.replace(',', '|') if is_authors else lambda x:x
+    val_map = {}
+    case_changes = {}
+    for vals in book_id_val_map.itervalues():
+        for val in vals:
+            get_db_id(val, db, m, table, kmap, rid_map, allow_case_change,
+                      case_changes, val_map, sql_val_map=sql_val_map)
+
+    if case_changes:
+        change_case(case_changes, dirtied, db, table, m,
+                    sql_val_map=sql_val_map)
+
+    book_id_item_id_map = {k:tuple(val_map[v] for v in vals)
+                           for k, vals in book_id_val_map.iteritems()}
+
+    # Ignore those items whose value is the same as the current value
+    book_id_item_id_map = {k:v for k, v in book_id_item_id_map.iteritems()
+        if v != table.book_col_map.get(k, None)}
+    dirtied |= set(book_id_item_id_map)
+
+    # Update the book->col and col->book maps
+    deleted = set()
+    updated = {}
+    for book_id, item_ids in book_id_item_id_map.iteritems():
+        old_item_ids = table.book_col_map.get(book_id, None)
+        if old_item_ids:
+            for old_item_id in old_item_ids:
+                table.col_book_map[old_item_id].discard(book_id)
+        if item_ids:
+            table.book_col_map[book_id] = item_ids
+            for item_id in item_ids:
+                table.col_book_map[item_id].add(book_id)
+            updated[book_id] = item_ids
+        else:
+            table.book_col_map.pop(book_id, None)
+            deleted.add(book_id)
+
+    # Update the db link table
+    if deleted:
+        db.conn.executemany('DELETE FROM %s WHERE book=?'%table.link_table,
+                            tuple((k,) for k in deleted))
+    if updated:
+        vals = tuple(
+            (book_id, book_id, val) for book_id, vals in updated.iteritems()
+            for val in vals
+        )
+        sql = (
+            'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
+        )
+        db.conn.executemany(sql.format(table.link_table, m['link_column']), vals)
+
+    # Remove no longer used items
+    remove = {item_id for item_id in table.id_map if not
+              table.col_book_map.get(item_id, False)}
+    if remove:
+        db.conn.executemany('DELETE FROM %s WHERE id=?'%m['table'],
+            tuple((item_id,) for item_id in remove))
+        for item_id in remove:
+            del table.id_map[item_id]
+            table.col_book_map.pop(item_id, None)
+
+    return dirtied
+
+
+# }}}
+
 def dummy(book_id_val_map, *args):
     return set()
 
@@ -311,9 +365,7 @@ class Writer(object):
         elif self.name[0] == '#' and self.name.endswith('_index'):
             self.set_books_func = custom_series_index
         elif field.is_many_many:
-            # TODO: Implement this
+            self.set_books_func = many_many
-            pass
-            # TODO: Remember to change commas to | when writing authors to sqlite
         elif field.is_many:
             self.set_books_func = (self.set_books_for_enum if dt ==
                                    'enumeration' else many_one)