diff --git a/resources/notes_sqlite.sql b/resources/notes_sqlite.sql
index 6b2c4e94e8..2897c21615 100644
--- a/resources/notes_sqlite.sql
+++ b/resources/notes_sqlite.sql
@@ -8,6 +8,8 @@ CREATE TABLE notes_db.notes ( id INTEGER PRIMARY KEY AUTOINCREMENT,
     UNIQUE(item, colname)
 );
 
+CREATE INDEX notes_db.notes_colname_idx ON notes (colname);
+
 CREATE TABLE notes_db.resources ( 
     hash TEXT NOT NULL PRIMARY KEY ON CONFLICT FAIL,
     name TEXT NOT NULL UNIQUE ON CONFLICT FAIL
diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py
index bb4b39267b..000ceccc3f 100644
--- a/src/calibre/db/backend.py
+++ b/src/calibre/db/backend.py
@@ -975,8 +975,8 @@ class DB:
     def notes_data_for(self, field_name, item_id):
         return self.notes.get_note_data(self.conn, field_name, item_id)
 
-    def get_notes_id_map(self):
-        return self.notes.get_note_id_map(self.conn)
+    def get_all_items_that_have_notes(self, field_name):
+        return self.notes.get_all_items_that_have_notes(self.conn, field_name)
 
     def set_notes_for(self, field, item_id, doc: str, searchable_text: str, resource_hashes, remove_unused_resources) -> int:
         id_val = self.tables[field].id_map[item_id]
diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index ea7c5790d3..8089968f8f 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -686,9 +686,9 @@ class Cache:
         ' Return all notes data as a dict or None if note does not exist '
         return self.backend.notes_data_for(field, item_id)
 
-    def get_notes_id_map(self) -> dict:
-        ' Return all item_id for each field assosiated to a notes. '
-        return self.backend.get_notes_id_map()
+    def get_all_items_that_have_notes(self, field_name=None) -> set[int] | dict[str, int]:
+        ' Return all item_ids for items that have notes in the specified field or all fields if field_name is None '
+        return self.backend.get_all_items_that_have_notes(field_name)
 
     @read_api
     def field_supports_notes(self, field) -> bool:
diff --git a/src/calibre/db/notes/connect.py b/src/calibre/db/notes/connect.py
index 558a25b85f..5ceed337ac 100644
--- a/src/calibre/db/notes/connect.py
+++ b/src/calibre/db/notes/connect.py
@@ -9,6 +9,7 @@ import time
 import xxhash
 from contextlib import suppress
 from itertools import count, repeat
+from collections import defaultdict
 from typing import Optional
 
 from calibre import sanitize_file_name
@@ -269,13 +270,13 @@ class Notes:
                 'resource_hashes': frozenset(self.resources_used_by(conn, note_id)),
             }
 
-    def get_note_id_map(self, conn):
-        rslt = {}
-        for (note_id, field_name) in conn.execute('SELECT id,colname FROM notes_db.notes'):
-            if field_name not in rslt:
-                rslt[field_name] = []
-            rslt[field_name].append(note_id)
-        return rslt
+    def get_all_items_that_have_notes(self, conn, field_name=None):
+        if field_name:
+            return {item_id for (item_id,) in conn.execute('SELECT item FROM notes_db.notes WHERE colname=?', (field_name,))}
+        ans = defaultdict(set)
+        for (note_id, field_name) in conn.execute('SELECT item, colname FROM notes_db.notes'):
+            ans[field_name].add(note_id)
+        return ans
 
     def rename_note(self, conn, field_name, old_item_id, new_item_id, new_item_value):
         note_id = self.note_id_for(conn, field_name, old_item_id)
diff --git a/src/calibre/db/tests/notes.py b/src/calibre/db/tests/notes.py
index a03aa4a99b..c9313835f5 100644
--- a/src/calibre/db/tests/notes.py
+++ b/src/calibre/db/tests/notes.py
@@ -117,6 +117,7 @@ def test_cache_api(self: 'NotesTest'):
     h1 = cache.add_notes_resource(b'resource1t', 'r1.jpg')
     h2 = cache.add_notes_resource(b'resource2t', 'r1.jpg')
     cache.set_notes_for('#tags', tag_id, doc, resource_hashes=(h1, h2))
+    self.ae(cache.get_all_items_that_have_notes(), {'#tags': {tag_id}, 'authors': {author_id}})
     self.ae(cache.notes_for('#tags', tag_id), doc)
     cache.delete_custom_column('tags')
     cache.close()