diff --git a/src/calibre/gui2/viewer/highlights.py b/src/calibre/gui2/viewer/highlights.py
index 1406cbb353..3776885efa 100644
--- a/src/calibre/gui2/viewer/highlights.py
+++ b/src/calibre/gui2/viewer/highlights.py
@@ -121,6 +121,38 @@ def decoration_for_style(palette, style, icon_size, device_pixel_ratio, is_dark)
     return ans
 
 
+class ChapterGroup:
+
+    def __init__(self, title='', level=0):
+        self.title = title
+        self.subgroups = {}
+        self.annotations = []
+        self.level = level
+
+    def add_annot(self, a):
+        titles = a.get('toc_family_titles', (_('Unknown chapter'),))
+        node = self
+        for title in titles:
+            node = node.group_for_title(title)
+        node.annotations.append(a)
+
+    def group_for_title(self, title):
+        ans = self.subgroups.get(title)
+        if ans is None:
+            ans = ChapterGroup(title, self.level+1)
+            self.subgroups[title] = ans
+        return ans
+
+    def render_as_text(self, lines, as_markdown, link_prefix):
+        if self.title:
+            lines.append('#' * self.level + ' ' + self.title)
+            lines.append('')
+        for hl in self.annotations:
+            render_highlight_as_text(hl, lines, as_markdown=as_markdown, link_prefix=link_prefix)
+        for sg in self.subgroups.values():
+            sg.render_as_text(lines, as_markdown, link_prefix)
+
+
 class Export(ExportBase):
     prefs = vprefs
     pref_name = 'highlight_export_format'
@@ -142,17 +174,10 @@ class Export(ExportBase):
         lines = []
         as_markdown = fmt == 'md'
         link_prefix = link_prefix_for_location_links()
-        chapter_groups = {}
-        def_chap = (_('Unknown chapter'),)
+        root = ChapterGroup()
         for a in self.annotations:
-            toc_titles = a.get('toc_family_titles', def_chap)
-            chapter_groups.setdefault(toc_titles[0], []).append(a)
-        for chapter, group in chapter_groups.items():
-            if len(chapter_groups) > 1:
-                lines.append('### ' + chapter)
-                lines.append('')
-            for hl in group:
-                render_highlight_as_text(hl, lines, as_markdown=as_markdown, link_prefix=link_prefix)
+            root.add_annot(a)
+        root.render_as_text(lines, as_markdown, link_prefix)
         return '\n'.join(lines).strip()
 
 
diff --git a/src/pyj/read_book/highlights.pyj b/src/pyj/read_book/highlights.pyj
index f1e9ea0f24..9f637acdc1 100644
--- a/src/pyj/read_book/highlights.pyj
+++ b/src/pyj/read_book/highlights.pyj
@@ -577,6 +577,43 @@ def render_highlight_as_text(hl, lines, link_prefix, current_query, as_markdown=
     lines.push('')
 
 
+class ChapterGroup:
+
+    def __init__(self, title, level):
+        self.level = level or 0
+        self.title = title or ''
+        self.subgroups = v'{}'
+        self.subgroups_in_order = v'[]'
+        self.annotations = v'[]'
+
+    def add_annot(self, a):
+        titles = a['toc_family_titles']
+        if not titles:
+            titles = [_('Unknown chapter')]
+        node = self
+        for title in titles:
+            node = node.group_for_title(title)
+        node.annotations.push(a)
+
+    def group_for_title(self, title):
+        ans = self.subgroups[title]
+        if not ans:
+            ans = ChapterGroup(title, self.level+1)
+            self.subgroups[title] = ans
+            self.subgroups_in_order.push(title)
+        return ans
+
+    def render_as_text(self, lines, link_prefix, current_query, as_markdown):
+        if self.title:
+            lines.push('#' * self.level + ' ' + self.title)
+            lines.push('')
+        for hl in self.annotations:
+            render_highlight_as_text(hl, lines, link_prefix, current_query, as_markdown)
+        for title in self.subgroups_in_order:
+            sg = self.subgroups[title]
+            sg.render_as_text(lines, link_prefix, current_query, as_markdown)
+
+
 def show_export_dialog(annotations_manager):
     sd = get_session_data()
     fmt = sd.get('highlights_export_format')
@@ -605,21 +642,10 @@ def show_export_dialog(annotations_manager):
             return
         as_markdown = fmt is 'markdown'
         lines = v'[]'
-        def_chap = [_('Unknown chapter')]
-        chapter_groups = {}
-        chapters = v'[]'
-        for hl in all_highlights:
-            toc_title = (hl['toc_family_titles'] or def_chap)[0]
-            if not chapter_groups[toc_title]:
-                chapter_groups[toc_title] = v'[]'
-                chapters.push(toc_title)
-            chapter_groups[toc_title].push(hl)
-        for chapter_title in chapters:
-            if chapters.length > 1:
-                lines.push('### ' + chapter_title)
-                lines.push('')
-            for hl in chapter_groups[chapter_title]:
-                render_highlight_as_text(hl, lines, link_prefix, current_query, as_markdown=as_markdown)
+        root = ChapterGroup()
+        for a in all_highlights:
+            root.add_annot(a)
+        root.render_as_text(lines, link_prefix, current_query, as_markdown)
         document.getElementById(ta_id).textContent = lines.join('\n')
 
     def fmt_item(text, val):