diff --git a/docs/.pages b/docs/.pages
index 1ef66ee..6b41c66 100644
--- a/docs/.pages
+++ b/docs/.pages
@@ -1,5 +1,4 @@
 nav:
     - README.md
-    - tags.md
     - colophon.md
     - ...
diff --git a/docs/colophon.md b/docs/colophon.md
index bcfe83e..d71db97 100644
--- a/docs/colophon.md
+++ b/docs/colophon.md
@@ -4,7 +4,7 @@ hide: [toc]
 
 # Colophon
 
-This site is built with [`mkdocs`](https://www.mkdocs.org/) and [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/). Sure, it's designed for docs, but with a few modifications (like the [`tags`](https://squidfunk.github.io/mkdocs-material/plugins/tags/) plugin) it can be used for notes.
+This site is built with [`mkdocs`](https://www.mkdocs.org/) and [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/). Sure, it's designed for docs, but with a few modifications (like the [`tags`](https://squidfunk.github.io/mkdocs-material/plugins/tags/) plugin) it can be used for notes. I made some fairly large modifications to how tags work, so the sidebar is structured by tag rather than by directory (directories are fairly arbitrary, and might get removed entirely in future).
 
 It's been through a few iterations in the past, starting with [`gatsby-theme-code-notes`](https://github.com/mrmartineau/gatsby-theme-code-notes/), on to [`hugo-theme-notes`](https://github.com/RealOrangeOne/hugo-theme-notes/), and finally here.
 
diff --git a/docs/notes/index.md b/docs/notes/index.md
index 77a71c8..1dee9f4 100644
--- a/docs/notes/index.md
+++ b/docs/notes/index.md
@@ -1,5 +1,4 @@
 ---
-template: all.html
 hide: [toc]
 ---
 
diff --git a/docs/tags.md b/docs/tags.md
deleted file mode 100644
index 3ca3bc4..0000000
--- a/docs/tags.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-hide: [toc]
----
-
-# Tags
diff --git a/hooks/notes.py b/hooks/notes.py
index 2e33233..efd6609 100644
--- a/hooks/notes.py
+++ b/hooks/notes.py
@@ -1,4 +1,8 @@
 import jinja2
+from mkdocs.structure.nav import Navigation, Section
+from collections import defaultdict
+from mkdocs.utils import meta
+from mkdocs.plugins import event_priority
 
 @jinja2.pass_context
 def get_page(context, slug):
@@ -10,17 +14,18 @@ def get_page(context, slug):
     raise ValueError("Unable to find page for '%s'", slug)
 
 
+def get_notes_from_nav(nav: Navigation):
+    notes_section = next(item for item in nav if item.is_section and item.title == "Notes")
+
+    notes = [item for item in nav.pages if item.is_page and notes_section in item.ancestors and item.file.src_uri != "notes/index.md"]
+
+    return notes_section, notes
+
+
 @jinja2.pass_context
 def get_notes(context):
-    notes = []
-    for page in context["nav"].pages:
-        if not page.file.src_uri.startswith("notes/"):
-            continue
+    notes_section, notes = get_notes_from_nav(context["nav"])
 
-        if page.file.src_uri == "notes/index.md":
-            continue
-
-        notes.append(page)
     return sorted(notes, key=lambda p: p.meta["git_creation_date_localized_raw_iso_date"], reverse=True)
 
 
@@ -28,3 +33,48 @@ def on_env(env, config, files):
     env.tests["startswith"] = str.startswith
     env.globals["get_page"] = get_page
     env.globals["get_notes"] = get_notes
+
+
+class TagSection(Section):
+    """
+    A modified section which is active if any of its children are active
+    (regardless of what mkdocs tells it).
+    """
+    @property
+    def active(self):
+        return any(c.active for c in self.children)
+
+    @active.setter
+    def active(self, value: bool):
+        pass
+
+
+@event_priority(100)
+def on_nav(nav: Navigation, config, files):
+    """
+    Structure the nav sections by
+    """
+    notes_section, notes = get_notes_from_nav(nav)
+
+    notes_page = notes_section.children[0]
+
+    pages_to_tag = defaultdict(list)
+
+    orphan_pages = []
+
+    for note in notes:
+        with open(note.file.abs_src_path) as f:
+            _, metadata = meta.get_data(f.read())
+
+        if tags := metadata.get("tags"):
+            for tag in tags:
+                pages_to_tag[tag].append(note)
+        else:
+            orphan_pages.append(note)
+
+    tag_sections = [
+        TagSection(tag, pages)
+        for tag, pages in pages_to_tag.items()
+    ]
+
+    notes_section.children = tag_sections + orphan_pages + [notes_page]
diff --git a/mkdocs.yml b/mkdocs.yml
index 8056434..1674616 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -95,12 +95,11 @@ plugins:
       order_by: title
   - tags:
       enabled: true
-      tags_file: tags.md
+      tags_file: notes/index.md
   - git-revision-date-localized:
       enable_creation_date: true
       fallback_to_build_date: true
       exclude:
-        - tags.md
         - README.md
         - notes/index.md
   - minify_html
diff --git a/theme/all.html b/theme/all.html
deleted file mode 100644
index f4c316c..0000000
--- a/theme/all.html
+++ /dev/null
@@ -1,16 +0,0 @@
-{% extends "base.html" %}
-
-{% block content %}
-    {{ super() }}
-
-    <ul>
-        {% for page in get_notes() %}
-            <li>
-                <a href="{{ page.url }}">{{ page.title }}</a> - {{ page.meta.git_creation_date_localized_raw_iso_date }}
-                {% for tag in page.meta.tags %}
-                    <span class="md-tag">{{ tag }}</span>
-                {% endfor %}
-            </li>
-        {% endfor %}
-    </ul>
-{% endblock %}