#!/tmp/ea-venv/bin/python3 """ hermes-wiki-generator: Static HTML generator for Lukas' karpathy-style Wiki. Architecture: 1. Watchdog observes /home/admin/my-karpathy-wiki/ for changes 2. On .md change → re-render single HTML file to /home/admin/.local/share/hermes-wiki/site/ 3. On Wiki startup → generate tree.json, graph.json, tag-cloud.json (one-time) 4. Python http.server serves /home/admin/.local/share/hermes-wiki/site/ on 127.0.0.1:8765 Why static HTML: - Mobile-first: load once, works offline (with service worker) - Fast: zero JS for content rendering, JSON data lazy-loaded - Bookmarkable: each page is a real URL like /concepts/agent-reference-model.html - Touch-friendly: no animation that fights tap-targets Run: python3 ~/repos/hermes-wiki-static/generator.py --start python3 ~/repos/hermes-wiki-static/generator.py --stop python3 ~/repos/hermes-wiki-static/generator.py --status python3 ~/repos/hermes-wiki-static/generator.py --once (one-shot regen, no watcher) """ import argparse import json import logging import os import re import signal import socket import sys import time from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from threading import Lock, Thread from urllib.parse import unquote import frontmatter import markdown as md import yaml from watchdog.events import FileSystemEvent, FileSystemEventHandler from watchdog.observers import Observer # ============================================================ # Configuration # ============================================================ WIKI_DIR = Path("/home/admin/my-karpathy-wiki") SITE_DIR = Path("/home/admin/.local/share/hermes-wiki/site") LOG_DIR = Path("/home/admin/.local/share/hermes-wiki/log") PID_FILE = Path("/tmp/hermes-wiki-static.pid") PORT = 8765 BIND = "127.0.0.1" LOG_DIR.mkdir(parents=True, exist_ok=True) SITE_DIR.mkdir(parents=True, exist_ok=True) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ logging.FileHandler(LOG_DIR / "generator.log"), logging.StreamHandler(sys.stdout), ], ) log = logging.getLogger("hermes-wiki") # ============================================================ # Markdown → HTML rendering # ============================================================ # WikiLink regex: [[entity-name]] or [[path/to/entity|display-text]] WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|([^\]]+))?\]\]") # Hashtag: #tag-name (max 30 chars, alphanumeric + dash) TAG_RE = re.compile(r"(? str: """Convert /concepts/agent-reference-model.md → concepts/agent-reference-model""" rel = path.relative_to(WIKI_DIR).with_suffix("") return str(rel).replace(os.sep, "/") def read_markdown(path: Path) -> frontmatter.Post: """Read .md with YAML frontmatter, fallback to plain text.""" try: return frontmatter.load(path) except yaml.YAMLError as e: log.warning(f"YAML-Fehler in {path}: {e} — versuche ohne Frontmatter") text = path.read_text(encoding="utf-8", errors="replace") return frontmatter.Post(text) def resolve_wikilinks(text: str, slug_to_path: dict) -> str: """Replace [[entity-name]] with entity-name.""" def replace(m: re.Match) -> str: target = m.group(1).strip() display = (m.group(2) or target).strip() # Try exact match first if target in slug_to_path: slug = slug_to_path[target] return f'{display}' # Try with .md suffix stripped target_no_ext = target.replace(".md", "") if target_no_ext in slug_to_path: slug = slug_to_path[target_no_ext] return f'{display}' # Not found — render as broken link return f'{display}' return WIKILINK_RE.sub(replace, text) def quote(s: str) -> str: """URL-encode path components.""" import urllib.parse return urllib.parse.quote(s, safe="/-_~.") def render_page(md_path: Path, slug_to_path: dict, all_meta: list) -> dict: """Render one .md to (HTML + metadata). Returns dict with html, title, slug, frontmatter.""" post = read_markdown(md_path) slug = slugify(md_path) # Replace wikilinks before markdown rendering body_with_links = resolve_wikilinks(post.content, slug_to_path) html_body = md.markdown( body_with_links, extensions=MD_EXTENSIONS, extension_configs={"toc": {"permalink": True}}, ) # Extract title from H1 if not in frontmatter title = post.metadata.get("title") or md_path.stem.replace("-", " ").title() if not post.metadata.get("title"): h1_match = re.search(r"]*>(.*?)", html_body, re.IGNORECASE) if h1_match: title = re.sub(r"<[^>]+>", "", h1_match.group(1)) return { "slug": slug, "title": title, "html": html_body, "frontmatter": dict(post.metadata), "path": str(md_path.relative_to(WIKI_DIR)), } # ============================================================ # Index data: tree, graph, tags, backlinks # ============================================================ def build_index(slug_meta_list: list) -> dict: """Build tree.json, graph.json, tags.json, backlinks.json.""" slug_to_meta = {s["slug"]: s for s in slug_meta_list} # Tree: hierarchical folder structure tree = {"name": "Vault", "children": {}, "files": []} for s in slug_meta_list: parts = s["slug"].split("/") node = tree for folder in parts[:-1]: node = node["children"].setdefault(folder, {"name": folder, "children": {}, "files": []}) node["files"].append({"name": parts[-1], "title": s["title"], "slug": s["slug"]}) # Recursively convert dict-of-children to list-of-children (easier JSON) def tree_to_list(node): result = { "name": node["name"], "files": sorted(node["files"], key=lambda f: f["name"].lower()), } result["folders"] = sorted( [tree_to_list(child) for child in node["children"].values()], key=lambda f: f["name"].lower() ) return result tree_list = tree_to_list(tree) # Graph: extract [[wikilinks]] from each rendered HTML nodes = [] edges = [] for s in slug_meta_list: nodes.append({"id": s["slug"], "title": s["title"], "size": 1}) # Find all wikilinks in the source wikilinks = WIKILINK_RE.findall(s["html"] + " " + str(s["frontmatter"])) for target, _disp in wikilinks: target_slug = target.replace(".md", "") if target_slug in slug_to_meta: edges.append({"source": s["slug"], "target": target_slug}) # Tags: collect from frontmatter.tags + inline #tags tags = {} # tag → [slugs] for s in slug_meta_list: # Frontmatter tags fm_tags = s["frontmatter"].get("tags", []) if isinstance(fm_tags, str): fm_tags = [t.strip() for t in fm_tags.split(",")] for tag in fm_tags or []: tags.setdefault(str(tag).lower(), []).append(s["slug"]) # Inline #tags in content inline_tags = TAG_RE.findall(s["html"]) for tag in inline_tags: tags.setdefault(tag.lower(), []).append(s["slug"]) # Backlinks: for each page, list of pages that link TO it backlinks = {s["slug"]: [] for s in slug_meta_list} for edge in edges: target = edge["target"] source = edge["source"] if source != target: # skip self-links backlinks[target].append(source) return { "tree": tree_list, "graph": {"nodes": nodes, "edges": edges}, "tags": tags, "backlinks": backlinks, } # ============================================================ # HTML page template # ============================================================ PAGE_TEMPLATE = """ {title} · Hermes Wiki
📓 Hermes Wiki
{frontmatter_card}
{body}
{path}
GRAPH VIEW · Drag to rotate · Tap a node to navigate
Legend
Current page
Connected
Other notes
""" def render_frontmatter_card(meta: dict) -> str: """Render frontmatter as a Catppuccin-styled card.""" if not meta: return "" rows = [] for key, value in meta.items(): if key in ("title", "tags"): continue # shown elsewhere if isinstance(value, list): value = ", ".join(str(v) for v in value) rows.append(f'
{key}{value}
') if not rows: return "" return f'' def write_page(meta: dict, site_dir: Path) -> None: """Write a single HTML page to SITE_DIR/.html.""" target = site_dir / (quote(meta["slug"]) + ".html") target.parent.mkdir(parents=True, exist_ok=True) kind = "concept" if "concepts" in meta["slug"] else \ "entity" if "entities" in meta["slug"] else \ "scratch" if "scratch" in meta["slug"] else "note" html = PAGE_TEMPLATE.format( title=meta["title"], body=meta["html"], frontmatter_card=render_frontmatter_card(meta["frontmatter"]), kind=kind, path=meta["path"], ) target.write_text(html, encoding="utf-8") def write_index_files(index_data: dict, site_dir: Path) -> None: """Write tree.json, graph.json, tags.json, backlinks.json.""" assets_dir = site_dir / "__" assets_dir.mkdir(parents=True, exist_ok=True) (assets_dir / "tree.json").write_text( json.dumps(index_data["tree"], ensure_ascii=False, indent=2), encoding="utf-8" ) (assets_dir / "graph.json").write_text( json.dumps(index_data["graph"], ensure_ascii=False, indent=2), encoding="utf-8" ) (assets_dir / "tags.json").write_text( json.dumps(index_data["tags"], ensure_ascii=False, indent=2), encoding="utf-8" ) (assets_dir / "backlinks.json").write_text( json.dumps(index_data["backlinks"], ensure_ascii=False, indent=2), encoding="utf-8" ) def write_data_js(slug_meta_list: list, index_data: dict, site_dir: Path) -> None: """Single data.js with all metadata + index baked in (avoids CORS issues).""" assets_dir = site_dir / "__" assets_dir.mkdir(parents=True, exist_ok=True) pages = [{ "slug": s["slug"], "title": s["title"], "tags": s["frontmatter"].get("tags", []) if isinstance(s["frontmatter"].get("tags"), list) else [], "type": s["frontmatter"].get("type", "note"), "path": s["path"], } for s in slug_meta_list] js = f"""// Auto-generated by hermes-wiki-generator window.HERMES_DATA = {{ pages: {json.dumps(pages, ensure_ascii=False)}, tree: {json.dumps(index_data['tree'], ensure_ascii=False)}, graph: {json.dumps(index_data['graph'], ensure_ascii=False)}, tags: {json.dumps(index_data['tags'], ensure_ascii=False)}, backlinks: {json.dumps(index_data['backlinks'], ensure_ascii=False)}, currentSlug: "{slug_meta_list[0]['slug'] if slug_meta_list else ''}", }}; """ (site_dir / "__/data.js").write_text(js, encoding="utf-8") # ============================================================ # Static assets (CSS, JS, PWA manifest) # ============================================================ # CSS is a placeholder — touch-first design comes in next commit CSS_PLACEHOLDER = """/* Hermes Wiki — placeholder CSS */ :root { --bg-primary: #0a0a14; --bg-secondary: #0e0e1a; --bg-surface: #1a1a2e; --text-primary: #cdd6f4; --text-secondary: #a6adc8; --text-muted: #6c7086; --accent: #FFD700; --link: #FFD700; --border: #313244; } * { margin: 0; padding: 0; box-sizing: border-box; } body { background: var(--bg-primary); color: var(--text-primary); font-family: -apple-system, 'Segoe UI', Inter, sans-serif; } .app-shell { display: grid; grid-template-columns: 280px 1fr 260px; grid-template-rows: 48px 1fr; height: 100vh; } .app-header { grid-column: 1 / -1; background: var(--bg-secondary); border-bottom: 1px solid var(--border); display: flex; align-items: center; padding: 0 16px; gap: 12px; } .brand { color: var(--accent); font-weight: 700; } #search { flex: 1; max-width: 400px; padding: 6px 12px; background: rgba(30,30,50,0.6); border: 1px solid var(--border); border-radius: 6px; color: var(--text-primary); } .icon-btn { background: none; border: 1px solid var(--border); color: var(--text-secondary); padding: 4px 10px; border-radius: 4px; cursor: pointer; } .app-tree { background: var(--bg-secondary); border-right: 1px solid var(--border); overflow-y: auto; padding: 8px; } .app-content { overflow-y: auto; padding: 32px 48px; } .app-graph { background: var(--bg-secondary); border-left: 1px solid var(--border); } .meta-card { background: var(--bg-surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 16px; margin-bottom: 24px; font-size: 12px; } .meta-row { display: flex; gap: 8px; padding: 2px 0; } .meta-key { color: var(--accent); font-weight: 600; min-width: 80px; } .wikilink-missing { color: var(--text-muted); text-decoration: line-through; } .note-body { line-height: 1.7; } .note-body h1, .note-body h2, .note-body h3 { margin-top: 1.5em; margin-bottom: 0.5em; } .note-body code { background: var(--bg-surface); padding: 2px 6px; border-radius: 3px; } .note-body pre { background: var(--bg-surface); padding: 12px; border-radius: 6px; overflow-x: auto; } .app-bottom-nav { display: none; } """ JS_PLACEHOLDER = """// Hermes Wiki — placeholder JS (function() { 'use strict'; // Bootstrap from data.js const data = window.HERMES_DATA || { pages: [], tree: [], graph: { nodes: [], edges: [] }, tags: {}, backlinks: {} }; // Render tree into sidebar function renderTree(node, container, basePath = '') { const ul = document.createElement('ul'); if (node.folders) { for (const folder of node.folders) { const li = document.createElement('li'); const header = document.createElement('div'); header.textContent = '📁 ' + folder.name; header.style.cursor = 'pointer'; const childrenContainer = document.createElement('div'); header.onclick = () => { childrenContainer.style.display = childrenContainer.style.display === 'none' ? 'block' : 'none'; }; childrenContainer.style.display = 'none'; childrenContainer.style.paddingLeft = '12px'; li.appendChild(header); li.appendChild(childrenContainer); renderTree(folder, childrenContainer, basePath + '/' + folder.name); ul.appendChild(li); } } if (node.files) { for (const file of node.files) { const li = document.createElement('li'); const a = document.createElement('a'); a.href = '/' + file.slug + '.html'; a.textContent = file.title; a.style.color = 'var(--text-primary)'; a.style.textDecoration = 'none'; a.style.display = 'block'; a.style.padding = '2px 8px'; li.appendChild(a); ul.appendChild(li); } } container.appendChild(ul); } const treeEl = document.getElementById('tree'); if (treeEl && data.tree) { renderTree(data.tree, treeEl); } // Search (client-side) const search = document.getElementById('search'); if (search) { search.addEventListener('input', (e) => { const q = e.target.value.toLowerCase().trim(); if (!q) return; // Simple substring search on titles + tags const matches = data.pages.filter(p => p.title.toLowerCase().includes(q) || (p.tags || []).some(t => t.toLowerCase().includes(q)) ); console.log('[Search]', q, '→', matches.length, 'results'); // TODO: show results dropdown }); } // Toggle buttons (placeholder) document.addEventListener('click', (e) => { if (e.target.dataset.action === 'toggle-tree') { document.querySelector('.app-tree').classList.toggle('open'); } if (e.target.dataset.action === 'toggle-graph') { document.querySelector('.app-graph').classList.toggle('open'); } }); })(); """ PWA_MANIFEST = """{ "name": "Hermes Wiki", "short_name": "Wiki", "description": "Lukas Huber's knowledge vault — mobile-first static wiki", "start_url": "/__/index.html", "display": "standalone", "background_color": "#0a0a14", "theme_color": "#0a0a14", "icons": [ { "src": "data:image/svg+xml,📓", "sizes": "any", "type": "image/svg+xml" } ] }""" def write_assets(site_dir: Path) -> None: """Copy static assets (CSS, JS, manifest) from repo to site dir.""" assets_dir = site_dir / "__" assets_dir.mkdir(parents=True, exist_ok=True) repo_root = Path(__file__).parent for asset in ("style.css", "app.js", "manifest.json"): src = repo_root / asset if src.exists(): content = src.read_text(encoding="utf-8") (assets_dir / asset).write_text(content, encoding="utf-8") else: log.warning(f"Asset not found in repo: {src}") # index page (redirect to first page or show all) index_html = """ Hermes Wiki Open Wiki""" first_slug = "index" # fall back to index.md (site_dir / "__/index.html").write_text( index_html.replace("{slug}", first_slug), encoding="utf-8" ) # ============================================================ # Generator orchestration # ============================================================ class WikiState: """Holds in-memory state of generated pages and index.""" def __init__(self): self.lock = Lock() self.slug_to_path = {} # slug → original .md path self.slug_meta_list = [] # list of {slug, title, html, frontmatter, path} self.index_data = {} # tree, graph, tags, backlinks def regenerate_all(self): """Full rebuild — used on startup.""" log.info(f"Full regenerate: scanning {WIKI_DIR}") with self.lock: self.slug_to_path.clear() self.slug_meta_list.clear() md_files = sorted(WIKI_DIR.rglob("*.md")) skipped_raw = 0 for md_file in md_files: rel = md_file.relative_to(WIKI_DIR) if rel.parts[0] == "raw": skipped_raw += 1 continue slug = slugify(md_file) self.slug_to_path[slug] = slug self.slug_to_path[md_file.stem] = slug # for [[agent-hermes]] lookups # Also index by basename without .md base = md_file.name[:-3] if base not in self.slug_to_path: self.slug_to_path[base] = slug log.info(f"Found {len(self.slug_to_path)} slugs ({skipped_raw} skipped in raw/)") # Render each page for md_file in md_files: rel = md_file.relative_to(WIKI_DIR) if rel.parts[0] == "raw": continue meta = render_page(md_file, self.slug_to_path, self.slug_meta_list) self.slug_meta_list.append(meta) write_page(meta, SITE_DIR) log.info(f"Rendered {len(self.slug_meta_list)} HTML pages") # Build index data self.index_data = build_index(self.slug_meta_list) write_index_files(self.index_data, SITE_DIR) write_data_js(self.slug_meta_list, self.index_data, SITE_DIR) write_assets(SITE_DIR) log.info("Index files written: tree.json, graph.json, tags.json, backlinks.json, data.js") def regenerate_one(self, md_path: Path): """Re-render one page (and its reverse-link targets if needed).""" rel = md_path.relative_to(WIKI_DIR) if rel.parts[0] == "raw": return with self.lock: slug = slugify(md_path) meta = render_page(md_path, self.slug_to_path, self.slug_meta_list) # Update or append for i, existing in enumerate(self.slug_meta_list): if existing["slug"] == slug: self.slug_meta_list[i] = meta break else: self.slug_meta_list.append(meta) write_page(meta, SITE_DIR) log.info(f"Re-rendered: {slug}") # Rebuild index (cheap for small wikis, ensures backlinks/tags stay consistent) self.index_data = build_index(self.slug_meta_list) write_index_files(self.index_data, SITE_DIR) write_data_js(self.slug_meta_list, self.index_data, SITE_DIR) class WikiFileWatcher(FileSystemEventHandler): def __init__(self, state: WikiState): self.state = state def on_modified(self, event: FileSystemEvent): if event.is_directory: return path = Path(event.src_path) if path.suffix == ".md": log.info(f"File modified: {path}") self.state.regenerate_one(path) def on_created(self, event: FileSystemEvent): if event.is_directory: return path = Path(event.src_path) if path.suffix == ".md": log.info(f"File created: {path}") self.state.regenerate_one(path) def on_deleted(self, event: FileSystemEvent): if event.is_directory: return path = Path(event.src_path) if path.suffix == ".md": log.info(f"File deleted: {path} (TODO: remove HTML)") # ============================================================ # HTTP server # ============================================================ class WikiHandler(SimpleHTTPRequestHandler): """Serves from SITE_DIR. Logs requests briefly.""" def log_message(self, format, *args): log.debug(f"HTTP {self.address_string()} {format % args}") def end_headers(self): # Cache headers for static assets if self.path.startswith("/__/"): self.send_header("Cache-Control", "public, max-age=300") super().end_headers() def start_server(): """Run HTTP server on BIND:PORT serving from SITE_DIR.""" os.chdir(SITE_DIR) server = HTTPServer((BIND, PORT), WikiHandler) log.info(f"HTTP server: http://{BIND}:{PORT} serving {SITE_DIR}") server.serve_forever() # ============================================================ # Daemon mode (PID file, signals) # ============================================================ def write_pid(): PID_FILE.write_text(str(os.getpid())) def is_running(): if not PID_FILE.exists(): return False try: pid = int(PID_FILE.read_text()) os.kill(pid, 0) # check if alive return True except (ValueError, ProcessLookupError, PermissionError): return False def stop_daemon(): if not PID_FILE.exists(): log.info("No PID file — daemon not running") return pid = int(PID_FILE.read_text()) try: os.kill(pid, signal.SIGTERM) log.info(f"Sent SIGTERM to PID {pid}") except ProcessLookupError: log.info(f"PID {pid} not found — already stopped?") PID_FILE.unlink(missing_ok=True) # ============================================================ # Main entry point # ============================================================ def main(): parser = argparse.ArgumentParser(description="Hermes Wiki Static Generator") parser.add_argument("--start", action="store_true", help="Run as daemon (watcher + HTTP server)") parser.add_argument("--stop", action="store_true", help="Stop daemon") parser.add_argument("--status", action="store_true", help="Show status") parser.add_argument("--once", action="store_true", help="Generate once and exit (no watcher)") parser.add_argument("--foreground", action="store_true", help="Run in foreground (default: --start)") args = parser.parse_args() if args.stop: stop_daemon() return if args.status: if is_running(): pid = int(PID_FILE.read_text()) print(f"Running: PID {pid}") else: print(f"Not running (no PID file: {PID_FILE})") return if args.once: state = WikiState() state.regenerate_all() print(f"Generated {len(state.slug_meta_list)} pages to {SITE_DIR}") return # Daemon mode if is_running(): log.error(f"Already running (PID {int(PID_FILE.read_text())}). Use --stop first.") sys.exit(1) # Initial full regen state = WikiState() state.regenerate_all() # Start file watcher observer = Observer() observer.schedule(WikiFileWatcher(state), str(WIKI_DIR), recursive=True) observer.start() log.info(f"Filesystem watcher started on {WIKI_DIR}") # Start HTTP server in background thread server_thread = Thread(target=start_server, daemon=True) server_thread.start() # Write PID write_pid() log.info(f"Daemon PID: {os.getpid()}") # Handle signals def handle_term(signum, frame): log.info(f"Received signal {signum}, shutting down") observer.stop() PID_FILE.unlink(missing_ok=True) sys.exit(0) signal.signal(signal.SIGTERM, handle_term) signal.signal(signal.SIGINT, handle_term) # Keep main thread alive try: while True: time.sleep(60) except KeyboardInterrupt: handle_term(0, None) if __name__ == "__main__": main()