zettelkasten.dashboard.backend.routes.graph_data

zettelkasten.dashboard.backend.routes.graph_data

Graph listing, graph-data, cluster, papers, and fulltext/pdf routes.

Split out of the former flat routes.py; behaviour is unchanged.

GraphDrilldownRequest

Bases: BaseModel

POST body for a single-graph drill (Approach B: membership in the payload).

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

class GraphDrilldownRequest(BaseModel):
    """POST body for a single-graph drill (Approach B: membership in the payload)."""
    group_id: str
    # The drilled glyph's EXACT build-time member id set, carried in the LOD payload
    # and echoed back here so drill resolution needs NO server-side store lookup.
    member_ids: "list[str] | None" = None

get_graph_data

get_graph_data(name: str, max_nodes: int = Query(0, ge=0, description='LOD cap: 0 = full graph; >0 keeps the top-N nodes by link_count and collapses the rest'), bbox: str = Query(None, description="Viewport bbox 'x0,y0,x1,y1' in t-SNE space; with max_nodes>0, on-screen notes are kept in full detail (server-side viewport LOD)"), exclude_types: str = Query(None, description="Comma-separated note types to omit from the payload + LOD (e.g. 'quote'); they stay reachable via the note-detail API"), request: Request = None, response: Response = None) -> dict

Returns nodes and edges for the force-directed graph.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/graph-data")
def get_graph_data(
    name: str,
    max_nodes: int = Query(0, ge=0, description="LOD cap: 0 = full graph; >0 keeps the top-N nodes by link_count and collapses the rest"),
    bbox: str = Query(None, description="Viewport bbox 'x0,y0,x1,y1' in t-SNE space; with max_nodes>0, on-screen notes are kept in full detail (server-side viewport LOD)"),
    exclude_types: str = Query(None, description="Comma-separated note types to omit from the payload + LOD (e.g. 'quote'); they stay reachable via the note-detail API"),
    request: Request = None,
    response: Response = None,
) -> dict:
    """Returns nodes and edges for the force-directed graph."""
    # The LOD cap only engages for an int > 0 supplied over HTTP (direct callers
    # pass the ``Query(...)`` sentinel), so normalize it to keep the full (0) and
    # LOD (>0) payloads as DISTINCT cache entries.
    lod_key = max_nodes if isinstance(max_nodes, int) and max_nodes > 0 else 0
    # Note types to drop from the payload (and thus from LOD ranking/collapse).
    # ``()`` when absent, so the default request stays byte-identical below.
    excluded_types = _parse_exclude_types(exclude_types if isinstance(exclude_types, str) else None)
    # Viewport-aware LOD only applies alongside a max_nodes cap. Snap the bbox to
    # the tile grid so it keys the cache stably AND filters detail identically
    # (mirrors the project route).
    qbbox = _quantize_bbox(_parse_bbox(bbox)) if lod_key else None

    # Structural-only read: this route touches only ``zg.notes`` + links, so
    # load without eagerly building the kglite embedding index (a wasteful,
    # crash-prone native build for a purely structural payload). A later
    # search/cluster caller still gets a fully-indexed box on demand.
    zg = _get_graph_structural(name)

    # Serve an unchanged graph from the payload cache: ``_get_graph`` above has
    # already refreshed ``_graph_sigs[name]`` from the on-disk note mtimes, so
    # keying on that signature invalidates exactly when a note changes. Use the
    # STABLE digest (not builtin ``hash()``) so the key — and the ETag derived
    # from it — is reproducible across process restarts.
    sig_component = _sig_digest(_graph_sigs.get(name, ()))
    # The LOD collapse groups semantically ONLY when the resolved cluster layout
    # carries ``node_clusters``; otherwise it falls back to structural grouping. So
    # the LOD cache key's embedding component must key on THAT SAME signal — not
    # ``_embeddings_ready(zg)``, which flips True once the box's kglite index slot
    # exists even though the t-SNE/cluster compute is a LATER step. There is a real
    # window where ``_embeddings_ready``==True but ``_graph_cluster_result``==None
    # (still warming): keying on readiness caches a structural-fallback payload
    # under ``emb=1`` in that window that never busts once the clusters warm (the
    # key never transitions, so semantic supers never appear for that graph).
    #
    # FIX-2 perf: derive that component from the CHEAP in-memory readiness probe
    # (``_graph_clusters_ready_cheap``) instead of eagerly calling
    # ``_graph_cluster_result`` here — that eager call forced a synchronous t-SNE on
    # a cold cluster cache even for a would-be 304/cache HIT. The probe reads only
    # the cluster cache (no t-SNE), and the actual (possibly expensive) layout is
    # computed below ONLY on the fresh-build path. The probe flips False→True on the
    # SAME cold→warm cluster transition the resolved-readiness key busted on, so a
    # structural-fallback payload is never served from the LRU after the clusters
    # warm. Mirrors ``projects.py::get_project_graph``.
    force_semantic_rebuild = False
    if lod_key:
        # LOD path only: the quantized viewport tile keys each pan tile separately,
        # and cluster-readiness busts the key on a cold→warm transition.
        emb_component = int(_graph_clusters_ready_cheap(name, _LOD_BASE_GRANULARITY))
        cache_key = ("graph", name, sig_component, lod_key, qbbox, emb_component)
        # FIX-2: break the stale-STRUCTURAL deadlock. The cheap probe flips True
        # only once a cluster build has CACHED clusters, which happens only on a
        # fresh-build (payload-cache MISS). So a box that warmed AFTER a structural
        # payload was cached (probe False → emb=0 → K0) stays stuck: the same sig +
        # emb=0 key HITs K0 forever, the build path is never re-entered, clusters
        # are never computed, and the probe never flips — a non-semantic client
        # (which never polls /clusters to self-heal) is served a permanently-stale
        # structural payload. When embeddings are now ready BUT no cluster build has
        # been attempted yet (``_graph_clusters_attempted`` False — distinct from
        # "attempted, genuinely no clusters", which must NOT thrash), bypass the
        # 304/payload-cache serve and rebuild ONCE: the build computes+caches the
        # clusters and returns the SEMANTIC payload; the next poll's probe is then
        # True (emb=1) and settles. This is cheap (a readiness flag + a dict lookup,
        # never a t-SNE), and NEVER fires on a genuine warm cluster-cache HIT (that
        # path has emb=1, so ``emb_component == 0`` is already False) — keeping the
        # fix-4 perf guarantee intact.
        force_semantic_rebuild = (
            emb_component == 0
            and _embeddings_ready(zg)
            and not _graph_clusters_attempted(name, _LOD_BASE_GRANULARITY)
        )
    else:
        # Full (non-LOD) payload never touches clusters or the viewport, so its key
        # — and the ETag derived from it — stays BYTE-IDENTICAL to the pre-LOD-v2
        # 4-tuple. Adding viewport/readiness components here would needlessly
        # invalidate every client's cached full-graph ETag.
        cache_key = ("graph", name, sig_component, lod_key)
    # Only extend the key when types are actually excluded, so a request WITHOUT
    # ``exclude_types`` keeps its historical key/ETag byte-identical.
    if excluded_types:
        cache_key = cache_key + ("ex",) + excluded_types
    if not force_semantic_rebuild:
        # ETag / conditional-GET on the FRESH signature: a matching If-None-Match
        # returns a bodyless 304 — skipping BOTH the node/edge assembly below AND
        # FastAPI's JSON serialization of the (large) payload. Skipped on a forced
        # rebuild so a client holding the stale structural ETag isn't 304'd back to
        # the stale payload.
        not_modified = _conditional_etag(request, response, cache_key)
        if not_modified is not None:
            return not_modified
        cached = _graph_cache_get(cache_key)
        if cached is not None:
            return cached

    nodes = []
    edges = []
    local_ids = set(zg.notes.keys())

    for note in zg.notes.values():
        nodes.append({
            "id": note.id,
            "title": note.title,
            "type": note.type,
            "tags": note.tags,
            "chapter": note.source.get("chapter", ""),
            "link_count": len(note.links) + len(zg.get_backlinks(note.id)),
        })
        for link in note.links:
            if link.target not in local_ids:
                continue
            edge: dict[str, Any] = {
                "source": note.id,
                "target": link.target,
                "relation": link.relation,
                "direction": link.direction,
            }
            if link.graph:
                edge["graph"] = link.graph
            edges.append(edge)

    # Drop excluded note types (e.g. supporting ``quote`` evidence) from the
    # payload BEFORE LOD so they never enter the ranking/collapse — the cluster
    # view's node budget is spent on substantive notes and no super-node's
    # ``type_mix`` is dominated by evidence. No-op when ``exclude_types`` is absent.
    nodes, edges = _apply_exclude_types(nodes, edges, excluded_types)

    # ``isinstance`` guard: FastAPI resolves ``max_nodes`` to an int over HTTP,
    # but direct callers (tests) get the ``Query(...)`` sentinel — treat that as
    # "no LOD" so the default path stays byte-identical.
    if isinstance(max_nodes, int) and max_nodes > 0:
        # SEMANTIC LOD (fresh-build path ONLY — reached past the 304/cache lookup):
        # resolve the embedding clusters HERE, not before the cache check, so a
        # cache hit never pays for t-SNE (FIX-2). Collapsed notes group by topical
        # cluster, super-nodes are stamped with their members' t-SNE centroid, and
        # on-screen notes stay in full detail. A warming/empty box returns no layout
        # → we fall back to plain structural rank-based LOD (graceful, never blocks
        # the response). This same call warms the cluster cache, so the NEXT
        # request's cheap probe agrees with this build's readiness.
        layout = _graph_cluster_result(name, _LOD_BASE_GRANULARITY)
        positions = (layout or {}).get("node_positions")
        node_clusters = (layout or {}).get("node_clusters")
        cluster_meta = _cluster_meta(layout)
        # A finer clustering supplies each super's child sub-cluster ids for
        # recursive drill-down (best-effort: only when the base clusters exist).
        fine = _graph_cluster_result(name, _LOD_FINE_GRANULARITY) if node_clusters else None
        child_clusters = (fine or {}).get("node_clusters")
        # Capture each super's exact member set so it can be co-located with the
        # cached payload (re-asserted on a payload HIT) — immune to the shared
        # member-set LRU's independent eviction.
        member_sets: dict = {}
        nodes, edges, _cits, lod = _apply_lod(
            nodes, edges, max_nodes, fallback_bucket=name,
            positions=positions,
            bbox=qbbox if positions else None,
            node_clusters=node_clusters,
            cluster_meta=cluster_meta,
            child_clusters=child_clusters,
            # Persist each super's exact member set so a later drill reveals the
            # EXACT members it was built from (root-cause fix for drill under/over-
            # reveal on cluster-readiness flips + t-SNE regeneration).
            remember_members=True,
            member_sets_out=member_sets,
        )
        result = {"nodes": nodes, "edges": edges, "lod": lod}
        if force_semantic_rebuild:
            # The forced rebuild was keyed under emb=0 (the PRE-build cheap probe),
            # but the build above just ran ``_graph_cluster_result``, so the probe
            # now reflects reality. Store the payload under the component the NEXT
            # poll will compute — so that poll HITs this entry instead of MISSing
            # under a different key and rebuilding the same semantic payload a second
            # time (wasting an LRU slot on the write-once emb=0 entry). Post-build the
            # probe is True (clusters materialized → next poll keys emb=1) or False
            # (structural fallback → next poll keys emb=0 and correctly serves this).
            cache_key = (
                "graph", name, sig_component, lod_key, qbbox,
                int(_graph_clusters_ready_cheap(name, _LOD_BASE_GRANULARITY)),
            )
        _graph_cache_put(cache_key, result, member_sets=member_sets)
    else:
        result = {"nodes": nodes, "edges": edges}
        _graph_cache_put(cache_key, result)
    return result

get_graph_clusters

get_graph_clusters(name: str, granularity: int = Query(50, ge=10, le=90, description='Cluster granularity (10=few large, 90=many small)'), request: Request = None, response: Response = None) -> dict

Embedding-based semantic clusters for a single graph's notes.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/clusters")
def get_graph_clusters(
    name: str,
    granularity: int = Query(50, ge=10, le=90, description="Cluster granularity (10=few large, 90=many small)"),
    request: Request = None,
    response: Response = None,
) -> dict:
    """Embedding-based semantic clusters for a single graph's notes."""
    # Never build the kglite index on the request thread. Load structurally (404s
    # an unknown name without an eager build), then require the box's embedding
    # index to be ALREADY warm (built by the background run_warmup). A cold box
    # returns the empty-cluster shape the frontend already tolerates plus an
    # additive ``embeddings_status`` flag, so the request is fast and the native
    # build stays off the request thread.
    zg = _get_graph_structural(name)
    if not _embeddings_ready(zg):
        # Cold box: schedule an idempotent background rebuild (never on the
        # request thread) so the NEXT poll returns real clusters, and return the
        # warming shape now. A note-less box needs no index — _maybe_rewarm skips
        # it — so it just reports warming without scheduling anything.
        _maybe_rewarm(name, zg)
        return {"node_clusters": {}, "clusters": [], "max_clusters": 0, "embeddings_status": "warming"}
    # STABLE digest, not builtin ``hash()``: this ``sig_component`` flows into the
    # ``tsne:…`` cluster cache_key below, which is what names the on-disk t-SNE
    # cache file — a per-process-salted hash would change the filename every
    # restart, so the persisted layout would never be re-read.
    sig_component = _sig_digest(_graph_sigs.get(name, ()))
    cache_key = f"graph:{name}:{granularity}:{sig_component}"
    # ETag / conditional-GET: a matching If-None-Match returns a bodyless 304,
    # skipping the (cached) cluster lookup + JSON serialization entirely.
    not_modified = _conditional_etag(request, response, cache_key)
    if not_modified is not None:
        return not_modified
    cached = _cluster_cache_get(cache_key)
    if cached is not None:
        return cached
    # Share the expensive TSNE with the papers-semantic projection at the same
    # scope+granularity+signature (papers keys the identical ``tsne:graph:…``).
    result = _embedding_clusters(
        _note_items(zg), granularity,
        cache_key=f"tsne:graph:{name}:{granularity}:{sig_component}",
        follow_map=_evidence_follow_map(zg),
    )
    _cluster_cache_put(cache_key, result)
    # Monotonic marker so the LOD readiness/attempted probes stay stable for this
    # warm graph even after the layout cache clears (prevents emb_component churn).
    _mark_cluster_result(cache_key, bool(result.get("node_clusters")))
    return result

get_graph_drilldown

get_graph_drilldown(name: str, group_id: str = Query(..., description='The super-node id (or its group_id) to drill into'), member_ids: 'list[str] | None' = Query(None, description="Approach B: the drilled glyph's exact build-time member id set (from the LOD payload), so drill resolution needs no server store. Prefer the POST route for large sets."), request: Request = None, response: Response = None) -> dict

Recursive drill-down for a single-graph LOD super-node.

Given a collapsed super-node's id, returns that group's CONNECTED sub-graph: its member notes PLUS their REAL edges (inter-member connectivity preserved, never stripped), re-clustered into finer semantic sub-supers at the next granularity tier — or the real members when the group is small / cannot subdivide (leaf=True). Degrades to an empty sub-graph while embeddings are warming (semantic grouping needs the cluster map).

When member_ids is supplied (Approach B) membership resolves directly from that set with no server-side store lookup; otherwise it degrades to the store/live re-derivation for back-compat with old clients.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/graph-drilldown")
def get_graph_drilldown(
    name: str,
    group_id: str = Query(..., description="The super-node id (or its group_id) to drill into"),
    member_ids: "list[str] | None" = Query(
        None,
        description="Approach B: the drilled glyph's exact build-time member id set (from the LOD payload), so drill resolution needs no server store. Prefer the POST route for large sets.",
    ),
    request: Request = None,
    response: Response = None,
) -> dict:
    """Recursive drill-down for a single-graph LOD super-node.

    Given a collapsed super-node's id, returns that group's CONNECTED sub-graph:
    its member notes PLUS their REAL edges (inter-member connectivity preserved,
    never stripped), re-clustered into finer semantic sub-supers at the next
    granularity tier — or the real members when the group is small / cannot
    subdivide (``leaf=True``). Degrades to an empty sub-graph while embeddings are
    warming (semantic grouping needs the cluster map).

    When ``member_ids`` is supplied (Approach B) membership resolves directly from
    that set with no server-side store lookup; otherwise it degrades to the
    store/live re-derivation for back-compat with old clients.
    """
    # ``isinstance`` guard: FastAPI resolves ``member_ids`` to a list (client sent
    # it) or None; a DIRECT caller that omits it gets the ``Query(...)`` sentinel,
    # which is not a list — treat that as "not supplied" so the back-compat path
    # stays byte-identical to before.
    supplied = member_ids if isinstance(member_ids, list) else None
    return _graph_drilldown(name, group_id, supplied)

post_graph_drilldown

post_graph_drilldown(name: str, req: GraphDrilldownRequest, request: Request = None, response: Response = None) -> dict

POST twin of :func:get_graph_drilldown — carries the glyph's member_ids in the body (preferred for large member sets that would bloat a query string).

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.post("/graphs/{name}/graph-drilldown")
def post_graph_drilldown(
    name: str,
    req: GraphDrilldownRequest,
    request: Request = None,
    response: Response = None,
) -> dict:
    """POST twin of :func:`get_graph_drilldown` — carries the glyph's ``member_ids``
    in the body (preferred for large member sets that would bloat a query string)."""
    return _graph_drilldown(name, req.group_id, req.member_ids)

get_project_clusters

get_project_clusters(name: str, granularity: int = Query(50, ge=10, le=90, description='Cluster granularity (10=few large, 90=many small)'), request: Request = None, response: Response = None) -> dict

Embedding-based semantic clusters across all notes in a project.

Clusters the composite of every source's notes (plus connecting _cross notes) so semantic colouring in project mode behaves like it does for a single source.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/projects/{name}/clusters")
def get_project_clusters(
    name: str,
    granularity: int = Query(50, ge=10, le=90, description="Cluster granularity (10=few large, 90=many small)"),
    request: Request = None,
    response: Response = None,
) -> dict:
    """Embedding-based semantic clusters across all notes in a project.

    Clusters the composite of every source's notes (plus connecting ``_cross``
    notes) so semantic colouring in project mode behaves like it does for a
    single source.
    """
    # ── Pre-built bundle fast path ──
    # A fresh per-project bundle already holds the base/fine semantic layouts, so
    # serve one straight from disk: no source resolution, no embedding-readiness
    # gate (so it NEVER defers with "warming"), and no t-SNE on the request thread.
    # Signature-gated — a bundle built from now-stale sources is ignored and we
    # fall through to the live compute below. Only the granularities the bundle
    # stores (base/fine) are covered here; any other slider value falls through to
    # the (disk-cached) live path.
    _bundled = _project_bundle_layout(name, granularity)
    if _bundled is not None:
        _layout, _bsig = _bundled
        not_modified = _conditional_etag(
            request, response, ("bundle", "project-clusters", name, granularity, _bsig)
        )
        if not_modified is not None:
            return not_modified
        return _layout
    else:
        # No fresh bundle layout covering this project: warm one in the background
        # (the build computes the t-SNE off the web server, in a separate process),
        # then fall through to the live path for this request. Guarded on a real
        # signature so an unknown project doesn't schedule a doomed build.
        #
        # Two miss shapes reach here and only ONE should schedule a build:
        #   * NO fresh bundle at all (``_b is None``) → build.
        #   * a fresh but STRUCTURE-ONLY bundle (``layout is None`` — built while the
        #     embedding stack was cold, yet signature-fresh) → schedule an UPGRADE
        #     build so a real layout is persisted and future cluster/LOD reads stop
        #     falling to live t-SNE on the request thread. Gated on
        #     ``_project_embeddings_ready`` so a machine that simply can't produce a
        #     layout doesn't loop rebuilding one every poll; it fires once the boxes
        #     warm (a later poll), then the bundle upgrades and this branch stops.
        #   * a fresh bundle WITH a layout but a mid-slider granularity the bundle
        #     doesn't store → ``_b.get("layout")`` is truthy → NO build (expected
        #     fall-through to the disk-cached live compute).
        try:
            from .projects import (
                project_signature,
                _load_project_bundle,
                _schedule_bundle_build,
                _project_embeddings_ready,
            )

            _psig = project_signature(name)
            if _psig is not None:
                _b = _load_project_bundle(name, _psig)
                if _b is None:
                    _schedule_bundle_build(name)
                elif _b.get("layout") is None and _project_embeddings_ready(name):
                    _schedule_bundle_build(name)
        except Exception:  # pragma: no cover - defensive
            pass

    _, repo = _resolve_ref(name)
    project_data = _load_project_or_404(name)
    gdir = repo.zettel_dir if repo is not None else GRAPHS_DIR
    sources = list(project_data.get("sources", []))
    if (gdir / "_cross").is_dir():
        sources.append("_cross")

    def _display(src: str) -> str:
        return federation.namespace_id(repo.id, src) if repo is not None else src

    # Resolve each source structurally (never an eager per-box kglite build), so
    # the request thread stays off the native construction path. Readiness is
    # checked below on the private index; ``.embeddings`` is only touched once a
    # box is confirmed warm.
    resolved: list[tuple] = []
    sig_parts: list[str] = []
    for source_name in sources:
        display = _display(source_name)
        zg = _try_get_graph_structural(display)
        if zg is None or not zg.path.is_dir():
            continue
        resolved.append((display, zg))
        sig_parts.append(f"{display}:{_sig_digest(_graph_sigs.get(display, ()))}")

    # If any non-empty source box's embedding index isn't warm yet, return the
    # empty-cluster shape (frontend-tolerated) + an additive warming flag rather
    # than synchronously building indexes here — run_warmup owns the build.
    # Empty boxes carry no notes to cluster, so they never gate the response.
    # For every cold-but-has-notes source, schedule an idempotent background
    # rebuild keyed by the display name actually queried (so federated/project
    # sources self-heal too); the NEXT poll then returns real clusters.
    cold = [(dn, zg) for dn, zg in resolved if zg.notes and not _embeddings_ready(zg)]
    if cold:
        for display_name, zg in cold:
            _maybe_rewarm(display_name, zg)
        return {"node_clusters": {}, "clusters": [], "max_clusters": 0, "embeddings_status": "warming"}

    sig_component = "|".join(sig_parts)
    cache_key = f"project:{name}:{granularity}:{sig_component}"
    # ETag / conditional-GET: a matching If-None-Match returns a bodyless 304,
    # skipping the (cached) cluster lookup + JSON serialization entirely. Checked
    # BEFORE the per-note ``items`` assembly below (which is O(notes) and touches
    # every box's embeddings), so a matching 304 pays none of that cost — the
    # ``cache_key`` uses only ``sig_parts`` + name + granularity, all resolved
    # above, so its ETag value is unchanged by this reorder.
    not_modified = _conditional_etag(request, response, cache_key)
    if not_modified is not None:
        return not_modified
    cached = _cluster_cache_get(cache_key)
    if cached is not None:
        return cached

    items: list[dict] = []
    follow_map: dict[str, str] = {}
    for _display_name, zg in resolved:
        if not zg.notes:
            continue  # no notes → no items, and no reason to touch .embeddings
        items.extend(_note_items(zg))
        # A quote's ``supports`` target lives in the SAME source, so the raw-id
        # follow map resolves within the composite union (items also carry raw ids).
        follow_map.update(_evidence_follow_map(zg))

    # Share the expensive TSNE with the papers-semantic projection at the same
    # scope+granularity+signature (papers keys the identical ``tsne:project:…``).
    result = _embedding_clusters(
        items, granularity,
        cache_key=f"tsne:project:{name}:{granularity}:{sig_component}",
        follow_map=follow_map,
    )
    _cluster_cache_put(cache_key, result)
    # Monotonic marker (project twin) so the project LOD readiness/attempted probes
    # stay stable across a layout-cache clear — no emb_component oscillation.
    _mark_cluster_result(cache_key, bool(result.get("node_clusters")))
    return result

graph_papers

graph_papers(name: str, budget: int = Query(12, ge=0, description='Max papers in the suggested set'), suggested_only: bool = Query(False, description='Return only the suggested set'), sort: str = Query('', description='importance|read_priority|interestingness|recency|influence|trajectory|type|title|author|date'), reverse: bool = Query(False, description='Reverse the resolved sort order'), as_of: str = Query('', description='Replay trends as of this year; empty = live')) -> dict

Papers view for a single source graph (its works, scored and situated).

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/papers")
def graph_papers(
    name: str,
    budget: int = Query(12, ge=0, description="Max papers in the suggested set"),
    suggested_only: bool = Query(False, description="Return only the suggested set"),
    sort: str = Query("", description="importance|read_priority|interestingness|recency|influence|trajectory|type|title|author|date"),
    reverse: bool = Query(False, description="Reverse the resolved sort order"),
    as_of: str = Query("", description="Replay trends as of this year; empty = live"),
) -> dict:
    """Papers view for a single source graph (its works, scored and situated)."""
    # Structural load (404s an unknown name) so a cold box is never eagerly
    # indexed on the request thread. The note-embedding projection below runs
    # ONLY when the box's kglite index is already warm (built by the background
    # run_warmup); a still-warming box degrades to the engine's theme-only signal
    # plus an additive ``embeddings_status="warming"`` flag rather than blocking
    # on a native build. When warm, the payload is byte-identical to before.
    zg = _get_graph_structural(name)  # 404s on an unknown/unsafe name
    local, repo = _resolve_ref(name)
    base = repo.zettel_dir if repo is not None else GRAPHS_DIR
    ns = (lambda s: federation.namespace_id(repo.id, s)) if repo is not None else (lambda s: s)
    loc = lambda d: _resolve_ref(d)[0]

    # Project note embeddings onto works only when the box has notes AND its index
    # is warm — never touch ``.embeddings`` on a not-ready box (that is the eager
    # native build this decouples). A note-bearing but cold box is "warming".
    projected = bool(zg.notes and _embeddings_ready(zg))
    warming = bool(zg.notes and not _embeddings_ready(zg))
    if warming:
        # Cold box with notes: schedule an idempotent background rebuild (off the
        # request thread) so a subsequent request gets the full semantic signal.
        _maybe_rewarm(name, zg)

    cov_sig = _coverage_signature([local], base)
    meta_sig = _meta_signature([local], base)
    # Graph-scope engine runs build_corpus(project="") = GLOBAL, so the key must
    # also track the citation pool and every other graph's signature. The
    # readiness state (projected/warming) is folded in too so a warming payload
    # is never served once the index warms — the warm request keys differently
    # and rebuilds with full semantic signals.
    cit_sig = _citations_signature(base)
    corpus_sig = _global_corpus_signature(base)
    # The cache key deliberately OMITS sort/reverse/suggested_only: those are
    # cheap post-cache view transforms (``apply_papers_view``), not inputs to the
    # scored payload. One scored base payload per (scope, budget, as_of,
    # signatures, readiness) therefore serves every sort/filter toggle as a cache
    # HIT + O(n) reorder rather than a full syllabus recompute.
    cache_key = (
        f"graph:{name}:{budget}:{as_of}:"
        f"{_sig_digest(_graph_sigs.get(name, ()))}:{cov_sig}:{meta_sig}:{cit_sig}:{corpus_sig}:{int(projected)}:{int(warming)}"
    )
    base_payload = _papers_cache_get(cache_key)
    if base_payload is None:
        # Graph scope runs build_corpus(project="") = GLOBAL, so ownership spans
        # ALL on-disk source folders — not just [name]. Pass the full local
        # folder set so a citation merged onto an out-of-scope owner keeps its
        # note-centroid.
        if projected:
            embeddings, clusters = _papers_semantic_inputs(
                [name], loc, base, owner_locals=set(note_graph_names(base)),
                cluster_cache_key=f"tsne:graph:{name}:50:{_sig_digest(_graph_sigs.get(name, ()))}",
            )
        else:
            embeddings, clusters = {}, {}
        # Assemble the base payload in the engine's DEFAULT order (no sort/filter)
        # so it can be re-viewed per request without rescoring.
        base_payload = papers.assemble_papers(
            _get_graph_structural,
            graph=local,
            budget=budget,
            suggested_only=False,
            sort="",
            reverse=False,
            as_of=as_of,
            scope_name=name,
            graphs_dir=base,
            namespace=ns,
            localize=loc,
            embeddings=embeddings,
            clusters=clusters,
        )
        if warming:
            base_payload["embeddings_status"] = "warming"
        _papers_cache_put(cache_key, base_payload)
    return papers.apply_papers_view(
        base_payload, sort=sort, reverse=reverse, suggested_only=suggested_only
    )

project_papers

project_papers(name: str, budget: int = Query(12, ge=0, description='Max papers in the suggested set'), suggested_only: bool = Query(False, description='Return only the suggested set'), sort: str = Query('', description='importance|read_priority|interestingness|recency|influence|trajectory|type|title|author|date'), reverse: bool = Query(False, description='Reverse the resolved sort order'), as_of: str = Query('', description='Replay trends as of this year; empty = live')) -> dict

Papers view across a project: the works cited by any of its sources.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/projects/{name}/papers")
def project_papers(
    name: str,
    budget: int = Query(12, ge=0, description="Max papers in the suggested set"),
    suggested_only: bool = Query(False, description="Return only the suggested set"),
    sort: str = Query("", description="importance|read_priority|interestingness|recency|influence|trajectory|type|title|author|date"),
    reverse: bool = Query(False, description="Reverse the resolved sort order"),
    as_of: str = Query("", description="Replay trends as of this year; empty = live"),
) -> dict:
    """Papers view across a project: the works cited by any of its sources."""
    local, repo = _resolve_ref(name)
    project_data = _load_project_or_404(name)
    base = repo.zettel_dir if repo is not None else GRAPHS_DIR
    ns = (lambda s: federation.namespace_id(repo.id, s)) if repo is not None else (lambda s: s)
    loc = lambda d: _resolve_ref(d)[0]

    sources = list(project_data.get("sources", []))
    if (base / "_cross").is_dir():
        sources.append("_cross")
    scoped_display = frame_search_sources(local, graphs_dir=base, namespace=ns)

    # Reload each scoped source BEFORE keying the cache so an out-of-band note
    # edit refreshes its `_graph_sigs` entry and changes the cache key — without
    # this the sig is read from a stale (or absent) entry and a note edit on disk
    # would hit a frozen key and serve a stale payload. Mirrors graph_papers /
    # get_project_clusters. Resolve STRUCTURALLY (never an eager per-box kglite
    # build) and, in the same pass, note whether every box's index is already
    # warm: the semantic projection below runs only when all are, so a cold box
    # is never indexed on the request thread. A note-bearing cold box is "warming".
    all_ready = True
    warming = False
    for display in scoped_display:
        zg = _try_get_graph_structural(display)
        if zg is None or not zg.path.is_dir():
            continue
        # Only a box that actually HAS notes can gate the project's semantics on
        # being warm. A note-less source (an empty ``_cross``, a promoted-but-
        # unextracted paper) carries nothing to embed, so treating it as
        # not-ready would make the WHOLE project warming-forever with no hint —
        # mirror get_project_clusters, which guards ``zg.notes and not
        # _embeddings_ready(zg)``. For a cold box that DOES have notes, schedule
        # an idempotent background rebuild (off the request thread), keyed by the
        # display name, so a later poll gets the full semantic projection.
        if zg.notes and not _embeddings_ready(zg):
            all_ready = False
            warming = True
            _maybe_rewarm(display, zg)

    cov_sig = _coverage_signature(sources, base)
    meta_sig = _meta_signature([loc(d) for d in scoped_display], base)
    sig = "|".join(f"{d}:{_sig_digest(_graph_sigs.get(d, ()))}" for d in scoped_display)
    # Project scope is covered by the per-source sigs above, but the referenced
    # citations come from the global store — fold its fingerprint in too. The
    # readiness state (all_ready/warming) is folded in so a warming payload is
    # never served once the indexes warm (the warm request keys differently).
    cit_sig = _citations_signature(base)
    # As in graph_papers, the key OMITS sort/reverse/suggested_only — those are
    # applied as a cheap post-cache view (``apply_papers_view``) over one scored
    # base payload, so a sort/filter toggle is a cache HIT + O(n) reorder.
    cache_key = (
        f"project:{name}:{budget}:{as_of}:{sig}:{cov_sig}:{meta_sig}:{cit_sig}:{int(all_ready)}:{int(warming)}"
    )
    base_payload = _papers_cache_get(cache_key)
    if base_payload is None:
        if all_ready:
            # Reuse the SAME cluster-cache key convention the /projects/{name}/
            # clusters route uses (scope:name:granularity:signature) so papers and
            # /clusters share ONE t-SNE clustering per (scope, granularity,
            # content). granularity is the route default (50) for both.
            embeddings, clusters = _papers_semantic_inputs(
                scoped_display, loc, base,
                cluster_cache_key=f"tsne:project:{name}:50:{sig}",
            )
        else:
            embeddings, clusters = {}, {}
        base_payload = papers.assemble_papers(
            _get_graph_structural,
            project=local,
            budget=budget,
            suggested_only=False,
            sort="",
            reverse=False,
            as_of=as_of,
            scope_name=name,
            graphs_dir=base,
            namespace=ns,
            localize=loc,
            embeddings=embeddings,
            clusters=clusters,
        )
        if warming:
            base_payload["embeddings_status"] = "warming"
        _papers_cache_put(cache_key, base_payload)
    return papers.apply_papers_view(
        base_payload, sort=sort, reverse=reverse, suggested_only=suggested_only
    )

graph_fulltext

graph_fulltext(name: str) -> dict

Extracted full text + saved highlights for a source's local Zotero PDF.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/fulltext")
def graph_fulltext(name: str) -> dict:
    """Extracted full text + saved highlights for a source's local Zotero PDF."""
    _get_graph_structural(name)  # 404s on an unknown/unsafe name (no embedding build)
    key, meta = _source_zotero_key(name)
    if not key:
        return {"available": False, "warning": "No Zotero key on this source (federated or unlinked)."}
    result = zotero.fetch_pdf(key, include_text=True, include_annotations=True)
    if "error" in result:
        return {"available": False, "warning": result["error"]}
    return {
        "available": True,
        "filename": result.get("filename", ""),
        "num_pages": result.get("num_pages", 0),
        "text": result.get("text", ""),
        "annotations": result.get("annotations", []),
        "warning": result.get("text_warning", ""),
    }

graph_pdf

graph_pdf(name: str)

Stream a source's local Zotero PDF bytes for an embedded viewer.

Source code in zettelkasten/dashboard/backend/routes/graph_data.py

@router.get("/graphs/{name}/pdf")
def graph_pdf(name: str):
    """Stream a source's local Zotero PDF bytes for an embedded viewer."""
    _get_graph_structural(name)  # 404s on an unknown/unsafe name (no embedding build)
    key, _meta = _source_zotero_key(name)
    if not key:
        raise HTTPException(status_code=404, detail="No Zotero key on this source.")
    result = zotero.fetch_pdf(key, include_text=False, include_annotations=False)
    if "error" in result or not result.get("path"):
        raise HTTPException(status_code=404, detail=result.get("error", "PDF not available locally."))
    return FileResponse(
        result["path"],
        media_type="application/pdf",
        filename=result.get("filename", f"{name}.pdf"),
    )