ignacio.ballesteros/quartz-org-roam
1
0
Fork 0
forked from github/quartz
Code Pull Requests Activity

feat: unified watch server under org-garden

Browse Source
This commit is contained in:
Ignacio Ballesteros
2026-02-21 00:36:31 +01:00
parent 1076bf31ed
commit a4582230b5
21 changed files with 679 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -22,6 +22,6 @@ scripts/pipeline/erl_crash.dump
# Test helpers (not needed in production)
scripts/test.bib
scripts/test_pipeline.exs
/pipeline/deps/
/pipeline/_build/
/pipeline/result
/org-garden/deps/
/org-garden/_build/
/org-garden/result

8
flake.lock generated
View File

@@ -68,17 +68,17 @@
"type": "github"
}
},
"pipeline": {
"org-garden": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_2"
},
"locked": {
"path": "./pipeline",
"path": "./org-garden",
"type": "path"
},
"original": {
"path": "./pipeline",
"path": "./org-garden",
"type": "path"
},
"parent": []
@@ -87,7 +87,7 @@
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"pipeline": "pipeline"
"org-garden": "org-garden"
}
},
"systems": {

55
flake.nix
View File

@@ -4,16 +4,16 @@
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
pipeline.url = "path:./pipeline";
org-garden.url = "path:./org-garden";
};
outputs = { self, nixpkgs, flake-utils, pipeline }:
outputs = { self, nixpkgs, flake-utils, org-garden }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
fs = pkgs.lib.fileset;
pipelineApp = pipeline.packages.${system}.default;
orgGardenApp = org-garden.packages.${system}.default;
# Pre-fetched npm dependency tree (node_modules).
# src is filtered to only package.json + package-lock.json so that
@@ -36,7 +36,7 @@
'';
};
# The build application wrapper script
# The build application wrapper script (one-shot build)
buildApp = pkgs.writeShellApplication {
name = "build";
runtimeInputs = [ pkgs.nodejs_22 ];
@@ -54,18 +54,43 @@
# Drop in pre-built node_modules
ln -s ${quartzDeps}/node_modules "$WORK/repo/node_modules"
# Run the pre-compiled pipeline escript (org → md, citations transform)
${pipelineApp}/bin/pipeline "$NOTES_DIR" \
# Pass paths via environment for org-garden
export QUARTZ_PATH="$WORK/repo"
export NODE_PATH="${pkgs.nodejs_22}/bin/node"
# Run org-garden build (org → md → static site)
${orgGardenApp}/bin/org-garden build "$NOTES_DIR" \
--output "$WORK/repo" \
--content-dir "$WORK/repo/content"
# Build the static site from within the repo copy so relative paths
# (e.g. ./package.json in constants.js) resolve correctly.
# --output is absolute so the result lands in the caller's cwd.
cd "$WORK/repo"
node quartz/bootstrap-cli.mjs build \
--directory "$WORK/repo/content" \
--output "$ORIG_CWD/public"
# Copy public output to caller's cwd
cp -r "$WORK/repo/public" "$ORIG_CWD/public"
'';
};
# Development server with watch + live reload
notesApp = pkgs.writeShellApplication {
name = "notes";
runtimeInputs = [ pkgs.nodejs_22 orgGardenApp ];
text = ''
NOTES_DIR="''${1:?Usage: notes <notes-dir>}"
NOTES_DIR=$(realpath "$NOTES_DIR")
# Set up writable working copy
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
cp -r ${self}/. "$WORK/repo"
chmod -R u+w "$WORK/repo"
ln -s ${quartzDeps}/node_modules "$WORK/repo/node_modules"
# Pass paths via environment
export QUARTZ_PATH="$WORK/repo"
export NODE_PATH="${pkgs.nodejs_22}/bin/node"
# org-garden reads these internally
org-garden serve "$NOTES_DIR" \
--output "$WORK/repo" \
--content-dir "$WORK/repo/content"
'';
};
in
@@ -85,9 +110,11 @@
packages.default = buildApp;
packages.build = buildApp;
packages.pipeline = pipelineApp;
packages.notes = notesApp;
packages.org-garden = orgGardenApp;
apps.default = { type = "app"; program = "${buildApp}/bin/build"; };
apps.build = { type = "app"; program = "${buildApp}/bin/build"; };
apps.notes = { type = "app"; program = "${notesApp}/bin/notes"; };
});
}

0
pipeline/flake.lock → org-garden/flake.lock generated
View File

30
pipeline/flake.nix → org-garden/flake.nix
View File

@@ -1,5 +1,5 @@
{
description = "Org-roam export pipeline — Elixir escript";
description = "Org-garden — org-roam to website publishing pipeline";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
@@ -12,7 +12,7 @@
pkgs = import nixpkgs { inherit system; };
fs = pkgs.lib.fileset;
# Emacs with ox-hugo — needed at runtime by the pipeline escript
# Emacs with ox-hugo — needed at runtime by the escript
# (export_org_files calls `emacs --batch` with ox-hugo).
emacsWithOxHugo = (pkgs.emacsPackagesFor pkgs.emacs-nox).emacsWithPackages
(epkgs: [ epkgs.ox-hugo ]);
@@ -21,7 +21,7 @@
# src is filtered to mix.exs + mix.lock so source edits don't
# invalidate this derivation.
mixDeps = pkgs.beamPackages.fetchMixDeps {
pname = "pipeline-mix-deps";
pname = "org-garden-mix-deps";
version = "0.1.0";
src = fs.toSource {
root = ./.;
@@ -33,37 +33,39 @@
sha256 = "sha256-si7JAomY1HZ33m6ihUJP5i6PO39CE1clYvuMtn0CbPU=";
};
# Compiled pipeline escript (without runtime wrappers).
pipelineEscript = pkgs.beamPackages.mixRelease {
pname = "pipeline";
# Compiled org-garden escript (without runtime wrappers).
# Note: escript name is org_garden (from app: :org_garden in mix.exs)
orgGardenEscript = pkgs.beamPackages.mixRelease {
pname = "org-garden";
version = "0.1.0";
src = ./.;
escriptBinName = "pipeline";
escriptBinName = "org_garden";
mixFodDeps = mixDeps;
stripDebug = true;
};
# Wrapped pipeline that puts emacs (with ox-hugo) on PATH so
# Wrapped org-garden that puts emacs (with ox-hugo) on PATH so
# the escript's System.cmd("emacs", ...) calls succeed.
pipelineApp = pkgs.writeShellApplication {
name = "pipeline";
runtimeInputs = [ emacsWithOxHugo pkgs.inotify-tools ];
orgGardenApp = pkgs.writeShellApplication {
name = "org-garden";
runtimeInputs = [ emacsWithOxHugo pkgs.inotify-tools pkgs.nodejs_22 ];
text = ''
exec ${pipelineEscript}/bin/pipeline "$@"
exec ${orgGardenEscript}/bin/org_garden "$@"
'';
};
in
{
packages.default = pipelineApp;
packages.escript = pipelineEscript;
packages.default = orgGardenApp;
packages.escript = orgGardenEscript;
devShells.default = pkgs.mkShell {
buildInputs = [
pkgs.elixir
pkgs.inotify-tools
emacsWithOxHugo
pkgs.nodejs_22
];
};
});

59
pipeline/lib/pipeline.ex → org-garden/lib/org_garden.ex
View File

@@ -1,12 +1,11 @@
defmodule Pipeline do
defmodule OrgGarden do
@moduledoc """
Post-export markdown transformation pipeline.
Org-roam to website publishing pipeline.
Applies a list of transform modules sequentially over markdown files.
Each transform module must implement the `Pipeline.Transform` behaviour.
Transforms are applied in the order given. A file is rewritten only
when at least one transform mutates its content (checked via equality).
Orchestrates:
1. Org → Markdown export (via Emacs + ox-hugo)
2. Markdown transforms (citations, etc.)
3. Markdown → HTML + serving (via Quartz)
## Usage
@@ -17,14 +16,14 @@ defmodule Pipeline do
}
# Batch: all .md files in a directory
Pipeline.run(content_dir, [Pipeline.Transforms.Citations], opts)
OrgGarden.run(content_dir, [OrgGarden.Transforms.Citations], opts)
# Targeted: specific files only
Pipeline.run_on_files(["content/foo.md"], [Pipeline.Transforms.Citations], opts)
OrgGarden.run_on_files(["content/foo.md"], [OrgGarden.Transforms.Citations], opts)
# With pre-initialized transforms (for watch mode, avoids re-init)
initialized = Pipeline.init_transforms([Pipeline.Transforms.Citations], opts)
Pipeline.run_on_files_with(["content/foo.md"], initialized, opts)
initialized = OrgGarden.init_transforms([OrgGarden.Transforms.Citations], opts)
OrgGarden.run_on_files_with(["content/foo.md"], initialized, opts)
"""
require Logger
@@ -33,6 +32,32 @@ defmodule Pipeline do
@type initialized_transform :: {module(), term()}
@type opts :: map()
@doc "One-shot build: org files → static site"
def build(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_build([notes_dir | opts_to_args(opts)])
end
@doc "Development server: watch + live reload"
def serve(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_serve([notes_dir | opts_to_args(opts)])
end
@doc "Export only: org files → markdown (no Quartz)"
def export(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_export([notes_dir | opts_to_args(opts)])
end
defp opts_to_args(opts) do
Enum.flat_map(opts, fn
{:output, v} -> ["--output", v]
{:port, v} -> ["--port", to_string(v)]
{:ws_port, v} -> ["--ws-port", to_string(v)]
{:watch, true} -> ["--watch"]
{:watch, false} -> []
_ -> []
end)
end
@doc """
Initialize transform modules. Returns a list of `{module, state}` tuples.
@@ -75,11 +100,11 @@ defmodule Pipeline do
|> Path.wildcard()
if md_files == [] do
Logger.warning("Pipeline: no .md files found in #{content_dir}")
Logger.warning("OrgGarden: no .md files found in #{content_dir}")
{:ok, %{}}
else
Logger.info(
"Pipeline: processing #{length(md_files)} markdown files " <>
"OrgGarden: processing #{length(md_files)} markdown files " <>
"with #{length(transforms)} transform(s)"
)
@@ -101,10 +126,10 @@ defmodule Pipeline do
existing = Enum.filter(file_paths, &File.exists?/1)
if existing == [] do
Logger.debug("Pipeline: no files to process")
Logger.debug("OrgGarden: no files to process")
{:ok, %{}}
else
Logger.info("Pipeline: processing #{length(existing)} file(s)")
Logger.info("OrgGarden: processing #{length(existing)} file(s)")
initialized = init_transforms(transforms, opts)
stats = apply_transforms(existing, initialized, opts)
teardown_transforms(initialized)
@@ -123,7 +148,7 @@ defmodule Pipeline do
existing = Enum.filter(file_paths, &File.exists?/1)
if existing == [] do
Logger.debug("Pipeline: no files to process")
Logger.debug("OrgGarden: no files to process")
{:ok, %{}}
else
stats = apply_transforms(existing, initialized, opts)
@@ -155,7 +180,7 @@ defmodule Pipeline do
if transformed != original do
File.write!(path, transformed)
Logger.debug("Pipeline: updated #{Path.relative_to_cwd(path)}")
Logger.debug("OrgGarden: updated #{Path.relative_to_cwd(path)}")
end
Map.merge(acc, file_stats, fn _k, a, b -> a + b end)

6
pipeline/lib/pipeline/application.ex → org-garden/lib/org_garden/application.ex
View File

@@ -1,14 +1,14 @@
defmodule Pipeline.Application do
defmodule OrgGarden.Application do
@moduledoc false
use Application
@impl true
def start(_type, _args) do
children = [
{Finch, name: Pipeline.Finch}
{Finch, name: OrgGarden.Finch}
]
opts = [strategy: :one_for_one, name: Pipeline.Supervisor]
opts = [strategy: :one_for_one, name: OrgGarden.AppSupervisor]
Supervisor.start_link(children, opts)
end
end

375
org-garden/lib/org_garden/cli.ex Normal file
View File

@@ -0,0 +1,375 @@
defmodule OrgGarden.CLI do
@moduledoc """
Escript entry point for the org-garden pipeline.
## Commands
org-garden serve <notes-dir> [--port 8080] [--ws-port 3001]
org-garden build <notes-dir> [--output <path>]
org-garden export <notes-dir> [--watch]
### serve
Development server with watch + live reload. Starts both the org→md
watcher and Quartz in serve mode.
### build
One-shot build for CI/production. Exports org files, runs transforms,
then builds static site with Quartz.
### export
Just export org→md (current pipeline behavior). Use --watch for
incremental re-export on file changes.
## Arguments
notes-dir Path to the directory containing `.org` notes (required).
Also accepts the `NOTES_DIR` env var.
## Options
--output <path> Output root directory (used as ox-hugo base dir).
Defaults to the `OUTPUT_DIR` env var, or the current
working directory.
--content-dir <p> Output directory for exported Markdown. Defaults to
`<output>/content`.
--port <n> HTTP server port (default: 8080). Only for `serve`.
--ws-port <n> WebSocket hot reload port (default: 3001). Only for `serve`.
--watch After initial batch, watch notes-dir for changes and
incrementally re-export affected files. Only for `export`.
## Environment Variables
BIBTEX_FILE Path to a `.bib` file used as citation fallback.
ZOTERO_URL Zotero Better BibTeX base URL (default: http://localhost:23119).
CITATION_MODE silent | warn (default) | strict.
QUARTZ_PATH Path to quartz directory (required for serve/build).
NODE_PATH Node.js executable (default: node).
"""
require Logger
@transforms [OrgGarden.Transforms.Citations]
def main(argv) do
Application.ensure_all_started(:org_garden)
case argv do
["serve" | rest] -> handle_serve(rest)
["build" | rest] -> handle_build(rest)
["export" | rest] -> handle_export(rest)
# Legacy: treat bare args as export command for backward compatibility
[_ | _] -> handle_export(argv)
_ -> abort("Usage: org-garden <serve|build|export> <notes-dir> [options]")
end
end
# ---------------------------------------------------------------------------
# Command: serve
# ---------------------------------------------------------------------------
def handle_serve(argv) do
require_quartz_env()
{notes_dir, output_dir, content_dir, opts} = parse_serve_args(argv)
pipeline_opts = build_pipeline_opts()
# Initial batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
IO.puts("==> Starting development server...")
{:ok, _pid} =
OrgGarden.Supervisor.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms,
port: opts[:port] || 8080,
ws_port: opts[:ws_port] || 3001
)
IO.puts("==> Server running at http://localhost:#{opts[:port] || 8080}")
IO.puts("==> Watching #{notes_dir} for changes (Ctrl+C to stop)")
Process.sleep(:infinity)
end
defp parse_serve_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [
output: :string,
content_dir: :string,
port: :integer,
ws_port: :integer
]
)
notes_dir = extract_notes_dir(positional, "serve")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
{notes_dir, output_dir, content_dir, opts}
end
# ---------------------------------------------------------------------------
# Command: build
# ---------------------------------------------------------------------------
def handle_build(argv) do
quartz_path = require_quartz_env()
{notes_dir, output_dir, content_dir, _opts} = parse_build_args(argv)
pipeline_opts = build_pipeline_opts()
# Full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
node_path = System.get_env("NODE_PATH", "node")
IO.puts("==> Building static site with Quartz...")
{output, status} =
System.cmd(
node_path,
[
Path.join(quartz_path, "quartz/bootstrap-cli.mjs"),
"build",
"--directory",
content_dir,
"--output",
Path.join(output_dir, "public")
],
cd: quartz_path,
stderr_to_stdout: true
)
IO.puts(output)
if status != 0 do
abort("Quartz build failed with status #{status}")
end
IO.puts("==> Build complete. Output: #{Path.join(output_dir, "public")}")
end
defp parse_build_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string]
)
notes_dir = extract_notes_dir(positional, "build")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
{notes_dir, output_dir, content_dir, opts}
end
# ---------------------------------------------------------------------------
# Command: export (original pipeline behavior)
# ---------------------------------------------------------------------------
def handle_export(argv) do
{notes_dir, output_dir, content_dir, watch?} = parse_export_args(argv)
pipeline_opts = build_pipeline_opts()
# Phase 1-4: full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
md_count =
content_dir
|> Path.join("**/*.md")
|> Path.wildcard()
|> length()
IO.puts("==> Done. #{md_count} markdown files in #{content_dir}")
# Phase 5: optional watch mode
if watch? do
IO.puts("==> Watching #{notes_dir} for .org changes... (Ctrl+C to stop)")
{:ok, _pid} =
OrgGarden.Watcher.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms
)
Process.sleep(:infinity)
end
end
defp parse_export_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string, watch: :boolean]
)
notes_dir = extract_notes_dir(positional, "export")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
watch? = Keyword.get(opts, :watch, false)
{notes_dir, output_dir, content_dir, watch?}
end
# ---------------------------------------------------------------------------
# Shared argument extraction
# ---------------------------------------------------------------------------
defp extract_notes_dir(positional, command) do
notes_dir =
case positional do
[dir | _] ->
dir
[] ->
System.get_env("NOTES_DIR") ||
abort("Usage: org-garden #{command} <notes-dir> [options]")
end
notes_dir = Path.expand(notes_dir)
unless File.dir?(notes_dir) do
abort("Error: notes directory does not exist: #{notes_dir}")
end
notes_dir
end
defp extract_output_dir(opts) do
(opts[:output] || System.get_env("OUTPUT_DIR") || File.cwd!())
|> Path.expand()
end
defp extract_content_dir(opts, output_dir) do
(opts[:content_dir] || Path.join(output_dir, "content"))
|> Path.expand()
end
# ---------------------------------------------------------------------------
# Phase 1: Wipe content/
# ---------------------------------------------------------------------------
defp wipe(content_dir) do
IO.puts("==> Wiping #{content_dir}")
File.mkdir_p!(content_dir)
content_dir
|> File.ls!()
|> Enum.reject(&(&1 == ".gitkeep"))
|> Enum.each(fn entry ->
Path.join(content_dir, entry) |> File.rm_rf!()
end)
end
# ---------------------------------------------------------------------------
# Phase 2: Export org files via Emacs + ox-hugo
# ---------------------------------------------------------------------------
defp export_all(notes_dir, output_dir) do
IO.puts("==> Exporting org files from #{notes_dir}")
case OrgGarden.Export.export_all(notes_dir, output_dir) do
{:ok, 0} ->
IO.puts("No .org files found in #{notes_dir}")
System.halt(0)
{:ok, count} ->
IO.puts(" exported #{count} file(s)")
{:error, failures} ->
IO.puts(:stderr, "\nFailed to export #{length(failures)} file(s):")
Enum.each(failures, fn {f, {:error, reason}} ->
IO.puts(:stderr, " #{f}: #{inspect(reason)}")
end)
System.halt(1)
end
end
# ---------------------------------------------------------------------------
# Phase 3: Markdown transformation pipeline
# ---------------------------------------------------------------------------
defp run_pipeline(content_dir, pipeline_opts) do
IO.puts("==> Running markdown pipeline")
{:ok, stats} = OrgGarden.run(content_dir, @transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} ->
IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
end)
end
# ---------------------------------------------------------------------------
# Phase 4: Generate default index.md if none was exported
# ---------------------------------------------------------------------------
defp generate_index(content_dir) do
IO.puts("==> Generating index")
OrgGarden.Index.generate(content_dir)
end
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
defp require_quartz_env do
case System.get_env("QUARTZ_PATH") do
nil ->
abort("""
Error: QUARTZ_PATH environment variable not set.
The 'serve' and 'build' commands require Quartz to be available.
Use the wrapper scripts that set up the environment:
nix run .#notes -- <notes-dir> # for serve
nix run .#build -- <notes-dir> # for build
Or set QUARTZ_PATH manually to point to a quartz-org-roam checkout
with node_modules installed.
For export-only mode (no Quartz), use:
org-garden export <notes-dir> [--watch]
""")
path ->
unless File.exists?(Path.join(path, "quartz/bootstrap-cli.mjs")) do
abort("Error: QUARTZ_PATH=#{path} does not contain quartz/bootstrap-cli.mjs")
end
path
end
end
defp build_pipeline_opts do
%{
zotero_url: System.get_env("ZOTERO_URL", "http://localhost:23119"),
bibtex_file: System.get_env("BIBTEX_FILE"),
citation_mode:
case System.get_env("CITATION_MODE", "warn") do
"silent" -> :silent
"strict" -> :strict
_ -> :warn
end
}
end
defp abort(message) do
IO.puts(:stderr, message)
System.halt(1)
end
end

6
pipeline/lib/pipeline/export.ex → org-garden/lib/org_garden/export.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Export do
defmodule OrgGarden.Export do
@moduledoc """
Org-to-Markdown export via Emacs batch + ox-hugo.
@@ -112,10 +112,10 @@ defmodule Pipeline.Export do
## Examples
iex> Pipeline.Export.expected_md_path("/notes/bus/emt.org", "/notes", "/out/content")
iex> OrgGarden.Export.expected_md_path("/notes/bus/emt.org", "/notes", "/out/content")
"/out/content/bus/emt.md"
iex> Pipeline.Export.expected_md_path("/notes/top-level.org", "/notes", "/out/content")
iex> OrgGarden.Export.expected_md_path("/notes/top-level.org", "/notes", "/out/content")
"/out/content/top-level.md"
"""
@spec expected_md_path(String.t(), String.t(), String.t()) :: String.t()

2
pipeline/lib/pipeline/index.ex → org-garden/lib/org_garden/index.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Index do
defmodule OrgGarden.Index do
@moduledoc """
Generates a fallback `index.md` in the content directory if none was
exported from an `.org` file.

118
org-garden/lib/org_garden/quartz.ex Normal file
View File

@@ -0,0 +1,118 @@
defmodule OrgGarden.Quartz do
@moduledoc """
Manages Quartz Node.js process as an Erlang Port.
Required environment:
- QUARTZ_PATH: path to quartz repo (with node_modules)
- NODE_PATH: path to node executable (default: "node")
Starts Quartz in serve mode (`npx quartz build --serve`) and forwards
all stdout/stderr output to the Logger with a `[quartz]` prefix.
If Quartz exits, this GenServer will stop, which triggers the supervisor
to restart the entire supervision tree (strategy: :one_for_all).
"""
use GenServer
require Logger
defstruct [:port, :quartz_path, :content_dir, :http_port, :ws_port]
# -------------------------------------------------------------------
# Client API
# -------------------------------------------------------------------
@doc """
Start the Quartz process as a linked GenServer.
## Options
* `:content_dir` — directory where markdown files are located (required)
* `:port` — HTTP server port (default: 8080)
* `:ws_port` — WebSocket hot reload port (default: 3001)
"""
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
end
# -------------------------------------------------------------------
# GenServer callbacks
# -------------------------------------------------------------------
@impl true
def init(opts) do
quartz_path =
System.get_env("QUARTZ_PATH") ||
raise "QUARTZ_PATH environment variable not set"
node_path = System.get_env("NODE_PATH", "node")
content_dir = Keyword.fetch!(opts, :content_dir)
http_port = Keyword.get(opts, :port, 8080)
ws_port = Keyword.get(opts, :ws_port, 3001)
cli_path = Path.join(quartz_path, "quartz/bootstrap-cli.mjs")
unless File.exists?(cli_path) do
raise "Quartz CLI not found at #{cli_path}. Check QUARTZ_PATH."
end
args = [
cli_path,
"build",
"--serve",
"--directory", content_dir,
"--port", to_string(http_port),
"--wsPort", to_string(ws_port)
]
Logger.info("[quartz] Starting: #{node_path} #{Enum.join(args, " ")}")
Logger.info("[quartz] Working directory: #{quartz_path}")
port =
Port.open({:spawn_executable, node_path}, [
:binary,
:exit_status,
:stderr_to_stdout,
args: args,
cd: quartz_path,
env: [{~c"NODE_NO_WARNINGS", ~c"1"}]
])
state = %__MODULE__{
port: port,
quartz_path: quartz_path,
content_dir: content_dir,
http_port: http_port,
ws_port: ws_port
}
{:ok, state}
end
@impl true
def handle_info({port, {:data, data}}, %{port: port} = state) do
data
|> String.split("\n", trim: true)
|> Enum.each(&Logger.info("[quartz] #{&1}"))
{:noreply, state}
end
@impl true
def handle_info({port, {:exit_status, status}}, %{port: port} = state) do
Logger.error("[quartz] Process exited with status #{status}")
{:stop, {:quartz_exit, status}, state}
end
@impl true
def terminate(_reason, %{port: port}) when is_port(port) do
# Attempt graceful shutdown
Port.close(port)
:ok
rescue
_ -> :ok
end
def terminate(_reason, _state), do: :ok
end

2
pipeline/lib/pipeline/resolvers/bibtex.ex → org-garden/lib/org_garden/resolvers/bibtex.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.BibTeX do
defmodule OrgGarden.Resolvers.BibTeX do
@moduledoc """
Resolves citation keys from a local BibTeX (.bib) file.

2
pipeline/lib/pipeline/resolvers/doi.ex → org-garden/lib/org_garden/resolvers/doi.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.DOI do
defmodule OrgGarden.Resolvers.DOI do
@moduledoc """
Last-resort citation resolver — always succeeds.

6
pipeline/lib/pipeline/resolvers/zotero.ex → org-garden/lib/org_garden/resolvers/zotero.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.Zotero do
defmodule OrgGarden.Resolvers.Zotero do
@moduledoc """
Resolves citation keys via Zotero Better BibTeX's JSON-RPC API.
@@ -39,7 +39,7 @@ defmodule Pipeline.Resolvers.Zotero do
body: payload,
headers: [{"content-type", "application/json"}],
receive_timeout: 5_000,
finch: Pipeline.Finch
finch: OrgGarden.Finch
) do
{:ok, %{status: 200, body: body}} ->
parse_response(body, key, base_url)
@@ -100,7 +100,7 @@ defmodule Pipeline.Resolvers.Zotero do
body: payload,
headers: [{"content-type", "application/json"}],
receive_timeout: 5_000,
finch: Pipeline.Finch
finch: OrgGarden.Finch
) do
{:ok, %{status: 200, body: %{"result" => attachments}}} when is_list(attachments) ->
attachments

40
org-garden/lib/org_garden/supervisor.ex Normal file
View File

@@ -0,0 +1,40 @@
defmodule OrgGarden.Supervisor do
@moduledoc """
Supervises development server components.
Strategy: :one_for_all
If either child fails, restart both to ensure consistent state.
Children:
1. OrgGarden.Watcher - watches .org files for changes
2. OrgGarden.Quartz - runs Quartz Node.js server
## Usage
OrgGarden.Supervisor.start_link(
notes_dir: "/path/to/notes",
output_dir: "/path/to/output",
content_dir: "/path/to/output/content",
pipeline_opts: %{zotero_url: "...", ...},
transforms: [OrgGarden.Transforms.Citations],
port: 8080,
ws_port: 3001
)
"""
use Supervisor
def start_link(opts) do
Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
end
@impl true
def init(opts) do
children = [
{OrgGarden.Watcher,
Keyword.take(opts, [:notes_dir, :output_dir, :content_dir, :pipeline_opts, :transforms])},
{OrgGarden.Quartz, Keyword.take(opts, [:content_dir, :port, :ws_port])}
]
Supervisor.init(children, strategy: :one_for_all)
end
end

8
pipeline/lib/pipeline/transform.ex → org-garden/lib/org_garden/transform.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Transform do
defmodule OrgGarden.Transform do
@moduledoc """
Behaviour that all markdown transform modules must implement.
@@ -12,7 +12,7 @@ defmodule Pipeline.Transform do
## Example
defmodule MyTransform do
@behaviour Pipeline.Transform
@behaviour OrgGarden.Transform
@impl true
def init(opts), do: %{some_state: opts[:value]}
@@ -37,9 +37,9 @@ defmodule Pipeline.Transform do
defmacro __using__(_) do
quote do
@behaviour Pipeline.Transform
@behaviour OrgGarden.Transform
@impl Pipeline.Transform
@impl OrgGarden.Transform
def init(opts), do: opts
defoverridable init: 1

14
pipeline/lib/pipeline/transforms/citations.ex → org-garden/lib/org_garden/transforms/citations.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Transforms.Citations do
defmodule OrgGarden.Transforms.Citations do
@moduledoc """
Markdown transform: resolves org-citar citation keys to hyperlinks.
@@ -37,13 +37,13 @@ defmodule Pipeline.Transforms.Citations do
and probes Zotero availability, emitting warnings as appropriate.
"""
@behaviour Pipeline.Transform
@behaviour OrgGarden.Transform
require Logger
alias Pipeline.Resolvers.Zotero
alias Pipeline.Resolvers.BibTeX
alias Pipeline.Resolvers.DOI
alias OrgGarden.Resolvers.Zotero
alias OrgGarden.Resolvers.BibTeX
alias OrgGarden.Resolvers.DOI
# Match [cite:@key] and [cite:@key1;@key2;...] (org-cite / citar style)
@cite_bracket_regex ~r/\[cite:(@[^\]]+)\]/
@@ -52,7 +52,7 @@ defmodule Pipeline.Transforms.Citations do
@cite_bare_regex ~r/(?<![(\[])cite:@?([a-zA-Z0-9_:-]+)/
# ------------------------------------------------------------------
# Pipeline callbacks
# OrgGarden callbacks
# ------------------------------------------------------------------
@doc """
@@ -202,7 +202,7 @@ defmodule Pipeline.Transforms.Citations do
body: payload,
headers: [{"content-type", "application/json"}],
receive_timeout: 3_000,
finch: Pipeline.Finch
finch: OrgGarden.Finch
)
rescue
e -> {:error, e}

26
pipeline/lib/pipeline/watcher.ex → org-garden/lib/org_garden/watcher.ex
View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Watcher do
defmodule OrgGarden.Watcher do
@moduledoc """
File-watching GenServer that detects `.org` file changes and triggers
incremental export + transform for only the affected files.
@@ -9,18 +9,18 @@ defmodule Pipeline.Watcher do
## Lifecycle
Started dynamically by `Pipeline.CLI` after the initial batch export.
Started dynamically by `OrgGarden.CLI` after the initial batch export.
Transforms are initialized once at startup and reused across all
incremental rebuilds to avoid repeated Zotero probes and BibTeX loads.
## Usage
Pipeline.Watcher.start_link(
OrgGarden.Watcher.start_link(
notes_dir: "/path/to/notes",
output_dir: "/path/to/output",
content_dir: "/path/to/output/content",
pipeline_opts: %{zotero_url: "...", ...},
transforms: [Pipeline.Transforms.Citations]
transforms: [OrgGarden.Transforms.Citations]
)
"""
@@ -43,7 +43,7 @@ defmodule Pipeline.Watcher do
* `:output_dir` — ox-hugo base dir (required)
* `:content_dir` — directory where `.md` files are written (required)
* `:pipeline_opts` — opts map passed to transforms (required)
* `:transforms` — list of transform modules (default: `[Pipeline.Transforms.Citations]`)
* `:transforms` — list of transform modules (default: `[OrgGarden.Transforms.Citations]`)
"""
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
@@ -59,10 +59,10 @@ defmodule Pipeline.Watcher do
output_dir = Keyword.fetch!(opts, :output_dir)
content_dir = Keyword.fetch!(opts, :content_dir)
pipeline_opts = Keyword.fetch!(opts, :pipeline_opts)
transforms = Keyword.get(opts, :transforms, [Pipeline.Transforms.Citations])
transforms = Keyword.get(opts, :transforms, [OrgGarden.Transforms.Citations])
# Initialize transforms once — reused for all incremental rebuilds
initialized_transforms = Pipeline.init_transforms(transforms, pipeline_opts)
initialized_transforms = OrgGarden.init_transforms(transforms, pipeline_opts)
# Start the file system watcher
{:ok, watcher_pid} = FileSystem.start_link(dirs: [notes_dir], recursive: true)
@@ -118,7 +118,7 @@ defmodule Pipeline.Watcher do
@impl true
def terminate(_reason, state) do
Pipeline.teardown_transforms(state.initialized_transforms)
OrgGarden.teardown_transforms(state.initialized_transforms)
:ok
end
@@ -135,14 +135,14 @@ defmodule Pipeline.Watcher do
initialized_transforms: initialized_transforms
} = state
md_path = Pipeline.Export.expected_md_path(orgfile, notes_dir, content_dir)
md_path = OrgGarden.Export.expected_md_path(orgfile, notes_dir, content_dir)
IO.puts("==> Changed: #{Path.relative_to(orgfile, notes_dir)}")
case Pipeline.Export.export_file(orgfile, notes_dir, output_dir) do
case OrgGarden.Export.export_file(orgfile, notes_dir, output_dir) do
{:ok, _} ->
IO.puts(" exported: #{Path.relative_to(md_path, content_dir)}")
{:ok, stats} = Pipeline.run_on_files_with([md_path], initialized_transforms, pipeline_opts)
{:ok, stats} = OrgGarden.run_on_files_with([md_path], initialized_transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} ->
if count > 0, do: IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
@@ -159,7 +159,7 @@ defmodule Pipeline.Watcher do
defp handle_delete(orgfile, state) do
%{notes_dir: notes_dir, content_dir: content_dir} = state
md_path = Pipeline.Export.expected_md_path(orgfile, notes_dir, content_dir)
md_path = OrgGarden.Export.expected_md_path(orgfile, notes_dir, content_dir)
IO.puts("==> Deleted: #{Path.relative_to(orgfile, notes_dir)}")
if File.exists?(md_path) do
@@ -179,7 +179,7 @@ defmodule Pipeline.Watcher do
# -------------------------------------------------------------------
defp regenerate_index(content_dir) do
Pipeline.Index.regenerate(content_dir)
OrgGarden.Index.regenerate(content_dir)
end
# -------------------------------------------------------------------

8
pipeline/mix.exs → org-garden/mix.exs
View File

@@ -1,9 +1,9 @@
defmodule Pipeline.MixProject do
defmodule OrgGarden.MixProject do
use Mix.Project
def project do
[
app: :pipeline,
app: :org_garden,
version: "0.1.0",
elixir: "~> 1.17",
start_permanent: Mix.env() == :prod,
@@ -15,12 +15,12 @@ defmodule Pipeline.MixProject do
def application do
[
extra_applications: [:logger],
mod: {Pipeline.Application, []}
mod: {OrgGarden.Application, []}
]
end
defp escript do
[main_module: Pipeline.CLI]
[main_module: OrgGarden.CLI]
end
defp deps do

0
pipeline/mix.lock → org-garden/mix.lock
View File

204
pipeline/lib/pipeline/cli.ex
View File

@@ -1,204 +0,0 @@
defmodule Pipeline.CLI do
@moduledoc """
Escript entry point for the org-roam export pipeline.
Runs four phases in sequence:
1. Wipe `content/` (preserving `.gitkeep`)
2. Export each `.org` file via `emacs --batch` + ox-hugo -> `content/**/*.md`
3. Run Elixir transform modules over every `.md` file
4. Generate a fallback `content/index.md` if none was exported
With `--watch`, after the initial batch the process stays alive and
incrementally re-exports only changed `.org` files.
## Usage
pipeline <notes-dir> [--output <path>] [--watch]
Arguments:
notes-dir Path to the directory containing `.org` notes (required).
Also accepts the `NOTES_DIR` env var.
Options:
--output <path> Output root directory (used as ox-hugo base dir).
Defaults to the `OUTPUT_DIR` env var, or the current
working directory.
--content-dir <p> Output directory for exported Markdown. Defaults to
`<output>/content`.
--watch After initial batch, watch notes-dir for changes and
incrementally re-export affected files.
Optional env vars:
BIBTEX_FILE Path to a `.bib` file used as citation fallback.
ZOTERO_URL Zotero Better BibTeX base URL (default: http://localhost:23119).
CITATION_MODE silent | warn (default) | strict.
"""
require Logger
@transforms [Pipeline.Transforms.Citations]
def main(argv) do
Application.ensure_all_started(:pipeline)
{notes_dir, output_dir, content_dir, watch?} = parse_args(argv)
pipeline_opts = build_pipeline_opts()
# Phase 1-4: full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
md_count =
content_dir
|> Path.join("**/*.md")
|> Path.wildcard()
|> length()
IO.puts("==> Done. #{md_count} markdown files in #{content_dir}")
# Phase 5: optional watch mode
if watch? do
IO.puts("==> Watching #{notes_dir} for .org changes... (Ctrl+C to stop)")
{:ok, _pid} =
Pipeline.Watcher.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms
)
Process.sleep(:infinity)
end
end
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
defp parse_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string, watch: :boolean]
)
notes_dir =
case positional do
[dir | _] ->
dir
[] ->
System.get_env("NOTES_DIR") ||
abort("Usage: pipeline <notes-dir> [--output <path>] [--watch]")
end
notes_dir = Path.expand(notes_dir)
unless File.dir?(notes_dir) do
abort("Error: notes directory does not exist: #{notes_dir}")
end
output_dir =
(opts[:output] || System.get_env("OUTPUT_DIR") || File.cwd!())
|> Path.expand()
content_dir =
(opts[:content_dir] || Path.join(output_dir, "content"))
|> Path.expand()
watch? = Keyword.get(opts, :watch, false)
{notes_dir, output_dir, content_dir, watch?}
end
# ---------------------------------------------------------------------------
# Phase 1: Wipe content/
# ---------------------------------------------------------------------------
defp wipe(content_dir) do
IO.puts("==> Wiping #{content_dir}")
File.mkdir_p!(content_dir)
content_dir
|> File.ls!()
|> Enum.reject(&(&1 == ".gitkeep"))
|> Enum.each(fn entry ->
Path.join(content_dir, entry) |> File.rm_rf!()
end)
end
# ---------------------------------------------------------------------------
# Phase 2: Export org files via Emacs + ox-hugo
# ---------------------------------------------------------------------------
defp export_all(notes_dir, output_dir) do
IO.puts("==> Exporting org files from #{notes_dir}")
case Pipeline.Export.export_all(notes_dir, output_dir) do
{:ok, 0} ->
IO.puts("No .org files found in #{notes_dir}")
System.halt(0)
{:ok, count} ->
IO.puts(" exported #{count} file(s)")
{:error, failures} ->
IO.puts(:stderr, "\nFailed to export #{length(failures)} file(s):")
Enum.each(failures, fn {f, {:error, reason}} ->
IO.puts(:stderr, " #{f}: #{inspect(reason)}")
end)
System.halt(1)
end
end
# ---------------------------------------------------------------------------
# Phase 3: Markdown transformation pipeline
# ---------------------------------------------------------------------------
defp run_pipeline(content_dir, pipeline_opts) do
IO.puts("==> Running markdown pipeline")
{:ok, stats} = Pipeline.run(content_dir, @transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} ->
IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
end)
end
# ---------------------------------------------------------------------------
# Phase 4: Generate default index.md if none was exported
# ---------------------------------------------------------------------------
defp generate_index(content_dir) do
IO.puts("==> Generating index")
Pipeline.Index.generate(content_dir)
end
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
defp build_pipeline_opts do
%{
zotero_url: System.get_env("ZOTERO_URL", "http://localhost:23119"),
bibtex_file: System.get_env("BIBTEX_FILE"),
citation_mode:
case System.get_env("CITATION_MODE", "warn") do
"silent" -> :silent
"strict" -> :strict
_ -> :warn
end
}
end
defp abort(message) do
IO.puts(:stderr, message)
System.halt(1)
end
end