feat: unified watch server under org-garden

This commit is contained in:
Ignacio Ballesteros
2026-02-21 00:36:31 +01:00
parent 1076bf31ed
commit a4582230b5
21 changed files with 679 additions and 296 deletions

6
.gitignore vendored
View File

@@ -22,6 +22,6 @@ scripts/pipeline/erl_crash.dump
# Test helpers (not needed in production) # Test helpers (not needed in production)
scripts/test.bib scripts/test.bib
scripts/test_pipeline.exs scripts/test_pipeline.exs
/pipeline/deps/ /org-garden/deps/
/pipeline/_build/ /org-garden/_build/
/pipeline/result /org-garden/result

8
flake.lock generated
View File

@@ -68,17 +68,17 @@
"type": "github" "type": "github"
} }
}, },
"pipeline": { "org-garden": {
"inputs": { "inputs": {
"flake-utils": "flake-utils_2", "flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_2" "nixpkgs": "nixpkgs_2"
}, },
"locked": { "locked": {
"path": "./pipeline", "path": "./org-garden",
"type": "path" "type": "path"
}, },
"original": { "original": {
"path": "./pipeline", "path": "./org-garden",
"type": "path" "type": "path"
}, },
"parent": [] "parent": []
@@ -87,7 +87,7 @@
"inputs": { "inputs": {
"flake-utils": "flake-utils", "flake-utils": "flake-utils",
"nixpkgs": "nixpkgs", "nixpkgs": "nixpkgs",
"pipeline": "pipeline" "org-garden": "org-garden"
} }
}, },
"systems": { "systems": {

View File

@@ -4,16 +4,16 @@
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils"; flake-utils.url = "github:numtide/flake-utils";
pipeline.url = "path:./pipeline"; org-garden.url = "path:./org-garden";
}; };
outputs = { self, nixpkgs, flake-utils, pipeline }: outputs = { self, nixpkgs, flake-utils, org-garden }:
flake-utils.lib.eachDefaultSystem (system: flake-utils.lib.eachDefaultSystem (system:
let let
pkgs = import nixpkgs { inherit system; }; pkgs = import nixpkgs { inherit system; };
fs = pkgs.lib.fileset; fs = pkgs.lib.fileset;
pipelineApp = pipeline.packages.${system}.default; orgGardenApp = org-garden.packages.${system}.default;
# Pre-fetched npm dependency tree (node_modules). # Pre-fetched npm dependency tree (node_modules).
# src is filtered to only package.json + package-lock.json so that # src is filtered to only package.json + package-lock.json so that
@@ -36,7 +36,7 @@
''; '';
}; };
# The build application wrapper script # The build application wrapper script (one-shot build)
buildApp = pkgs.writeShellApplication { buildApp = pkgs.writeShellApplication {
name = "build"; name = "build";
runtimeInputs = [ pkgs.nodejs_22 ]; runtimeInputs = [ pkgs.nodejs_22 ];
@@ -54,18 +54,43 @@
# Drop in pre-built node_modules # Drop in pre-built node_modules
ln -s ${quartzDeps}/node_modules "$WORK/repo/node_modules" ln -s ${quartzDeps}/node_modules "$WORK/repo/node_modules"
# Run the pre-compiled pipeline escript (org md, citations transform) # Pass paths via environment for org-garden
${pipelineApp}/bin/pipeline "$NOTES_DIR" \ export QUARTZ_PATH="$WORK/repo"
export NODE_PATH="${pkgs.nodejs_22}/bin/node"
# Run org-garden build (org md static site)
${orgGardenApp}/bin/org-garden build "$NOTES_DIR" \
--output "$WORK/repo" \ --output "$WORK/repo" \
--content-dir "$WORK/repo/content" --content-dir "$WORK/repo/content"
# Build the static site from within the repo copy so relative paths # Copy public output to caller's cwd
# (e.g. ./package.json in constants.js) resolve correctly. cp -r "$WORK/repo/public" "$ORIG_CWD/public"
# --output is absolute so the result lands in the caller's cwd. '';
cd "$WORK/repo" };
node quartz/bootstrap-cli.mjs build \
--directory "$WORK/repo/content" \ # Development server with watch + live reload
--output "$ORIG_CWD/public" notesApp = pkgs.writeShellApplication {
name = "notes";
runtimeInputs = [ pkgs.nodejs_22 orgGardenApp ];
text = ''
NOTES_DIR="''${1:?Usage: notes <notes-dir>}"
NOTES_DIR=$(realpath "$NOTES_DIR")
# Set up writable working copy
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
cp -r ${self}/. "$WORK/repo"
chmod -R u+w "$WORK/repo"
ln -s ${quartzDeps}/node_modules "$WORK/repo/node_modules"
# Pass paths via environment
export QUARTZ_PATH="$WORK/repo"
export NODE_PATH="${pkgs.nodejs_22}/bin/node"
# org-garden reads these internally
org-garden serve "$NOTES_DIR" \
--output "$WORK/repo" \
--content-dir "$WORK/repo/content"
''; '';
}; };
in in
@@ -85,9 +110,11 @@
packages.default = buildApp; packages.default = buildApp;
packages.build = buildApp; packages.build = buildApp;
packages.pipeline = pipelineApp; packages.notes = notesApp;
packages.org-garden = orgGardenApp;
apps.default = { type = "app"; program = "${buildApp}/bin/build"; }; apps.default = { type = "app"; program = "${buildApp}/bin/build"; };
apps.build = { type = "app"; program = "${buildApp}/bin/build"; }; apps.build = { type = "app"; program = "${buildApp}/bin/build"; };
apps.notes = { type = "app"; program = "${notesApp}/bin/notes"; };
}); });
} }

View File

@@ -1,5 +1,5 @@
{ {
description = "Org-roam export pipeline Elixir escript"; description = "Org-garden org-roam to website publishing pipeline";
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
@@ -12,7 +12,7 @@
pkgs = import nixpkgs { inherit system; }; pkgs = import nixpkgs { inherit system; };
fs = pkgs.lib.fileset; fs = pkgs.lib.fileset;
# Emacs with ox-hugo — needed at runtime by the pipeline escript # Emacs with ox-hugo — needed at runtime by the escript
# (export_org_files calls `emacs --batch` with ox-hugo). # (export_org_files calls `emacs --batch` with ox-hugo).
emacsWithOxHugo = (pkgs.emacsPackagesFor pkgs.emacs-nox).emacsWithPackages emacsWithOxHugo = (pkgs.emacsPackagesFor pkgs.emacs-nox).emacsWithPackages
(epkgs: [ epkgs.ox-hugo ]); (epkgs: [ epkgs.ox-hugo ]);
@@ -21,7 +21,7 @@
# src is filtered to mix.exs + mix.lock so source edits don't # src is filtered to mix.exs + mix.lock so source edits don't
# invalidate this derivation. # invalidate this derivation.
mixDeps = pkgs.beamPackages.fetchMixDeps { mixDeps = pkgs.beamPackages.fetchMixDeps {
pname = "pipeline-mix-deps"; pname = "org-garden-mix-deps";
version = "0.1.0"; version = "0.1.0";
src = fs.toSource { src = fs.toSource {
root = ./.; root = ./.;
@@ -33,37 +33,39 @@
sha256 = "sha256-si7JAomY1HZ33m6ihUJP5i6PO39CE1clYvuMtn0CbPU="; sha256 = "sha256-si7JAomY1HZ33m6ihUJP5i6PO39CE1clYvuMtn0CbPU=";
}; };
# Compiled pipeline escript (without runtime wrappers). # Compiled org-garden escript (without runtime wrappers).
pipelineEscript = pkgs.beamPackages.mixRelease { # Note: escript name is org_garden (from app: :org_garden in mix.exs)
pname = "pipeline"; orgGardenEscript = pkgs.beamPackages.mixRelease {
pname = "org-garden";
version = "0.1.0"; version = "0.1.0";
src = ./.; src = ./.;
escriptBinName = "pipeline"; escriptBinName = "org_garden";
mixFodDeps = mixDeps; mixFodDeps = mixDeps;
stripDebug = true; stripDebug = true;
}; };
# Wrapped pipeline that puts emacs (with ox-hugo) on PATH so # Wrapped org-garden that puts emacs (with ox-hugo) on PATH so
# the escript's System.cmd("emacs", ...) calls succeed. # the escript's System.cmd("emacs", ...) calls succeed.
pipelineApp = pkgs.writeShellApplication { orgGardenApp = pkgs.writeShellApplication {
name = "pipeline"; name = "org-garden";
runtimeInputs = [ emacsWithOxHugo pkgs.inotify-tools ]; runtimeInputs = [ emacsWithOxHugo pkgs.inotify-tools pkgs.nodejs_22 ];
text = '' text = ''
exec ${pipelineEscript}/bin/pipeline "$@" exec ${orgGardenEscript}/bin/org_garden "$@"
''; '';
}; };
in in
{ {
packages.default = pipelineApp; packages.default = orgGardenApp;
packages.escript = pipelineEscript; packages.escript = orgGardenEscript;
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
buildInputs = [ buildInputs = [
pkgs.elixir pkgs.elixir
pkgs.inotify-tools pkgs.inotify-tools
emacsWithOxHugo emacsWithOxHugo
pkgs.nodejs_22
]; ];
}; };
}); });

View File

@@ -1,12 +1,11 @@
defmodule Pipeline do defmodule OrgGarden do
@moduledoc """ @moduledoc """
Post-export markdown transformation pipeline. Org-roam to website publishing pipeline.
Applies a list of transform modules sequentially over markdown files. Orchestrates:
Each transform module must implement the `Pipeline.Transform` behaviour. 1. Org Markdown export (via Emacs + ox-hugo)
2. Markdown transforms (citations, etc.)
Transforms are applied in the order given. A file is rewritten only 3. Markdown HTML + serving (via Quartz)
when at least one transform mutates its content (checked via equality).
## Usage ## Usage
@@ -17,14 +16,14 @@ defmodule Pipeline do
} }
# Batch: all .md files in a directory # Batch: all .md files in a directory
Pipeline.run(content_dir, [Pipeline.Transforms.Citations], opts) OrgGarden.run(content_dir, [OrgGarden.Transforms.Citations], opts)
# Targeted: specific files only # Targeted: specific files only
Pipeline.run_on_files(["content/foo.md"], [Pipeline.Transforms.Citations], opts) OrgGarden.run_on_files(["content/foo.md"], [OrgGarden.Transforms.Citations], opts)
# With pre-initialized transforms (for watch mode, avoids re-init) # With pre-initialized transforms (for watch mode, avoids re-init)
initialized = Pipeline.init_transforms([Pipeline.Transforms.Citations], opts) initialized = OrgGarden.init_transforms([OrgGarden.Transforms.Citations], opts)
Pipeline.run_on_files_with(["content/foo.md"], initialized, opts) OrgGarden.run_on_files_with(["content/foo.md"], initialized, opts)
""" """
require Logger require Logger
@@ -33,6 +32,32 @@ defmodule Pipeline do
@type initialized_transform :: {module(), term()} @type initialized_transform :: {module(), term()}
@type opts :: map() @type opts :: map()
@doc "One-shot build: org files → static site"
def build(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_build([notes_dir | opts_to_args(opts)])
end
@doc "Development server: watch + live reload"
def serve(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_serve([notes_dir | opts_to_args(opts)])
end
@doc "Export only: org files → markdown (no Quartz)"
def export(notes_dir, opts \\ []) do
OrgGarden.CLI.handle_export([notes_dir | opts_to_args(opts)])
end
defp opts_to_args(opts) do
Enum.flat_map(opts, fn
{:output, v} -> ["--output", v]
{:port, v} -> ["--port", to_string(v)]
{:ws_port, v} -> ["--ws-port", to_string(v)]
{:watch, true} -> ["--watch"]
{:watch, false} -> []
_ -> []
end)
end
@doc """ @doc """
Initialize transform modules. Returns a list of `{module, state}` tuples. Initialize transform modules. Returns a list of `{module, state}` tuples.
@@ -75,11 +100,11 @@ defmodule Pipeline do
|> Path.wildcard() |> Path.wildcard()
if md_files == [] do if md_files == [] do
Logger.warning("Pipeline: no .md files found in #{content_dir}") Logger.warning("OrgGarden: no .md files found in #{content_dir}")
{:ok, %{}} {:ok, %{}}
else else
Logger.info( Logger.info(
"Pipeline: processing #{length(md_files)} markdown files " <> "OrgGarden: processing #{length(md_files)} markdown files " <>
"with #{length(transforms)} transform(s)" "with #{length(transforms)} transform(s)"
) )
@@ -101,10 +126,10 @@ defmodule Pipeline do
existing = Enum.filter(file_paths, &File.exists?/1) existing = Enum.filter(file_paths, &File.exists?/1)
if existing == [] do if existing == [] do
Logger.debug("Pipeline: no files to process") Logger.debug("OrgGarden: no files to process")
{:ok, %{}} {:ok, %{}}
else else
Logger.info("Pipeline: processing #{length(existing)} file(s)") Logger.info("OrgGarden: processing #{length(existing)} file(s)")
initialized = init_transforms(transforms, opts) initialized = init_transforms(transforms, opts)
stats = apply_transforms(existing, initialized, opts) stats = apply_transforms(existing, initialized, opts)
teardown_transforms(initialized) teardown_transforms(initialized)
@@ -123,7 +148,7 @@ defmodule Pipeline do
existing = Enum.filter(file_paths, &File.exists?/1) existing = Enum.filter(file_paths, &File.exists?/1)
if existing == [] do if existing == [] do
Logger.debug("Pipeline: no files to process") Logger.debug("OrgGarden: no files to process")
{:ok, %{}} {:ok, %{}}
else else
stats = apply_transforms(existing, initialized, opts) stats = apply_transforms(existing, initialized, opts)
@@ -155,7 +180,7 @@ defmodule Pipeline do
if transformed != original do if transformed != original do
File.write!(path, transformed) File.write!(path, transformed)
Logger.debug("Pipeline: updated #{Path.relative_to_cwd(path)}") Logger.debug("OrgGarden: updated #{Path.relative_to_cwd(path)}")
end end
Map.merge(acc, file_stats, fn _k, a, b -> a + b end) Map.merge(acc, file_stats, fn _k, a, b -> a + b end)

View File

@@ -1,14 +1,14 @@
defmodule Pipeline.Application do defmodule OrgGarden.Application do
@moduledoc false @moduledoc false
use Application use Application
@impl true @impl true
def start(_type, _args) do def start(_type, _args) do
children = [ children = [
{Finch, name: Pipeline.Finch} {Finch, name: OrgGarden.Finch}
] ]
opts = [strategy: :one_for_one, name: Pipeline.Supervisor] opts = [strategy: :one_for_one, name: OrgGarden.AppSupervisor]
Supervisor.start_link(children, opts) Supervisor.start_link(children, opts)
end end
end end

View File

@@ -0,0 +1,375 @@
defmodule OrgGarden.CLI do
@moduledoc """
Escript entry point for the org-garden pipeline.
## Commands
org-garden serve <notes-dir> [--port 8080] [--ws-port 3001]
org-garden build <notes-dir> [--output <path>]
org-garden export <notes-dir> [--watch]
### serve
Development server with watch + live reload. Starts both the org→md
watcher and Quartz in serve mode.
### build
One-shot build for CI/production. Exports org files, runs transforms,
then builds static site with Quartz.
### export
Just export org→md (current pipeline behavior). Use --watch for
incremental re-export on file changes.
## Arguments
notes-dir Path to the directory containing `.org` notes (required).
Also accepts the `NOTES_DIR` env var.
## Options
--output <path> Output root directory (used as ox-hugo base dir).
Defaults to the `OUTPUT_DIR` env var, or the current
working directory.
--content-dir <p> Output directory for exported Markdown. Defaults to
`<output>/content`.
--port <n> HTTP server port (default: 8080). Only for `serve`.
--ws-port <n> WebSocket hot reload port (default: 3001). Only for `serve`.
--watch After initial batch, watch notes-dir for changes and
incrementally re-export affected files. Only for `export`.
## Environment Variables
BIBTEX_FILE Path to a `.bib` file used as citation fallback.
ZOTERO_URL Zotero Better BibTeX base URL (default: http://localhost:23119).
CITATION_MODE silent | warn (default) | strict.
QUARTZ_PATH Path to quartz directory (required for serve/build).
NODE_PATH Node.js executable (default: node).
"""
require Logger
@transforms [OrgGarden.Transforms.Citations]
def main(argv) do
Application.ensure_all_started(:org_garden)
case argv do
["serve" | rest] -> handle_serve(rest)
["build" | rest] -> handle_build(rest)
["export" | rest] -> handle_export(rest)
# Legacy: treat bare args as export command for backward compatibility
[_ | _] -> handle_export(argv)
_ -> abort("Usage: org-garden <serve|build|export> <notes-dir> [options]")
end
end
# ---------------------------------------------------------------------------
# Command: serve
# ---------------------------------------------------------------------------
def handle_serve(argv) do
require_quartz_env()
{notes_dir, output_dir, content_dir, opts} = parse_serve_args(argv)
pipeline_opts = build_pipeline_opts()
# Initial batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
IO.puts("==> Starting development server...")
{:ok, _pid} =
OrgGarden.Supervisor.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms,
port: opts[:port] || 8080,
ws_port: opts[:ws_port] || 3001
)
IO.puts("==> Server running at http://localhost:#{opts[:port] || 8080}")
IO.puts("==> Watching #{notes_dir} for changes (Ctrl+C to stop)")
Process.sleep(:infinity)
end
defp parse_serve_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [
output: :string,
content_dir: :string,
port: :integer,
ws_port: :integer
]
)
notes_dir = extract_notes_dir(positional, "serve")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
{notes_dir, output_dir, content_dir, opts}
end
# ---------------------------------------------------------------------------
# Command: build
# ---------------------------------------------------------------------------
def handle_build(argv) do
quartz_path = require_quartz_env()
{notes_dir, output_dir, content_dir, _opts} = parse_build_args(argv)
pipeline_opts = build_pipeline_opts()
# Full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
node_path = System.get_env("NODE_PATH", "node")
IO.puts("==> Building static site with Quartz...")
{output, status} =
System.cmd(
node_path,
[
Path.join(quartz_path, "quartz/bootstrap-cli.mjs"),
"build",
"--directory",
content_dir,
"--output",
Path.join(output_dir, "public")
],
cd: quartz_path,
stderr_to_stdout: true
)
IO.puts(output)
if status != 0 do
abort("Quartz build failed with status #{status}")
end
IO.puts("==> Build complete. Output: #{Path.join(output_dir, "public")}")
end
defp parse_build_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string]
)
notes_dir = extract_notes_dir(positional, "build")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
{notes_dir, output_dir, content_dir, opts}
end
# ---------------------------------------------------------------------------
# Command: export (original pipeline behavior)
# ---------------------------------------------------------------------------
def handle_export(argv) do
{notes_dir, output_dir, content_dir, watch?} = parse_export_args(argv)
pipeline_opts = build_pipeline_opts()
# Phase 1-4: full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
md_count =
content_dir
|> Path.join("**/*.md")
|> Path.wildcard()
|> length()
IO.puts("==> Done. #{md_count} markdown files in #{content_dir}")
# Phase 5: optional watch mode
if watch? do
IO.puts("==> Watching #{notes_dir} for .org changes... (Ctrl+C to stop)")
{:ok, _pid} =
OrgGarden.Watcher.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms
)
Process.sleep(:infinity)
end
end
defp parse_export_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string, watch: :boolean]
)
notes_dir = extract_notes_dir(positional, "export")
output_dir = extract_output_dir(opts)
content_dir = extract_content_dir(opts, output_dir)
watch? = Keyword.get(opts, :watch, false)
{notes_dir, output_dir, content_dir, watch?}
end
# ---------------------------------------------------------------------------
# Shared argument extraction
# ---------------------------------------------------------------------------
defp extract_notes_dir(positional, command) do
notes_dir =
case positional do
[dir | _] ->
dir
[] ->
System.get_env("NOTES_DIR") ||
abort("Usage: org-garden #{command} <notes-dir> [options]")
end
notes_dir = Path.expand(notes_dir)
unless File.dir?(notes_dir) do
abort("Error: notes directory does not exist: #{notes_dir}")
end
notes_dir
end
defp extract_output_dir(opts) do
(opts[:output] || System.get_env("OUTPUT_DIR") || File.cwd!())
|> Path.expand()
end
defp extract_content_dir(opts, output_dir) do
(opts[:content_dir] || Path.join(output_dir, "content"))
|> Path.expand()
end
# ---------------------------------------------------------------------------
# Phase 1: Wipe content/
# ---------------------------------------------------------------------------
defp wipe(content_dir) do
IO.puts("==> Wiping #{content_dir}")
File.mkdir_p!(content_dir)
content_dir
|> File.ls!()
|> Enum.reject(&(&1 == ".gitkeep"))
|> Enum.each(fn entry ->
Path.join(content_dir, entry) |> File.rm_rf!()
end)
end
# ---------------------------------------------------------------------------
# Phase 2: Export org files via Emacs + ox-hugo
# ---------------------------------------------------------------------------
defp export_all(notes_dir, output_dir) do
IO.puts("==> Exporting org files from #{notes_dir}")
case OrgGarden.Export.export_all(notes_dir, output_dir) do
{:ok, 0} ->
IO.puts("No .org files found in #{notes_dir}")
System.halt(0)
{:ok, count} ->
IO.puts(" exported #{count} file(s)")
{:error, failures} ->
IO.puts(:stderr, "\nFailed to export #{length(failures)} file(s):")
Enum.each(failures, fn {f, {:error, reason}} ->
IO.puts(:stderr, " #{f}: #{inspect(reason)}")
end)
System.halt(1)
end
end
# ---------------------------------------------------------------------------
# Phase 3: Markdown transformation pipeline
# ---------------------------------------------------------------------------
defp run_pipeline(content_dir, pipeline_opts) do
IO.puts("==> Running markdown pipeline")
{:ok, stats} = OrgGarden.run(content_dir, @transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} ->
IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
end)
end
# ---------------------------------------------------------------------------
# Phase 4: Generate default index.md if none was exported
# ---------------------------------------------------------------------------
defp generate_index(content_dir) do
IO.puts("==> Generating index")
OrgGarden.Index.generate(content_dir)
end
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
defp require_quartz_env do
case System.get_env("QUARTZ_PATH") do
nil ->
abort("""
Error: QUARTZ_PATH environment variable not set.
The 'serve' and 'build' commands require Quartz to be available.
Use the wrapper scripts that set up the environment:
nix run .#notes -- <notes-dir> # for serve
nix run .#build -- <notes-dir> # for build
Or set QUARTZ_PATH manually to point to a quartz-org-roam checkout
with node_modules installed.
For export-only mode (no Quartz), use:
org-garden export <notes-dir> [--watch]
""")
path ->
unless File.exists?(Path.join(path, "quartz/bootstrap-cli.mjs")) do
abort("Error: QUARTZ_PATH=#{path} does not contain quartz/bootstrap-cli.mjs")
end
path
end
end
defp build_pipeline_opts do
%{
zotero_url: System.get_env("ZOTERO_URL", "http://localhost:23119"),
bibtex_file: System.get_env("BIBTEX_FILE"),
citation_mode:
case System.get_env("CITATION_MODE", "warn") do
"silent" -> :silent
"strict" -> :strict
_ -> :warn
end
}
end
defp abort(message) do
IO.puts(:stderr, message)
System.halt(1)
end
end

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Export do defmodule OrgGarden.Export do
@moduledoc """ @moduledoc """
Org-to-Markdown export via Emacs batch + ox-hugo. Org-to-Markdown export via Emacs batch + ox-hugo.
@@ -112,10 +112,10 @@ defmodule Pipeline.Export do
## Examples ## Examples
iex> Pipeline.Export.expected_md_path("/notes/bus/emt.org", "/notes", "/out/content") iex> OrgGarden.Export.expected_md_path("/notes/bus/emt.org", "/notes", "/out/content")
"/out/content/bus/emt.md" "/out/content/bus/emt.md"
iex> Pipeline.Export.expected_md_path("/notes/top-level.org", "/notes", "/out/content") iex> OrgGarden.Export.expected_md_path("/notes/top-level.org", "/notes", "/out/content")
"/out/content/top-level.md" "/out/content/top-level.md"
""" """
@spec expected_md_path(String.t(), String.t(), String.t()) :: String.t() @spec expected_md_path(String.t(), String.t(), String.t()) :: String.t()

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Index do defmodule OrgGarden.Index do
@moduledoc """ @moduledoc """
Generates a fallback `index.md` in the content directory if none was Generates a fallback `index.md` in the content directory if none was
exported from an `.org` file. exported from an `.org` file.

View File

@@ -0,0 +1,118 @@
defmodule OrgGarden.Quartz do
@moduledoc """
Manages Quartz Node.js process as an Erlang Port.
Required environment:
- QUARTZ_PATH: path to quartz repo (with node_modules)
- NODE_PATH: path to node executable (default: "node")
Starts Quartz in serve mode (`npx quartz build --serve`) and forwards
all stdout/stderr output to the Logger with a `[quartz]` prefix.
If Quartz exits, this GenServer will stop, which triggers the supervisor
to restart the entire supervision tree (strategy: :one_for_all).
"""
use GenServer
require Logger
defstruct [:port, :quartz_path, :content_dir, :http_port, :ws_port]
# -------------------------------------------------------------------
# Client API
# -------------------------------------------------------------------
@doc """
Start the Quartz process as a linked GenServer.
## Options
* `:content_dir` — directory where markdown files are located (required)
* `:port` — HTTP server port (default: 8080)
* `:ws_port` — WebSocket hot reload port (default: 3001)
"""
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
end
# -------------------------------------------------------------------
# GenServer callbacks
# -------------------------------------------------------------------
@impl true
def init(opts) do
quartz_path =
System.get_env("QUARTZ_PATH") ||
raise "QUARTZ_PATH environment variable not set"
node_path = System.get_env("NODE_PATH", "node")
content_dir = Keyword.fetch!(opts, :content_dir)
http_port = Keyword.get(opts, :port, 8080)
ws_port = Keyword.get(opts, :ws_port, 3001)
cli_path = Path.join(quartz_path, "quartz/bootstrap-cli.mjs")
unless File.exists?(cli_path) do
raise "Quartz CLI not found at #{cli_path}. Check QUARTZ_PATH."
end
args = [
cli_path,
"build",
"--serve",
"--directory", content_dir,
"--port", to_string(http_port),
"--wsPort", to_string(ws_port)
]
Logger.info("[quartz] Starting: #{node_path} #{Enum.join(args, " ")}")
Logger.info("[quartz] Working directory: #{quartz_path}")
port =
Port.open({:spawn_executable, node_path}, [
:binary,
:exit_status,
:stderr_to_stdout,
args: args,
cd: quartz_path,
env: [{~c"NODE_NO_WARNINGS", ~c"1"}]
])
state = %__MODULE__{
port: port,
quartz_path: quartz_path,
content_dir: content_dir,
http_port: http_port,
ws_port: ws_port
}
{:ok, state}
end
@impl true
def handle_info({port, {:data, data}}, %{port: port} = state) do
data
|> String.split("\n", trim: true)
|> Enum.each(&Logger.info("[quartz] #{&1}"))
{:noreply, state}
end
@impl true
def handle_info({port, {:exit_status, status}}, %{port: port} = state) do
Logger.error("[quartz] Process exited with status #{status}")
{:stop, {:quartz_exit, status}, state}
end
@impl true
def terminate(_reason, %{port: port}) when is_port(port) do
# Attempt graceful shutdown
Port.close(port)
:ok
rescue
_ -> :ok
end
def terminate(_reason, _state), do: :ok
end

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.BibTeX do defmodule OrgGarden.Resolvers.BibTeX do
@moduledoc """ @moduledoc """
Resolves citation keys from a local BibTeX (.bib) file. Resolves citation keys from a local BibTeX (.bib) file.

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.DOI do defmodule OrgGarden.Resolvers.DOI do
@moduledoc """ @moduledoc """
Last-resort citation resolver always succeeds. Last-resort citation resolver always succeeds.

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Resolvers.Zotero do defmodule OrgGarden.Resolvers.Zotero do
@moduledoc """ @moduledoc """
Resolves citation keys via Zotero Better BibTeX's JSON-RPC API. Resolves citation keys via Zotero Better BibTeX's JSON-RPC API.
@@ -39,7 +39,7 @@ defmodule Pipeline.Resolvers.Zotero do
body: payload, body: payload,
headers: [{"content-type", "application/json"}], headers: [{"content-type", "application/json"}],
receive_timeout: 5_000, receive_timeout: 5_000,
finch: Pipeline.Finch finch: OrgGarden.Finch
) do ) do
{:ok, %{status: 200, body: body}} -> {:ok, %{status: 200, body: body}} ->
parse_response(body, key, base_url) parse_response(body, key, base_url)
@@ -100,7 +100,7 @@ defmodule Pipeline.Resolvers.Zotero do
body: payload, body: payload,
headers: [{"content-type", "application/json"}], headers: [{"content-type", "application/json"}],
receive_timeout: 5_000, receive_timeout: 5_000,
finch: Pipeline.Finch finch: OrgGarden.Finch
) do ) do
{:ok, %{status: 200, body: %{"result" => attachments}}} when is_list(attachments) -> {:ok, %{status: 200, body: %{"result" => attachments}}} when is_list(attachments) ->
attachments attachments

View File

@@ -0,0 +1,40 @@
defmodule OrgGarden.Supervisor do
@moduledoc """
Supervises development server components.
Strategy: :one_for_all
If either child fails, restart both to ensure consistent state.
Children:
1. OrgGarden.Watcher - watches .org files for changes
2. OrgGarden.Quartz - runs Quartz Node.js server
## Usage
OrgGarden.Supervisor.start_link(
notes_dir: "/path/to/notes",
output_dir: "/path/to/output",
content_dir: "/path/to/output/content",
pipeline_opts: %{zotero_url: "...", ...},
transforms: [OrgGarden.Transforms.Citations],
port: 8080,
ws_port: 3001
)
"""
use Supervisor
def start_link(opts) do
Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
end
@impl true
def init(opts) do
children = [
{OrgGarden.Watcher,
Keyword.take(opts, [:notes_dir, :output_dir, :content_dir, :pipeline_opts, :transforms])},
{OrgGarden.Quartz, Keyword.take(opts, [:content_dir, :port, :ws_port])}
]
Supervisor.init(children, strategy: :one_for_all)
end
end

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Transform do defmodule OrgGarden.Transform do
@moduledoc """ @moduledoc """
Behaviour that all markdown transform modules must implement. Behaviour that all markdown transform modules must implement.
@@ -12,7 +12,7 @@ defmodule Pipeline.Transform do
## Example ## Example
defmodule MyTransform do defmodule MyTransform do
@behaviour Pipeline.Transform @behaviour OrgGarden.Transform
@impl true @impl true
def init(opts), do: %{some_state: opts[:value]} def init(opts), do: %{some_state: opts[:value]}
@@ -37,9 +37,9 @@ defmodule Pipeline.Transform do
defmacro __using__(_) do defmacro __using__(_) do
quote do quote do
@behaviour Pipeline.Transform @behaviour OrgGarden.Transform
@impl Pipeline.Transform @impl OrgGarden.Transform
def init(opts), do: opts def init(opts), do: opts
defoverridable init: 1 defoverridable init: 1

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Transforms.Citations do defmodule OrgGarden.Transforms.Citations do
@moduledoc """ @moduledoc """
Markdown transform: resolves org-citar citation keys to hyperlinks. Markdown transform: resolves org-citar citation keys to hyperlinks.
@@ -37,13 +37,13 @@ defmodule Pipeline.Transforms.Citations do
and probes Zotero availability, emitting warnings as appropriate. and probes Zotero availability, emitting warnings as appropriate.
""" """
@behaviour Pipeline.Transform @behaviour OrgGarden.Transform
require Logger require Logger
alias Pipeline.Resolvers.Zotero alias OrgGarden.Resolvers.Zotero
alias Pipeline.Resolvers.BibTeX alias OrgGarden.Resolvers.BibTeX
alias Pipeline.Resolvers.DOI alias OrgGarden.Resolvers.DOI
# Match [cite:@key] and [cite:@key1;@key2;...] (org-cite / citar style) # Match [cite:@key] and [cite:@key1;@key2;...] (org-cite / citar style)
@cite_bracket_regex ~r/\[cite:(@[^\]]+)\]/ @cite_bracket_regex ~r/\[cite:(@[^\]]+)\]/
@@ -52,7 +52,7 @@ defmodule Pipeline.Transforms.Citations do
@cite_bare_regex ~r/(?<![(\[])cite:@?([a-zA-Z0-9_:-]+)/ @cite_bare_regex ~r/(?<![(\[])cite:@?([a-zA-Z0-9_:-]+)/
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Pipeline callbacks # OrgGarden callbacks
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@doc """ @doc """
@@ -202,7 +202,7 @@ defmodule Pipeline.Transforms.Citations do
body: payload, body: payload,
headers: [{"content-type", "application/json"}], headers: [{"content-type", "application/json"}],
receive_timeout: 3_000, receive_timeout: 3_000,
finch: Pipeline.Finch finch: OrgGarden.Finch
) )
rescue rescue
e -> {:error, e} e -> {:error, e}

View File

@@ -1,4 +1,4 @@
defmodule Pipeline.Watcher do defmodule OrgGarden.Watcher do
@moduledoc """ @moduledoc """
File-watching GenServer that detects `.org` file changes and triggers File-watching GenServer that detects `.org` file changes and triggers
incremental export + transform for only the affected files. incremental export + transform for only the affected files.
@@ -9,18 +9,18 @@ defmodule Pipeline.Watcher do
## Lifecycle ## Lifecycle
Started dynamically by `Pipeline.CLI` after the initial batch export. Started dynamically by `OrgGarden.CLI` after the initial batch export.
Transforms are initialized once at startup and reused across all Transforms are initialized once at startup and reused across all
incremental rebuilds to avoid repeated Zotero probes and BibTeX loads. incremental rebuilds to avoid repeated Zotero probes and BibTeX loads.
## Usage ## Usage
Pipeline.Watcher.start_link( OrgGarden.Watcher.start_link(
notes_dir: "/path/to/notes", notes_dir: "/path/to/notes",
output_dir: "/path/to/output", output_dir: "/path/to/output",
content_dir: "/path/to/output/content", content_dir: "/path/to/output/content",
pipeline_opts: %{zotero_url: "...", ...}, pipeline_opts: %{zotero_url: "...", ...},
transforms: [Pipeline.Transforms.Citations] transforms: [OrgGarden.Transforms.Citations]
) )
""" """
@@ -43,7 +43,7 @@ defmodule Pipeline.Watcher do
* `:output_dir` ox-hugo base dir (required) * `:output_dir` ox-hugo base dir (required)
* `:content_dir` directory where `.md` files are written (required) * `:content_dir` directory where `.md` files are written (required)
* `:pipeline_opts` opts map passed to transforms (required) * `:pipeline_opts` opts map passed to transforms (required)
* `:transforms` list of transform modules (default: `[Pipeline.Transforms.Citations]`) * `:transforms` list of transform modules (default: `[OrgGarden.Transforms.Citations]`)
""" """
def start_link(opts) do def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__) GenServer.start_link(__MODULE__, opts, name: __MODULE__)
@@ -59,10 +59,10 @@ defmodule Pipeline.Watcher do
output_dir = Keyword.fetch!(opts, :output_dir) output_dir = Keyword.fetch!(opts, :output_dir)
content_dir = Keyword.fetch!(opts, :content_dir) content_dir = Keyword.fetch!(opts, :content_dir)
pipeline_opts = Keyword.fetch!(opts, :pipeline_opts) pipeline_opts = Keyword.fetch!(opts, :pipeline_opts)
transforms = Keyword.get(opts, :transforms, [Pipeline.Transforms.Citations]) transforms = Keyword.get(opts, :transforms, [OrgGarden.Transforms.Citations])
# Initialize transforms once — reused for all incremental rebuilds # Initialize transforms once — reused for all incremental rebuilds
initialized_transforms = Pipeline.init_transforms(transforms, pipeline_opts) initialized_transforms = OrgGarden.init_transforms(transforms, pipeline_opts)
# Start the file system watcher # Start the file system watcher
{:ok, watcher_pid} = FileSystem.start_link(dirs: [notes_dir], recursive: true) {:ok, watcher_pid} = FileSystem.start_link(dirs: [notes_dir], recursive: true)
@@ -118,7 +118,7 @@ defmodule Pipeline.Watcher do
@impl true @impl true
def terminate(_reason, state) do def terminate(_reason, state) do
Pipeline.teardown_transforms(state.initialized_transforms) OrgGarden.teardown_transforms(state.initialized_transforms)
:ok :ok
end end
@@ -135,14 +135,14 @@ defmodule Pipeline.Watcher do
initialized_transforms: initialized_transforms initialized_transforms: initialized_transforms
} = state } = state
md_path = Pipeline.Export.expected_md_path(orgfile, notes_dir, content_dir) md_path = OrgGarden.Export.expected_md_path(orgfile, notes_dir, content_dir)
IO.puts("==> Changed: #{Path.relative_to(orgfile, notes_dir)}") IO.puts("==> Changed: #{Path.relative_to(orgfile, notes_dir)}")
case Pipeline.Export.export_file(orgfile, notes_dir, output_dir) do case OrgGarden.Export.export_file(orgfile, notes_dir, output_dir) do
{:ok, _} -> {:ok, _} ->
IO.puts(" exported: #{Path.relative_to(md_path, content_dir)}") IO.puts(" exported: #{Path.relative_to(md_path, content_dir)}")
{:ok, stats} = Pipeline.run_on_files_with([md_path], initialized_transforms, pipeline_opts) {:ok, stats} = OrgGarden.run_on_files_with([md_path], initialized_transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} -> Enum.each(stats, fn {mod, count} ->
if count > 0, do: IO.puts(" #{inspect(mod)}: #{count} file(s) modified") if count > 0, do: IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
@@ -159,7 +159,7 @@ defmodule Pipeline.Watcher do
defp handle_delete(orgfile, state) do defp handle_delete(orgfile, state) do
%{notes_dir: notes_dir, content_dir: content_dir} = state %{notes_dir: notes_dir, content_dir: content_dir} = state
md_path = Pipeline.Export.expected_md_path(orgfile, notes_dir, content_dir) md_path = OrgGarden.Export.expected_md_path(orgfile, notes_dir, content_dir)
IO.puts("==> Deleted: #{Path.relative_to(orgfile, notes_dir)}") IO.puts("==> Deleted: #{Path.relative_to(orgfile, notes_dir)}")
if File.exists?(md_path) do if File.exists?(md_path) do
@@ -179,7 +179,7 @@ defmodule Pipeline.Watcher do
# ------------------------------------------------------------------- # -------------------------------------------------------------------
defp regenerate_index(content_dir) do defp regenerate_index(content_dir) do
Pipeline.Index.regenerate(content_dir) OrgGarden.Index.regenerate(content_dir)
end end
# ------------------------------------------------------------------- # -------------------------------------------------------------------

View File

@@ -1,9 +1,9 @@
defmodule Pipeline.MixProject do defmodule OrgGarden.MixProject do
use Mix.Project use Mix.Project
def project do def project do
[ [
app: :pipeline, app: :org_garden,
version: "0.1.0", version: "0.1.0",
elixir: "~> 1.17", elixir: "~> 1.17",
start_permanent: Mix.env() == :prod, start_permanent: Mix.env() == :prod,
@@ -15,12 +15,12 @@ defmodule Pipeline.MixProject do
def application do def application do
[ [
extra_applications: [:logger], extra_applications: [:logger],
mod: {Pipeline.Application, []} mod: {OrgGarden.Application, []}
] ]
end end
defp escript do defp escript do
[main_module: Pipeline.CLI] [main_module: OrgGarden.CLI]
end end
defp deps do defp deps do

View File

@@ -1,204 +0,0 @@
defmodule Pipeline.CLI do
@moduledoc """
Escript entry point for the org-roam export pipeline.
Runs four phases in sequence:
1. Wipe `content/` (preserving `.gitkeep`)
2. Export each `.org` file via `emacs --batch` + ox-hugo -> `content/**/*.md`
3. Run Elixir transform modules over every `.md` file
4. Generate a fallback `content/index.md` if none was exported
With `--watch`, after the initial batch the process stays alive and
incrementally re-exports only changed `.org` files.
## Usage
pipeline <notes-dir> [--output <path>] [--watch]
Arguments:
notes-dir Path to the directory containing `.org` notes (required).
Also accepts the `NOTES_DIR` env var.
Options:
--output <path> Output root directory (used as ox-hugo base dir).
Defaults to the `OUTPUT_DIR` env var, or the current
working directory.
--content-dir <p> Output directory for exported Markdown. Defaults to
`<output>/content`.
--watch After initial batch, watch notes-dir for changes and
incrementally re-export affected files.
Optional env vars:
BIBTEX_FILE Path to a `.bib` file used as citation fallback.
ZOTERO_URL Zotero Better BibTeX base URL (default: http://localhost:23119).
CITATION_MODE silent | warn (default) | strict.
"""
require Logger
@transforms [Pipeline.Transforms.Citations]
def main(argv) do
Application.ensure_all_started(:pipeline)
{notes_dir, output_dir, content_dir, watch?} = parse_args(argv)
pipeline_opts = build_pipeline_opts()
# Phase 1-4: full batch export
wipe(content_dir)
export_all(notes_dir, output_dir)
run_pipeline(content_dir, pipeline_opts)
generate_index(content_dir)
md_count =
content_dir
|> Path.join("**/*.md")
|> Path.wildcard()
|> length()
IO.puts("==> Done. #{md_count} markdown files in #{content_dir}")
# Phase 5: optional watch mode
if watch? do
IO.puts("==> Watching #{notes_dir} for .org changes... (Ctrl+C to stop)")
{:ok, _pid} =
Pipeline.Watcher.start_link(
notes_dir: notes_dir,
output_dir: output_dir,
content_dir: content_dir,
pipeline_opts: pipeline_opts,
transforms: @transforms
)
Process.sleep(:infinity)
end
end
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
defp parse_args(argv) do
{opts, positional, _invalid} =
OptionParser.parse(argv,
strict: [output: :string, content_dir: :string, watch: :boolean]
)
notes_dir =
case positional do
[dir | _] ->
dir
[] ->
System.get_env("NOTES_DIR") ||
abort("Usage: pipeline <notes-dir> [--output <path>] [--watch]")
end
notes_dir = Path.expand(notes_dir)
unless File.dir?(notes_dir) do
abort("Error: notes directory does not exist: #{notes_dir}")
end
output_dir =
(opts[:output] || System.get_env("OUTPUT_DIR") || File.cwd!())
|> Path.expand()
content_dir =
(opts[:content_dir] || Path.join(output_dir, "content"))
|> Path.expand()
watch? = Keyword.get(opts, :watch, false)
{notes_dir, output_dir, content_dir, watch?}
end
# ---------------------------------------------------------------------------
# Phase 1: Wipe content/
# ---------------------------------------------------------------------------
defp wipe(content_dir) do
IO.puts("==> Wiping #{content_dir}")
File.mkdir_p!(content_dir)
content_dir
|> File.ls!()
|> Enum.reject(&(&1 == ".gitkeep"))
|> Enum.each(fn entry ->
Path.join(content_dir, entry) |> File.rm_rf!()
end)
end
# ---------------------------------------------------------------------------
# Phase 2: Export org files via Emacs + ox-hugo
# ---------------------------------------------------------------------------
defp export_all(notes_dir, output_dir) do
IO.puts("==> Exporting org files from #{notes_dir}")
case Pipeline.Export.export_all(notes_dir, output_dir) do
{:ok, 0} ->
IO.puts("No .org files found in #{notes_dir}")
System.halt(0)
{:ok, count} ->
IO.puts(" exported #{count} file(s)")
{:error, failures} ->
IO.puts(:stderr, "\nFailed to export #{length(failures)} file(s):")
Enum.each(failures, fn {f, {:error, reason}} ->
IO.puts(:stderr, " #{f}: #{inspect(reason)}")
end)
System.halt(1)
end
end
# ---------------------------------------------------------------------------
# Phase 3: Markdown transformation pipeline
# ---------------------------------------------------------------------------
defp run_pipeline(content_dir, pipeline_opts) do
IO.puts("==> Running markdown pipeline")
{:ok, stats} = Pipeline.run(content_dir, @transforms, pipeline_opts)
Enum.each(stats, fn {mod, count} ->
IO.puts(" #{inspect(mod)}: #{count} file(s) modified")
end)
end
# ---------------------------------------------------------------------------
# Phase 4: Generate default index.md if none was exported
# ---------------------------------------------------------------------------
defp generate_index(content_dir) do
IO.puts("==> Generating index")
Pipeline.Index.generate(content_dir)
end
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
defp build_pipeline_opts do
%{
zotero_url: System.get_env("ZOTERO_URL", "http://localhost:23119"),
bibtex_file: System.get_env("BIBTEX_FILE"),
citation_mode:
case System.get_env("CITATION_MODE", "warn") do
"silent" -> :silent
"strict" -> :strict
_ -> :warn
end
}
end
defp abort(message) do
IO.puts(:stderr, message)
System.halt(1)
end
end