Add parallel export with configurable concurrency (default: 8)
Use Task.async_stream for parallel org->md export. Configurable via EXPORT_CONCURRENCY env var or :export_concurrency config.
This commit is contained in:
@@ -71,9 +71,15 @@ defmodule OrgGarden.Export do
|
||||
e -> {:error, e}
|
||||
end
|
||||
|
||||
@default_max_concurrency 8
|
||||
|
||||
@doc """
|
||||
Export all `.org` files found under `notes_dir`.
|
||||
|
||||
Exports files in parallel for improved performance. The concurrency level
|
||||
can be configured via the `:export_concurrency` application config or
|
||||
the `EXPORT_CONCURRENCY` environment variable. Defaults to #{@default_max_concurrency}.
|
||||
|
||||
Returns `{:ok, count}` where `count` is the number of successfully
|
||||
exported files, or `{:error, failures}` if any files failed.
|
||||
"""
|
||||
@@ -87,13 +93,21 @@ defmodule OrgGarden.Export do
|
||||
Logger.warning("No .org files found in #{notes_dir}")
|
||||
{:ok, 0}
|
||||
else
|
||||
Logger.info("Exporting #{length(org_files)} org file(s) from #{notes_dir}")
|
||||
max_concurrency = get_concurrency()
|
||||
Logger.info("Exporting #{length(org_files)} org file(s) from #{notes_dir} (concurrency: #{max_concurrency})")
|
||||
|
||||
results =
|
||||
Enum.map(org_files, fn orgfile ->
|
||||
IO.puts(" exporting: #{orgfile}")
|
||||
{orgfile, export_file(orgfile, notes_dir, output_dir)}
|
||||
end)
|
||||
org_files
|
||||
|> Task.async_stream(
|
||||
fn orgfile ->
|
||||
Logger.info(" exporting: #{orgfile}")
|
||||
{orgfile, export_file(orgfile, notes_dir, output_dir)}
|
||||
end,
|
||||
max_concurrency: max_concurrency,
|
||||
timeout: :infinity,
|
||||
ordered: false
|
||||
)
|
||||
|> Enum.map(fn {:ok, result} -> result end)
|
||||
|
||||
failures =
|
||||
Enum.filter(results, fn
|
||||
@@ -109,6 +123,10 @@ defmodule OrgGarden.Export do
|
||||
end
|
||||
end
|
||||
|
||||
defp get_concurrency do
|
||||
Application.get_env(:org_garden, :export_concurrency, @default_max_concurrency)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Compute the expected `.md` path for a given `.org` file.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user