Compare commits
16 Commits
v2025.1.3
...
v2025.1.27
| Author | SHA1 | Date | |
|---|---|---|---|
 Kieran Eglin
|
b62d5c201b | ||
|
Kieran
|
6ead29182d | ||
|
Kieran
|
62214b80a6 | ||
|
Kieran
|
704d29dc7e | ||
|
Kieran
|
3dd20141e0 | ||
|
Kieran Eglin
|
993c57f853 | ||
|
Kieran
|
63bb4d2327 | ||
|
Kieran
|
80406c9e0e | ||
|
Kieran Eglin
|
61ae50735f | ||
|
Kieran
|
d8d7353228 | ||
|
Kieran Eglin
|
03a0afd657 | ||
|
Kieran
|
ca90da49f5 | ||
|
Kieran Eglin
|
40cde43be1 | ||
|
Kieran
|
e9f6b45953 | ||
|
Kieran
|
fb27988963 | ||
|
Kieran
|
8a40d296c4 |
@@ -15,7 +15,8 @@
|
||||
<div align="center">
|
||||
|
||||
[](LICENSE)
|
||||
[](https://github.com/kieraneglin/pinchflat/releases)
|
||||
[](https://github.com/kieraneglin/pinchflat/releases)
|
||||
[](https://discord.gg/j7T6dCuwU4)
|
||||
[](#)
|
||||
[](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/kieraneglin/pinchflat)
|
||||
|
||||
@@ -33,6 +34,7 @@
|
||||
- [Portainer](#portainer)
|
||||
- [Docker](#docker)
|
||||
- [Environment Variables](#environment-variables)
|
||||
- [A note on reverse proxies](#reverse-proxies)
|
||||
- [Username and Password (authentication)](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password)
|
||||
- [Frequently asked questions](https://github.com/kieraneglin/pinchflat/wiki/Frequently-Asked-Questions)
|
||||
- [Documentation](https://github.com/kieraneglin/pinchflat/wiki)
|
||||
@@ -142,18 +144,24 @@ If you change this setting and it works well for you, please leave a comment on
|
||||
|
||||
### Environment variables
|
||||
|
||||
| Name | Required? | Default | Notes |
|
||||
| --------------------------- | --------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `TZ` | No | `UTC` | Must follow IANA TZ format |
|
||||
| `LOG_LEVEL` | No | `debug` | Can be set to `info` but `debug` is strongly recommended |
|
||||
| `BASIC_AUTH_USERNAME` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `BASIC_AUTH_PASSWORD` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `EXPOSE_FEED_ENDPOINTS` | No | `false` | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) |
|
||||
| `ENABLE_IPV6` | No | `false` | Setting to _any_ non-blank value will enable IPv6 |
|
||||
| `JOURNAL_MODE` | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) |
|
||||
| `TZ_DATA_DIR` | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored |
|
||||
| `BASE_ROUTE_PATH` | No | `/` | The base path for route generation. Useful when running behind certain reverse proxies, but prefix must be stripped. |
|
||||
| `YT_DLP_WORKER_CONCURRENCY` | No | `2` | The number of concurrent workers that use `yt-dlp` _per queue_. Set to 1 if you're getting IP limited, otherwise don't touch it |
|
||||
| Name | Required? | Default | Notes |
|
||||
| --------------------------- | --------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `TZ` | No | `UTC` | Must follow IANA TZ format |
|
||||
| `LOG_LEVEL` | No | `debug` | Can be set to `info` but `debug` is strongly recommended |
|
||||
| `UMASK` | No | `022` | Unraid users may want to set this to `000` |
|
||||
| `BASIC_AUTH_USERNAME` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `BASIC_AUTH_PASSWORD` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `EXPOSE_FEED_ENDPOINTS` | No | `false` | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) |
|
||||
| `ENABLE_IPV6` | No | `false` | Setting to _any_ non-blank value will enable IPv6 |
|
||||
| `JOURNAL_MODE` | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) |
|
||||
| `TZ_DATA_DIR` | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored |
|
||||
| `BASE_ROUTE_PATH` | No | `/` | The base path for route generation. Useful when running behind certain reverse proxies - prefixes must be stripped. |
|
||||
| `YT_DLP_WORKER_CONCURRENCY` | No | `2` | The number of concurrent workers that use `yt-dlp` _per queue_. Set to 1 if you're getting IP limited, otherwise don't touch it |
|
||||
| `ENABLE_PROMETHEUS` | No | `false` | Setting to _any_ non-blank value will enable Prometheus. See [docs](https://github.com/kieraneglin/pinchflat/wiki/Prometheus-and-Grafana) |
|
||||
|
||||
### Reverse Proxies
|
||||
|
||||
Pinchflat makes heavy use of websockets for real-time updates. If you're running Pinchflat behind a reverse proxy then you'll need to make sure it's configured to support websockets.
|
||||
|
||||
## EFF donations
|
||||
|
||||
|
||||
@@ -347,6 +347,38 @@ module.exports = {
|
||||
},
|
||||
{ values }
|
||||
)
|
||||
}),
|
||||
plugin(function ({ matchComponents, theme }) {
|
||||
let iconsDir = path.join(__dirname, './vendor/simple-icons')
|
||||
let values = {}
|
||||
|
||||
fs.readdirSync(iconsDir).forEach((file) => {
|
||||
let name = path.basename(file, '.svg')
|
||||
values[name] = { name, fullPath: path.join(iconsDir, file) }
|
||||
})
|
||||
|
||||
matchComponents(
|
||||
{
|
||||
si: ({ name, fullPath }) => {
|
||||
let content = fs
|
||||
.readFileSync(fullPath)
|
||||
.toString()
|
||||
.replace(/\r?\n|\r/g, '')
|
||||
return {
|
||||
[`--si-${name}`]: `url('data:image/svg+xml;utf8,${content}')`,
|
||||
'-webkit-mask': `var(--si-${name})`,
|
||||
mask: `var(--si-${name})`,
|
||||
'mask-repeat': 'no-repeat',
|
||||
'background-color': 'currentColor',
|
||||
'vertical-align': 'middle',
|
||||
display: 'inline-block',
|
||||
width: theme('spacing.5'),
|
||||
height: theme('spacing.5')
|
||||
}
|
||||
}
|
||||
},
|
||||
{ values }
|
||||
)
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Discord</title><path d="M20.317 4.3698a19.7913 19.7913 0 00-4.8851-1.5152.0741.0741 0 00-.0785.0371c-.211.3753-.4447.8648-.6083 1.2495-1.8447-.2762-3.68-.2762-5.4868 0-.1636-.3933-.4058-.8742-.6177-1.2495a.077.077 0 00-.0785-.037 19.7363 19.7363 0 00-4.8852 1.515.0699.0699 0 00-.0321.0277C.5334 9.0458-.319 13.5799.0992 18.0578a.0824.0824 0 00.0312.0561c2.0528 1.5076 4.0413 2.4228 5.9929 3.0294a.0777.0777 0 00.0842-.0276c.4616-.6304.8731-1.2952 1.226-1.9942a.076.076 0 00-.0416-.1057c-.6528-.2476-1.2743-.5495-1.8722-.8923a.077.077 0 01-.0076-.1277c.1258-.0943.2517-.1923.3718-.2914a.0743.0743 0 01.0776-.0105c3.9278 1.7933 8.18 1.7933 12.0614 0a.0739.0739 0 01.0785.0095c.1202.099.246.1981.3728.2924a.077.077 0 01-.0066.1276 12.2986 12.2986 0 01-1.873.8914.0766.0766 0 00-.0407.1067c.3604.698.7719 1.3628 1.225 1.9932a.076.076 0 00.0842.0286c1.961-.6067 3.9495-1.5219 6.0023-3.0294a.077.077 0 00.0313-.0552c.5004-5.177-.8382-9.6739-3.5485-13.6604a.061.061 0 00-.0312-.0286zM8.02 15.3312c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9555-2.4189 2.157-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.9555 2.4189-2.1569 2.4189zm7.9748 0c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9554-2.4189 2.1569-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.946 2.4189-2.1568 2.4189Z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
+1
@@ -0,0 +1 @@
|
||||
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>GitHub</title><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg>
|
||||
|
After Width: | Height: | Size: 823 B |
+8
-10
@@ -10,6 +10,7 @@ import Config
|
||||
config :pinchflat,
|
||||
ecto_repos: [Pinchflat.Repo],
|
||||
generators: [timestamp_type: :utc_datetime],
|
||||
env: config_env(),
|
||||
# Specifying backend data here makes mocking and local testing SUPER easy
|
||||
yt_dlp_executable: System.find_executable("yt-dlp"),
|
||||
apprise_executable: System.find_executable("apprise"),
|
||||
@@ -49,16 +50,7 @@ config :pinchflat, PinchflatWeb.Endpoint,
|
||||
|
||||
config :pinchflat, Oban,
|
||||
engine: Oban.Engines.Lite,
|
||||
repo: Pinchflat.Repo,
|
||||
# Keep old jobs for 30 days for display in the UI
|
||||
plugins: [
|
||||
{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
|
||||
{Oban.Plugins.Cron,
|
||||
crontab: [
|
||||
{"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
|
||||
{"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
|
||||
]}
|
||||
]
|
||||
repo: Pinchflat.Repo
|
||||
|
||||
# Configures the mailer
|
||||
#
|
||||
@@ -99,6 +91,12 @@ config :logger, :default_formatter,
|
||||
# Use Jason for JSON parsing in Phoenix
|
||||
config :phoenix, :json_library, Jason
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx,
|
||||
disabled: true,
|
||||
manual_metrics_start_delay: :no_delay,
|
||||
drop_metrics_groups: [],
|
||||
metrics_server: :disabled
|
||||
|
||||
# Import environment specific config. This must remain at the bottom
|
||||
# of this file so it overrides the configuration defined above.
|
||||
import_config "#{config_env()}.exs"
|
||||
|
||||
@@ -81,3 +81,5 @@ config :phoenix_live_view, :debug_heex_annotations, true
|
||||
|
||||
# Disable swoosh api client as it is only required for production adapters.
|
||||
config :swoosh, :api_client, false
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx, disabled: false
|
||||
|
||||
+19
-1
@@ -43,15 +43,30 @@ config :pinchflat, Pinchflat.Repo,
|
||||
# Some users may want to increase the number of workers that use yt-dlp to improve speeds
|
||||
# Others may want to decrease the number of these workers to lessen the chance of an IP ban
|
||||
{yt_dlp_worker_count, _} = Integer.parse(System.get_env("YT_DLP_WORKER_CONCURRENCY", "2"))
|
||||
# Used to set the cron for the yt-dlp update worker. The reason for this is
|
||||
# to avoid all instances of PF updating yt-dlp at the same time, which 1)
|
||||
# could result in rate limiting and 2) gives me time to react if an update
|
||||
# breaks something
|
||||
%{hour: current_hour, minute: current_minute} = DateTime.utc_now()
|
||||
|
||||
config :pinchflat, Oban,
|
||||
queues: [
|
||||
default: 10,
|
||||
fast_indexing: 6,
|
||||
fast_indexing: yt_dlp_worker_count,
|
||||
media_collection_indexing: yt_dlp_worker_count,
|
||||
media_fetching: yt_dlp_worker_count,
|
||||
remote_metadata: yt_dlp_worker_count,
|
||||
local_data: 8
|
||||
],
|
||||
plugins: [
|
||||
# Keep old jobs for 30 days for display in the UI
|
||||
{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
|
||||
{Oban.Plugins.Cron,
|
||||
crontab: [
|
||||
{"#{current_minute} #{current_hour} * * *", Pinchflat.YtDlp.UpdateWorker},
|
||||
{"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
|
||||
{"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
|
||||
]}
|
||||
]
|
||||
|
||||
if config_env() == :prod do
|
||||
@@ -72,6 +87,7 @@ if config_env() == :prod do
|
||||
# For running PF in a subdirectory via a reverse proxy
|
||||
base_route_path = System.get_env("BASE_ROUTE_PATH", "/")
|
||||
enable_ipv6 = String.length(System.get_env("ENABLE_IPV6", "")) > 0
|
||||
enable_prometheus = String.length(System.get_env("ENABLE_PROMETHEUS", "")) > 0
|
||||
|
||||
config :logger, level: String.to_existing_atom(System.get_env("LOG_LEVEL", "debug"))
|
||||
|
||||
@@ -95,6 +111,8 @@ if config_env() == :prod do
|
||||
database: db_path,
|
||||
journal_mode: journal_mode
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx, disabled: !enable_prometheus
|
||||
|
||||
# The secret key base is used to sign/encrypt cookies and other secrets.
|
||||
# A default value is used in config/dev.exs and config/test.exs but you
|
||||
# want to use a different value for prod and you most likely don't want
|
||||
|
||||
@@ -7,7 +7,7 @@ ARG DEBIAN_VERSION=bookworm-20240612-slim
|
||||
ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
|
||||
ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
|
||||
|
||||
FROM ${BUILDER_IMAGE} as builder
|
||||
FROM ${BUILDER_IMAGE} AS builder
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
RUN echo "Building for ${TARGETPLATFORM:?}"
|
||||
@@ -110,9 +110,9 @@ RUN apt-get update -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# More locale setup
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LANGUAGE en_US:en
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
ENV LANG=en_US.UTF-8
|
||||
ENV LANGUAGE=en_US:en
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
WORKDIR "/app"
|
||||
|
||||
@@ -124,12 +124,13 @@ RUN mkdir -p /config /downloads /etc/elixir_tzdata_data /etc/yt-dlp/plugins && \
|
||||
ENV MIX_ENV="prod"
|
||||
ENV PORT=${PORT}
|
||||
ENV RUN_CONTEXT="selfhosted"
|
||||
ENV UMASK=022
|
||||
EXPOSE ${PORT}
|
||||
|
||||
# Only copy the final release from the build stage
|
||||
COPY --from=builder /app/_build/${MIX_ENV}/rel/pinchflat ./
|
||||
|
||||
HEALTHCHECK --interval=120s --start-period=10s \
|
||||
HEALTHCHECK --interval=30s --start-period=15s \
|
||||
CMD curl --fail http://localhost:${PORT}/healthcheck || exit 1
|
||||
|
||||
# Start the app
|
||||
|
||||
@@ -9,8 +9,13 @@ defmodule Pinchflat.Application do
|
||||
@impl true
|
||||
def start(_type, _args) do
|
||||
check_and_update_timezone()
|
||||
attach_oban_telemetry()
|
||||
Logger.add_handlers(:pinchflat)
|
||||
|
||||
children = [
|
||||
# See https://hexdocs.pm/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
[
|
||||
Pinchflat.PromEx,
|
||||
PinchflatWeb.Telemetry,
|
||||
Pinchflat.Repo,
|
||||
# Must be before startup tasks
|
||||
@@ -23,17 +28,11 @@ defmodule Pinchflat.Application do
|
||||
{Finch, name: Pinchflat.Finch},
|
||||
# Start a worker by calling: Pinchflat.Worker.start_link(arg)
|
||||
# {Pinchflat.Worker, arg},
|
||||
# Start to serve requests, typically the last entry
|
||||
PinchflatWeb.Endpoint
|
||||
# Start to serve requests, typically the last entry (except for the post-boot tasks)
|
||||
PinchflatWeb.Endpoint,
|
||||
Pinchflat.Boot.PostBootStartupTasks
|
||||
]
|
||||
|
||||
attach_oban_telemetry()
|
||||
Logger.add_handlers(:pinchflat)
|
||||
|
||||
# See https://hexdocs.pm/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
opts = [strategy: :one_for_one, name: Pinchflat.Supervisor]
|
||||
Supervisor.start_link(children, opts)
|
||||
|> Supervisor.start_link(strategy: :one_for_one, name: Pinchflat.Supervisor)
|
||||
end
|
||||
|
||||
# Tell Phoenix to update the endpoint configuration
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
defmodule Pinchflat.Boot.PostBootStartupTasks do
|
||||
@moduledoc """
|
||||
This module is responsible for running startup tasks on app boot
|
||||
AFTER all other boot steps have taken place and the app is ready to serve requests.
|
||||
|
||||
It's a GenServer because that plays REALLY nicely with the existing
|
||||
Phoenix supervision tree.
|
||||
"""
|
||||
|
||||
alias Pinchflat.YtDlp.UpdateWorker, as: YtDlpUpdateWorker
|
||||
|
||||
# restart: :temporary means that this process will never be restarted (ie: will run once and then die)
|
||||
use GenServer, restart: :temporary
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Runs post-boot application startup tasks.
|
||||
|
||||
Any code defined here will run every time the application starts. You must
|
||||
make sure that the code is idempotent and safe to run multiple times.
|
||||
|
||||
This is a good place to set up default settings, create initial records, stuff like that.
|
||||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
update_yt_dlp()
|
||||
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
defp update_yt_dlp do
|
||||
YtDlpUpdateWorker.kickoff()
|
||||
end
|
||||
end
|
||||
@@ -1,7 +1,7 @@
|
||||
defmodule Pinchflat.Boot.PostJobStartupTasks do
|
||||
@moduledoc """
|
||||
This module is responsible for running startup tasks on app boot
|
||||
AFTER the job runner has initiallized.
|
||||
AFTER the job runner has initialized.
|
||||
|
||||
It's a GenServer because that plays REALLY nicely with the existing
|
||||
Phoenix supervision tree.
|
||||
@@ -12,7 +12,7 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{}, opts)
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
@@ -25,6 +25,13 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
|
||||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
# Nothing at the moment!
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
|
||||
alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{}, opts)
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
@@ -32,6 +32,13 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
|
||||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
ensure_tmpfile_directory()
|
||||
reset_executing_jobs()
|
||||
|
||||
@@ -27,13 +27,15 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
|
||||
def enqueue_pending_download_tasks(source, job_opts \\ [])
|
||||
|
||||
def enqueue_pending_download_tasks(%Source{download_media: true} = source, job_opts) do
|
||||
source
|
||||
|> Media.list_pending_media_items_for()
|
||||
|> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
|
||||
|> Enum.each(&MediaDownloadWorker.kickoff_with_task(&1, %{}, job_opts))
|
||||
end
|
||||
|
||||
def enqueue_pending_download_tasks(%Source{download_media: false}) do
|
||||
def enqueue_pending_download_tasks(%Source{download_media: false}, _job_opts) do
|
||||
:ok
|
||||
end
|
||||
|
||||
@@ -55,13 +57,13 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
||||
|
||||
Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
|
||||
"""
|
||||
def kickoff_download_if_pending(%MediaItem{} = media_item) do
|
||||
def kickoff_download_if_pending(%MediaItem{} = media_item, job_opts \\ []) do
|
||||
media_item = Repo.preload(media_item, :source)
|
||||
|
||||
if media_item.source.download_media && Media.pending_download?(media_item) do
|
||||
Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
|
||||
|
||||
MediaDownloadWorker.kickoff_with_task(media_item)
|
||||
MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
|
||||
else
|
||||
{:error, :should_not_download}
|
||||
end
|
||||
|
||||
@@ -3,6 +3,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :media_fetching,
|
||||
priority: 5,
|
||||
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
|
||||
tags: ["media_item", "media_fetching", "show_in_dashboard"]
|
||||
|
||||
@@ -49,8 +50,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
||||
|
||||
media_item = fetch_and_run_prevent_download_user_script(media_item_id)
|
||||
|
||||
# If the source or media item is set to not download media, perform a no-op unless forced
|
||||
if (media_item.source.download_media && !media_item.prevent_download) || should_force do
|
||||
if should_download_media?(media_item, should_force, is_quality_upgrade) do
|
||||
download_media_and_schedule_jobs(media_item, is_quality_upgrade, should_force)
|
||||
else
|
||||
:ok
|
||||
@@ -60,6 +60,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
||||
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
|
||||
end
|
||||
|
||||
# If this is a quality upgrade, only check if the source is set to download media
|
||||
# or that the media item's download hasn't been prevented
|
||||
defp should_download_media?(media_item, should_force, true = _is_quality_upgrade) do
|
||||
(media_item.source.download_media && !media_item.prevent_download) || should_force
|
||||
end
|
||||
|
||||
# If it's not a quality upgrade, additionally check if the media item is pending download
|
||||
defp should_download_media?(media_item, should_force, _is_quality_upgrade) do
|
||||
source = media_item.source
|
||||
is_pending = Media.pending_download?(media_item)
|
||||
|
||||
(is_pending && source.download_media && !media_item.prevent_download) || should_force
|
||||
end
|
||||
|
||||
# If a user script exists and, when run, returns a non-zero exit code, prevent this and all future downloads
|
||||
# of the media item.
|
||||
defp fetch_and_run_prevent_download_user_script(media_item_id) do
|
||||
@@ -115,7 +129,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
||||
defp action_on_error(message) do
|
||||
# This will attempt re-download at the next indexing, but it won't be retried
|
||||
# immediately as part of job failure logic
|
||||
non_retryable_errors = ["Video unavailable"]
|
||||
non_retryable_errors = ["Video unavailable", "Sign in to confirm"]
|
||||
|
||||
if String.contains?(to_string(message), non_retryable_errors) do
|
||||
Logger.error("yt-dlp download will not be retried: #{inspect(message)}")
|
||||
|
||||
@@ -40,7 +40,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
||||
Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
|
||||
downloaded_.
|
||||
"""
|
||||
def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
|
||||
def index_and_kickoff_downloads(%Source{} = source) do
|
||||
# The media_profile is needed to determine the quality options to _then_ determine a more
|
||||
# accurate predicted filepath
|
||||
source = Repo.preload(source, [:media_profile])
|
||||
@@ -53,6 +53,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
||||
Enum.map(new_media_ids, fn media_id ->
|
||||
case create_media_item_from_media_id(source, media_id) do
|
||||
{:ok, media_item} ->
|
||||
DownloadingHelpers.kickoff_download_if_pending(media_item, priority: 0)
|
||||
media_item
|
||||
|
||||
err ->
|
||||
@@ -61,7 +62,9 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
||||
end
|
||||
end)
|
||||
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
# Pick up any stragglers. Intentionally has a lower priority than the per-media item
|
||||
# kickoff above
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source, priority: 1)
|
||||
|
||||
Enum.filter(maybe_new_media_items, & &1)
|
||||
end
|
||||
|
||||
@@ -38,8 +38,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
|
||||
|
||||
Order of operations:
|
||||
1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
|
||||
with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
|
||||
2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
|
||||
with `FastIndexingHelpers.index_and_kickoff_downloads`
|
||||
2. If the above `index_and_kickoff_downloads` finds new media items in the RSS feed,
|
||||
it indexes them with a yt-dlp call to create the media item records then kicks off downloading
|
||||
tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
|
||||
3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
|
||||
@@ -67,7 +67,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
|
||||
|
||||
new_media_items =
|
||||
source
|
||||
|> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
|
||||
|> FastIndexingHelpers.index_and_kickoff_downloads()
|
||||
|> Enum.filter(&Media.pending_download?(&1))
|
||||
|
||||
if source.download_media do
|
||||
|
||||
@@ -112,6 +112,9 @@ defmodule Pinchflat.Media.MediaItem do
|
||||
|> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
|
||||
|> update_upload_date_index()
|
||||
|> validate_required(@required_fields)
|
||||
# Validate that the title does NOT start with "youtube video #" since that indicates a restriction by YouTube.
|
||||
# See issue #549 for more information.
|
||||
|> validate_format(:title, ~r/^(?!youtube video #)/)
|
||||
|> unique_constraint([:media_id, :source_id])
|
||||
end
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ defmodule Pinchflat.Profiles.MediaProfileDeletionWorker do
|
||||
Starts the profile deletion worker. Does not attach it to a task like `kickoff_with_task/2`
|
||||
since deletion also cancels all tasks for the profile
|
||||
|
||||
Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
|
||||
Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff(profile, job_args \\ %{}, job_opts \\ []) do
|
||||
%{id: profile.id}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
defmodule Pinchflat.PromEx do
|
||||
@moduledoc """
|
||||
Configuration for the PromEx library which provides Prometheus metrics
|
||||
"""
|
||||
|
||||
use PromEx, otp_app: :pinchflat
|
||||
|
||||
alias PromEx.Plugins
|
||||
|
||||
@impl true
|
||||
def plugins do
|
||||
[
|
||||
Plugins.Application,
|
||||
Plugins.Beam,
|
||||
{Plugins.Phoenix, router: PinchflatWeb.Router, endpoint: PinchflatWeb.Endpoint},
|
||||
Plugins.Ecto,
|
||||
Plugins.Oban,
|
||||
Plugins.PhoenixLiveView
|
||||
]
|
||||
end
|
||||
|
||||
@impl true
|
||||
def dashboard_assigns do
|
||||
[
|
||||
default_selected_interval: "30s"
|
||||
]
|
||||
end
|
||||
|
||||
@impl true
|
||||
def dashboards do
|
||||
[
|
||||
{:prom_ex, "application.json"},
|
||||
{:prom_ex, "beam.json"},
|
||||
{:prom_ex, "phoenix.json"},
|
||||
{:prom_ex, "ecto.json"},
|
||||
{:prom_ex, "oban.json"},
|
||||
{:prom_ex, "phoenix_live_view.json"}
|
||||
]
|
||||
end
|
||||
end
|
||||
@@ -14,15 +14,17 @@ defmodule Pinchflat.Settings.Setting do
|
||||
:apprise_server,
|
||||
:video_codec_preference,
|
||||
:audio_codec_preference,
|
||||
:youtube_api_key
|
||||
:youtube_api_key,
|
||||
:extractor_sleep_interval_seconds
|
||||
]
|
||||
|
||||
@required_fields ~w(
|
||||
onboarding
|
||||
pro_enabled
|
||||
video_codec_preference
|
||||
audio_codec_preference
|
||||
)a
|
||||
@required_fields [
|
||||
:onboarding,
|
||||
:pro_enabled,
|
||||
:video_codec_preference,
|
||||
:audio_codec_preference,
|
||||
:extractor_sleep_interval_seconds
|
||||
]
|
||||
|
||||
schema "settings" do
|
||||
field :onboarding, :boolean, default: true
|
||||
@@ -32,6 +34,7 @@ defmodule Pinchflat.Settings.Setting do
|
||||
field :apprise_server, :string
|
||||
field :youtube_api_key, :string
|
||||
field :route_token, :string
|
||||
field :extractor_sleep_interval_seconds, :integer, default: 0
|
||||
|
||||
field :video_codec_preference, :string
|
||||
field :audio_codec_preference, :string
|
||||
@@ -42,5 +45,6 @@ defmodule Pinchflat.Settings.Setting do
|
||||
setting
|
||||
|> cast(attrs, @allowed_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> validate_number(:extractor_sleep_interval_seconds, greater_than_or_equal_to: 0)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -39,7 +39,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
||||
def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
|
||||
job_offset_seconds = if job_args[:force], do: 0, else: calculate_job_offset_seconds(source)
|
||||
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
|
||||
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
|
||||
|
||||
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts ++ [schedule_in: job_offset_seconds])
|
||||
@@ -231,8 +230,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
||||
# The download archive isn't useful for playlists (since those are ordered arbitrarily)
|
||||
# and we don't want to use it if the indexing was forced by the user. In other words,
|
||||
# only create an archive for channels that are being indexed as part of their regular
|
||||
# indexing schedule
|
||||
# indexing schedule. The first indexing pass should also not create an archive.
|
||||
defp build_download_archive_options(%Source{collection_type: :playlist}, _was_forced), do: []
|
||||
defp build_download_archive_options(%Source{last_indexed_at: nil}, _was_forced), do: []
|
||||
defp build_download_archive_options(_source, true), do: []
|
||||
|
||||
defp build_download_archive_options(source, _was_forced) do
|
||||
|
||||
@@ -180,9 +180,12 @@ defmodule Pinchflat.Sources do
|
||||
end
|
||||
|
||||
defp add_source_details_to_changeset(source, changeset) do
|
||||
original_url = changeset.changes.original_url
|
||||
use_cookies = Ecto.Changeset.get_field(changeset, :use_cookies)
|
||||
# Skipping sleep interval since this is UI blocking and we want to keep this as fast as possible
|
||||
addl_opts = [use_cookies: use_cookies, skip_sleep_interval: true]
|
||||
|
||||
case MediaCollection.get_source_details(changeset.changes.original_url, [], use_cookies: use_cookies) do
|
||||
case MediaCollection.get_source_details(original_url, [], addl_opts) do
|
||||
{:ok, source_details} ->
|
||||
add_source_details_by_collection_type(source, changeset, source_details)
|
||||
|
||||
@@ -297,6 +300,10 @@ defmodule Pinchflat.Sources do
|
||||
%{__meta__: %{state: :built}} ->
|
||||
SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
if Ecto.Changeset.get_field(changeset, :fast_index) do
|
||||
FastIndexingHelpers.kickoff_indexing_task(source)
|
||||
end
|
||||
|
||||
# If the record has been persisted, only run indexing if the
|
||||
# indexing frequency has been changed and is now greater than 0
|
||||
%{__meta__: %{state: :loaded}} ->
|
||||
|
||||
@@ -36,4 +36,18 @@ defmodule Pinchflat.Utils.NumberUtils do
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Adds jitter to a number based on a percentage. Returns 0 if the number is less than or equal to 0.
|
||||
|
||||
Returns integer()
|
||||
"""
|
||||
def add_jitter(num, jitter_percentage \\ 0.5)
|
||||
def add_jitter(num, _jitter_percentage) when num <= 0, do: 0
|
||||
|
||||
def add_jitter(num, jitter_percentage) do
|
||||
jitter = :rand.uniform(round(num * jitter_percentage))
|
||||
|
||||
round(num + jitter)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,7 +5,9 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Utils.CliUtils
|
||||
alias Pinchflat.Utils.NumberUtils
|
||||
alias Pinchflat.YtDlp.YtDlpCommandRunner
|
||||
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
|
||||
|
||||
@@ -22,23 +24,23 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
||||
for a file watcher.
|
||||
- :use_cookies - if true, will add a cookie file to the command options. Will not
|
||||
attach a cookie file if the user hasn't set one up.
|
||||
- :skip_sleep_interval - if true, will not add the sleep interval options to the command.
|
||||
Usually only used for commands that would be UI-blocking
|
||||
|
||||
Returns {:ok, binary()} | {:error, output, status}.
|
||||
"""
|
||||
@impl YtDlpCommandRunner
|
||||
def run(url, action_name, command_opts, output_template, addl_opts \\ []) do
|
||||
Logger.debug("Running yt-dlp command for action: #{action_name}")
|
||||
# This approach lets us mock the command for testing
|
||||
command = backend_executable()
|
||||
|
||||
output_filepath = generate_output_filepath(addl_opts)
|
||||
print_to_file_opts = [{:print_to_file, output_template}, output_filepath]
|
||||
user_configured_opts = cookie_file_options(addl_opts)
|
||||
user_configured_opts = cookie_file_options(addl_opts) ++ sleep_interval_opts(addl_opts)
|
||||
# These must stay in exactly this order, hence why I'm giving it its own variable.
|
||||
all_opts = command_opts ++ print_to_file_opts ++ user_configured_opts ++ global_options()
|
||||
formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts)
|
||||
|
||||
case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do
|
||||
case CliUtils.wrap_cmd(backend_executable(), formatted_command_opts, stderr_to_stdout: true) do
|
||||
# yt-dlp exit codes:
|
||||
# 0 = Everything is successful
|
||||
# 100 = yt-dlp must restart for update to complete
|
||||
@@ -74,6 +76,24 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates yt-dlp to the latest version
|
||||
|
||||
Returns {:ok, binary()} | {:error, binary()}
|
||||
"""
|
||||
@impl YtDlpCommandRunner
|
||||
def update do
|
||||
command = backend_executable()
|
||||
|
||||
case CliUtils.wrap_cmd(command, ["--update"]) do
|
||||
{output, 0} ->
|
||||
{:ok, String.trim(output)}
|
||||
|
||||
{output, _} ->
|
||||
{:error, output}
|
||||
end
|
||||
end
|
||||
|
||||
defp generate_output_filepath(addl_opts) do
|
||||
case Keyword.get(addl_opts, :output_filepath) do
|
||||
nil -> FSUtils.generate_metadata_tmpfile(:json)
|
||||
@@ -96,6 +116,20 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
||||
end
|
||||
end
|
||||
|
||||
defp sleep_interval_opts(addl_opts) do
|
||||
sleep_interval = Settings.get!(:extractor_sleep_interval_seconds)
|
||||
|
||||
if sleep_interval <= 0 || Keyword.get(addl_opts, :skip_sleep_interval) do
|
||||
[]
|
||||
else
|
||||
[
|
||||
sleep_requests: NumberUtils.add_jitter(sleep_interval),
|
||||
sleep_interval: NumberUtils.add_jitter(sleep_interval),
|
||||
sleep_subtitles: NumberUtils.add_jitter(sleep_interval)
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
defp add_cookie_file do
|
||||
base_dir = Application.get_env(:pinchflat, :extras_directory)
|
||||
filename_options_map = %{cookies: "cookies.txt"}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
defmodule Pinchflat.YtDlp.UpdateWorker do
|
||||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
tags: ["local_data"]
|
||||
|
||||
require Logger
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Settings
|
||||
|
||||
@doc """
|
||||
Starts the yt-dlp update worker. Does not attach it to a task like `kickoff_with_task/2`
|
||||
|
||||
Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff do
|
||||
Oban.insert(UpdateWorker.new(%{}))
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates yt-dlp and saves the version to the settings.
|
||||
|
||||
This worker is scheduled to run via the Oban Cron plugin as well as on app boot.
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
Logger.info("Updating yt-dlp")
|
||||
|
||||
yt_dlp_runner().update()
|
||||
|
||||
{:ok, yt_dlp_version} = yt_dlp_runner().version()
|
||||
Settings.set(yt_dlp_version: yt_dlp_version)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp yt_dlp_runner do
|
||||
Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
end
|
||||
end
|
||||
@@ -9,4 +9,5 @@ defmodule Pinchflat.YtDlp.YtDlpCommandRunner do
|
||||
@callback run(binary(), atom(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback run(binary(), atom(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback version() :: {:ok, binary()} | {:error, binary()}
|
||||
@callback update() :: {:ok, binary()} | {:error, binary()}
|
||||
end
|
||||
|
||||
@@ -700,7 +700,7 @@ defmodule PinchflatWeb.CoreComponents do
|
||||
attr :class, :string, default: nil
|
||||
attr :rest, :global
|
||||
|
||||
def icon(%{name: "hero-" <> _} = assigns) do
|
||||
def icon(assigns) do
|
||||
~H"""
|
||||
<span class={[@name, @class]} {@rest} />
|
||||
"""
|
||||
|
||||
@@ -15,11 +15,12 @@ defmodule PinchflatWeb.Layouts do
|
||||
attr :text, :string, required: true
|
||||
attr :href, :any, required: true
|
||||
attr :target, :any, default: "_self"
|
||||
attr :icon_class, :string, default: ""
|
||||
|
||||
def sidebar_item(assigns) do
|
||||
~H"""
|
||||
<li class="text-bodydark1">
|
||||
<.sidebar_link icon={@icon} text={@text} href={@href} target={@target} />
|
||||
<.sidebar_link icon={@icon} text={@text} href={@href} target={@target} icon_class={@icon_class} />
|
||||
</li>
|
||||
"""
|
||||
end
|
||||
@@ -89,6 +90,7 @@ defmodule PinchflatWeb.Layouts do
|
||||
attr :href, :any, required: true
|
||||
attr :target, :any, default: "_self"
|
||||
attr :class, :string, default: ""
|
||||
attr :icon_class, :string, default: ""
|
||||
|
||||
def sidebar_link(assigns) do
|
||||
~H"""
|
||||
@@ -103,7 +105,7 @@ defmodule PinchflatWeb.Layouts do
|
||||
@class
|
||||
]}
|
||||
>
|
||||
<.icon :if={@icon} name={@icon} /> {@text}
|
||||
<.icon :if={@icon} name={@icon} class={@icon_class} /> {@text}
|
||||
</.link>
|
||||
"""
|
||||
end
|
||||
|
||||
@@ -47,8 +47,10 @@
|
||||
text="Docs"
|
||||
target="_blank"
|
||||
href="https://github.com/kieraneglin/pinchflat/wiki"
|
||||
icon_class="scale-110"
|
||||
/>
|
||||
<.sidebar_item icon="hero-cog" text="Github" target="_blank" href="https://github.com/kieraneglin/pinchflat" />
|
||||
<.sidebar_item icon="si-github" text="Github" target="_blank" href="https://github.com/kieraneglin/pinchflat" />
|
||||
<.sidebar_item icon="si-discord" text="Discord" target="_blank" href="https://discord.gg/j7T6dCuwU4" />
|
||||
<li>
|
||||
<span
|
||||
class={[
|
||||
@@ -59,7 +61,7 @@
|
||||
]}
|
||||
phx-click={show_modal("donate-modal")}
|
||||
>
|
||||
<.icon name="hero-currency-dollar" /> Donate
|
||||
<.icon name="hero-currency-dollar" class="scale-110" /> Donate
|
||||
</span>
|
||||
</li>
|
||||
<li>
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<section class="mt-8">
|
||||
<section>
|
||||
<h3 class="text-2xl text-black dark:text-white">
|
||||
Indexing Settings
|
||||
Extractor Settings
|
||||
</h3>
|
||||
|
||||
<.input
|
||||
@@ -41,6 +41,14 @@
|
||||
html_help={true}
|
||||
inputclass="font-mono text-sm mr-4"
|
||||
/>
|
||||
|
||||
<.input
|
||||
field={f[:extractor_sleep_interval_seconds]}
|
||||
placeholder="0"
|
||||
type="number"
|
||||
label="Sleep Interval (seconds)"
|
||||
help="Sleep interval in seconds between each extractor request. Must be a positive whole number (or set to 0 to disable)"
|
||||
/>
|
||||
</section>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ defmodule PinchflatWeb.Endpoint do
|
||||
plug Plug.Static,
|
||||
at: "/",
|
||||
from: :pinchflat,
|
||||
gzip: Mix.env() == :prod,
|
||||
gzip: Application.compile_env(:pinchflat, :env) == :prod,
|
||||
only: PinchflatWeb.static_paths()
|
||||
|
||||
# Code reloading can be explicitly enabled under the
|
||||
@@ -32,6 +32,8 @@ defmodule PinchflatWeb.Endpoint do
|
||||
plug Phoenix.Ecto.CheckRepoStatus, otp_app: :pinchflat
|
||||
end
|
||||
|
||||
plug PromEx.Plug, prom_ex_module: Pinchflat.PromEx
|
||||
|
||||
plug Phoenix.LiveDashboard.RequestLogger,
|
||||
param_key: "request_logger",
|
||||
cookie_key: "request_logger"
|
||||
|
||||
@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
|
||||
def project do
|
||||
[
|
||||
app: :pinchflat,
|
||||
version: "2025.1.3",
|
||||
version: "2025.1.27",
|
||||
elixir: "~> 1.17",
|
||||
elixirc_paths: elixirc_paths(Mix.env()),
|
||||
start_permanent: Mix.env() == :prod,
|
||||
@@ -60,9 +60,9 @@ defmodule Pinchflat.MixProject do
|
||||
{:esbuild, "~> 0.8", runtime: Mix.env() == :dev},
|
||||
{:tailwind, "~> 0.2.0", runtime: Mix.env() == :dev},
|
||||
{:swoosh, "~> 1.3"},
|
||||
{:finch, "~> 0.13"},
|
||||
{:telemetry_metrics, "~> 0.6"},
|
||||
{:telemetry_poller, "~> 1.0"},
|
||||
{:finch, "~> 0.18"},
|
||||
{:telemetry_metrics, "~> 1.0"},
|
||||
{:telemetry_poller, "~> 1.1"},
|
||||
{:gettext, "~> 0.20"},
|
||||
{:jason, "~> 1.2"},
|
||||
{:dns_cluster, "~> 0.1.1"},
|
||||
@@ -70,6 +70,7 @@ defmodule Pinchflat.MixProject do
|
||||
{:oban, "~> 2.17"},
|
||||
{:nimble_parsec, "~> 1.4"},
|
||||
{:timex, "~> 3.0"},
|
||||
{:prom_ex, "~> 1.11.0"},
|
||||
{:mox, "~> 1.0", only: :test},
|
||||
{:credo, "~> 1.7.7", only: [:dev, :test], runtime: false},
|
||||
{:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
%{
|
||||
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
|
||||
"castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"},
|
||||
"castore": {:hex, :castore, "1.0.11", "4bbd584741601eb658007339ea730b082cc61f3554cf2e8f39bf693a11b49073", [:mix], [], "hexpm", "e03990b4db988df56262852f20de0f659871c35154691427a5047f4967a16a62"},
|
||||
"cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"},
|
||||
"certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
|
||||
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
|
||||
@@ -23,23 +23,25 @@
|
||||
"exqlite": {:hex, :exqlite, "0.23.0", "6e851c937a033299d0784994c66da24845415072adbc455a337e20087bce9033", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "404341cceec5e6466aaed160cf0b58be2019b60af82588c215e1224ebd3ec831"},
|
||||
"faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"},
|
||||
"file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
|
||||
"finch": {:hex, :finch, "0.17.0", "17d06e1d44d891d20dbd437335eebe844e2426a0cd7e3a3e220b461127c73f70", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8d014a661bb6a437263d4b5abf0bcbd3cf0deb26b1e8596f2a271d22e48934c7"},
|
||||
"finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"},
|
||||
"floki": {:hex, :floki, "0.36.3", "1102f93b16a55bc5383b85ae3ec470f82dee056eaeff9195e8afdf0ef2a43c30", [:mix], [], "hexpm", "fe0158bff509e407735f6d40b3ee0d7deb47f3f3ee7c6c182ad28599f9f6b27a"},
|
||||
"gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"},
|
||||
"hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
|
||||
"hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
|
||||
"hpax": {:hex, :hpax, "1.0.2", "762df951b0c399ff67cc57c3995ec3cf46d696e41f0bba17da0518d94acd4aac", [:mix], [], "hexpm", "2f09b4c1074e0abd846747329eaa26d535be0eb3d189fa69d812bfb8bfefd32f"},
|
||||
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
|
||||
"jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
|
||||
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
|
||||
"mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
|
||||
"mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"},
|
||||
"mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
|
||||
"mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"},
|
||||
"mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
|
||||
"nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
|
||||
"nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
|
||||
"nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
|
||||
"nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
|
||||
"nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
|
||||
"oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"},
|
||||
"octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"},
|
||||
"parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
|
||||
"peep": {:hex, :peep, "3.4.1", "0e5263710fa0b42675bd0a11fdcdd3ee4f484e319105b6ad9a576c91a5d3cb55", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "7a9b8c1f17b8b9475efb27b7048afa4d89ab84ef33a3d1df13696c85c12cd632"},
|
||||
"phoenix": {:hex, :phoenix, "1.7.17", "2fcdceecc6fb90bec26fab008f96abbd0fd93bc9956ec7985e5892cf545152ca", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "50e8ad537f3f7b0efb1509b2f75b5c918f697be6a45d48e49a30d3b7c0e464c9"},
|
||||
"phoenix_ecto": {:hex, :phoenix_ecto, "4.4.3", "86e9878f833829c3f66da03d75254c155d91d72a201eb56ae83482328dc7ca93", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d36c401206f3011fefd63d04e8ef626ec8791975d9d107f9a0817d426f61ac07"},
|
||||
"phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"},
|
||||
@@ -51,6 +53,7 @@
|
||||
"plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"},
|
||||
"plug_cowboy": {:hex, :plug_cowboy, "2.7.2", "fdadb973799ae691bf9ecad99125b16625b1c6039999da5fe544d99218e662e4", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "245d8a11ee2306094840c000e8816f0cbed69a23fc0ac2bcf8d7835ae019bb2f"},
|
||||
"plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
|
||||
"prom_ex": {:hex, :prom_ex, "1.11.0", "1f6d67f2dead92224cb4f59beb3e4d319257c5728d9638b4a5e8ceb51a4f9c7e", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "76b074bc3730f0802978a7eb5c7091a65473eaaf07e99ec9e933138dcc327805"},
|
||||
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
|
||||
"sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"},
|
||||
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
|
||||
@@ -58,8 +61,9 @@
|
||||
"table_rex": {:hex, :table_rex, "4.0.0", "3c613a68ebdc6d4d1e731bc973c233500974ec3993c99fcdabb210407b90959b", [:mix], [], "hexpm", "c35c4d5612ca49ebb0344ea10387da4d2afe278387d4019e4d8111e815df8f55"},
|
||||
"tailwind": {:hex, :tailwind, "0.2.2", "9e27288b568ede1d88517e8c61259bc214a12d7eed271e102db4c93fcca9b2cd", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "ccfb5025179ea307f7f899d1bb3905cd0ac9f687ed77feebc8f67bdca78565c4"},
|
||||
"telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
|
||||
"telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"},
|
||||
"telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},
|
||||
"telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"},
|
||||
"telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
|
||||
"telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"},
|
||||
"timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"},
|
||||
"tzdata": {:hex, :tzdata, "1.1.2", "45e5f1fcf8729525ec27c65e163be5b3d247ab1702581a94674e008413eef50b", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cec7b286e608371602318c414f344941d5eb0375e14cfdab605cca2fe66cba8b"},
|
||||
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
|
||||
|
||||
@@ -0,0 +1,607 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
},
|
||||
{
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "#73BF69",
|
||||
"limit": 100,
|
||||
"name": "PromEx service start",
|
||||
"showIn": 0,
|
||||
"tags": ["prom_ex", "pinchflat", "start"],
|
||||
"type": "tags"
|
||||
},
|
||||
{
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "#FF9830",
|
||||
"limit": 100,
|
||||
"name": "PromEx service stop",
|
||||
"showIn": 0,
|
||||
"tags": ["prom_ex", "pinchflat", "stop"],
|
||||
"type": "tags"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "All the data that is presented here is captured by the PromEx Application plugin (https://github.com/akoutmos/prom_ex/blob/master/lib/prom_ex/plugins/application.ex)",
|
||||
"editable": false,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "bolt",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Sponsor PromEx",
|
||||
"tooltip": "",
|
||||
"type": "link",
|
||||
"url": "https://github.com/sponsors/akoutmos"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "doc",
|
||||
"includeVars": false,
|
||||
"keepTime": false,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Application Plugin Docs",
|
||||
"tooltip": "",
|
||||
"type": "link",
|
||||
"url": "https://hexdocs.pm/prom_ex/PromEx.Plugins.Application.html"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The amount of time that the application has been running.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"decimals": 1,
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "dtdurationms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_uptime_milliseconds_count{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Uptime",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The data is populated by the PromEx Application plugin and provides information regarding your application's dependencies.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "left",
|
||||
"displayMode": "auto"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Status"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.displayMode",
|
||||
"value": "color-background"
|
||||
},
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"from": "",
|
||||
"id": 0,
|
||||
"text": "Started",
|
||||
"to": "",
|
||||
"type": 1,
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"from": "",
|
||||
"id": 1,
|
||||
"text": "Loaded",
|
||||
"to": "",
|
||||
"type": 1,
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "custom.align",
|
||||
"value": "center"
|
||||
},
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 202
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Name"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 349
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Version"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 187
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 36,
|
||||
"w": 16,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [
|
||||
{
|
||||
"desc": false,
|
||||
"displayName": "Name"
|
||||
}
|
||||
]
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_dependency_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Dependency Information",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"instance": true,
|
||||
"job": true,
|
||||
"Value": true
|
||||
},
|
||||
"indexByName": {
|
||||
"Time": 0,
|
||||
"Value": 4,
|
||||
"__name__": 1,
|
||||
"instance": 2,
|
||||
"job": 3,
|
||||
"modules": 7,
|
||||
"name": 5,
|
||||
"version": 6
|
||||
},
|
||||
"renameByName": {
|
||||
"Value": "Status",
|
||||
"modules": "Number of Modules Loaded",
|
||||
"name": "Name",
|
||||
"version": "Version"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The name of the primary application that is running.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "/^name$/",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Name",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The Git SHA of the application.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "/^sha$/",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_git_sha_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Git SHA",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The author of the application's last Git commit.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "/^author$/",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_git_author_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Git Author",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The version of the primary application that is running.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "/^version$/",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Version",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"description": "The number of modules loaded by the primary application that is running.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 30
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["last"],
|
||||
"fields": "/^modules$/",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.1.3",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "pinchflat_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Application Modules Loaded",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 26,
|
||||
"style": "dark",
|
||||
"tags": ["PromEx", "Application", "pinchflat"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": null,
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values(pinchflat_prom_ex_prom_ex_status_info, job)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Prometheus Job",
|
||||
"multi": false,
|
||||
"name": "job",
|
||||
"options": [],
|
||||
"query": "label_values(pinchflat_prom_ex_prom_ex_status_info, job)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 6,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"allValue": null,
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values(pinchflat_prom_ex_prom_ex_status_info, instance)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Application Instance",
|
||||
"multi": false,
|
||||
"name": "instance",
|
||||
"options": [],
|
||||
"query": "label_values(pinchflat_prom_ex_prom_ex_status_info{job=\"$job\"}, instance)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "Pinchflat - PromEx Application Dashboard",
|
||||
"uid": "7DBBC471C5775585391E8F24D1E62319",
|
||||
"version": 1
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
Before Width: | Height: | Size: 442 KiB After Width: | Height: | Size: 443 KiB |
@@ -0,0 +1,9 @@
|
||||
defmodule Pinchflat.Repo.Migrations.AddExtractorSleepIntervalToSettings do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
alter table(:settings) do
|
||||
add :extractor_sleep_interval_seconds, :number, null: false, default: 0
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -6,6 +6,9 @@ if [ $? -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting umask to ${UMASK}"
|
||||
umask ${UMASK}
|
||||
|
||||
/app/bin/migrate
|
||||
|
||||
cd -P -- "$(dirname -- "$0")"
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
defmodule Pinchflat.Boot.PostBootStartupTasksTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
alias Pinchflat.YtDlp.UpdateWorker
|
||||
alias Pinchflat.Boot.PostBootStartupTasks
|
||||
|
||||
describe "update_yt_dlp" do
|
||||
test "enqueues an update job" do
|
||||
assert [] = all_enqueued(worker: UpdateWorker)
|
||||
|
||||
PostBootStartupTasks.init(%{})
|
||||
|
||||
assert [%Oban.Job{}] = all_enqueued(worker: UpdateWorker)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -10,7 +10,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
alias Pinchflat.Downloading.MediaDownloadWorker
|
||||
|
||||
describe "enqueue_pending_download_tasks/1" do
|
||||
test "it enqueues a job for each pending media item" do
|
||||
test "enqueues a job for each pending media item" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
@@ -19,7 +19,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
|
||||
end
|
||||
|
||||
test "it does not enqueue a job for media items with a filepath" do
|
||||
test "does not enqueue a job for media items with a filepath" do
|
||||
source = source_fixture()
|
||||
_media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
|
||||
|
||||
@@ -28,7 +28,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "it attaches a task to each enqueued job" do
|
||||
test "attaches a task to each enqueued job" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
@@ -39,7 +39,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
assert [_] = Tasks.list_tasks_for(media_item)
|
||||
end
|
||||
|
||||
test "it does not create a job if the source is set to not download" do
|
||||
test "does not create a job if the source is set to not download" do
|
||||
source = source_fixture(download_media: false)
|
||||
|
||||
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
@@ -47,17 +47,26 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "it does not attach tasks if the source is set to not download" do
|
||||
test "does not attach tasks if the source is set to not download" do
|
||||
source = source_fixture(download_media: false)
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
assert [] = Tasks.list_tasks_for(media_item)
|
||||
end
|
||||
|
||||
test "can pass job options" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source, priority: 1)
|
||||
|
||||
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}, priority: 1)
|
||||
end
|
||||
end
|
||||
|
||||
describe "dequeue_pending_download_tasks/1" do
|
||||
test "it deletes all pending tasks for a source's media items" do
|
||||
test "deletes all pending tasks for a source's media items" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
@@ -109,6 +118,14 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
|
||||
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "can pass job options" do
|
||||
media_item = media_item_fixture(media_filepath: nil)
|
||||
|
||||
assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item, priority: 1)
|
||||
|
||||
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}, priority: 1)
|
||||
end
|
||||
end
|
||||
|
||||
describe "kickoff_redownload_for_existing_media/1" do
|
||||
|
||||
@@ -46,13 +46,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id, "force" => true})
|
||||
end
|
||||
|
||||
test "can be called with additional job options", %{media_item: media_item} do
|
||||
job_opts = [max_attempts: 5]
|
||||
|
||||
assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
|
||||
test "has a priority of 5 by default", %{media_item: media_item} do
|
||||
assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item)
|
||||
|
||||
[job] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
|
||||
assert job.max_attempts == 5
|
||||
|
||||
assert job.priority == 5
|
||||
end
|
||||
|
||||
test "priority can be set", %{media_item: media_item} do
|
||||
assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{}, priority: 0)
|
||||
|
||||
[job] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
|
||||
|
||||
assert job.priority == 0
|
||||
end
|
||||
end
|
||||
|
||||
@@ -67,7 +74,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
:ok
|
||||
end
|
||||
|
||||
test "it saves attributes to the media_item", %{media_item: media_item} do
|
||||
test "saves attributes to the media_item", %{media_item: media_item} do
|
||||
assert media_item.media_filepath == nil
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
media_item = Repo.reload(media_item)
|
||||
@@ -75,20 +82,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
assert media_item.media_filepath != nil
|
||||
end
|
||||
|
||||
test "it saves the metadata to the media_item", %{media_item: media_item} do
|
||||
test "saves the metadata to the media_item", %{media_item: media_item} do
|
||||
assert media_item.metadata == nil
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
assert Repo.reload(media_item).metadata != nil
|
||||
end
|
||||
|
||||
test "it won't double-schedule downloading jobs", %{media_item: media_item} do
|
||||
test "won't double-schedule downloading jobs", %{media_item: media_item} do
|
||||
Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
|
||||
Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
|
||||
|
||||
assert [_] = all_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "it sets the job to retryable if the download fails", %{media_item: media_item} do
|
||||
test "sets the job to retryable if the download fails", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 2, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:error, "error"}
|
||||
@@ -127,7 +134,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
end)
|
||||
end
|
||||
|
||||
test "it ensures error are returned in a 2-item tuple", %{media_item: media_item} do
|
||||
test "does not set the job to retryable if youtube thinks you're a bot", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 2, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:error, "Sign in to confirm you're not a bot", 1}
|
||||
end)
|
||||
|
||||
Oban.Testing.with_testing_mode(:inline, fn ->
|
||||
{:ok, job} = Oban.insert(MediaDownloadWorker.new(%{id: media_item.id, quality_upgrade?: true}))
|
||||
|
||||
assert job.state == "completed"
|
||||
end)
|
||||
end
|
||||
|
||||
test "ensures error are returned in a 2-item tuple", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 2, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:error, "error", 1}
|
||||
@@ -136,7 +156,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
assert {:error, :download_failed} = perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
end
|
||||
|
||||
test "it does not download if the source is set to not download", %{media_item: media_item} do
|
||||
test "does not download if the source is set to not download", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
|
||||
|
||||
Sources.update_source(media_item.source, %{download_media: false})
|
||||
@@ -152,7 +172,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
end
|
||||
|
||||
test "it saves the file's size to the database", %{media_item: media_item} do
|
||||
test "saves the file's size to the database", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl ->
|
||||
{:ok, "{}"}
|
||||
@@ -201,6 +221,14 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
end
|
||||
|
||||
test "does not download if the media item isn't pending download", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
|
||||
|
||||
Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id})
|
||||
end
|
||||
end
|
||||
|
||||
describe "perform/1 when testing non-downloadable media" do
|
||||
@@ -219,12 +247,30 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
|
||||
describe "perform/1 when testing forced downloads" do
|
||||
test "ignores 'prevent_download' if forced", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
|
||||
_url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
|
||||
end)
|
||||
|
||||
Sources.update_source(media_item.source, %{download_media: false})
|
||||
Media.update_media_item(media_item, %{prevent_download: true})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
|
||||
end
|
||||
|
||||
test "ignores whether the media item is pending when forced", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
|
||||
_url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
|
||||
end)
|
||||
|
||||
Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
|
||||
end
|
||||
|
||||
test "sets force_overwrites runner option", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl ->
|
||||
@@ -252,6 +298,34 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
|
||||
assert media_item.media_redownloaded_at != nil
|
||||
end
|
||||
|
||||
test "ignores whether the media item is pending when re-downloaded", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
|
||||
_url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
|
||||
_url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
|
||||
end)
|
||||
|
||||
Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
|
||||
end
|
||||
|
||||
test "doesn't redownload if the source is set to not download", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
|
||||
|
||||
Sources.update_source(media_item.source, %{download_media: false})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
|
||||
end
|
||||
|
||||
test "doesn't redownload if the media item is set to not download", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
|
||||
|
||||
Media.update_media_item(media_item, %{prevent_download: true})
|
||||
|
||||
perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
|
||||
end
|
||||
|
||||
test "sets force_overwrites runner option", %{media_item: media_item} do
|
||||
expect(YtDlpRunnerMock, :run, 3, fn
|
||||
_url, :get_downloadable_status, _opts, _ot, _addl ->
|
||||
|
||||
@@ -38,36 +38,48 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "kickoff_download_tasks_from_youtube_rss_feed/1" do
|
||||
test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
|
||||
describe "index_and_kickoff_downloads/1" do
|
||||
test "enqueues a worker for each new media_id in the source's RSS feed", %{source: source} do
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
|
||||
assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [media_item] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
assert [worker] = all_enqueued(worker: MediaDownloadWorker)
|
||||
assert worker.args["id"] == media_item.id
|
||||
assert worker.priority == 0
|
||||
end
|
||||
|
||||
test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
media_item_fixture(source_id: source.id, media_id: "test_1")
|
||||
|
||||
assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "kicks off a download task for all pending media but at a lower priority", %{source: source} do
|
||||
pending_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
assert [worker_1, _worker_2] = all_enqueued(worker: MediaDownloadWorker)
|
||||
assert worker_1.args["id"] == pending_item.id
|
||||
assert worker_1.priority == 1
|
||||
end
|
||||
|
||||
test "returns the found media items", %{source: source} do
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "does not enqueue a download job if the source does not allow it" do
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
source = source_fixture(%{download_media: false})
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
@@ -75,7 +87,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
test "creates a download task record", %{source: source} do
|
||||
expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
|
||||
|
||||
assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [media_item] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
|
||||
end
|
||||
@@ -89,7 +101,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
{:ok, media_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "sets use_cookies if the source uses cookies" do
|
||||
@@ -103,7 +115,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
source = source_fixture(%{use_cookies: true})
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "does not set use_cookies if the source does not use cookies" do
|
||||
@@ -117,7 +129,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
source = source_fixture(%{use_cookies: false})
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "does not enqueue a download job if the media item does not match the format rules" do
|
||||
@@ -142,7 +154,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
{:ok, output}
|
||||
end)
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
@@ -154,7 +166,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
{:ok, "{}"}
|
||||
end)
|
||||
|
||||
assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "does not blow up if a media item causes a yt-dlp error", %{source: source} do
|
||||
@@ -164,11 +176,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
{:error, "message", 1}
|
||||
end)
|
||||
|
||||
assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
end
|
||||
|
||||
describe "kickoff_download_tasks_from_youtube_rss_feed/1 when testing backends" do
|
||||
describe "index_and_kickoff_downloads/1 when testing backends" do
|
||||
test "uses the YouTube API if it is enabled", %{source: source} do
|
||||
expect(HTTPClientMock, :get, fn url, _headers ->
|
||||
assert url =~ "https://youtube.googleapis.com/youtube/v3/playlistItems"
|
||||
@@ -178,7 +190,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
Settings.set(youtube_api_key: "test_key")
|
||||
|
||||
assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "the YouTube API creates records as expected", %{source: source} do
|
||||
@@ -188,7 +200,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
Settings.set(youtube_api_key: "test_key")
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "RSS is used as a backup if the API fails", %{source: source} do
|
||||
@@ -197,7 +209,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
Settings.set(youtube_api_key: "test_key")
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
|
||||
test "RSS is used if the API is not enabled", %{source: source} do
|
||||
@@ -209,7 +221,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
|
||||
|
||||
Settings.set(youtube_api_key: nil)
|
||||
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
|
||||
assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -921,6 +921,14 @@ defmodule Pinchflat.MediaTest do
|
||||
media_item = media_item_fixture()
|
||||
assert %Ecto.Changeset{} = Media.change_media_item(media_item)
|
||||
end
|
||||
|
||||
test "validates the title doesn't start with 'youtube video #'" do
|
||||
# This is to account for youtube restricting indexing. See issue #549 for more
|
||||
media_item = media_item_fixture()
|
||||
|
||||
assert %Ecto.Changeset{valid?: false} = Media.change_media_item(media_item, %{title: "youtube video #123"})
|
||||
assert %Ecto.Changeset{valid?: true} = Media.change_media_item(media_item, %{title: "any other title"})
|
||||
end
|
||||
end
|
||||
|
||||
describe "change_media_item/1 when testing upload_date_index and source is a channel" do
|
||||
|
||||
@@ -77,5 +77,20 @@ defmodule Pinchflat.SettingsTest do
|
||||
|
||||
assert %Ecto.Changeset{} = Settings.change_setting(setting, %{onboarding: true})
|
||||
end
|
||||
|
||||
test "ensures the extractor sleep interval is positive" do
|
||||
setting = Settings.record()
|
||||
|
||||
assert %Ecto.Changeset{valid?: true} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: 1})
|
||||
assert %Ecto.Changeset{valid?: true} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: 0})
|
||||
assert %Ecto.Changeset{valid?: false} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: -1})
|
||||
end
|
||||
|
||||
test "allows you to reset the extractor sleep interval" do
|
||||
setting = Settings.record()
|
||||
assert {:ok, setting} = Settings.update_setting(setting, %{extractor_sleep_interval_seconds: 1})
|
||||
|
||||
assert %Ecto.Changeset{valid?: true} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: 0})
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -96,26 +96,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "deletes any pending media tasks for the source" do
|
||||
source = source_fixture()
|
||||
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "deletes any fast indexing tasks for the source" do
|
||||
source = source_fixture()
|
||||
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "can be called with additional job arguments" do
|
||||
source = source_fixture(index_frequency_minutes: 1)
|
||||
job_args = %{"force" => true}
|
||||
@@ -290,6 +270,29 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
assert %Ecto.Changeset{} = changeset
|
||||
end
|
||||
|
||||
test "doesn't blow up if the media item cannot be saved", %{source: source} do
|
||||
stub(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, _opts, _ot, _addl_opts ->
|
||||
response =
|
||||
Phoenix.json_library().encode!(%{
|
||||
id: "video1",
|
||||
# This is a disallowed title - see MediaItem changeset or issue #549
|
||||
title: "youtube video #123",
|
||||
original_url: "https://example.com/video1",
|
||||
live_status: "not_live",
|
||||
description: "desc1",
|
||||
aspect_ratio: 1.67,
|
||||
duration: 12.34,
|
||||
upload_date: "20210101"
|
||||
})
|
||||
|
||||
{:ok, response}
|
||||
end)
|
||||
|
||||
assert [changeset] = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
assert %Ecto.Changeset{} = changeset
|
||||
end
|
||||
|
||||
test "passes the source's download options to the yt-dlp runner", %{source: source} do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, opts, _ot, _addl_opts ->
|
||||
assert {:output, "/tmp/test/media/%(title)S.%(ext)S"} in opts
|
||||
@@ -452,7 +455,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
end
|
||||
|
||||
describe "index_and_enqueue_download_for_media_items when testing the download archive" do
|
||||
test "a download archive is used if the source is a channel", %{source: source} do
|
||||
test "a download archive is used if the source is a channel that has been indexed before" do
|
||||
source = source_fixture(%{collection_type: :channel, last_indexed_at: now()})
|
||||
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, opts, _ot, _addl_opts ->
|
||||
assert :break_on_existing in opts
|
||||
assert Keyword.has_key?(opts, :download_archive)
|
||||
@@ -476,6 +481,19 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
end
|
||||
|
||||
test "a download archive is not used if the source has never been indexed before" do
|
||||
source = source_fixture(%{collection_type: :channel, last_indexed_at: nil})
|
||||
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, opts, _ot, _addl_opts ->
|
||||
refute :break_on_existing in opts
|
||||
refute Keyword.has_key?(opts, :download_archive)
|
||||
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
end
|
||||
|
||||
test "a download archive is not used if the index has been forced to run" do
|
||||
source = source_fixture(%{collection_type: :channel})
|
||||
|
||||
@@ -489,7 +507,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source, was_forced: true)
|
||||
end
|
||||
|
||||
test "the download archive is formatted correctly and contains the right video", %{source: source} do
|
||||
test "the download archive is formatted correctly and contains the right video" do
|
||||
source = source_fixture(%{collection_type: :channel, last_indexed_at: now()})
|
||||
|
||||
media_items =
|
||||
1..21
|
||||
|> Enum.map(fn n ->
|
||||
|
||||
@@ -294,6 +294,34 @@ defmodule Pinchflat.SourcesTest do
|
||||
assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
|
||||
end
|
||||
|
||||
test "creation will schedule a fast indexing job if the fast_index option is set" do
|
||||
expect(YtDlpRunnerMock, :run, &channel_mock/5)
|
||||
|
||||
valid_attrs = %{
|
||||
media_profile_id: media_profile_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/abc123",
|
||||
fast_index: true
|
||||
}
|
||||
|
||||
assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
|
||||
|
||||
assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
|
||||
end
|
||||
|
||||
test "creation will not schedule a fast indexing job if the fast_index option is not set" do
|
||||
expect(YtDlpRunnerMock, :run, &channel_mock/5)
|
||||
|
||||
valid_attrs = %{
|
||||
media_profile_id: media_profile_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/abc123",
|
||||
fast_index: false
|
||||
}
|
||||
|
||||
assert {:ok, %Source{}} = Sources.create_source(valid_attrs)
|
||||
|
||||
refute_enqueued(worker: FastIndexingWorker)
|
||||
end
|
||||
|
||||
test "creation schedules an index test even if the index frequency is 0" do
|
||||
expect(YtDlpRunnerMock, :run, &channel_mock/5)
|
||||
|
||||
@@ -354,7 +382,56 @@ defmodule Pinchflat.SourcesTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "create_source/2 when testing options" do
|
||||
describe "create_source/2 when testing yt-dlp options" do
|
||||
test "sets use_cookies to true if the source has been set to use cookies" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_source_details, _opts, _ot, addl ->
|
||||
assert Keyword.get(addl, :use_cookies)
|
||||
|
||||
{:ok, playlist_return()}
|
||||
end)
|
||||
|
||||
valid_attrs = %{
|
||||
media_profile_id: media_profile_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/abc123",
|
||||
use_cookies: true
|
||||
}
|
||||
|
||||
assert {:ok, %Source{}} = Sources.create_source(valid_attrs)
|
||||
end
|
||||
|
||||
test "does not set use_cookies to false if the source has not been set to use cookies" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_source_details, _opts, _ot, addl ->
|
||||
refute Keyword.get(addl, :use_cookies)
|
||||
|
||||
{:ok, playlist_return()}
|
||||
end)
|
||||
|
||||
valid_attrs = %{
|
||||
media_profile_id: media_profile_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/abc123",
|
||||
use_cookies: false
|
||||
}
|
||||
|
||||
assert {:ok, %Source{}} = Sources.create_source(valid_attrs)
|
||||
end
|
||||
|
||||
test "skips sleep interval" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, :get_source_details, _opts, _ot, addl ->
|
||||
assert Keyword.get(addl, :skip_sleep_interval)
|
||||
|
||||
{:ok, playlist_return()}
|
||||
end)
|
||||
|
||||
valid_attrs = %{
|
||||
media_profile_id: media_profile_fixture().id,
|
||||
original_url: "https://www.youtube.com/channel/abc123"
|
||||
}
|
||||
|
||||
assert {:ok, %Source{}} = Sources.create_source(valid_attrs)
|
||||
end
|
||||
end
|
||||
|
||||
describe "create_source/2 when testing its options" do
|
||||
test "run_post_commit_tasks: false won't enqueue post-commit tasks" do
|
||||
expect(YtDlpRunnerMock, :run, &channel_mock/5)
|
||||
|
||||
@@ -902,28 +979,30 @@ defmodule Pinchflat.SourcesTest do
|
||||
end
|
||||
|
||||
defp playlist_mock(_url, :get_source_details, _opts, _ot, _addl) do
|
||||
{
|
||||
:ok,
|
||||
Phoenix.json_library().encode!(%{
|
||||
channel: nil,
|
||||
channel_id: nil,
|
||||
playlist_id: "some_playlist_id_#{:rand.uniform(1_000_000)}",
|
||||
playlist_title: "some playlist name"
|
||||
})
|
||||
}
|
||||
{:ok, playlist_return()}
|
||||
end
|
||||
|
||||
defp channel_mock(_url, :get_source_details, _opts, _ot, _addl) do
|
||||
{:ok, channel_return()}
|
||||
end
|
||||
|
||||
defp playlist_return do
|
||||
Phoenix.json_library().encode!(%{
|
||||
channel: nil,
|
||||
channel_id: nil,
|
||||
playlist_id: "some_playlist_id_#{:rand.uniform(1_000_000)}",
|
||||
playlist_title: "some playlist name"
|
||||
})
|
||||
end
|
||||
|
||||
defp channel_return do
|
||||
channel_id = "some_channel_id_#{:rand.uniform(1_000_000)}"
|
||||
|
||||
{
|
||||
:ok,
|
||||
Phoenix.json_library().encode!(%{
|
||||
channel: "some channel name",
|
||||
channel_id: channel_id,
|
||||
playlist_id: channel_id,
|
||||
playlist_title: "some channel name - videos"
|
||||
})
|
||||
}
|
||||
Phoenix.json_library().encode!(%{
|
||||
channel: "some channel name",
|
||||
channel_id: channel_id,
|
||||
playlist_id: channel_id,
|
||||
playlist_title: "some channel name - videos"
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
@@ -47,4 +47,21 @@ defmodule Pinchflat.Utils.NumberUtilsTest do
|
||||
assert NumberUtils.human_byte_size(nil) == {0, "B"}
|
||||
end
|
||||
end
|
||||
|
||||
describe "add_jitter/2" do
|
||||
test "returns 0 when the number is less than or equal to 0" do
|
||||
assert NumberUtils.add_jitter(0) == 0
|
||||
assert NumberUtils.add_jitter(-1) == 0
|
||||
end
|
||||
|
||||
test "returns the number with jitter added" do
|
||||
assert NumberUtils.add_jitter(100) in 100..150
|
||||
end
|
||||
|
||||
test "optionally takes a jitter percentage" do
|
||||
assert NumberUtils.add_jitter(100, 0.1) in 90..110
|
||||
assert NumberUtils.add_jitter(100, 0.5) in 50..150
|
||||
assert NumberUtils.add_jitter(100, 1) in 0..200
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
defmodule Pinchflat.YtDlp.CommandRunnerTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
|
||||
alias Pinchflat.YtDlp.CommandRunner, as: Runner
|
||||
@@ -95,6 +96,36 @@ defmodule Pinchflat.YtDlp.CommandRunnerTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "run/4 when testing sleep interval options" do
|
||||
test "includes sleep interval options by default" do
|
||||
Settings.set(extractor_sleep_interval_seconds: 5)
|
||||
|
||||
assert {:ok, output} = Runner.run(@media_url, :foo, [], "")
|
||||
|
||||
assert String.contains?(output, "--sleep-interval")
|
||||
assert String.contains?(output, "--sleep-requests")
|
||||
assert String.contains?(output, "--sleep-subtitles")
|
||||
end
|
||||
|
||||
test "doesn't include sleep interval options when skip_sleep_interval is true" do
|
||||
assert {:ok, output} = Runner.run(@media_url, :foo, [], "", skip_sleep_interval: true)
|
||||
|
||||
refute String.contains?(output, "--sleep-interval")
|
||||
refute String.contains?(output, "--sleep-requests")
|
||||
refute String.contains?(output, "--sleep-subtitles")
|
||||
end
|
||||
|
||||
test "doesn't include sleep interval options when extractor_sleep_interval_seconds is 0" do
|
||||
Settings.set(extractor_sleep_interval_seconds: 0)
|
||||
|
||||
assert {:ok, output} = Runner.run(@media_url, :foo, [], "")
|
||||
|
||||
refute String.contains?(output, "--sleep-interval")
|
||||
refute String.contains?(output, "--sleep-requests")
|
||||
refute String.contains?(output, "--sleep-subtitles")
|
||||
end
|
||||
end
|
||||
|
||||
describe "run/4 when testing global options" do
|
||||
test "creates windows-safe filenames" do
|
||||
assert {:ok, output} = Runner.run(@media_url, :foo, [], "")
|
||||
@@ -123,6 +154,14 @@ defmodule Pinchflat.YtDlp.CommandRunnerTest do
|
||||
end
|
||||
end
|
||||
|
||||
describe "update/0" do
|
||||
test "adds the update arg" do
|
||||
assert {:ok, output} = Runner.update()
|
||||
|
||||
assert String.contains?(output, "--update")
|
||||
end
|
||||
end
|
||||
|
||||
defp wrap_executable(new_executable, fun) do
|
||||
Application.put_env(:pinchflat, :yt_dlp_executable, new_executable)
|
||||
fun.()
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
defmodule Pinchflat.YtDlp.UpdateWorkerTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.YtDlp.UpdateWorker
|
||||
|
||||
describe "perform/1" do
|
||||
test "calls the yt-dlp runner to update yt-dlp" do
|
||||
expect(YtDlpRunnerMock, :update, fn -> {:ok, ""} end)
|
||||
expect(YtDlpRunnerMock, :version, fn -> {:ok, ""} end)
|
||||
|
||||
perform_job(UpdateWorker, %{})
|
||||
end
|
||||
|
||||
test "saves the new version to the database" do
|
||||
expect(YtDlpRunnerMock, :update, fn -> {:ok, ""} end)
|
||||
expect(YtDlpRunnerMock, :version, fn -> {:ok, "1.2.3"} end)
|
||||
|
||||
perform_job(UpdateWorker, %{})
|
||||
|
||||
assert {:ok, "1.2.3"} = Settings.get(:yt_dlp_version)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user