bumped version

* Update youtube shorts detection to support youtube pants

* Updates a test

README.md

@@ -1,3 +1,6 @@
+> [!IMPORTANT]  
+> (2025-02-14) [zakkarry](https://github.com/sponsors/zakkarry), who is a collaborator on [cross-seed](https://github.com/cross-seed/cross-seed) and an extremely helpful community member in general, is facing hard times due to medical debt and family illness. If you're able, please consider [sponsoring him on GitHub](https://github.com/sponsors/zakkarry) or donating via [buymeacoffee](https://tip.ary.dev). Tell him I sent you!
+
 <p align="center">  
   <img 
     src="priv/static/images/originals/logo-white-wordmark-with-background.png" 
@@ -148,6 +151,7 @@ If you change this setting and it works well for you, please leave a comment on
 | --------------------------- | --------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
 | `TZ`                        | No        | `UTC`                     | Must follow IANA TZ format                                                                                                                |
 | `LOG_LEVEL`                 | No        | `debug`                   | Can be set to `info` but `debug` is strongly recommended                                                                                  |
+| `UMASK`                     | No        | `022`                     | Unraid users may want to set this to `000`                                                                                                |
 | `BASIC_AUTH_USERNAME`       | No        |                           | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password)                                            |
 | `BASIC_AUTH_PASSWORD`       | No        |                           | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password)                                            |
 | `EXPOSE_FEED_ENDPOINTS`     | No        | `false`                   | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds)                                                      |

config/config.exs

@@ -10,6 +10,7 @@ import Config
 config :pinchflat,
   ecto_repos: [Pinchflat.Repo],
   generators: [timestamp_type: :utc_datetime],
+  env: config_env(),
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
   apprise_executable: System.find_executable("apprise"),
@@ -49,16 +50,7 @@ config :pinchflat, PinchflatWeb.Endpoint,
 
 config :pinchflat, Oban,
   engine: Oban.Engines.Lite,
-  repo: Pinchflat.Repo,
-  # Keep old jobs for 30 days for display in the UI
-  plugins: [
-    {Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
-    {Oban.Plugins.Cron,
-     crontab: [
-       {"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
-       {"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
-     ]}
-  ]
+  repo: Pinchflat.Repo
 
 # Configures the mailer
 #

config/runtime.exs

@@ -43,15 +43,30 @@ config :pinchflat, Pinchflat.Repo,
 # Some users may want to increase the number of workers that use yt-dlp to improve speeds
 # Others may want to decrease the number of these workers to lessen the chance of an IP ban
 {yt_dlp_worker_count, _} = Integer.parse(System.get_env("YT_DLP_WORKER_CONCURRENCY", "2"))
+# Used to set the cron for the yt-dlp update worker. The reason for this is
+# to avoid all instances of PF updating yt-dlp at the same time, which 1)
+# could result in rate limiting and 2) gives me time to react if an update
+# breaks something
+%{hour: current_hour, minute: current_minute} = DateTime.utc_now()
 
 config :pinchflat, Oban,
   queues: [
     default: 10,
-    fast_indexing: 6,
+    fast_indexing: yt_dlp_worker_count,
     media_collection_indexing: yt_dlp_worker_count,
     media_fetching: yt_dlp_worker_count,
     remote_metadata: yt_dlp_worker_count,
     local_data: 8
+  ],
+  plugins: [
+    # Keep old jobs for 30 days for display in the UI
+    {Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
+    {Oban.Plugins.Cron,
+     crontab: [
+       {"#{current_minute} #{current_hour} * * *", Pinchflat.YtDlp.UpdateWorker},
+       {"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
+       {"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
+     ]}
   ]
 
 if config_env() == :prod do

docker/selfhosted.Dockerfile

@@ -7,7 +7,7 @@ ARG DEBIAN_VERSION=bookworm-20240612-slim
 ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
 ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
 
-FROM ${BUILDER_IMAGE} as builder
+FROM ${BUILDER_IMAGE} AS builder
 
 ARG TARGETPLATFORM
 RUN echo "Building for ${TARGETPLATFORM:?}"
@@ -110,9 +110,9 @@ RUN apt-get update -y && \
     rm -rf /var/lib/apt/lists/*
 
 # More locale setup
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
 
 WORKDIR "/app"
 
@@ -124,6 +124,7 @@ RUN mkdir -p /config /downloads /etc/elixir_tzdata_data /etc/yt-dlp/plugins && \
 ENV MIX_ENV="prod"
 ENV PORT=${PORT}
 ENV RUN_CONTEXT="selfhosted"
+ENV UMASK=022
 EXPOSE ${PORT}
 
 # Only copy the final release from the build stage

lib/pinchflat/application.ex

@@ -9,8 +9,12 @@ defmodule Pinchflat.Application do
   @impl true
   def start(_type, _args) do
     check_and_update_timezone()
+    attach_oban_telemetry()
+    Logger.add_handlers(:pinchflat)
 
-    children = [
+    # See https://hexdocs.pm/elixir/Supervisor.html
+    # for other strategies and supported options
+    [
       Pinchflat.PromEx,
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
@@ -24,17 +28,11 @@ defmodule Pinchflat.Application do
       {Finch, name: Pinchflat.Finch},
       # Start a worker by calling: Pinchflat.Worker.start_link(arg)
       # {Pinchflat.Worker, arg},
-      # Start to serve requests, typically the last entry
-      PinchflatWeb.Endpoint
+      # Start to serve requests, typically the last entry (except for the post-boot tasks)
+      PinchflatWeb.Endpoint,
+      Pinchflat.Boot.PostBootStartupTasks
     ]
-
-    attach_oban_telemetry()
-    Logger.add_handlers(:pinchflat)
-
-    # See https://hexdocs.pm/elixir/Supervisor.html
-    # for other strategies and supported options
-    opts = [strategy: :one_for_one, name: Pinchflat.Supervisor]
-    Supervisor.start_link(children, opts)
+    |> Supervisor.start_link(strategy: :one_for_one, name: Pinchflat.Supervisor)
   end
 
   # Tell Phoenix to update the endpoint configuration

lib/pinchflat/boot/post_boot_startup_tasks.ex

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Boot.PostBootStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  AFTER all other boot steps have taken place and the app is ready to serve requests.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  alias Pinchflat.YtDlp.UpdateWorker, as: YtDlpUpdateWorker
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
+  end
+
+  @doc """
+  Runs post-boot application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(%{env: :test} = state) do
+    # Do nothing _as part of the app bootup process_.
+    # Since bootup calls `start_link` and that's where the `env` state is injected,
+    # you can still call `.init()` manually to run these tasks for testing purposes
+    {:ok, state}
+  end
+
+  def init(state) do
+    update_yt_dlp()
+
+    {:ok, state}
+  end
+
+  defp update_yt_dlp do
+    YtDlpUpdateWorker.kickoff()
+  end
+end

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,7 +1,7 @@
 defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
   This module is responsible for running startup tasks on app boot
-  AFTER the job runner has initiallized.
+  AFTER the job runner has initialized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -12,7 +12,7 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   import Ecto.Query, warn: false
 
   def start_link(opts \\ []) do
-    GenServer.start_link(__MODULE__, %{}, opts)
+    GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
   end
 
   @doc """
@@ -25,6 +25,13 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   Should be fast - anything with the potential to be slow should be kicked off as a job instead.
   """
   @impl true
+  def init(%{env: :test} = state) do
+    # Do nothing _as part of the app bootup process_.
+    # Since bootup calls `start_link` and that's where the `env` state is injected,
+    # you can still call `.init()` manually to run these tasks for testing purposes
+    {:ok, state}
+  end
+
   def init(state) do
     # Nothing at the moment!
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -19,7 +19,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
   alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
 
   def start_link(opts \\ []) do
-    GenServer.start_link(__MODULE__, %{}, opts)
+    GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
   end
 
   @doc """
@@ -32,6 +32,13 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
   Should be fast - anything with the potential to be slow should be kicked off as a job instead.
   """
   @impl true
+  def init(%{env: :test} = state) do
+    # Do nothing _as part of the app bootup process_.
+    # Since bootup calls `start_link` and that's where the `env` state is injected,
+    # you can still call `.init()` manually to run these tasks for testing purposes
+    {:ok, state}
+  end
+
   def init(state) do
     ensure_tmpfile_directory()
     reset_executing_jobs()

lib/pinchflat/downloading/downloading_helpers.ex

@@ -27,13 +27,15 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
 
   Returns :ok
   """
-  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+  def enqueue_pending_download_tasks(source, job_opts \\ [])
+
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source, job_opts) do
     source
     |> Media.list_pending_media_items_for()
-    |> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
+    |> Enum.each(&MediaDownloadWorker.kickoff_with_task(&1, %{}, job_opts))
   end
 
-  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+  def enqueue_pending_download_tasks(%Source{download_media: false}, _job_opts) do
     :ok
   end
 
@@ -55,13 +57,13 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
 
   Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
   """
-  def kickoff_download_if_pending(%MediaItem{} = media_item) do
+  def kickoff_download_if_pending(%MediaItem{} = media_item, job_opts \\ []) do
     media_item = Repo.preload(media_item, :source)
 
     if media_item.source.download_media && Media.pending_download?(media_item) do
       Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
 
-      MediaDownloadWorker.kickoff_with_task(media_item)
+      MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
     else
       {:error, :should_not_download}
     end

lib/pinchflat/downloading/media_download_worker.ex

@@ -3,6 +3,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
   use Oban.Worker,
     queue: :media_fetching,
+    priority: 5,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching", "show_in_dashboard"]
 
@@ -49,8 +50,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
     media_item = fetch_and_run_prevent_download_user_script(media_item_id)
 
-    # If the source or media item is set to not download media, perform a no-op unless forced
-    if (media_item.source.download_media && !media_item.prevent_download) || should_force do
+    if should_download_media?(media_item, should_force, is_quality_upgrade) do
       download_media_and_schedule_jobs(media_item, is_quality_upgrade, should_force)
     else
       :ok
@@ -60,6 +60,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
     Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
+  # If this is a quality upgrade, only check if the source is set to download media
+  # or that the media item's download hasn't been prevented
+  defp should_download_media?(media_item, should_force, true = _is_quality_upgrade) do
+    (media_item.source.download_media && !media_item.prevent_download) || should_force
+  end
+
+  # If it's not a quality upgrade, additionally check if the media item is pending download
+  defp should_download_media?(media_item, should_force, _is_quality_upgrade) do
+    source = media_item.source
+    is_pending = Media.pending_download?(media_item)
+
+    (is_pending && source.download_media && !media_item.prevent_download) || should_force
+  end
+
   # If a user script exists and, when run, returns a non-zero exit code, prevent this and all future downloads
   # of the media item.
   defp fetch_and_run_prevent_download_user_script(media_item_id) do
@@ -91,13 +105,13 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
         :ok
 
-      {:recovered, _} ->
+      {:recovered, _media_item, _message} ->
         {:error, :retry}
 
-      {:error, :unsuitable_for_download} ->
+      {:error, :unsuitable_for_download, _message} ->
         {:ok, :non_retry}
 
-      {:error, message} ->
+      {:error, _error_atom, message} ->
         action_on_error(message)
     end
   end
@@ -115,7 +129,11 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
   defp action_on_error(message) do
     # This will attempt re-download at the next indexing, but it won't be retried
     # immediately as part of job failure logic
-    non_retryable_errors = ["Video unavailable", "Sign in to confirm"]
+    non_retryable_errors = [
+      "Video unavailable",
+      "Sign in to confirm",
+      "This video is available to this channel's members"
+    ]
 
     if String.contains?(to_string(message), non_retryable_errors) do
       Logger.error("yt-dlp download will not be retried: #{inspect(message)}")

lib/pinchflat/downloading/media_downloader.ex

@@ -10,6 +10,7 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Utils.StringUtils
   alias Pinchflat.Metadata.NfoBuilder
   alias Pinchflat.Metadata.MetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers
@@ -20,16 +21,53 @@ defmodule Pinchflat.Downloading.MediaDownloader do
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
-  returned by yt-dlp. Also saves the entire metadata response to the associated
-  media_metadata record.
+  returned by yt-dlp. Encountered errors are saved to the Media Item record. Saves
+  the entire metadata response to the associated media_metadata record.
 
-  NOTE: related methods (like the download worker) won't download if the media item's source
+  NOTE: related methods (like the download worker) won't download if Pthe media item's source
   is set to not download media. However, I'm not enforcing that here since I need this for testing.
   This may change in the future but I'm not stressed.
 
-  Returns {:ok, %MediaItem{}} | {:error, any, ...any}
+  Returns {:ok, %MediaItem{}} | {:error, atom(), String.t()} | {:recovered, %MediaItem{}, String.t()}
   """
   def download_for_media_item(%MediaItem{} = media_item, override_opts \\ []) do
+    case attempt_download_and_update_for_media_item(media_item, override_opts) do
+      {:ok, media_item} ->
+        # Returns {:ok, %MediaItem{}}
+        Media.update_media_item(media_item, %{last_error: nil})
+
+      {:error, error_atom, message} ->
+        Media.update_media_item(media_item, %{last_error: StringUtils.wrap_string(message)})
+
+        {:error, error_atom, message}
+
+      {:recovered, media_item, message} ->
+        {:ok, updated_media_item} = Media.update_media_item(media_item, %{last_error: StringUtils.wrap_string(message)})
+
+        {:recovered, updated_media_item, message}
+    end
+  end
+
+  # Looks complicated, but here's the key points:
+  # - download_with_options runs a pre-check to see if the media item is suitable for download.
+  # - If the media item fails the precheck, it returns {:error, :unsuitable_for_download, message}
+  # - If the precheck passes but the download fails, it normally returns {:error, :download_failed, message}
+  #   - However, there are some errors we can recover from (eg: failure to communicate with SponsorBlock).
+  #     In this case, we attempt the download anyway and update the media item with what details we do have.
+  #     This case returns {:recovered, updated_media_item, message}
+  #   - If we attempt a retry but it fails, we return {:error, :unrecoverable, message}
+  # - If there is an unknown error unrelated to the above, we return {:error, :unknown, message}
+  # - Finally, if there is no error, we update the media item with the parsed JSON and return {:ok, updated_media_item}
+  #
+  # Restated, here are the return values for each case:
+  # - On success: {:ok, updated_media_item}
+  # - On initial failure but successfully recovered: {:recovered, updated_media_item, message}
+  # - On error: {:error, error_atom, message} where error_atom is one of:
+  #   - `:unsuitable_for_download` if the media item fails the precheck
+  #   - `:unrecoverable` if there was an initial failure and the recovery attempt failed
+  #   - `:download_failed` for all other yt-dlp-related downloading errors
+  #   - `:unknown` for any other errors, including those not related to yt-dlp
+  defp attempt_download_and_update_for_media_item(media_item, override_opts) do
     output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
     media_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
 
@@ -38,31 +76,30 @@ defmodule Pinchflat.Downloading.MediaDownloader do
         update_media_item_from_parsed_json(media_with_preloads, parsed_json)
 
       {:error, :unsuitable_for_download} ->
-        Logger.warning(
+        message =
           "Media item ##{media_with_preloads.id} isn't suitable for download yet. May be an active or processing live stream"
-        )
 
-        {:error, :unsuitable_for_download}
+        Logger.warning(message)
+
+        {:error, :unsuitable_for_download, message}
 
       {:error, message, _exit_code} ->
         Logger.error("yt-dlp download error for media item ##{media_with_preloads.id}: #{inspect(message)}")
 
         if String.contains?(to_string(message), recoverable_errors()) do
-          attempt_update_media_item(media_with_preloads, output_filepath)
-
-          {:recovered, message}
+          attempt_recovery_from_error(media_with_preloads, output_filepath, message)
         else
-          {:error, message}
+          {:error, :download_failed, message}
         end
 
       err ->
         Logger.error("Unknown error downloading media item ##{media_with_preloads.id}: #{inspect(err)}")
 
-        {:error, "Unknown error: #{inspect(err)}"}
+        {:error, :unknown, "Unknown error: #{inspect(err)}"}
     end
   end
 
-  defp attempt_update_media_item(media_with_preloads, output_filepath) do
+  defp attempt_recovery_from_error(media_with_preloads, output_filepath, error_message) do
     with {:ok, contents} <- File.read(output_filepath),
          {:ok, parsed_json} <- Phoenix.json_library().decode(contents) do
       Logger.info("""
@@ -71,12 +108,13 @@ defmodule Pinchflat.Downloading.MediaDownloader do
       anyway
       """)
 
-      update_media_item_from_parsed_json(media_with_preloads, parsed_json)
+      {:ok, updated_media_item} = update_media_item_from_parsed_json(media_with_preloads, parsed_json)
+      {:recovered, updated_media_item, error_message}
     else
       err ->
         Logger.error("Unable to recover error for media item ##{media_with_preloads.id}: #{inspect(err)}")
 
-        {:error, :retry_failed}
+        {:error, :unrecoverable, error_message}
     end
   end
 

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -40,7 +40,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
   Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
   downloaded_.
   """
-  def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
+  def index_and_kickoff_downloads(%Source{} = source) do
     # The media_profile is needed to determine the quality options to _then_ determine a more
     # accurate predicted filepath
     source = Repo.preload(source, [:media_profile])
@@ -53,6 +53,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
       Enum.map(new_media_ids, fn media_id ->
         case create_media_item_from_media_id(source, media_id) do
           {:ok, media_item} ->
+            DownloadingHelpers.kickoff_download_if_pending(media_item, priority: 0)
             media_item
 
           err ->
@@ -61,7 +62,9 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
         end
       end)
 
-    DownloadingHelpers.enqueue_pending_download_tasks(source)
+    # Pick up any stragglers. Intentionally has a lower priority than the per-media item
+    # kickoff above
+    DownloadingHelpers.enqueue_pending_download_tasks(source, priority: 1)
 
     Enum.filter(maybe_new_media_items, & &1)
   end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -38,8 +38,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
 
   Order of operations:
     1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
-       with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
-    2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
+       with `FastIndexingHelpers.index_and_kickoff_downloads`
+    2. If the above `index_and_kickoff_downloads` finds new media items in the RSS feed,
        it indexes them with a yt-dlp call to create the media item records then kicks off downloading
        tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
     3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
@@ -67,7 +67,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
 
     new_media_items =
       source
-      |> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
+      |> FastIndexingHelpers.index_and_kickoff_downloads()
       |> Enum.filter(&Media.pending_download?(&1))
 
     if source.download_media do

lib/pinchflat/fast_indexing/youtube_api.ex

@@ -12,6 +12,8 @@ defmodule Pinchflat.FastIndexing.YoutubeApi do
 
   @behaviour YoutubeBehaviour
 
+  @agent_name {:global, __MODULE__.KeyIndex}
+
   @doc """
   Determines if the YouTube API is enabled for fast indexing by checking
   if the user has an API key set
@@ -19,7 +21,7 @@ defmodule Pinchflat.FastIndexing.YoutubeApi do
   Returns boolean()
   """
   @impl YoutubeBehaviour
-  def enabled?(), do: is_binary(api_key())
+  def enabled?, do: Enum.any?(api_keys())
 
   @doc """
   Fetches the recent media IDs from the YouTube API for a given source.
@@ -74,8 +76,45 @@ defmodule Pinchflat.FastIndexing.YoutubeApi do
     |> FunctionUtils.wrap_ok()
   end
 
-  defp api_key do
-    Settings.get!(:youtube_api_key)
+  defp api_keys do
+    case Settings.get!(:youtube_api_key) do
+      nil ->
+        []
+
+      keys ->
+        keys
+        |> String.split(",")
+        |> Enum.map(&String.trim/1)
+        |> Enum.reject(&(&1 == ""))
+    end
+  end
+
+  defp get_or_start_api_key_agent do
+    case Agent.start(fn -> 0 end, name: @agent_name) do
+      {:ok, pid} -> pid
+      {:error, {:already_started, pid}} -> pid
+    end
+  end
+
+  # Gets the next API key in round-robin fashion
+  defp next_api_key do
+    keys = api_keys()
+
+    case keys do
+      [] ->
+        nil
+
+      keys ->
+        pid = get_or_start_api_key_agent()
+
+        current_index =
+          Agent.get_and_update(pid, fn current ->
+            {current, rem(current + 1, length(keys))}
+          end)
+
+        Logger.debug("Using YouTube API key: #{Enum.at(keys, current_index)}")
+        Enum.at(keys, current_index)
+    end
   end
 
   defp construct_api_endpoint(playlist_id) do
@@ -83,7 +122,7 @@ defmodule Pinchflat.FastIndexing.YoutubeApi do
     property_type = "contentDetails"
     max_results = 50
 
-    "#{api_base}?part=#{property_type}&maxResults=#{max_results}&playlistId=#{playlist_id}&key=#{api_key()}"
+    "#{api_base}?part=#{property_type}&maxResults=#{max_results}&playlistId=#{playlist_id}&key=#{next_api_key()}"
   end
 
   defp http_client do

lib/pinchflat/media/media_item.ex

@@ -40,6 +40,7 @@ defmodule Pinchflat.Media.MediaItem do
     :thumbnail_filepath,
     :metadata_filepath,
     :nfo_filepath,
+    :last_error,
     # These are user or system controlled fields
     :prevent_download,
     :prevent_culling,
@@ -88,6 +89,7 @@ defmodule Pinchflat.Media.MediaItem do
     # Will very likely revisit because I can't leave well-enough alone.
     field :subtitle_filepaths, {:array, {:array, :string}}, default: []
 
+    field :last_error, :string
     field :prevent_download, :boolean, default: false
     field :prevent_culling, :boolean, default: false
     field :culled_at, :utc_datetime

lib/pinchflat/profiles/media_profile_deletion_worker.ex

@@ -14,7 +14,7 @@ defmodule Pinchflat.Profiles.MediaProfileDeletionWorker do
   Starts the profile deletion worker. Does not attach it to a task like `kickoff_with_task/2`
   since deletion also cancels all tasks for the profile
 
-  Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
+  Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
   """
   def kickoff(profile, job_args \\ %{}, job_opts \\ []) do
     %{id: profile.id}

lib/pinchflat/settings/setting.ex

@@ -45,6 +45,6 @@ defmodule Pinchflat.Settings.Setting do
     setting
     |> cast(attrs, @allowed_fields)
     |> validate_required(@required_fields)
-    |> validate_number(:extractor_sleep_interval_seconds, greater_than: 0)
+    |> validate_number(:extractor_sleep_interval_seconds, greater_than_or_equal_to: 0)
   end
 end

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -39,7 +39,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
     job_offset_seconds = if job_args[:force], do: 0, else: calculate_job_offset_seconds(source)
 
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
 
     MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts ++ [schedule_in: job_offset_seconds])
@@ -231,8 +230,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   # The download archive isn't useful for playlists (since those are ordered arbitrarily)
   # and we don't want to use it if the indexing was forced by the user. In other words,
   # only create an archive for channels that are being indexed as part of their regular
-  # indexing schedule
+  # indexing schedule. The first indexing pass should also not create an archive.
   defp build_download_archive_options(%Source{collection_type: :playlist}, _was_forced), do: []
+  defp build_download_archive_options(%Source{last_indexed_at: nil}, _was_forced), do: []
   defp build_download_archive_options(_source, true), do: []
 
   defp build_download_archive_options(source, _was_forced) do

lib/pinchflat/sources/sources.ex

@@ -300,6 +300,10 @@ defmodule Pinchflat.Sources do
       %{__meta__: %{state: :built}} ->
         SlowIndexingHelpers.kickoff_indexing_task(source)
 
+        if Ecto.Changeset.get_field(changeset, :fast_index) do
+          FastIndexingHelpers.kickoff_indexing_task(source)
+        end
+
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
       %{__meta__: %{state: :loaded}} ->

lib/pinchflat/utils/string_utils.ex

@@ -35,4 +35,13 @@ defmodule Pinchflat.Utils.StringUtils do
   def double_brace(string) do
     "{{ #{string} }}"
   end
+
+  @doc """
+  Wraps a string in quotes if it's not already a string. Useful for working with
+  error messages whose types can vary.
+
+  Returns binary()
+  """
+  def wrap_string(message) when is_binary(message), do: message
+  def wrap_string(message), do: "#{inspect(message)}"
 end

lib/pinchflat/yt_dlp/command_runner.ex

@@ -76,6 +76,24 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     end
   end
 
+  @doc """
+  Updates yt-dlp to the latest version
+
+  Returns {:ok, binary()} | {:error, binary()}
+  """
+  @impl YtDlpCommandRunner
+  def update do
+    command = backend_executable()
+
+    case CliUtils.wrap_cmd(command, ["--update"]) do
+      {output, 0} ->
+        {:ok, String.trim(output)}
+
+      {output, _} ->
+        {:error, output}
+    end
+  end
+
   defp generate_output_filepath(addl_opts) do
     case Keyword.get(addl_opts, :output_filepath) do
       nil -> FSUtils.generate_metadata_tmpfile(:json)

lib/pinchflat/yt_dlp/media.ex

@@ -151,7 +151,7 @@ defmodule Pinchflat.YtDlp.Media do
       #
       # These don't fail if duration or aspect_ratio are missing
       # due to Elixir's comparison semantics
-      response["duration"] <= 60 && response["aspect_ratio"] <= 0.85
+      response["duration"] <= 180 && response["aspect_ratio"] <= 0.85
     end
   end
 

lib/pinchflat/yt_dlp/update_worker.ex

@@ -0,0 +1,44 @@
+defmodule Pinchflat.YtDlp.UpdateWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_data,
+    tags: ["local_data"]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Settings
+
+  @doc """
+  Starts the yt-dlp update worker. Does not attach it to a task like `kickoff_with_task/2`
+
+  Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff do
+    Oban.insert(UpdateWorker.new(%{}))
+  end
+
+  @doc """
+  Updates yt-dlp and saves the version to the settings.
+
+  This worker is scheduled to run via the Oban Cron plugin as well as on app boot.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    Logger.info("Updating yt-dlp")
+
+    yt_dlp_runner().update()
+
+    {:ok, yt_dlp_version} = yt_dlp_runner().version()
+    Settings.set(yt_dlp_version: yt_dlp_version)
+
+    :ok
+  end
+
+  defp yt_dlp_runner do
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

lib/pinchflat/yt_dlp/yt_dlp_command_runner.ex

@@ -9,4 +9,5 @@ defmodule Pinchflat.YtDlp.YtDlpCommandRunner do
   @callback run(binary(), atom(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
   @callback run(binary(), atom(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
   @callback version() :: {:ok, binary()} | {:error, binary()}
+  @callback update() :: {:ok, binary()} | {:error, binary()}
 end

lib/pinchflat_web/components/custom_components/button_components.ex

@@ -3,6 +3,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
   use Phoenix.Component, global_prefixes: ~w(x-)
 
   alias PinchflatWeb.CoreComponents
+  alias PinchflatWeb.CustomComponents.TextComponents
 
   @doc """
   Render a button
@@ -104,7 +105,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
 
   def icon_button(assigns) do
     ~H"""
-    <div class="group relative inline-block">
+    <TextComponents.tooltip position="bottom" tooltip={@tooltip} tooltip_class="text-nowrap">
       <button
         class={[
           "flex justify-center items-center rounded-lg ",
@@ -117,18 +118,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
       >
         <CoreComponents.icon name={@icon_name} class="text-stroke" />
       </button>
-      <div
-        :if={@tooltip}
-        class={[
-          "hidden absolute left-1/2 top-full z-20 mt-3 -translate-x-1/2 whitespace-nowrap rounded-md",
-          "px-4.5 py-1.5 text-sm font-medium opacity-0 drop-shadow-4 group-hover:opacity-100 group-hover:block bg-meta-4"
-        ]}
-      >
-        <span class="border-light absolute -top-1 left-1/2 -z-10 h-2 w-2 -translate-x-1/2 rotate-45 rounded-sm bg-meta-4">
-        </span>
-        <span>{@tooltip}</span>
-      </div>
-    </div>
+    </TextComponents.tooltip>
     """
   end
 end

lib/pinchflat_web/components/custom_components/text_components.ex

@@ -146,4 +146,60 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
     <.localized_number number={@num} /> {@suffix}
     """
   end
+
+  @doc """
+  Renders a tooltip with the given content
+  """
+
+  attr :tooltip, :string, required: true
+  attr :position, :string, default: ""
+  attr :tooltip_class, :any, default: ""
+  attr :tooltip_arrow_class, :any, default: ""
+  slot :inner_block
+
+  def tooltip(%{position: "bottom-right"} = assigns) do
+    ~H"""
+    <.tooltip tooltip={@tooltip} tooltip_class={@tooltip_class} tooltip_arrow_class={["-top-1", @tooltip_arrow_class]}>
+      {render_slot(@inner_block)}
+    </.tooltip>
+    """
+  end
+
+  def tooltip(%{position: "bottom"} = assigns) do
+    ~H"""
+    <.tooltip
+      tooltip={@tooltip}
+      tooltip_class={["left-1/2 -translate-x-1/2", @tooltip_class]}
+      tooltip_arrow_class={["-top-1 left-1/2 -translate-x-1/2", @tooltip_arrow_class]}
+    >
+      {render_slot(@inner_block)}
+    </.tooltip>
+    """
+  end
+
+  def tooltip(assigns) do
+    ~H"""
+    <div class="group relative inline-block cursor-pointer">
+      <div>
+        {render_slot(@inner_block)}
+      </div>
+      <div
+        :if={@tooltip}
+        class={[
+          "hidden absolute top-full z-20 mt-3 whitespace-nowrap rounded-md",
+          "p-1.5 text-sm font-medium opacity-0 drop-shadow-4 group-hover:opacity-100 group-hover:block bg-meta-4",
+          "border border-form-strokedark text-wrap",
+          @tooltip_class
+        ]}
+      >
+        <span class={[
+          "border-t border-l border-form-strokedark absolute -z-10 h-2 w-2 rotate-45 rounded-sm bg-meta-4",
+          @tooltip_arrow_class
+        ]}>
+        </span>
+        <div class="px-3">{@tooltip}</div>
+      </div>
+    </div>
+    """
+  end
 end

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -16,7 +16,7 @@
     </.link>
   </nav>
 </div>
-<div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark px-7.5">
+<div class="rounded-sm border py-5 pt-6 shadow-default border-strokedark bg-boxdark px-7.5">
   <div class="max-w-full">
     <.tabbed_layout>
       <:tab_append>
@@ -24,7 +24,15 @@
       </:tab_append>
 
       <:tab title="Media" id="media">
-        <div class="flex flex-col gap-10 dark:text-white">
+        <div class="flex flex-col gap-10 text-white">
+          <section :if={@media_item.last_error} class="mt-6">
+            <div class="flex items-center gap-1 mb-2">
+              <.icon name="hero-exclamation-circle-solid" class="text-red-500" />
+              <h3 class="font-bold text-xl">Last Error</h3>
+            </div>
+            <span>{@media_item.last_error}</span>
+          </section>
+
           <%= if media_file_exists?(@media_item) do %>
             <section class="grid grid-cols-1 xl:gap-6 mt-6">
               <div>
@@ -54,7 +62,8 @@
             </section>
           <% end %>
 
-          <h3 class="font-bold text-xl mt-6">Raw Attributes</h3>
+          <section>
+            <h3 class="font-bold text-xl mb-2">Raw Attributes</h3>
             <section>
               <strong>Source:</strong>
               <.subtle_link href={~p"/sources/#{@media_item.source_id}"}>
@@ -62,11 +71,12 @@
               </.subtle_link>
               <.list_items_from_map map={Map.from_struct(@media_item)} />
             </section>
+          </section>
         </div>
       </:tab>
       <:tab title="Tasks" id="tasks">
         <%= if match?([_|_], @media_item.tasks) do %>
-          <.table rows={@media_item.tasks} table_class="text-black dark:text-white">
+          <.table rows={@media_item.tasks} table_class="text-white">
             <:col :let={task} label="Worker">
               {task.job.worker}
             </:col>

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -25,8 +25,10 @@
 
       <:tab title="Media Profile" id="media-profile">
         <div class="flex flex-col gap-10 text-white">
-          <h3 class="font-bold text-xl mt-6">Raw Attributes</h3>
+          <section>
+            <h3 class="font-bold text-xl mt-6 mb-2">Raw Attributes</h3>
             <.list_items_from_map map={Map.from_struct(@media_profile)} />
+          </section>
         </div>
       </:tab>
       <:tab title="Sources" id="sources">

lib/pinchflat_web/controllers/pages/page_html/history_table_live.ex

@@ -28,10 +28,22 @@ defmodule Pinchflat.Pages.HistoryTableLive do
       </span>
       <div class="max-w-full overflow-x-auto">
         <.table rows={@records} table_class="text-white">
-          <:col :let={media_item} label="Title" class="truncate max-w-xs">
-            <.subtle_link href={~p"/sources/#{media_item.source_id}/media/#{media_item}"}>
+          <:col :let={media_item} label="Title" class="max-w-xs">
+            <section class="flex items-center space-x-1">
+              <.tooltip
+                :if={media_item.last_error}
+                tooltip={media_item.last_error}
+                position="bottom-right"
+                tooltip_class="w-64"
+              >
+                <.icon name="hero-exclamation-circle-solid" class="text-red-500" />
+              </.tooltip>
+              <span class="truncate">
+                <.subtle_link href={~p"/sources/#{media_item.source_id}/media/#{media_item.id}"}>
                   {media_item.title}
                 </.subtle_link>
+              </span>
+            </section>
           </:col>
           <:col :let={media_item} label="Upload Date">
             {DateTime.to_date(media_item.uploaded_at)}

lib/pinchflat_web/controllers/settings/setting_html/setting_form.html.heex

@@ -34,9 +34,9 @@
 
       <.input
         field={f[:youtube_api_key]}
-        placeholder="ABC123"
+        placeholder="ABC123,DEF456"
         type="text"
-        label="YouTube API Key"
+        label="YouTube API Key(s)"
         help={youtube_api_help()}
         html_help={true}
         inputclass="font-mono text-sm mr-4"

lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex

@@ -46,10 +46,22 @@ defmodule PinchflatWeb.Sources.MediaItemTableLive do
         </div>
       </header>
       <.table rows={@records} table_class="text-white">
-        <:col :let={media_item} label="Title" class="truncate max-w-xs">
+        <:col :let={media_item} label="Title" class="max-w-xs">
+          <section class="flex items-center space-x-1">
+            <.tooltip
+              :if={media_item.last_error}
+              tooltip={media_item.last_error}
+              position="bottom-right"
+              tooltip_class="w-64"
+            >
+              <.icon name="hero-exclamation-circle-solid" class="text-red-500" />
+            </.tooltip>
+            <span class="truncate">
               <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
                 {media_item.title}
               </.subtle_link>
+            </span>
+          </section>
         </:col>
         <:col :let={media_item} :if={@media_state == "other"} label="Manually Ignored?">
           <.icon name={if media_item.prevent_download, do: "hero-check", else: "hero-x-mark"} />
@@ -205,6 +217,6 @@ defmodule PinchflatWeb.Sources.MediaItemTableLive do
 
   # Selecting only what we need GREATLY speeds up queries on large tables
   defp select_fields do
-    [:id, :title, :uploaded_at, :prevent_download]
+    [:id, :title, :uploaded_at, :prevent_download, :last_error]
   end
 end

lib/pinchflat_web/controllers/sources/source_html/show.html.heex

@@ -24,8 +24,9 @@
       </:tab_append>
 
       <:tab title="Source" id="source">
-        <div class="flex flex-col gap-10 text-white">
-          <h3 class="font-bold text-xl mt-6">Raw Attributes</h3>
+        <div class="flex flex-col text-white gap-10">
+          <section>
+            <h3 class="font-bold text-xl mb-2 mt-6">Raw Attributes</h3>
             <section>
               <strong>Media Profile:</strong>
               <.subtle_link href={~p"/media_profiles/#{@source.media_profile_id}"}>
@@ -34,6 +35,7 @@
             </section>
 
             <.list_items_from_map map={Map.from_struct(@source)} />
+          </section>
         </div>
       </:tab>
       <:tab title="Pending" id="pending">

lib/pinchflat_web/endpoint.ex

@@ -20,7 +20,7 @@ defmodule PinchflatWeb.Endpoint do
   plug Plug.Static,
     at: "/",
     from: :pinchflat,
-    gzip: Mix.env() == :prod,
+    gzip: Application.compile_env(:pinchflat, :env) == :prod,
     only: PinchflatWeb.static_paths()
 
   # Code reloading can be explicitly enabled under the

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "2025.1.14",
+      version: "2025.2.20",
       elixir: "~> 1.17",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

priv/repo/migrations/20250210201413_add_last_error_to_media_item.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddLastErrorToMediaItem do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :last_error, :string
+    end
+  end
+end

rel/overlays/bin/docker_start

@@ -6,6 +6,9 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
+echo "Setting umask to ${UMASK}"
+umask ${UMASK}
+
 /app/bin/migrate
 
 cd -P -- "$(dirname -- "$0")"

test/pinchflat/boot/post_boot_startup_tasks_test.exs

@@ -0,0 +1,16 @@
+defmodule Pinchflat.Boot.PostBootStartupTasksTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.YtDlp.UpdateWorker
+  alias Pinchflat.Boot.PostBootStartupTasks
+
+  describe "update_yt_dlp" do
+    test "enqueues an update job" do
+      assert [] = all_enqueued(worker: UpdateWorker)
+
+      PostBootStartupTasks.init(%{})
+
+      assert [%Oban.Job{}] = all_enqueued(worker: UpdateWorker)
+    end
+  end
+end

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -10,7 +10,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
   alias Pinchflat.Downloading.MediaDownloadWorker
 
   describe "enqueue_pending_download_tasks/1" do
-    test "it enqueues a job for each pending media item" do
+    test "enqueues a job for each pending media item" do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
@@ -19,7 +19,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
     end
 
-    test "it does not enqueue a job for media items with a filepath" do
+    test "does not enqueue a job for media items with a filepath" do
       source = source_fixture()
       _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
 
@@ -28,7 +28,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
-    test "it attaches a task to each enqueued job" do
+    test "attaches a task to each enqueued job" do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
@@ -39,7 +39,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       assert [_] = Tasks.list_tasks_for(media_item)
     end
 
-    test "it does not create a job if the source is set to not download" do
+    test "does not create a job if the source is set to not download" do
       source = source_fixture(download_media: false)
 
       assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
@@ -47,17 +47,26 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
-    test "it does not attach tasks if the source is set to not download" do
+    test "does not attach tasks if the source is set to not download" do
       source = source_fixture(download_media: false)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
       assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
       assert [] = Tasks.list_tasks_for(media_item)
     end
+
+    test "can pass job options" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source, priority: 1)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}, priority: 1)
+    end
   end
 
   describe "dequeue_pending_download_tasks/1" do
-    test "it deletes all pending tasks for a source's media items" do
+    test "deletes all pending tasks for a source's media items" do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
@@ -109,6 +118,14 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
 
       refute_enqueued(worker: MediaDownloadWorker)
     end
+
+    test "can pass job options" do
+      media_item = media_item_fixture(media_filepath: nil)
+
+      assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item, priority: 1)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id}, priority: 1)
+    end
   end
 
   describe "kickoff_redownload_for_existing_media/1" do

test/pinchflat/downloading/media_download_worker_test.exs

@@ -46,13 +46,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id, "force" => true})
     end
 
-    test "can be called with additional job options", %{media_item: media_item} do
-      job_opts = [max_attempts: 5]
-
-      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
+    test "has a priority of 5 by default", %{media_item: media_item} do
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item)
 
       [job] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-      assert job.max_attempts == 5
+
+      assert job.priority == 5
+    end
+
+    test "priority can be set", %{media_item: media_item} do
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{}, priority: 0)
+
+      [job] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+
+      assert job.priority == 0
     end
   end
 
@@ -67,7 +74,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       :ok
     end
 
-    test "it saves attributes to the media_item", %{media_item: media_item} do
+    test "saves attributes to the media_item", %{media_item: media_item} do
       assert media_item.media_filepath == nil
       perform_job(MediaDownloadWorker, %{id: media_item.id})
       media_item = Repo.reload(media_item)
@@ -75,20 +82,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert media_item.media_filepath != nil
     end
 
-    test "it saves the metadata to the media_item", %{media_item: media_item} do
+    test "saves the metadata to the media_item", %{media_item: media_item} do
       assert media_item.metadata == nil
       perform_job(MediaDownloadWorker, %{id: media_item.id})
       assert Repo.reload(media_item).metadata != nil
     end
 
-    test "it won't double-schedule downloading jobs", %{media_item: media_item} do
+    test "won't double-schedule downloading jobs", %{media_item: media_item} do
       Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
       Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
 
       assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
 
-    test "it sets the job to retryable if the download fails", %{media_item: media_item} do
+    test "sets the job to retryable if the download fails", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 2, fn
         _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
         _url, :download, _opts, _ot, _addl -> {:error, "error"}
@@ -140,7 +147,23 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       end)
     end
 
-    test "it ensures error are returned in a 2-item tuple", %{media_item: media_item} do
+    test "does not set the job to retryable you aren't a member", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 2, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl ->
+          {:ok, "{}"}
+
+        _url, :download, _opts, _ot, _addl ->
+          {:error, "This video is available to this channel's members on level: foo", 1}
+      end)
+
+      Oban.Testing.with_testing_mode(:inline, fn ->
+        {:ok, job} = Oban.insert(MediaDownloadWorker.new(%{id: media_item.id, quality_upgrade?: true}))
+
+        assert job.state == "completed"
+      end)
+    end
+
+    test "ensures error are returned in a 2-item tuple", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 2, fn
         _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
         _url, :download, _opts, _ot, _addl -> {:error, "error", 1}
@@ -149,7 +172,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert {:error, :download_failed} = perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
-    test "it does not download if the source is set to not download", %{media_item: media_item} do
+    test "does not download if the source is set to not download", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
 
       Sources.update_source(media_item.source, %{download_media: false})
@@ -165,7 +188,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
-    test "it saves the file's size to the database", %{media_item: media_item} do
+    test "saves the file's size to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 3, fn
         _url, :get_downloadable_status, _opts, _ot, _addl ->
           {:ok, "{}"}
@@ -214,6 +237,14 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
 
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
+
+    test "does not download if the media item isn't pending download", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
+
+      Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id})
+    end
   end
 
   describe "perform/1 when testing non-downloadable media" do
@@ -232,12 +263,30 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
 
   describe "perform/1 when testing forced downloads" do
     test "ignores 'prevent_download' if forced", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 3, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
+        _url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
+        _url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
+      end)
+
       Sources.update_source(media_item.source, %{download_media: false})
       Media.update_media_item(media_item, %{prevent_download: true})
 
       perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
     end
 
+    test "ignores whether the media item is pending when forced", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 3, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
+        _url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
+        _url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
+      end)
+
+      Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
+    end
+
     test "sets force_overwrites runner option", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 3, fn
         _url, :get_downloadable_status, _opts, _ot, _addl ->
@@ -265,6 +314,34 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert media_item.media_redownloaded_at != nil
     end
 
+    test "ignores whether the media item is pending when re-downloaded", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 3, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
+        _url, :download, _opts, _ot, _addl -> {:ok, render_metadata(:media_metadata)}
+        _url, :download_thumbnail, _opts, _ot, _addl -> {:ok, ""}
+      end)
+
+      Media.update_media_item(media_item, %{media_filepath: "foo.mp4"})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
+    end
+
+    test "doesn't redownload if the source is set to not download", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
+
+      Sources.update_source(media_item.source, %{download_media: false})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
+    end
+
+    test "doesn't redownload if the media item is set to not download", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> :ok end)
+
+      Media.update_media_item(media_item, %{prevent_download: true})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id, quality_upgrade?: true})
+    end
+
     test "sets force_overwrites runner option", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, 3, fn
         _url, :get_downloadable_status, _opts, _ot, _addl ->

test/pinchflat/downloading/media_downloader_test.exs

@@ -60,7 +60,8 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
         {:ok, Phoenix.json_library().encode!(%{"live_status" => "is_live"})}
       end)
 
-      assert {:error, :unsuitable_for_download} = MediaDownloader.download_for_media_item(media_item)
+      assert {:error, :unsuitable_for_download, message} = MediaDownloader.download_for_media_item(media_item)
+      assert message =~ "Media item ##{media_item.id} isn't suitable for download yet."
     end
 
     test "non-recoverable errors are passed through", %{media_item: media_item} do
@@ -69,7 +70,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
         _url, :download, _opts, _ot, _addl -> {:error, :some_error, 1}
       end)
 
-      assert {:error, :some_error} = MediaDownloader.download_for_media_item(media_item)
+      assert {:error, :download_failed, :some_error} = MediaDownloader.download_for_media_item(media_item)
     end
 
     test "unknown errors are passed through", %{media_item: media_item} do
@@ -78,7 +79,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
         _url, :download, _opts, _ot, _addl -> {:error, :some_error}
       end)
 
-      assert {:error, message} = MediaDownloader.download_for_media_item(media_item)
+      assert {:error, :unknown, message} = MediaDownloader.download_for_media_item(media_item)
       assert message == "Unknown error: {:error, :some_error}"
     end
   end
@@ -107,13 +108,15 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
 
       expect(YtDlpRunnerMock, :run, 0, fn _url, :download, _opts, _ot, _addl -> {:ok, ""} end)
 
-      assert {:error, :unsuitable_for_download} = MediaDownloader.download_for_media_item(media_item)
+      assert {:error, :unsuitable_for_download, message} = MediaDownloader.download_for_media_item(media_item)
+      assert message =~ "Media item ##{media_item.id} isn't suitable for download yet."
     end
 
     test "returns unexpected errors from the download status determination method", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, :get_downloadable_status, _opts, _ot, _addl -> {:error, :what_tha} end)
 
-      assert {:error, "Unknown error: {:error, :what_tha}"} = MediaDownloader.download_for_media_item(media_item)
+      assert {:error, :unknown, "Unknown error: {:error, :what_tha}"} =
+               MediaDownloader.download_for_media_item(media_item)
     end
   end
 
@@ -184,15 +187,21 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
     test "returns a recovered tuple on recoverable errors", %{media_item: media_item} do
       message = "Unable to communicate with SponsorBlock"
 
-      expect(YtDlpRunnerMock, :run, 2, fn
+      expect(YtDlpRunnerMock, :run, 3, fn
         _url, :get_downloadable_status, _opts, _ot, _addl ->
           {:ok, "{}"}
 
-        _url, :download, _opts, _ot, _addl ->
+        _url, :download, _opts, _ot, addl ->
+          [{:output_filepath, filepath} | _] = addl
+          File.write(filepath, render_metadata(:media_metadata))
+
           {:error, message, 1}
+
+        _url, :download_thumbnail, _opts, _ot, _addl ->
+          {:ok, ""}
       end)
 
-      assert {:recovered, ^message} = MediaDownloader.download_for_media_item(media_item)
+      assert {:recovered, _media_item, ^message} = MediaDownloader.download_for_media_item(media_item)
     end
 
     test "attempts to update the media item on recoverable errors", %{media_item: media_item} do
@@ -212,11 +221,59 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
           {:ok, ""}
       end)
 
-      assert {:recovered, ^message} = MediaDownloader.download_for_media_item(media_item)
+      assert {:recovered, updated_media_item, ^message} = MediaDownloader.download_for_media_item(media_item)
+
+      assert DateTime.diff(DateTime.utc_now(), updated_media_item.media_downloaded_at) < 2
+      assert String.ends_with?(updated_media_item.media_filepath, ".mkv")
+    end
+
+    test "returns an unrecoverable tuple if recovery fails", %{media_item: media_item} do
+      message = "Unable to communicate with SponsorBlock"
+
+      expect(YtDlpRunnerMock, :run, 2, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl ->
+          {:ok, "{}"}
+
+        _url, :download, _opts, _ot, _addl ->
+          # This errors because the metadata is not written to the file so JSON parsing fails
+          {:error, message, 1}
+      end)
+
+      assert {:error, :unrecoverable, ^message} = MediaDownloader.download_for_media_item(media_item)
+    end
+
+    test "sets the last_error appropriately when recovered", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 3, fn
+        _url, :download, _opts, _ot, addl ->
+          [{:output_filepath, filepath} | _] = addl
+          File.write(filepath, render_metadata(:media_metadata))
+
+          {:error, "Unable to communicate with SponsorBlock", 1}
+
+        _url, :get_downloadable_status, _opts, _ot, _addl ->
+          {:ok, "{}"}
+
+        _url, :download_thumbnail, _opts, _ot, _addl ->
+          {:ok, ""}
+      end)
+
+      assert {:recovered, updated_media_item, _message} = MediaDownloader.download_for_media_item(media_item)
+      assert updated_media_item.last_error == "Unable to communicate with SponsorBlock"
+    end
+
+    test "sets the last_error appropriately when unrecoverable", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 2, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl ->
+          {:ok, "{}"}
+
+        _url, :download, _opts, _ot, _addl ->
+          {:error, "Unable to communicate with SponsorBlock", 1}
+      end)
+
+      assert {:error, :unrecoverable, _message} = MediaDownloader.download_for_media_item(media_item)
       media_item = Repo.reload(media_item)
 
-      assert DateTime.diff(DateTime.utc_now(), media_item.media_downloaded_at) < 2
-      assert String.ends_with?(media_item.media_filepath, ".mkv")
+      assert media_item.last_error == "Unable to communicate with SponsorBlock"
     end
   end
 
@@ -324,6 +381,25 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
 
       File.rm(updated_media_item.metadata_filepath)
     end
+
+    test "sets the last_error to nil on success" do
+      media_item = media_item_fixture(%{last_error: "Some error"})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+      assert updated_media_item.last_error == nil
+    end
+
+    test "sets the last_error to the error message on failure", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 2, fn
+        _url, :get_downloadable_status, _opts, _ot, _addl -> {:ok, "{}"}
+        _url, :download, _opts, _ot, _addl -> {:error, :some_error}
+      end)
+
+      assert {:error, :unknown, _message} = MediaDownloader.download_for_media_item(media_item)
+      media_item = Repo.reload(media_item)
+
+      assert media_item.last_error == "Unknown error: {:error, :some_error}"
+    end
   end
 
   describe "download_for_media_item/3 when testing NFO generation" do

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -38,36 +38,48 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
     end
   end
 
-  describe "kickoff_download_tasks_from_youtube_rss_feed/1" do
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+  describe "index_and_kickoff_downloads/1" do
+    test "enqueues a worker for each new media_id in the source's RSS feed", %{source: source} do
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
 
-      assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [media_item] = FastIndexingHelpers.index_and_kickoff_downloads(source)
 
       assert [worker] = all_enqueued(worker: MediaDownloadWorker)
       assert worker.args["id"] == media_item.id
+      assert worker.priority == 0
     end
 
     test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
       media_item_fixture(source_id: source.id, media_id: "test_1")
 
-      assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
 
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
+    test "kicks off a download task for all pending media but at a lower priority", %{source: source} do
+      pending_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
+
+      assert [worker_1, _worker_2] = all_enqueued(worker: MediaDownloadWorker)
+      assert worker_1.args["id"] == pending_item.id
+      assert worker_1.priority == 1
+    end
+
     test "returns the found media items", %{source: source} do
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "does not enqueue a download job if the source does not allow it" do
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
       source = source_fixture(%{download_media: false})
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
 
       refute_enqueued(worker: MediaDownloadWorker)
     end
@@ -75,7 +87,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
     test "creates a download task record", %{source: source} do
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
 
-      assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [media_item] = FastIndexingHelpers.index_and_kickoff_downloads(source)
 
       assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
     end
@@ -89,7 +101,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
         {:ok, media_attributes_return_fixture()}
       end)
 
-      FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "sets use_cookies if the source uses cookies" do
@@ -103,7 +115,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       source = source_fixture(%{use_cookies: true})
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "does not set use_cookies if the source does not use cookies" do
@@ -117,7 +129,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       source = source_fixture(%{use_cookies: false})
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "does not enqueue a download job if the media item does not match the format rules" do
@@ -142,7 +154,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
         {:ok, output}
       end)
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
 
       refute_enqueued(worker: MediaDownloadWorker)
     end
@@ -154,7 +166,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
         {:ok, "{}"}
       end)
 
-      assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "does not blow up if a media item causes a yt-dlp error", %{source: source} do
@@ -164,11 +176,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
         {:error, "message", 1}
       end)
 
-      assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
   end
 
-  describe "kickoff_download_tasks_from_youtube_rss_feed/1 when testing backends" do
+  describe "index_and_kickoff_downloads/1 when testing backends" do
     test "uses the YouTube API if it is enabled", %{source: source} do
       expect(HTTPClientMock, :get, fn url, _headers ->
         assert url =~ "https://youtube.googleapis.com/youtube/v3/playlistItems"
@@ -178,7 +190,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       Settings.set(youtube_api_key: "test_key")
 
-      assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "the YouTube API creates records as expected", %{source: source} do
@@ -188,7 +200,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       Settings.set(youtube_api_key: "test_key")
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "RSS is used as a backup if the API fails", %{source: source} do
@@ -197,7 +209,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       Settings.set(youtube_api_key: "test_key")
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
 
     test "RSS is used if the API is not enabled", %{source: source} do
@@ -209,7 +221,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
 
       Settings.set(youtube_api_key: nil)
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert [%MediaItem{}] = FastIndexingHelpers.index_and_kickoff_downloads(source)
     end
   end
 end

test/pinchflat/fast_indexing/youtube_api_test.exs

@@ -7,31 +7,67 @@ defmodule Pinchflat.FastIndexing.YoutubeApiTest do
   alias Pinchflat.FastIndexing.YoutubeApi
 
   describe "enabled?/0" do
-    test "returns true if the user has set a YouTube API key" do
+    test "returns true if the user has set YouTube API keys" do
+      Settings.set(youtube_api_key: "key1, key2")
+      assert YoutubeApi.enabled?()
+    end
+
+    test "returns true with a single API key" do
       Settings.set(youtube_api_key: "test_key")
 
       assert YoutubeApi.enabled?()
     end
 
-    test "returns false if the user has not set an API key" do
+    test "returns false if the user has not set any API keys" do
       Settings.set(youtube_api_key: nil)
+      refute YoutubeApi.enabled?()
+    end
 
+    test "returns false if only empty or whitespace keys are provided" do
+      Settings.set(youtube_api_key: "  ,  ,")
       refute YoutubeApi.enabled?()
     end
   end
 
   describe "get_recent_media_ids/1" do
     setup do
+      case :global.whereis_name(YoutubeApi.KeyIndex) do
+        :undefined -> :ok
+        pid -> Agent.stop(pid)
+      end
+
       source = source_fixture()
-      Settings.set(youtube_api_key: "test_key")
+      Settings.set(youtube_api_key: "key1, key2")
 
       {:ok, source: source}
     end
 
+    test "rotates through API keys", %{source: source} do
+      expect(HTTPClientMock, :get, fn url, _headers ->
+        assert url =~ "key=key1"
+        {:ok, "{}"}
+      end)
+
+      expect(HTTPClientMock, :get, fn url, _headers ->
+        assert url =~ "key=key2"
+        {:ok, "{}"}
+      end)
+
+      expect(HTTPClientMock, :get, fn url, _headers ->
+        assert url =~ "key=key1"
+        {:ok, "{}"}
+      end)
+
+      # three calls to verify rotation
+      YoutubeApi.get_recent_media_ids(source)
+      YoutubeApi.get_recent_media_ids(source)
+      YoutubeApi.get_recent_media_ids(source)
+    end
+
     test "calls the expected URL", %{source: source} do
       expect(HTTPClientMock, :get, fn url, headers ->
         api_base = "https://youtube.googleapis.com/youtube/v3/playlistItems"
-        request_url = "#{api_base}?part=contentDetails&maxResults=50&playlistId=#{source.collection_id}&key=test_key"
+        request_url = "#{api_base}?part=contentDetails&maxResults=50&playlistId=#{source.collection_id}&key=key1"
 
         assert url == request_url
         assert headers == [accept: "application/json"]

test/pinchflat/settings_test.exs

@@ -85,5 +85,12 @@ defmodule Pinchflat.SettingsTest do
       assert %Ecto.Changeset{valid?: true} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: 0})
       assert %Ecto.Changeset{valid?: false} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: -1})
     end
+
+    test "allows you to reset the extractor sleep interval" do
+      setting = Settings.record()
+      assert {:ok, setting} = Settings.update_setting(setting, %{extractor_sleep_interval_seconds: 1})
+
+      assert %Ecto.Changeset{valid?: true} = Settings.change_setting(setting, %{extractor_sleep_interval_seconds: 0})
+    end
   end
 end

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -96,26 +96,6 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
 
-    test "deletes any pending media tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-
-    test "deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-
     test "can be called with additional job arguments" do
       source = source_fixture(index_frequency_minutes: 1)
       job_args = %{"force" => true}
@@ -475,7 +455,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
   end
 
   describe "index_and_enqueue_download_for_media_items when testing the download archive" do
-    test "a download archive is used if the source is a channel", %{source: source} do
+    test "a download archive is used if the source is a channel that has been indexed before" do
+      source = source_fixture(%{collection_type: :channel, last_indexed_at: now()})
+
       expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, opts, _ot, _addl_opts ->
         assert :break_on_existing in opts
         assert Keyword.has_key?(opts, :download_archive)
@@ -499,6 +481,19 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
       SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
     end
 
+    test "a download archive is not used if the source has never been indexed before" do
+      source = source_fixture(%{collection_type: :channel, last_indexed_at: nil})
+
+      expect(YtDlpRunnerMock, :run, fn _url, :get_media_attributes_for_collection, opts, _ot, _addl_opts ->
+        refute :break_on_existing in opts
+        refute Keyword.has_key?(opts, :download_archive)
+
+        {:ok, source_attributes_return_fixture()}
+      end)
+
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+    end
+
     test "a download archive is not used if the index has been forced to run" do
       source = source_fixture(%{collection_type: :channel})
 
@@ -512,7 +507,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
       SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source, was_forced: true)
     end
 
-    test "the download archive is formatted correctly and contains the right video", %{source: source} do
+    test "the download archive is formatted correctly and contains the right video" do
+      source = source_fixture(%{collection_type: :channel, last_indexed_at: now()})
+
       media_items =
         1..21
         |> Enum.map(fn n ->

test/pinchflat/sources_test.exs

@@ -294,6 +294,34 @@ defmodule Pinchflat.SourcesTest do
       assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
+    test "creation will schedule a fast indexing job if the fast_index option is set" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/5)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: true
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "creation will not schedule a fast indexing job if the fast_index option is not set" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/5)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: false
+      }
+
+      assert {:ok, %Source{}} = Sources.create_source(valid_attrs)
+
+      refute_enqueued(worker: FastIndexingWorker)
+    end
+
     test "creation schedules an index test even if the index frequency is 0" do
       expect(YtDlpRunnerMock, :run, &channel_mock/5)
 

test/pinchflat/utils/string_utils_test.exs

@@ -33,4 +33,14 @@ defmodule Pinchflat.Utils.StringUtilsTest do
       assert StringUtils.double_brace("hello") == "{{ hello }}"
     end
   end
+
+  describe "wrap_string/1" do
+    test "returns strings as-is" do
+      assert StringUtils.wrap_string("hello") == "hello"
+    end
+
+    test "returns other values as inspected strings" do
+      assert StringUtils.wrap_string(1) == "1"
+    end
+  end
 end

test/pinchflat/yt_dlp/command_runner_test.exs

@@ -154,6 +154,14 @@ defmodule Pinchflat.YtDlp.CommandRunnerTest do
     end
   end
 
+  describe "update/0" do
+    test "adds the update arg" do
+      assert {:ok, output} = Runner.update()
+
+      assert String.contains?(output, "--update")
+    end
+  end
+
   defp wrap_executable(new_executable, fun) do
     Application.put_env(:pinchflat, :yt_dlp_executable, new_executable)
     fun.()

test/pinchflat/yt_dlp/media_test.exs

@@ -269,7 +269,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
       response = %{
         "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
         "aspect_ratio" => 0.5,
-        "duration" => 59,
+        "duration" => 150,
         "upload_date" => "20210101"
       }
 

test/pinchflat/yt_dlp/update_worker_test.exs

@@ -0,0 +1,24 @@
+defmodule Pinchflat.YtDlp.UpdateWorkerTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Settings
+  alias Pinchflat.YtDlp.UpdateWorker
+
+  describe "perform/1" do
+    test "calls the yt-dlp runner to update yt-dlp" do
+      expect(YtDlpRunnerMock, :update, fn -> {:ok, ""} end)
+      expect(YtDlpRunnerMock, :version, fn -> {:ok, ""} end)
+
+      perform_job(UpdateWorker, %{})
+    end
+
+    test "saves the new version to the database" do
+      expect(YtDlpRunnerMock, :update, fn -> {:ok, ""} end)
+      expect(YtDlpRunnerMock, :version, fn -> {:ok, "1.2.3"} end)
+
+      perform_job(UpdateWorker, %{})
+
+      assert {:ok, "1.2.3"} = Settings.get(:yt_dlp_version)
+    end
+  end
+end