Bumped version

* Updated downloader and runner to handle sponsorblock failures more gracefully

* stopped download worker from running if a media item is preventing download

.github/workflows/docker_release.yml

@@ -77,3 +77,5 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

README.md

@@ -48,6 +48,7 @@ If it doesn't work for your use case, please make a feature request! You can als
   - Uses a novel approach to download new content more quickly than other apps
 - Supports downloading audio content
 - Custom rules for handling YouTube Shorts and livestreams
+- Optionally automatically delete old content ([docs](https://github.com/kieraneglin/pinchflat/wiki/Automatically-Delete-Media))
 - Advanced options like setting cutoff dates and filtering by title
 - Reliable hands-off operation
 - Can pass cookies to YouTube to download your private playlists ([docs](https://github.com/kieraneglin/pinchflat/wiki/YouTube-Cookies))

assets/js/app.js

@@ -22,6 +22,7 @@ import { Socket } from 'phoenix'
 import { LiveSocket } from 'phoenix_live_view'
 import topbar from '../vendor/topbar'
 import Alpine from 'alpinejs'
+import './tabs'
 
 window.Alpine = Alpine
 Alpine.start()

assets/js/tabs.js

@@ -0,0 +1,20 @@
+window.setTabIndex = (index) => {
+  window.location.hash = `tab-${index}`
+
+  return index
+}
+
+// The conditionals and currIndex stuff ensures that
+// the tab index is always set to 0 if the hash is empty
+// AND other hash values are ignored
+window.getTabIndex = (currIndex) => {
+  if (window.location.hash === '' || window.location.hash === '#') {
+    return 0
+  }
+
+  if (window.location.hash.startsWith('#tab-')) {
+    return parseInt(window.location.hash.replace('#tab-', ''))
+  }
+
+  return currIndex
+}

config/config.exs

@@ -46,7 +46,13 @@ config :pinchflat, Oban,
   engine: Oban.Engines.Lite,
   repo: Pinchflat.Repo,
   # Keep old jobs for 30 days for display in the UI
-  plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}],
+  plugins: [
+    {Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
+    {Oban.Plugins.Cron,
+     crontab: [
+       {"@daily", Pinchflat.Downloading.MediaRetentionWorker}
+     ]}
+  ],
   # TODO: consider making this an env var or something?
   queues: [
     default: 10,
@@ -79,7 +85,7 @@ config :esbuild,
 
 # Configure tailwind (the version is required)
 config :tailwind,
-  version: "3.3.2",
+  version: "3.4.3",
   default: [
     args: ~w(
       --config=tailwind.config.js

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,62 +0,0 @@
-defmodule Pinchflat.Boot.DataBackfillWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :local_metadata,
-    unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
-    tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
-
-  # This one is going to be a little more self-contained
-  # instead of relying on outside modules for the methods.
-  # That's because, for now, these methods are not intended
-  # to be used elsewhere.
-  #
-  # I'm just trying out that pattern and seeing if I like it better
-  # so this may change.
-  import Ecto.Query, warn: false
-  require Logger
-
-  alias __MODULE__
-  alias Pinchflat.Repo
-
-  @doc """
-  Cancels all pending backfill jobs. Useful for ensuring worker runs immediately
-  on app boot.
-
-  Returns {:ok, integer()}
-  """
-  def cancel_pending_backfill_jobs do
-    Oban.Job
-    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
-    |> Oban.cancel_all_jobs()
-  end
-
-  @impl Oban.Worker
-  @doc """
-  Performs one-off tasks to get data in the right shape.
-  This can be needed when we add new features or change the way
-  we store data. Must be idempotent. All new data should already
-  conform to the expected schema so this should only be needed
-  for existing data. Still runs periodically to be safe.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{}) do
-    Logger.info("Running data backfill worker")
-    # Nothing to do for now - just reschedule
-    # Keeping in-place because we _will_ need it in the future
-
-    reschedule_backfill()
-
-    :ok
-  end
-
-  defp reschedule_backfill do
-    # Run hourly
-    next_run_in = 60 * 60
-
-    %{}
-    |> DataBackfillWorker.new(schedule_in: next_run_in)
-    |> Repo.insert_unique_job()
-  end
-end

lib/pinchflat/boot/nfo_backfill_worker.ex

@@ -0,0 +1,70 @@
+defmodule Pinchflat.Boot.NfoBackfillWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_metadata,
+    # This should have it running once _ever_ (until the job is pruned, anyway)
+    # NOTE: remove within the next month
+    unique: [period: :infinity, states: Oban.Job.states()],
+    tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
+
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Media
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Metadata.MetadataFileHelpers
+
+  @doc """
+  Runs a one-off backfill job to regenerate NFO files for media items that have
+  both an NFO file and a metadata file. This is needed because NFO files weren't
+  escaping characters properly so we need to regenerate them.
+
+  This job will only run once as long as I remove it before the jobs are pruned in a month.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    Logger.info("Running NFO backfill worker")
+
+    media_items = get_media_items_to_backfill()
+
+    Enum.each(media_items, fn media_item ->
+      nfo_exists = File.exists?(media_item.nfo_filepath)
+      metadata_exists = File.exists?(media_item.metadata.metadata_filepath)
+
+      if nfo_exists && metadata_exists do
+        Logger.info("NFO and metadata exist for media item #{media_item.id} - proceeding")
+
+        regenerate_nfo_for_media_item(media_item)
+      end
+    end)
+
+    :ok
+  end
+
+  defp get_media_items_to_backfill do
+    from(m in MediaItem, where: not is_nil(m.nfo_filepath))
+    |> Repo.all()
+    |> Repo.preload([:metadata, source: :media_profile])
+  end
+
+  defp regenerate_nfo_for_media_item(media_item) do
+    try do
+      case MetadataFileHelpers.read_compressed_metadata(media_item.metadata.metadata_filepath) do
+        {:ok, metadata} ->
+          Media.update_media_item(media_item, %{
+            nfo_filepath: NfoBuilder.build_and_store_for_media_item(media_item.nfo_filepath, metadata)
+          })
+
+        _err ->
+          Logger.error("Failed to read metadata for media item #{media_item.id}")
+      end
+    rescue
+      e -> Logger.error("Unknown error regenerating NFO file for MI ##{media_item.id}: #{inspect(e)}")
+    end
+  end
+end

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -7,13 +7,13 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   Phoenix supervision tree.
   """
 
+  alias Pinchflat.Repo
+  alias Pinchflat.Boot.NfoBackfillWorker
+
   # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
   use GenServer, restart: :temporary
   import Ecto.Query, warn: false
 
-  alias Pinchflat.Repo
-  alias Pinchflat.Boot.DataBackfillWorker
-
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
   end
@@ -29,16 +29,8 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   """
   @impl true
   def init(state) do
-    enqueue_backfill_worker()
+    Repo.insert_unique_job(NfoBackfillWorker.new(%{}))
 
     {:ok, state}
   end
-
-  defp enqueue_backfill_worker do
-    DataBackfillWorker.cancel_pending_backfill_jobs()
-
-    %{}
-    |> DataBackfillWorker.new()
-    |> Repo.insert_unique_job()
-  end
 end

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -14,6 +14,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.YtDlp.CommandRunner
   alias Pinchflat.Filesystem.FilesystemHelpers
 
   def start_link(opts \\ []) do
@@ -62,7 +63,8 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
   end
 
   defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
+    {:ok, yt_dlp_version} = CommandRunner.version()
-    Settings.fetch!(:pro_enabled, false)
+
+    Settings.set(yt_dlp_version: yt_dlp_version)
   end
 end

lib/pinchflat/downloading/media_download_worker.ex

@@ -19,27 +19,29 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
   Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
   """
-  def kickoff_with_task(media_item, opts \\ []) do
+  def kickoff_with_task(media_item, job_args \\ %{}, job_opts \\ []) do
     %{id: media_item.id}
-    |> MediaDownloadWorker.new(opts)
+    |> Map.merge(job_args)
+    |> MediaDownloadWorker.new(job_opts)
     |> Tasks.create_job_with_task(media_item)
   end
 
   @doc """
   For a given media item, download the media alongside any options.
-  Does not download media if its source is set to not download media.
+  Does not download media if its source is set to not download media
+  (unless forced).
 
   Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   @impl Oban.Worker
-  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
+  def perform(%Oban.Job{args: %{"id" => media_item_id} = args}) do
     media_item =
       media_item_id
       |> Media.get_media_item!()
       |> Repo.preload(:source)
 
-    # If the source is set to not download media, perform a no-op
+    # If the source or media item is set to not download media, perform a no-op unless forced
-    if media_item.source.download_media do
+    if (media_item.source.download_media && !media_item.prevent_download) || args["force"] do
       download_media_and_schedule_jobs(media_item)
     else
       :ok
@@ -56,9 +58,10 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
         {:ok, updated_media_item}
 
-      err ->
+      {:recovered, _} ->
-        Logger.error("Failed to download media for media item #{media_item.id}: #{inspect(err)}")
+        {:error, :retry}
 
+      {:error, _message} ->
         {:error, :download_failed}
     end
   end

lib/pinchflat/downloading/media_downloader.ex

@@ -5,12 +5,15 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   to download the media with the desired options.
   """
 
+  require Logger
+
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.NfoBuilder
   alias Pinchflat.Metadata.MetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.Downloading.DownloadOptionBuilder
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@@ -27,31 +30,67 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def download_for_media_item(%MediaItem{} = media_item) do
-    item_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+    media_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
 
-    case download_with_options(media_item.original_url, item_with_preloads) do
+    case download_with_options(media_item.original_url, media_with_preloads, output_filepath) do
       {:ok, parsed_json} ->
+        update_media_item_from_parsed_json(media_with_preloads, parsed_json)
+
+      {:error, message, _exit_code} ->
+        Logger.error("yt-dlp download error for media item ##{media_with_preloads.id}: #{inspect(message)}")
+
+        if String.contains?(to_string(message), recoverable_errors()) do
+          attempt_update_media_item(media_with_preloads, output_filepath)
+
+          {:recovered, message}
+        else
+          {:error, message}
+        end
+
+      err ->
+        Logger.error("Unknown error downloading media item ##{media_with_preloads.id}: #{inspect(err)}")
+
+        {:error, "Unknown error: #{inspect(err)}"}
+    end
+  end
+
+  defp attempt_update_media_item(media_with_preloads, output_filepath) do
+    with {:ok, contents} <- File.read(output_filepath),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(contents) do
+      Logger.info("""
+      Recovery from yt-dlp error seems possible. Updating media item ##{media_with_preloads.id}
+      with parsed JSON from partial download attempt. Full download will be re-attemted in future
+      anyway
+      """)
+
+      update_media_item_from_parsed_json(media_with_preloads, parsed_json)
+    else
+      err ->
+        Logger.error("Unable to recover error for media item ##{media_with_preloads.id}: #{inspect(err)}")
+
+        {:error, :retry_failed}
+    end
+  end
+
+  defp update_media_item_from_parsed_json(media_with_preloads, parsed_json) do
     parsed_attrs =
       parsed_json
       |> MetadataParser.parse_for_media_item()
       |> Map.merge(%{
         media_downloaded_at: DateTime.utc_now(),
-            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
+        nfo_filepath: determine_nfo_filepath(media_with_preloads, parsed_json),
         metadata: %{
           # IDEA: might be worth kicking off a job for this since thumbnail fetching
           # could fail and I want to handle that in isolation
-              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_with_preloads, parsed_json),
-              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
+          thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_with_preloads, parsed_json)
         }
       })
 
     # Don't forgor to use preloaded associations or updates to
     # associations won't work!
-        Media.update_media_item(item_with_preloads, parsed_attrs)
+    Media.update_media_item(media_with_preloads, parsed_attrs)
-
-      err ->
-        err
-    end
   end
 
   defp determine_nfo_filepath(media_item, parsed_json) do
@@ -64,9 +103,15 @@ defmodule Pinchflat.Downloading.MediaDownloader do
     end
   end
 
-  defp download_with_options(url, item_with_preloads) do
+  defp download_with_options(url, item_with_preloads, output_filepath) do
     {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
-    YtDlpMedia.download(url, options)
+    YtDlpMedia.download(url, options, output_filepath: output_filepath)
+  end
+
+  defp recoverable_errors do
+    [
+      "Unable to communicate with SponsorBlock"
+    ]
   end
 end

lib/pinchflat/downloading/media_retention_worker.ex

@@ -0,0 +1,33 @@
+defmodule Pinchflat.Downloading.MediaRetentionWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_metadata,
+    unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
+    tags: ["media_item", "local_metadata"]
+
+  require Logger
+
+  alias Pinchflat.Media
+
+  @doc """
+  Deletes media items that are past their retention date and prevents
+  them from being re-downloaded.
+
+  This worker is scheduled to run daily via the Oban Cron plugin.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    cullable_media = Media.list_cullable_media_items()
+    Logger.info("Culling #{length(cullable_media)} media items past their retention date")
+
+    Enum.each(cullable_media, fn media_item ->
+      Media.delete_media_files(media_item, %{
+        prevent_download: true,
+        culled_at: DateTime.utc_now()
+      })
+    end)
+  end
+end

lib/pinchflat/media/media.ex

@@ -23,38 +23,32 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Returns a list of pending media_items for a given source, where
+  Returns a list of media_items that are cullable based on the retention period
-  pending means the `media_filepath` is `nil` AND the media_item
+  of the source they belong to.
-  matches the format selection rules of the parent media_profile.
 
-  See `build_format_clauses` but tl;dr is it _may_ filter based
+  Returns [%MediaItem{}, ...]
-  on shorts or livestreams depending on the media_profile settings.
-
-  Returns [%MediaItem{}, ...].
   """
-  def list_pending_media_items_for(%Source{} = source, opts \\ []) do
+  def list_cullable_media_items do
-    limit = Keyword.get(opts, :limit, nil)
-    source = Repo.preload(source, :media_profile)
-
     MediaQuery.new()
-    |> MediaQuery.for_source(source)
+    |> MediaQuery.with_media_filepath()
-    |> matching_download_criteria_for(source)
+    |> MediaQuery.with_passed_retention_period()
-    |> Repo.maybe_limit(limit)
+    |> MediaQuery.with_no_culling_prevention()
     |> Repo.all()
   end
 
   @doc """
-  Returns a list of downloaded media_items for a given source.
+  Returns a list of pending media_items for a given source, where
+  pending means the `media_filepath` is `nil` AND the media_item
+  matches satisfies `MediaQuery.with_media_pending_download`. You
+  should really check out that function if you need to know more
+  because it has a lot going on.
 
   Returns [%MediaItem{}, ...].
   """
-  def list_downloaded_media_items_for(%Source{} = source, opts \\ []) do
+  def list_pending_media_items_for(%Source{} = source) do
-    limit = Keyword.get(opts, :limit, nil)
-
     MediaQuery.new()
     |> MediaQuery.for_source(source)
-    |> MediaQuery.with_media_filepath()
+    |> MediaQuery.with_media_pending_download()
-    |> Repo.maybe_limit(limit)
     |> Repo.all()
   end
 
@@ -72,7 +66,7 @@ defmodule Pinchflat.Media do
 
     MediaQuery.new()
     |> MediaQuery.with_id(media_item.id)
-    |> matching_download_criteria_for(media_item.source)
+    |> MediaQuery.with_media_pending_download()
     |> Repo.exists?()
   end
 
@@ -161,7 +155,7 @@ defmodule Pinchflat.Media do
     Tasks.delete_tasks_for(media_item)
 
     if delete_files do
-      {:ok, _} = delete_media_files(media_item)
+      {:ok, _} = do_delete_media_files(media_item)
     end
 
     # Should delete these no matter what
@@ -169,6 +163,25 @@ defmodule Pinchflat.Media do
     Repo.delete(media_item)
   end
 
+  @doc """
+  Deletes the tasks and media files associated with a media_item but leaves the
+  media_item in the database. Does not delete anything to do with associated metadata.
+
+  Optionally accepts a second argument `addl_attrs` which will be merged into the
+  media_item before it is updated. Useful for setting things like `prevent_download`
+  and `culled_at`, if wanted
+
+  Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
+  """
+  def delete_media_files(%MediaItem{} = media_item, addl_attrs \\ %{}) do
+    filepath_attrs = MediaItem.filepath_attribute_defaults()
+
+    Tasks.delete_tasks_for(media_item)
+    {:ok, _} = do_delete_media_files(media_item)
+
+    update_media_item(media_item, Map.merge(filepath_attrs, addl_attrs))
+  end
+
   @doc """
   Returns an `%Ecto.Changeset{}` for tracking media_item changes.
   """
@@ -176,7 +189,7 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_media_files(media_item) do
+  defp do_delete_media_files(media_item) do
     mapped_struct = Map.from_struct(media_item)
 
     MediaItem.filepath_attributes()
@@ -200,12 +213,4 @@ defmodule Pinchflat.Media do
     |> Enum.filter(&is_binary/1)
     |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
   end
-
-  defp matching_download_criteria_for(query, source_with_preloads) do
-    query
-    |> MediaQuery.with_no_media_filepath()
-    |> MediaQuery.with_upload_date_after(source_with_preloads.download_cutoff_date)
-    |> MediaQuery.with_format_preference(source_with_preloads.media_profile)
-    |> MediaQuery.matching_title_regex(source_with_preloads.title_filter_regex)
-  end
 end

lib/pinchflat/media/media_item.ex

@@ -30,7 +30,11 @@ defmodule Pinchflat.Media.MediaItem do
     :subtitle_filepaths,
     :thumbnail_filepath,
     :metadata_filepath,
-    :nfo_filepath
+    :nfo_filepath,
+    # These are user or system controlled fields
+    :prevent_download,
+    :prevent_culling,
+    :culled_at
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -70,6 +74,10 @@ defmodule Pinchflat.Media.MediaItem do
     # Will very likely revisit because I can't leave well-enough alone.
     field :subtitle_filepaths, {:array, {:array, :string}}, default: []
 
+    field :prevent_download, :boolean, default: false
+    field :prevent_culling, :boolean, default: false
+    field :culled_at, :utc_datetime
+
     field :matching_search_term, :string, virtual: true
 
     belongs_to :source, Source
@@ -96,4 +104,14 @@ defmodule Pinchflat.Media.MediaItem do
   def filepath_attributes do
     ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
+
+  @doc false
+  def filepath_attribute_defaults do
+    filepath_attributes()
+    |> Enum.map(fn
+      :subtitle_filepaths -> {:subtitle_filepaths, []}
+      field -> {field, nil}
+    end)
+    |> Enum.into(%{})
+  end
 end

lib/pinchflat/media/media_query.ex

@@ -3,13 +3,10 @@ defmodule Pinchflat.Media.MediaQuery do
   Query helpers for the Media context.
 
   These methods are made to be one-ish liners used
-  to compose queries for media items. Each method should
+  to compose queries. Each method should strive to do
-  strive to do _one_ thing. These don't need to be tested
+  _one_ thing. These don't need to be tested as
-  as they are just building blocks for other functionality
+  they are just building blocks for other functionality
   which, itself, will be tested.
-
-  ALSO, this is me trying something new. If I like it,
-  I'll refactor other contexts to use this pattern.
   """
   import Ecto.Query, warn: false
 
@@ -17,6 +14,7 @@ defmodule Pinchflat.Media.MediaQuery do
 
   # Prefixes:
   # - for_* - belonging to a certain record
+  # - join_* - for joining on a certain record
   # - with_* - for filtering based on full, concrete attributes
   # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search)
   #
@@ -31,6 +29,28 @@ defmodule Pinchflat.Media.MediaQuery do
     where(query, [mi], mi.source_id == ^source.id)
   end
 
+  def join_sources(query) do
+    from(mi in query, join: s in assoc(mi, :source), as: :sources)
+  end
+
+  def with_passed_retention_period(query) do
+    query
+    |> require_assoc(:source)
+    |> where(
+      [mi, source],
+      fragment(
+        "IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?",
+        source.retention_period_days,
+        source.retention_period_days,
+        mi.media_downloaded_at
+      )
+    )
+  end
+
+  def with_no_culling_prevention(query) do
+    where(query, [mi], mi.prevent_culling == false)
+  end
+
   def with_id(query, id) do
     where(query, [mi], mi.id == ^id)
   end
@@ -47,16 +67,23 @@ defmodule Pinchflat.Media.MediaQuery do
     where(query, [mi], is_nil(mi.media_filepath))
   end
 
-  def with_upload_date_after(query, nil), do: query
+  def with_upload_date_after_source_cutoff(query) do
-
+    query
-  def with_upload_date_after(query, date) do
+    |> require_assoc(:source)
-    where(query, [mi], mi.upload_date >= ^date)
+    |> where([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date)
   end
 
-  def matching_title_regex(query, nil), do: query
+  def with_no_prevented_download(query) do
+    where(query, [mi], mi.prevent_download == false)
+  end
 
-  def matching_title_regex(query, regex) do
+  def matching_source_title_regex(query) do
-    where(query, [mi], fragment("regexp_like(?, ?)", mi.title, ^regex))
+    query
+    |> require_assoc(:source)
+    |> where(
+      [mi, source],
+      is_nil(source.title_filter_regex) or fragment("regexp_like(?, ?)", mi.title, source.title_filter_regex)
+    )
   end
 
   def matching_search_term(query, nil), do: query
@@ -77,44 +104,55 @@ defmodule Pinchflat.Media.MediaQuery do
     )
   end
 
-  # NOTE: this method breaks the contract set by other methods in that it
+  def with_format_matching_profile_preference(query) do
-  # takes a media_profile struct instead of taking just the attributes it
+    query
-  # cares about. Consider refactoring but low priority.
+    |> require_assoc(:media_profile)
-  def with_format_preference(query, media_profile) do
+    |> where(
-    mapped_struct = Map.from_struct(media_profile)
+      fragment("""
-
+        CASE
-    finders =
+          WHEN shorts_behaviour = 'only' AND livestream_behaviour = 'only' THEN
-      Enum.reduce(mapped_struct, dynamic(true), fn attr, dynamic ->
+            livestream = true OR short_form_content = true
-        case {attr, media_profile} do
+          WHEN shorts_behaviour = 'only' THEN
-          {{:shorts_behaviour, :only}, %{livestream_behaviour: :only}} ->
+            short_form_content = true
-            dynamic(
+          WHEN livestream_behaviour = 'only' THEN
-              [mi],
+            livestream = true
-              ^dynamic and (mi.livestream == true or mi.short_form_content == true)
+          WHEN shorts_behaviour = 'exclude' AND livestream_behaviour = 'exclude' THEN
+            short_form_content = false AND livestream = false
+          WHEN shorts_behaviour = 'exclude' THEN
+            short_form_content = false
+          WHEN livestream_behaviour = 'exclude' THEN
+            livestream = false
+          ELSE
+            true
+        END
+      """)
     )
-
-          # Technically redundant, but makes the other clauses easier to parse
-          # (redundant because this condition is the same as the condition above, just flipped)
-          {{:livestream_behaviour, :only}, %{shorts_behaviour: :only}} ->
-            dynamic
-
-          {{:shorts_behaviour, :only}, _} ->
-            dynamic([mi], ^dynamic and mi.short_form_content == true)
-
-          {{:livestream_behaviour, :only}, _} ->
-            dynamic([mi], ^dynamic and mi.livestream == true)
-
-          {{:shorts_behaviour, :exclude}, %{livestream_behaviour: lb}} when lb != :only ->
-            dynamic([mi], ^dynamic and mi.short_form_content == false)
-
-          {{:livestream_behaviour, :exclude}, %{shorts_behaviour: sb}} when sb != :only ->
-            # return records with livestream: false
-            dynamic([mi], ^dynamic and mi.livestream == false)
-
-          _ ->
-            dynamic
   end
-      end)
 
-    where(query, ^finders)
+  def with_media_pending_download(query) do
+    query
+    |> with_no_prevented_download()
+    |> with_no_media_filepath()
+    |> with_upload_date_after_source_cutoff()
+    |> with_format_matching_profile_preference()
+    |> matching_source_title_regex()
+  end
+
+  defp require_assoc(query, identifier) do
+    if has_named_binding?(query, identifier) do
+      query
+    else
+      do_require_assoc(query, identifier)
+    end
+  end
+
+  defp do_require_assoc(query, :source) do
+    from(mi in query, join: s in assoc(mi, :source), as: :source)
+  end
+
+  defp do_require_assoc(query, :media_profile) do
+    query
+    |> require_assoc(:source)
+    |> join(:inner, [mi, source], mp in assoc(source, :media_profile), as: :media_profile)
   end
 end

lib/pinchflat/metadata/nfo_builder.ex

@@ -4,6 +4,8 @@ defmodule Pinchflat.Metadata.NfoBuilder do
   use by Kodi/Jellyfin and other media center software.
   """
 
+  import Pinchflat.Utils.XmlUtils, only: [safe: 1]
+
   alias Pinchflat.Metadata.MetadataFileHelpers
   alias Pinchflat.Filesystem.FilesystemHelpers
 
@@ -42,12 +44,12 @@ defmodule Pinchflat.Metadata.NfoBuilder do
     """
     <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
     <episodedetails>
-      <title>#{metadata["title"]}</title>
+      <title>#{safe(metadata["title"])}</title>
-      <showtitle>#{metadata["uploader"]}</showtitle>
+      <showtitle>#{safe(metadata["uploader"])}</showtitle>
-      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <uniqueid type="youtube" default="true">#{safe(metadata["id"])}</uniqueid>
-      <plot>#{metadata["description"]}</plot>
+      <plot>#{safe(metadata["description"])}</plot>
-      <aired>#{upload_date}</aired>
+      <aired>#{safe(upload_date)}</aired>
-      <season>#{upload_date.year}</season>
+      <season>#{safe(upload_date.year)}</season>
       <episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
       <genre>YouTube</genre>
     </episodedetails>
@@ -58,9 +60,9 @@ defmodule Pinchflat.Metadata.NfoBuilder do
     """
     <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
     <tvshow>
-      <title>#{metadata["title"]}</title>
+      <title>#{safe(metadata["title"])}</title>
-      <plot>#{metadata["description"]}</plot>
+      <plot>#{safe(metadata["description"])}</plot>
-      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <uniqueid type="youtube" default="true">#{safe(metadata["id"])}</uniqueid>
       <genre>YouTube</genre>
     </tvshow>
     """

lib/pinchflat/podcasts/podcast_helpers.ex

@@ -5,7 +5,7 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
   """
 
   alias Pinchflat.Repo
-  alias Pinchflat.Media
+  alias Pinchflat.Media.MediaQuery
   alias Pinchflat.Metadata.MediaMetadata
   alias Pinchflat.Metadata.SourceMetadata
 
@@ -25,8 +25,11 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
   def persisted_media_items_for(source, opts \\ []) do
     limit = Keyword.get(opts, :limit, 500)
 
-    source
+    MediaQuery.new()
-    |> Media.list_downloaded_media_items_for(limit: limit)
+    |> MediaQuery.for_source(source)
+    |> MediaQuery.with_media_filepath()
+    |> Repo.maybe_limit(limit)
+    |> Repo.all()
     |> Enum.filter(fn media_item -> File.exists?(media_item.media_filepath) end)
   end
 

lib/pinchflat/podcasts/rss_feed_builder.ex

@@ -5,6 +5,8 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
 
   @datetime_format "%a, %d %b %Y %H:%M:%S %z"
 
+  import Pinchflat.Utils.XmlUtils, only: [safe: 1]
+
   alias Pinchflat.Utils.DatetimeUtils
   alias Pinchflat.Podcasts.PodcastHelpers
   alias PinchflatWeb.Router.Helpers, as: Routes
@@ -94,14 +96,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
     """
   end
 
-  defp safe(nil), do: ""
-
-  defp safe(value) do
-    value
-    |> Phoenix.HTML.html_escape()
-    |> Phoenix.HTML.safe_to_string()
-  end
-
   defp generate_self_link(url_base, source) do
     Path.join(url_base, "#{podcast_route(:rss_feed, source.uuid)}.xml")
   end

lib/pinchflat/settings/setting.ex

@@ -1,24 +1,32 @@
 defmodule Pinchflat.Settings.Setting do
   @moduledoc """
-  A Setting is a key-value pair with a datatype used to track user-level settings.
+  The Setting schema.
   """
 
   use Ecto.Schema
   import Ecto.Changeset
 
-  schema "settings" do
+  @allowed_fields [
-    field :name, :string
+    :onboarding,
-    field :value, :string
+    :pro_enabled,
-    field :datatype, Ecto.Enum, values: ~w(boolean string integer float)a
+    :yt_dlp_version
+  ]
 
-    timestamps(type: :utc_datetime)
+  @required_fields ~w(
+    onboarding
+    pro_enabled
+  )a
+
+  schema "settings" do
+    field :onboarding, :boolean, default: true
+    field :pro_enabled, :boolean, default: false
+    field :yt_dlp_version, :string
   end
 
   @doc false
   def changeset(setting, attrs) do
     setting
-    |> cast(attrs, [:name, :value, :datatype])
+    |> cast(attrs, @allowed_fields)
-    |> validate_required([:name, :value, :datatype])
+    |> validate_required(@required_fields)
-    |> unique_constraint([:name])
   end
 end

lib/pinchflat/settings/settings.ex

@@ -2,94 +2,63 @@ defmodule Pinchflat.Settings do
   @moduledoc """
   The Settings context.
   """
-
   import Ecto.Query, warn: false
-  alias Pinchflat.Repo
 
+  alias Pinchflat.Repo
   alias Pinchflat.Settings.Setting
 
   @doc """
-  Returns the list of settings.
+  Returns the only setting record. It _should_ be impossible
+  to create or delete this record, so it's assertive about
+  assuming it's the only one.
 
-  Returns [%Setting{}, ...]
+  Returns %Setting{}
   """
-  def list_settings do
+  def record do
-    Repo.all(Setting)
+    Setting
+    |> limit(1)
+    |> Repo.one()
   end
 
   @doc """
-  Creates or updates a setting, returning the parsed value.
+  Updates a setting, returning the new value.
-  Raises if an unsupported datatype is used. Optionally allows
+  Is setup to take a keyword list argument so you
-  specifying the datatype.
+  can call it like `Settings.set(onboarding: true)`
 
-  Returns value in type of `Ecto.Enum.mappings(Setting, :datatype)`
+  Returns {:ok, value} | {:error, :invalid_key} | {:error, %Ecto.Changeset{}}
   """
-  def set!(name, value) do
+  def set([{attr, value}]) do
-    set!(name, value, infer_datatype(value))
+    record()
-  end
+    |> Setting.changeset(%{attr => value})
-
+    |> Repo.update()
-  def set!(name, value, datatype) do
+    |> case do
-    # Only create if doesn't exist
+      {:ok, %{^attr => _}} -> {:ok, value}
-    case Repo.get_by(Setting, name: to_string(name)) do
+      {:ok, _} -> {:error, :invalid_key}
-      nil -> create_setting!(name, value, datatype)
+      {:error, changeset} -> {:error, changeset}
-      setting -> update_setting!(setting, value, datatype)
     end
   end
 
   @doc """
-  Gets the parsed value of a setting. Raises if the setting does not exist.
+  Gets the value of a setting.
 
-  Returns value in type of `Ecto.Enum.mappings(Setting, :datatype)`
+  Returns {:ok, value} | {:error, :invalid_key}
+  """
+  def get(name) do
+    case Map.fetch(record(), name) do
+      {:ok, value} -> {:ok, value}
+      :error -> {:error, :invalid_key}
+    end
+  end
+
+  @doc """
+  Gets the value of a setting, raising if it doesn't exist.
+
+  Returns value
   """
   def get!(name) do
-    Setting
+    case get(name) do
-    |> Repo.get_by!(name: to_string(name))
+      {:ok, value} -> value
-    |> read_setting()
+      {:error, _} -> raise "Setting `#{name}` not found"
-  end
-
-  @doc """
-  Attempts to find a setting by name or creates a setting with value
-  if one doesn't exist, returning the parsed value. Optionally allows
-  specifying the datatype.
-
-  Returns value in type of `Ecto.Enum.mappings(Setting, :datatype)`
-  """
-  def fetch!(name, value) do
-    fetch!(name, value, infer_datatype(value))
-  end
-
-  def fetch!(name, value, datatype) do
-    case Repo.get_by(Setting, name: to_string(name)) do
-      nil -> create_setting!(name, value, datatype)
-      setting -> read_setting(setting)
     end
   end
-
-  defp change_setting(setting, attrs) do
-    Setting.changeset(setting, attrs)
-  end
-
-  defp create_setting!(name, value, datatype) do
-    %Setting{}
-    |> change_setting(%{name: to_string(name), value: to_string(value), datatype: datatype})
-    |> Repo.insert!()
-    |> read_setting()
-  end
-
-  defp update_setting!(setting, value, datatype) do
-    setting
-    |> change_setting(%{value: to_string(value), datatype: datatype})
-    |> Repo.update!()
-    |> read_setting()
-  end
-
-  defp read_setting(%{value: value, datatype: :string}), do: value
-  defp read_setting(%{value: value, datatype: :boolean}), do: value in ["true", "t", "1"]
-  defp read_setting(%{value: value, datatype: :integer}), do: String.to_integer(value)
-  defp read_setting(%{value: value, datatype: :float}), do: String.to_float(value)
-
-  defp infer_datatype(value) when is_boolean(value), do: :boolean
-  defp infer_datatype(value) when is_integer(value), do: :integer
-  defp infer_datatype(value) when is_float(value), do: :float
-  defp infer_datatype(value) when is_binary(value), do: :string
 end

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -20,9 +20,10 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
 
   Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
   """
-  def kickoff_with_task(source, opts \\ []) do
+  def kickoff_with_task(source, job_args \\ %{}, job_opts \\ []) do
     %{id: source.id}
-    |> MediaCollectionIndexingWorker.new(opts)
+    |> Map.merge(job_args)
+    |> MediaCollectionIndexingWorker.new(job_opts)
     |> Tasks.create_job_with_task(source)
   end
 
@@ -30,8 +31,8 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   The ID is that of a source _record_, not a YouTube channel/playlist ID. Indexes
   the provided source, kicks off downloads for each new MediaItem, and
   reschedules the job to run again in the future. It will ALWAYS index a source
-  if it's never been indexed before, but rescheduling is determined by the
+  if it's never been indexed before or if `force` is set to `true`, but rescheduling
-  `index_frequency_minutes` field.
+  is determined by the `index_frequency_minutes` field.
 
   README: Re-scheduling here works a little different than you may expect.
   The reschedule time is relative to the time the job has actually _completed_.
@@ -71,7 +72,7 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   Returns :ok | {:ok, %Task{}}
   """
   @impl Oban.Worker
-  def perform(%Oban.Job{args: %{"id" => source_id}}) do
+  def perform(%Oban.Job{args: %{"id" => source_id} = args}) do
     source = Sources.get_source!(source_id)
 
     case {source.index_frequency_minutes, source.last_indexed_at} do
@@ -89,7 +90,11 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
 
       _ ->
         # If the source HAS been indexed and is not meant to reschedule,
-        # perform a no-op
+        # perform a no-op (unless forced)
+        if args["force"] do
+          SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+        end
+
         :ok
     end
   rescue

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -27,12 +27,12 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
 
   Returns {:ok, %Task{}}.
   """
-  def kickoff_indexing_task(%Source{} = source) do
+  def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
     Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
-    MediaCollectionIndexingWorker.kickoff_with_task(source)
+    MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
   end
 
   @doc """

lib/pinchflat/sources/source.ex

@@ -29,6 +29,7 @@ defmodule Pinchflat.Sources.Source do
     last_indexed_at
     original_url
     download_cutoff_date
+    retention_period_days
     title_filter_regex
     media_profile_id
   )a
@@ -72,6 +73,7 @@ defmodule Pinchflat.Sources.Source do
     field :last_indexed_at, :utc_datetime
     # Only download media items that were published after this date
     field :download_cutoff_date, :date
+    field :retention_period_days, :integer
     field :original_url, :string
     field :title_filter_regex, :string
 
@@ -106,7 +108,9 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
+    |> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
+    |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
   end
 
   @doc false

lib/pinchflat/sources/sources_query.ex

@@ -0,0 +1,31 @@
+defmodule Pinchflat.Sources.SourcesQuery do
+  @moduledoc """
+  Query helpers for the Sources context.
+
+  These methods are made to be one-ish liners used
+  to compose queries. Each method should strive to do
+  _one_ thing. These don't need to be tested as
+  they are just building blocks for other functionality
+  which, itself, will be tested.
+  """
+  import Ecto.Query, warn: false
+
+  alias Pinchflat.Sources.Source
+
+  # Prefixes:
+  # - for_* - belonging to a certain record
+  # - join_* - for joining on a certain record
+  # - with_* - for filtering based on full, concrete attributes
+  # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search)
+  #
+  # Suffixes:
+  # - _for - the arg passed is an association record
+
+  def new do
+    Source
+  end
+
+  def for_media_profile(query, media_profile) do
+    where(query, [s], s.media_profile_id == ^media_profile.id)
+  end
+end

lib/pinchflat/utils/xml_utils.ex

@@ -0,0 +1,17 @@
+defmodule Pinchflat.Utils.XmlUtils do
+  @moduledoc """
+  Utility methods for working with XML documents
+  """
+
+  @doc """
+  Escapes invalid XML characters in a string
+
+  Returns binary()
+  """
+  def safe(value) do
+    value
+    |> to_string()
+    |> Phoenix.HTML.html_escape()
+    |> Phoenix.HTML.safe_to_string()
+  end
+end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -8,4 +8,5 @@ defmodule Pinchflat.YtDlp.BackendCommandRunner do
 
   @callback run(binary(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
   @callback run(binary(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
+  @callback version() :: {:ok, binary()} | {:error, binary()}
 end

lib/pinchflat/yt_dlp/command_runner.ex

@@ -29,7 +29,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     command = backend_executable()
     # These must stay in exactly this order, hence why I'm giving it its own variable.
     # Also, can't use RAM file since yt-dlp needs a concrete filepath.
-    output_filepath = Keyword.get(addl_opts, :output_filepath, FSUtils.generate_metadata_tmpfile(:json))
+    output_filepath = generate_output_filepath(addl_opts)
     print_to_file_opts = [{:print_to_file, output_template}, output_filepath]
     cookie_opts = build_cookie_options()
     formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts ++ cookie_opts)
@@ -48,6 +48,26 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     end
   end
 
+  @impl BackendCommandRunner
+  def version do
+    command = backend_executable()
+
+    case System.cmd(command, ["--version"]) do
+      {output, 0} ->
+        {:ok, String.trim(output)}
+
+      {output, _} ->
+        {:error, output}
+    end
+  end
+
+  defp generate_output_filepath(addl_opts) do
+    case Keyword.get(addl_opts, :output_filepath) do
+      nil -> FSUtils.generate_metadata_tmpfile(:json)
+      path -> path
+    end
+  end
+
   defp build_cookie_options do
     base_dir = Application.get_env(:pinchflat, :extras_directory)
     cookie_file = Path.join(base_dir, "cookies.txt")

lib/pinchflat/yt_dlp/media.ex

@@ -35,10 +35,10 @@ defmodule Pinchflat.YtDlp.Media do
 
   Returns {:ok, map()} | {:error, any, ...}.
   """
-  def download(url, command_opts \\ []) do
+  def download(url, command_opts \\ [], addl_opts \\ []) do
     opts = [:no_simulate] ++ command_opts
 
-    with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j"),
+    with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j", addl_opts),
          {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
       {:ok, parsed_json}
     else

lib/pinchflat_web/components/custom_components/tab_components.ex

@@ -14,7 +14,8 @@ defmodule PinchflatWeb.CustomComponents.TabComponents do
   def tabbed_layout(assigns) do
     ~H"""
     <div
-      x-data="{ openTab: 0, activeClasses: 'text-meta-5 border-meta-5', inactiveClasses: 'border-transparent' }"
+      x-data="{ openTab: getTabIndex(0), activeClasses: 'text-meta-5 border-meta-5', inactiveClasses: 'border-transparent' }"
+      @hashchange.window="openTab = getTabIndex(openTab)"
       class="w-full"
     >
       <header class="flex flex-col md:flex-row md:justify-between border-b border-strokedark">
@@ -22,7 +23,7 @@ defmodule PinchflatWeb.CustomComponents.TabComponents do
           <a
             :for={{tab, idx} <- Enum.with_index(@tab)}
             href="#"
-            @click.prevent={"openTab = #{idx}"}
+            @click.prevent={"openTab = setTabIndex(#{idx})"}
             x-bind:class={"openTab === #{idx} ? activeClasses : inactiveClasses"}
             class="border-b-2 py-4 w-full sm:w-fit text-sm font-medium hover:text-meta-5 md:text-base"
           >
@@ -33,7 +34,7 @@ defmodule PinchflatWeb.CustomComponents.TabComponents do
           <%= render_slot(@tab_append) %>
         </div>
       </header>
-      <div class="mt-4">
+      <div class="mt-4 min-h-60">
         <div :for={{tab, idx} <- Enum.with_index(@tab)} x-show={"openTab === #{idx}"} class="font-medium leading-relaxed">
           <%= render_slot(tab) %>
         </div>

lib/pinchflat_web/components/layouts/partials/sidebar.html.heex

@@ -58,8 +58,11 @@
           </span>
         </li>
         <li>
-          <span class="group relative flex items-center gap-2.5 px-4 py-2 text-sm">
+          <span class="group relative flex items-center gap-2.5 px-4 pt-2 text-sm">
-            v<%= Application.spec(:pinchflat)[:vsn] %>
+            Pinchflat v<%= Application.spec(:pinchflat)[:vsn] %>
+          </span>
+          <span class="group relative flex items-center gap-2.5 px-4 text-sm">
+            yt-dlp <%= Settings.get!(:yt_dlp_version) %>
           </span>
         </li>
       </ul>

lib/pinchflat_web/components/layouts/partials/upgrade_button_live.ex

@@ -30,7 +30,7 @@ defmodule Pinchflat.UpgradeButtonLive do
       |> String.downcase()
 
     if normalized_text == "got it!" do
-      Settings.set!(:pro_enabled, true)
+      Settings.set(pro_enabled: true)
 
       {:noreply, update(socket, :button_disabled, fn _ -> false end)}
     else

lib/pinchflat_web/controllers/health_controller.ex

@@ -0,0 +1,9 @@
+defmodule PinchflatWeb.HealthController do
+  use PinchflatWeb, :controller
+
+  def check(conn, _params) do
+    conn
+    |> put_status(:ok)
+    |> json(%{status: "ok"})
+  end
+end

lib/pinchflat_web/controllers/media_items/media_item_controller.ex

@@ -6,6 +6,7 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   def show(conn, %{"id" => id}) do
     media_item =
@@ -16,23 +17,46 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
     render(conn, :show, media_item: media_item)
   end
 
-  def delete(conn, %{"id" => id} = params) do
+  def edit(conn, %{"id" => id}) do
-    delete_files = Map.get(params, "delete_files", false)
     media_item = Media.get_media_item!(id)
-    {:ok, _} = Media.delete_media_item(media_item, delete_files: delete_files)
+    changeset = Media.change_media_item(media_item)
 
-    flash_message =
+    render(conn, :edit, media_item: media_item, changeset: changeset)
-      if delete_files do
-        "Record and files deleted successfully."
-      else
-        "Record deleted successfully. Files were not deleted."
   end
 
+  def update(conn, %{"id" => id, "media_item" => params}) do
+    media_item = Media.get_media_item!(id)
+
+    case Media.update_media_item(media_item, params) do
+      {:ok, media_item} ->
         conn
-    |> put_flash(:info, flash_message)
+        |> put_flash(:info, "Media Item updated successfully.")
+        |> redirect(to: ~p"/sources/#{media_item.source_id}/media/#{media_item}")
+
+      {:error, %Ecto.Changeset{} = changeset} ->
+        render(conn, :edit, media_item: media_item, changeset: changeset)
+    end
+  end
+
+  def delete(conn, %{"id" => id} = params) do
+    prevent_download = Map.get(params, "prevent_download", false)
+    media_item = Media.get_media_item!(id)
+    {:ok, _} = Media.delete_media_files(media_item, %{prevent_download: prevent_download})
+
+    conn
+    |> put_flash(:info, "Files deleted successfully.")
     |> redirect(to: ~p"/sources/#{media_item.source_id}")
   end
 
+  def force_download(conn, %{"media_item_id" => id}) do
+    media_item = Media.get_media_item!(id)
+    {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{force: true})
+
+    conn
+    |> put_flash(:info, "Download task enqueued.")
+    |> redirect(to: ~p"/sources/#{media_item.source_id}/media/#{media_item}")
+  end
+
   # See here for details on streaming files and range requests:
   # https://www.zeng.dev/post/2023-http-range-and-play-mp4-in-browser/
   #

lib/pinchflat_web/controllers/media_items/media_item_html.ex

@@ -3,6 +3,14 @@ defmodule PinchflatWeb.MediaItems.MediaItemHTML do
 
   embed_templates "media_item_html/*"
 
+  @doc """
+  Renders a media item form.
+  """
+  attr :changeset, Ecto.Changeset, required: true
+  attr :action, :string, required: true
+
+  def media_item_form(assigns)
+
   def media_file_exists?(media_item) do
     !!media_item.media_filepath and File.exists?(media_item.media_filepath)
   end

lib/pinchflat_web/controllers/media_items/media_item_html/actions_dropdown.html.heex

@@ -0,0 +1,32 @@
+<.button_dropdown text="Actions" class="justify-center w-full sm:w-50">
+  <:option>
+    <.link
+      href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}/force_download"}
+      method="post"
+      data-confirm="Are you sure you force a download of this media?"
+    >
+      Force Download
+    </.link>
+  </:option>
+  <:option>
+    <div class="h-px w-full bg-bodydark2"></div>
+  </:option>
+  <:option>
+    <.link
+      href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}"}
+      method="delete"
+      data-confirm="Are you sure you want to delete all files for this media item? This cannot be undone."
+    >
+      Delete Files
+    </.link>
+  </:option>
+  <:option>
+    <.link
+      href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}?prevent_download=true"}
+      method="delete"
+      data-confirm="Are you sure you want to delete all files for this media item and prevent it from re-downloading in the future? This cannot be undone."
+    >
+      Delete and Ignore
+    </.link>
+  </:option>
+</.button_dropdown>

lib/pinchflat_web/controllers/media_items/media_item_html/edit.html.heex

@@ -0,0 +1,13 @@
+<div class="mb-6 flex gap-3 flex-row items-center">
+  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
+    Editing "<%= StringUtils.truncate(@media_item.title, 35) %>"
+  </h2>
+</div>
+
+<div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">
+  <div class="max-w-full overflow-x-auto">
+    <div class="flex flex-col gap-10">
+      <.media_item_form changeset={@changeset} action={~p"/sources/#{@media_item.source_id}/media/#{@media_item}"} />
+    </div>
+  </div>
+</div>

lib/pinchflat_web/controllers/media_items/media_item_html/media_item_form.html.heex

@@ -0,0 +1,31 @@
+<.simple_form
+  :let={f}
+  for={@changeset}
+  action={@action}
+  x-data="{ advancedMode: !!JSON.parse(localStorage.getItem('advancedMode')) }"
+  x-init="$watch('advancedMode', value => localStorage.setItem('advancedMode', JSON.stringify(value)))"
+>
+  <.error :if={@changeset.action}>
+    Oops, something went wrong! Please check the errors below.
+  </.error>
+
+  <h3 class=" text-2xl text-black dark:text-white">
+    General Options
+  </h3>
+
+  <.input
+    field={f[:prevent_download]}
+    type="toggle"
+    label="Prevent Download"
+    help="Checking excludes this media item from automatic download. Download can still be manually forced"
+  />
+
+  <.input
+    field={f[:prevent_culling]}
+    type="toggle"
+    label="Prevent Automatic Deletion"
+    help="Checking excludes media from being automatically deleted based on media retention rules"
+  />
+
+  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Media Item</.button>
+</.simple_form>

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -4,25 +4,23 @@
       <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
     </.link>
     <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
-      Media Item #<%= @media_item.id %>
+      <%= StringUtils.truncate(@media_item.title, 35) %>
     </h2>
   </div>
+
+  <nav>
+    <.link href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}/edit"}>
+      <.button color="bg-primary" rounding="rounded-lg">
+        <.icon name="hero-pencil-square" class="mr-2" /> Edit
+      </.button>
+    </.link>
+  </nav>
 </div>
 <div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark px-7.5">
   <div class="max-w-full overflow-x-auto">
     <.tabbed_layout>
       <:tab_append>
-        <.button_dropdown text="Actions" class="justify-center w-full sm:w-50">
+        <.actions_dropdown media_item={@media_item} />
-          <:option>
-            <.link
-              href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}?delete_files=true"}
-              method="delete"
-              data-confirm="Are you sure you want to delete this record and all associated files on disk? This cannot be undone."
-            >
-              Delete Files
-            </.link>
-          </:option>
-        </.button_dropdown>
       </:tab_append>
 
       <:tab title="Attributes">
@@ -32,12 +30,13 @@
             <.media_preview media_item={@media_item} />
           <% end %>
 
+          <h2 class="font-bold text-2xl"><%= @media_item.title %></h2>
           <h3 class="font-bold text-xl">Attributes</h3>
           <section>
             <strong>Source:</strong>
-            <.inline_link href={~p"/sources/#{@media_item.source_id}"}>
+            <.subtle_link href={~p"/sources/#{@media_item.source_id}"}>
               <%= @media_item.source.custom_name %>
-            </.inline_link>
+            </.subtle_link>
           </section>
 
           <.list_items_from_map map={Map.from_struct(@media_item)} />

lib/pinchflat_web/controllers/media_profiles/media_profile_controller.ex

@@ -1,12 +1,19 @@
 defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
   use PinchflatWeb, :controller
 
+  import Ecto.Query, warn: false
+
   alias Pinchflat.Repo
   alias Pinchflat.Profiles
+  alias Pinchflat.Sources.SourcesQuery
   alias Pinchflat.Profiles.MediaProfile
 
   def index(conn, _params) do
-    media_profiles = Profiles.list_media_profiles()
+    media_profiles =
+      MediaProfile
+      |> order_by(asc: :name)
+      |> Repo.all()
+
     render(conn, :index, media_profiles: media_profiles)
   end
 
@@ -32,12 +39,15 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
   end
 
   def show(conn, %{"id" => id}) do
-    media_profile =
+    media_profile = Profiles.get_media_profile!(id)
-      id
-      |> Profiles.get_media_profile!()
-      |> Repo.preload(:sources)
 
-    render(conn, :show, media_profile: media_profile)
+    sources =
+      SourcesQuery.new()
+      |> SourcesQuery.for_media_profile(media_profile)
+      |> order_by(asc: :custom_name)
+      |> Repo.all()
+
+    render(conn, :show, media_profile: media_profile, sources: sources)
   end
 
   def edit(conn, %{"id" => id}) do

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -50,9 +50,11 @@
         </div>
       </:tab>
       <:tab title="Sources">
-        <.table rows={@media_profile.sources} table_class="text-black dark:text-white">
+        <.table rows={@sources} table_class="text-black dark:text-white">
           <:col :let={source} label="Name">
+            <.subtle_link href={~p"/sources/#{source.id}"}>
               <%= source.custom_name || source.collection_name %>
+            </.subtle_link>
           </:col>
           <:col :let={source} label="Type"><%= source.collection_type %></:col>
           <:col :let={source} label="Should Download?">

lib/pinchflat_web/controllers/pages/page_controller.ex

@@ -11,7 +11,7 @@ defmodule PinchflatWeb.Pages.PageController do
     done_onboarding = params["onboarding"] == "0"
     force_onboarding = params["onboarding"] == "1"
 
-    if done_onboarding, do: Settings.set!(:onboarding, false)
+    if done_onboarding, do: Settings.set(onboarding: false)
 
     if force_onboarding || Settings.get!(:onboarding) do
       render_onboarding_page(conn)
@@ -30,7 +30,7 @@ defmodule PinchflatWeb.Pages.PageController do
   end
 
   defp render_onboarding_page(conn) do
-    Settings.set!(:onboarding, true)
+    Settings.set(onboarding: true)
 
     conn
     |> render(:onboarding_checklist,

lib/pinchflat_web/controllers/podcasts/podcast_controller.ex

@@ -2,7 +2,7 @@ defmodule PinchflatWeb.Podcasts.PodcastController do
   use PinchflatWeb, :controller
 
   alias Pinchflat.Repo
-  alias Pinchflat.Media
+  alias Pinchflat.Media.MediaQuery
   alias Pinchflat.Sources.Source
   alias Pinchflat.Podcasts.RssFeedBuilder
   alias Pinchflat.Podcasts.PodcastHelpers
@@ -20,10 +20,15 @@ defmodule PinchflatWeb.Podcasts.PodcastController do
 
   def feed_image(conn, %{"uuid" => uuid}) do
     source = Repo.get_by!(Source, uuid: uuid)
-    # This provides a fallback image if the source has none.
+
-    # We only need one since we're using the internal metadata image which
+    # This is used to fetch a fallback cover image
-    # we know exists.
+    # if the source doesn't have any usable images
-    media_items = Media.list_downloaded_media_items_for(source, limit: 1)
+    media_items =
+      MediaQuery.new()
+      |> MediaQuery.for_source(source)
+      |> MediaQuery.with_media_filepath()
+      |> Repo.maybe_limit(1)
+      |> Repo.all()
 
     case PodcastHelpers.select_cover_image(source, media_items) do
       {:error, _} ->

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -4,15 +4,21 @@ defmodule PinchflatWeb.Sources.SourceController do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
-  alias Pinchflat.Profiles
+  alias Pinchflat.MediaQuery
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaQuery
+  alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
   def index(conn, _params) do
-    sources = Repo.preload(Sources.list_sources(), :media_profile)
+    sources =
+      Source
+      |> order_by(asc: :custom_name)
+      |> Repo.all()
+      |> Repo.preload(:media_profile)
 
     render(conn, :index, sources: sources)
   end
@@ -49,9 +55,26 @@ defmodule PinchflatWeb.Sources.SourceController do
   def show(conn, %{"id" => id}) do
     source = Repo.preload(Sources.get_source!(id), :media_profile)
 
-    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(source), :job)
+    pending_tasks =
-    pending_media = Media.list_pending_media_items_for(source, limit: 100)
+      source
-    downloaded_media = Media.list_downloaded_media_items_for(source, limit: 100)
+      |> Tasks.list_tasks_for(nil, [:executing, :available, :scheduled, :retryable])
+      |> Repo.preload(:job)
+
+    pending_media =
+      MediaQuery.new()
+      |> MediaQuery.for_source(source)
+      |> MediaQuery.with_media_pending_download()
+      |> order_by(desc: :id)
+      |> limit(100)
+      |> Repo.all()
+
+    downloaded_media =
+      MediaQuery.new()
+      |> MediaQuery.for_source(source)
+      |> MediaQuery.with_media_filepath()
+      |> order_by(desc: :id)
+      |> limit(100)
+      |> Repo.all()
 
     render(conn, :show,
       source: source,
@@ -104,12 +127,30 @@ defmodule PinchflatWeb.Sources.SourceController do
     |> redirect(to: ~p"/sources")
   end
 
+  def force_download(conn, %{"source_id" => id}) do
+    source = Sources.get_source!(id)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+    conn
+    |> put_flash(:info, "Forced download of pending media items.")
+    |> redirect(to: ~p"/sources/#{source}")
+  end
+
+  def force_index(conn, %{"source_id" => id}) do
+    source = Sources.get_source!(id)
+    SlowIndexingHelpers.kickoff_indexing_task(source, %{force: true})
+
+    conn
+    |> put_flash(:info, "Index enqueued.")
+    |> redirect(to: ~p"/sources/#{source}")
+  end
+
   defp media_profiles do
-    Profiles.list_media_profiles()
+    MediaProfile
+    |> order_by(asc: :name)
+    |> Repo.all()
   end
 
-  # NOTE: should move this out of the controller
-  # once I finally add some query fragment layer
   defp total_downloaded_for(source) do
     MediaQuery.new()
     |> MediaQuery.for_source(source)

lib/pinchflat_web/controllers/sources/source_html/actions_dropdown.html.heex

@@ -0,0 +1,55 @@
+<.button_dropdown text="Actions" class="justify-center w-full sm:w-50">
+  <:option>
+    <span
+      x-data="{ copied: false }"
+      x-on:click={"
+                window.copyTextToClipboard('#{rss_feed_url(@conn, @source)}')
+                copied = true
+                setTimeout(() => copied = false, 4000)
+              "}
+    >
+      Copy RSS Feed
+      <span x-show="copied" x-transition.duration.150ms><.icon name="hero-check" class="ml-2 h-4 w-4" /></span>
+    </span>
+  </:option>
+  <:option :if={@source.download_media}>
+    <.link
+      href={~p"/sources/#{@source}/force_download"}
+      method="post"
+      data-confirm="Are you sure you want to force a download of all *pending* media items? This isn't normally needed."
+    >
+      Force Download
+    </.link>
+  </:option>
+  <:option>
+    <.link
+      href={~p"/sources/#{@source}/force_index"}
+      method="post"
+      data-confirm="Are you sure you want to force an index of this source? This isn't normally needed."
+    >
+      Force Index
+    </.link>
+  </:option>
+  <:option>
+    <div class="h-px w-full bg-bodydark2"></div>
+  </:option>
+  <:option>
+    <.link
+      href={~p"/sources/#{@source}"}
+      method="delete"
+      data-confirm="Are you sure you want to delete this source (leaving files in place)? This cannot be undone."
+    >
+      Delete Source
+    </.link>
+  </:option>
+  <:option>
+    <.link
+      href={~p"/sources/#{@source}?delete_files=true"}
+      method="delete"
+      data-confirm="Are you sure you want to delete this source and it's files on disk? This cannot be undone."
+      class="mt-5 md:mt-0"
+    >
+      Delete Source + Files
+    </.link>
+  </:option>
+</.button_dropdown>

lib/pinchflat_web/controllers/sources/source_html/index.html.heex

@@ -22,6 +22,13 @@
         <:col :let={source} label="Should Download?">
           <.icon name={if source.download_media, do: "hero-check", else: "hero-x-mark"} />
         </:col>
+        <:col :let={source} label="Retention">
+          <%= if source.retention_period_days && source.retention_period_days > 0 do %>
+            <%= source.retention_period_days %> day(s)
+          <% else %>
+            <span class="text-lg">∞</span>
+          <% end %>
+        </:col>
         <:col :let={source} label="Media Profile">
           <.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
             <%= source.media_profile.name %>

lib/pinchflat_web/controllers/sources/source_html/show.html.heex

@@ -20,43 +20,7 @@
   <div class="max-w-full overflow-x-auto">
     <.tabbed_layout>
       <:tab_append>
-        <.button_dropdown text="Actions" class="justify-center w-full sm:w-50">
+        <.actions_dropdown source={@source} conn={@conn} />
-          <:option>
-            <span
-              x-data="{ copied: false }"
-              x-on:click={"
-                window.copyTextToClipboard('#{rss_feed_url(@conn, @source)}')
-                copied = true
-                setTimeout(() => copied = false, 4000)
-              "}
-            >
-              Copy RSS Feed
-              <span x-show="copied" x-transition.duration.150ms><.icon name="hero-check" class="ml-2 h-4 w-4" /></span>
-            </span>
-          </:option>
-          <:option>
-            <div class="h-px w-full bg-bodydark2"></div>
-          </:option>
-          <:option>
-            <.link
-              href={~p"/sources/#{@source}"}
-              method="delete"
-              data-confirm="Are you sure you want to delete this source (leaving files in place)? This cannot be undone."
-            >
-              Delete Source
-            </.link>
-          </:option>
-          <:option>
-            <.link
-              href={~p"/sources/#{@source}?delete_files=true"}
-              method="delete"
-              data-confirm="Are you sure you want to delete this source and it's files on disk? This cannot be undone."
-              class="mt-5 md:mt-0"
-            >
-              Delete Source + Files
-            </.link>
-          </:option>
-        </.button_dropdown>
       </:tab_append>
 
       <:tab title="Attributes">
@@ -64,9 +28,9 @@
           <h3 class="font-bold text-lg">Attributes</h3>
           <section>
             <strong>Media Profile:</strong>
-            <.inline_link href={~p"/media_profiles/#{@source.media_profile_id}"}>
+            <.subtle_link href={~p"/media_profiles/#{@source.media_profile_id}"}>
               <%= @source.media_profile.name %>
-            </.inline_link>
+            </.subtle_link>
           </section>
 
           <.list_items_from_map map={Map.from_struct(@source)} />
@@ -77,10 +41,17 @@
           <h4 class="text-white text-lg mb-6">Shows a maximum of 100 media items</h4>
           <.table rows={@pending_media} table_class="text-black dark:text-white">
             <:col :let={media_item} label="Title">
+              <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
                 <%= StringUtils.truncate(media_item.title, 50) %>
+              </.subtle_link>
             </:col>
             <:col :let={media_item} label="" class="flex place-content-evenly">
-              <.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" />
+              <.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" class="mx-1" />
+              <.icon_link
+                href={~p"/sources/#{@source.id}/media/#{media_item.id}/edit"}
+                icon="hero-pencil-square"
+                class="mx-1"
+              />
             </:col>
           </.table>
         <% else %>
@@ -92,10 +63,17 @@
           <h4 class="text-white text-lg mb-6">Shows a maximum of 100 media items (<%= @total_downloaded %> total)</h4>
           <.table rows={@downloaded_media} table_class="text-black dark:text-white">
             <:col :let={media_item} label="Title">
+              <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
                 <%= StringUtils.truncate(media_item.title, 50) %>
+              </.subtle_link>
             </:col>
             <:col :let={media_item} label="" class="flex place-content-evenly">
-              <.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" />
+              <.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" class="mx-1" />
+              <.icon_link
+                href={~p"/sources/#{@source.id}/media/#{media_item.id}/edit"}
+                icon="hero-pencil-square"
+                class="mx-1"
+              />
             </:col>
           </.table>
         <% else %>

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -88,6 +88,14 @@
     help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
   />
 
+  <.input
+    field={f[:retention_period_days]}
+    type="number"
+    label="Retention Period (days)"
+    min="0"
+    help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
+  />
+
   <section x-show="advancedMode">
     <h3 class="mt-8 text-2xl text-black dark:text-white">
       Advanced Options

lib/pinchflat_web/endpoint.ex

@@ -20,7 +20,7 @@ defmodule PinchflatWeb.Endpoint do
   plug Plug.Static,
     at: "/",
     from: :pinchflat,
-    gzip: false,
+    gzip: Mix.env() == :prod,
     only: PinchflatWeb.static_paths()
 
   # Code reloading can be explicitly enabled under the

lib/pinchflat_web/router.ex

@@ -32,7 +32,12 @@ defmodule PinchflatWeb.Router do
     resources "/search", Searches.SearchController, only: [:show], singleton: true
 
     resources "/sources", Sources.SourceController do
-      resources "/media", MediaItems.MediaItemController, only: [:show, :delete]
+      post "/force_download", Sources.SourceController, :force_download
+      post "/force_index", Sources.SourceController, :force_index
+
+      resources "/media", MediaItems.MediaItemController, only: [:show, :edit, :update, :delete] do
+        post "/force_download", MediaItems.MediaItemController, :force_download
+      end
     end
   end
 
@@ -47,6 +52,13 @@ defmodule PinchflatWeb.Router do
     get "/media/:uuid/stream", MediaItems.MediaItemController, :stream
   end
 
+  # No auth or CSRF protection for the health check endpoint
+  scope "/", PinchflatWeb do
+    pipe_through :api
+
+    get "/healthcheck", HealthController, :check
+  end
+
   scope "/dev" do
     pipe_through :browser
 

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.6",
+      version: "0.1.9",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

priv/repo/migrations/20240401170440_re_re_add_source_uniqueness_index.exs

@@ -0,0 +1,19 @@
+defmodule Pinchflat.Repo.Migrations.ReReAddSourceUniquenessIndex do
+  use Ecto.Migration
+
+  def up do
+    execute """
+      CREATE UNIQUE INDEX sources_collection_id_media_profile_id_title_filter_regex_index ON sources (
+        collection_id,
+        media_profile_id,
+        IFNULL(title_filter_regex, '')
+      );
+    """
+  end
+
+  def down do
+    execute """
+      DROP INDEX sources_collection_id_media_profile_id_title_filter_regex_index;
+    """
+  end
+end

priv/repo/migrations/20240402192417_add_prevent_download_to_media_items.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddPreventDownloadToMediaItems do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :prevent_download, :boolean, default: false, null: false
+    end
+  end
+end

priv/repo/migrations/20240402221638_add_retention_period_to_sources.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddRetentionPeriodToSources do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :retention_period_days, :integer
+    end
+  end
+end

priv/repo/migrations/20240403164943_add_culled_at_to_media_items.exs

@@ -0,0 +1,10 @@
+defmodule Pinchflat.Repo.Migrations.AddCulledAtToMediaItems do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :culled_at, :utc_datetime
+      add :prevent_culling, :boolean, default: false
+    end
+  end
+end

priv/repo/migrations/20240404172256_rename_settings_table.exs

@@ -0,0 +1,7 @@
+defmodule Pinchflat.Repo.Migrations.RenameSettingsTable do
+  use Ecto.Migration
+
+  def change do
+    rename table(:settings), to: table(:settings_backup)
+  end
+end

priv/repo/migrations/20240404174144_create_new_settings.exs

@@ -0,0 +1,29 @@
+defmodule Pinchflat.Repo.Migrations.CreateNewSettings do
+  use Ecto.Migration
+
+  def up do
+    create table(:settings) do
+      add :onboarding, :boolean, default: true, null: false
+      add :pro_enabled, :boolean, default: false, null: false
+      add :yt_dlp_version, :string
+    end
+
+    # Make an initial record because this will be the only one ever inserted
+    execute "INSERT INTO settings (onboarding, pro_enabled, yt_dlp_version) VALUES (true, false, NULL)"
+
+    # Set the value of onboarding to the previous version set in `settings_backup`
+    execute """
+      UPDATE settings
+      SET onboarding = COALESCE((SELECT value = 'true' FROM settings_backup WHERE name = 'onboarding'), true)
+    """
+
+    execute """
+      UPDATE settings
+      SET pro_enabled = COALESCE((SELECT value = 'true' FROM settings_backup WHERE name = 'pro_enabled'), false)
+    """
+  end
+
+  def down do
+    drop table(:settings)
+  end
+end

selfhosted.Dockerfile

@@ -95,11 +95,9 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
 WORKDIR "/app"
-RUN chown nobody /app
 
 # Set up data volumes
 RUN mkdir /config /downloads
-RUN chown nobody /config /downloads
 
 # set runner ENV
 ENV MIX_ENV="prod"
@@ -108,7 +106,7 @@ ENV RUN_CONTEXT="selfhosted"
 EXPOSE ${PORT}
 
 # Only copy the final release from the build stage
-COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./
+COPY --from=builder /app/_build/${MIX_ENV}/rel/pinchflat ./
 
 # NEVER do this if you're running in an environment where you don't trust the user
 # (ie: most environments). This is only acceptable in a self-hosted environment.
@@ -121,12 +119,13 @@ COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./
 # root, use --user 0:0 or something.
 RUN passwd -d root
 
-USER nobody
-
 # If using an environment that doesn't automatically reap zombie processes, it is
 # advised to add an init process such as tini via `apt-get install`
 # above and adding an entrypoint. See https://github.com/krallin/tini for details
 # ENTRYPOINT ["/tini", "--"]
 
+HEALTHCHECK --interval=120s --start-period=10s \
+  CMD curl --fail http://localhost:${PORT}/healthcheck || exit 1
+
 # Start the app
 CMD ["/app/bin/docker_start"]

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -1,46 +0,0 @@
-defmodule Pinchflat.Boot.DataBackfillWorkerTest do
-  use Pinchflat.DataCase
-
-  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.JobFixtures.TestJobWorker
-
-  describe "cancel_pending_backfill_jobs/0" do
-    test "cancels all pending backfill jobs" do
-      %{}
-      |> DataBackfillWorker.new()
-      |> Repo.insert_unique_job()
-
-      assert_enqueued(worker: DataBackfillWorker)
-
-      DataBackfillWorker.cancel_pending_backfill_jobs()
-
-      refute_enqueued(worker: DataBackfillWorker)
-    end
-
-    test "does not cancel jobs for other workers" do
-      %{id: 0}
-      |> TestJobWorker.new()
-      |> Repo.insert_unique_job()
-
-      assert_enqueued(worker: TestJobWorker)
-
-      DataBackfillWorker.cancel_pending_backfill_jobs()
-
-      assert_enqueued(worker: TestJobWorker)
-    end
-  end
-
-  describe "perform/1" do
-    setup do
-      DataBackfillWorker.cancel_pending_backfill_jobs()
-
-      :ok
-    end
-
-    test "reschedules itself once complete" do
-      perform_job(DataBackfillWorker, %{})
-
-      assert_enqueued(worker: DataBackfillWorker, scheduled_at: now_plus(60, :minutes))
-    end
-  end
-end

test/pinchflat/boot/pre_job_startup_tasks_test.exs

@@ -1,25 +1,47 @@
 defmodule Pinchflat.Boot.PreJobStartupTasksTest do
   use Pinchflat.DataCase
 
+  import Pinchflat.JobFixtures
+
   alias Pinchflat.Settings
-  alias Pinchflat.Settings.Setting
   alias Pinchflat.Boot.PreJobStartupTasks
 
-  describe "apply_default_settings" do
+  describe "reset_executing_jobs" do
-    setup do
+    test "resets executing jobs" do
-      Repo.delete_all(Setting)
+      job = job_fixture()
+      Repo.update_all(Oban.Job, set: [state: "executing"])
 
-      :ok
+      assert Repo.reload!(job).state == "executing"
-    end
-
-    test "sets default settings" do
-      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
-      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
 
       PreJobStartupTasks.start_link()
 
-      assert Settings.get!(:onboarding)
+      assert Repo.reload!(job).state == "retryable"
-      refute Settings.get!(:pro_enabled)
+    end
+  end
+
+  describe "create_blank_cookie_file" do
+    test "creates a blank cookie file" do
+      base_dir = Application.get_env(:pinchflat, :extras_directory)
+      filepath = Path.join(base_dir, "cookies.txt")
+      File.rm(filepath)
+
+      refute File.exists?(filepath)
+
+      PreJobStartupTasks.start_link()
+
+      assert File.exists?(filepath)
+    end
+  end
+
+  describe "apply_default_settings" do
+    test "sets default settings" do
+      Settings.set(yt_dlp_version: nil)
+
+      refute Settings.get!(:yt_dlp_version)
+
+      PreJobStartupTasks.start_link()
+
+      assert Settings.get!(:yt_dlp_version)
     end
   end
 end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -4,6 +4,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   import Mox
   import Pinchflat.MediaFixtures
 
+  alias Pinchflat.Media
   alias Pinchflat.Sources
   alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.Downloading.MediaDownloadWorker
@@ -34,11 +35,28 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert {:ok, task} = MediaDownloadWorker.kickoff_with_task(media_item)
       assert task.media_item_id == media_item.id
     end
+
+    test "can be called with additional job arguments", %{media_item: media_item} do
+      job_args = %{"force" => true}
+
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, job_args)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id, "force" => true})
+    end
+
+    test "can be called with additional job options", %{media_item: media_item} do
+      job_opts = [max_attempts: 5]
+
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
+
+      [job] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+      assert job.max_attempts == 5
+    end
   end
 
   describe "perform/1" do
     test "it saves attributes to the media_item", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
@@ -48,7 +66,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
     end
 
     test "it saves the metadata to the media_item", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
@@ -65,7 +83,19 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
     end
 
     test "it sets the job to retryable if the download fails", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "error"} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl -> {:error, "error"} end)
+
+      Oban.Testing.with_testing_mode(:inline, fn ->
+        {:ok, job} = Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
+
+        assert job.state == "retryable"
+      end)
+    end
+
+    test "sets the job to retryable if the download failed and was retried", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
+        {:error, "Unable to communicate with SponsorBlock", 1}
+      end)
 
       Oban.Testing.with_testing_mode(:inline, fn ->
         {:ok, job} = Oban.insert(MediaDownloadWorker.new(%{id: media_item.id}))
@@ -75,21 +105,38 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
     end
 
     test "it ensures error are returned in a 2-item tuple", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "error", 1} end)
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl -> {:error, "error", 1} end)
 
       assert {:error, :download_failed} = perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
     test "it does not download if the source is set to not download", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot -> :ok end)
+      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl -> :ok end)
 
       Sources.update_source(media_item.source, %{download_media: false})
 
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
+    test "does not download if the media item is set to not download", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl -> :ok end)
+
+      Media.update_media_item(media_item, %{prevent_download: true})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id})
+    end
+
+    test "downloads anyway if forced", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl -> :ok end)
+
+      Sources.update_source(media_item.source, %{download_media: false})
+      Media.update_media_item(media_item, %{prevent_download: true})
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
+    end
+
     test "it saves the file's size to the database", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         metadata = render_parsed_metadata(:media_metadata)
         FilesystemHelpers.write_p!(metadata["filepath"], "test")
 

test/pinchflat/downloading/media_downloader_test.exs

@@ -25,9 +25,11 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
 
   describe "download_for_media_item/3" do
     test "it calls the backend runner", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn url, _opts, ot ->
+      expect(YtDlpRunnerMock, :run, fn url, _opts, ot, addl ->
         assert url == media_item.original_url
         assert ot == "after_move:%()j"
+        assert [{:output_filepath, filepath}] = addl
+        assert is_binary(filepath)
 
         {:ok, render_metadata(:media_metadata)}
       end)
@@ -36,7 +38,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
     end
 
     test "it saves the metadata filepath to the database", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
@@ -47,18 +49,56 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
       assert updated_media_item.metadata.thumbnail_filepath =~ "media_items/#{media_item.id}/maxresdefault.jpg"
     end
 
-    test "errors are passed through", %{media_item: media_item} do
+    test "non-recoverable errors are passed through", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
-        {:error, :some_error}
+        {:error, :some_error, 1}
       end)
 
       assert {:error, :some_error} = MediaDownloader.download_for_media_item(media_item)
     end
+
+    test "unknown errors are passed through", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
+        {:error, :some_error}
+      end)
+
+      assert {:error, message} = MediaDownloader.download_for_media_item(media_item)
+      assert message == "Unknown error: {:error, :some_error}"
+    end
+  end
+
+  describe "download_for_media_item/3 when testing retries" do
+    test "returns a recovered tuple on recoverable errors", %{media_item: media_item} do
+      message = "Unable to communicate with SponsorBlock"
+
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
+        {:error, message, 1}
+      end)
+
+      assert {:recovered, ^message} = MediaDownloader.download_for_media_item(media_item)
+    end
+
+    test "attempts to update the media item on recoverable errors", %{media_item: media_item} do
+      message = "Unable to communicate with SponsorBlock"
+
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl ->
+        [{:output_filepath, filepath}] = addl
+        File.write(filepath, render_metadata(:media_metadata))
+
+        {:error, message, 1}
+      end)
+
+      assert {:recovered, ^message} = MediaDownloader.download_for_media_item(media_item)
+      media_item = Repo.reload(media_item)
+
+      assert DateTime.diff(DateTime.utc_now(), media_item.media_downloaded_at) < 2
+      assert String.ends_with?(media_item.media_filepath, ".mkv")
+    end
   end
 
   describe "download_for_media_item/3 when testing media_item attributes" do
     setup do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
@@ -100,7 +140,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
     end
 
     test "it extracts the thumbnail_filepath", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         metadata = render_parsed_metadata(:media_metadata)
 
         thumbnail_filepath =
@@ -124,7 +164,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
     end
 
     test "it extracts the metadata_filepath", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         metadata = render_parsed_metadata(:media_metadata)
 
         infojson_filepath = metadata["infojson_filename"]
@@ -143,7 +183,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
 
   describe "download_for_media_item/3 when testing NFO generation" do
     setup do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 

test/pinchflat/downloading/media_retention_worker_test.exs

@@ -0,0 +1,70 @@
+defmodule Pinchflat.Downloading.MediaRetentionWorkerTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Media
+  alias Pinchflat.Downloading.MediaRetentionWorker
+
+  describe "perform/1" do
+    test "deletes media files that are past their retention date" do
+      {_source, old_media_item, new_media_item} = prepare_records()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(new_media_item.media_filepath)
+      refute File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(new_media_item).media_filepath
+      refute Repo.reload!(old_media_item).media_filepath
+    end
+
+    test "sets deleted media to not re-download" do
+      {_source, old_media_item, new_media_item} = prepare_records()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(new_media_item).prevent_download
+      assert Repo.reload!(old_media_item).prevent_download
+    end
+
+    test "sets culled_at timestamp on deleted media" do
+      {_source, old_media_item, new_media_item} = prepare_records()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(new_media_item).culled_at
+      assert Repo.reload!(old_media_item).culled_at
+      assert DateTime.diff(now(), Repo.reload!(old_media_item).culled_at) < 1
+    end
+
+    test "doesn't cull media items that have prevent_culling set" do
+      {_source, old_media_item, _new_media_item} = prepare_records()
+
+      Media.update_media_item(old_media_item, %{prevent_culling: true})
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(old_media_item).media_filepath
+    end
+  end
+
+  defp prepare_records do
+    source = source_fixture(%{retention_period_days: 2})
+
+    old_media_item =
+      media_item_with_attachments(%{
+        source_id: source.id,
+        media_downloaded_at: now_minus(3, :days)
+      })
+
+    new_media_item =
+      media_item_with_attachments(%{
+        source_id: source.id,
+        media_downloaded_at: now_minus(1, :day)
+      })
+
+    {source, old_media_item, new_media_item}
+  end
+end

test/pinchflat/media_test.exs

@@ -38,6 +38,98 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "list_cullable_media_items/0" do
+    test "returns media items where the source has a retention period" do
+      source_one = source_fixture(%{retention_period_days: 2})
+      source_two = source_fixture(%{retention_period_days: 0})
+      source_three = source_fixture(%{retention_period_days: nil})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source_two.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source_three.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source_one.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+
+    test "returns media_items with a media_filepath" do
+      source = source_fixture(%{retention_period_days: 2})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: nil,
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+
+    test "returns items that have passed their retention period" do
+      source = source_fixture(%{retention_period_days: 2})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(2, :days)
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+
+    test "doesn't return items that are set to prevent culling" do
+      source = source_fixture(%{retention_period_days: 2})
+
+      _media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days),
+          prevent_culling: true
+        })
+
+      expected_media_item =
+        media_item_fixture(%{
+          source_id: source.id,
+          media_filepath: "/video/#{Faker.File.file_name(:video)}",
+          media_downloaded_at: now_minus(3, :days)
+        })
+
+      assert Media.list_cullable_media_items() == [expected_media_item]
+    end
+  end
+
   describe "list_pending_media_items_for/1" do
     test "it returns pending without a filepath for a given source" do
       source = source_fixture()
@@ -57,14 +149,6 @@ defmodule Pinchflat.MediaTest do
 
       assert Media.list_pending_media_items_for(source) == []
     end
-
-    test "optionally accepts a limit" do
-      source = source_fixture()
-      media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
-
-      assert Media.list_pending_media_items_for(source, limit: 1) == [media_item]
-      assert Media.list_pending_media_items_for(source, limit: 0) == []
-    end
   end
 
   describe "list_pending_media_items_for/1 when testing shorts" do
@@ -233,22 +317,13 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
-  describe "list_downloaded_media_items_for/1" do
+  describe "list_pending_media_items_for/1 when testing download prevention" do
-    test "returns only media items with a media_filepath" do
+    test "returns only media items that are not prevented from downloading" do
       source = source_fixture()
-      _media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      _prevented_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, prevent_download: true})
-      media_item = media_item_fixture(%{source_id: source.id, media_filepath: "/video/#{Faker.File.file_name(:video)}"})
+      media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, prevent_download: false})
 
-      assert Media.list_downloaded_media_items_for(source) == [media_item]
+      assert Media.list_pending_media_items_for(source) == [media_item]
-    end
-
-    test "optionally accepts a limit" do
-      source = source_fixture()
-      _media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
-      media_item = media_item_fixture(%{source_id: source.id, media_filepath: "/video/#{Faker.File.file_name(:video)}"})
-
-      assert Media.list_downloaded_media_items_for(source, limit: 1) == [media_item]
-      assert Media.list_downloaded_media_items_for(source, limit: 0) == []
     end
   end
 
@@ -320,6 +395,18 @@ defmodule Pinchflat.MediaTest do
 
       assert Media.pending_download?(media_item)
     end
+
+    test "returns true if the media item is not prevented from downloading" do
+      media_item = media_item_fixture(%{media_filepath: nil, prevent_download: false})
+
+      assert Media.pending_download?(media_item)
+    end
+
+    test "returns false if the media item is prevented from downloading" do
+      media_item = media_item_fixture(%{media_filepath: nil, prevent_download: true})
+
+      refute Media.pending_download?(media_item)
+    end
   end
 
   describe "search/1" do
@@ -587,6 +674,63 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "delete_media_files/2" do
+    test "does not delete the media_item" do
+      media_item = media_item_fixture()
+
+      assert {:ok, %MediaItem{}} = Media.delete_media_files(media_item)
+      assert Repo.reload!(media_item)
+    end
+
+    test "deletes attached tasks" do
+      media_item = media_item_fixture()
+      task = task_fixture(%{media_item_id: media_item.id})
+
+      assert {:ok, %MediaItem{}} = Media.delete_media_files(media_item)
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+
+    test "deletes the media_item's files" do
+      media_item = media_item_with_attachments()
+
+      assert File.exists?(media_item.media_filepath)
+      assert {:ok, _} = Media.delete_media_files(media_item)
+      refute File.exists?(media_item.media_filepath)
+    end
+
+    test "does not delete the media item's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      media_item = Repo.preload(media_item_with_attachments(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
+          thumbnail_filepath:
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
+        }
+      }
+
+      {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs)
+      metadata = Repo.preload(updated_media_item, :metadata).metadata
+
+      assert {:ok, _} = Media.delete_media_files(updated_media_item)
+      assert Repo.reload(metadata)
+      assert File.exists?(updated_media_item.metadata.metadata_filepath)
+
+      # cleanup
+      Media.delete_media_item(updated_media_item, delete_files: true)
+    end
+
+    test "can take additional attributes update media item" do
+      media_item = media_item_with_attachments()
+
+      assert {:ok, updated_media_item} = Media.delete_media_files(media_item, %{prevent_download: true})
+      assert updated_media_item.prevent_download
+    end
+  end
+
   describe "change_media_item/1" do
     test "change_media_item/1 returns a media_item changeset" do
       media_item = media_item_fixture()

test/pinchflat/metadata/nfo_builder_test.exs

@@ -30,6 +30,21 @@ defmodule Pinchflat.Metadata.NfoBuilderTest do
       assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
       assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
     end
+
+    test "escapes invalid characters", %{filepath: filepath} do
+      metadata = %{
+        "title" => "hello' & <world>",
+        "uploader" => "uploader",
+        "id" => "id",
+        "description" => "description",
+        "upload_date" => "20210101"
+      }
+
+      result = NfoBuilder.build_and_store_for_media_item(filepath, metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, "hello&#39; &amp; &lt;world&gt;")
+    end
   end
 
   describe "build_and_store_for_source/2" do
@@ -46,5 +61,18 @@ defmodule Pinchflat.Metadata.NfoBuilderTest do
       assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
       assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
     end
+
+    test "escapes invalid characters", %{filepath: filepath} do
+      metadata = %{
+        "title" => "hello' & <world>",
+        "description" => "description",
+        "id" => "id"
+      }
+
+      result = NfoBuilder.build_and_store_for_source(filepath, metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, "hello&#39; &amp; &lt;world&gt;")
+    end
   end
 end

test/pinchflat/settings_test.exs

@@ -9,100 +9,55 @@ defmodule Pinchflat.SettingsTest do
   # are always created on app boot (including in the test env),
   # so we can't treat these like a clean slate.
 
-  describe "list_settings/0" do
+  setup do
-    test "returns all settings" do
+    # Ensure we have a clean slate
-      Settings.set!("foo", "bar")
+    Settings.set(onboarding: false)
-      results = Settings.list_settings()
+    Settings.set(pro_enabled: false)
+    Settings.set(yt_dlp_version: nil)
 
-      assert Enum.all?(results, fn setting -> match?(%Setting{}, setting) end)
+    :ok
+  end
+
+  describe "record/0" do
+    test "returns the only setting" do
+      assert %Setting{} = Settings.record()
     end
   end
 
-  describe "set/2" do
+  describe "set/1" do
-    test "creates a new setting if one does not exist" do
+    test "updates the setting" do
-      original = Repo.aggregate(Setting, :count, :id)
+      assert {:ok, true} = Settings.set(onboarding: true)
-      Settings.set!("foo", "bar")
+      assert {:ok, true} = Settings.get(:onboarding)
-      assert Repo.aggregate(Setting, :count, :id) == original + 1
     end
 
-    test "updates an existing setting if one exists" do
+    test "returns an error if the setting key doesn't exist" do
-      Settings.set!("foo", "bar")
+      assert {:error, :invalid_key} = Settings.set(foo: "bar")
-      original = Repo.aggregate(Setting, :count, :id)
-      Settings.set!("foo", "baz")
-      assert Repo.aggregate(Setting, :count, :id) == original
-      assert Settings.get!("foo") == "baz"
     end
 
-    test "returns the parsed value" do
+    test "returns an error if the setting value is invalid" do
-      assert Settings.set!("foo", true) == true
+      assert {:error, %Ecto.Changeset{}} = Settings.set(onboarding: "bar")
-      assert Settings.set!("foo", false) == false
-      assert Settings.set!("foo", 123) == 123
-      assert Settings.set!("foo", 12.34) == 12.34
-      assert Settings.set!("foo", "bar") == "bar"
-    end
-
-    test "allows for atom keys" do
-      assert Settings.set!(:foo, "bar") == "bar"
-    end
-
-    test "blows up when an unsupported datatype is used" do
-      assert_raise FunctionClauseError, fn ->
-        Settings.set!("foo", nil)
-      end
-    end
-  end
-
-  describe "set/3" do
-    test "allows manual specification of datatype" do
-      assert Settings.set!("foo", "true", :boolean) == true
-      assert Settings.set!("foo", "false", :boolean) == false
-      assert Settings.set!("foo", "123", :integer) == 123
-      assert Settings.set!("foo", "12.34", :float) == 12.34
     end
   end
 
   describe "get/1" do
-    test "returns the value of the setting" do
+    test "returns the setting value" do
-      Settings.set!("str", "bar")
+      assert {:ok, false} = Settings.get(:onboarding)
-      Settings.set!("bool", true)
-      Settings.set!("int", 123)
-      Settings.set!("float", 12.34)
-
-      assert Settings.get!("str") == "bar"
-      assert Settings.get!("bool") == true
-      assert Settings.get!("int") == 123
-      assert Settings.get!("float") == 12.34
     end
 
-    test "allows for atom keys" do
+    test "returns an error if the setting key doesn't exist" do
-      Settings.set!("str", "bar")
+      assert {:error, :invalid_key} = Settings.get(:foo)
-      assert Settings.get!(:str) == "bar"
-    end
-
-    test "blows up when the setting does not exist" do
-      assert_raise Ecto.NoResultsError, fn ->
-        Settings.get!("foo")
-      end
     end
   end
 
-  describe "fetch/2" do
+  describe "get!/1" do
-    test "creates a setting if one doesn't exist" do
+    test "returns the setting value" do
-      original = Repo.aggregate(Setting, :count, :id)
+      assert Settings.get!(:onboarding) == false
-      assert Settings.fetch!("foo", "bar") == "bar"
-      assert Repo.aggregate(Setting, :count, :id) == original + 1
     end
 
-    test "returns an existing setting if one does exist" do
+    test "raises an error if the setting key doesn't exist" do
-      Settings.set!("foo", "bar")
+      assert_raise RuntimeError, "Setting `foo` not found", fn ->
-
+        Settings.get!(:foo)
-      assert Settings.fetch!("foo", "baz") == "bar"
       end
     end
-
-  describe "fetch/3" do
-    test "allows manual specification of datatype" do
-      assert Settings.fetch!("foo", "true", :boolean) == true
-    end
   end
 end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -14,6 +14,42 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
 
   setup :verify_on_exit!
 
+  describe "kickoff_with_task/3" do
+    setup do
+      source = source_fixture(index_frequency_minutes: 10)
+
+      {:ok, %{source: source}}
+    end
+
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: MediaCollectionIndexingWorker)
+      assert {:ok, _} = MediaCollectionIndexingWorker.kickoff_with_task(source)
+      assert [_] = all_enqueued(worker: MediaCollectionIndexingWorker)
+    end
+
+    test "attaches a task", %{source: source} do
+      assert {:ok, task} = MediaCollectionIndexingWorker.kickoff_with_task(source)
+      assert task.source_id == source.id
+    end
+
+    test "can be called with additional job arguments", %{source: source} do
+      job_args = %{"force" => true}
+
+      assert {:ok, _} = MediaCollectionIndexingWorker.kickoff_with_task(source, job_args)
+
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id, "force" => true})
+    end
+
+    test "can be called with additional job options", %{source: source} do
+      job_opts = [max_attempts: 5]
+
+      assert {:ok, _} = MediaCollectionIndexingWorker.kickoff_with_task(source, %{}, job_opts)
+
+      [job] = all_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
+      assert job.max_attempts == 5
+    end
+  end
+
   describe "perform/1" do
     test "it indexes the source if it should be indexed" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
@@ -31,6 +67,14 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
       perform_job(MediaCollectionIndexingWorker, %{id: source.id})
     end
 
+    test "it indexes the source no matter what if the 'force' arg is passed" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 0, last_indexed_at: DateTime.utc_now())
+
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id, force: true})
+    end
+
     test "it does not do any indexing if the source has been indexed and shouldn't be rescheduled" do
       expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
 

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -18,7 +18,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
 
   setup :verify_on_exit!
 
-  describe "kickoff_indexing_task/1" do
+  describe "kickoff_indexing_task/3" do
     test "it schedules a job" do
       source = source_fixture(index_frequency_minutes: 1)
 
@@ -64,6 +64,25 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
+
+    test "can be called with additional job arguments" do
+      source = source_fixture(index_frequency_minutes: 1)
+      job_args = %{"force" => true}
+
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source, job_args)
+
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id, "force" => true})
+    end
+
+    test "can be called with additional job options" do
+      source = source_fixture(index_frequency_minutes: 1)
+      job_opts = [max_attempts: 5]
+
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source, %{}, job_opts)
+
+      [job] = all_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
+      assert job.max_attempts == 5
+    end
   end
 
   describe "index_and_enqueue_download_for_media_items/1" do

test/pinchflat/sources_test.exs

@@ -137,6 +137,75 @@ defmodule Pinchflat.SourcesTest do
       assert source.custom_name == "some channel name"
     end
 
+    test "creation enforces uniqueness of collection_id scoped to the media_profile and title regex" do
+      expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
+        {:ok,
+         Phoenix.json_library().encode!(%{
+           channel: "some channel name",
+           channel_id: "some_channel_id_12345678",
+           playlist_id: "some_channel_id_12345678",
+           playlist_title: "some channel name - videos"
+         })}
+      end)
+
+      valid_once_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        title_filter_regex: nil
+      }
+
+      assert {:ok, %Source{}} = Sources.create_source(valid_once_attrs)
+      assert {:error, %Ecto.Changeset{}} = Sources.create_source(valid_once_attrs)
+    end
+
+    test "creation lets you duplicate collection_ids and profiles as long as the regex is different" do
+      expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
+        {:ok,
+         Phoenix.json_library().encode!(%{
+           channel: "some channel name",
+           channel_id: "some_channel_id_12345678",
+           playlist_id: "some_channel_id_12345678",
+           playlist_title: "some channel name - videos"
+         })}
+      end)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        name: "some name",
+        original_url: "https://www.youtube.com/channel/abc123"
+      }
+
+      source_1_attrs = Map.merge(valid_attrs, %{title_filter_regex: "foo"})
+      source_2_attrs = Map.merge(valid_attrs, %{title_filter_regex: "bar"})
+
+      assert {:ok, %Source{}} = Sources.create_source(source_1_attrs)
+      assert {:ok, %Source{}} = Sources.create_source(source_2_attrs)
+    end
+
+    test "creation lets you duplicate collection_ids as long as the media profile is different" do
+      expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
+        {:ok,
+         Phoenix.json_library().encode!(%{
+           channel: "some channel name",
+           channel_id: "some_channel_id_12345678",
+           playlist_id: "some_channel_id_12345678",
+           playlist_title: "some channel name - videos"
+         })}
+      end)
+
+      valid_attrs = %{
+        name: "some name",
+        original_url: "https://www.youtube.com/channel/abc123",
+        title_filter_regex: "TEST"
+      }
+
+      source_1_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
+      source_2_attrs = Map.merge(valid_attrs, %{media_profile_id: media_profile_fixture().id})
+
+      assert {:ok, %Source{}} = Sources.create_source(source_1_attrs)
+      assert {:ok, %Source{}} = Sources.create_source(source_2_attrs)
+    end
+
     test "collection_type is inferred from source details" do
       expect(YtDlpRunnerMock, :run, &channel_mock/3)
       expect(YtDlpRunnerMock, :run, &playlist_mock/3)

test/pinchflat/utils/xml_utils_test.exs

@@ -0,0 +1,16 @@
+defmodule Pinchflat.Utils.XmlUtilsTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.Utils.XmlUtils
+
+  describe "safe/1" do
+    test "escapes invalid characters" do
+      assert XmlUtils.safe("hello' & <world>") == "hello&#39; &amp; &lt;world&gt;"
+    end
+
+    test "converts input to string" do
+      assert XmlUtils.safe(42) == "42"
+      assert XmlUtils.safe(nil) == ""
+    end
+  end
+end

test/pinchflat/yt_dlp/command_runner_test.exs

@@ -102,6 +102,14 @@ defmodule Pinchflat.YtDlp.CommandRunnerTest do
     end
   end
 
+  describe "version/0" do
+    test "adds the version arg" do
+      assert {:ok, output} = Runner.version()
+
+      assert String.contains?(output, "--version")
+    end
+  end
+
   defp wrap_executable(new_executable, fun) do
     Application.put_env(:pinchflat, :yt_dlp_executable, new_executable)
     fun.()

test/pinchflat/yt_dlp/media_test.exs

@@ -11,9 +11,10 @@ defmodule Pinchflat.YtDlp.MediaTest do
 
   describe "download/2" do
     test "it calls the backend runner with the expected arguments" do
-      expect(YtDlpRunnerMock, :run, fn @media_url, opts, ot ->
+      expect(YtDlpRunnerMock, :run, fn @media_url, opts, ot, addl ->
         assert [:no_simulate] = opts
         assert "after_move:%()j" = ot
+        assert addl == []
 
         {:ok, render_metadata(:media_metadata)}
       end)
@@ -22,17 +23,18 @@ defmodule Pinchflat.YtDlp.MediaTest do
     end
 
     test "it passes along additional options" do
-      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, addl ->
         assert [:no_simulate, :custom_arg] = opts
+        assert [addl_arg: true] = addl
 
         {:ok, "{}"}
       end)
 
-      assert {:ok, _} = Media.download(@media_url, [:custom_arg])
+      assert {:ok, _} = Media.download(@media_url, [:custom_arg], addl_arg: true)
     end
 
     test "it parses and returns the generated file as JSON" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
@@ -41,7 +43,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
     end
 
     test "it returns errors" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opt, _ot ->
+      expect(YtDlpRunnerMock, :run, fn _url, _opt, _ot, _addl ->
         {:error, "something"}
       end)
 

test/pinchflat_web/controllers/health_controller_test.exs

@@ -0,0 +1,10 @@
+defmodule PinchflatWeb.HealthControllerTest do
+  use PinchflatWeb.ConnCase
+
+  describe "GET /healthcheck" do
+    test "returns ok", %{conn: conn} do
+      conn = get(conn, "/healthcheck")
+      assert json_response(conn, 200) == %{"status" => "ok"}
+    end
+  end
+end

test/pinchflat_web/controllers/media_item_controller_test.exs

@@ -4,33 +4,65 @@ defmodule PinchflatWeb.MediaItemControllerTest do
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Repo
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   describe "show media" do
     setup [:create_media_item]
 
     test "renders the page", %{conn: conn, media_item: media_item} do
       conn = get(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item}")
-      assert html_response(conn, 200) =~ "Media Item ##{media_item.id}"
+
+      assert html_response(conn, 200) =~ "#{media_item.title}"
     end
   end
 
-  describe "delete media when just deleting the records" do
+  describe "edit media" do
+    setup [:create_media_item]
+
+    test "renders form for editing chosen media_item", %{conn: conn, media_item: media_item} do
+      conn = get(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item}/edit")
+
+      assert html_response(conn, 200) =~ "Editing"
+    end
+  end
+
+  describe "update media" do
+    setup [:create_media_item]
+
+    test "redirects when data is valid", %{conn: conn, media_item: media_item} do
+      update_attrs = %{title: "New Title"}
+
+      conn = put(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item}", media_item: update_attrs)
+      assert redirected_to(conn) == ~p"/sources/#{media_item.source_id}/media/#{media_item}"
+
+      conn = get(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item}")
+      assert html_response(conn, 200) =~ update_attrs[:title]
+    end
+
+    test "renders errors when data is invalid", %{conn: conn, media_item: media_item} do
+      conn = put(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item}", media_item: %{title: nil})
+
+      assert html_response(conn, 200) =~ "Editing"
+    end
+  end
+
+  describe "delete media" do
     setup do
       media_item = media_item_with_attachments()
 
       %{media_item: media_item}
     end
 
-    test "the media item is deleted", %{conn: conn, media_item: media_item} do
+    test "the media item not is deleted", %{conn: conn, media_item: media_item} do
       delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}")
 
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      assert Repo.reload!(media_item)
     end
 
-    test "the files are not deleted", %{conn: conn, media_item: media_item} do
+    test "the files are deleted", %{conn: conn, media_item: media_item} do
       delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}")
 
-      assert File.exists?(media_item.media_filepath)
+      refute File.exists?(media_item.media_filepath)
     end
 
     test "redirects to the source page", %{conn: conn, media_item: media_item} do
@@ -38,31 +70,46 @@ defmodule PinchflatWeb.MediaItemControllerTest do
 
       assert redirected_to(conn) == ~p"/sources/#{media_item.source_id}"
     end
+
+    test "doesn't prevent re-download by default", %{conn: conn, media_item: media_item} do
+      delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}")
+
+      media_item = Repo.reload(media_item)
+
+      refute media_item.prevent_download
     end
 
-  describe "delete media when deleting the records and files" do
+    test "can optionally prevent re-download", %{conn: conn, media_item: media_item} do
-    setup do
+      delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}?prevent_download=true")
-      media_item = media_item_with_attachments()
 
-      %{media_item: media_item}
+      media_item = Repo.reload(media_item)
+
+      assert media_item.prevent_download
+    end
   end
 
-    test "the media item is deleted", %{conn: conn, media_item: media_item} do
+  describe "force_download" do
-      delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}?delete_files=true")
+    test "enqueues download task", %{conn: conn} do
+      media_item = media_item_fixture()
 
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      assert [] = all_enqueued(worker: MediaDownloadWorker)
+      post(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}/force_download")
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
 
-    test "the files are deleted", %{conn: conn, media_item: media_item} do
+    test "forces a download even if one wouldn't normally run", %{conn: conn} do
-      delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}?delete_files=true")
+      media_item = media_item_fixture(%{media_filepath: nil})
 
-      refute File.exists?(media_item.media_filepath)
+      post(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}/force_download")
+      assert [_] = all_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id, "force" => true})
     end
 
-    test "redirects to the source page", %{conn: conn, media_item: media_item} do
+    test "redirects to the show page", %{conn: conn} do
-      conn = delete(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}?delete_files=true")
+      media_item = media_item_fixture()
 
-      assert redirected_to(conn) == ~p"/sources/#{media_item.source_id}"
+      conn = post(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}/force_download")
+
+      assert redirected_to(conn) == ~p"/sources/#{media_item.source_id}/media/#{media_item.id}"
     end
   end
 

test/pinchflat_web/controllers/media_profile_controller_test.exs

@@ -16,7 +16,7 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
   @invalid_attrs %{name: nil, output_path_template: nil}
 
   setup do
-    Settings.set!(:onboarding, false)
+    Settings.set(onboarding: false)
 
     :ok
   end
@@ -35,7 +35,7 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
     end
 
     test "renders correct layout when onboarding", %{conn: conn} do
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = get(conn, ~p"/media_profiles/new")
 
       refute html_response(conn, 200) =~ "MENU"
@@ -59,14 +59,14 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
     end
 
     test "redirects to onboarding when onboarding", %{conn: conn} do
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = post(conn, ~p"/media_profiles", media_profile: @create_attrs)
 
       assert redirected_to(conn) == ~p"/?onboarding=1"
     end
 
     test "renders correct layout on error when onboarding", %{conn: conn} do
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = post(conn, ~p"/media_profiles", media_profile: @invalid_attrs)
 
       refute html_response(conn, 200) =~ "MENU"

test/pinchflat_web/controllers/page_controller_test.exs

@@ -10,7 +10,7 @@ defmodule PinchflatWeb.PageControllerTest do
     end
 
     test "displays the onboarding page when onboarding is forced", %{conn: conn} do
-      Settings.set!(:onboarding, false)
+      Settings.set(onboarding: false)
 
       conn = get(conn, ~p"/?onboarding=1")
       assert html_response(conn, 200) =~ "Welcome to Pinchflat"
@@ -25,7 +25,7 @@ defmodule PinchflatWeb.PageControllerTest do
     end
 
     test "displays the home page when not onboarding", %{conn: conn} do
-      Settings.set!(:onboarding, false)
+      Settings.set(onboarding: false)
 
       conn = get(conn, ~p"/")
       assert html_response(conn, 200) =~ "MENU"

test/pinchflat_web/controllers/source_controller_test.exs

@@ -8,10 +8,12 @@ defmodule PinchflatWeb.SourceControllerTest do
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup do
     media_profile = media_profile_fixture()
-    Settings.set!(:onboarding, false)
+    Settings.set(onboarding: false)
 
     {
       :ok,
@@ -45,7 +47,7 @@ defmodule PinchflatWeb.SourceControllerTest do
     end
 
     test "renders correct layout when onboarding", %{conn: conn} do
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = get(conn, ~p"/sources/new")
 
       refute html_response(conn, 200) =~ "MENU"
@@ -72,14 +74,14 @@ defmodule PinchflatWeb.SourceControllerTest do
     test "redirects to onboarding when onboarding", %{conn: conn, create_attrs: create_attrs} do
       expect(YtDlpRunnerMock, :run, 1, &runner_function_mock/3)
 
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = post(conn, ~p"/sources", source: create_attrs)
 
       assert redirected_to(conn) == ~p"/?onboarding=1"
     end
 
     test "renders correct layout on error when onboarding", %{conn: conn, invalid_attrs: invalid_attrs} do
-      Settings.set!(:onboarding, true)
+      Settings.set(onboarding: true)
       conn = post(conn, ~p"/sources", source: invalid_attrs)
 
       refute html_response(conn, 200) =~ "MENU"
@@ -160,6 +162,59 @@ defmodule PinchflatWeb.SourceControllerTest do
     end
   end
 
+  describe "force_download" do
+    test "enqueues pending download tasks", %{conn: conn} do
+      source = source_fixture()
+      _media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
+
+      assert [] = all_enqueued(worker: MediaDownloadWorker)
+      post(conn, ~p"/sources/#{source.id}/force_download")
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "redirects to the source page", %{conn: conn} do
+      source = source_fixture()
+
+      conn = post(conn, ~p"/sources/#{source.id}/force_download")
+      assert redirected_to(conn) == ~p"/sources/#{source.id}"
+    end
+  end
+
+  describe "force_index" do
+    test "forces an index", %{conn: conn} do
+      source = source_fixture()
+
+      assert [] = all_enqueued(worker: MediaCollectionIndexingWorker)
+      post(conn, ~p"/sources/#{source.id}/force_index")
+      assert [_] = all_enqueued(worker: MediaCollectionIndexingWorker)
+    end
+
+    test "forces an index even if one wouldn't normally run", %{conn: conn} do
+      source = source_fixture(index_frequency_minutes: 0, last_indexed_at: DateTime.utc_now())
+
+      post(conn, ~p"/sources/#{source.id}/force_index")
+      assert [job] = all_enqueued(worker: MediaCollectionIndexingWorker)
+      assert job.args == %{"id" => source.id, "force" => true}
+    end
+
+    test "deletes pending indexing tasks", %{conn: conn} do
+      source = source_fixture()
+      {:ok, task} = MediaCollectionIndexingWorker.kickoff_with_task(source)
+      job = Repo.preload(task, :job).job
+
+      assert job.state == "available"
+      post(conn, ~p"/sources/#{source.id}/force_index")
+      assert Repo.reload!(job).state == "cancelled"
+    end
+
+    test "redirects to the source page", %{conn: conn} do
+      source = source_fixture()
+
+      conn = post(conn, ~p"/sources/#{source.id}/force_index")
+      assert redirected_to(conn) == ~p"/sources/#{source.id}"
+    end
+  end
+
   defp create_source(_) do
     source = source_fixture()
     media_item = media_item_with_attachments(%{source_id: source.id})

test/support/conn_case.ex

@@ -21,8 +21,10 @@ defmodule PinchflatWeb.ConnCase do
     quote do
       # The default endpoint for testing
       @endpoint PinchflatWeb.Endpoint
+      alias Pinchflat.Repo
 
       use PinchflatWeb, :verified_routes
+      use Oban.Testing, repo: Repo
 
       # Import conveniences for testing with connections
       import Plug.Conn

test/support/scripts/yt-dlp-mocks/repeater.sh

@@ -12,5 +12,9 @@ for ((i = 1; i <= $#; i++)); do
   fi
 done
 
-# Write all args to the file
+if [ "${!i}" == "--print-to-file" ]; then
-echo "$@" >"$file_location"
+  # Write all args to the file
+  echo "$@" >"$file_location"
+else
+  echo "$@"
+fi