Bumped version

* Added option for yt-dlp config file usage

* renamed yt-dlp config file

* refactored to use a precedence-based approach

* Updated README

assets/js/app.js

@@ -36,17 +36,6 @@ let liveSocket = new LiveSocket('/live', Socket, {
         window.Alpine.clone(from, to)
       }
     }
-  },
-  hooks: {
-    supressEnterSubmission: {
-      mounted() {
-        this.el.addEventListener('keypress', (event) => {
-          if (event.key === 'Enter') {
-            event.preventDefault()
-          }
-        })
-      }
-    }
   }
 })
 

dev.Dockerfile

@@ -8,7 +8,7 @@ FROM ${DEV_IMAGE}
 # Install debian packages
 RUN apt-get update -qq
 RUN apt-get install -y inotify-tools ffmpeg curl git openssh-client \
-  python3 python3-pip python3-setuptools python3-wheel python3-dev locales procps
+  python3 python3-pip python3-setuptools python3-wheel python3-dev locales
 
 # Install nodejs
 RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
@@ -29,9 +29,6 @@ RUN yt-dlp -U
 # Download Apprise
 RUN python3 -m pip install -U apprise --break-system-packages
 
-# Download Mutagen for music thumbnail generation
-RUN python3 -m pip install -U mutagen --break-system-packages
-
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8

lib/pinchflat/downloading/download_option_builder.ex

@@ -3,7 +3,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
-  alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Downloading.OutputPathBuilder
@@ -31,25 +30,19 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
 
   @doc """
   Builds the output path for yt-dlp to download media based on the given source's
-  media profile. Uses the source's override output path template if it exists.
+  media profile.
 
   Returns binary()
   """
   def build_output_path_for(%Source{} = source_with_preloads) do
-    output_path_template = Sources.output_path_template(source_with_preloads)
+    output_path_template = source_with_preloads.media_profile.output_path_template
 
     build_output_path(output_path_template, source_with_preloads)
   end
 
   defp default_options do
-    [
-      :no_progress,
-      :windows_filenames,
-      # Add force-overwrites to make sure redownloading works
-      :force_overwrites,
-      # This makes the date metadata conform to what jellyfin expects
-      parse_metadata: "%(upload_date>%Y-%m-%d)s:(?P<meta_date>.+)"
-    ]
+    # Add force-overwrites to make sure redownloading works
+    [:no_progress, :windows_filenames, :force_overwrites]
   end
 
   defp subtitle_options(media_profile) do
@@ -112,18 +105,16 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   end
 
   defp quality_options(media_profile) do
-    video_codec_option = fn res ->
-      [format_sort: "res:#{res},+codec:avc:m4a", remux_video: "mp4"]
-    end
+    video_codec_options = "+codec:avc:m4a"
 
     case media_profile.preferred_resolution do
       # Also be aware that :audio disabled all embedding options for subtitles
       :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
-      :"360p" -> video_codec_option.("360")
-      :"480p" -> video_codec_option.("480")
-      :"720p" -> video_codec_option.("720")
-      :"1080p" -> video_codec_option.("1080")
-      :"2160p" -> video_codec_option.("2160")
+      :"360p" -> [format_sort: "res:360,#{video_codec_options}"]
+      :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
+      :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
+      :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
+      :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
     end
   end
 
@@ -193,7 +184,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   # It's dependent on the output_path_template being a string ending `.{{ ext }}`
   # (or equivalent), but that's validated by the MediaProfile schema.
   defp determine_thumbnail_location(media_item_with_preloads) do
-    output_path_template = Sources.output_path_template(media_item_with_preloads.source)
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
 
     output_path_template
     |> String.split(~r{\.}, include_captures: true)

lib/pinchflat/downloading/downloading_helpers.ex

@@ -7,11 +7,9 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
 
   require Logger
 
-  alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Downloading.MediaDownloadWorker
 
   @doc """
@@ -45,23 +43,4 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
     |> Media.list_pending_media_items_for()
     |> Enum.each(&Tasks.delete_pending_tasks_for/1)
   end
-
-  @doc """
-  Takes a single media item and enqueues a download job if the media should be
-  downloaded, based on the source's download settings and whether media is
-  considered pending.
-
-  Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
-  """
-  def kickoff_download_if_pending(%MediaItem{} = media_item) do
-    media_item = Repo.preload(media_item, :source)
-
-    if media_item.source.download_media && Media.pending_download?(media_item) do
-      Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
-
-      MediaDownloadWorker.kickoff_with_task(media_item)
-    else
-      {:error, :should_not_download}
-    end
-  end
 end

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -5,45 +5,64 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
   Many of these methods are made to be kickoff or be consumed by workers.
   """
 
-  require Logger
-
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaQuery
   alias Pinchflat.FastIndexing.YoutubeRss
-  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
-  Fetches new media IDs from a source's YouTube RSS feed, indexes them, and kicks off downloading
-  tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
   order of operations and how this fits into the indexing process.
 
-  Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
-  downloaded_.
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns [binary()] where each binary is the media ID of a new media item.
   """
-  def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
     {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
     existing_media_items = list_media_items_by_media_id_for(source, media_ids)
     new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
 
-    maybe_new_media_items =
-      Enum.map(new_media_ids, fn media_id ->
-        case create_media_item_from_media_id(source, media_id) do
-          {:ok, media_item} ->
-            media_item
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
 
-          err ->
-            Logger.error("Error creating media item '#{media_id}' from URL: #{inspect(err)}")
-            nil
+      MediaIndexingWorker.kickoff_with_task(source, url)
+    end)
+
+    new_media_ids
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          MediaDownloadWorker.kickoff_with_task(media_item)
         end
-      end)
 
-    DownloadingHelpers.enqueue_pending_download_tasks(source)
+        {:ok, media_item}
 
-    Enum.filter(maybe_new_media_items, & &1)
+      err ->
+        err
+    end
   end
 
   defp list_media_items_by_media_id_for(source, media_ids) do
@@ -53,15 +72,9 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
     |> Repo.all()
   end
 
-  defp create_media_item_from_media_id(source, media_id) do
-    url = "https://www.youtube.com/watch?v=#{media_id}"
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
 
-    case YtDlpMedia.get_media_attributes(url) do
-      {:ok, media_attrs} ->
-        Media.create_media_item_from_backend_attrs(source, media_attrs)
-
-      err ->
-        err
-    end
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
   end
 end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -10,7 +10,6 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
 
   alias __MODULE__
   alias Pinchflat.Tasks
-  alias Pinchflat.Media
   alias Pinchflat.Sources
   alias Pinchflat.Settings
   alias Pinchflat.Sources.Source
@@ -29,21 +28,9 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   end
 
   @doc """
-  Similar to `MediaCollectionIndexingWorker`, but for working with RSS feeds.
-  `MediaCollectionIndexingWorker` should be preferred in general, but this is
-  useful for downloading small batches of media items via fast indexing.
-
-  Only kicks off downloads for media that _should_ be downloaded
-  (ie: the source is set to download and the media matches the profile's format preferences)
-
-  Order of operations:
-    1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
-       with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
-    2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
-       it indexes them with a yt-dlp call to create the media item records then kicks off downloading
-       tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
-    3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
-       then reschedules itself to run again in the future.
+  Kicks off the fast indexing process for a source, reschedules the job to run again
+  once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
+  for more
 
   Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
   """
@@ -52,7 +39,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      perform_indexing_and_send_notification(source)
+      perform_indexing_and_notification(source)
       reschedule_indexing(source)
     else
       :ok
@@ -62,17 +49,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
     Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
-  defp perform_indexing_and_send_notification(source) do
+  defp perform_indexing_and_notification(source) do
     apprise_server = Settings.get!(:apprise_server)
+    new_media_items = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
-    new_media_items =
-      source
-      |> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
-      |> Enum.filter(&Media.pending_download?(&1))
-
-    if source.download_media do
-      SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
-    end
+    SourceNotifications.send_new_media_notification(apprise_server, source, length(new_media_items))
   end
 
   defp reschedule_indexing(source) do

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :media_indexing,
+    unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
+    tags: ["media_source", "media_indexing"]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the fast media indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, media_url, opts \\ []) do
+    %{id: source.id, media_url: media_url}
+    |> MediaIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Similar to `MediaCollectionIndexingWorker`, but for individual media items.
+  Does not reschedule or check anything to do with a source's indexing
+  frequency - only collects initial metadata then kicks off a download.
+  `MediaCollectionIndexingWorker` should be preferred in general, but this is
+  useful for downloading one-off media items based on a URL (like for fast indexing).
+
+  Only downloads media that _should_ be downloaded (ie: the source is set to download
+  and the media matches the profile's format preferences)
+
+  Order of operations:
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+       in its own worker) periodically checks the YouTube RSS feed for new media
+    2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
+       item
+    3. This worker fetches the media metadata and uses that to determine if it should be
+       downloaded. If so, it enqueues a MediaDownloadWorker
+
+  Each is a worker because they all either need to be scheduled periodically or call out to
+  an external service and will be long-running. They're split into different jobs to separate
+  retry logic for each step and allow us to better optimize various queues (eg: the indexing
+  steps can keep running while the slow download steps are worked through).
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
+    source = Sources.get_source!(source_id)
+
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
+      {:ok, media_item} ->
+        Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
+
+      {:error, reason} ->
+        Logger.debug("Failed to index and enqueue download for url: #{media_url} (reason: #{inspect(reason)})")
+    end
+
+    :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
+  end
+end

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -54,18 +54,11 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
 
   @doc """
   Downloads and stores a thumbnail for a media item, returning the filepath.
-  Chooses the highest quality jpg thumbnail available.
 
   Returns binary()
   """
   def download_and_store_thumbnail_for(database_record, metadata_map) do
-    thumbnail_url =
-      metadata_map["thumbnails"]
-      |> Enum.filter(&(&1["preference"] && String.ends_with?(&1["url"], ".jpg")))
-      |> Enum.sort(&(&1["preference"] >= &2["preference"]))
-      |> List.first()
-      |> Map.get("url")
-
+    thumbnail_url = metadata_map["thumbnail"]
     filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
     thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 

lib/pinchflat/notifications/command_runner.ex

@@ -45,7 +45,7 @@ defmodule Pinchflat.Notifications.CommandRunner do
   """
   @impl AppriseCommandRunner
   def version do
-    case CliUtils.wrap_cmd(backend_executable(), ["--version"]) do
+    case System.cmd(backend_executable(), ["--version"]) do
       {output, 0} ->
         output
         |> String.split(~r{\r?\n})

lib/pinchflat/profiles/media_profile.ex

@@ -80,8 +80,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
     |> unique_constraint(:name)
   end
 
-  @doc false
-  def ext_regex do
+  defp ext_regex do
     ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
   end
 end

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -61,8 +61,9 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
     5. If the source uses fast indexing, that job is kicked off as well. It
        uses RSS to run a smaller, faster, and more frequent index. That job
        handles rescheduling itself but largely has a similar behaviour to this
-       job in that it runs and index and maybe kicks off media download jobs.
-       Check out `FastIndexingWorker` comments for more.
+       job in that it kicks off index and maybe download jobs. The biggest difference
+       is that an index job is kicked off _for each new media item_ as opposed
+       to one larger index job. Check out `MediaIndexingWorker` comments for more.
     6. If the job reschedules, the cycle from step 3 repeats until the heat death
        of the universe. The user changing things like the index frequency can
        dequeue or reschedule jobs as well

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -16,6 +16,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   alias Pinchflat.YtDlp.MediaCollection
   alias Pinchflat.Downloading.DownloadingHelpers
   alias Pinchflat.SlowIndexing.FileFollowerServer
+  alias Pinchflat.Downloading.MediaDownloadWorker
   alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@@ -28,6 +29,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   """
   def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
     Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+    Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
     MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
@@ -125,7 +127,11 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
 
     case Media.create_media_item_from_backend_attrs(source, media_attrs) do
       {:ok, %MediaItem{} = media_item} ->
-        DownloadingHelpers.kickoff_download_if_pending(media_item)
+        if source.download_media && Media.pending_download?(media_item) do
+          Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
+
+          MediaDownloadWorker.kickoff_with_task(media_item)
+        end
 
       {:error, changeset} ->
         changeset

lib/pinchflat/sources/source.ex

@@ -32,7 +32,6 @@ defmodule Pinchflat.Sources.Source do
     retention_period_days
     title_filter_regex
     media_profile_id
-    output_path_template_override
   )a
 
   # Expensive API calls are made when a source is inserted/updated so
@@ -77,7 +76,6 @@ defmodule Pinchflat.Sources.Source do
     field :retention_period_days, :integer
     field :original_url, :string
     field :title_filter_regex, :string
-    field :output_path_template_override, :string
 
     field :series_directory, :string
     field :nfo_filepath, :string
@@ -111,8 +109,6 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
     |> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
-    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
-    |> validate_format(:output_path_template_override, MediaProfile.ext_regex(), message: "must end with .{{ ext }}")
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
   end

lib/pinchflat/sources/sources.ex

@@ -19,19 +19,6 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
-  @doc """
-  Returns the relevant output path template for a source.
-  Pulls from the source's override if present, otherwise uses the media profile's.
-
-  Returns binary()
-  """
-  def output_path_template(source) do
-    source = Repo.preload(source, :media_profile)
-    media_profile = source.media_profile
-
-    source.output_path_template_override || media_profile.output_path_template
-  end
-
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
   """
@@ -288,6 +275,7 @@ defmodule Pinchflat.Sources do
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+        Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
         Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
       _ ->

lib/pinchflat/utils/cli_utils.ex

@@ -5,23 +5,6 @@ defmodule Pinchflat.Utils.CliUtils do
 
   alias Pinchflat.Utils.StringUtils
 
-  @doc """
-  Wraps a command in a shell script that will terminate
-  the command if stdin is closed. Useful for stopping
-  commands if the job runner is cancelled.
-
-  Delegates to `System.cmd/3` and any options/output
-  are passed through.
-
-  Returns {binary(), integer()}
-  """
-  def wrap_cmd(command, args, opts \\ []) do
-    wrapper_command = Path.join(:code.priv_dir(:pinchflat), "cmd_wrapper.sh")
-    actual_command = [command] ++ args
-
-    System.cmd(wrapper_command, actual_command, opts)
-  end
-
   @doc """
   Parses a list of command options into a list of strings suitable for passing to
   `System.cmd/3`.

lib/pinchflat/yt_dlp/command_runner.ex

@@ -37,7 +37,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts)
     Logger.info("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
 
-    case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do
+    case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
         # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
         # a huge deal, but it's still a good idea to clean up after ourselves.

lib/pinchflat_web/components/layouts/partials/upgrade_button_live.ex

@@ -3,11 +3,10 @@ defmodule Pinchflat.UpgradeButtonLive do
 
   def render(assigns) do
     ~H"""
-    <form id="upgradeForm" phx-change="check_matching_text" phx-hook="supressEnterSubmission">
+    <form phx-change="check_matching_text">
       <.input type="text" name="unlock-pro-textbox" value="" />
     </form>
 
-    <%!-- The setTimeout is so the modal has time to disappear before it's removed --%>
     <.button
       class="w-full mt-4"
       type="button"

lib/pinchflat_web/controllers/media_items/media_item_controller.ex

@@ -50,13 +50,7 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
 
   def force_download(conn, %{"media_item_id" => id}) do
     media_item = Media.get_media_item!(id)
-
-    :ok =
-      case MediaDownloadWorker.kickoff_with_task(media_item, %{force: true}) do
-        {:ok, _} -> :ok
-        {:error, :duplicate_job} -> :ok
-        err -> err
-      end
+    {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item, %{force: true})
 
     conn
     |> put_flash(:info, "Download task enqueued.")

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -28,20 +28,4 @@ defmodule PinchflatWeb.Sources.SourceHTML do
   def rss_feed_url(conn, source) do
     url(conn, ~p"/sources/#{source.uuid}/feed") <> ".xml"
   end
-
-  def output_path_template_override_placeholders(media_profiles) do
-    media_profiles
-    |> Enum.map(&{&1.id, &1.output_path_template})
-    |> Map.new()
-    |> Phoenix.json_library().encode!()
-  end
-
-  def output_path_template_override_help do
-    help_button_classes = "underline decoration-bodydark decoration-1 hover:decoration-white cursor-pointer"
-    help_button = ~s{<span class="#{help_button_classes}" x-on:click="$dispatch('load-template')">Click here</span>}
-
-    """
-    Must end with .{{ ext }}. Same rules as Media Profile output path templates. #{help_button} to load your media profile's output template
-    """
-  end
 end

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -9,139 +9,113 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <section x-data="{ mediaProfileId: null }">
-    <section class="flex justify-between items-center mt-8">
-      <h3 class=" text-2xl text-black dark:text-white">
-        General Options
-      </h3>
-      <span class="cursor-pointer hover:underline" x-on:click="advancedMode = !advancedMode">
-        Editing Mode: <span x-text="advancedMode ? 'Advanced' : 'Basic'"></span>
-      </span>
-    </section>
-
-    <.input
-      field={f[:custom_name]}
-      type="text"
-      label="Custom Name"
-      help="Something descriptive. Does not impact indexing or downloading"
-    />
-
-    <.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
-
-    <.input
-      field={f[:media_profile_id]}
-      options={Enum.map(@media_profiles, &{&1.name, &1.id})}
-      type="select"
-      label="Media Profile"
-      help="Sets your preferences for what media to look for and how to store it"
-      x-model.fill="mediaProfileId"
-    />
-
-    <h3 class="mt-8 text-2xl text-black dark:text-white">
-      Indexing Options
+  <section class="flex justify-between items-center mt-8">
+    <h3 class=" text-2xl text-black dark:text-white">
+      General Options
     </h3>
+    <span class="cursor-pointer hover:underline" x-on:click="advancedMode = !advancedMode">
+      Editing Mode: <span x-text="advancedMode ? 'Advanced' : 'Basic'"></span>
+    </span>
+  </section>
 
-    <section x-data="{ fastIndexingEnabled: null }">
+  <.input
+    field={f[:custom_name]}
+    type="text"
+    label="Custom Name"
+    help="Something descriptive. Does not impact indexing or downloading"
+  />
+
+  <.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
+
+  <.input
+    field={f[:media_profile_id]}
+    options={Enum.map(@media_profiles, &{&1.name, &1.id})}
+    type="select"
+    label="Media Profile"
+    help="Sets your preferences for what media to look for and how to store it"
+  />
+
+  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    Indexing Options
+  </h3>
+
+  <section x-data="{ fastIndexingEnabled: null }">
+    <.input
+      field={f[:index_frequency_minutes]}
+      options={friendly_index_frequencies()}
+      type="select"
+      label="Index Frequency"
+      x-bind:disabled="fastIndexingEnabled == true"
+      x-init="$watch('fastIndexingEnabled', v => v && ($el.value = 30 * 24 * 60))"
+      help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
+    />
+
+    <div phx-click={show_modal("upgrade-modal")}>
       <.input
-        field={f[:index_frequency_minutes]}
-        options={friendly_index_frequencies()}
-        type="select"
-        label="Index Frequency"
-        x-bind:disabled="fastIndexingEnabled == true"
-        x-init="$watch('fastIndexingEnabled', v => v && ($el.value = 30 * 24 * 60))"
-        help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
-      />
-
-      <div phx-click={show_modal("upgrade-modal")}>
-        <.input
-          field={f[:fast_index]}
-          type="toggle"
-          label="Use Fast Indexing"
-          label_suffix="(pro)"
-          help="Experimental. Overrides 'Index Frequency'. Recommended for large channels that upload frequently. Does not work with private playlists. See below for more info"
-          x-init="
+        field={f[:fast_index]}
+        type="toggle"
+        label="Use Fast Indexing"
+        label_suffix="(pro)"
+        help="Experimental. Overrides 'Index Frequency'. Recommended for large channels that upload frequently. Does not work with private playlists. See below for more info"
+        x-init="
           // `enabled` is the data attribute that the toggle uses internally
           fastIndexingEnabled = enabled
           $watch('enabled', value => fastIndexingEnabled = !!value)
         "
-        />
-      </div>
-    </section>
-
-    <h3 class="mt-8 text-2xl text-black dark:text-white">
-      Downloading Options
-    </h3>
-
-    <.input
-      field={f[:download_media]}
-      type="toggle"
-      label="Download Media"
-      help="Unchecking still indexes media but it won't be downloaded until you enable this option"
-    />
-
-    <.input
-      field={f[:download_cutoff_date]}
-      type="text"
-      label="Download Cutoff Date"
-      placeholder="YYYY-MM-DD"
-      maxlength="10"
-      pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
-      title="YYYY-MM-DD"
-      help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
-    />
-
-    <.input
-      field={f[:retention_period_days]}
-      type="number"
-      label="Retention Period (days)"
-      min="0"
-      help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
-    />
-
-    <section x-show="advancedMode">
-      <h3 class="mt-8 text-2xl text-black dark:text-white">
-        Advanced Options
-      </h3>
-      <p class="text-sm mt-2">
-        Tread carefully
-      </p>
-
-      <.input
-        field={f[:title_filter_regex]}
-        type="text"
-        label="Title Filter Regex"
-        placeholder="(?i)^How to Bike$"
-        help="A PCRE-compatible regex. Only media with titles that match this regex will be downloaded. Look up 'SQLean Regex docs' for more"
       />
-
-      <section
-        x-data={
-          """
-          { 
-            placeholders: JSON.parse('#{output_path_template_override_placeholders(@media_profiles)}'),
-            inputValue: null
-          }
-          """
-        }
-        x-on:load-template="inputValue = placeholders[mediaProfileId]"
-      >
-        <.input
-          field={f[:output_path_template_override]}
-          type="text"
-          inputclass="font-mono"
-          label="Output path template override"
-          help={output_path_template_override_help()}
-          html_help={true}
-          x-bind:placeholder="placeholders[mediaProfileId]"
-          x-model.fill="inputValue"
-        />
-      </section>
-    </section>
-
-    <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Source</.button>
-
-    <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
-      <.fast_indexing_help />
     </div>
   </section>
+
+  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    Downloading Options
+  </h3>
+
+  <.input
+    field={f[:download_media]}
+    type="toggle"
+    label="Download Media"
+    help="Unchecking still indexes media but it won't be downloaded until you enable this option"
+  />
+
+  <.input
+    field={f[:download_cutoff_date]}
+    type="text"
+    label="Download Cutoff Date"
+    placeholder="YYYY-MM-DD"
+    maxlength="10"
+    pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
+    title="YYYY-MM-DD"
+    help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
+  />
+
+  <.input
+    field={f[:retention_period_days]}
+    type="number"
+    label="Retention Period (days)"
+    min="0"
+    help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
+  />
+
+  <section x-show="advancedMode">
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
+      Advanced Options
+    </h3>
+    <p class="text-sm mt-2">
+      Tread carefully
+    </p>
+
+    <.input
+      field={f[:title_filter_regex]}
+      type="text"
+      label="Title Filter Regex"
+      placeholder="(?i)^How to Bike$"
+      help="A PCRE-compatible regex. Only media with titles that match this regex will be downloaded. Look up 'SQLean Regex docs' for more"
+    />
+  </section>
+
+  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Source</.button>
+
+  <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
+    <.fast_indexing_help />
+  </div>
 </.simple_form>

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.14",
+      version: "0.1.12",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

priv/cmd_wrapper.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is a wrapper for other programs
-# that ensures they are killed when stdin closes
-# (eg: a job terminates)
-
-# Start the program in the background
-exec "$@" &
-pid1=$!
-
-# Silence warnings from here on
-exec >/dev/null 2>&1
-
-# Read from stdin in the background and
-# kill running program when stdin closes
-exec 0<&0 $(
-  while read; do :; done
-  kill -KILL $pid1
-) &
-pid2=$!
-
-# Clean up
-wait $pid1
-ret=$?
-kill -KILL $pid2
-exit $ret

priv/repo/migrations/20240411154245_add_output_template_to_sources.exs

@@ -1,9 +0,0 @@
-defmodule Pinchflat.Repo.Migrations.AddOutputTemplateToSources do
-  use Ecto.Migration
-
-  def change do
-    alter table(:sources) do
-      add :output_path_template_override, :string
-    end
-  end
-end

selfhosted.Dockerfile

@@ -90,9 +90,6 @@ RUN yt-dlp -U
 # Download Apprise
 RUN python3 -m pip install -U apprise --break-system-packages
 
-# Download Mutagen for music thumbnail generation
-RUN python3 -m pip install -U mutagen --break-system-packages
-
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8

test/pinchflat/downloading/download_option_builder_test.exs

@@ -4,7 +4,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   import Pinchflat.SourcesFixtures
   import Pinchflat.ProfilesFixtures
 
-  alias Pinchflat.Sources
   alias Pinchflat.Profiles
   alias Pinchflat.Utils.FilesystemUtils
   alias Pinchflat.Downloading.DownloadOptionBuilder
@@ -32,20 +31,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert {:output, "/tmp/test/media/#{media_item.source.custom_name}.%(ext)s"} in res
     end
-
-    test "uses source's output override if present", %{media_item: media_item} do
-      source = media_item.source
-      {:ok, _} = Sources.update_source(source, %{output_path_template_override: "override.%(ext)s"})
-
-      media_item =
-        media_item
-        |> Repo.reload()
-        |> Repo.preload(source: :media_profile)
-
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
-
-      assert {:output, "/tmp/test/media/override.%(ext)s"} in res
-    end
   end
 
   describe "build/1 when testing default options" do
@@ -55,7 +40,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
       assert :no_progress in res
       assert :windows_filenames in res
       assert :force_overwrites in res
-      assert {:parse_metadata, "%(upload_date>%Y-%m-%d)s:(?P<meta_date>.+)"} in res
     end
   end
 
@@ -151,20 +135,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
       assert {:output, "thumbnail:/tmp/test/media/%(title)S-thumb.%(ext)s"} in res
     end
 
-    test "appends -thumb to source's output path override, if present", %{media_item: media_item} do
-      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
-      {:ok, _} = Sources.update_source(media_item.source, %{output_path_template_override: "override.%(ext)s"})
-
-      media_item =
-        media_item
-        |> Repo.reload()
-        |> Repo.preload(source: :media_profile)
-
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
-
-      assert {:output, "thumbnail:/tmp/test/media/override-thumb.%(ext)s"} in res
-    end
-
     test "converts thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
 
@@ -241,7 +211,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
         assert {:ok, res} = DownloadOptionBuilder.build(media_item)
         assert {:format_sort, "res:#{resolution},+codec:avc:m4a"} in res
-        assert {:remux_video, "mp4"} in res
       end)
     end
 
@@ -252,8 +221,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert :extract_audio in res
       assert {:format, "bestaudio[ext=m4a]"} in res
-
-      refute {:remux_video, "mp4"} in res
     end
   end
 
@@ -392,15 +359,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert path == "/tmp/test/media/%(title)S.%(ext)s"
     end
-
-    test "uses source's output override if present", %{media_item: media_item} do
-      source = media_item.source
-      {:ok, source} = Sources.update_source(source, %{output_path_template_override: "override.%(ext)s"})
-
-      path = DownloadOptionBuilder.build_output_path_for(source)
-
-      assert path == "/tmp/test/media/override.%(ext)s"
-    end
   end
 
   defp update_media_profile_attribute(media_item_with_preloads, attrs) do

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -4,7 +4,6 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
   import Mox
   import Pinchflat.MediaFixtures
   import Pinchflat.SourcesFixtures
-  import Pinchflat.ProfilesFixtures
 
   alias Pinchflat.Tasks
   alias Pinchflat.Downloading.DownloadingHelpers
@@ -73,44 +72,4 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       assert [] = Tasks.list_tasks_for(media_item)
     end
   end
-
-  describe "kickoff_download_if_pending/1" do
-    setup do
-      media_item = media_item_fixture(media_filepath: nil)
-
-      {:ok, media_item: media_item}
-    end
-
-    test "enqueues a download job", %{media_item: media_item} do
-      assert {:ok, _} = DownloadingHelpers.kickoff_download_if_pending(media_item)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "creates and returns a download task record", %{media_item: media_item} do
-      assert {:ok, task} = DownloadingHelpers.kickoff_download_if_pending(media_item)
-
-      assert [found_task] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
-      assert task.id == found_task.id
-    end
-
-    test "does not enqueue a download job if the source does not allow it" do
-      source = source_fixture(%{download_media: false})
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "does not enqueue a download job if the media item does not match the format rules" do
-      profile = media_profile_fixture(%{livestream_behaviour: :exclude})
-      source = source_fixture(%{media_profile_id: profile.id})
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil, livestream: true)
-
-      assert {:error, :should_not_download} = DownloadingHelpers.kickoff_download_if_pending(media_item)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-  end
 end

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -9,11 +9,45 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
   alias Pinchflat.Tasks
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
   alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   setup :verify_on_exit!
 
-  describe "kickoff_download_tasks_from_youtube_rss_feed/1" do
+  @media_url "https://www.youtube.com/watch?v=test_1"
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert [_] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert [] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+
+    test "returns the IDs of the found media items", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert ["test_1"] = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+    end
+  end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
     setup do
       stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
         {:ok, media_attributes_return_fixture()}
@@ -22,50 +56,41 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
       {:ok, [source: source_fixture()]}
     end
 
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
-
-      assert [worker] = all_enqueued(worker: MediaDownloadWorker)
-      assert worker.args["id"] == media_item.id
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
     end
 
-    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      media_item_fixture(source_id: source.id, media_id: "test_1")
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
 
-      assert [] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
+      assert Repo.aggregate(MediaItem, :count) == 1
+      assert mi_1.id == mi_2.id
     end
 
-    test "returns the found media items", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
-    end
-
-    test "does not enqueue a download job if the source does not allow it" do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      source = source_fixture(%{download_media: false})
-
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
     end
 
     test "creates a download task record", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert [media_item] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
 
       assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
     end
 
-    test "does not enqueue a download job if the media item does not match the format rules" do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
 
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
       profile = media_profile_fixture(%{shorts_behaviour: :exclude})
       source = source_fixture(%{media_profile_id: profile.id})
 
@@ -85,8 +110,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
         {:ok, output}
       end)
 
-      assert [%MediaItem{}] = FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source)
-
+      assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
       refute_enqueued(worker: MediaDownloadWorker)
     end
   end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -87,7 +87,6 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
       source = source_fixture(fast_index: true)
 
       expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
 
       expect(AppriseRunnerMock, :run, fn servers, opts ->
         assert "server_1" = servers
@@ -99,34 +98,5 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
 
       perform_job(FastIndexingWorker, %{id: source.id})
     end
-
-    test "doesn't send a notification if new media is not found" do
-      source = source_fixture(fast_index: true)
-
-      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
-      expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
-
-      perform_job(FastIndexingWorker, %{id: source.id})
-    end
-
-    test "doesn't send a notification if the source doesn't download media" do
-      source = source_fixture(fast_index: true, download_media: false)
-
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
-      expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
-
-      perform_job(FastIndexingWorker, %{id: source.id})
-    end
-
-    test "doesn't send a notification if the media isn't pending download" do
-      source = source_fixture(fast_index: true, title_filter_regex: "foobar")
-
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, render_metadata(:media_metadata)} end)
-      expect(AppriseRunnerMock, :run, 0, fn _servers, _opts -> {:ok, ""} end)
-
-      perform_job(FastIndexingWorker, %{id: source.id})
-    end
   end
 end

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -0,0 +1,61 @@
+defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  @media_url "https://www.youtube.com/watch?v=1234567890"
+
+  setup :verify_on_exit!
+
+  setup do
+    source = source_fixture()
+
+    {:ok, source: source}
+  end
+
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: MediaIndexingWorker)
+      assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert [_] = all_enqueued(worker: MediaIndexingWorker)
+    end
+
+    test "attaches a task", %{source: source} do
+      assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "indexes the media item and saves it to the database", %{source: source} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      before = Repo.aggregate(MediaItem, :count, :id)
+      perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
+
+      assert Repo.aggregate(MediaItem, :count, :id) == before + 1
+    end
+
+    test "enqueues a download job for the media item", %{source: source} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
+
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
+    end
+  end
+end

test/pinchflat/media_test.exs

@@ -731,7 +731,9 @@ defmodule Pinchflat.MediaTest do
         metadata: %{
           metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
           thumbnail_filepath:
-            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, render_parsed_metadata(:media_metadata))
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
         }
       }
 
@@ -758,7 +760,9 @@ defmodule Pinchflat.MediaTest do
         metadata: %{
           metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
           thumbnail_filepath:
-            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, render_parsed_metadata(:media_metadata))
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
         }
       }
 
@@ -827,7 +831,9 @@ defmodule Pinchflat.MediaTest do
         metadata: %{
           metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
           thumbnail_filepath:
-            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, render_parsed_metadata(:media_metadata))
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
         }
       }
 

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -54,11 +54,13 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
   describe "download_and_store_thumbnail_for/2" do
     setup do
       # This tests that the HTTP endpoint is being called with every test
-      expect(HTTPClientMock, :get, fn _url, _headers, _opts ->
+      expect(HTTPClientMock, :get, fn url, _headers, _opts ->
+        assert url =~ "example.com"
+
         {:ok, "thumbnail data"}
       end)
 
-      metadata = render_parsed_metadata(:media_metadata)
+      metadata = %{"thumbnail" => "example.com/thumbnail.jpg"}
 
       {:ok, %{metadata: metadata}}
     end
@@ -66,7 +68,7 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
     test "returns the filepath", %{media_item: media_item, metadata: metadata} do
       filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
 
-      assert filepath =~ ~r{/media_items/#{media_item.id}/maxresdefault.jpg}
+      assert filepath =~ ~r{/media_items/#{media_item.id}/thumbnail.jpg}
     end
 
     test "creates folder structure based on passed record", %{media_item: media_item, metadata: metadata} do
@@ -75,19 +77,11 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
       assert File.exists?(Path.dirname(filepath))
     end
 
-    test "chooses the highest preference jpg thumbnail available", %{media_item: media_item} do
-      metadata = %{
-        "thumbnails" => [
-          %{"url" => "https://i.ytimg.com/vi/ABC123/img_1.jpg", "preference" => -1},
-          %{"url" => "https://i.ytimg.com/vi/ABC123/img_2.jpg", "preference" => 1},
-          %{"url" => "https://i.ytimg.com/vi/ABC123/img_3.jpg", "preference" => -10},
-          %{"url" => "https://i.ytimg.com/vi/ABC123/img_4.webp", "preference" => 10}
-        ]
-      }
-
+    test "the filename and extension is based on the URL", %{media_item: media_item} do
+      metadata = %{"thumbnail" => "example.com/maxres.webp"}
       filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
 
-      assert filepath =~ ~r{/media_items/#{media_item.id}/img_2.jpg}
+      assert Path.basename(filepath) == "maxres.webp"
     end
   end
 

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -12,6 +12,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
   alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
@@ -46,7 +47,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
 
     test "it deletes any pending media tasks for the source" do
       source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
       assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)

test/pinchflat/sources_test.exs

@@ -13,6 +13,7 @@ defmodule Pinchflat.SourcesTest do
   alias Pinchflat.Downloading.DownloadingHelpers
   alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
   alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
@@ -34,28 +35,6 @@ defmodule Pinchflat.SourcesTest do
     end
   end
 
-  describe "output_path_template/1" do
-    test "returns the source's override if present" do
-      source = source_fixture(%{output_path_template_override: "/override/{{ title }}.{{ ext }}"})
-
-      assert Sources.output_path_template(source) == "/override/{{ title }}.{{ ext }}"
-    end
-
-    test "returns the media profile's template if no override is present" do
-      media_profile = media_profile_fixture(%{output_path_template: "/profile/{{ title }}.{{ ext }}"})
-      source = source_fixture(%{media_profile_id: media_profile.id})
-
-      assert Sources.output_path_template(source) == "/profile/{{ title }}.{{ ext }}"
-    end
-
-    test "Treats empty strings as being blank" do
-      media_profile = media_profile_fixture(%{output_path_template: "/profile/{{ title }}.{{ ext }}"})
-      source = source_fixture(%{media_profile_id: media_profile.id, output_path_template_override: "  "})
-
-      assert Sources.output_path_template(source) == "/profile/{{ title }}.{{ ext }}"
-    end
-  end
-
   describe "list_sources/0" do
     test "it returns all sources" do
       source = source_fixture()
@@ -414,13 +393,16 @@ defmodule Pinchflat.SourcesTest do
 
       {:ok, job_1} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
       task_1 = task_fixture(source_id: source.id, job_id: job_1.id)
-      {:ok, job_2} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
+      {:ok, job_2} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task_2 = task_fixture(source_id: source.id, job_id: job_2.id)
+      {:ok, job_3} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
+      task_3 = task_fixture(source_id: source.id, job_id: job_3.id)
 
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_1) end
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_2) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_3) end
     end
 
     test "not updating the index frequency will not re-schedule the indexing task or delete tasks" do

test/pinchflat/utils/cli_utils_test.exs

@@ -3,12 +3,6 @@ defmodule Pinchflat.Utils.CliUtilsTest do
 
   alias Pinchflat.Utils.CliUtils
 
-  describe "wrap_cmd/3" do
-    test "delegates to System.cmd/3" do
-      assert {"output\n", 0} = CliUtils.wrap_cmd("echo", ["output"])
-    end
-  end
-
   describe "parse_options/1" do
     test "it converts symbol k-v arg keys to kebab case" do
       assert ["--buffer-size", "1024"] = CliUtils.parse_options(buffer_size: 1024)

test/pinchflat_web/controllers/media_item_controller_test.exs

@@ -97,15 +97,6 @@ defmodule PinchflatWeb.MediaItemControllerTest do
       assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
 
-    test "doesn't freak out if the task is a duplicate", %{conn: conn} do
-      media_item = media_item_fixture()
-
-      MediaDownloadWorker.kickoff_with_task(media_item, %{force: true})
-
-      post(conn, ~p"/sources/#{media_item.source_id}/media/#{media_item.id}/force_download")
-      assert [_] = all_enqueued(worker: MediaDownloadWorker)
-    end
-
     test "forces a download even if one wouldn't normally run", %{conn: conn} do
       media_item = media_item_fixture(%{media_filepath: nil})