Bumped version

* Fixed TV show output template

* Discard jobs if record not found; hopefully addressed timing-related source deletion bug

.check.exs

@@ -9,18 +9,15 @@
   fix: true,
 
   ## don't retry automatically even if last run resulted in failures
-  # retry: false,
+  retry: false,
 
   ## list of tools (see `mix check` docs for a list of default curated tools)
   tools: [
     {:compiler, env: %{"MIX_ENV" => "test"}},
     {:formatter, env: %{"MIX_ENV" => "test"}},
     {:sobelow, "mix sobelow --config"},
-    # TODO: delete these and replace them with builtin ex_unit and formatter tools
+    {:prettier_formatting, "yarn run prettier . --check", fix: "yarn run prettier . --write"},
-    # once Elixir 1.16.2 is released (see: https://github.com/karolsluszniak/ex_check/issues/41#issuecomment-1921390413)
+    {:npm_test, false}
-    {:elixir_tests, "mix test"},
-    {:elixir_formatting, "mix format --check-formatted", fix: "mix format"},
-    {:prettier_formatting, "yarn run prettier . --check", fix: "yarn run prettier . --write"}
 
     ## curated tools may be disabled (e.g. the check for compilation warnings)
     # {:compiler, false},

.credo.exs

@@ -128,7 +128,7 @@
           {Credo.Check.Refactor.MatchInCondition, []},
           {Credo.Check.Refactor.NegatedConditionsInUnless, []},
           {Credo.Check.Refactor.NegatedConditionsWithElse, []},
-          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.Nesting, [max_nesting: 4]},
           {Credo.Check.Refactor.RedundantWithClauseResult, []},
           {Credo.Check.Refactor.RejectReject, []},
           {Credo.Check.Refactor.UnlessWithElse, []},
@@ -157,7 +157,16 @@
           {Credo.Check.Warning.UnusedRegexOperation, []},
           {Credo.Check.Warning.UnusedStringOperation, []},
           {Credo.Check.Warning.UnusedTupleOperation, []},
-          {Credo.Check.Warning.WrongTestFileExtension, []}
+          {Credo.Check.Warning.WrongTestFileExtension, []},
+
+          #
+          ## Naming Checks
+          #
+          {CredoNaming.Check.Consistency.ModuleFilename,
+           [
+             priority: :normal,
+             excluded_paths: [~r/test\/support/, ~r/priv/, ~r/lib\/pinchflat_web/, ~r/test\/pinchflat_web/]
+           ]}
         ],
         disabled: [
           #

.iex.exs

@@ -4,8 +4,7 @@ alias Pinchflat.Repo
 alias Pinchflat.Tasks.Task
 alias Pinchflat.Sources.Source
 alias Pinchflat.Media.MediaItem
-alias Pinchflat.Tasks.SourceTasks
+alias Pinchflat.Metadata.MediaMetadata
-alias Pinchflat.Media.MediaMetadata
 alias Pinchflat.Profiles.MediaProfile
 
 alias Pinchflat.Tasks
@@ -14,14 +13,14 @@ alias Pinchflat.Profiles
 alias Pinchflat.Sources
 alias Pinchflat.Settings
 
-alias Pinchflat.MediaClient.MediaDownloader
+alias Pinchflat.Downloading.MediaDownloader
-alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
+alias Pinchflat.YtDlp.MediaCollection, as: YtDlpCollection
 
-alias Pinchflat.Api.YoutubeRss
+alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
-alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+alias Pinchflat.SlowIndexing.FileFollowerServer
 
 defmodule IexHelpers do
   def playlist_url do

config/config.exs

@@ -12,7 +12,7 @@ config :pinchflat,
   generators: [timestamp_type: :utc_datetime],
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
-  yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
+  yt_dlp_runner: Pinchflat.YtDlp.CommandRunner,
   media_directory: "/downloads",
   # The user may or may not store metadata for their needs, but the app will always store its copy
   metadata_directory: "/config/metadata",
@@ -48,7 +48,8 @@ config :pinchflat, Oban,
     media_indexing: 2,
     media_collection_indexing: 2,
     media_fetching: 2,
-    media_local_metadata: 8
+    local_metadata: 8,
+    remote_metadata: 4
   ]
 
 # Configures the mailer

dev.Dockerfile

@@ -1,9 +1,18 @@
-# Extend from the official Elixir image.
+ARG ELIXIR_VERSION=1.16.2
-FROM elixir:latest
+ARG OTP_VERSION=26.2.2
+ARG DEBIAN_VERSION=bookworm-20240130
+ARG DEV_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
+
+FROM ${DEV_IMAGE}
+
+# Set the locale deets
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 # Install debian packages
 RUN apt-get update -qq
-RUN apt-get install -y inotify-tools ffmpeg \
+RUN apt-get install -y inotify-tools ffmpeg curl git openssh-client \
   python3 python3-pip python3-setuptools python3-wheel python3-dev
 
 # Install nodejs
@@ -28,6 +37,7 @@ COPY . ./
 RUN chmod +x ./docker-run.dev.sh
 
 # Install Elixir deps
+# RUN mix archive.install github hexpm/hex branch latest
 RUN mix deps.get
 # Gives us iex shell history
 ENV ERL_AFLAGS="-kernel shell_history enabled"

ideas.md

@@ -2,5 +2,3 @@
   - Use a UUID for the media database ID (or at least alongside it)
 - Look into this and its recommended plugins https://hexdocs.pm/ex_check/readme.html
 - Add output template option for the source's friendly name
-- TODO: Install Elixir 1.16.2 when available to fix bug with `ex_check` https://github.com/karolsluszniak/ex_check/issues/41#issuecomment-1921390413
-  - delete `{:elixir_tests, "mix test"}` and formatting check

lib/pinchflat/application.ex

@@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
       # Must be before startup tasks
+      Pinchflat.Boot.PreJobStartupTasks,
       {Oban, Application.fetch_env!(:pinchflat, Oban)},
-      Pinchflat.StartupTasks,
+      Pinchflat.Boot.PostJobStartupTasks,
       {DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: Pinchflat.PubSub},
       # Start the Finch HTTP client for sending emails

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Workers.DataBackfillWorker do
+defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
-    queue: :media_local_metadata,
+    queue: :local_metadata,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
 
@@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
   """
   def cancel_pending_backfill_jobs do
     Oban.Job
-    |> where(worker: "Pinchflat.Workers.DataBackfillWorker")
+    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
     |> Oban.cancel_all_jobs()
   end
 

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,6 +1,7 @@
-defmodule Pinchflat.StartupTasks do
+defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
-  This module is responsible for running startup tasks on app boot.
+  This module is responsible for running startup tasks on app boot
+  AFTER the job runner has initiallized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Settings
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.DataBackfillWorker
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
   """
   @impl true
   def init(state) do
-    apply_default_settings()
     enqueue_backfill_worker()
 
     {:ok, state}
   end
 
-  defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
-    Settings.fetch!(:pro_enabled, false)
-  end
-
   defp enqueue_backfill_worker do
     DataBackfillWorker.cancel_pending_backfill_jobs()
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -0,0 +1,89 @@
+defmodule Pinchflat.Boot.PreJobStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  BEFORE the job runner has initiallized.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Settings
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{}, opts)
+  end
+
+  @doc """
+  Runs application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(state) do
+    apply_default_settings()
+    ensure_directories_are_writeable()
+    rename_old_job_workers()
+
+    {:ok, state}
+  end
+
+  defp apply_default_settings do
+    Settings.fetch!(:onboarding, true)
+    Settings.fetch!(:pro_enabled, false)
+  end
+
+  defp ensure_directories_are_writeable do
+    directories = [
+      Application.get_env(:pinchflat, :media_directory),
+      Application.get_env(:pinchflat, :tmpfile_directory),
+      Application.get_env(:pinchflat, :metadata_directory)
+    ]
+
+    Enum.each(directories, fn dir ->
+      file = Path.join([dir, ".keep"])
+
+      # This will fail if the directory is not writeable, stopping boot
+      FilesystemHelpers.write_p!(file, "")
+    end)
+  end
+
+  # As part of a large refactor, I ended up moving a bunch of workers around. This
+  # is a problem because the workers are stored in the database and the runner
+  # will try to run the OLD jobs. This is also why these tasks run before the job
+  # runner starts up.
+  #
+  # Can be removed after a few months (created: 2024-03-12)
+  defp rename_old_job_workers do
+    # [ [old_name, new_name], ...]
+    rename_map = [
+      ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
+      ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
+      ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
+      ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
+      ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
+    ]
+
+    jobs_renamed =
+      Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
+        {count, _} =
+          Oban.Job
+          |> where(worker: ^old_name)
+          |> Repo.update_all(set: [worker: new_name])
+
+        acc + count
+      end)
+
+    Logger.info("Renamed #{jobs_renamed} old job workers")
+  end
+end

lib/pinchflat/downloading/download_option_builder.ex

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   @doc """
   Builds the options for yt-dlp to download media based on the given media's profile.
@@ -18,7 +18,7 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
-        thumbnail_options(media_profile) ++
+        thumbnail_options(media_item_with_preloads) ++
         metadata_options(media_profile) ++
         quality_options(media_profile) ++
         output_options(media_item_with_preloads)
@@ -57,16 +57,19 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     end)
   end
 
-  defp thumbnail_options(media_profile) do
+  defp thumbnail_options(media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
     mapped_struct = Map.from_struct(media_profile)
 
     Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
+      case attr do
-        {{:download_thumbnail, true}, _} ->
+        {:download_thumbnail, true} ->
-          acc ++ [:write_thumbnail]
+          thumbnail_save_location = determine_thumbnail_location(media_item_with_preloads)
 
-        {{:embed_thumbnail, true}, %{preferred_resolution: pr}} when pr != :audio ->
+          acc ++ [:write_thumbnail, convert_thumbnail: "jpg", output: "thumbnail:#{thumbnail_save_location}"]
-          acc ++ [:embed_thumbnail]
+
+        {:embed_thumbnail, true} ->
+          acc ++ [:embed_thumbnail, convert_thumbnail: "jpg"]
 
         _ ->
           acc
@@ -78,44 +81,43 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     mapped_struct = Map.from_struct(media_profile)
 
     Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
+      case attr do
-        {{:download_metadata, true}, _} ->
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
-          acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
-
+        _ -> acc
-        {{:embed_metadata, true}, %{preferred_resolution: pr}} when pr != :audio ->
-          acc ++ [:embed_metadata]
-
-        _ ->
-          acc
       end
     end)
   end
 
   defp quality_options(media_profile) do
-    codec_options = "+codec:avc:m4a"
+    video_codec_options = "+codec:avc:m4a"
 
     case media_profile.preferred_resolution do
-      # Also be aware that :audio disabled all embedding options for thumbnails, subtitles, and metadata
+      # Also be aware that :audio disabled all embedding options for subtitles
-      :audio -> [format_sort: "ext", format: "bestaudio"]
+      :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
-      :"360p" -> [format_sort: "res:360,#{codec_options}"]
+      :"360p" -> [format_sort: "res:360,#{video_codec_options}"]
-      :"480p" -> [format_sort: "res:480,#{codec_options}"]
+      :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
-      :"720p" -> [format_sort: "res:720,#{codec_options}"]
+      :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
-      :"1080p" -> [format_sort: "res:1080,#{codec_options}"]
+      :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
-      :"1440p" -> [format_sort: "res:1440,#{codec_options}"]
+      :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
-      :"2160p" -> [format_sort: "res:2160,#{codec_options}"]
     end
   end
 
   defp output_options(media_item_with_preloads) do
-    media_profile = media_item_with_preloads.source.media_profile
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
-    additional_options_map = output_options_map(media_item_with_preloads)
-    {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
 
     [
-      output: Path.join(base_directory(), output_path)
+      output: build_output_path(output_path_template, media_item_with_preloads)
     ]
   end
 
+  defp build_output_path(string, media_item_with_preloads) do
+    additional_options_map = output_options_map(media_item_with_preloads)
+    {:ok, output_path} = OutputPathBuilder.build(string, additional_options_map)
+
+    Path.join(base_directory(), output_path)
+  end
+
   defp output_options_map(media_item_with_preloads) do
     source = media_item_with_preloads.source
 
@@ -125,6 +127,19 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     }
   end
 
+  # I don't love the string manipulation here, but what can ya' do.
+  # It's dependent on the output_path_template being a string ending `.{{ ext }}`
+  # (or equivalent), but that's validated by the MediaProfile schema.
+  defp determine_thumbnail_location(media_item_with_preloads) do
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
+
+    output_path_template
+    |> String.split(~r{\.}, include_captures: true)
+    |> List.insert_at(-3, "-thumb")
+    |> Enum.join()
+    |> build_output_path(media_item_with_preloads)
+  end
+
   defp base_directory do
     Application.get_env(:pinchflat, :media_directory)
   end

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaDownloadWorker do
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,19 +6,32 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Repo
   alias Pinchflat.Media
-  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.MediaDownloader
-  alias Pinchflat.MediaClient.MediaDownloader
+
-  alias Pinchflat.Workers.FilesystemDataWorker
+  @doc """
+  Starts the media_item media download worker and creates a task for the media_item.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(media_item, opts \\ []) do
+    %{id: media_item.id}
+    |> MediaDownloadWorker.new(opts)
+    |> Tasks.create_job_with_task(media_item)
+  end
 
-  @impl Oban.Worker
   @doc """
   For a given media item, download the media alongside any options.
   Does not download media if its source is set to not download media.
 
   Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item =
       media_item_id
@@ -31,26 +44,30 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
   defp download_media_and_schedule_jobs(media_item) do
     case MediaDownloader.download_for_media_item(media_item) do
-      {:ok, _} ->
+      {:ok, updated_media_item} ->
-        schedule_filesystem_data_worker(media_item)
+        compute_and_save_media_filesize(updated_media_item)
-        {:ok, media_item}
+
+        {:ok, updated_media_item}
 
       err ->
         err
     end
   end
 
-  defp schedule_filesystem_data_worker(media_item) do
+  defp compute_and_save_media_filesize(media_item) do
-    %{id: media_item.id}
+    case File.stat(media_item.media_filepath) do
-    |> FilesystemDataWorker.new()
+      {:ok, %{size: size}} ->
-    |> Tasks.create_job_with_task(media_item)
+        Media.update_media_item(media_item, %{media_size_bytes: size})
-    |> case do
+
-      {:ok, task} -> {:ok, task}
+      _ ->
-      {:error, :duplicate_job} -> {:ok, :job_exists}
+        :ok
     end
   end
 end

lib/pinchflat/downloading/media_downloader.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.MediaDownloader do
+defmodule Pinchflat.Downloading.MediaDownloader do
   @moduledoc """
   This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
@@ -8,11 +8,12 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Metadata.MetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
-  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
-  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
@@ -30,16 +31,17 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
 
     case download_with_options(media_item.original_url, item_with_preloads) do
       {:ok, parsed_json} ->
-        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
-
         parsed_attrs =
           parsed_json
-          |> parser.parse_for_media_item()
+          |> MetadataParser.parse_for_media_item()
           |> Map.merge(%{
             media_downloaded_at: DateTime.utc_now(),
+            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
             metadata: %{
-              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
+              # IDEA: might be worth kicking off a job for this since thumbnail fetching
-              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+              # could fail and I want to handle that in isolation
+              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
             }
           })
 
@@ -52,13 +54,16 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
     end
   end
 
-  # def download_for_source(source, url) do
+  defp determine_nfo_filepath(media_item, parsed_json) do
-  #   # Create MI from source and URL
+    if media_item.source.media_profile.download_nfo do
-  #   media_item = nil
+      NfoBuilder.build_and_store_for_media_item(parsed_json)
-  # end
+    else
+      nil
+    end
+  end
 
   defp download_with_options(url, item_with_preloads) do
-    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
+    {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
     YtDlpMedia.download(url, options)
   end

lib/pinchflat/downloading/output_path/base.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.RenderedString.Base do
+defmodule Pinchflat.Downloading.OutputPath.Base do
   @moduledoc """
   A base module for parsing rendered strings, designed as a macro to be used
   in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
   NOTE: if the needs here get any more complicated, look into using a Liquid
   template parser. No need to reinvent the wheel any more than I already have.
 
-  NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
   """
 
   defmacro __using__(_opts) do

lib/pinchflat/downloading/output_path/parser.ex

@@ -1,11 +1,11 @@
-defmodule Pinchflat.RenderedString.Parser do
+defmodule Pinchflat.Downloading.OutputPath.Parser do
   @moduledoc """
   Parses liquid-ish-style strings into a rendered string
 
   Used for turning filepath templates into real filepaths
   """
 
-  use Pinchflat.RenderedString.Base
+  use Pinchflat.Downloading.OutputPath.Base
 
   @doc """
   Parses a string into a rendered string, using the provided variables. Optionally

lib/pinchflat/downloading/output_path_builder.ex

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Profiles.OutputPathBuilder do
+defmodule Pinchflat.Downloading.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
   """
 
-  alias Pinchflat.RenderedString.Parser, as: TemplateParser
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
 
   @doc """
   Builds the actual final filepath from a given template. Optionally, you can pass in
@@ -39,7 +39,10 @@ defmodule Pinchflat.Profiles.OutputPathBuilder do
       # Individual parts of the upload date
       "upload_year" => "%(upload_date>%Y)S",
       "upload_month" => "%(upload_date>%m)S",
-      "upload_day" => "%(upload_date>%d)S"
+      "upload_day" => "%(upload_date>%d)S",
+      "upload_yyyy_mm_dd" => "%(upload_date>%Y-%m-%d)S",
+      "season_from_date" => "%(upload_date>%Y)S",
+      "season_episode_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S"
     }
   end
 end

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  @moduledoc """
+  Methods for performing fast indexing tasks and managing the fast indexing process.
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  alias Pinchflat.Media
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      MediaIndexingWorker.kickoff_with_task(source, url)
+    end)
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          MediaDownloadWorker.kickoff_with_task(media_item)
+        end
+
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,13 +6,25 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "fast_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the source fast indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> FastIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Kicks off the fast indexing process for a source, reschedules the job to run again
   once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
@@ -20,25 +32,26 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
 
   Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do
     next_run_in = Source.fast_index_frequency() * 60
 
-    %{id: source.id}
+    case kickoff_with_task(source, schedule_in: next_run_in) do
-    |> FastIndexingWorker.new(schedule_in: next_run_in)
-    |> Tasks.create_job_with_task(source)
-    |> case do
       {:ok, task} -> {:ok, task}
       {:error, :duplicate_job} -> {:ok, :job_exists}
     end

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorker do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -8,10 +8,22 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   require Logger
 
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the fast media indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, media_url, opts \\ []) do
+    %{id: source.id, media_url: media_url}
+    |> MediaIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Similar to `MediaCollectionIndexingWorker`, but for individual media items.
   Does not reschedule or check anything to do with a source's indexing
@@ -23,7 +35,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item
@@ -37,10 +49,11 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   Returns :ok
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
-    case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
       {:ok, media_item} ->
         Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
 
@@ -49,5 +62,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
     end
 
     :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 end

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -0,0 +1,83 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpers do
+  @moduledoc """
+  Utility methods for working with the filesystem
+  """
+  alias Pinchflat.Media
+  alias Pinchflat.Utils.StringUtils
+
+  @doc """
+  Generates a temporary file and returns its path. The file is empty and has the given type.
+  Generates all the directories in the path if they don't exist.
+
+  Returns binary()
+  """
+  def generate_metadata_tmpfile(type) do
+    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
+
+    :ok = write_p!(filepath, "")
+
+    filepath
+  end
+
+  @doc """
+  Writes content to a file, creating directories as needed.
+  Takes the same args as File.write!/3.
+
+  Returns :ok | raises on error
+  """
+  def write_p!(filepath, content, modes \\ []) do
+    filepath
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.write!(filepath, content, modes)
+  end
+
+  @doc """
+  Fetches the file size of a media item and saves it to the database.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      err ->
+        err
+    end
+  end
+
+  @doc """
+  Deletes a file and removes any empty directories in the path.
+  Does NOT remove any directories that are not empty.
+
+  Returns :ok | {:error, any()}
+  """
+  def delete_file_and_remove_empty_directories(filepath) do
+    case File.rm(filepath) do
+      :ok ->
+        filepath
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+  end
+
+  defp recursively_delete_empty_directories(directory) do
+    case File.rmdir(directory) do
+      :ok ->
+        directory
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+
+    :ok
+  end
+end

lib/pinchflat/media/media.ex

@@ -7,9 +7,10 @@ defmodule Pinchflat.Media do
 
   alias Pinchflat.Repo
   alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @doc """
   Returns the list of media_items.
@@ -141,38 +142,6 @@ defmodule Pinchflat.Media do
   """
   def get_media_item!(id), do: Repo.get!(MediaItem, id)
 
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's downloaded files
-
-  Returns [binary()]
-  """
-  def media_filepaths(media_item) do
-    mapped_struct = Map.from_struct(media_item)
-
-    MediaItem.filepath_attributes()
-    |> Enum.map(fn
-      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-      field -> List.wrap(mapped_struct[field])
-    end)
-    |> List.flatten()
-    |> Enum.filter(&is_binary/1)
-  end
-
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's metadata files.
-  Returns an empty list if the media_item has no metadata.
-
-  Returns [binary()] | []
-  """
-  def metadata_filepaths(media_item) do
-    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
-    mapped_struct = Map.from_struct(metadata)
-
-    MediaMetadata.filepath_attributes()
-    |> Enum.map(fn field -> mapped_struct[field] end)
-    |> Enum.filter(&is_binary/1)
-  end
-
   @doc """
   Creates a media_item.
 
@@ -219,19 +188,22 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Deletes a media_item and its associated tasks.
+  Deletes a media_item, its associated tasks, and our internal metadata files.
-  Can optionally delete the media_item's files.
+  Can optionally delete the media_item's media files (media, thumbnail, subtitles, etc).
 
   Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
   """
   def delete_media_item(%MediaItem{} = media_item, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(media_item)
+
     if delete_files do
-      {:ok, _} = delete_all_attachments(media_item)
+      {:ok, _} = delete_media_files(media_item)
     end
 
-    Tasks.delete_tasks_for(media_item)
+    # Should delete these no matter what
+    delete_internal_metadata_files(media_item)
     Repo.delete(media_item)
   end
 
@@ -242,26 +214,31 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_all_attachments(media_item) do
+  defp delete_media_files(media_item) do
-    media_item = Repo.preload(media_item, :metadata)
+    mapped_struct = Map.from_struct(media_item)
 
-    media_item
+    MediaItem.filepath_attributes()
-    |> media_filepaths()
+    |> Enum.map(fn
-    |> Enum.concat(metadata_filepaths(media_item))
+      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-    |> Enum.each(&File.rm/1)
+      field -> List.wrap(mapped_struct[field])
-
+    end)
-    # rmdir will attempt to delete the directory, but only if it is empty
+    |> List.flatten()
-    if media_item.media_filepath do
+    |> Enum.filter(&is_binary/1)
-      File.rmdir(Path.dirname(media_item.media_filepath))
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
-    end
-
-    if media_item.metadata && media_item.metadata.metadata_filepath do
-      File.rmdir(Path.dirname(media_item.metadata.metadata_filepath))
-    end
 
     {:ok, media_item}
   end
 
+  defp delete_internal_metadata_files(media_item) do
+    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    MediaMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp maybe_apply_cutoff_date(source) do
     if source.download_cutoff_date do
       dynamic([mi], mi.upload_date >= ^source.download_cutoff_date)

lib/pinchflat/media/media_item.ex

@@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
-  alias Pinchflat.Media.MediaItemSearchIndex
+  alias Pinchflat.Media.MediaItemsSearchIndex
 
   @allowed_fields [
     # these fields are captured on indexing (and again on download)
@@ -27,7 +27,8 @@ defmodule Pinchflat.Media.MediaItem do
     :media_size_bytes,
     :subtitle_filepaths,
     :thumbnail_filepath,
-    :metadata_filepath
+    :metadata_filepath,
+    :nfo_filepath
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -54,6 +55,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :media_size_bytes, :integer
     field :thumbnail_filepath, :string
     field :metadata_filepath, :string
+    field :nfo_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.
@@ -64,7 +66,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
-    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
+    has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 
@@ -82,6 +84,6 @@ defmodule Pinchflat.Media.MediaItem do
 
   @doc false
   def filepath_attributes do
-    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths)a
+    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
 end

lib/pinchflat/media/media_items_search_index.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaItemSearchIndex do
+defmodule Pinchflat.Media.MediaItemsSearchIndex do
   @moduledoc """
   The MediaItem fts5 search index. Not made to be directly interacted with,
   but I figured it'd be better to have it in-app so it's not a mystery.

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -9,6 +9,8 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
   needed
   """
 
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
   @doc """
   Compresses and stores metadata for a media item, returning the filepath.
 
@@ -18,8 +20,7 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, "metadata.json.gz")
     {:ok, json} = Phoenix.json_library().encode(metadata_map)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, json, [:compressed])
-    :ok = File.write(filepath, json, [:compressed])
 
     filepath
   end
@@ -45,12 +46,23 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
     thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, thumbnail_blob)
-    :ok = File.write(filepath, thumbnail_blob)
 
     filepath
   end
 
+  @doc """
+  Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
+  and returns a Date struct.
+
+  Returns Date.t()
+  """
+  def parse_upload_date(upload_date) do
+    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
+
+    Date.from_iso8601!("#{year}-#{month}-#{day}")
+  end
+
   defp fetch_thumbnail_from_url(url) do
     http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
     {:ok, body} = http_client.get(url, [], body_format: :binary)

lib/pinchflat/metadata/metadata_parser.ex

@@ -54,9 +54,22 @@ defmodule Pinchflat.Metadata.MetadataParser do
       |> Enum.reverse()
       |> Enum.find_value(fn attrs -> attrs["filepath"] end)
 
-    %{
+    if thumbnail_filepath do
-      thumbnail_filepath: thumbnail_filepath
+      # NOTE: whole ordeal needed due to a bug I found in yt-dlp
-    }
+      # https://github.com/yt-dlp/yt-dlp/issues/9445
+      # Can be reverted to remove this entire conditional once fixed
+      %{
+        thumbnail_filepath:
+          thumbnail_filepath
+          |> String.split(~r{\.}, include_captures: true)
+          |> List.insert_at(-3, "-thumb")
+          |> Enum.join()
+      }
+    else
+      %{
+        thumbnail_filepath: thumbnail_filepath
+      }
+    end
   end
 
   defp parse_infojson_metadata(metadata) do

lib/pinchflat/metadata/nfo_builder.ex

@@ -0,0 +1,44 @@
+defmodule Pinchflat.Metadata.NfoBuilder do
+  @moduledoc """
+  Provides methods for building and storing NFO files for
+  use by Kodi/Jellyfin and other media center software.
+  """
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Builds an NFO file for a media item (read: single "episode") and
+  stores it in the same directory as the media file. Has the same name
+  as the media file, but with a .nfo extension.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_media_item(metadata) do
+    filepath = Path.rootname(metadata["filepath"]) <> ".nfo"
+    nfo = build_for_media_item(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  defp build_for_media_item(metadata) do
+    upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
+    # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
+    # WHO NEEDS A FANCY XML PARSER ANYWAY?!
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <episodedetails>
+      <title>#{metadata["title"]}</title>
+      <showtitle>#{metadata["uploader"]}</showtitle>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <plot>#{metadata["description"]}</plot>
+      <premiered>#{upload_date}</premiered>
+      <season>#{upload_date.year}</season>
+      <episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
+      <genre>YouTube</genre>
+    </episodedetails>
+    """
+  end
+end

lib/pinchflat/metadata/source_metadata.ex

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Metadata.SourceMetadata do
+  @moduledoc """
+  The SourceMetadata schema.
+
+  Look. Don't @ me about Metadata vs. Metadatum. I'm very sensitive.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Pinchflat.Sources.Source
+
+  @allowed_fields ~w(metadata_filepath)a
+  @required_fields ~w(metadata_filepath)a
+
+  schema "source_metadata" do
+    field :metadata_filepath, :string
+
+    belongs_to :source, Source
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(source_metadata, attrs) do
+    source_metadata
+    |> cast(attrs, @allowed_fields)
+    |> validate_required(@required_fields)
+    |> unique_constraint([:source_id])
+  end
+
+  @doc false
+  def filepath_attributes do
+    ~w(metadata_filepath)a
+  end
+end

lib/pinchflat/metadata/source_metadata_storage_worker.ex

@@ -0,0 +1,56 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :remote_metadata,
+    tags: ["media_source", "source_metadata", "remote_metadata"],
+    max_attempts: 1,
+    # This is the only thing stopping this job from calling itself
+    # in an infinite loop.
+    unique: [period: 600]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Repo
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.MetadataFileHelpers
+
+  @doc """
+  Starts the source metadata storage worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> SourceMetadataStorageWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches and stores metadata for a source in the secret metadata location.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id}}) do
+    source = Repo.preload(Sources.get_source!(source_id), :metadata)
+    {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url)
+
+    # Since updating a source kicks this job off again, we enforce job uniqueness (above)
+    # to once, per source, per x minutes. This is to prevent a job from calling itself
+    # in an infinite loop.
+    Sources.update_source(source, %{
+      metadata: %{
+        metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, metadata)
+      }
+    })
+
+    :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
+  end
+end

lib/pinchflat/profiles/media_profile.ex

@@ -19,6 +19,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
     embed_thumbnail
     download_metadata
     embed_metadata
+    download_nfo
     shorts_behaviour
     livestream_behaviour
     preferred_resolution
@@ -30,19 +31,20 @@ defmodule Pinchflat.Profiles.MediaProfile do
     field :name, :string
 
     field :output_path_template, :string,
-      default: "/{{ source_custom_name }}/{{ title }}/{{ title }} [{{ id }}].{{ ext }}"
+      default: "/{{ source_custom_name }}/{{ upload_yyyy_mm_dd }} {{ title }}/{{ title }} [{{ id }}].{{ ext }}"
 
-    field :download_subs, :boolean, default: true
+    field :download_subs, :boolean, default: false
-    field :download_auto_subs, :boolean, default: true
+    field :download_auto_subs, :boolean, default: false
-    field :embed_subs, :boolean, default: true
+    field :embed_subs, :boolean, default: false
     field :sub_langs, :string, default: "en"
 
-    field :download_thumbnail, :boolean, default: true
+    field :download_thumbnail, :boolean, default: false
-    field :embed_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: false
 
-    field :download_metadata, :boolean, default: true
+    field :download_metadata, :boolean, default: false
-    field :embed_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: false
 
+    field :download_nfo, :boolean, default: false
     # NOTE: these do NOT speed up indexing - the indexer still has to go
     # through the entire collection to determine if a media is a short or
     # a livestream.
@@ -65,6 +67,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     media_profile
     |> cast(attrs, @allowed_fields)
     |> validate_required(@required_fields)
+    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
+    |> validate_format(:output_path_template, ext_regex(), message: "must end with .{{ ext }}")
     |> unique_constraint(:name)
   end
+
+  defp ext_regex do
+    ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
+  end
 end

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
+defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   @moduledoc """
   A GenServer that watches a file for new lines and processes them as they come in.
   This is useful for tailing log files and other similar tasks. If there's no activity

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,14 +6,26 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "media_collection_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+
+  @doc """
+  Starts the source slow indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> MediaCollectionIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   The ID is that of a source _record_, not a YouTube channel/playlist ID. Indexes
   the provided source, kicks off downloads for each new MediaItem, and
@@ -40,7 +52,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
        by the `download_media` field on the source as well as the profile's
        shorts/livestream behaviour. At this step we also attach a file reader
        to the `yt-dlp` output file so we can create media items as they come in
-       for a little speedup (see SourceTasks comments for more)
+       for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
     4. If this job is meant to reschedule (ie: has an index frequency > 0),
        it reschedules itself. If not, it runs once and does not reschedule
     5. If the source uses fast indexing, that job is kicked off as well. It
@@ -56,24 +68,23 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   NOTE: Since indexing can take a LONG time, I should check what happens if an
   application restart occurs while a job is running. Will the job be lost?
 
-  IDEA: Should I use paging and do indexing in chunks? Is that even faster?
-
   Returns :ok | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         :ok
 
       _ ->
@@ -81,6 +92,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
         # perform a no-op
         :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -1,9 +1,8 @@
-defmodule Pinchflat.Tasks.SourceTasks do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   @moduledoc """
-  Contains methods used by OR used to create/manage tasks for sources.
+  Methods for performing slow indexing tasks and managing the indexing process.
 
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
+  Many of these methods are made to be kickoff or be consumed by workers.
-  do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
   """
 
   require Logger
@@ -12,16 +11,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.YtDlp.MediaCollection
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.FileFollowerServer
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Starts tasks for indexing a source's media regardless of the source's indexing
@@ -34,56 +31,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
     Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
-    %{id: source.id}
+    MediaCollectionIndexingWorker.kickoff_with_task(source)
-    # Schedule this one immediately, but future ones will be on an interval
-    |> MediaCollectionIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
   end
 
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
   item belonging to the source. You can't tell me the method name isn't descriptive!
+  Returns a list of media items or changesets (if the media item couldn't be created).
 
   Indexing is slow and usually returns a list of all media data at once for record creation.
   To help with this, we use a file follower to watch the file that yt-dlp writes to
@@ -99,56 +54,27 @@ defmodule Pinchflat.Tasks.SourceTasks do
   Since indexing returns all media data EVERY TIME, we that that opportunity to update
   indexing metadata for media items that have already been created.
 
-  Returns [%MediaItem{}, ...]
+  Returns [%MediaItem{} | %Ecto.Changeset{}]
   """
   def index_and_enqueue_download_for_media_items(%Source{} = source) do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
-    result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
+
+    result =
+      Enum.map(media_attributes, fn media_attrs ->
+        case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+          {:ok, media_item} -> media_item
+          {:error, changeset} -> changeset
+        end
+      end)
 
     Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
-    enqueue_pending_media_tasks(source)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
 
     result
   end
 
-  @doc """
-  Starts tasks for downloading media for any of a sources _pending_ media items.
-  Jobs are not enqueued if the source is set to not download media. This will return :ok.
-
-  NOTE: this starts a download for each media item that is pending,
-  not just the ones that were indexed in this job run. This should ensure
-  that any stragglers are caught if, for some reason, they weren't enqueued
-  or somehow got de-queued.
-
-  Returns :ok
-  """
-  def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
-
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
-    :ok
-  end
-
-  @doc """
-  Deletes ALL pending tasks for a source's media items.
-
-  Returns :ok
-  """
-  def dequeue_pending_media_tasks(%Source{} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -191,27 +117,16 @@ defmodule Pinchflat.Tasks.SourceTasks do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
-    maybe_media_item = create_media_item_from_attributes(source, media_attrs)
+    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-
+      {:ok, %MediaItem{} = media_item} ->
-    case maybe_media_item do
-      %MediaItem{} = media_item ->
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
-          %{id: media_item.id}
+          MediaDownloadWorker.kickoff_with_task(media_item)
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
         end
 
-      changeset ->
+      {:error, changeset} ->
         changeset
     end
   end
-
-  defp create_media_item_from_attributes(source, media_attrs) do
-    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-      {:ok, media_item} -> media_item
-      {:error, changeset} -> changeset
-    end
-  end
 end

lib/pinchflat/sources/source.ex

@@ -10,6 +10,7 @@ defmodule Pinchflat.Sources.Source do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Metadata.SourceMetadata
 
   @allowed_fields ~w(
     collection_name
@@ -28,7 +29,8 @@ defmodule Pinchflat.Sources.Source do
   # Expensive API calls are made when a source is inserted/updated so
   # we want to ensure that the source is valid before making the call.
   # This way, we check that the other attributes are valid before ensuring
-  # that all fields are valid.
+  # that all fields are valid. This is still only one DB insert but it's
+  # a two-stage validation process to fail fast before the API call.
   @initially_required_fields ~w(
     index_frequency_minutes
     fast_index
@@ -60,6 +62,8 @@ defmodule Pinchflat.Sources.Source do
 
     belongs_to :media_profile, MediaProfile
 
+    has_one :metadata, SourceMetadata, on_replace: :update
+
     has_many :tasks, Task
     has_many :media_items, MediaItem, foreign_key: :source_id
 
@@ -80,6 +84,7 @@ defmodule Pinchflat.Sources.Source do
     |> cast(attrs, @allowed_fields)
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> validate_required(required_fields)
+    |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id])
   end
 

lib/pinchflat/sources/sources.ex

@@ -9,9 +9,14 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -52,7 +57,7 @@ defmodule Pinchflat.Sources do
     case change_source(%Source{}, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         %Source{}
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
         |> commit_and_handle_tasks()
 
@@ -67,7 +72,7 @@ defmodule Pinchflat.Sources do
   media if the indexing frequency has been changed.
 
   Existing indexing tasks will be cancelled if the indexing frequency has been
-  changed (logic in `SourceTasks.kickoff_indexing_task`)
+  changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
 
   Runs an initial `change_source` check to ensure most of the source is valid
   before making an expensive API call. Runs it through `Repo.update` even
@@ -80,7 +85,7 @@ defmodule Pinchflat.Sources do
     case change_source(source, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         source
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
         |> commit_and_handle_tasks()
 
@@ -98,13 +103,15 @@ defmodule Pinchflat.Sources do
   def delete_source(%Source{} = source, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(source)
+
     source
     |> Media.list_media_items_for()
     |> Enum.each(fn media_item ->
       Media.delete_media_item(media_item, delete_files: delete_files)
     end)
 
-    Tasks.delete_tasks_for(source)
+    delete_source_metadata_files(source)
     Repo.delete(source)
   end
 
@@ -115,19 +122,9 @@ defmodule Pinchflat.Sources do
     Source.changeset(source, attrs, validation_stage)
   end
 
-  @doc """
+  # NOTE: When operating in the ideal path, this effectively adds an API call
-  Returns an `%Ecto.Changeset{}` for tracking source changes and additionally
+  # to the source creation/update process. Should be used only when needed.
-  fetches source details from the original_url (if provided). If the source
+  defp maybe_change_source_from_url(%Source{} = source, attrs) do
-  details cannot be fetched, an error is added to the changeset.
-
-  Note that this fetches source details as long as the `original_url` is present.
-  This means that it'll go for it even if a changeset is otherwise invalid. This
-  is pretty easy to change, but for MVP I'm not concerned.
-
-  NOTE: When operating in the ideal path, this effectively adds an API call
-  to the source creation/update process. Should be used only when needed.
-  """
-  def change_source_from_url(%Source{} = source, attrs) do
     case change_source(source, attrs) do
       %Ecto.Changeset{changes: %{original_url: _}} = changeset ->
         add_source_details_to_changeset(source, changeset)
@@ -137,6 +134,18 @@ defmodule Pinchflat.Sources do
     end
   end
 
+  defp delete_source_metadata_files(source) do
+    metadata = Repo.preload(source, :metadata).metadata || %SourceMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    filepaths =
+      SourceMetadata.filepath_attributes()
+      |> Enum.map(fn field -> mapped_struct[field] end)
+      |> Enum.filter(&is_binary/1)
+
+    Enum.each(filepaths, &FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp add_source_details_to_changeset(source, changeset) do
     %Ecto.Changeset{changes: changes} = changeset
 
@@ -194,6 +203,9 @@ defmodule Pinchflat.Sources do
       {:ok, %Source{} = source} ->
         maybe_handle_media_tasks(changeset, source)
         maybe_run_indexing_task(changeset, source)
+        run_metadata_storage_task(source)
+
+        {:ok, source}
 
       err ->
         err
@@ -205,23 +217,21 @@ defmodule Pinchflat.Sources do
   defp maybe_handle_media_tasks(changeset, source) do
     case {changeset.data, changeset.changes} do
       {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
-        SourceTasks.enqueue_pending_media_tasks(source)
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
-        SourceTasks.dequeue_pending_media_tasks(source)
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
         :ok
     end
-
-    {:ok, source}
   end
 
   defp maybe_run_indexing_task(changeset, source) do
     case changeset.data do
       # If the changeset is new (not persisted), attempt indexing no matter what
       %{__meta__: %{state: :built}} ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
@@ -229,14 +239,17 @@ defmodule Pinchflat.Sources do
         maybe_update_slow_indexing_task(changeset, source)
         maybe_update_fast_indexing_task(changeset, source)
     end
+  end
 
-    {:ok, source}
+  # This runs every time to pick up any changes to the metadata
+  defp run_metadata_storage_task(source) do
+    SourceMetadataStorageWorker.kickoff_with_task(source)
   end
 
   defp maybe_update_slow_indexing_task(changeset, source) do
     case changeset.changes do
       %{index_frequency_minutes: mins} when mins > 0 ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@@ -251,7 +264,8 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+        FastIndexingWorker.kickoff_with_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/media_item_tasks.ex

@@ -1,61 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasks do
-  @moduledoc """
-  Contains methods used by OR used to create/manage tasks for media items.
-
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
-  """
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Fetches the file size of a media item and saves it to the database.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def compute_and_save_media_filesize(media_item) do
-    case File.stat(media_item.media_filepath) do
-      {:ok, %{size: size}} ->
-        Media.update_media_item(media_item, %{media_size_bytes: size})
-
-      err ->
-        err
-    end
-  end
-
-  @doc """
-  Indexes a single media item for a source and enqueues a download job if the
-  media should be downloaded. This method creates the media item record so it's
-  the one-stop-shop for adding a media item (and possibly downloading it) just
-  by a URL and source.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
-    maybe_media_item = create_media_item_from_url(source, url)
-
-    case maybe_media_item do
-      {:ok, media_item} ->
-        if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
-        end
-
-        {:ok, media_item}
-
-      err ->
-        err
-    end
-  end
-
-  defp create_media_item_from_url(source, url) do
-    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
-
-    Media.create_media_item_from_backend_attrs(source, media_attrs)
-  end
-end

lib/pinchflat/tasks/tasks.ex

@@ -22,9 +22,15 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil, job_states \\ Oban.Job.states()) do
+  def list_tasks_for(record, worker_name \\ nil, job_states \\ Oban.Job.states()) do
     stringified_states = Enum.map(job_states, &to_string/1)
 
+    record_type =
+      case record do
+        %Source{} -> :source_id
+        %MediaItem{} -> :media_item_id
+      end
+
     worker_name_finder =
       if worker_name do
         # Workers are the full module name - we want to match on the string ENDING with
@@ -41,7 +47,7 @@ defmodule Pinchflat.Tasks do
     Repo.all(
       from t in Task,
         join: j in assoc(t, :job),
-        where: field(t, ^attached_record_type) == ^attached_record_id,
+        where: field(t, ^record_type) == ^record.id,
         where: ^worker_name_finder,
         where: j.state in ^stringified_states
     )
@@ -53,10 +59,9 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_pending_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil) do
+  def list_pending_tasks_for(record, worker_name \\ nil) do
     list_tasks_for(
-      attached_record_type,
+      record,
-      attached_record_id,
       worker_name,
       [:available, :scheduled, :retryable]
     )
@@ -126,14 +131,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_tasks_for(worker_name)
-        %Source{} = source -> list_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """
@@ -142,14 +143,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_pending_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_pending_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_pending_tasks_for(worker_name)
-        %Source{} = source -> list_pending_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_pending_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """

lib/pinchflat/utils/filesystem_utils.ex

@@ -1,23 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtils do
-  @moduledoc """
-  Utility methods for working with the filesystem
-  """
-
-  alias Pinchflat.Utils.StringUtils
-
-  @doc """
-  Generates a temporary file and returns its path. The file is empty and has the given type.
-  Generates all the directories in the path if they don't exist.
-
-  Returns binary()
-  """
-  def generate_metadata_tmpfile(type) do
-    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
-    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
-
-    :ok = File.mkdir_p!(Path.dirname(filepath))
-    :ok = File.write(filepath, "")
-
-    filepath
-  end
-end

lib/pinchflat/workers/filesystem_data_worker.ex

@@ -1,27 +0,0 @@
-defmodule Pinchflat.Workers.FilesystemDataWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :media_local_metadata,
-    tags: ["media_item", "media_metadata", "local_metadata"],
-    max_attempts: 1
-
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks.MediaItemTasks
-
-  @impl Oban.Worker
-  @doc """
-  For a given media item, compute and save metadata about the file on-disk.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
-
-    MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-    # Don't retry on failure - if it didn't work immediately there's no
-    # reason to believe it will work later.
-    :ok
-  end
-end

lib/pinchflat/yt_dlp/backend/media_collection.ex

@@ -1,81 +0,0 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollection do
-  @moduledoc """
-  Contains utilities for working with collections of
-  media (aka: a source [ie: channels, playlists]).
-  """
-
-  require Logger
-
-  alias Pinchflat.Utils.FunctionUtils
-  alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Returns a list of maps representing the media in the collection.
-
-  Options:
-    - :file_listener_handler - a function that will be called with the path to the
-      file that will be written to when yt-dlp is done. This is useful for
-      setting up a file watcher to know when the file is ready to be read.
-
-  Returns {:ok, [map()]} | {:error, any, ...}.
-  """
-  def get_media_attributes_for_collection(url, addl_opts \\ []) do
-    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
-    command_opts = [:simulate, :skip_download]
-    output_template = YtDlpMedia.indexing_output_template()
-    output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
-    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
-
-    if file_listener_handler do
-      file_listener_handler.(output_filepath)
-    end
-
-    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
-      {:ok, output} ->
-        output
-        |> String.split("\n", trim: true)
-        |> Enum.map(&Phoenix.json_library().decode!/1)
-        |> Enum.map(&YtDlpMedia.response_to_struct/1)
-        |> FunctionUtils.wrap_ok()
-
-      res ->
-        res
-    end
-  end
-
-  @doc """
-  Gets a source's ID and name from its URL.
-
-  yt-dlp does not _really_ have source-specific functions, so
-  instead we're fetching just the first video (using playlist_end: 1)
-  and parsing the source ID and name from _its_ metadata
-
-  Returns {:ok, map()} | {:error, any, ...}.
-  """
-  def get_source_details(source_url) do
-    opts = [:simulate, :skip_download, playlist_end: 1]
-    output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
-
-    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
-         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
-      {:ok, format_source_details(parsed_json)}
-    else
-      err -> err
-    end
-  end
-
-  defp format_source_details(response) do
-    %{
-      channel_id: response["channel_id"],
-      channel_name: response["channel"],
-      playlist_id: response["playlist_id"],
-      playlist_name: response["playlist_title"]
-    }
-  end
-
-  defp backend_runner do
-    # This approach lets us mock the command for testing
-    Application.get_env(:pinchflat, :yt_dlp_runner)
-  end
-end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.BackendCommandRunner do
   @moduledoc """
   A behaviour for running CLI commands against a downloader backend (yt-dlp).
 

lib/pinchflat/yt_dlp/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunner do
+defmodule Pinchflat.YtDlp.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   require Logger
 
   alias Pinchflat.Utils.StringUtils
-  alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
+  alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
-  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
+  alias Pinchflat.YtDlp.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 
@@ -37,7 +37,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
 
     case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
-        # IDEA: consider deleting the file after reading it
+        # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
+        # a huge deal, but it's still a good idea to clean up after ourselves.
         # (even on error? especially on error?)
         File.read(output_filepath)
 

lib/pinchflat/yt_dlp/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.Media do
+defmodule Pinchflat.YtDlp.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """
@@ -25,6 +25,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do
 
   alias __MODULE__
   alias Pinchflat.Utils.FunctionUtils
+  alias Pinchflat.Metadata.MetadataFileHelpers
 
   @doc """
   Downloads a single piece of media (and possibly its metadata) directly to its
@@ -85,8 +86,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       description: response["description"],
       original_url: response["webpage_url"],
       livestream: response["was_live"],
-      short_form_content: short_form_content?(response),
+      short_form_content: response["webpage_url"] && short_form_content?(response),
-      upload_date: parse_upload_date(response["upload_date"])
+      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
     }
   end
 
@@ -99,16 +100,13 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       # WILL returns false positives, but it's a best-effort approach
       # that should work for most cases. The aspect_ratio check is
       # based on a gut feeling and may need to be tweaked.
+      #
+      # These don't fail if duration or aspect_ratio are missing
+      # due to Elixir's comparison semantics
       response["duration"] <= 60 && response["aspect_ratio"] < 0.8
     end
   end
 
-  defp parse_upload_date(upload_date) do
-    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
-
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
-  end
-
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

lib/pinchflat/yt_dlp/media_collection.ex

@@ -0,0 +1,123 @@
+defmodule Pinchflat.YtDlp.MediaCollection do
+  @moduledoc """
+  Contains utilities for working with collections of
+  media (aka: a source [ie: channels, playlists]).
+  """
+
+  require Logger
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Returns a list of maps representing the media in the collection.
+
+  Options:
+    - :file_listener_handler - a function that will be called with the path to the
+      file that will be written to when yt-dlp is done. This is useful for
+      setting up a file watcher to know when the file is ready to be read.
+
+  Returns {:ok, [map()]} | {:error, any, ...}.
+  """
+  def get_media_attributes_for_collection(url, addl_opts \\ []) do
+    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the media details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error]
+    output_template = YtDlpMedia.indexing_output_template()
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
+
+    if file_listener_handler do
+      file_listener_handler.(output_filepath)
+    end
+
+    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
+      {:ok, output} ->
+        parsed_lines =
+          output
+          |> String.split("\n", trim: true)
+          |> Enum.map(fn line ->
+            case Phoenix.json_library().decode(line) do
+              {:ok, parsed_json} ->
+                YtDlpMedia.response_to_struct(parsed_json)
+
+              _ ->
+                nil
+            end
+          end)
+
+        {:ok, Enum.filter(parsed_lines, &(&1 != nil))}
+
+      res ->
+        res
+    end
+  end
+
+  @doc """
+  Gets a source's ID and name from its URL.
+
+  yt-dlp does not _really_ have source-specific functions that return what
+  we need, so instead we're fetching just the first video (using playlist_end: 1)
+  and parsing the source ID and name from _its_ metadata
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_details(source_url) do
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the source details
+    opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1]
+    output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
+
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, format_source_details(parsed_json)}
+    else
+      err -> err
+    end
+  end
+
+  @doc """
+  Gets a source's metadata from its URL.
+
+  This is mostly for things like getting the source's avatar and banner image
+  (if applicable). However, this yt-dlp call doesn't have enough overlap with
+  `get_source_details/1` to allow combining them - this one has _almost_ everything
+  we need, but it doesn't contain enough information to tell 100% if the url is a channel
+  or a playlist.
+
+  The main purpose of this (past using as a fetcher for _other_ metadata) is to live
+  as a compressed blob for possible future use. That's why it's not getting formatted like
+  `get_source_details/1`
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_metadata(source_url) do
+    opts = [playlist_items: 0]
+    output_template = "playlist:%()j"
+
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, parsed_json}
+    else
+      err -> err
+    end
+  end
+
+  defp format_source_details(response) do
+    # NOTE: I should probably make this a struct some day
+    %{
+      channel_id: response["channel_id"],
+      channel_name: response["channel"],
+      playlist_id: response["playlist_id"],
+      playlist_name: response["playlist_title"]
+    }
+  end
+
+  defp backend_runner do
+    # This approach lets us mock the command for testing
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

lib/pinchflat_web/components/core_components.ex

@@ -317,17 +317,7 @@ defmodule PinchflatWeb.CoreComponents do
         <span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
       <div class="relative">
-        <input type="hidden" name={@name} value="false" />
+        <input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
-        <input
-          type="checkbox"
-          id={@id}
-          name={@name}
-          value="true"
-          x-bind:checked="enabled"
-          class="sr-only"
-          @change="enabled = !enabled"
-          {@rest}
-        />
         <div class="inline-block cursor-pointer" @click="enabled = !enabled">
           <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
           <div
@@ -349,23 +339,26 @@ defmodule PinchflatWeb.CoreComponents do
   def input(%{type: "select"} = assigns) do
     ~H"""
     <div phx-feedback-for={@name}>
-      <.label for={@id}>
+      <.label :if={@label} for={@id}>
         <%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
-      <select
+      <div class="flex">
-        id={@id}
+        <select
-        name={@name}
+          id={@id}
-        class={[
+          name={@name}
-          "relative z-20 w-full appearance-none rounded border border-stroke bg-transparent py-3 pl-5 pr-12 outline-none transition",
+          class={[
-          "focus:border-primary active:border-primary dark:border-form-strokedark dark:bg-form-input text-black dark:text-white",
+            "relative z-20 w-full appearance-none rounded border border-stroke bg-transparent py-3 pl-5 pr-12 outline-none transition",
-          @inputclass
+            "focus:border-primary active:border-primary dark:border-form-strokedark dark:bg-form-input text-black dark:text-white",
-        ]}
+            @inputclass
-        multiple={@multiple}
+          ]}
-        {@rest}
+          multiple={@multiple}
-      >
+          {@rest}
-        <option :if={@prompt} value=""><%= @prompt %></option>
+        >
-        <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
+          <option :if={@prompt} value=""><%= @prompt %></option>
-      </select>
+          <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
+        </select>
+        <%= render_slot(@inner_block) %>
+      </div>
       <.help :if={@help}><%= @help %></.help>
       <.error :for={msg <- @errors}><%= msg %></.error>
     </div>

lib/pinchflat_web/components/custom_components/button_components.ex

@@ -28,9 +28,10 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
         "#{@rounding} inline-flex items-center justify-center px-8 py-4",
         "#{@color}",
         "hover:bg-opacity-90 lg:px-8 xl:px-10",
-        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-gray-2",
+        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-grey-5",
         @class
       ]}
+      type={@type}
       disabled={@disabled}
       {@rest}
     >

lib/pinchflat_web/components/layouts/root.html.heex

@@ -13,13 +13,17 @@
     </script>
   </head>
   <body
-    x-data={"{ sidebarVisible: false, proEnabled: #{Settings.get!(:pro_enabled)} }"}
+    x-data={"{ 
+      sidebarVisible: false,
+      proEnabled: #{Settings.get!(:pro_enabled)},
+      onboarding: #{Settings.get!(:onboarding)}
+    }"}
     class="dark text-bodydark bg-boxdark-2"
   >
     <%= @inner_content %>
 
     <.donate_modal conn={@conn} />
-    <template x-if="!proEnabled">
+    <template x-if="!proEnabled && !onboarding">
       <.upgrade_modal conn={@conn} />
     </template>
   </body>

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -1,6 +1,8 @@
 defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   use PinchflatWeb, :html
 
+  alias Pinchflat.Profiles.MediaProfile
+
   embed_templates "media_profile_html/*"
 
   @doc """
@@ -35,6 +37,7 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       upload_day: nil,
       upload_month: nil,
       upload_year: nil,
+      upload_yyyy_mm_dd: "the upload date in the format YYYY-MM-DD",
       source_custom_name: "the name of the sources that use this profile",
       source_collection_type: "the collection type of the sources that use this profile. Either 'channel' or 'playlist'"
     }
@@ -52,4 +55,25 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       duration_string
     )a
   end
+
+  def preset_options do
+    [
+      {"Default", "default"},
+      {"Media Center (Plex, Jellyfin, Kodi, etc.)", "media_center"},
+      {"Music", "audio"},
+      {"Archiving", "archiving"}
+    ]
+  end
+
+  defp default_output_template do
+    %MediaProfile{}.output_path_template
+  end
+
+  defp media_center_output_template do
+    "/shows/{{ source_custom_name }}/Season {{ season_from_date }}/{{ season_episode_from_date }} - {{ title }}.{{ ext }}"
+  end
+
+  defp audio_output_template do
+    "/music/{{ source_custom_name }}/{{ title }}.{{ ext }}"
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -3,116 +3,218 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <h3 class="my-4 text-2xl text-black dark:text-white">
+  <section x-data="{ selectedPreset: null }">
-    General Options
+    <h3 class="my-4 text-2xl text-black dark:text-white">
-  </h3>
+      Use a Preset
-  <.input
+    </h3>
-    field={f[:name]}
+    <section x-data="{ selection: null }">
-    type="text"
+      <.input
-    label="Name"
+        prompt="Select preset"
-    placeholder="New Profile"
+        name="media_profile_preset"
-    help="Something descriptive. Does not impact indexing or downloading (required)"
+        value=""
-  />
+        options={preset_options()}
+        type="select"
+        x-model="selection"
+        inputclass="w-full"
+        help="You can further customize the settings after selecting a preset. This is just a starting point"
+      >
+        <.button
+          class="h-13 w-2/5 lg:w-1/5 ml-2 md:ml-4"
+          rounding="rounded"
+          type="button"
+          x-on:click="selectedPreset = selection; selection = null"
+          x-bind:disabled="!selection"
+        >
+          <span x-text="selection ? 'Load' : 'Select'">Select</span><span class="hidden lg:inline ml-1">Preset</span>
+        </.button>
+      </.input>
+    </section>
 
-  <.input
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
-    field={f[:output_path_template]}
+      General Options
-    type="text"
+    </h3>
-    inputclass="font-mono"
-    label="Output path template"
-    help="Must end with .{{ ext }}. See below for more details. I promise the default is good for most cases (required)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ 
-    Subtitle Options
+        presets: { 
-  </h3>
+          default: 'Default',
-  <.input
+          media_center: 'TV Shows',
-    field={f[:download_subs]}
+          audio: 'Audio',
-    type="toggle"
+          archiving: 'Archiving'
-    label="Download Subtitles"
+        }
-    help="Downloads subtitle files alongside media file"
+    }">
-  />
+      <.input
-  <.input
+        field={f[:name]}
-    field={f[:download_auto_subs]}
+        type="text"
-    type="toggle"
+        label="Name"
-    label="Download Autogenerated Subtitles"
+        placeholder="New Profile"
-    help="Prefers normal subs but will download autogenerated if needed"
+        help="Something descriptive. Does not impact indexing or downloading (required)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
-  <.input
+      />
-    field={f[:embed_subs]}
+    </section>
-    type="toggle"
-    label="Embed Subtitles"
-    help="Embeds subtitles in the video file itself, if supported (recommended)"
-  />
-  <.input
-    field={f[:sub_langs]}
-    type="text"
-    label="Subtitle Languages"
-    help="Use commas for multiple languages (eg: en,de)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data={"{ 
-    Thumbnail Options
+        presets: { 
-  </h3>
+          default: '#{default_output_template()}',
-  <.input
+          media_center: '#{media_center_output_template()}',
-    field={f[:download_thumbnail]}
+          audio: '#{audio_output_template()}',
-    type="toggle"
+          archiving: '#{default_output_template()}'
-    label="Download Thumbnail"
+        }
-    help="Downloads thumbnail alongside media file"
+    }"}>
-  />
+      <.input
-  <.input
+        field={f[:output_path_template]}
-    field={f[:embed_thumbnail]}
+        type="text"
-    type="toggle"
+        inputclass="font-mono"
-    label="Embed Thumbnail"
+        label="Output path template"
-    help="Embeds thumbnail in the video file itself, if supported (recommended)"
+        help="Must end with .{{ ext }}. See below for more details. The default is good for most cases (required)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
-    Metadata Options
+      Subtitle Options
-  </h3>
+    </h3>
-  <.input
-    field={f[:download_metadata]}
-    type="toggle"
-    label="Download Metadata"
-    help="Downloads metadata file alongside media file"
-  />
-  <.input
-    field={f[:embed_metadata]}
-    type="toggle"
-    label="Embed Metadata"
-    help="Embeds metadata in the video file itself, if supported (recommended)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
-    Release Format Options
+      <.input
-  </h3>
+        field={f[:download_subs]}
+        type="toggle"
+        label="Download Subtitles"
+        help="Downloads subtitle files alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
 
-  <.input
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: false } }">
-    field={f[:shorts_behaviour]}
+      <.input
-    options={friendly_format_type_options()}
+        field={f[:download_auto_subs]}
-    type="select"
+        type="toggle"
-    label="Include Shorts?"
+        label="Download Autogenerated Subtitles"
-    help="Experimental. Please report any issues on GitHub"
+        help="Prefers normal subs but will download autogenerated if needed. Requires 'Download Subtitles' to be enabled"
-  />
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
-  <.input
+      />
-    field={f[:livestream_behaviour]}
+    </section>
-    options={friendly_format_type_options()}
-    type="select"
-    label="Include Livestreams?"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
-    Quality Options
+      <.input
-  </h3>
+        field={f[:embed_subs]}
+        type="toggle"
+        label="Embed Subtitles"
+        help="Downloads and embeds subtitles in the media file itself, if supported. Uneffected by 'Download Subtitles' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
 
-  <.input
+    <section x-data="{ presets: { default: 'en', media_center: 'en', audio: '', archiving: 'all' } }">
-    field={f[:preferred_resolution]}
+      <.input
-    options={friendly_resolution_options()}
+        field={f[:sub_langs]}
-    type="select"
+        type="text"
-    label="Preferred Resolution"
+        label="Subtitle Languages"
-    help="Will grab the closest available resolution if your preferred is not available. Setting to 'Audio Only' negates embedding options."
+        help="Use commas for multiple languages (eg: en,de)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
 
-  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Thumbnail Options
+    </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_thumbnail]}
+        type="toggle"
+        label="Download Thumbnail"
+        help="Downloads thumbnail alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
+      <.input
+        field={f[:embed_thumbnail]}
+        type="toggle"
+        label="Embed Thumbnail"
+        help="Downloads and embeds thumbnail in the media file itself, if supported. Uneffected by 'Download Thumbnail' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Metadata Options
+    </h3>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_metadata]}
+        type="toggle"
+        label="Download Metadata"
+        help="Downloads metadata file alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
+      <.input
+        field={f[:embed_metadata]}
+        type="toggle"
+        label="Embed Metadata"
+        help="Downloads and embeds metadata in the media file itself, if supported. Uneffected by 'Download Metadata' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: false, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_nfo]}
+        type="toggle"
+        label="Download NFO data"
+        help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc."
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Release Format Options
+    </h3>
+
+    <section x-data="{ presets: { default: 'include', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
+      <.input
+        field={f[:shorts_behaviour]}
+        options={friendly_format_type_options()}
+        type="select"
+        label="Include Shorts?"
+        help="Experimental. Please report any issues on GitHub"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: 'exclude', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
+      <.input
+        field={f[:livestream_behaviour]}
+        options={friendly_format_type_options()}
+        type="select"
+        label="Include Livestreams?"
+        help="Excludes media that comes from a past livestream"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Quality Options
+    </h3>
+
+    <section x-data="{ presets: { default: '1080p', media_center: '1080p', audio: 'audio', archiving: '2160p' } }">
+      <.input
+        field={f[:preferred_resolution]}
+        options={friendly_resolution_options()}
+        type="select"
+        label="Preferred Resolution"
+        help="Will grab the closest available resolution if your preferred is not available. 'Audio Only' grabs the highest quality m4a"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+  </section>
 
   <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
     <.output_template_help />

lib/pinchflat_web/controllers/pages/page_controller.ex

@@ -8,38 +8,34 @@ defmodule PinchflatWeb.Pages.PageController do
   alias Pinchflat.Profiles.MediaProfile
 
   def home(conn, params) do
-    force_onboarding = params["onboarding"]
+    done_onboarding = params["onboarding"] == "0"
-    media_profiles_exist = Repo.exists?(MediaProfile)
+    force_onboarding = params["onboarding"] == "1"
-    sources_exist = Repo.exists?(Source)
 
-    if !force_onboarding && media_profiles_exist && sources_exist do
+    if done_onboarding, do: Settings.set!(:onboarding, false)
-      render_home_page(conn)
+
+    if force_onboarding || Settings.get!(:onboarding) do
+      render_onboarding_page(conn)
     else
-      render_onboarding_page(conn, media_profiles_exist, sources_exist)
+      render_home_page(conn)
     end
   end
 
   defp render_home_page(conn) do
-    Settings.set!(:onboarding, false)
-    media_profile_count = Repo.aggregate(MediaProfile, :count, :id)
-    source_count = Repo.aggregate(Source, :count, :id)
-    media_item_count = Repo.aggregate(MediaItem, :count, :id)
-
     conn
     |> render(:home,
-      media_profile_count: media_profile_count,
+      media_profile_count: Repo.aggregate(MediaProfile, :count, :id),
-      source_count: source_count,
+      source_count: Repo.aggregate(Source, :count, :id),
-      media_item_count: media_item_count
+      media_item_count: Repo.aggregate(MediaItem, :count, :id)
     )
   end
 
-  defp render_onboarding_page(conn, media_profiles_exist, sources_exist) do
+  defp render_onboarding_page(conn) do
     Settings.set!(:onboarding, true)
 
     conn
     |> render(:onboarding_checklist,
-      media_profiles_exist: media_profiles_exist,
+      media_profiles_exist: Repo.exists?(MediaProfile),
-      sources_exist: sources_exist,
+      sources_exist: Repo.exists?(Source),
       layout: {Layouts, :onboarding}
     )
   end

lib/pinchflat_web/controllers/pages/page_html/onboarding_checklist.html.heex

@@ -39,7 +39,7 @@
         </p>
         <p class="text-md text-bodydark">Feel free to add more Media Profiles or Sources in the meantime!</p>
         <div class="mt-8">
-          <.link href={~p"/"}>
+          <.link href={~p"/?onboarding=0"}>
             <.button color="bg-primary" rounding="rounded-full" disabled={not @sources_exist}>
               Let's Go <span class="font-bold mx-2">🚀</span>
             </.button>

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -46,7 +46,7 @@ defmodule PinchflatWeb.Sources.SourceController do
   def show(conn, %{"id" => id}) do
     source = Repo.preload(Sources.get_source!(id), :media_profile)
 
-    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(:source_id, source.id), :job)
+    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(source), :job)
     pending_media = Media.list_pending_media_items_for(source, limit: 100)
     downloaded_media = Media.list_downloaded_media_items_for(source, limit: 100)
 

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -14,7 +14,7 @@ defmodule PinchflatWeb.Sources.SourceHTML do
 
   def friendly_index_frequencies do
     [
-      {"On Create", -1},
+      {"Only once when first created", -1},
       {"1 Hour", 60},
       {"3 Hours", 3 * 60},
       {"6 Hours", 6 * 60},

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -21,6 +21,7 @@
     options={Enum.map(@media_profiles, &{&1.name, &1.id})}
     type="select"
     label="Media Profile"
+    help="Sets your preferences for what media to look for and how to store it"
   />
 
   <h3 class="mt-8 text-2xl text-black dark:text-white">
@@ -32,7 +33,7 @@
     options={friendly_index_frequencies()}
     type="select"
     label="Index Frequency"
-    help="Time between one index of this source finishing and the next one starting. Setting to 'On Create' will still run an initial index but no subsequent ones"
+    help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
   />
 
   <%!-- TODO: use Alpine to disable the index frequency when fast indexing is enabled --%>
@@ -62,6 +63,9 @@
     type="text"
     label="Download Cutoff Date"
     placeholder="YYYY-MM-DD"
+    maxlength="10"
+    pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
+    title="YYYY-MM-DD"
     help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
   />
 

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.0-alpha.1",
+      version: "0.1.0-alpha.5",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -13,6 +13,16 @@ defmodule Pinchflat.MixProject do
       preferred_cli_env: [
         check: :test,
         credo: :test
+      ],
+      test_coverage: [
+        ignore_modules: [
+          Pinchflat.HTTP.HTTPClient,
+          PinchflatWeb.Layouts,
+          Pinchflat.DataCase,
+          Pinchflat.Release,
+          ~r/Fixtures/,
+          ~r/HTML$/
+        ]
       ]
     ]
   end
@@ -59,6 +69,7 @@ defmodule Pinchflat.MixProject do
       {:nimble_parsec, "~> 1.4"},
       {:mox, "~> 1.0", only: :test},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
+      {:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
       {:ex_check, "~> 0.14.0", only: [:dev, :test], runtime: false},
       {:faker, "~> 0.17", only: :test},
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false}

mix.lock

@@ -6,6 +6,7 @@
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
+  "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
   "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},

priv/repo/migrations/20240313175638_add_nfo_filepath_to_media_item.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddNfoFilepathToMediaItem do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :nfo_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240313175744_add_download_nfo_to_media_profile.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddDownloadNfoToMediaProfile do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_nfo, :boolean, default: false, null: false
+    end
+  end
+end

priv/repo/migrations/20240314174731_create_source_metadata.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.CreateSourceMetadata do
+  use Ecto.Migration
+
+  def change do
+    create table(:source_metadata) do
+      add :metadata_filepath, :string, null: false
+      add :source_id, references(:sources, on_delete: :delete_all), null: false
+
+      timestamps(type: :utc_datetime)
+    end
+
+    create unique_index(:source_metadata, [:source_id])
+  end
+end

selfhosted.Dockerfile

@@ -11,9 +11,9 @@
 #   - https://pkgs.org/ - resource for finding needed packages
 #   - Ex: hexpm/elixir:1.16.0-erlang-26.2.1-debian-bullseye-20231009-slim
 #
-ARG ELIXIR_VERSION=1.16.0
+ARG ELIXIR_VERSION=1.16.2
-ARG OTP_VERSION=26.2.1
+ARG OTP_VERSION=26.2.2
-ARG DEBIAN_VERSION=bullseye-20231009-slim
+ARG DEBIAN_VERSION=bookworm-20240130-slim
 
 ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
 ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
@@ -76,12 +76,13 @@ FROM ${RUNNER_IMAGE}
 
 RUN apt-get update -y
 RUN apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \
-  python3 python3-pip ffmpeg
+  ffmpeg curl git openssh-client
 RUN apt-get clean && rm -f /var/lib/apt/lists/*_*
 
-# Download YT-DLP
+# Download and update YT-DLP
-# NOTE: If you're seeing weird issues, consider using the FFMPEG released by yt-dlp
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
-RUN python3 -m pip install -U --pre yt-dlp
+RUN chmod a+rx /usr/local/bin/yt-dlp
+RUN yt-dlp -U
 
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
@@ -106,6 +107,17 @@ ENV RUN_CONTEXT="selfhosted"
 # Only copy the final release from the build stage
 COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./
 
+# NEVER do this if you're running in an environment where you don't trust the user
+# (ie: most environments). This is only acceptable in a self-hosted environment.
+# The user could just run the whole container as root and bypass this anyway so
+# it's not a huge deal.
+# This removes the root password to allow users to assume root if needed. This is
+# preferrable to running the whole container as root so that the files/directories
+# created by the app aren't owned by root and are therefore easier for other users
+# and processes to interact with. If you want to just run the whole container as
+# root, use --user 0:0 or something.
+RUN passwd -d root
+
 USER nobody
 
 # If using an environment that doesn't automatically reap zombie processes, it is

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.Workers.DataBackfillWorkerTest do
+defmodule Pinchflat.Boot.DataBackfillWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.DataBackfillWorker
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.JobFixtures.TestJobWorker
 
   describe "cancel_pending_backfill_jobs/0" do
     test "cancels all pending backfill jobs" do
@@ -21,14 +21,14 @@ defmodule Pinchflat.Workers.DataBackfillWorkerTest do
 
     test "does not cancel jobs for other workers" do
       %{id: 0}
-      |> FilesystemDataWorker.new()
+      |> TestJobWorker.new()
       |> Repo.insert_unique_job()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
 
       DataBackfillWorker.cancel_pending_backfill_jobs()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
     end
   end
 

test/pinchflat/boot/pre_job_startup_tasks_test.exs

@@ -0,0 +1,25 @@
+defmodule Pinchflat.Boot.PreJobStartupTasksTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Settings
+  alias Pinchflat.Settings.Setting
+  alias Pinchflat.Boot.PreJobStartupTasks
+
+  describe "apply_default_settings" do
+    setup do
+      Repo.delete_all(Setting)
+
+      :ok
+    end
+
+    test "sets default settings" do
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
+
+      PreJobStartupTasks.start_link()
+
+      assert Settings.get!(:onboarding)
+      refute Settings.get!(:pro_enabled)
+    end
+  end
+end

test/pinchflat/downloading/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})
@@ -125,6 +125,22 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :write_thumbnail in res
     end
 
+    test "appends -thumb to the thumbnail name when download_thumbnail is true", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:output, "thumbnail:/tmp/test/media/%(title)S-thumb.%(ext)s"} in res
+    end
+
+    test "converts thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:convert_thumbnail, "jpg"} in res
+    end
+
     test "includes :embed_thumbnail option when specified", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true})
 
@@ -133,12 +149,12 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :embed_thumbnail in res
     end
 
-    test "doesn't include :embed_thumbnail option when preferred_resolution is :audio", %{media_item: media_item} do
+    test "convertes thumbnail to jpg when embed_thumbnail is true", %{media_item: media_item} do
-      media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true, preferred_resolution: :audio})
+      media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true})
 
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
 
-      refute :embed_thumbnail in res
+      assert {:convert_thumbnail, "jpg"} in res
     end
 
     test "doesn't include these options when not specified", %{media_item: media_item} do
@@ -169,14 +185,6 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :embed_metadata in res
     end
 
-    test "doesn't include :embed_metadata option when preferred_resolution is :audio", %{media_item: media_item} do
-      media_item = update_media_profile_attribute(media_item, %{embed_metadata: true, preferred_resolution: :audio})
-
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
-
-      refute :embed_metadata in res
-    end
-
     test "doesn't include these options when not specified", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{embed_metadata: false, download_metadata: false})
 
@@ -189,12 +197,19 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
   end
 
   describe "build/1 when testing quality options" do
-    test "it includes quality options", %{media_item: media_item} do
+    test "it includes quality options" do
-      media_item = update_media_profile_attribute(media_item, %{preferred_resolution: :"1080p"})
+      resolutions = ["360", "480", "720", "1080", "2160"]
 
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+      Enum.each(resolutions, fn resolution ->
+        resolution_atom = String.to_existing_atom(resolution <> "p")
 
-      assert {:format_sort, "res:1080,+codec:avc:m4a"} in res
+        media_profile = media_profile_fixture(%{preferred_resolution: resolution_atom})
+        source = source_fixture(%{media_profile_id: media_profile.id})
+        media_item = Repo.preload(media_item_fixture(source_id: source.id), source: :media_profile)
+
+        assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+        assert {:format_sort, "res:#{resolution},+codec:avc:m4a"} in res
+      end)
     end
 
     test "it includes quality options for audio only", %{media_item: media_item} do
@@ -202,18 +217,18 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
 
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
 
-      assert {:format, "bestaudio"} in res
+      assert :extract_audio in res
-      assert {:format_sort, "ext"} in res
+      assert {:format, "bestaudio[ext=m4a]"} in res
     end
   end
 
   defp update_media_profile_attribute(media_item_with_preloads, attrs) do
     media_item_with_preloads.source.media_profile
     |> Profiles.change_media_profile(attrs)
-    |> Repo.update!()
+    |> Repo.update()
 
     media_item_with_preloads
     |> Repo.reload()
-    |> Repo.preload(source: :media_profile)
+    |> Repo.preload([source: :media_profile], force: true)
   end
 end

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -0,0 +1,75 @@
+defmodule Pinchflat.Downloading.DownloadingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  setup :verify_on_exit!
+
+  describe "enqueue_pending_download_tasks/1" do
+    test "it enqueues a job for each pending media item" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "it does not enqueue a job for media items with a filepath" do
+      source = source_fixture()
+      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it attaches a task to each enqueued job" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert [] = Tasks.list_tasks_for(media_item)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert [_] = Tasks.list_tasks_for(media_item)
+    end
+
+    test "it does not create a job if the source is set to not download" do
+      source = source_fixture(download_media: false)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it does not attach tasks if the source is set to not download" do
+      source = source_fixture(download_media: false)
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert [] = Tasks.list_tasks_for(media_item)
+    end
+  end
+
+  describe "dequeue_pending_download_tasks/1" do
+    test "it deletes all pending tasks for a source's media items" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+
+      assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+      assert [] = Tasks.list_tasks_for(media_item)
+    end
+  end
+end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -1,29 +1,41 @@
-defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
+defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   setup :verify_on_exit!
 
   setup do
-    media_item =
-      Repo.preload(
-        media_item_fixture(%{media_filepath: nil}),
-        [:metadata, source: :media_profile]
-      )
-
     stub(HTTPClientMock, :get, fn _url, _headers, _opts ->
       {:ok, ""}
     end)
 
+    media_item =
+      %{media_filepath: nil}
+      |> media_item_fixture()
+      |> Repo.preload([:metadata, source: :media_profile])
+
     {:ok, %{media_item: media_item}}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{media_item: media_item} do
+      assert [] = all_enqueued(worker: MediaDownloadWorker)
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "attaches a task", %{media_item: media_item} do
+      assert {:ok, task} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert task.media_item_id == media_item.id
+    end
+  end
+
   describe "perform/1" do
     test "it saves attributes to the media_item", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -70,16 +82,22 @@ defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
-    test "it schedules a filesystem data worker", %{media_item: media_item} do
+    test "it saves the file's size to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, render_metadata(:media_metadata)}
+        metadata = render_parsed_metadata(:media_metadata)
+        FilesystemHelpers.write_p!(metadata["filepath"], "test")
+
+        {:ok, Phoenix.json_library().encode!(metadata)}
       end)
 
-      assert [] = all_enqueued(worker: FilesystemDataWorker)
-
       perform_job(MediaDownloadWorker, %{id: media_item.id})
+      media_item = Repo.reload(media_item)
 
-      assert [_] = all_enqueued(worker: FilesystemDataWorker)
+      assert media_item.media_size_bytes > 0
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0})
     end
   end
 end

test/pinchflat/downloading/media_downloader_test.exs

@@ -1,9 +1,11 @@
-defmodule Pinchflat.MediaClient.MediaDownloaderTest do
+defmodule Pinchflat.Downloading.MediaDownloaderTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
 
-  alias Pinchflat.MediaClient.MediaDownloader
+  alias Pinchflat.Downloading.MediaDownloader
 
   setup :verify_on_exit!
 
@@ -103,4 +105,37 @@ defmodule Pinchflat.MediaClient.MediaDownloaderTest do
       assert String.ends_with?(updated_media_item.metadata_filepath, ".info.json")
     end
   end
+
+  describe "download_for_media_item/3 when testing NFO generation" do
+    setup do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      :ok
+    end
+
+    test "it generates an NFO file if the source is set to download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: true})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert String.ends_with?(updated_media_item.nfo_filepath, ".nfo")
+      assert File.exists?(updated_media_item.nfo_filepath)
+
+      File.rm!(updated_media_item.nfo_filepath)
+    end
+
+    test "it does not generate an NFO file if the source is set to not download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: false})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert updated_media_item.nfo_filepath == nil
+    end
+  end
 end

test/pinchflat/downloading/output_path/parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.RenderedString.ParserTest do
+defmodule Pinchflat.Downloading.OutputPath.ParserTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.RenderedString.Parser
+  alias Pinchflat.Downloading.OutputPath.Parser
 
   describe "parse/3" do
     test "it returns the rendered string when the string is valid" do

test/pinchflat/downloading/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.Profiles.OutputPathBuilderTest do
+defmodule Pinchflat.Downloading.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -0,0 +1,111 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  setup :verify_on_exit!
+
+  @media_url "https://www.youtube.com/watch?v=test_1"
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+  end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert Repo.aggregate(MediaItem, :count) == 1
+      assert mi_1.id == mi_2.id
+    end
+
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "creates a download task record", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
+    end
+
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
+
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
+      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
+      source = source_fixture(%{media_profile_id: profile.id})
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        output =
+          Phoenix.json_library().encode!(%{
+            id: "video2",
+            title: "Video 2",
+            webpage_url: "https://example.com/shorts/video2",
+            was_live: true,
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67,
+            upload_date: "20210101"
+          })
+
+        {:ok, output}
+      end)
+
+      assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+  end
+end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -0,0 +1,77 @@
+defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+
+  setup :verify_on_exit!
+
+  describe "kickoff_with_task/2" do
+    test "starts the worker" do
+      source = source_fixture(fast_index: true)
+
+      assert [] = all_enqueued(worker: FastIndexingWorker)
+      assert {:ok, _} = FastIndexingWorker.kickoff_with_task(source)
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "attaches a task" do
+      source = source_fixture(fast_index: true)
+
+      assert {:ok, task} = FastIndexingWorker.kickoff_with_task(source)
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "calls out to Youtube RSS if enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+    end
+
+    test "reschedules itself if fast indexing is enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      assert_enqueued(
+        worker: FastIndexingWorker,
+        args: %{"id" => source.id},
+        scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
+      )
+    end
+
+    test "does not reschedule if that would create a duplicate job" do
+      stub(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "does not call out to Youtube RSS if disabled" do
+      expect(HTTPClientMock, :get, 0, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: false)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+    end
+
+    test "does not reschedule itself if fast indexing is disabled" do
+      source = source_fixture(fast_index: false)
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      refute_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(FastIndexingWorker, %{id: 0})
+    end
+  end
+end

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -6,8 +6,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
 
   @media_url "https://www.youtube.com/watch?v=1234567890"
 
@@ -19,6 +19,19 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
     {:ok, source: source}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: MediaIndexingWorker)
+      assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert [_] = all_enqueued(worker: MediaIndexingWorker)
+    end
+
+    test "attaches a task", %{source: source} do
+      assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert task.source_id == source.id
+    end
+  end
+
   describe "perform/1" do
     test "indexes the media item and saves it to the database", %{source: source} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -40,5 +53,9 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
 
       assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
+    end
   end
 end

test/pinchflat/fast_indexing/youtube_rss_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Api.YoutubeRssTest do
+defmodule Pinchflat.FastIndexing.YoutubeRssTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
 
   setup :verify_on_exit!
 

test/pinchflat/filesystem/filesystem_helpers_test.exs

@@ -0,0 +1,110 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  describe "generate_metadata_tmpfile/1" do
+    test "creates a tmpfile and returns its path" do
+      res = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert String.ends_with?(res, ".json")
+      assert File.exists?(res)
+
+      File.rm!(res)
+    end
+  end
+
+  describe "compute_and_save_media_filesize/1" do
+    test "updates the media item with the file size" do
+      media_item = media_item_with_attachments()
+
+      refute media_item.media_size_bytes
+
+      assert {:ok, media_item} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+
+      assert Repo.reload!(media_item).media_size_bytes
+    end
+
+    test "returns the error if operation fails" do
+      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
+
+      assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+    end
+  end
+
+  describe "write_p!/3" do
+    test "writes content to a file" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+
+    test "creates directories as needed" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "file.json"])
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+  end
+
+  describe "delete_file_and_remove_empty_directories/1" do
+    test "deletes file at the provided filepath" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert File.exists?(filepath)
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+    end
+
+    test "deletes empty directories" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      FilesystemHelpers.write_p!(filepath, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo"]))
+    end
+
+    test "does not delete directories with other files in them" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath_1 = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      filepath_2 = Path.join([tmpfile_directory, "foo", "baz.json"])
+      FilesystemHelpers.write_p!(filepath_1, "")
+      FilesystemHelpers.write_p!(filepath_2, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_1)
+
+      refute File.exists?(filepath_1)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+
+      assert File.exists?(filepath_2)
+      assert File.exists?(Path.join([tmpfile_directory, "foo"]))
+
+      # cleanup
+      FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_2)
+    end
+
+    test "returns an error if file could not be deleted" do
+      filepath = "/nonexistent/file.json"
+
+      assert {:error, _} = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+    end
+  end
+end

test/pinchflat/media_test.exs

@@ -11,7 +11,7 @@ defmodule Pinchflat.MediaTest do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.MetadataFileHelpers
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   setup :verify_on_exit!
 
@@ -360,57 +360,6 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
-  describe "media_filepaths/1" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: "/video/test.jpg",
-        subtitle_filepaths: [["en", "video/test.srt"]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == [
-               "/video/test.mp4",
-               "/video/test.jpg",
-               "video/test.srt"
-             ]
-    end
-
-    test "strips out nil values" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: nil,
-        subtitle_filepaths: [["en", nil]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == ["/video/test.mp4"]
-    end
-  end
-
-  describe "metadata_filepaths" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        metadata_filepath: "/metadata.json.gz",
-        thumbnail_filepath: "/thumbnail.jpg"
-      }
-
-      media_item = media_item_fixture(%{metadata: filepaths})
-
-      assert Media.metadata_filepaths(media_item) == [
-               "/metadata.json.gz",
-               "/thumbnail.jpg"
-             ]
-    end
-
-    test "returns an empty list when there is no metadata" do
-      media_item = media_item_fixture()
-      assert Media.metadata_filepaths(media_item) == []
-    end
-  end
-
   describe "create_media_item/1" do
     test "creating with valid data creates a media_item" do
       valid_attrs = %{
@@ -515,6 +464,26 @@ defmodule Pinchflat.MediaTest do
       assert {:ok, _} = Media.delete_media_item(media_item)
       assert File.exists?(media_item.media_filepath)
     end
+
+    test "does delete the media item's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      media_item = Repo.preload(media_item_with_attachments(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
+          thumbnail_filepath:
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
+        }
+      }
+
+      {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs)
+
+      assert {:ok, _} = Media.delete_media_item(updated_media_item)
+      refute File.exists?(updated_media_item.metadata.metadata_filepath)
+    end
   end
 
   describe "delete_media_item/2 when testing file deletion" do

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -84,4 +84,12 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
       assert Path.basename(filepath) == "maxres.webp"
     end
   end
+
+  describe "parse_upload_date/1" do
+    test "returns a date from the given metadata upload date" do
+      upload_date = "20210101"
+
+      assert Helpers.parse_upload_date(upload_date) == ~D[2021-01-01]
+    end
+  end
 end

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,25 +1,10 @@
-defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
+defmodule Pinchflat.Metadata.MetadataParserTest do
   use Pinchflat.DataCase
 
   alias Pinchflat.Metadata.MetadataParser, as: Parser
 
   setup do
-    json_filepath =
+    {:ok, %{metadata: render_parsed_metadata(:media_metadata)}}
-      Path.join([
-        File.cwd!(),
-        "test",
-        "support",
-        "files",
-        "media_metadata.json"
-      ])
-
-    {:ok, file_body} = File.read(json_filepath)
-    {:ok, parsed_json} = Phoenix.json_library().decode(file_body)
-
-    {:ok,
-     %{
-       metadata: parsed_json
-     }}
   end
 
   describe "parse_for_media_item/1 when testing media metadata" do
@@ -109,6 +94,15 @@ defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
       assert String.ends_with?(result.thumbnail_filepath, ".webp")
     end
 
+    # NOTE: this can be removed once this bug is fixed
+    # https://github.com/yt-dlp/yt-dlp/issues/9445
+    # and the associated conditional in the parser is removed
+    test "automatically appends `-thumb` to the thumbnail filename", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert String.contains?(result.thumbnail_filepath, "-thumb.webp")
+    end
+
     test "doesn't freak out if the media has no thumbnails", %{metadata: metadata} do
       metadata = Map.put(metadata, "thumbnails", %{})
 

test/pinchflat/metadata/nfo_builder_test.exs

@@ -0,0 +1,38 @@
+defmodule Pinchflat.Metadata.NfoBuilderTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Metadata.NfoBuilder
+
+  setup do
+    {:ok, %{metadata: render_parsed_metadata(:media_metadata)}}
+  end
+
+  describe "build_and_store_for_media_item/1" do
+    test "returns the filepath", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+
+      assert File.exists?(result)
+
+      File.rm!(result)
+    end
+
+    test "builds filepath based on media location", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+
+      assert String.contains?(result, Path.rootname(metadata["filepath"]))
+      assert String.ends_with?(result, ".nfo")
+
+      File.rm!(result)
+    end
+
+    test "builds an NFO file", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
+      assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
+
+      File.rm!(result)
+    end
+  end
+end

test/pinchflat/metadata/source_metadata_storage_worker_test.exs

@@ -0,0 +1,82 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
+
+  setup :verify_on_exit!
+
+  describe "kickoff_with_task/1" do
+    test "enqueues a new worker for the source" do
+      source = source_fixture()
+
+      assert {:ok, _} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
+
+    test "creates a new task for the source" do
+      source = source_fixture()
+
+      assert {:ok, task} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "sets metadata location for source" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source = Repo.preload(source_fixture(), :metadata)
+
+      refute source.metadata
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+
+      assert source.metadata.metadata_filepath
+
+      File.rm!(source.metadata.metadata_filepath)
+    end
+
+    test "fetches and stores returned metadata for source" do
+      source = source_fixture()
+      file_contents = Phoenix.json_library().encode!(%{"title" => "test"})
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, file_contents} end)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+      {:ok, metadata} = MetadataFileHelpers.read_compressed_metadata(source.metadata.metadata_filepath)
+
+      assert metadata == %{"title" => "test"}
+    end
+
+    test "won't call itself in an infinite loop" do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source = source_fixture()
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+
+      assert [_] = all_enqueued(worker: SourceMetadataStorageWorker)
+    end
+
+    test "doesn't prevent over source jobs from running" do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source_1 = source_fixture()
+      source_2 = source_fixture()
+
+      perform_job(SourceMetadataStorageWorker, %{id: source_1.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_1.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_2.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_2.id})
+
+      assert [_, _] = all_enqueued(worker: SourceMetadataStorageWorker)
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(SourceMetadataStorageWorker, %{id: 0})
+    end
+  end
+end

test/pinchflat/profiles_test.exs

@@ -26,11 +26,11 @@ defmodule Pinchflat.ProfilesTest do
 
   describe "create_media_profile/1" do
     test "creation with valid data creates a media_profile" do
-      valid_attrs = %{name: "some name", output_path_template: "some output_path_template"}
+      valid_attrs = %{name: "some name", output_path_template: "output_template.{{ ext }}"}
 
       assert {:ok, %MediaProfile{} = media_profile} = Profiles.create_media_profile(valid_attrs)
       assert media_profile.name == "some name"
-      assert media_profile.output_path_template == "some output_path_template"
+      assert media_profile.output_path_template == "output_template.{{ ext }}"
     end
 
     test "creation with invalid data returns error changeset" do
@@ -44,14 +44,14 @@ defmodule Pinchflat.ProfilesTest do
 
       update_attrs = %{
         name: "some updated name",
-        output_path_template: "some updated output_path_template"
+        output_path_template: "new_output_template.{{ ext }}"
       }
 
       assert {:ok, %MediaProfile{} = media_profile} =
                Profiles.update_media_profile(media_profile, update_attrs)
 
       assert media_profile.name == "some updated name"
-      assert media_profile.output_path_template == "some updated output_path_template"
+      assert media_profile.output_path_template == "new_output_template.{{ ext }}"
     end
 
     test "updating with invalid data returns error changeset" do
@@ -132,5 +132,41 @@ defmodule Pinchflat.ProfilesTest do
       media_profile = media_profile_fixture()
       assert %Ecto.Changeset{} = Profiles.change_media_profile(media_profile)
     end
+
+    test "it ensures the media profile's output template ends with an extension" do
+      valid_templates = [
+        "output_template.{{ ext }}",
+        "output_template.{{ext}}",
+        "output_template.%(ext)s",
+        "output_template.%(ext)S",
+        "output_template.%( ext )s",
+        "output_template.%( ext )S"
+      ]
+
+      for template <- valid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        assert cs.valid?
+      end
+    end
+
+    test "it does not allow invalid output templates" do
+      invalid_templates = [
+        "output_template.{{ ext }}.something",
+        "output_template.{{   ext   }}",
+        "output_template{{ ext }}",
+        "output_template.%(ext)s.something",
+        "output_template.txt",
+        "output_template%(ext)s",
+        "output_template.%(nope)s",
+        "output_template"
+      ]
+
+      for template <- invalid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        refute cs.valid?
+      end
+    end
   end
 end

test/pinchflat/slow_indexing/file_follower_server_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServerTest do
+defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
   use ExUnit.Case, async: true
 
-  alias alias Pinchflat.Utils.FilesystemUtils
+  alias alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias Pinchflat.SlowIndexing.FileFollowerServer
 
   setup do
     {:ok, pid} = FileFollowerServer.start_link()
-    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
+    tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
 
     {:ok, %{pid: pid, tmpfile: tmpfile}}
   end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -8,9 +8,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -101,7 +101,10 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
 
       source = source_fixture(index_frequency_minutes: 10)
-      task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
+
+      task_count_fetcher = fn ->
+        Enum.count(Tasks.list_tasks_for(source, "MediaCollectionIndexingWorker"))
+      end
 
       assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
@@ -158,5 +161,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
       end)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaCollectionIndexingWorker, %{id: 0})
+    end
   end
 end

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Tasks.SourceTasksTest do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -9,12 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Tasks.Task
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -22,7 +22,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it schedules a job" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
@@ -30,7 +30,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it creates and attaches a task" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, %Task{} = task} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert task.source_id == source.id
     end
@@ -40,7 +40,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -50,7 +50,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -60,65 +60,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
   end
 
-  describe "kickoff_fast_indexing_task/1" do
-    test "it schedules a job" do
-      source = source_fixture()
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it creates and attaches a task" do
-      source = source_fixture()
-
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert task.source_id == source.id
-    end
-
-    test "it deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-  end
-
-  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
-    setup do
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
-      assert worker.args["id"] == source.id
-      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
-    end
-
-    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      media_item_fixture(source_id: source.id, media_id: "test_1")
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaIndexingWorker)
-    end
-  end
-
   describe "index_and_enqueue_download_for_media_items/1" do
     setup do
       stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
@@ -129,7 +76,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it creates a media_item record for each media ID returned", %{source: source} do
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
@@ -141,15 +88,15 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
     test "it attaches all media_items to the given source", %{source: source} do
       source_id = source.id
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
     end
 
     test "it won't duplicate media_items based on media_id and source", %{source: source} do
-      _first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      _first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      _duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      _duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       media_items = Repo.preload(source, :media_items).media_items
       assert Enum.count(media_items) == 3
@@ -158,8 +105,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it can duplicate media_ids for different sources", %{source: source} do
       other_source = source_fixture()
 
-      media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      media_items_other_source = SourceTasks.index_and_enqueue_download_for_media_items(other_source)
+      media_items_other_source = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(other_source)
 
       assert Enum.count(media_items) == 3
       assert Enum.count(media_items_other_source) == 3
@@ -169,8 +116,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it returns a list of media_items", %{source: source} do
-      first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       first_ids = Enum.map(first_run, & &1.id)
       duplicate_ids = Enum.map(duplicate_run, & &1.id)
@@ -181,7 +128,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it updates the source's last_indexed_at field", %{source: source} do
       assert source.last_indexed_at == nil
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       source = Repo.reload!(source)
 
       assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
@@ -191,7 +138,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
     end
@@ -200,9 +147,33 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture(download_media: false)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [] = Tasks.list_tasks_for(media_item)
+    end
+
+    test "it doesn't blow up if a media item cannot be coerced into a struct", %{source: source} do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        response =
+          Phoenix.json_library().encode!(%{
+            id: "video3",
+            title: "Video 3",
+            was_live: false,
+            description: "desc3",
+            # Only focusing on these because these are passed to functions that
+            # could fail if they're not present
+            webpage_url: nil,
+            aspect_ratio: nil,
+            duration: nil,
+            upload_date: nil
+          })
+
+        {:ok, response}
+      end)
+
+      assert [changeset] = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+
+      assert %Ecto.Changeset{} = changeset
     end
   end
 
@@ -226,7 +197,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       assert Repo.aggregate(MediaItem, :count, :id) == 0
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
     end
 
@@ -245,7 +216,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       refute_enqueued(worker: MediaDownloadWorker)
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -264,7 +235,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -297,7 +268,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -315,71 +286,26 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, source_attributes_return_fixture()}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
       assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
     end
-  end
 
-  describe "enqueue_pending_media_tasks/1" do
+    test "does not blow up if the file returns invalid json", %{source: source} do
-    test "it enqueues a job for each pending media item" do
+      watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
+        filepath = Keyword.get(addl_opts, :output_filepath)
+        File.write(filepath, "INVALID")
 
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+        # Need to add a delay to ensure the file watcher has time to read the file
-    end
+        :timer.sleep(watcher_poll_interval * 2)
+        # We know we're testing the file watcher since the syncronous call will only
+        # return an empty string (creating no records)
+        {:ok, ""}
+      end)
 
-    test "it does not enqueue a job for media items with a filepath" do
+      assert [] = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      source = source_fixture()
-      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it attaches a task to each enqueued job" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-
-    test "it does not create a job if the source is set to not download" do
-      source = source_fixture(download_media: false)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it does not attach tasks if the source is set to not download" do
-      source = source_fixture(download_media: false)
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-  end
-
-  describe "dequeue_pending_media_tasks/1" do
-    test "it deletes all pending tasks for a source's media items" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      SourceTasks.enqueue_pending_media_tasks(source)
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-
-      assert :ok = SourceTasks.dequeue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
     end
   end
 end

test/pinchflat/sources_test.exs

@@ -7,17 +7,33 @@ defmodule Pinchflat.SourcesTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.Metadata.MetadataFileHelpers
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   @invalid_source_attrs %{name: nil, collection_id: nil}
 
   setup :verify_on_exit!
 
+  describe "schema" do
+    test "source_metadata is deleted when the source is deleted" do
+      source =
+        source_fixture(%{metadata: %{metadata_filepath: "/metadata.json.gz"}})
+
+      metadata = source.metadata
+      assert {:ok, %Source{}} = Sources.delete_source(source)
+
+      assert_raise Ecto.NoResultsError, fn ->
+        Repo.reload!(metadata)
+      end
+    end
+  end
+
   describe "list_sources/0" do
     test "it returns all sources" do
       source = source_fixture()
@@ -220,6 +236,21 @@ defmodule Pinchflat.SourcesTest do
 
       assert source.index_frequency_minutes == 0
     end
+
+    test "creating will kickoff a metadata storage worker" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: false,
+        index_frequency_minutes: 0
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
   end
 
   describe "update_source/2" do
@@ -331,7 +362,7 @@ defmodule Pinchflat.SourcesTest do
       source = source_fixture(download_media: true)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
       update_attrs = %{download_media: false}
-      SourceTasks.enqueue_pending_media_tasks(source)
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
@@ -384,6 +415,15 @@ defmodule Pinchflat.SourcesTest do
 
       assert source.index_frequency_minutes == 0
     end
+
+    test "updating will kickoff a metadata storage worker" do
+      source = source_fixture()
+      update_attrs = %{name: "some updated name"}
+
+      assert {:ok, %Source{} = source} = Sources.update_source(source, update_attrs)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
   end
 
   describe "delete_source/2" do
@@ -421,6 +461,22 @@ defmodule Pinchflat.SourcesTest do
       assert {:ok, %Source{}} = Sources.delete_source(source)
       assert File.exists?(media_item.media_filepath)
     end
+
+    test "deletes the source's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      source = Repo.preload(source_fixture(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, %{})
+        }
+      }
+
+      {:ok, updated_source} = Sources.update_source(source, update_attrs)
+
+      assert {:ok, _} = Sources.delete_source(updated_source, delete_files: true)
+      refute File.exists?(updated_source.metadata.metadata_filepath)
+    end
   end
 
   describe "delete_source/2 when deleting files" do
@@ -452,70 +508,6 @@ defmodule Pinchflat.SourcesTest do
     end
   end
 
-  describe "change_source_from_url/2" do
-    test "it returns a changeset" do
-      stub(YtDlpRunnerMock, :run, &channel_mock/3)
-      source = source_fixture()
-
-      assert %Ecto.Changeset{} = Sources.change_source_from_url(source, %{})
-    end
-
-    test "it does not fetch source details if the original_url isn't in the changeset" do
-      expect(YtDlpRunnerMock, :run, 0, &channel_mock/3)
-
-      changeset = Sources.change_source_from_url(%Source{}, %{name: "some updated name"})
-
-      assert %Ecto.Changeset{} = changeset
-    end
-
-    test "it fetches source details if the original_url is in the changeset" do
-      expect(YtDlpRunnerMock, :run, &channel_mock/3)
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123"
-        })
-
-      assert %Ecto.Changeset{} = changeset
-    end
-
-    test "it adds source details to the changeset, keeping the orignal details" do
-      expect(YtDlpRunnerMock, :run, &channel_mock/3)
-
-      media_profile = media_profile_fixture()
-      media_profile_id = media_profile.id
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123",
-          media_profile_id: media_profile.id
-        })
-
-      assert %Ecto.Changeset{} = changeset
-      assert String.starts_with?(changeset.changes.collection_id, "some_channel_id_")
-
-      assert %{
-               collection_name: "some channel name",
-               media_profile_id: ^media_profile_id,
-               original_url: "https://www.youtube.com/channel/abc123"
-             } = changeset.changes
-    end
-
-    test "it adds an error to the changeset if the runner fails" do
-      expect(YtDlpRunnerMock, :run, 1, fn _url, _opts, _ot ->
-        {:error, "some error", 1}
-      end)
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123"
-        })
-
-      assert %Ecto.Changeset{} = changeset
-      assert errors_on(changeset).original_url == ["could not fetch source details from URL"]
-    end
-  end
-
   defp playlist_mock(_url, _opts, _ot) do
     {
       :ok,

test/pinchflat/startup_tasks_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.StartupTasksTest do
-  use Pinchflat.DataCase
-
-  alias Pinchflat.Settings
-
-  # Since this runs on app boot (even in the test env),
-  # any actions in the `init/1` function will already have
-  # run. So we can only test the side effects of those actions,
-  # rather than the actions themselves.
-
-  describe "apply_default_settings" do
-    test "sets default settings" do
-      assert Settings.get!(:onboarding) == true
-    end
-  end
-end

test/pinchflat/tasks/media_items_tasks_test.exs

@@ -1,103 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasksTest do
-  use Pinchflat.DataCase
-
-  import Mox
-  import Pinchflat.MediaFixtures
-  import Pinchflat.SourcesFixtures
-  import Pinchflat.ProfilesFixtures
-
-  alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Tasks.MediaItemTasks
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  setup :verify_on_exit!
-
-  @media_url "https://www.youtube.com/watch?v=1234"
-
-  describe "compute_and_save_media_filesize/1" do
-    test "updates the media item with the file size" do
-      media_item = media_item_with_attachments()
-
-      refute media_item.media_size_bytes
-
-      assert {:ok, media_item} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-      assert Repo.reload!(media_item).media_size_bytes
-    end
-
-    test "returns the error if operation fails" do
-      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
-
-      assert {:error, _} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-    end
-  end
-
-  describe "index_and_enqueue_download_for_media_item/2" do
-    setup do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, media_attributes_return_fixture()}
-      end)
-
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "creates a new media item based on the URL", %{source: source} do
-      assert Repo.aggregate(MediaItem, :count) == 0
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert Repo.aggregate(MediaItem, :count) == 1
-    end
-
-    test "won't duplicate media_items based on media_id and source", %{source: source} do
-      assert {:ok, mi_1} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert {:ok, mi_2} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert Repo.aggregate(MediaItem, :count) == 1
-      assert mi_1.id == mi_2.id
-    end
-
-    test "enqueues a download job", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "creates a download task record", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
-    end
-
-    test "does not enqueue a download job if the source does not allow it" do
-      source = source_fixture(%{download_media: false})
-
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "does not enqueue a download job if the media item does not match the format rules" do
-      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
-      source = source_fixture(%{media_profile_id: profile.id})
-
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        output =
-          Phoenix.json_library().encode!(%{
-            id: "video2",
-            title: "Video 2",
-            webpage_url: "https://example.com/shorts/video2",
-            was_live: true,
-            description: "desc2",
-            aspect_ratio: 1.67,
-            duration: 345.67,
-            upload_date: "20210101"
-          })
-
-        {:ok, output}
-      end)
-
-      assert {:ok, _media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-  end
-end

test/pinchflat/tasks_test.exs

@@ -36,53 +36,59 @@ defmodule Pinchflat.TasksTest do
     end
   end
 
-  describe "list_tasks_for/4" do
+  describe "list_tasks_for/3" do
     test "it lets you specify which record type/ID to join on" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id) == [task]
+      assert Tasks.list_tasks_for(source, nil, [:available]) == [task]
     end
 
     test "it lets you specify which job states to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil, [:available]) == [task]
+      assert Tasks.list_tasks_for(source, nil, [:available]) == [task]
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil, [:cancelled]) == []
+      assert Tasks.list_tasks_for(source, nil, [:cancelled]) == []
     end
 
     test "it lets you specify which worker to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, "TestJobWorker") == [task]
+      assert Tasks.list_tasks_for(source, "TestJobWorker") == [task]
-      assert Tasks.list_tasks_for(:source_id, task.source_id, "FooBarWorker") == []
+      assert Tasks.list_tasks_for(source, "FooBarWorker") == []
     end
 
     test "it includes all workers if no worker is specified" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil) == [task]
+      assert Tasks.list_tasks_for(source, nil) == [task]
     end
   end
 
   describe "list_pending_tasks_for/3" do
     test "it lists pending tasks" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id) == [task]
+      assert Tasks.list_pending_tasks_for(source) == [task]
     end
 
     test "it does not list non-pending tasks" do
-      task = Repo.preload(task_fixture(), :job)
+      task = Repo.preload(task_fixture(), [:job, :source])
       :ok = Oban.cancel_job(task.job)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id) == []
+      assert Tasks.list_pending_tasks_for(task.source) == []
     end
 
     test "it lets you specify which worker to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id, "TestJobWorker") == [task]
+      assert Tasks.list_pending_tasks_for(source, "TestJobWorker") == [task]
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id, "FooBarWorker") == []
+      assert Tasks.list_pending_tasks_for(source, "FooBarWorker") == []
     end
   end
 

test/pinchflat/utils/filesystem_utils_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtilsTest do
-  use ExUnit.Case, async: true
-
-  alias Pinchflat.Utils.FilesystemUtils
-
-  describe "generate_metadata_tmpfile/1" do
-    test "creates a tmpfile and returns its path" do
-      res = FilesystemUtils.generate_metadata_tmpfile(:json)
-
-      assert String.ends_with?(res, ".json")
-      assert File.exists?(res)
-
-      File.rm!(res)
-    end
-  end
-end

test/pinchflat/workers/fast_indexing_worker_test.exs

@@ -1,46 +0,0 @@
-defmodule Pinchflat.Workers.FastIndexingWorkerTest do
-  use Pinchflat.DataCase
-
-  import Mox
-  import Pinchflat.SourcesFixtures
-
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
-
-  setup :verify_on_exit!
-
-  describe "perform/1" do
-    test "calls out to Youtube RSS if enabled" do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
-      source = source_fixture(fast_index: true)
-
-      perform_job(FastIndexingWorker, %{"id" => source.id})
-    end
-
-    test "reschedules itself if fast indexing is enabled" do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
-      source = source_fixture(fast_index: true)
-      perform_job(FastIndexingWorker, %{"id" => source.id})
-
-      assert_enqueued(
-        worker: FastIndexingWorker,
-        args: %{"id" => source.id},
-        scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
-      )
-    end
-
-    test "does not call out to Youtube RSS if disabled" do
-      expect(HTTPClientMock, :get, 0, fn _url -> {:ok, ""} end)
-      source = source_fixture(fast_index: false)
-
-      perform_job(FastIndexingWorker, %{"id" => source.id})
-    end
-
-    test "does not reschedule itself if fast indexing is disabled" do
-      source = source_fixture(fast_index: false)
-      perform_job(FastIndexingWorker, %{"id" => source.id})
-
-      refute_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-  end
-end

test/pinchflat/workers/filesystem_data_worker_test.exs

@@ -1,19 +0,0 @@
-defmodule Pinchflat.Workers.FilesystemDataWorkerTest do
-  use Pinchflat.DataCase
-
-  import Pinchflat.MediaFixtures
-
-  alias Pinchflat.Workers.FilesystemDataWorker
-
-  describe "perform/1" do
-    test "Computes and stores the media file size" do
-      media_item = media_item_with_attachments()
-
-      refute media_item.media_size_bytes
-
-      perform_job(FilesystemDataWorker, %{id: media_item.id})
-
-      assert Repo.reload!(media_item).media_size_bytes
-    end
-  end
-end

test/pinchflat/yt_dlp/command_runner_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
+defmodule Pinchflat.YtDlp.CommandRunnerTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
+  alias Pinchflat.YtDlp.CommandRunner, as: Runner
 
   @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

test/pinchflat/yt_dlp/media_collection_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
+defmodule Pinchflat.YtDlp.MediaCollectionTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Media
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
 
   @channel_url "https://www.youtube.com/c/TheUselessTrials"
 
@@ -22,7 +22,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
 
     test "it passes the expected default args" do
       expect(YtDlpRunnerMock, :run, fn _url, opts, ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
+        assert opts == [:simulate, :skip_download, :ignore_no_formats_error]
         assert ot == Media.indexing_output_template()
 
         {:ok, ""}
@@ -62,6 +62,15 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
       assert_receive {:handler, filename}
       assert String.ends_with?(filename, ".json")
     end
+
+    test "gracefully handles partially failed responses" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        {:ok, "INVALID\n\n" <> source_attributes_return_fixture() <> "\nINVALID\n"}
+      end)
+
+      assert {:ok, [%Media{media_id: "video1"}, %Media{media_id: "video2"}, %Media{media_id: "video3"}]} =
+               MediaCollection.get_media_attributes_for_collection(@channel_url)
+    end
   end
 
   describe "get_source_details/1" do
@@ -87,7 +96,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
 
     test "it passes the expected args to the backend runner" do
       expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
-        assert opts == [:simulate, :skip_download, playlist_end: 1]
+        assert opts == [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1]
         assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
 
         {:ok, "{}"}
@@ -108,4 +117,39 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
       assert {:error, %Jason.DecodeError{}} = MediaCollection.get_source_details(@channel_url)
     end
   end
+
+  describe "get_source_metadata/1" do
+    test "it returns a map with data on success" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        Phoenix.json_library().encode(%{channel: "TheUselessTrials"})
+      end)
+
+      assert {:ok, res} = MediaCollection.get_source_metadata(@channel_url)
+
+      assert %{"channel" => "TheUselessTrials"} = res
+    end
+
+    test "it passes the expected args to the backend runner" do
+      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
+        assert opts == [playlist_items: 0]
+        assert ot == "playlist:%()j"
+
+        {:ok, "{}"}
+      end)
+
+      assert {:ok, _} = MediaCollection.get_source_metadata(@channel_url)
+    end
+
+    test "it returns an error if the runner returns an error" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
+
+      assert {:error, "Big issue", 1} = MediaCollection.get_source_metadata(@channel_url)
+    end
+
+    test "it returns an error if the output is not JSON" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "Not JSON"} end)
+
+      assert {:error, %Jason.DecodeError{}} = MediaCollection.get_source_metadata(@channel_url)
+    end
+  end
 end

test/pinchflat/yt_dlp/media_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.YtDlp.Backend.MediaTest do
+defmodule Pinchflat.YtDlp.MediaTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Media
 
   @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
 
@@ -141,6 +141,16 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
       assert %Media{short_form_content: false} = Media.response_to_struct(response)
     end
 
+    test "doesn't blow up if short form content-related fields are missing" do
+      response = %{
+        "webpage_url" => nil,
+        "aspect_ratio" => nil,
+        "duration" => nil
+      }
+
+      assert %Media{short_form_content: nil} = Media.response_to_struct(response)
+    end
+
     test "parses the upload date" do
       response = %{
         "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
@@ -153,5 +163,16 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
 
       assert %Media{upload_date: ^expected_date} = Media.response_to_struct(response)
     end
+
+    test "doesn't blow up if upload date is missing" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61,
+        "upload_date" => nil
+      }
+
+      assert %Media{upload_date: nil} = Media.response_to_struct(response)
+    end
   end
 end

test/pinchflat_web/controllers/media_profile_controller_test.exs

@@ -8,10 +8,10 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
   alias Pinchflat.Repo
   alias Pinchflat.Settings
 
-  @create_attrs %{name: "some name", output_path_template: "some output_path_template"}
+  @create_attrs %{name: "some name", output_path_template: "output_template.{{ ext }}"}
   @update_attrs %{
     name: "some updated name",
-    output_path_template: "some updated output_path_template"
+    output_path_template: "new_output_template.{{ ext }}"
   }
   @invalid_attrs %{name: nil, output_path_template: nil}
 

test/pinchflat_web/controllers/page_controller_test.exs

@@ -1,51 +1,31 @@
 defmodule PinchflatWeb.PageControllerTest do
   use PinchflatWeb.ConnCase
 
-  import Pinchflat.ProfilesFixtures
-  import Pinchflat.SourcesFixtures
-
   alias Pinchflat.Settings
 
   describe "GET / when testing onboarding" do
-    test "sets the onboarding session to true when onboarding", %{conn: conn} do
+    test "sets the onboarding setting to true when onboarding", %{conn: conn} do
       _conn = get(conn, ~p"/")
       assert Settings.get!(:onboarding)
     end
 
-    test "displays the onboarding page when no media profiles exist", %{conn: conn} do
-      conn = get(conn, ~p"/")
-      assert html_response(conn, 200) =~ "Welcome to Pinchflat"
-    end
-
-    test "displays the onboarding page when no sources exist", %{conn: conn} do
-      _ = media_profile_fixture()
-
-      conn = get(conn, ~p"/")
-      assert html_response(conn, 200) =~ "Welcome to Pinchflat"
-    end
-
     test "displays the onboarding page when onboarding is forced", %{conn: conn} do
-      _ = media_profile_fixture()
+      Settings.set!(:onboarding, false)
-      _ = source_fixture()
 
       conn = get(conn, ~p"/?onboarding=1")
       assert html_response(conn, 200) =~ "Welcome to Pinchflat"
     end
 
-    test "sets the onboarding session to false when not onboarding", %{conn: conn} do
+    test "sets the onboarding setting to false if you pass the corrent query param", %{conn: conn} do
       conn = get(conn, ~p"/")
       assert Settings.get!(:onboarding)
 
-      _ = media_profile_fixture()
+      _conn = get(conn, ~p"/?onboarding=0")
-      _ = source_fixture()
-
-      _conn = get(conn, ~p"/")
       refute Settings.get!(:onboarding)
     end
 
     test "displays the home page when not onboarding", %{conn: conn} do
-      _ = media_profile_fixture()
+      Settings.set!(:onboarding, false)
-      _ = source_fixture()
 
       conn = get(conn, ~p"/")
       assert html_response(conn, 200) =~ "MENU"

test/support/fixtures/sources_fixtures.ex

@@ -34,6 +34,20 @@ defmodule Pinchflat.SourcesFixtures do
     source
   end
 
+  @doc """
+  Generate a source with metadata.
+  """
+  def source_with_metadata(attrs \\ %{}) do
+    merged_attrs =
+      Map.merge(attrs, %{
+        metadata: %{
+          metadata_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/metadata.json.gz"
+        }
+      })
+
+    source_fixture(merged_attrs)
+  end
+
   def source_attributes_return_fixture do
     source_attributes = [
       %{

test/support/testing_helper_methods.ex

@@ -48,4 +48,10 @@ defmodule Pinchflat.TestingHelperMethods do
 
     File.read!(json_filepath)
   end
+
+  def render_parsed_metadata(metadata_name) do
+    metadata_name
+    |> render_metadata()
+    |> Phoenix.json_library().decode!()
+  end
 end