Bumped version; added link to donation receipts

* Simplifies index pages

* Improved UX of more pages

.check.exs

@@ -9,18 +9,15 @@
   fix: true,
 
   ## don't retry automatically even if last run resulted in failures
-  # retry: false,
+  retry: false,
 
   ## list of tools (see `mix check` docs for a list of default curated tools)
   tools: [
     {:compiler, env: %{"MIX_ENV" => "test"}},
     {:formatter, env: %{"MIX_ENV" => "test"}},
     {:sobelow, "mix sobelow --config"},
-    # TODO: delete these and replace them with builtin ex_unit and formatter tools
+    {:prettier_formatting, "yarn run prettier . --check", fix: "yarn run prettier . --write"},
-    # once Elixir 1.16.2 is released (see: https://github.com/karolsluszniak/ex_check/issues/41#issuecomment-1921390413)
+    {:npm_test, false}
-    {:elixir_tests, "mix test"},
-    {:elixir_formatting, "mix format --check-formatted", fix: "mix format"},
-    {:prettier_formatting, "yarn run prettier . --check", fix: "yarn run prettier . --write"}
 
     ## curated tools may be disabled (e.g. the check for compilation warnings)
     # {:compiler, false},

.credo.exs

@@ -128,7 +128,7 @@
           {Credo.Check.Refactor.MatchInCondition, []},
           {Credo.Check.Refactor.NegatedConditionsInUnless, []},
           {Credo.Check.Refactor.NegatedConditionsWithElse, []},
-          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.Nesting, [max_nesting: 4]},
           {Credo.Check.Refactor.RedundantWithClauseResult, []},
           {Credo.Check.Refactor.RejectReject, []},
           {Credo.Check.Refactor.UnlessWithElse, []},
@@ -157,7 +157,16 @@
           {Credo.Check.Warning.UnusedRegexOperation, []},
           {Credo.Check.Warning.UnusedStringOperation, []},
           {Credo.Check.Warning.UnusedTupleOperation, []},
-          {Credo.Check.Warning.WrongTestFileExtension, []}
+          {Credo.Check.Warning.WrongTestFileExtension, []},
+
+          #
+          ## Naming Checks
+          #
+          {CredoNaming.Check.Consistency.ModuleFilename,
+           [
+             priority: :normal,
+             excluded_paths: [~r/test\/support/, ~r/priv/, ~r/lib\/pinchflat_web/, ~r/test\/pinchflat_web/]
+           ]}
         ],
         disabled: [
           #

.iex.exs

@@ -4,8 +4,7 @@ alias Pinchflat.Repo
 alias Pinchflat.Tasks.Task
 alias Pinchflat.Sources.Source
 alias Pinchflat.Media.MediaItem
-alias Pinchflat.Tasks.SourceTasks
+alias Pinchflat.Metadata.MediaMetadata
-alias Pinchflat.Media.MediaMetadata
 alias Pinchflat.Profiles.MediaProfile
 
 alias Pinchflat.Tasks
@@ -14,14 +13,14 @@ alias Pinchflat.Profiles
 alias Pinchflat.Sources
 alias Pinchflat.Settings
 
-alias Pinchflat.MediaClient.MediaDownloader
+alias Pinchflat.Downloading.MediaDownloader
-alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
+alias Pinchflat.YtDlp.MediaCollection, as: YtDlpCollection
 
-alias Pinchflat.Api.YoutubeRss
+alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
-alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+alias Pinchflat.SlowIndexing.FileFollowerServer
 
 defmodule IexHelpers do
   def playlist_url do

README.md

@@ -2,6 +2,10 @@
 
 This is alpha software and anything can break at any time. I make not guarantees about the stability of this software, forward-compatibility of updates, or integrity (both related to and independent of Pinchflat). Essentially, use at your own risk and expect there will be rough edges.
 
+## EFF Donation Receipts
+
+A portion of all donations to Pinchflat will be donated to the Electronic Frontier Foundation. [See here](https://github.com/kieraneglin/pinchflat/wiki/EFF-Donation-Receipts) for a list of donation receipts.
+
 ## What is Pinchflat?
 
 TODO: expand on this.

config/config.exs

@@ -12,7 +12,7 @@ config :pinchflat,
   generators: [timestamp_type: :utc_datetime],
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
-  yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
+  yt_dlp_runner: Pinchflat.YtDlp.CommandRunner,
   media_directory: "/downloads",
   # The user may or may not store metadata for their needs, but the app will always store its copy
   metadata_directory: "/config/metadata",
@@ -48,7 +48,8 @@ config :pinchflat, Oban,
     media_indexing: 2,
     media_collection_indexing: 2,
     media_fetching: 2,
-    media_local_metadata: 8
+    local_metadata: 8,
+    remote_metadata: 4
   ]
 
 # Configures the mailer

dev.Dockerfile

@@ -1,9 +1,18 @@
-# Extend from the official Elixir image.
+ARG ELIXIR_VERSION=1.16.2
-FROM elixir:latest
+ARG OTP_VERSION=26.2.2
+ARG DEBIAN_VERSION=bookworm-20240130
+ARG DEV_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
+
+FROM ${DEV_IMAGE}
+
+# Set the locale deets
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 # Install debian packages
 RUN apt-get update -qq
-RUN apt-get install -y inotify-tools ffmpeg \
+RUN apt-get install -y inotify-tools ffmpeg curl git openssh-client \
   python3 python3-pip python3-setuptools python3-wheel python3-dev
 
 # Install nodejs
@@ -28,6 +37,7 @@ COPY . ./
 RUN chmod +x ./docker-run.dev.sh
 
 # Install Elixir deps
+# RUN mix archive.install github hexpm/hex branch latest
 RUN mix deps.get
 # Gives us iex shell history
 ENV ERL_AFLAGS="-kernel shell_history enabled"

ideas.md

@@ -2,5 +2,3 @@
   - Use a UUID for the media database ID (or at least alongside it)
 - Look into this and its recommended plugins https://hexdocs.pm/ex_check/readme.html
 - Add output template option for the source's friendly name
-- TODO: Install Elixir 1.16.2 when available to fix bug with `ex_check` https://github.com/karolsluszniak/ex_check/issues/41#issuecomment-1921390413
-  - delete `{:elixir_tests, "mix test"}` and formatting check

lib/pinchflat/application.ex

@@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
       # Must be before startup tasks
+      Pinchflat.Boot.PreJobStartupTasks,
       {Oban, Application.fetch_env!(:pinchflat, Oban)},
-      Pinchflat.StartupTasks,
+      Pinchflat.Boot.PostJobStartupTasks,
       {DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: Pinchflat.PubSub},
       # Start the Finch HTTP client for sending emails

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Workers.DataBackfillWorker do
+defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
-    queue: :media_local_metadata,
+    queue: :local_metadata,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
 
@@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
   """
   def cancel_pending_backfill_jobs do
     Oban.Job
-    |> where(worker: "Pinchflat.Workers.DataBackfillWorker")
+    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
     |> Oban.cancel_all_jobs()
   end
 

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,6 +1,7 @@
-defmodule Pinchflat.StartupTasks do
+defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
-  This module is responsible for running startup tasks on app boot.
+  This module is responsible for running startup tasks on app boot
+  AFTER the job runner has initiallized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Settings
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.DataBackfillWorker
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
   """
   @impl true
   def init(state) do
-    apply_default_settings()
     enqueue_backfill_worker()
 
     {:ok, state}
   end
 
-  defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
-    Settings.fetch!(:pro_enabled, false)
-  end
-
   defp enqueue_backfill_worker do
     DataBackfillWorker.cancel_pending_backfill_jobs()
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -0,0 +1,89 @@
+defmodule Pinchflat.Boot.PreJobStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  BEFORE the job runner has initiallized.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Settings
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{}, opts)
+  end
+
+  @doc """
+  Runs application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(state) do
+    apply_default_settings()
+    ensure_directories_are_writeable()
+    rename_old_job_workers()
+
+    {:ok, state}
+  end
+
+  defp apply_default_settings do
+    Settings.fetch!(:onboarding, true)
+    Settings.fetch!(:pro_enabled, false)
+  end
+
+  defp ensure_directories_are_writeable do
+    directories = [
+      Application.get_env(:pinchflat, :media_directory),
+      Application.get_env(:pinchflat, :tmpfile_directory),
+      Application.get_env(:pinchflat, :metadata_directory)
+    ]
+
+    Enum.each(directories, fn dir ->
+      file = Path.join([dir, ".keep"])
+
+      # This will fail if the directory is not writeable, stopping boot
+      FilesystemHelpers.write_p!(file, "")
+    end)
+  end
+
+  # As part of a large refactor, I ended up moving a bunch of workers around. This
+  # is a problem because the workers are stored in the database and the runner
+  # will try to run the OLD jobs. This is also why these tasks run before the job
+  # runner starts up.
+  #
+  # Can be removed after a few months (created: 2024-03-12)
+  defp rename_old_job_workers do
+    # [ [old_name, new_name], ...]
+    rename_map = [
+      ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
+      ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
+      ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
+      ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
+      ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
+    ]
+
+    jobs_renamed =
+      Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
+        {count, _} =
+          Oban.Job
+          |> where(worker: ^old_name)
+          |> Repo.update_all(set: [worker: new_name])
+
+        acc + count
+      end)
+
+    Logger.info("Renamed #{jobs_renamed} old job workers")
+  end
+end

lib/pinchflat/downloading/download_option_builder.ex

@@ -0,0 +1,155 @@
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to download media based on the given media profile.
+  """
+
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.OutputPathBuilder
+
+  @doc """
+  Builds the options for yt-dlp to download media based on the given media's profile.
+
+  IDEA: consider adding the ability to pass in a second argument to override
+        these options
+  """
+  def build(%MediaItem{} = media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
+
+    built_options =
+      default_options() ++
+        subtitle_options(media_profile) ++
+        thumbnail_options(media_item_with_preloads) ++
+        metadata_options(media_profile) ++
+        quality_options(media_profile) ++
+        output_options(media_item_with_preloads)
+
+    {:ok, built_options}
+  end
+
+  @doc """
+  Builds the output path for yt-dlp to download media based on the given source's
+  media profile.
+
+  Returns binary()
+  """
+  def build_output_path_for(%Source{} = source_with_preloads) do
+    output_path_template = source_with_preloads.media_profile.output_path_template
+
+    build_output_path(output_path_template, source_with_preloads)
+  end
+
+  defp default_options do
+    [:no_progress, :windows_filenames]
+  end
+
+  defp subtitle_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:download_subs, true}, _} ->
+          # Force SRT for now - MAY provide as an option in the future
+          acc ++ [:write_subs, convert_subs: "srt"]
+
+        {{:download_auto_subs, true}, %{download_subs: true}} ->
+          acc ++ [:write_auto_subs]
+
+        {{:embed_subs, true}, %{preferred_resolution: pr}} when pr != :audio ->
+          acc ++ [:embed_subs]
+
+        {{:sub_langs, sub_langs}, %{download_subs: true}} ->
+          acc ++ [sub_langs: sub_langs]
+
+        {{:sub_langs, sub_langs}, %{embed_subs: true}} ->
+          acc ++ [sub_langs: sub_langs]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
+  defp thumbnail_options(media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} ->
+          thumbnail_save_location = determine_thumbnail_location(media_item_with_preloads)
+
+          acc ++ [:write_thumbnail, convert_thumbnail: "jpg", output: "thumbnail:#{thumbnail_save_location}"]
+
+        {:embed_thumbnail, true} ->
+          acc ++ [:embed_thumbnail, convert_thumbnail: "jpg"]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp quality_options(media_profile) do
+    video_codec_options = "+codec:avc:m4a"
+
+    case media_profile.preferred_resolution do
+      # Also be aware that :audio disabled all embedding options for subtitles
+      :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
+      :"360p" -> [format_sort: "res:360,#{video_codec_options}"]
+      :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
+      :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
+      :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
+      :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
+    end
+  end
+
+  defp output_options(media_item_with_preloads) do
+    [
+      output: build_output_path_for(media_item_with_preloads.source)
+    ]
+  end
+
+  defp build_output_path(string, source) do
+    additional_options_map = output_options_map(source)
+    {:ok, output_path} = OutputPathBuilder.build(string, additional_options_map)
+
+    Path.join(base_directory(), output_path)
+  end
+
+  defp output_options_map(source) do
+    %{
+      "source_custom_name" => source.custom_name,
+      "source_collection_type" => source.collection_type
+    }
+  end
+
+  # I don't love the string manipulation here, but what can ya' do.
+  # It's dependent on the output_path_template being a string ending `.{{ ext }}`
+  # (or equivalent), but that's validated by the MediaProfile schema.
+  defp determine_thumbnail_location(media_item_with_preloads) do
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
+
+    output_path_template
+    |> String.split(~r{\.}, include_captures: true)
+    |> List.insert_at(-3, "-thumb")
+    |> Enum.join()
+    |> build_output_path(media_item_with_preloads.source)
+  end
+
+  defp base_directory do
+    Application.get_env(:pinchflat, :media_directory)
+  end
+end

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaDownloadWorker do
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,19 +6,32 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Repo
   alias Pinchflat.Media
-  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.MediaDownloader
-  alias Pinchflat.MediaClient.MediaDownloader
+
-  alias Pinchflat.Workers.FilesystemDataWorker
+  @doc """
+  Starts the media_item media download worker and creates a task for the media_item.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(media_item, opts \\ []) do
+    %{id: media_item.id}
+    |> MediaDownloadWorker.new(opts)
+    |> Tasks.create_job_with_task(media_item)
+  end
 
-  @impl Oban.Worker
   @doc """
   For a given media item, download the media alongside any options.
   Does not download media if its source is set to not download media.
 
   Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item =
       media_item_id
@@ -31,26 +44,30 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
   defp download_media_and_schedule_jobs(media_item) do
     case MediaDownloader.download_for_media_item(media_item) do
-      {:ok, _} ->
+      {:ok, updated_media_item} ->
-        schedule_filesystem_data_worker(media_item)
+        compute_and_save_media_filesize(updated_media_item)
-        {:ok, media_item}
+
+        {:ok, updated_media_item}
 
       err ->
         err
     end
   end
 
-  defp schedule_filesystem_data_worker(media_item) do
+  defp compute_and_save_media_filesize(media_item) do
-    %{id: media_item.id}
+    case File.stat(media_item.media_filepath) do
-    |> FilesystemDataWorker.new()
+      {:ok, %{size: size}} ->
-    |> Tasks.create_job_with_task(media_item)
+        Media.update_media_item(media_item, %{media_size_bytes: size})
-    |> case do
+
-      {:ok, task} -> {:ok, task}
+      _ ->
-      {:error, :duplicate_job} -> {:ok, :job_exists}
+        :ok
     end
   end
 end

lib/pinchflat/downloading/media_downloader.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.MediaDownloader do
+defmodule Pinchflat.Downloading.MediaDownloader do
   @moduledoc """
   This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
@@ -8,11 +8,12 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Metadata.MetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
-  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
-  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
@@ -30,16 +31,17 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
 
     case download_with_options(media_item.original_url, item_with_preloads) do
       {:ok, parsed_json} ->
-        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
-
         parsed_attrs =
           parsed_json
-          |> parser.parse_for_media_item()
+          |> MetadataParser.parse_for_media_item()
           |> Map.merge(%{
             media_downloaded_at: DateTime.utc_now(),
+            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
             metadata: %{
-              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
+              # IDEA: might be worth kicking off a job for this since thumbnail fetching
-              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+              # could fail and I want to handle that in isolation
+              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
             }
           })
 
@@ -52,13 +54,18 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
     end
   end
 
-  # def download_for_source(source, url) do
+  defp determine_nfo_filepath(media_item, parsed_json) do
-  #   # Create MI from source and URL
+    if media_item.source.media_profile.download_nfo do
-  #   media_item = nil
+      filepath = Path.rootname(parsed_json["filepath"]) <> ".nfo"
-  # end
+
+      NfoBuilder.build_and_store_for_media_item(filepath, parsed_json)
+    else
+      nil
+    end
+  end
 
   defp download_with_options(url, item_with_preloads) do
-    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
+    {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
     YtDlpMedia.download(url, options)
   end

lib/pinchflat/downloading/output_path/base.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.RenderedString.Base do
+defmodule Pinchflat.Downloading.OutputPath.Base do
   @moduledoc """
   A base module for parsing rendered strings, designed as a macro to be used
   in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
   NOTE: if the needs here get any more complicated, look into using a Liquid
   template parser. No need to reinvent the wheel any more than I already have.
 
-  NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
   """
 
   defmacro __using__(_opts) do

lib/pinchflat/downloading/output_path/parser.ex

@@ -1,11 +1,11 @@
-defmodule Pinchflat.RenderedString.Parser do
+defmodule Pinchflat.Downloading.OutputPath.Parser do
   @moduledoc """
   Parses liquid-ish-style strings into a rendered string
 
   Used for turning filepath templates into real filepaths
   """
 
-  use Pinchflat.RenderedString.Base
+  use Pinchflat.Downloading.OutputPath.Base
 
   @doc """
   Parses a string into a rendered string, using the provided variables. Optionally

lib/pinchflat/downloading/output_path_builder.ex

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Profiles.OutputPathBuilder do
+defmodule Pinchflat.Downloading.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
   """
 
-  alias Pinchflat.RenderedString.Parser, as: TemplateParser
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
 
   @doc """
   Builds the actual final filepath from a given template. Optionally, you can pass in
@@ -39,7 +39,10 @@ defmodule Pinchflat.Profiles.OutputPathBuilder do
       # Individual parts of the upload date
       "upload_year" => "%(upload_date>%Y)S",
       "upload_month" => "%(upload_date>%m)S",
-      "upload_day" => "%(upload_date>%d)S"
+      "upload_day" => "%(upload_date>%d)S",
+      "upload_yyyy_mm_dd" => "%(upload_date>%Y-%m-%d)S",
+      "season_from_date" => "%(upload_date>%Y)S",
+      "season_episode_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S"
     }
   end
 end

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  @moduledoc """
+  Methods for performing fast indexing tasks and managing the fast indexing process.
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  alias Pinchflat.Media
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      MediaIndexingWorker.kickoff_with_task(source, url)
+    end)
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          MediaDownloadWorker.kickoff_with_task(media_item)
+        end
+
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,13 +6,25 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "fast_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the source fast indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> FastIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Kicks off the fast indexing process for a source, reschedules the job to run again
   once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
@@ -20,25 +32,26 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
 
   Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do
     next_run_in = Source.fast_index_frequency() * 60
 
-    %{id: source.id}
+    case kickoff_with_task(source, schedule_in: next_run_in) do
-    |> FastIndexingWorker.new(schedule_in: next_run_in)
-    |> Tasks.create_job_with_task(source)
-    |> case do
       {:ok, task} -> {:ok, task}
       {:error, :duplicate_job} -> {:ok, :job_exists}
     end

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorker do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -8,10 +8,22 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   require Logger
 
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the fast media indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, media_url, opts \\ []) do
+    %{id: source.id, media_url: media_url}
+    |> MediaIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Similar to `MediaCollectionIndexingWorker`, but for individual media items.
   Does not reschedule or check anything to do with a source's indexing
@@ -23,7 +35,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item
@@ -37,10 +49,11 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   Returns :ok
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
-    case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
       {:ok, media_item} ->
         Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
 
@@ -49,5 +62,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
     end
 
     :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 end

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -0,0 +1,96 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpers do
+  @moduledoc """
+  Utility methods for working with the filesystem
+  """
+  alias Pinchflat.Media
+  alias Pinchflat.Utils.StringUtils
+
+  @doc """
+  Generates a temporary file and returns its path. The file is empty and has the given type.
+  Generates all the directories in the path if they don't exist.
+
+  Returns binary()
+  """
+  def generate_metadata_tmpfile(type) do
+    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
+
+    :ok = write_p!(filepath, "")
+
+    filepath
+  end
+
+  @doc """
+  Writes content to a file, creating directories as needed.
+  Takes the same args as File.write!/3.
+
+  Returns :ok | raises on error
+  """
+  def write_p!(filepath, content, modes \\ []) do
+    filepath
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.write!(filepath, content, modes)
+  end
+
+  @doc """
+  Copies a file from source to destination, creating directories as needed.
+
+  Returns :ok | raises on error
+  """
+  def cp_p!(source, destination) do
+    destination
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.cp!(source, destination)
+  end
+
+  @doc """
+  Fetches the file size of a media item and saves it to the database.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      err ->
+        err
+    end
+  end
+
+  @doc """
+  Deletes a file and removes any empty directories in the path.
+  Does NOT remove any directories that are not empty.
+
+  Returns :ok | {:error, any()}
+  """
+  def delete_file_and_remove_empty_directories(filepath) do
+    case File.rm(filepath) do
+      :ok ->
+        filepath
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+  end
+
+  defp recursively_delete_empty_directories(directory) do
+    case File.rmdir(directory) do
+      :ok ->
+        directory
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+
+    :ok
+  end
+end

lib/pinchflat/media/media.ex

@@ -7,9 +7,10 @@ defmodule Pinchflat.Media do
 
   alias Pinchflat.Repo
   alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @doc """
   Returns the list of media_items.
@@ -141,38 +142,6 @@ defmodule Pinchflat.Media do
   """
   def get_media_item!(id), do: Repo.get!(MediaItem, id)
 
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's downloaded files
-
-  Returns [binary()]
-  """
-  def media_filepaths(media_item) do
-    mapped_struct = Map.from_struct(media_item)
-
-    MediaItem.filepath_attributes()
-    |> Enum.map(fn
-      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-      field -> List.wrap(mapped_struct[field])
-    end)
-    |> List.flatten()
-    |> Enum.filter(&is_binary/1)
-  end
-
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's metadata files.
-  Returns an empty list if the media_item has no metadata.
-
-  Returns [binary()] | []
-  """
-  def metadata_filepaths(media_item) do
-    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
-    mapped_struct = Map.from_struct(metadata)
-
-    MediaMetadata.filepath_attributes()
-    |> Enum.map(fn field -> mapped_struct[field] end)
-    |> Enum.filter(&is_binary/1)
-  end
-
   @doc """
   Creates a media_item.
 
@@ -219,19 +188,22 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Deletes a media_item and its associated tasks.
+  Deletes a media_item, its associated tasks, and our internal metadata files.
-  Can optionally delete the media_item's files.
+  Can optionally delete the media_item's media files (media, thumbnail, subtitles, etc).
 
   Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
   """
   def delete_media_item(%MediaItem{} = media_item, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(media_item)
+
     if delete_files do
-      {:ok, _} = delete_all_attachments(media_item)
+      {:ok, _} = delete_media_files(media_item)
     end
 
-    Tasks.delete_tasks_for(media_item)
+    # Should delete these no matter what
+    delete_internal_metadata_files(media_item)
     Repo.delete(media_item)
   end
 
@@ -242,26 +214,31 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_all_attachments(media_item) do
+  defp delete_media_files(media_item) do
-    media_item = Repo.preload(media_item, :metadata)
+    mapped_struct = Map.from_struct(media_item)
 
-    media_item
+    MediaItem.filepath_attributes()
-    |> media_filepaths()
+    |> Enum.map(fn
-    |> Enum.concat(metadata_filepaths(media_item))
+      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-    |> Enum.each(&File.rm/1)
+      field -> List.wrap(mapped_struct[field])
-
+    end)
-    # rmdir will attempt to delete the directory, but only if it is empty
+    |> List.flatten()
-    if media_item.media_filepath do
+    |> Enum.filter(&is_binary/1)
-      File.rmdir(Path.dirname(media_item.media_filepath))
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
-    end
-
-    if media_item.metadata && media_item.metadata.metadata_filepath do
-      File.rmdir(Path.dirname(media_item.metadata.metadata_filepath))
-    end
 
     {:ok, media_item}
   end
 
+  defp delete_internal_metadata_files(media_item) do
+    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    MediaMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp maybe_apply_cutoff_date(source) do
     if source.download_cutoff_date do
       dynamic([mi], mi.upload_date >= ^source.download_cutoff_date)

lib/pinchflat/media/media_item.ex

@@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
-  alias Pinchflat.Media.MediaItemSearchIndex
+  alias Pinchflat.Media.MediaItemsSearchIndex
 
   @allowed_fields [
     # these fields are captured on indexing (and again on download)
@@ -27,7 +27,8 @@ defmodule Pinchflat.Media.MediaItem do
     :media_size_bytes,
     :subtitle_filepaths,
     :thumbnail_filepath,
-    :metadata_filepath
+    :metadata_filepath,
+    :nfo_filepath
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -54,6 +55,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :media_size_bytes, :integer
     field :thumbnail_filepath, :string
     field :metadata_filepath, :string
+    field :nfo_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.
@@ -64,7 +66,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
-    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
+    has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 
@@ -82,6 +84,6 @@ defmodule Pinchflat.Media.MediaItem do
 
   @doc false
   def filepath_attributes do
-    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths)a
+    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
 end

lib/pinchflat/media/media_items_search_index.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaItemSearchIndex do
+defmodule Pinchflat.Media.MediaItemsSearchIndex do
   @moduledoc """
   The MediaItem fts5 search index. Not made to be directly interacted with,
   but I figured it'd be better to have it in-app so it's not a mystery.

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -9,6 +9,8 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
   needed
   """
 
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
   @doc """
   Compresses and stores metadata for a media item, returning the filepath.
 
@@ -18,8 +20,7 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, "metadata.json.gz")
     {:ok, json} = Phoenix.json_library().encode(metadata_map)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, json, [:compressed])
-    :ok = File.write(filepath, json, [:compressed])
 
     filepath
   end
@@ -45,12 +46,60 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
     thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, thumbnail_blob)
-    :ok = File.write(filepath, thumbnail_blob)
 
     filepath
   end
 
+  @doc """
+  Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
+  and returns a Date struct.
+
+  Returns Date.t()
+  """
+  def parse_upload_date(upload_date) do
+    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
+
+    Date.from_iso8601!("#{year}-#{month}-#{day}")
+  end
+
+  @doc """
+  Attempts to determine the series directory from a media filepath.
+  The series directory is the "root" directory for a given source
+  which should contain all the season-level folders of that source.
+
+  Used for determining where to store things like NFO data and banners
+  for media center apps. Not useful without a media center app.
+
+  Returns {:ok, binary()} | {:error, :indeterminable}
+  """
+  def series_directory_from_media_filepath(media_filepath) do
+    # Matches "s" or "season" (case-insensitive)
+    # followed by an optional non-word character (. or _ or <space>, etc)
+    # followed by at least one digit
+    # followed immediately by the end of the string
+    # Example matches: s1, s.1, s01 season 1, Season.01, Season_1, Season 1, Season1
+    # Example non-matches: s01e01, season, series 1,
+    season_regex = ~r/^s(eason)?(\W|_)?\d{1,}$/i
+
+    {series_directory, found_series_directory} =
+      media_filepath
+      |> Path.split()
+      |> Enum.reduce_while({[], false}, fn part, {directory_acc, _} ->
+        if String.match?(part, season_regex) do
+          {:halt, {directory_acc, true}}
+        else
+          {:cont, {directory_acc ++ [part], false}}
+        end
+      end)
+
+    if found_series_directory do
+      {:ok, Path.join(series_directory)}
+    else
+      {:error, :indeterminable}
+    end
+  end
+
   defp fetch_thumbnail_from_url(url) do
     http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
     {:ok, body} = http_client.get(url, [], body_format: :binary)

lib/pinchflat/metadata/metadata_parser.ex

@@ -54,9 +54,22 @@ defmodule Pinchflat.Metadata.MetadataParser do
       |> Enum.reverse()
       |> Enum.find_value(fn attrs -> attrs["filepath"] end)
 
-    %{
+    if thumbnail_filepath do
-      thumbnail_filepath: thumbnail_filepath
+      # NOTE: whole ordeal needed due to a bug I found in yt-dlp
-    }
+      # https://github.com/yt-dlp/yt-dlp/issues/9445
+      # Can be reverted to remove this entire conditional once fixed
+      %{
+        thumbnail_filepath:
+          thumbnail_filepath
+          |> String.split(~r{\.}, include_captures: true)
+          |> List.insert_at(-3, "-thumb")
+          |> Enum.join()
+      }
+    else
+      %{
+        thumbnail_filepath: thumbnail_filepath
+      }
+    end
   end
 
   defp parse_infojson_metadata(metadata) do

lib/pinchflat/metadata/nfo_builder.ex

@@ -0,0 +1,68 @@
+defmodule Pinchflat.Metadata.NfoBuilder do
+  @moduledoc """
+  Provides methods for building and storing NFO files for
+  use by Kodi/Jellyfin and other media center software.
+  """
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Builds an NFO file for a media item (read: single "episode") and
+  stores it at the specified location.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_media_item(filepath, metadata) do
+    nfo = build_for_media_item(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  @doc """
+  Builds an NFO file for a souce and stores it at the specified location.
+  Technically works for playlists, but it's really made for channels.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_source(filepath, metadata) do
+    nfo = build_for_source(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  defp build_for_media_item(metadata) do
+    upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
+    # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
+    # WHO NEEDS A FANCY XML PARSER ANYWAY?!
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <episodedetails>
+      <title>#{metadata["title"]}</title>
+      <showtitle>#{metadata["uploader"]}</showtitle>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <plot>#{metadata["description"]}</plot>
+      <aired>#{upload_date}</aired>
+      <season>#{upload_date.year}</season>
+      <episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
+      <genre>YouTube</genre>
+    </episodedetails>
+    """
+  end
+
+  defp build_for_source(metadata) do
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <tvshow>
+      <title>#{metadata["title"]}</title>
+      <plot>#{metadata["description"]}</plot>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <genre>YouTube</genre>
+    </tvshow>
+    """
+  end
+end

lib/pinchflat/metadata/source_image_parser.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.Metadata.SourceImageParser do
+  @moduledoc """
+  Functions for parsing and storing source images.
+  """
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Given a base directory and source metadata, look for the appropriate images
+  and move them to the base directory. Returns a map of image types and their
+  filepaths (specifically for a source). If a given image type is not found,
+  the key will not be present in the map.
+
+  The metadata is expected to contain the `filepath` key for images, implying
+  that the images have already been downloaded by yt-dlp. This method will NOT
+  download images from the internet - it just copys them around.
+
+  Returns a map with the possible keys :poster_filepath, :fanart_filepath, and
+  :banner_filepath.
+  """
+  def store_source_images(base_directory, source_metadata) do
+    (source_metadata["thumbnails"] || [])
+    |> Enum.filter(&(&1["filepath"] != nil))
+    |> select_useful_images()
+    |> Enum.map(&move_image(&1, base_directory))
+    |> Enum.into(%{})
+  end
+
+  defp select_useful_images(images) do
+    labelled_images =
+      Enum.reduce(images, [], fn image_map, acc ->
+        case image_map do
+          %{"id" => "avatar_uncropped"} ->
+            acc ++ [{:poster, :poster_filepath, image_map["filepath"]}]
+
+          %{"id" => "banner_uncropped"} ->
+            acc ++ [{:fanart, :fanart_filepath, image_map["filepath"]}]
+
+          _ ->
+            acc
+        end
+      end)
+
+    labelled_images
+    |> Enum.concat([{:banner, :banner_filepath, determine_best_banner(images)}])
+    |> Enum.filter(fn {_, _, tmp_filepath} -> tmp_filepath end)
+  end
+
+  defp determine_best_banner(images) do
+    best_candidate =
+      images
+      # Filter out images that don't have a width and height attribute
+      |> Enum.filter(&(&1["width"] && &1["height"]))
+      # Sort images with the highest width first
+      |> Enum.sort(&(&1["width"] >= &2["width"]))
+      # Find the first image where the ratio of width to height is greater than 3
+      |> Enum.find(&(&1["width"] / &1["height"] > 3))
+
+    Map.get(best_candidate || %{}, "filepath")
+  end
+
+  defp move_image({filename, source_attr_name, tmp_filepath}, base_directory) do
+    extension = Path.extname(tmp_filepath)
+    final_filepath = Path.join([base_directory, "#{filename}#{extension}"])
+
+    FilesystemHelpers.cp_p!(tmp_filepath, final_filepath)
+
+    {source_attr_name, final_filepath}
+  end
+end

lib/pinchflat/metadata/source_metadata.ex

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Metadata.SourceMetadata do
+  @moduledoc """
+  The SourceMetadata schema.
+
+  Look. Don't @ me about Metadata vs. Metadatum. I'm very sensitive.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Pinchflat.Sources.Source
+
+  @allowed_fields ~w(metadata_filepath)a
+  @required_fields ~w(metadata_filepath)a
+
+  schema "source_metadata" do
+    field :metadata_filepath, :string
+
+    belongs_to :source, Source
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(source_metadata, attrs) do
+    source_metadata
+    |> cast(attrs, @allowed_fields)
+    |> validate_required(@required_fields)
+    |> unique_constraint([:source_id])
+  end
+
+  @doc false
+  def filepath_attributes do
+    ~w(metadata_filepath)a
+  end
+end

lib/pinchflat/metadata/source_metadata_storage_worker.ex

@@ -0,0 +1,111 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :remote_metadata,
+    tags: ["media_source", "source_metadata", "remote_metadata"],
+    max_attempts: 3
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Repo
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.Utils.StringUtils
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceImageParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
+
+  @doc """
+  Starts the source metadata storage worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> SourceMetadataStorageWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches and stores various forms of metadata for a source:
+    - JSON metadata for internal use
+    - The series directory for the source
+    - The NFO file for the source (if specified)
+    - Downloads and stores source images (if specified)
+
+  The worker is kicked off after a source is inserted/updated - this can
+  take an unknown amount of time so don't rely on this data being here
+  before, say, the first indexing or downloading task is complete.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id}}) do
+    source = Repo.preload(Sources.get_source!(source_id), [:metadata, :media_profile])
+    series_directory = determine_series_directory(source)
+    {source_metadata, image_filepath_attrs} = fetch_source_metadata_and_images(series_directory, source)
+
+    # `run_post_commit_tasks: false` prevents this from running in an infinite loop
+    Sources.update_source(
+      source,
+      Map.merge(
+        %{
+          series_directory: series_directory,
+          nfo_filepath: store_source_nfo(source, series_directory, source_metadata),
+          metadata: %{
+            metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, source_metadata)
+          }
+        },
+        image_filepath_attrs
+      ),
+      run_post_commit_tasks: false
+    )
+
+    :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
+  end
+
+  defp fetch_source_metadata_and_images(series_directory, source) do
+    if source.media_profile.download_source_images && series_directory do
+      output_path = "#{tmp_directory()}/#{StringUtils.random_string(16)}/source_image.%(ext)S"
+      opts = [:write_all_thumbnails, convert_thumbnails: "jpg", output: output_path]
+
+      {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url, opts)
+      image_attrs = SourceImageParser.store_source_images(series_directory, metadata)
+
+      {metadata, image_attrs}
+    else
+      {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url, [])
+
+      {metadata, %{}}
+    end
+  end
+
+  defp determine_series_directory(source) do
+    output_path = DownloadOptionBuilder.build_output_path_for(source)
+    {:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, output: output_path)
+
+    case MetadataFileHelpers.series_directory_from_media_filepath(filepath) do
+      {:ok, series_directory} -> series_directory
+      {:error, _} -> nil
+    end
+  end
+
+  defp store_source_nfo(source, series_directory, metadata) do
+    if source.media_profile.download_nfo && series_directory do
+      nfo_filepath = Path.join(series_directory, "tvshow.nfo")
+
+      NfoBuilder.build_and_store_for_source(nfo_filepath, metadata)
+    end
+  end
+
+  defp tmp_directory do
+    Application.get_env(:pinchflat, :tmpfile_directory)
+  end
+end

lib/pinchflat/profiles/media_profile.ex

@@ -17,8 +17,10 @@ defmodule Pinchflat.Profiles.MediaProfile do
     sub_langs
     download_thumbnail
     embed_thumbnail
+    download_source_images
     download_metadata
     embed_metadata
+    download_nfo
     shorts_behaviour
     livestream_behaviour
     preferred_resolution
@@ -30,19 +32,21 @@ defmodule Pinchflat.Profiles.MediaProfile do
     field :name, :string
 
     field :output_path_template, :string,
-      default: "/{{ source_custom_name }}/{{ title }}/{{ title }} [{{ id }}].{{ ext }}"
+      default: "/{{ source_custom_name }}/{{ upload_yyyy_mm_dd }} {{ title }}/{{ title }} [{{ id }}].{{ ext }}"
 
-    field :download_subs, :boolean, default: true
+    field :download_subs, :boolean, default: false
-    field :download_auto_subs, :boolean, default: true
+    field :download_auto_subs, :boolean, default: false
-    field :embed_subs, :boolean, default: true
+    field :embed_subs, :boolean, default: false
     field :sub_langs, :string, default: "en"
 
-    field :download_thumbnail, :boolean, default: true
+    field :download_thumbnail, :boolean, default: false
-    field :embed_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: false
+    field :download_source_images, :boolean, default: false
 
-    field :download_metadata, :boolean, default: true
+    field :download_metadata, :boolean, default: false
-    field :embed_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: false
 
+    field :download_nfo, :boolean, default: false
     # NOTE: these do NOT speed up indexing - the indexer still has to go
     # through the entire collection to determine if a media is a short or
     # a livestream.
@@ -65,6 +69,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     media_profile
     |> cast(attrs, @allowed_fields)
     |> validate_required(@required_fields)
+    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
+    |> validate_format(:output_path_template, ext_regex(), message: "must end with .{{ ext }}")
     |> unique_constraint(:name)
   end
+
+  defp ext_regex do
+    ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
+  end
 end

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
+defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   @moduledoc """
   A GenServer that watches a file for new lines and processes them as they come in.
   This is useful for tailing log files and other similar tasks. If there's no activity

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,14 +6,26 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "media_collection_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+
+  @doc """
+  Starts the source slow indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> MediaCollectionIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   The ID is that of a source _record_, not a YouTube channel/playlist ID. Indexes
   the provided source, kicks off downloads for each new MediaItem, and
@@ -40,7 +52,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
        by the `download_media` field on the source as well as the profile's
        shorts/livestream behaviour. At this step we also attach a file reader
        to the `yt-dlp` output file so we can create media items as they come in
-       for a little speedup (see SourceTasks comments for more)
+       for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
     4. If this job is meant to reschedule (ie: has an index frequency > 0),
        it reschedules itself. If not, it runs once and does not reschedule
     5. If the source uses fast indexing, that job is kicked off as well. It
@@ -56,24 +68,23 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   NOTE: Since indexing can take a LONG time, I should check what happens if an
   application restart occurs while a job is running. Will the job be lost?
 
-  IDEA: Should I use paging and do indexing in chunks? Is that even faster?
-
   Returns :ok | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         :ok
 
       _ ->
@@ -81,6 +92,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
         # perform a no-op
         :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -1,9 +1,8 @@
-defmodule Pinchflat.Tasks.SourceTasks do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   @moduledoc """
-  Contains methods used by OR used to create/manage tasks for sources.
+  Methods for performing slow indexing tasks and managing the indexing process.
 
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
+  Many of these methods are made to be kickoff or be consumed by workers.
-  do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
   """
 
   require Logger
@@ -12,16 +11,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.YtDlp.MediaCollection
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.FileFollowerServer
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Starts tasks for indexing a source's media regardless of the source's indexing
@@ -34,56 +31,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
     Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
-    %{id: source.id}
+    MediaCollectionIndexingWorker.kickoff_with_task(source)
-    # Schedule this one immediately, but future ones will be on an interval
-    |> MediaCollectionIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
   end
 
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
   item belonging to the source. You can't tell me the method name isn't descriptive!
+  Returns a list of media items or changesets (if the media item couldn't be created).
 
   Indexing is slow and usually returns a list of all media data at once for record creation.
   To help with this, we use a file follower to watch the file that yt-dlp writes to
@@ -99,56 +54,27 @@ defmodule Pinchflat.Tasks.SourceTasks do
   Since indexing returns all media data EVERY TIME, we that that opportunity to update
   indexing metadata for media items that have already been created.
 
-  Returns [%MediaItem{}, ...]
+  Returns [%MediaItem{} | %Ecto.Changeset{}]
   """
   def index_and_enqueue_download_for_media_items(%Source{} = source) do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
-    result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
+
+    result =
+      Enum.map(media_attributes, fn media_attrs ->
+        case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+          {:ok, media_item} -> media_item
+          {:error, changeset} -> changeset
+        end
+      end)
 
     Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
-    enqueue_pending_media_tasks(source)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
 
     result
   end
 
-  @doc """
-  Starts tasks for downloading media for any of a sources _pending_ media items.
-  Jobs are not enqueued if the source is set to not download media. This will return :ok.
-
-  NOTE: this starts a download for each media item that is pending,
-  not just the ones that were indexed in this job run. This should ensure
-  that any stragglers are caught if, for some reason, they weren't enqueued
-  or somehow got de-queued.
-
-  Returns :ok
-  """
-  def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
-
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
-    :ok
-  end
-
-  @doc """
-  Deletes ALL pending tasks for a source's media items.
-
-  Returns :ok
-  """
-  def dequeue_pending_media_tasks(%Source{} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -191,27 +117,16 @@ defmodule Pinchflat.Tasks.SourceTasks do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
-    maybe_media_item = create_media_item_from_attributes(source, media_attrs)
+    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-
+      {:ok, %MediaItem{} = media_item} ->
-    case maybe_media_item do
-      %MediaItem{} = media_item ->
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
-          %{id: media_item.id}
+          MediaDownloadWorker.kickoff_with_task(media_item)
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
         end
 
-      changeset ->
+      {:error, changeset} ->
         changeset
     end
   end
-
-  defp create_media_item_from_attributes(source, media_attrs) do
-    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-      {:ok, media_item} -> media_item
-      {:error, changeset} -> changeset
-    end
-  end
 end

lib/pinchflat/sources/source.ex

@@ -10,12 +10,18 @@ defmodule Pinchflat.Sources.Source do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Metadata.SourceMetadata
 
   @allowed_fields ~w(
     collection_name
     collection_id
     collection_type
     custom_name
+    nfo_filepath
+    poster_filepath
+    fanart_filepath
+    banner_filepath
+    series_directory
     index_frequency_minutes
     fast_index
     download_media
@@ -28,7 +34,8 @@ defmodule Pinchflat.Sources.Source do
   # Expensive API calls are made when a source is inserted/updated so
   # we want to ensure that the source is valid before making the call.
   # This way, we check that the other attributes are valid before ensuring
-  # that all fields are valid.
+  # that all fields are valid. This is still only one DB insert but it's
+  # a two-stage validation process to fail fast before the API call.
   @initially_required_fields ~w(
     index_frequency_minutes
     fast_index
@@ -50,6 +57,11 @@ defmodule Pinchflat.Sources.Source do
     field :collection_name, :string
     field :collection_id, :string
     field :collection_type, Ecto.Enum, values: [:channel, :playlist]
+    field :nfo_filepath, :string
+    field :poster_filepath, :string
+    field :fanart_filepath, :string
+    field :banner_filepath, :string
+    field :series_directory, :string
     field :index_frequency_minutes, :integer, default: 60 * 24
     field :fast_index, :boolean, default: false
     field :download_media, :boolean, default: true
@@ -60,6 +72,8 @@ defmodule Pinchflat.Sources.Source do
 
     belongs_to :media_profile, MediaProfile
 
+    has_one :metadata, SourceMetadata, on_replace: :update
+
     has_many :tasks, Task
     has_many :media_items, MediaItem, foreign_key: :source_id
 
@@ -80,6 +94,7 @@ defmodule Pinchflat.Sources.Source do
     |> cast(attrs, @allowed_fields)
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> validate_required(required_fields)
+    |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id])
   end
 
@@ -94,4 +109,9 @@ defmodule Pinchflat.Sources.Source do
     # minutes
     15
   end
+
+  @doc false
+  def filepath_attributes do
+    ~w(nfo_filepath fanart_filepath poster_filepath banner_filepath)a
+  end
 end

lib/pinchflat/sources/sources.ex

@@ -9,9 +9,14 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -46,15 +51,19 @@ defmodule Pinchflat.Sources do
   though we know it's going to fail so it picks up any addl. database errors
   and fulfills our return contract.
 
+  You can pass options to control the behavior of the function:
+    - `run_post_commit_tasks` (default: true) - If false, the function will not
+      enqueue any tasks in `commit_and_handle_tasks`.
+
   Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}}
   """
-  def create_source(attrs) do
+  def create_source(attrs, opts \\ []) do
     case change_source(%Source{}, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         %Source{}
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
-        |> commit_and_handle_tasks()
+        |> commit_and_handle_tasks(opts)
 
       changeset ->
         Repo.insert(changeset)
@@ -67,22 +76,26 @@ defmodule Pinchflat.Sources do
   media if the indexing frequency has been changed.
 
   Existing indexing tasks will be cancelled if the indexing frequency has been
-  changed (logic in `SourceTasks.kickoff_indexing_task`)
+  changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
 
   Runs an initial `change_source` check to ensure most of the source is valid
   before making an expensive API call. Runs it through `Repo.update` even
   though we know it's going to fail so it picks up any addl. database errors
   and fulfills our return contract.
 
+  You can pass options to control the behavior of the function:
+    - `run_post_commit_tasks` (default: true) - If false, the function will not
+      enqueue any tasks in `commit_and_handle_tasks`.
+
   Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}}
   """
-  def update_source(%Source{} = source, attrs) do
+  def update_source(%Source{} = source, attrs, opts \\ []) do
     case change_source(source, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         source
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
-        |> commit_and_handle_tasks()
+        |> commit_and_handle_tasks(opts)
 
       changeset ->
         Repo.update(changeset)
@@ -97,6 +110,7 @@ defmodule Pinchflat.Sources do
   """
   def delete_source(%Source{} = source, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
+    Tasks.delete_tasks_for(source)
 
     source
     |> Media.list_media_items_for()
@@ -104,7 +118,11 @@ defmodule Pinchflat.Sources do
       Media.delete_media_item(media_item, delete_files: delete_files)
     end)
 
-    Tasks.delete_tasks_for(source)
+    if delete_files do
+      delete_source_files(source)
+    end
+
+    delete_internal_metadata_files(source)
     Repo.delete(source)
   end
 
@@ -115,19 +133,9 @@ defmodule Pinchflat.Sources do
     Source.changeset(source, attrs, validation_stage)
   end
 
-  @doc """
+  # NOTE: When operating in the ideal path, this effectively adds an API call
-  Returns an `%Ecto.Changeset{}` for tracking source changes and additionally
+  # to the source creation/update process. Should be used only when needed.
-  fetches source details from the original_url (if provided). If the source
+  defp maybe_change_source_from_url(%Source{} = source, attrs) do
-  details cannot be fetched, an error is added to the changeset.
-
-  Note that this fetches source details as long as the `original_url` is present.
-  This means that it'll go for it even if a changeset is otherwise invalid. This
-  is pretty easy to change, but for MVP I'm not concerned.
-
-  NOTE: When operating in the ideal path, this effectively adds an API call
-  to the source creation/update process. Should be used only when needed.
-  """
-  def change_source_from_url(%Source{} = source, attrs) do
     case change_source(source, attrs) do
       %Ecto.Changeset{changes: %{original_url: _}} = changeset ->
         add_source_details_to_changeset(source, changeset)
@@ -137,10 +145,27 @@ defmodule Pinchflat.Sources do
     end
   end
 
-  defp add_source_details_to_changeset(source, changeset) do
+  defp delete_source_files(source) do
-    %Ecto.Changeset{changes: changes} = changeset
+    mapped_struct = Map.from_struct(source)
 
-    case MediaCollection.get_source_details(changes.original_url) do
+    Source.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
+  defp delete_internal_metadata_files(source) do
+    metadata = Repo.preload(source, :metadata).metadata || %SourceMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    SourceMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
+  defp add_source_details_to_changeset(source, changeset) do
+    case MediaCollection.get_source_details(changeset.changes.original_url) do
       {:ok, source_details} ->
         add_source_details_by_collection_type(source, changeset, source_details)
 
@@ -189,11 +214,18 @@ defmodule Pinchflat.Sources do
     end
   end
 
-  defp commit_and_handle_tasks(changeset) do
+  defp commit_and_handle_tasks(changeset, opts) do
+    run_post_commit_tasks = Keyword.get(opts, :run_post_commit_tasks, true)
+
     case Repo.insert_or_update(changeset) do
       {:ok, %Source{} = source} ->
-        maybe_handle_media_tasks(changeset, source)
+        if run_post_commit_tasks do
-        maybe_run_indexing_task(changeset, source)
+          maybe_handle_media_tasks(changeset, source)
+          maybe_run_indexing_task(changeset, source)
+          run_metadata_storage_task(source)
+        end
+
+        {:ok, source}
 
       err ->
         err
@@ -205,23 +237,21 @@ defmodule Pinchflat.Sources do
   defp maybe_handle_media_tasks(changeset, source) do
     case {changeset.data, changeset.changes} do
       {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
-        SourceTasks.enqueue_pending_media_tasks(source)
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
-        SourceTasks.dequeue_pending_media_tasks(source)
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
         :ok
     end
-
-    {:ok, source}
   end
 
   defp maybe_run_indexing_task(changeset, source) do
     case changeset.data do
       # If the changeset is new (not persisted), attempt indexing no matter what
       %{__meta__: %{state: :built}} ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
@@ -229,14 +259,17 @@ defmodule Pinchflat.Sources do
         maybe_update_slow_indexing_task(changeset, source)
         maybe_update_fast_indexing_task(changeset, source)
     end
+  end
 
-    {:ok, source}
+  # This runs every time to pick up any changes to the metadata
+  defp run_metadata_storage_task(source) do
+    SourceMetadataStorageWorker.kickoff_with_task(source)
   end
 
   defp maybe_update_slow_indexing_task(changeset, source) do
     case changeset.changes do
       %{index_frequency_minutes: mins} when mins > 0 ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@@ -251,7 +284,8 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+        FastIndexingWorker.kickoff_with_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/media_item_tasks.ex

@@ -1,61 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasks do
-  @moduledoc """
-  Contains methods used by OR used to create/manage tasks for media items.
-
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
-  """
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Fetches the file size of a media item and saves it to the database.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def compute_and_save_media_filesize(media_item) do
-    case File.stat(media_item.media_filepath) do
-      {:ok, %{size: size}} ->
-        Media.update_media_item(media_item, %{media_size_bytes: size})
-
-      err ->
-        err
-    end
-  end
-
-  @doc """
-  Indexes a single media item for a source and enqueues a download job if the
-  media should be downloaded. This method creates the media item record so it's
-  the one-stop-shop for adding a media item (and possibly downloading it) just
-  by a URL and source.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
-    maybe_media_item = create_media_item_from_url(source, url)
-
-    case maybe_media_item do
-      {:ok, media_item} ->
-        if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
-        end
-
-        {:ok, media_item}
-
-      err ->
-        err
-    end
-  end
-
-  defp create_media_item_from_url(source, url) do
-    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
-
-    Media.create_media_item_from_backend_attrs(source, media_attrs)
-  end
-end

lib/pinchflat/tasks/tasks.ex

@@ -22,9 +22,15 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil, job_states \\ Oban.Job.states()) do
+  def list_tasks_for(record, worker_name \\ nil, job_states \\ Oban.Job.states()) do
     stringified_states = Enum.map(job_states, &to_string/1)
 
+    record_type =
+      case record do
+        %Source{} -> :source_id
+        %MediaItem{} -> :media_item_id
+      end
+
     worker_name_finder =
       if worker_name do
         # Workers are the full module name - we want to match on the string ENDING with
@@ -41,7 +47,7 @@ defmodule Pinchflat.Tasks do
     Repo.all(
       from t in Task,
         join: j in assoc(t, :job),
-        where: field(t, ^attached_record_type) == ^attached_record_id,
+        where: field(t, ^record_type) == ^record.id,
         where: ^worker_name_finder,
         where: j.state in ^stringified_states
     )
@@ -53,10 +59,9 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_pending_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil) do
+  def list_pending_tasks_for(record, worker_name \\ nil) do
     list_tasks_for(
-      attached_record_type,
+      record,
-      attached_record_id,
       worker_name,
       [:available, :scheduled, :retryable]
     )
@@ -126,14 +131,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_tasks_for(worker_name)
-        %Source{} = source -> list_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """
@@ -142,14 +143,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_pending_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_pending_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_pending_tasks_for(worker_name)
-        %Source{} = source -> list_pending_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_pending_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """

lib/pinchflat/utils/filesystem_utils.ex

@@ -1,23 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtils do
-  @moduledoc """
-  Utility methods for working with the filesystem
-  """
-
-  alias Pinchflat.Utils.StringUtils
-
-  @doc """
-  Generates a temporary file and returns its path. The file is empty and has the given type.
-  Generates all the directories in the path if they don't exist.
-
-  Returns binary()
-  """
-  def generate_metadata_tmpfile(type) do
-    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
-    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
-
-    :ok = File.mkdir_p!(Path.dirname(filepath))
-    :ok = File.write(filepath, "")
-
-    filepath
-  end
-end

lib/pinchflat/workers/filesystem_data_worker.ex

@@ -1,27 +0,0 @@
-defmodule Pinchflat.Workers.FilesystemDataWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :media_local_metadata,
-    tags: ["media_item", "media_metadata", "local_metadata"],
-    max_attempts: 1
-
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks.MediaItemTasks
-
-  @impl Oban.Worker
-  @doc """
-  For a given media item, compute and save metadata about the file on-disk.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
-
-    MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-    # Don't retry on failure - if it didn't work immediately there's no
-    # reason to believe it will work later.
-    :ok
-  end
-end

lib/pinchflat/yt_dlp/backend/media_collection.ex

@@ -1,81 +0,0 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollection do
-  @moduledoc """
-  Contains utilities for working with collections of
-  media (aka: a source [ie: channels, playlists]).
-  """
-
-  require Logger
-
-  alias Pinchflat.Utils.FunctionUtils
-  alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Returns a list of maps representing the media in the collection.
-
-  Options:
-    - :file_listener_handler - a function that will be called with the path to the
-      file that will be written to when yt-dlp is done. This is useful for
-      setting up a file watcher to know when the file is ready to be read.
-
-  Returns {:ok, [map()]} | {:error, any, ...}.
-  """
-  def get_media_attributes_for_collection(url, addl_opts \\ []) do
-    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
-    command_opts = [:simulate, :skip_download]
-    output_template = YtDlpMedia.indexing_output_template()
-    output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
-    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
-
-    if file_listener_handler do
-      file_listener_handler.(output_filepath)
-    end
-
-    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
-      {:ok, output} ->
-        output
-        |> String.split("\n", trim: true)
-        |> Enum.map(&Phoenix.json_library().decode!/1)
-        |> Enum.map(&YtDlpMedia.response_to_struct/1)
-        |> FunctionUtils.wrap_ok()
-
-      res ->
-        res
-    end
-  end
-
-  @doc """
-  Gets a source's ID and name from its URL.
-
-  yt-dlp does not _really_ have source-specific functions, so
-  instead we're fetching just the first video (using playlist_end: 1)
-  and parsing the source ID and name from _its_ metadata
-
-  Returns {:ok, map()} | {:error, any, ...}.
-  """
-  def get_source_details(source_url) do
-    opts = [:simulate, :skip_download, playlist_end: 1]
-    output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
-
-    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
-         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
-      {:ok, format_source_details(parsed_json)}
-    else
-      err -> err
-    end
-  end
-
-  defp format_source_details(response) do
-    %{
-      channel_id: response["channel_id"],
-      channel_name: response["channel"],
-      playlist_id: response["playlist_id"],
-      playlist_name: response["playlist_title"]
-    }
-  end
-
-  defp backend_runner do
-    # This approach lets us mock the command for testing
-    Application.get_env(:pinchflat, :yt_dlp_runner)
-  end
-end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.BackendCommandRunner do
   @moduledoc """
   A behaviour for running CLI commands against a downloader backend (yt-dlp).
 

lib/pinchflat/yt_dlp/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunner do
+defmodule Pinchflat.YtDlp.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   require Logger
 
   alias Pinchflat.Utils.StringUtils
-  alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
+  alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
-  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
+  alias Pinchflat.YtDlp.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 
@@ -37,7 +37,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
 
     case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
-        # IDEA: consider deleting the file after reading it
+        # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
+        # a huge deal, but it's still a good idea to clean up after ourselves.
         # (even on error? especially on error?)
         File.read(output_filepath)
 

lib/pinchflat/yt_dlp/download_option_builder.ex

@@ -1,131 +0,0 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
-  @moduledoc """
-  Builds the options for yt-dlp to download media based on the given media profile.
-  """
-
-  alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.OutputPathBuilder
-
-  @doc """
-  Builds the options for yt-dlp to download media based on the given media's profile.
-
-  IDEA: consider adding the ability to pass in a second argument to override
-        these options
-  """
-  def build(%MediaItem{} = media_item_with_preloads) do
-    media_profile = media_item_with_preloads.source.media_profile
-
-    built_options =
-      default_options() ++
-        subtitle_options(media_profile) ++
-        thumbnail_options(media_profile) ++
-        metadata_options(media_profile) ++
-        quality_options(media_profile) ++
-        output_options(media_item_with_preloads)
-
-    {:ok, built_options}
-  end
-
-  defp default_options do
-    [:no_progress, :windows_filenames]
-  end
-
-  defp subtitle_options(media_profile) do
-    mapped_struct = Map.from_struct(media_profile)
-
-    Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
-        {{:download_subs, true}, _} ->
-          # Force SRT for now - MAY provide as an option in the future
-          acc ++ [:write_subs, convert_subs: "srt"]
-
-        {{:download_auto_subs, true}, %{download_subs: true}} ->
-          acc ++ [:write_auto_subs]
-
-        {{:embed_subs, true}, %{preferred_resolution: pr}} when pr != :audio ->
-          acc ++ [:embed_subs]
-
-        {{:sub_langs, sub_langs}, %{download_subs: true}} ->
-          acc ++ [sub_langs: sub_langs]
-
-        {{:sub_langs, sub_langs}, %{embed_subs: true}} ->
-          acc ++ [sub_langs: sub_langs]
-
-        _ ->
-          acc
-      end
-    end)
-  end
-
-  defp thumbnail_options(media_profile) do
-    mapped_struct = Map.from_struct(media_profile)
-
-    Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
-        {{:download_thumbnail, true}, _} ->
-          acc ++ [:write_thumbnail]
-
-        {{:embed_thumbnail, true}, %{preferred_resolution: pr}} when pr != :audio ->
-          acc ++ [:embed_thumbnail]
-
-        _ ->
-          acc
-      end
-    end)
-  end
-
-  defp metadata_options(media_profile) do
-    mapped_struct = Map.from_struct(media_profile)
-
-    Enum.reduce(mapped_struct, [], fn attr, acc ->
-      case {attr, media_profile} do
-        {{:download_metadata, true}, _} ->
-          acc ++ [:write_info_json, :clean_info_json]
-
-        {{:embed_metadata, true}, %{preferred_resolution: pr}} when pr != :audio ->
-          acc ++ [:embed_metadata]
-
-        _ ->
-          acc
-      end
-    end)
-  end
-
-  defp quality_options(media_profile) do
-    codec_options = "+codec:avc:m4a"
-
-    case media_profile.preferred_resolution do
-      # Also be aware that :audio disabled all embedding options for thumbnails, subtitles, and metadata
-      :audio -> [format_sort: "ext", format: "bestaudio"]
-      :"360p" -> [format_sort: "res:360,#{codec_options}"]
-      :"480p" -> [format_sort: "res:480,#{codec_options}"]
-      :"720p" -> [format_sort: "res:720,#{codec_options}"]
-      :"1080p" -> [format_sort: "res:1080,#{codec_options}"]
-      :"1440p" -> [format_sort: "res:1440,#{codec_options}"]
-      :"2160p" -> [format_sort: "res:2160,#{codec_options}"]
-    end
-  end
-
-  defp output_options(media_item_with_preloads) do
-    media_profile = media_item_with_preloads.source.media_profile
-    additional_options_map = output_options_map(media_item_with_preloads)
-    {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
-
-    [
-      output: Path.join(base_directory(), output_path)
-    ]
-  end
-
-  defp output_options_map(media_item_with_preloads) do
-    source = media_item_with_preloads.source
-
-    %{
-      "source_custom_name" => source.custom_name,
-      "source_collection_type" => source.collection_type
-    }
-  end
-
-  defp base_directory do
-    Application.get_env(:pinchflat, :media_directory)
-  end
-end

lib/pinchflat/yt_dlp/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.Media do
+defmodule Pinchflat.YtDlp.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """
@@ -25,6 +25,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do
 
   alias __MODULE__
   alias Pinchflat.Utils.FunctionUtils
+  alias Pinchflat.Metadata.MetadataFileHelpers
 
   @doc """
   Downloads a single piece of media (and possibly its metadata) directly to its
@@ -85,8 +86,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       description: response["description"],
       original_url: response["webpage_url"],
       livestream: response["was_live"],
-      short_form_content: short_form_content?(response),
+      short_form_content: response["webpage_url"] && short_form_content?(response),
-      upload_date: parse_upload_date(response["upload_date"])
+      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
     }
   end
 
@@ -99,16 +100,13 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       # WILL returns false positives, but it's a best-effort approach
       # that should work for most cases. The aspect_ratio check is
       # based on a gut feeling and may need to be tweaked.
+      #
+      # These don't fail if duration or aspect_ratio are missing
+      # due to Elixir's comparison semantics
       response["duration"] <= 60 && response["aspect_ratio"] < 0.8
     end
   end
 
-  defp parse_upload_date(upload_date) do
-    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
-
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
-  end
-
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

lib/pinchflat/yt_dlp/media_collection.ex

@@ -0,0 +1,127 @@
+defmodule Pinchflat.YtDlp.MediaCollection do
+  @moduledoc """
+  Contains utilities for working with collections of
+  media (aka: a source [ie: channels, playlists]).
+  """
+
+  require Logger
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Returns a list of maps representing the media in the collection.
+
+  Options:
+    - :file_listener_handler - a function that will be called with the path to the
+      file that will be written to when yt-dlp is done. This is useful for
+      setting up a file watcher to know when the file is ready to be read.
+
+  Returns {:ok, [map()]} | {:error, any, ...}.
+  """
+  def get_media_attributes_for_collection(url, addl_opts \\ []) do
+    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the media details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error]
+    output_template = YtDlpMedia.indexing_output_template()
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
+
+    if file_listener_handler do
+      file_listener_handler.(output_filepath)
+    end
+
+    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
+      {:ok, output} ->
+        parsed_lines =
+          output
+          |> String.split("\n", trim: true)
+          |> Enum.map(fn line ->
+            case Phoenix.json_library().decode(line) do
+              {:ok, parsed_json} ->
+                YtDlpMedia.response_to_struct(parsed_json)
+
+              _ ->
+                nil
+            end
+          end)
+
+        {:ok, Enum.filter(parsed_lines, &(&1 != nil))}
+
+      res ->
+        res
+    end
+  end
+
+  @doc """
+  Gets a source's ID and name from its URL.
+
+  yt-dlp does not _really_ have source-specific functions that return what
+  we need, so instead we're fetching just the first video (using playlist_end: 1)
+  and parsing the source ID and name from _its_ metadata
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_details(source_url, addl_opts \\ []) do
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the source details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1] ++ addl_opts
+    output_template = "%(.{channel,channel_id,playlist_id,playlist_title,filename})j"
+
+    with {:ok, output} <- backend_runner().run(source_url, command_opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, format_source_details(parsed_json)}
+    else
+      err -> err
+    end
+  end
+
+  @doc """
+  Gets a source's metadata from its URL.
+
+  This is mostly for things like getting the source's avatar and banner image
+  (if applicable). However, this yt-dlp call doesn't have enough overlap with
+  `get_source_details/1` to allow combining them - this one has _almost_ everything
+  we need, but it doesn't contain enough information to tell 100% if the url is a channel
+  or a playlist.
+
+  The main purpose of this (past using as a fetcher for _other_ metadata) is to live
+  as a compressed blob for possible future use. That's why it's not getting formatted like
+  `get_source_details/1`
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_metadata(source_url, addl_opts \\ []) do
+    opts = [playlist_items: 0] ++ addl_opts
+    output_template = "playlist:%()j"
+
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, parsed_json}
+    else
+      err -> err
+    end
+  end
+
+  defp format_source_details(response) do
+    # NOTE: I should probably make this a struct some day
+    %{
+      channel_id: response["channel_id"],
+      channel_name: response["channel"],
+      playlist_id: response["playlist_id"],
+      playlist_name: response["playlist_title"],
+      # It's not a name, it's a path dammit!
+      # This actually isn't used for the inital response - it's
+      # used later to update a source's metadata
+      filepath: response["filename"]
+    }
+  end
+
+  defp backend_runner do
+    # This approach lets us mock the command for testing
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

lib/pinchflat_web/components/core_components.ex

@@ -317,17 +317,7 @@ defmodule PinchflatWeb.CoreComponents do
         <span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
       <div class="relative">
-        <input type="hidden" name={@name} value="false" />
+        <input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
-        <input
-          type="checkbox"
-          id={@id}
-          name={@name}
-          value="true"
-          x-bind:checked="enabled"
-          class="sr-only"
-          @change="enabled = !enabled"
-          {@rest}
-        />
         <div class="inline-block cursor-pointer" @click="enabled = !enabled">
           <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
           <div
@@ -349,23 +339,26 @@ defmodule PinchflatWeb.CoreComponents do
   def input(%{type: "select"} = assigns) do
     ~H"""
     <div phx-feedback-for={@name}>
-      <.label for={@id}>
+      <.label :if={@label} for={@id}>
         <%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
-      <select
+      <div class="flex">
-        id={@id}
+        <select
-        name={@name}
+          id={@id}
-        class={[
+          name={@name}
-          "relative z-20 w-full appearance-none rounded border border-stroke bg-transparent py-3 pl-5 pr-12 outline-none transition",
+          class={[
-          "focus:border-primary active:border-primary dark:border-form-strokedark dark:bg-form-input text-black dark:text-white",
+            "relative z-20 w-full appearance-none rounded border border-stroke bg-transparent py-3 pl-5 pr-12 outline-none transition",
-          @inputclass
+            "focus:border-primary active:border-primary dark:border-form-strokedark dark:bg-form-input text-black dark:text-white",
-        ]}
+            @inputclass
-        multiple={@multiple}
+          ]}
-        {@rest}
+          multiple={@multiple}
-      >
+          {@rest}
-        <option :if={@prompt} value=""><%= @prompt %></option>
+        >
-        <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
+          <option :if={@prompt} value=""><%= @prompt %></option>
-      </select>
+          <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
+        </select>
+        <%= render_slot(@inner_block) %>
+      </div>
       <.help :if={@help}><%= @help %></.help>
       <.error :for={msg <- @errors}><%= msg %></.error>
     </div>

lib/pinchflat_web/components/custom_components/button_components.ex

@@ -28,9 +28,10 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
         "#{@rounding} inline-flex items-center justify-center px-8 py-4",
         "#{@color}",
         "hover:bg-opacity-90 lg:px-8 xl:px-10",
-        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-gray-2",
+        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-grey-5",
         @class
       ]}
+      type={@type}
       disabled={@disabled}
       {@rest}
     >

lib/pinchflat_web/components/custom_components/text_components.ex

@@ -31,6 +31,20 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
     """
   end
 
+  @doc """
+  Renders a subtle link with the given href and content.
+  """
+  attr :href, :string, required: true
+  slot :inner_block
+
+  def subtle_link(assigns) do
+    ~H"""
+    <.link href={@href} class="underline decoration-bodydark decoration-1 hover:decoration-white">
+      <%= render_slot(@inner_block) %>
+    </.link>
+    """
+  end
+
   @doc """
   Renders an icon as a link with the given href.
   """

lib/pinchflat_web/components/layouts/root.html.heex

@@ -13,13 +13,17 @@
     </script>
   </head>
   <body
-    x-data={"{ sidebarVisible: false, proEnabled: #{Settings.get!(:pro_enabled)} }"}
+    x-data={"{ 
+      sidebarVisible: false,
+      proEnabled: #{Settings.get!(:pro_enabled)},
+      onboarding: #{Settings.get!(:onboarding)}
+    }"}
     class="dark text-bodydark bg-boxdark-2"
   >
     <%= @inner_content %>
 
     <.donate_modal conn={@conn} />
-    <template x-if="!proEnabled">
+    <template x-if="!proEnabled && !onboarding">
       <.upgrade_modal conn={@conn} />
     </template>
   </body>

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -30,7 +30,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this record and all associated files on disk? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -1,6 +1,8 @@
 defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   use PinchflatWeb, :html
 
+  alias Pinchflat.Profiles.MediaProfile
+
   embed_templates "media_profile_html/*"
 
   @doc """
@@ -35,6 +37,7 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       upload_day: nil,
       upload_month: nil,
       upload_year: nil,
+      upload_yyyy_mm_dd: "the upload date in the format YYYY-MM-DD",
       source_custom_name: "the name of the sources that use this profile",
       source_collection_type: "the collection type of the sources that use this profile. Either 'channel' or 'playlist'"
     }
@@ -52,4 +55,25 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       duration_string
     )a
   end
+
+  def preset_options do
+    [
+      {"Default", "default"},
+      {"Media Center (Plex, Jellyfin, Kodi, etc.)", "media_center"},
+      {"Music", "audio"},
+      {"Archiving", "archiving"}
+    ]
+  end
+
+  defp default_output_template do
+    %MediaProfile{}.output_path_template
+  end
+
+  defp media_center_output_template do
+    "/shows/{{ source_custom_name }}/Season {{ season_from_date }}/{{ season_episode_from_date }} - {{ title }}.{{ ext }}"
+  end
+
+  defp audio_output_template do
+    "/music/{{ source_custom_name }}/{{ title }}.{{ ext }}"
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/edit.html.heex

@@ -2,7 +2,9 @@
   <.link href={~p"/media_profiles"}>
     <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
   </.link>
-  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">Edit Media Profile</h2>
+  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
+    Editing "<%= @media_profile.name %>"
+  </h2>
 </div>
 
 <div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">

lib/pinchflat_web/controllers/media_profiles/media_profile_html/index.html.heex

@@ -4,7 +4,7 @@
   </h2>
   <nav>
     <.link href={~p"/media_profiles/new"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <span class="font-bold text-xl mx-2">+</span> New <span class="hidden sm:inline pl-1">Media Profile</span>
       </.button>
     </.link>
@@ -16,10 +16,9 @@
     <div class="flex flex-col gap-10 min-w-max">
       <.table rows={@media_profiles} table_class="text-black dark:text-white">
         <:col :let={media_profile} label="Name">
-          <%= media_profile.name %>
+          <.subtle_link href={~p"/media_profiles/#{media_profile.id}"}>
-        </:col>
+            <%= media_profile.name %>
-        <:col :let={media_profile} label="Output Template">
+          </.subtle_link>
-          <code class="text-sm"><%= media_profile.output_path_template %></code>
         </:col>
         <:col :let={media_profile} label="Preferred Resolution">
           <%= media_profile.preferred_resolution %>

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -3,116 +3,243 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <h3 class="my-4 text-2xl text-black dark:text-white">
+  <section x-data="{ selectedPreset: null }">
-    General Options
+    <h3 class="my-4 text-2xl text-black dark:text-white">
-  </h3>
+      Use a Preset
-  <.input
+    </h3>
-    field={f[:name]}
+    <section x-data="{ selection: null }">
-    type="text"
+      <.input
-    label="Name"
+        prompt="Select preset"
-    placeholder="New Profile"
+        name="media_profile_preset"
-    help="Something descriptive. Does not impact indexing or downloading (required)"
+        value=""
-  />
+        options={preset_options()}
+        type="select"
+        x-model="selection"
+        inputclass="w-full"
+        help="You can further customize the settings after selecting a preset. This is just a starting point"
+      >
+        <.button
+          class="h-13 w-2/5 lg:w-1/5 ml-2 md:ml-4"
+          rounding="rounded"
+          type="button"
+          x-on:click="selectedPreset = selection; selection = null"
+          x-bind:disabled="!selection"
+        >
+          <span x-text="selection ? 'Load' : 'Select'">Select</span><span class="hidden lg:inline ml-1">Preset</span>
+        </.button>
+      </.input>
+    </section>
 
-  <.input
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
-    field={f[:output_path_template]}
+      General Options
-    type="text"
+    </h3>
-    inputclass="font-mono"
-    label="Output path template"
-    help="Must end with .{{ ext }}. See below for more details. I promise the default is good for most cases (required)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ 
-    Subtitle Options
+        presets: { 
-  </h3>
+          default: 'Default',
-  <.input
+          media_center: 'TV Shows',
-    field={f[:download_subs]}
+          audio: 'Audio',
-    type="toggle"
+          archiving: 'Archiving'
-    label="Download Subtitles"
+        }
-    help="Downloads subtitle files alongside media file"
+    }">
-  />
+      <.input
-  <.input
+        field={f[:name]}
-    field={f[:download_auto_subs]}
+        type="text"
-    type="toggle"
+        label="Name"
-    label="Download Autogenerated Subtitles"
+        placeholder="New Profile"
-    help="Prefers normal subs but will download autogenerated if needed"
+        help="Something descriptive. Does not impact indexing or downloading (required)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
-  <.input
+      />
-    field={f[:embed_subs]}
+    </section>
-    type="toggle"
-    label="Embed Subtitles"
-    help="Embeds subtitles in the video file itself, if supported (recommended)"
-  />
-  <.input
-    field={f[:sub_langs]}
-    type="text"
-    label="Subtitle Languages"
-    help="Use commas for multiple languages (eg: en,de)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data={"{ 
-    Thumbnail Options
+        presets: { 
-  </h3>
+          default: '#{default_output_template()}',
-  <.input
+          media_center: '#{media_center_output_template()}',
-    field={f[:download_thumbnail]}
+          audio: '#{audio_output_template()}',
-    type="toggle"
+          archiving: '#{default_output_template()}'
-    label="Download Thumbnail"
+        }
-    help="Downloads thumbnail alongside media file"
+    }"}>
-  />
+      <.input
-  <.input
+        field={f[:output_path_template]}
-    field={f[:embed_thumbnail]}
+        type="text"
-    type="toggle"
+        inputclass="font-mono"
-    label="Embed Thumbnail"
+        label="Output path template"
-    help="Embeds thumbnail in the video file itself, if supported (recommended)"
+        help="Must end with .{{ ext }}. See below for more details. The default is good for most cases (required)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
-    Metadata Options
+      Subtitle Options
-  </h3>
+    </h3>
-  <.input
-    field={f[:download_metadata]}
-    type="toggle"
-    label="Download Metadata"
-    help="Downloads metadata file alongside media file"
-  />
-  <.input
-    field={f[:embed_metadata]}
-    type="toggle"
-    label="Embed Metadata"
-    help="Embeds metadata in the video file itself, if supported (recommended)"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
-    Release Format Options
+      <.input
-  </h3>
+        field={f[:download_subs]}
+        type="toggle"
+        label="Download Subtitles"
+        help="Downloads subtitle files alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
 
-  <.input
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: false } }">
-    field={f[:shorts_behaviour]}
+      <.input
-    options={friendly_format_type_options()}
+        field={f[:download_auto_subs]}
-    type="select"
+        type="toggle"
-    label="Include Shorts?"
+        label="Download Autogenerated Subtitles"
-    help="Experimental. Please report any issues on GitHub"
+        help="Prefers normal subs but will download autogenerated if needed. Requires 'Download Subtitles' to be enabled"
-  />
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
-  <.input
+      />
-    field={f[:livestream_behaviour]}
+    </section>
-    options={friendly_format_type_options()}
-    type="select"
-    label="Include Livestreams?"
-  />
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
-    Quality Options
+      <.input
-  </h3>
+        field={f[:embed_subs]}
+        type="toggle"
+        label="Embed Subtitles"
+        help="Downloads and embeds subtitles in the media file itself, if supported. Uneffected by 'Download Subtitles' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
 
-  <.input
+    <section x-data="{ presets: { default: 'en', media_center: 'en', audio: '', archiving: 'all' } }">
-    field={f[:preferred_resolution]}
+      <.input
-    options={friendly_resolution_options()}
+        field={f[:sub_langs]}
-    type="select"
+        type="text"
-    label="Preferred Resolution"
+        label="Subtitle Languages"
-    help="Will grab the closest available resolution if your preferred is not available. Setting to 'Audio Only' negates embedding options."
+        help="Use commas for multiple languages (eg: en,de)"
-  />
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
 
-  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Thumbnail Options
+    </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_thumbnail]}
+        type="toggle"
+        label="Download Thumbnail"
+        help="Downloads thumbnail alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
+      <.input
+        field={f[:embed_thumbnail]}
+        type="toggle"
+        label="Embed Thumbnail"
+        help="Downloads and embeds thumbnail in the media file itself, if supported. Uneffected by 'Download Thumbnail' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Metadata Options
+    </h3>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_metadata]}
+        type="toggle"
+        label="Download Metadata"
+        help="Downloads metadata file alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
+      <.input
+        field={f[:embed_metadata]}
+        type="toggle"
+        label="Embed Metadata"
+        help="Downloads and embeds metadata in the media file itself, if supported. Uneffected by 'Download Metadata' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Release Format Options
+    </h3>
+
+    <section x-data="{ presets: { default: 'include', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
+      <.input
+        field={f[:shorts_behaviour]}
+        options={friendly_format_type_options()}
+        type="select"
+        label="Include Shorts"
+        help="Experimental. Please report any issues on GitHub"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: 'exclude', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
+      <.input
+        field={f[:livestream_behaviour]}
+        options={friendly_format_type_options()}
+        type="select"
+        label="Include Livestreams"
+        help="Excludes media that comes from a past livestream"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
+      Quality Options
+    </h3>
+
+    <section x-data="{ presets: { default: '1080p', media_center: '1080p', audio: 'audio', archiving: '2160p' } }">
+      <.input
+        field={f[:preferred_resolution]}
+        options={friendly_resolution_options()}
+        type="select"
+        label="Preferred Resolution"
+        help="Will grab the closest available resolution if your preferred is not available. 'Audio Only' grabs the highest quality m4a"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
+      Media Center Options
+    </h3>
+    <p class="text-sm mt-2 max-w-prose">
+      Everything in this section is experimental - please open a GitHub issue if you see something odd.
+      <strong>
+        These options only work if this Media Profile's output template is set to split media into seasons.
+      </strong>
+      Try the "Media Center" preset if you're not sure.
+    </p>
+
+    <section
+      phx-click={show_modal("upgrade-modal")}
+      x-data="{ presets: { default: false, media_center: true, audio: false, archiving: false } }"
+    >
+      <.input
+        field={f[:download_nfo]}
+        type="toggle"
+        label="Download NFO data"
+        label_suffix="(pro)"
+        help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc."
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: false, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_source_images]}
+        type="toggle"
+        label="Download Series Images"
+        help="Downloads poster and banner images for use with Jellyfin, Kodi, etc. Only works for full channels (not playlists)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+  </section>
 
   <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
     <.output_template_help />

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -4,13 +4,13 @@
       <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
     </.link>
     <h2 class="text-title-md2 font-bold text-black dark:text-white ml-2">
-      Media Profile <span class="hidden sm:inline">#<%= @media_profile.id %></span>
+      <%= @media_profile.name %>
     </h2>
   </div>
 
   <nav>
     <.link href={~p"/media_profiles/#{@media_profile}/edit"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <.icon name="hero-pencil-square" class="mr-2" />Edit <span class="hidden sm:inline pl-1">Media Profile</span>
       </.button>
     </.link>
@@ -31,7 +31,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this profile and all its sources (leaving files in place)? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Profile and its Sources
             </.button>
           </.link>
@@ -41,7 +41,7 @@
             data-confirm="Are you sure you want to delete this profile, all its sources, and its files on disk? This cannot be undone."
             class="mt-5 md:mt-0"
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Profile, Sources, and Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/pages/page_controller.ex

@@ -8,38 +8,34 @@ defmodule PinchflatWeb.Pages.PageController do
   alias Pinchflat.Profiles.MediaProfile
 
   def home(conn, params) do
-    force_onboarding = params["onboarding"]
+    done_onboarding = params["onboarding"] == "0"
-    media_profiles_exist = Repo.exists?(MediaProfile)
+    force_onboarding = params["onboarding"] == "1"
-    sources_exist = Repo.exists?(Source)
 
-    if !force_onboarding && media_profiles_exist && sources_exist do
+    if done_onboarding, do: Settings.set!(:onboarding, false)
-      render_home_page(conn)
+
+    if force_onboarding || Settings.get!(:onboarding) do
+      render_onboarding_page(conn)
     else
-      render_onboarding_page(conn, media_profiles_exist, sources_exist)
+      render_home_page(conn)
     end
   end
 
   defp render_home_page(conn) do
-    Settings.set!(:onboarding, false)
-    media_profile_count = Repo.aggregate(MediaProfile, :count, :id)
-    source_count = Repo.aggregate(Source, :count, :id)
-    media_item_count = Repo.aggregate(MediaItem, :count, :id)
-
     conn
     |> render(:home,
-      media_profile_count: media_profile_count,
+      media_profile_count: Repo.aggregate(MediaProfile, :count, :id),
-      source_count: source_count,
+      source_count: Repo.aggregate(Source, :count, :id),
-      media_item_count: media_item_count
+      media_item_count: Repo.aggregate(MediaItem, :count, :id)
     )
   end
 
-  defp render_onboarding_page(conn, media_profiles_exist, sources_exist) do
+  defp render_onboarding_page(conn) do
     Settings.set!(:onboarding, true)
 
     conn
     |> render(:onboarding_checklist,
-      media_profiles_exist: media_profiles_exist,
+      media_profiles_exist: Repo.exists?(MediaProfile),
-      sources_exist: sources_exist,
+      sources_exist: Repo.exists?(Source),
       layout: {Layouts, :onboarding}
     )
   end

lib/pinchflat_web/controllers/pages/page_html/onboarding_checklist.html.heex

@@ -11,7 +11,7 @@
         <p class="text-md text-bodydark">Don't worry, you can create more Media Profiles later!</p>
         <div class="mt-8">
           <.link href={~p"/media_profiles/new"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={@media_profiles_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={@media_profiles_exist}>
               <span class="font-bold mx-2">+</span> New Media Profile
             </.button>
           </.link>
@@ -25,7 +25,7 @@
         </p>
         <div class="mt-8">
           <.link href={~p"/sources/new"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={not @media_profiles_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={not @media_profiles_exist}>
               <span class="font-bold mx-2">+</span> New Source
             </.button>
           </.link>
@@ -39,8 +39,8 @@
         </p>
         <p class="text-md text-bodydark">Feel free to add more Media Profiles or Sources in the meantime!</p>
         <div class="mt-8">
-          <.link href={~p"/"}>
+          <.link href={~p"/?onboarding=0"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={not @sources_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={not @sources_exist}>
               Let's Go <span class="font-bold mx-2">🚀</span>
             </.button>
           </.link>

lib/pinchflat_web/controllers/searches/search_html/show.html.heex

@@ -1,6 +1,6 @@
 <div class="mb-6 flex gap-3 flex-row items-center justify-between">
   <h2 class="text-title-md2 font-bold text-black dark:text-white">
-    <span class="hidden sm:inline">Search </span>Results for "<%= StringUtils.truncate(@search_term, 50) %>"
+    Results for "<%= StringUtils.truncate(@search_term, 50) %>"
   </h2>
 </div>
 
@@ -10,7 +10,7 @@
       <%= if match?([_|_], @search_results) do %>
         <.table rows={@search_results} table_class="text-black dark:text-white">
           <:col :let={result} label="Title">
-            <%= StringUtils.truncate(result.title, 40) %>
+            <%= result.title %>
           </:col>
           <:col :let={result} label="Excerpt">
             <.highlight_search_terms text={result.matching_search_term} />

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -46,7 +46,7 @@ defmodule PinchflatWeb.Sources.SourceController do
   def show(conn, %{"id" => id}) do
     source = Repo.preload(Sources.get_source!(id), :media_profile)
 
-    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(:source_id, source.id), :job)
+    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(source), :job)
     pending_media = Media.list_pending_media_items_for(source, limit: 100)
     downloaded_media = Media.list_downloaded_media_items_for(source, limit: 100)
 

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -14,7 +14,7 @@ defmodule PinchflatWeb.Sources.SourceHTML do
 
   def friendly_index_frequencies do
     [
-      {"On Create", -1},
+      {"Only once when first created", -1},
       {"1 Hour", 60},
       {"3 Hours", 3 * 60},
       {"6 Hours", 6 * 60},

lib/pinchflat_web/controllers/sources/source_html/edit.html.heex

@@ -2,7 +2,9 @@
   <.link href={~p"/sources"}>
     <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
   </.link>
-  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">Edit Source</h2>
+  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
+    Editing "<%= @source.custom_name %>"
+  </h2>
 </div>
 
 <div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">

lib/pinchflat_web/controllers/sources/source_html/index.html.heex

@@ -2,7 +2,7 @@
   <h2 class="text-title-md2 font-bold text-black dark:text-white">Sources</h2>
   <nav>
     <.link href={~p"/sources/new"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <span class="font-bold mx-2">+</span> New <span class="hidden sm:inline pl-1">Source</span>
       </.button>
     </.link>
@@ -14,19 +14,18 @@
     <div class="flex flex-col gap-10 min-w-max">
       <.table rows={@sources} table_class="text-black dark:text-white">
         <:col :let={source} label="Name">
-          <%= source.custom_name || source.collection_name %>
+          <.subtle_link href={~p"/sources/#{source.id}"}>
+            <%= source.custom_name || source.collection_name %>
+          </.subtle_link>
         </:col>
         <:col :let={source} label="Type"><%= source.collection_type %></:col>
         <:col :let={source} label="Should Download?">
           <.icon name={if source.download_media, do: "hero-check", else: "hero-x-mark"} />
         </:col>
         <:col :let={source} label="Media Profile">
-          <.link
+          <.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
-            href={~p"/media_profiles/#{source.media_profile_id}"}
-            class="hover:text-secondary duration-200 ease-in-out"
-          >
             <%= source.media_profile.name %>
-          </.link>
+          </.subtle_link>
         </:col>
         <:col :let={source} label="" class="flex place-content-evenly">
           <.icon_link href={~p"/sources/#{source.id}"} icon="hero-eye" class="mx-1" />

lib/pinchflat_web/controllers/sources/source_html/show.html.heex

@@ -4,13 +4,13 @@
       <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
     </.link>
     <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
-      Source <span class="hidden sm:inline">#<%= @source.id %></span>
+      <%= @source.custom_name %>
     </h2>
   </div>
 
   <nav>
     <.link href={~p"/sources/#{@source}/edit"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <.icon name="hero-pencil-square" class="mr-2" /> Edit <span class="hidden sm:inline pl-1">Source</span>
       </.button>
     </.link>
@@ -38,7 +38,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this source (leaving files in place)? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Source
             </.button>
           </.link>
@@ -48,7 +48,7 @@
             data-confirm="Are you sure you want to delete this source and it's files on disk? This cannot be undone."
             class="mt-5 md:mt-0"
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Source and Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -21,6 +21,7 @@
     options={Enum.map(@media_profiles, &{&1.name, &1.id})}
     type="select"
     label="Media Profile"
+    help="Sets your preferences for what media to look for and how to store it"
   />
 
   <h3 class="mt-8 text-2xl text-black dark:text-white">
@@ -32,7 +33,7 @@
     options={friendly_index_frequencies()}
     type="select"
     label="Index Frequency"
-    help="Time between one index of this source finishing and the next one starting. Setting to 'On Create' will still run an initial index but no subsequent ones"
+    help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
   />
 
   <%!-- TODO: use Alpine to disable the index frequency when fast indexing is enabled --%>
@@ -40,7 +41,7 @@
     <.input
       field={f[:fast_index]}
       type="toggle"
-      label="Use Fast Indexing?"
+      label="Use Fast Indexing"
       label_suffix="(pro)"
       help="Experimental. Ignores 'Index Frequency'. Recommended for large channels that upload frequently. See below for more info"
     />
@@ -53,7 +54,7 @@
   <.input
     field={f[:download_media]}
     type="toggle"
-    label="Download Media?"
+    label="Download Media"
     help="Unchecking still indexes media but it won't be downloaded until you enable this option"
   />
 
@@ -62,6 +63,9 @@
     type="text"
     label="Download Cutoff Date"
     placeholder="YYYY-MM-DD"
+    maxlength="10"
+    pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
+    title="YYYY-MM-DD"
     help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
   />
 

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.0-alpha.1",
+      version: "0.1.0-alpha.6",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -13,6 +13,16 @@ defmodule Pinchflat.MixProject do
       preferred_cli_env: [
         check: :test,
         credo: :test
+      ],
+      test_coverage: [
+        ignore_modules: [
+          Pinchflat.HTTP.HTTPClient,
+          PinchflatWeb.Layouts,
+          Pinchflat.DataCase,
+          Pinchflat.Release,
+          ~r/Fixtures/,
+          ~r/HTML$/
+        ]
       ]
     ]
   end
@@ -59,6 +69,7 @@ defmodule Pinchflat.MixProject do
       {:nimble_parsec, "~> 1.4"},
       {:mox, "~> 1.0", only: :test},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
+      {:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
       {:ex_check, "~> 0.14.0", only: [:dev, :test], runtime: false},
       {:faker, "~> 0.17", only: :test},
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false}

mix.lock

@@ -6,6 +6,7 @@
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
+  "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
   "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},

priv/repo/migrations/20240313175638_add_nfo_filepath_to_media_item.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddNfoFilepathToMediaItem do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :nfo_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240313175744_add_download_nfo_to_media_profile.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddDownloadNfoToMediaProfile do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_nfo, :boolean, default: false, null: false
+    end
+  end
+end

priv/repo/migrations/20240314174731_create_source_metadata.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.CreateSourceMetadata do
+  use Ecto.Migration
+
+  def change do
+    create table(:source_metadata) do
+      add :metadata_filepath, :string, null: false
+      add :source_id, references(:sources, on_delete: :delete_all), null: false
+
+      timestamps(type: :utc_datetime)
+    end
+
+    create unique_index(:source_metadata, [:source_id])
+  end
+end

priv/repo/migrations/20240318161405_add_nfo_path_to_sources.exs

@@ -0,0 +1,10 @@
+defmodule Pinchflat.Repo.Migrations.AddNfoPathToSources do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :nfo_filepath, :string
+      add :series_directory, :string
+    end
+  end
+end

priv/repo/migrations/20240319200823_add_source_photos_fields.exs

@@ -0,0 +1,15 @@
+defmodule Pinchflat.Repo.Migrations.AddSourcePhotosFields do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :fanart_filepath, :string
+      add :poster_filepath, :string
+      add :banner_filepath, :string
+    end
+
+    alter table(:media_profiles) do
+      add :download_source_images, :boolean, default: false, null: false
+    end
+  end
+end

selfhosted.Dockerfile

@@ -11,9 +11,9 @@
 #   - https://pkgs.org/ - resource for finding needed packages
 #   - Ex: hexpm/elixir:1.16.0-erlang-26.2.1-debian-bullseye-20231009-slim
 #
-ARG ELIXIR_VERSION=1.16.0
+ARG ELIXIR_VERSION=1.16.2
-ARG OTP_VERSION=26.2.1
+ARG OTP_VERSION=26.2.2
-ARG DEBIAN_VERSION=bullseye-20231009-slim
+ARG DEBIAN_VERSION=bookworm-20240130-slim
 
 ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
 ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
@@ -76,12 +76,13 @@ FROM ${RUNNER_IMAGE}
 
 RUN apt-get update -y
 RUN apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \
-  python3 python3-pip ffmpeg
+  ffmpeg curl git openssh-client
 RUN apt-get clean && rm -f /var/lib/apt/lists/*_*
 
-# Download YT-DLP
+# Download and update YT-DLP
-# NOTE: If you're seeing weird issues, consider using the FFMPEG released by yt-dlp
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
-RUN python3 -m pip install -U --pre yt-dlp
+RUN chmod a+rx /usr/local/bin/yt-dlp
+RUN yt-dlp -U
 
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
@@ -106,6 +107,17 @@ ENV RUN_CONTEXT="selfhosted"
 # Only copy the final release from the build stage
 COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./
 
+# NEVER do this if you're running in an environment where you don't trust the user
+# (ie: most environments). This is only acceptable in a self-hosted environment.
+# The user could just run the whole container as root and bypass this anyway so
+# it's not a huge deal.
+# This removes the root password to allow users to assume root if needed. This is
+# preferrable to running the whole container as root so that the files/directories
+# created by the app aren't owned by root and are therefore easier for other users
+# and processes to interact with. If you want to just run the whole container as
+# root, use --user 0:0 or something.
+RUN passwd -d root
+
 USER nobody
 
 # If using an environment that doesn't automatically reap zombie processes, it is

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.Workers.DataBackfillWorkerTest do
+defmodule Pinchflat.Boot.DataBackfillWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.DataBackfillWorker
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.JobFixtures.TestJobWorker
 
   describe "cancel_pending_backfill_jobs/0" do
     test "cancels all pending backfill jobs" do
@@ -21,14 +21,14 @@ defmodule Pinchflat.Workers.DataBackfillWorkerTest do
 
     test "does not cancel jobs for other workers" do
       %{id: 0}
-      |> FilesystemDataWorker.new()
+      |> TestJobWorker.new()
       |> Repo.insert_unique_job()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
 
       DataBackfillWorker.cancel_pending_backfill_jobs()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
     end
   end
 

test/pinchflat/boot/pre_job_startup_tasks_test.exs

@@ -0,0 +1,25 @@
+defmodule Pinchflat.Boot.PreJobStartupTasksTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Settings
+  alias Pinchflat.Settings.Setting
+  alias Pinchflat.Boot.PreJobStartupTasks
+
+  describe "apply_default_settings" do
+    setup do
+      Repo.delete_all(Setting)
+
+      :ok
+    end
+
+    test "sets default settings" do
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
+
+      PreJobStartupTasks.start_link()
+
+      assert Settings.get!(:onboarding)
+      refute Settings.get!(:pro_enabled)
+    end
+  end
+end

test/pinchflat/downloading/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})
@@ -125,6 +125,22 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :write_thumbnail in res
     end
 
+    test "appends -thumb to the thumbnail name when download_thumbnail is true", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:output, "thumbnail:/tmp/test/media/%(title)S-thumb.%(ext)s"} in res
+    end
+
+    test "converts thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:convert_thumbnail, "jpg"} in res
+    end
+
     test "includes :embed_thumbnail option when specified", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true})
 
@@ -133,12 +149,12 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :embed_thumbnail in res
     end
 
-    test "doesn't include :embed_thumbnail option when preferred_resolution is :audio", %{media_item: media_item} do
+    test "convertes thumbnail to jpg when embed_thumbnail is true", %{media_item: media_item} do
-      media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true, preferred_resolution: :audio})
+      media_item = update_media_profile_attribute(media_item, %{embed_thumbnail: true})
 
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
 
-      refute :embed_thumbnail in res
+      assert {:convert_thumbnail, "jpg"} in res
     end
 
     test "doesn't include these options when not specified", %{media_item: media_item} do
@@ -169,14 +185,6 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
       assert :embed_metadata in res
     end
 
-    test "doesn't include :embed_metadata option when preferred_resolution is :audio", %{media_item: media_item} do
-      media_item = update_media_profile_attribute(media_item, %{embed_metadata: true, preferred_resolution: :audio})
-
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
-
-      refute :embed_metadata in res
-    end
-
     test "doesn't include these options when not specified", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{embed_metadata: false, download_metadata: false})
 
@@ -189,12 +197,19 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
   end
 
   describe "build/1 when testing quality options" do
-    test "it includes quality options", %{media_item: media_item} do
+    test "it includes quality options" do
-      media_item = update_media_profile_attribute(media_item, %{preferred_resolution: :"1080p"})
+      resolutions = ["360", "480", "720", "1080", "2160"]
 
-      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+      Enum.each(resolutions, fn resolution ->
+        resolution_atom = String.to_existing_atom(resolution <> "p")
 
-      assert {:format_sort, "res:1080,+codec:avc:m4a"} in res
+        media_profile = media_profile_fixture(%{preferred_resolution: resolution_atom})
+        source = source_fixture(%{media_profile_id: media_profile.id})
+        media_item = Repo.preload(media_item_fixture(source_id: source.id), source: :media_profile)
+
+        assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+        assert {:format_sort, "res:#{resolution},+codec:avc:m4a"} in res
+      end)
     end
 
     test "it includes quality options for audio only", %{media_item: media_item} do
@@ -202,18 +217,26 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
 
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
 
-      assert {:format, "bestaudio"} in res
+      assert :extract_audio in res
-      assert {:format_sort, "ext"} in res
+      assert {:format, "bestaudio[ext=m4a]"} in res
+    end
+  end
+
+  describe "build_output_path_for/1" do
+    test "builds an output path for a source", %{media_item: media_item} do
+      path = DownloadOptionBuilder.build_output_path_for(media_item.source)
+
+      assert path == "/tmp/test/media/%(title)S.%(ext)s"
     end
   end
 
   defp update_media_profile_attribute(media_item_with_preloads, attrs) do
     media_item_with_preloads.source.media_profile
     |> Profiles.change_media_profile(attrs)
-    |> Repo.update!()
+    |> Repo.update()
 
     media_item_with_preloads
     |> Repo.reload()
-    |> Repo.preload(source: :media_profile)
+    |> Repo.preload([source: :media_profile], force: true)
   end
 end

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -0,0 +1,75 @@
+defmodule Pinchflat.Downloading.DownloadingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  setup :verify_on_exit!
+
+  describe "enqueue_pending_download_tasks/1" do
+    test "it enqueues a job for each pending media item" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "it does not enqueue a job for media items with a filepath" do
+      source = source_fixture()
+      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it attaches a task to each enqueued job" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert [] = Tasks.list_tasks_for(media_item)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert [_] = Tasks.list_tasks_for(media_item)
+    end
+
+    test "it does not create a job if the source is set to not download" do
+      source = source_fixture(download_media: false)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it does not attach tasks if the source is set to not download" do
+      source = source_fixture(download_media: false)
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert [] = Tasks.list_tasks_for(media_item)
+    end
+  end
+
+  describe "dequeue_pending_download_tasks/1" do
+    test "it deletes all pending tasks for a source's media items" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+
+      assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+      assert [] = Tasks.list_tasks_for(media_item)
+    end
+  end
+end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -1,29 +1,41 @@
-defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
+defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   setup :verify_on_exit!
 
   setup do
-    media_item =
-      Repo.preload(
-        media_item_fixture(%{media_filepath: nil}),
-        [:metadata, source: :media_profile]
-      )
-
     stub(HTTPClientMock, :get, fn _url, _headers, _opts ->
       {:ok, ""}
     end)
 
+    media_item =
+      %{media_filepath: nil}
+      |> media_item_fixture()
+      |> Repo.preload([:metadata, source: :media_profile])
+
     {:ok, %{media_item: media_item}}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{media_item: media_item} do
+      assert [] = all_enqueued(worker: MediaDownloadWorker)
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "attaches a task", %{media_item: media_item} do
+      assert {:ok, task} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert task.media_item_id == media_item.id
+    end
+  end
+
   describe "perform/1" do
     test "it saves attributes to the media_item", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -70,16 +82,22 @@ defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
-    test "it schedules a filesystem data worker", %{media_item: media_item} do
+    test "it saves the file's size to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, render_metadata(:media_metadata)}
+        metadata = render_parsed_metadata(:media_metadata)
+        FilesystemHelpers.write_p!(metadata["filepath"], "test")
+
+        {:ok, Phoenix.json_library().encode!(metadata)}
       end)
 
-      assert [] = all_enqueued(worker: FilesystemDataWorker)
-
       perform_job(MediaDownloadWorker, %{id: media_item.id})
+      media_item = Repo.reload(media_item)
 
-      assert [_] = all_enqueued(worker: FilesystemDataWorker)
+      assert media_item.media_size_bytes > 0
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0})
     end
   end
 end

test/pinchflat/downloading/media_downloader_test.exs

@@ -1,9 +1,11 @@
-defmodule Pinchflat.MediaClient.MediaDownloaderTest do
+defmodule Pinchflat.Downloading.MediaDownloaderTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
 
-  alias Pinchflat.MediaClient.MediaDownloader
+  alias Pinchflat.Downloading.MediaDownloader
 
   setup :verify_on_exit!
 
@@ -103,4 +105,37 @@ defmodule Pinchflat.MediaClient.MediaDownloaderTest do
       assert String.ends_with?(updated_media_item.metadata_filepath, ".info.json")
     end
   end
+
+  describe "download_for_media_item/3 when testing NFO generation" do
+    setup do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      :ok
+    end
+
+    test "it generates an NFO file if the source is set to download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: true})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert String.ends_with?(updated_media_item.nfo_filepath, ".nfo")
+      assert File.exists?(updated_media_item.nfo_filepath)
+
+      File.rm!(updated_media_item.nfo_filepath)
+    end
+
+    test "it does not generate an NFO file if the source is set to not download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: false})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert updated_media_item.nfo_filepath == nil
+    end
+  end
 end

test/pinchflat/downloading/output_path/parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.RenderedString.ParserTest do
+defmodule Pinchflat.Downloading.OutputPath.ParserTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.RenderedString.Parser
+  alias Pinchflat.Downloading.OutputPath.Parser
 
   describe "parse/3" do
     test "it returns the rendered string when the string is valid" do

test/pinchflat/downloading/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.Profiles.OutputPathBuilderTest do
+defmodule Pinchflat.Downloading.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -0,0 +1,111 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  setup :verify_on_exit!
+
+  @media_url "https://www.youtube.com/watch?v=test_1"
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+  end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert Repo.aggregate(MediaItem, :count) == 1
+      assert mi_1.id == mi_2.id
+    end
+
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "creates a download task record", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
+    end
+
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
+
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
+      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
+      source = source_fixture(%{media_profile_id: profile.id})
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        output =
+          Phoenix.json_library().encode!(%{
+            id: "video2",
+            title: "Video 2",
+            webpage_url: "https://example.com/shorts/video2",
+            was_live: true,
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67,
+            upload_date: "20210101"
+          })
+
+        {:ok, output}
+      end)
+
+      assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+  end
+end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -0,0 +1,77 @@
+defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+
+  setup :verify_on_exit!
+
+  describe "kickoff_with_task/2" do
+    test "starts the worker" do
+      source = source_fixture(fast_index: true)
+
+      assert [] = all_enqueued(worker: FastIndexingWorker)
+      assert {:ok, _} = FastIndexingWorker.kickoff_with_task(source)
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "attaches a task" do
+      source = source_fixture(fast_index: true)
+
+      assert {:ok, task} = FastIndexingWorker.kickoff_with_task(source)
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "calls out to Youtube RSS if enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+    end
+
+    test "reschedules itself if fast indexing is enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      assert_enqueued(
+        worker: FastIndexingWorker,
+        args: %{"id" => source.id},
+        scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
+      )
+    end
+
+    test "does not reschedule if that would create a duplicate job" do
+      stub(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "does not call out to Youtube RSS if disabled" do
+      expect(HTTPClientMock, :get, 0, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: false)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+    end
+
+    test "does not reschedule itself if fast indexing is disabled" do
+      source = source_fixture(fast_index: false)
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      refute_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(FastIndexingWorker, %{id: 0})
+    end
+  end
+end

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -6,8 +6,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
 
   @media_url "https://www.youtube.com/watch?v=1234567890"
 
@@ -19,6 +19,19 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
     {:ok, source: source}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: MediaIndexingWorker)
+      assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert [_] = all_enqueued(worker: MediaIndexingWorker)
+    end
+
+    test "attaches a task", %{source: source} do
+      assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert task.source_id == source.id
+    end
+  end
+
   describe "perform/1" do
     test "indexes the media item and saves it to the database", %{source: source} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -40,5 +53,9 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
 
       assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
+    end
   end
 end

test/pinchflat/fast_indexing/youtube_rss_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Api.YoutubeRssTest do
+defmodule Pinchflat.FastIndexing.YoutubeRssTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
 
   setup :verify_on_exit!
 

test/pinchflat/filesystem/filesystem_helpers_test.exs

@@ -0,0 +1,143 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  describe "generate_metadata_tmpfile/1" do
+    test "creates a tmpfile and returns its path" do
+      res = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert String.ends_with?(res, ".json")
+      assert File.exists?(res)
+
+      File.rm!(res)
+    end
+  end
+
+  describe "compute_and_save_media_filesize/1" do
+    test "updates the media item with the file size" do
+      media_item = media_item_with_attachments()
+
+      refute media_item.media_size_bytes
+
+      assert {:ok, media_item} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+
+      assert Repo.reload!(media_item).media_size_bytes
+    end
+
+    test "returns the error if operation fails" do
+      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
+
+      assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+    end
+  end
+
+  describe "write_p!/3" do
+    test "writes content to a file" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+
+    test "creates directories as needed" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "file.json"])
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+  end
+
+  describe "delete_file_and_remove_empty_directories/1" do
+    test "deletes file at the provided filepath" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert File.exists?(filepath)
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+    end
+
+    test "deletes empty directories" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      FilesystemHelpers.write_p!(filepath, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo"]))
+    end
+
+    test "does not delete directories with other files in them" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath_1 = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      filepath_2 = Path.join([tmpfile_directory, "foo", "baz.json"])
+      FilesystemHelpers.write_p!(filepath_1, "")
+      FilesystemHelpers.write_p!(filepath_2, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_1)
+
+      refute File.exists?(filepath_1)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+
+      assert File.exists?(filepath_2)
+      assert File.exists?(Path.join([tmpfile_directory, "foo"]))
+
+      # cleanup
+      FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_2)
+    end
+
+    test "returns an error if file could not be deleted" do
+      filepath = "/nonexistent/file.json"
+
+      assert {:error, _} = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+    end
+  end
+
+  describe "cp_p!/2" do
+    test "copies a file from source to destination" do
+      source = "#{tmpfile_directory()}/source.json"
+      FilesystemHelpers.write_p!(source, "TEST")
+      destination = "#{tmpfile_directory()}/destination.json"
+
+      refute File.exists?(destination)
+      FilesystemHelpers.cp_p!(source, destination)
+      assert File.exists?(destination)
+      assert File.read!(destination) == "TEST"
+
+      File.rm!(source)
+      File.rm!(destination)
+    end
+
+    test "creates directories as needed" do
+      source = "#{tmpfile_directory()}/source.json"
+      FilesystemHelpers.write_p!(source, "TEST")
+      destination = "#{tmpfile_directory()}/foo/bar/destination.json"
+
+      refute File.exists?(destination)
+      FilesystemHelpers.cp_p!(source, destination)
+      assert File.exists?(destination)
+
+      File.rm!(source)
+      File.rm!(destination)
+    end
+  end
+
+  defp tmpfile_directory do
+    Application.get_env(:pinchflat, :tmpfile_directory)
+  end
+end

test/pinchflat/media_test.exs

@@ -11,7 +11,7 @@ defmodule Pinchflat.MediaTest do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.MetadataFileHelpers
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   setup :verify_on_exit!
 
@@ -360,57 +360,6 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
-  describe "media_filepaths/1" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: "/video/test.jpg",
-        subtitle_filepaths: [["en", "video/test.srt"]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == [
-               "/video/test.mp4",
-               "/video/test.jpg",
-               "video/test.srt"
-             ]
-    end
-
-    test "strips out nil values" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: nil,
-        subtitle_filepaths: [["en", nil]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == ["/video/test.mp4"]
-    end
-  end
-
-  describe "metadata_filepaths" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        metadata_filepath: "/metadata.json.gz",
-        thumbnail_filepath: "/thumbnail.jpg"
-      }
-
-      media_item = media_item_fixture(%{metadata: filepaths})
-
-      assert Media.metadata_filepaths(media_item) == [
-               "/metadata.json.gz",
-               "/thumbnail.jpg"
-             ]
-    end
-
-    test "returns an empty list when there is no metadata" do
-      media_item = media_item_fixture()
-      assert Media.metadata_filepaths(media_item) == []
-    end
-  end
-
   describe "create_media_item/1" do
     test "creating with valid data creates a media_item" do
       valid_attrs = %{
@@ -515,6 +464,26 @@ defmodule Pinchflat.MediaTest do
       assert {:ok, _} = Media.delete_media_item(media_item)
       assert File.exists?(media_item.media_filepath)
     end
+
+    test "does delete the media item's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      media_item = Repo.preload(media_item_with_attachments(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
+          thumbnail_filepath:
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
+        }
+      }
+
+      {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs)
+
+      assert {:ok, _} = Media.delete_media_item(updated_media_item)
+      refute File.exists?(updated_media_item.metadata.metadata_filepath)
+    end
   end
 
   describe "delete_media_item/2 when testing file deletion" do

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -84,4 +84,60 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
       assert Path.basename(filepath) == "maxres.webp"
     end
   end
+
+  describe "parse_upload_date/1" do
+    test "returns a date from the given metadata upload date" do
+      upload_date = "20210101"
+
+      assert Helpers.parse_upload_date(upload_date) == ~D[2021-01-01]
+    end
+  end
+
+  describe "series_directory_from_media_filepath/1" do
+    test "returns base series directory if filepaths are setup as expected" do
+      good_filepaths = [
+        "/media/season1/episode.mp4",
+        "/media/season 1/episode.mp4",
+        "/media/season.1/episode.mp4",
+        "/media/season_1/episode.mp4",
+        "/media/season-1/episode.mp4",
+        "/media/SEASON 1/episode.mp4",
+        "/media/SEASON.1/episode.mp4",
+        "/media/s1/episode.mp4",
+        "/media/s.1/episode.mp4",
+        "/media/s_1/episode.mp4",
+        "/media/s-1/episode.mp4",
+        "/media/s 1/episode.mp4",
+        "/media/S1/episode.mp4",
+        "/media/S.1/episode.mp4"
+      ]
+
+      for filepath <- good_filepaths do
+        assert {:ok, "/media"} = Helpers.series_directory_from_media_filepath(filepath)
+      end
+    end
+
+    test "returns an error if the season filepath can't be determined" do
+      bad_filepaths = [
+        "/media/1/episode.mp4",
+        "/media/(s1)/episode.mp4",
+        "/media/episode.mp4",
+        "/media/s1e1/episode.mp4",
+        "/media/s1 e1/episode.mp4",
+        "/media/s1 (something else)/episode.mp4",
+        "/media/season1e1/episode.mp4",
+        "/media/season1 e1/episode.mp4",
+        "/media/seasoning1/episode.mp4",
+        "/media/season/episode.mp4",
+        "/media/series1/episode.mp4",
+        "/media/s/episode.mp4",
+        "/media/foo",
+        "/media/bar/"
+      ]
+
+      for filepath <- bad_filepaths do
+        assert {:error, :indeterminable} = Helpers.series_directory_from_media_filepath(filepath)
+      end
+    end
+  end
 end

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,25 +1,10 @@
-defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
+defmodule Pinchflat.Metadata.MetadataParserTest do
   use Pinchflat.DataCase
 
   alias Pinchflat.Metadata.MetadataParser, as: Parser
 
   setup do
-    json_filepath =
+    {:ok, %{metadata: render_parsed_metadata(:media_metadata)}}
-      Path.join([
-        File.cwd!(),
-        "test",
-        "support",
-        "files",
-        "media_metadata.json"
-      ])
-
-    {:ok, file_body} = File.read(json_filepath)
-    {:ok, parsed_json} = Phoenix.json_library().decode(file_body)
-
-    {:ok,
-     %{
-       metadata: parsed_json
-     }}
   end
 
   describe "parse_for_media_item/1 when testing media metadata" do
@@ -109,6 +94,15 @@ defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
       assert String.ends_with?(result.thumbnail_filepath, ".webp")
     end
 
+    # NOTE: this can be removed once this bug is fixed
+    # https://github.com/yt-dlp/yt-dlp/issues/9445
+    # and the associated conditional in the parser is removed
+    test "automatically appends `-thumb` to the thumbnail filename", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert String.contains?(result.thumbnail_filepath, "-thumb.webp")
+    end
+
     test "doesn't freak out if the media has no thumbnails", %{metadata: metadata} do
       metadata = Map.put(metadata, "thumbnails", %{})
 

test/pinchflat/metadata/nfo_builder_test.exs

@@ -0,0 +1,50 @@
+defmodule Pinchflat.Metadata.NfoBuilderTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  setup do
+    filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+    on_exit(fn -> File.rm!(filepath) end)
+
+    {:ok,
+     %{
+       metadata: render_parsed_metadata(:media_metadata),
+       filepath: filepath
+     }}
+  end
+
+  describe "build_and_store_for_media_item/2" do
+    test "returns the filepath", %{metadata: metadata, filepath: filepath} do
+      result = NfoBuilder.build_and_store_for_media_item(filepath, metadata)
+
+      assert File.exists?(result)
+    end
+
+    test "builds an NFO file", %{metadata: metadata, filepath: filepath} do
+      result = NfoBuilder.build_and_store_for_media_item(filepath, metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
+      assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
+    end
+  end
+
+  describe "build_and_store_for_source/2" do
+    test "returns the filepath", %{metadata: metadata, filepath: filepath} do
+      result = NfoBuilder.build_and_store_for_source(filepath, metadata)
+
+      assert File.exists?(result)
+    end
+
+    test "builds an NFO file", %{metadata: metadata, filepath: filepath} do
+      result = NfoBuilder.build_and_store_for_source(filepath, metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
+      assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
+    end
+  end
+end

test/pinchflat/metadata/source_image_parser_test.exs

@@ -0,0 +1,78 @@
+defmodule Pinchflat.Metadata.SourceImageParserTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Metadata.SourceImageParser
+
+  @base_dir Application.compile_env(:pinchflat, :tmpfile_directory)
+
+  describe "store_source_images/2" do
+    test "returns a map of image types and locations" do
+      metadata = render_parsed_metadata(:channel_source_metadata)
+
+      expected = %{
+        banner_filepath: "#{@base_dir}/banner.jpg",
+        fanart_filepath: "#{@base_dir}/fanart.jpg",
+        poster_filepath: "#{@base_dir}/poster.jpg"
+      }
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == expected
+    end
+
+    test "returns the avatar_uncropped as the poster" do
+      metadata = %{
+        "thumbnails" => [
+          %{"id" => "avatar_uncropped", "filepath" => "/app/test/support/files/channel_photos/a.0.jpg"}
+        ]
+      }
+
+      expected = %{
+        poster_filepath: "#{@base_dir}/poster.jpg"
+      }
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == expected
+    end
+
+    test "returns the banner_uncropped as the fanart" do
+      metadata = %{
+        "thumbnails" => [
+          %{"id" => "banner_uncropped", "filepath" => "/app/test/support/files/channel_photos/a.0.jpg"}
+        ]
+      }
+
+      expected = %{
+        fanart_filepath: "#{@base_dir}/fanart.jpg"
+      }
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == expected
+    end
+
+    test "doesn't return a banner if no suitable images are found" do
+      metadata = %{
+        "thumbnails" => [
+          %{
+            "id" => "1",
+            "filepath" => "foo.jpg",
+            "width" => 100,
+            "height" => 100
+          }
+        ]
+      }
+
+      expected = %{}
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == expected
+    end
+
+    test "doesn't blow up if empty metadata is passed" do
+      metadata = %{}
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == %{}
+    end
+
+    test "doesn't blow up if no thumbnails are passed" do
+      metadata = %{"thumbnails" => []}
+
+      assert SourceImageParser.store_source_images(@base_dir, metadata) == %{}
+    end
+  end
+end

test/pinchflat/metadata/source_metadata_storage_worker_test.exs

@@ -0,0 +1,262 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Sources
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
+
+  @source_details_ot "%(.{channel,channel_id,playlist_id,playlist_title,filename})j"
+  @metadata_ot "playlist:%()j"
+
+  setup :verify_on_exit!
+
+  describe "kickoff_with_task/1" do
+    test "enqueues a new worker for the source" do
+      source = source_fixture()
+
+      assert {:ok, _} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
+
+    test "creates a new task for the source" do
+      source = source_fixture()
+
+      assert {:ok, task} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "won't call itself in an infinite loop" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()}
+        _url, _opts, ot when ot == @metadata_ot -> {:ok, "{}"}
+      end)
+
+      source = source_fixture()
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+
+      assert [] = all_enqueued(worker: SourceMetadataStorageWorker)
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(SourceMetadataStorageWorker, %{id: 0})
+    end
+  end
+
+  describe "perform/1 when testing metadata storage" do
+    test "sets metadata location for source" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()}
+        _url, _opts, ot when ot == @metadata_ot -> {:ok, "{}"}
+      end)
+
+      source = Repo.preload(source_fixture(), :metadata)
+
+      refute source.metadata
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+
+      assert source.metadata.metadata_filepath
+
+      File.rm!(source.metadata.metadata_filepath)
+    end
+
+    test "fetches and stores returned metadata for source" do
+      source = source_fixture()
+      file_contents = Phoenix.json_library().encode!(%{"title" => "test"})
+
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot -> {:ok, source_details_return_fixture()}
+        _url, _opts, ot when ot == @metadata_ot -> {:ok, file_contents}
+      end)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+      {:ok, metadata} = MetadataFileHelpers.read_compressed_metadata(source.metadata.metadata_filepath)
+
+      assert metadata == %{"title" => "test"}
+    end
+  end
+
+  describe "perform/1 when testing source image downloading" do
+    test "downloads and stores source images" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, render_metadata(:channel_source_metadata)}
+      end)
+
+      profile = media_profile_fixture(%{download_source_images: true})
+      source = source_fixture(media_profile_id: profile.id)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      assert source.fanart_filepath
+      assert source.poster_filepath
+      assert source.banner_filepath
+
+      assert File.exists?(source.fanart_filepath)
+      assert File.exists?(source.poster_filepath)
+      assert File.exists?(source.banner_filepath)
+
+      Sources.delete_source(source, delete_files: true)
+    end
+
+    test "does not store source images if the profile is not set to" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, render_metadata(:channel_source_metadata)}
+      end)
+
+      profile = media_profile_fixture(%{download_source_images: false})
+      source = source_fixture(media_profile_id: profile.id)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      refute source.fanart_filepath
+      refute source.poster_filepath
+      refute source.banner_filepath
+    end
+
+    test "does not store source images if the series directory cannot be determined" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, render_metadata(:channel_source_metadata)}
+      end)
+
+      profile = media_profile_fixture(%{download_source_images: true})
+      source = source_fixture(media_profile_id: profile.id)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      refute source.fanart_filepath
+      refute source.poster_filepath
+      refute source.banner_filepath
+    end
+  end
+
+  describe "perform/1 when determining the series_directory" do
+    test "sets the series directory based on the returned media filepath" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, "{}"}
+      end)
+
+      source = source_fixture(%{series_directory: nil})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      assert source.series_directory
+    end
+
+    test "does not set the series directory if it cannot be determined" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, "{}"}
+      end)
+
+      source = source_fixture(%{series_directory: nil})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      refute source.series_directory
+    end
+  end
+
+  describe "perform/1 when storing the series NFO" do
+    test "stores the NFO if specified" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, "{}"}
+      end)
+
+      profile = media_profile_fixture(%{download_nfo: true})
+      source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      assert source.nfo_filepath
+      assert source.nfo_filepath == Path.join([source.series_directory, "tvshow.nfo"])
+      assert File.exists?(source.nfo_filepath)
+
+      File.rm!(source.nfo_filepath)
+    end
+
+    test "does not store the NFO if not specified" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "Season 1", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, "{}"}
+      end)
+
+      profile = media_profile_fixture(%{download_nfo: false})
+      source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      refute source.nfo_filepath
+    end
+
+    test "does not store the NFO if the series directory cannot be determined" do
+      stub(YtDlpRunnerMock, :run, fn
+        _url, _opts, ot when ot == @source_details_ot ->
+          filename = Path.join([Application.get_env(:pinchflat, :media_directory), "foo", "bar.mp4"])
+
+          {:ok, source_details_return_fixture(%{filename: filename})}
+
+        _url, _opts, ot when ot == @metadata_ot ->
+          {:ok, "{}"}
+      end)
+
+      profile = media_profile_fixture(%{download_nfo: true})
+      source = source_fixture(%{nfo_filepath: nil, media_profile_id: profile.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.reload(source)
+
+      refute source.nfo_filepath
+    end
+  end
+end

test/pinchflat/profiles_test.exs

@@ -26,11 +26,11 @@ defmodule Pinchflat.ProfilesTest do
 
   describe "create_media_profile/1" do
     test "creation with valid data creates a media_profile" do
-      valid_attrs = %{name: "some name", output_path_template: "some output_path_template"}
+      valid_attrs = %{name: "some name", output_path_template: "output_template.{{ ext }}"}
 
       assert {:ok, %MediaProfile{} = media_profile} = Profiles.create_media_profile(valid_attrs)
       assert media_profile.name == "some name"
-      assert media_profile.output_path_template == "some output_path_template"
+      assert media_profile.output_path_template == "output_template.{{ ext }}"
     end
 
     test "creation with invalid data returns error changeset" do
@@ -44,14 +44,14 @@ defmodule Pinchflat.ProfilesTest do
 
       update_attrs = %{
         name: "some updated name",
-        output_path_template: "some updated output_path_template"
+        output_path_template: "new_output_template.{{ ext }}"
       }
 
       assert {:ok, %MediaProfile{} = media_profile} =
                Profiles.update_media_profile(media_profile, update_attrs)
 
       assert media_profile.name == "some updated name"
-      assert media_profile.output_path_template == "some updated output_path_template"
+      assert media_profile.output_path_template == "new_output_template.{{ ext }}"
     end
 
     test "updating with invalid data returns error changeset" do
@@ -132,5 +132,41 @@ defmodule Pinchflat.ProfilesTest do
       media_profile = media_profile_fixture()
       assert %Ecto.Changeset{} = Profiles.change_media_profile(media_profile)
     end
+
+    test "it ensures the media profile's output template ends with an extension" do
+      valid_templates = [
+        "output_template.{{ ext }}",
+        "output_template.{{ext}}",
+        "output_template.%(ext)s",
+        "output_template.%(ext)S",
+        "output_template.%( ext )s",
+        "output_template.%( ext )S"
+      ]
+
+      for template <- valid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        assert cs.valid?
+      end
+    end
+
+    test "it does not allow invalid output templates" do
+      invalid_templates = [
+        "output_template.{{ ext }}.something",
+        "output_template.{{   ext   }}",
+        "output_template{{ ext }}",
+        "output_template.%(ext)s.something",
+        "output_template.txt",
+        "output_template%(ext)s",
+        "output_template.%(nope)s",
+        "output_template"
+      ]
+
+      for template <- invalid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        refute cs.valid?
+      end
+    end
   end
 end

test/pinchflat/slow_indexing/file_follower_server_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServerTest do
+defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
   use ExUnit.Case, async: true
 
-  alias alias Pinchflat.Utils.FilesystemUtils
+  alias alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias Pinchflat.SlowIndexing.FileFollowerServer
 
   setup do
     {:ok, pid} = FileFollowerServer.start_link()
-    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
+    tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
 
     {:ok, %{pid: pid, tmpfile: tmpfile}}
   end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -8,9 +8,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -101,7 +101,10 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
 
       source = source_fixture(index_frequency_minutes: 10)
-      task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
+
+      task_count_fetcher = fn ->
+        Enum.count(Tasks.list_tasks_for(source, "MediaCollectionIndexingWorker"))
+      end
 
       assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
@@ -158,5 +161,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
       end)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaCollectionIndexingWorker, %{id: 0})
+    end
   end
 end