Bumped version

* Fixed TV show output template

* Discard jobs if record not found; hopefully addressed timing-related source deletion bug

.credo.exs

@@ -128,7 +128,7 @@
           {Credo.Check.Refactor.MatchInCondition, []},
           {Credo.Check.Refactor.NegatedConditionsInUnless, []},
           {Credo.Check.Refactor.NegatedConditionsWithElse, []},
-          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.Nesting, [max_nesting: 4]},
           {Credo.Check.Refactor.RedundantWithClauseResult, []},
           {Credo.Check.Refactor.RejectReject, []},
           {Credo.Check.Refactor.UnlessWithElse, []},

config/config.exs

@@ -48,7 +48,8 @@ config :pinchflat, Oban,
     media_indexing: 2,
     media_collection_indexing: 2,
     media_fetching: 2,
-    media_local_metadata: 8
+    local_metadata: 8,
+    remote_metadata: 4
   ]
 
 # Configures the mailer

lib/pinchflat/boot/data_backfill_worker.ex

@@ -2,7 +2,7 @@ defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
-    queue: :media_local_metadata,
+    queue: :local_metadata,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -14,6 +14,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -31,6 +32,7 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
   @impl true
   def init(state) do
     apply_default_settings()
+    ensure_directories_are_writeable()
     rename_old_job_workers()
 
     {:ok, state}
@@ -41,6 +43,21 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
     Settings.fetch!(:pro_enabled, false)
   end
 
+  defp ensure_directories_are_writeable do
+    directories = [
+      Application.get_env(:pinchflat, :media_directory),
+      Application.get_env(:pinchflat, :tmpfile_directory),
+      Application.get_env(:pinchflat, :metadata_directory)
+    ]
+
+    Enum.each(directories, fn dir ->
+      file = Path.join([dir, ".keep"])
+
+      # This will fail if the directory is not writeable, stopping boot
+      FilesystemHelpers.write_p!(file, "")
+    end)
+  end
+
   # As part of a large refactor, I ended up moving a bunch of workers around. This
   # is a problem because the workers are stored in the database and the runner
   # will try to run the OLD jobs. This is also why these tasks run before the job
@@ -52,7 +69,6 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
     rename_map = [
       ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
       ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
-      ["Pinchflat.Workers.FilesystemDataWorker", "Pinchflat.Filesystem.FilesystemDataWorker"],
       ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
       ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
       ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]

lib/pinchflat/downloading/download_option_builder.ex

@@ -18,7 +18,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
-        thumbnail_options(media_profile) ++
+        thumbnail_options(media_item_with_preloads) ++
         metadata_options(media_profile) ++
         quality_options(media_profile) ++
         output_options(media_item_with_preloads)
@@ -57,13 +57,16 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
     end)
   end
 
-  defp thumbnail_options(media_profile) do
+  defp thumbnail_options(media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
     mapped_struct = Map.from_struct(media_profile)
 
     Enum.reduce(mapped_struct, [], fn attr, acc ->
       case attr do
         {:download_thumbnail, true} ->
-          acc ++ [:write_thumbnail, convert_thumbnail: "jpg"]
+          thumbnail_save_location = determine_thumbnail_location(media_item_with_preloads)
+
+          acc ++ [:write_thumbnail, convert_thumbnail: "jpg", output: "thumbnail:#{thumbnail_save_location}"]
 
         {:embed_thumbnail, true} ->
           acc ++ [:embed_thumbnail, convert_thumbnail: "jpg"]
@@ -96,21 +99,25 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
       :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
       :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
       :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
-      :"1440p" -> [format_sort: "res:1440,#{video_codec_options}"]
       :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
     end
   end
 
   defp output_options(media_item_with_preloads) do
-    media_profile = media_item_with_preloads.source.media_profile
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
-    additional_options_map = output_options_map(media_item_with_preloads)
-    {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
 
     [
-      output: Path.join(base_directory(), output_path)
+      output: build_output_path(output_path_template, media_item_with_preloads)
     ]
   end
 
+  defp build_output_path(string, media_item_with_preloads) do
+    additional_options_map = output_options_map(media_item_with_preloads)
+    {:ok, output_path} = OutputPathBuilder.build(string, additional_options_map)
+
+    Path.join(base_directory(), output_path)
+  end
+
   defp output_options_map(media_item_with_preloads) do
     source = media_item_with_preloads.source
 
@@ -120,6 +127,19 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
     }
   end
 
+  # I don't love the string manipulation here, but what can ya' do.
+  # It's dependent on the output_path_template being a string ending `.{{ ext }}`
+  # (or equivalent), but that's validated by the MediaProfile schema.
+  defp determine_thumbnail_location(media_item_with_preloads) do
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
+
+    output_path_template
+    |> String.split(~r{\.}, include_captures: true)
+    |> List.insert_at(-3, "-thumb")
+    |> Enum.join()
+    |> build_output_path(media_item_with_preloads)
+  end
+
   defp base_directory do
     Application.get_env(:pinchflat, :media_directory)
   end

lib/pinchflat/downloading/downloading_helpers.ex

@@ -26,11 +26,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
   def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
     source
     |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
+    |> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
   end
 
   def enqueue_pending_download_tasks(%Source{download_media: false}) do

lib/pinchflat/downloading/media_download_worker.ex

@@ -6,19 +6,32 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Repo
   alias Pinchflat.Media
-  alias Pinchflat.Tasks
   alias Pinchflat.Downloading.MediaDownloader
-  alias Pinchflat.Filesystem.FilesystemDataWorker
 
-  @impl Oban.Worker
+  @doc """
+  Starts the media_item media download worker and creates a task for the media_item.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(media_item, opts \\ []) do
+    %{id: media_item.id}
+    |> MediaDownloadWorker.new(opts)
+    |> Tasks.create_job_with_task(media_item)
+  end
+
   @doc """
   For a given media item, download the media alongside any options.
   Does not download media if its source is set to not download media.
 
   Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item =
       media_item_id
@@ -31,26 +44,30 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
   defp download_media_and_schedule_jobs(media_item) do
     case MediaDownloader.download_for_media_item(media_item) do
-      {:ok, _} ->
+      {:ok, updated_media_item} ->
-        schedule_filesystem_data_worker(media_item)
+        compute_and_save_media_filesize(updated_media_item)
-        {:ok, media_item}
+
+        {:ok, updated_media_item}
 
       err ->
         err
     end
   end
 
-  defp schedule_filesystem_data_worker(media_item) do
+  defp compute_and_save_media_filesize(media_item) do
-    %{id: media_item.id}
+    case File.stat(media_item.media_filepath) do
-    |> FilesystemDataWorker.new()
+      {:ok, %{size: size}} ->
-    |> Tasks.create_job_with_task(media_item)
+        Media.update_media_item(media_item, %{media_size_bytes: size})
-    |> case do
+
-      {:ok, task} -> {:ok, task}
+      _ ->
-      {:error, :duplicate_job} -> {:ok, :job_exists}
+        :ok
     end
   end
 end

lib/pinchflat/downloading/media_downloader.ex

@@ -8,11 +8,12 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Metadata.MetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-  alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
-  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
-  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
@@ -30,16 +31,17 @@ defmodule Pinchflat.Downloading.MediaDownloader do
 
     case download_with_options(media_item.original_url, item_with_preloads) do
       {:ok, parsed_json} ->
-        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
-
         parsed_attrs =
           parsed_json
-          |> parser.parse_for_media_item()
+          |> MetadataParser.parse_for_media_item()
           |> Map.merge(%{
             media_downloaded_at: DateTime.utc_now(),
+            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
             metadata: %{
-              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
+              # IDEA: might be worth kicking off a job for this since thumbnail fetching
-              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+              # could fail and I want to handle that in isolation
+              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
             }
           })
 
@@ -52,13 +54,16 @@ defmodule Pinchflat.Downloading.MediaDownloader do
     end
   end
 
-  # def download_for_source(source, url) do
+  defp determine_nfo_filepath(media_item, parsed_json) do
-  #   # Create MI from source and URL
+    if media_item.source.media_profile.download_nfo do
-  #   media_item = nil
+      NfoBuilder.build_and_store_for_media_item(parsed_json)
-  # end
+    else
+      nil
+    end
+  end
 
   defp download_with_options(url, item_with_preloads) do
-    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
+    {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
     YtDlpMedia.download(url, options)
   end

lib/pinchflat/downloading/output_path_builder.ex

@@ -40,7 +40,9 @@ defmodule Pinchflat.Downloading.OutputPathBuilder do
       "upload_year" => "%(upload_date>%Y)S",
       "upload_month" => "%(upload_date>%m)S",
       "upload_day" => "%(upload_date>%d)S",
-      "upload_yyyy_mm_dd" => "%(upload_date>%Y)S-%(upload_date>%m)S-%(upload_date>%d)S"
+      "upload_yyyy_mm_dd" => "%(upload_date>%Y-%m-%d)S",
+      "season_from_date" => "%(upload_date>%Y)S",
+      "season_episode_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S"
     }
   end
 end

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -6,34 +6,13 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
   """
 
   alias Pinchflat.Media
-  alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
   alias Pinchflat.FastIndexing.YoutubeRss
-  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
   alias Pinchflat.FastIndexing.MediaIndexingWorker
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
   @doc """
   Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
   for any new media items. See comments in `MediaIndexingWorker` for more info on the
@@ -54,9 +33,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
     Enum.each(new_media_ids, fn media_id ->
       url = "https://www.youtube.com/watch?v=#{media_id}"
 
-      %{id: source.id, media_url: url}
+      MediaIndexingWorker.kickoff_with_task(source, url)
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
     end)
   end
 
@@ -74,9 +51,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
     case maybe_media_item do
       {:ok, media_item} ->
         if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
+          MediaDownloadWorker.kickoff_with_task(media_item)
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
         end
 
         {:ok, media_item}

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -6,13 +6,25 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "fast_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
   alias Pinchflat.FastIndexing.FastIndexingHelpers
 
-  @impl Oban.Worker
+  @doc """
+  Starts the source fast indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> FastIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
   @doc """
   Kicks off the fast indexing process for a source, reschedules the job to run again
   once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
@@ -20,6 +32,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
 
   Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
@@ -30,15 +43,15 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do
     next_run_in = Source.fast_index_frequency() * 60
 
-    %{id: source.id}
+    case kickoff_with_task(source, schedule_in: next_run_in) do
-    |> FastIndexingWorker.new(schedule_in: next_run_in)
-    |> Tasks.create_job_with_task(source)
-    |> case do
       {:ok, task} -> {:ok, task}
       {:error, :duplicate_job} -> {:ok, :job_exists}
     end

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -8,10 +8,22 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
 
   require Logger
 
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.FastIndexing.FastIndexingHelpers
 
-  @impl Oban.Worker
+  @doc """
+  Starts the fast media indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, media_url, opts \\ []) do
+    %{id: source.id, media_url: media_url}
+    |> MediaIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
   @doc """
   Similar to `MediaCollectionIndexingWorker`, but for individual media items.
   Does not reschedule or check anything to do with a source's indexing
@@ -37,6 +49,7 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
 
   Returns :ok
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
@@ -49,5 +62,8 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
     end
 
     :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 end

lib/pinchflat/filesystem/filesystem_data_worker.ex

@@ -1,27 +0,0 @@
-defmodule Pinchflat.Filesystem.FilesystemDataWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :media_local_metadata,
-    tags: ["media_item", "media_metadata", "local_metadata"],
-    max_attempts: 1
-
-  alias Pinchflat.Media
-  alias Pinchflat.Filesystem.FilesystemHelpers
-
-  @impl Oban.Worker
-  @doc """
-  For a given media item, compute and save metadata about the file on-disk.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
-
-    FilesystemHelpers.compute_and_save_media_filesize(media_item)
-
-    # Don't retry on failure - if it didn't work immediately there's no
-    # reason to believe it will work later.
-    :ok
-  end
-end

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -15,12 +15,25 @@ defmodule Pinchflat.Filesystem.FilesystemHelpers do
     tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
     filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
 
-    :ok = File.mkdir_p!(Path.dirname(filepath))
+    :ok = write_p!(filepath, "")
-    :ok = File.write(filepath, "")
 
     filepath
   end
 
+  @doc """
+  Writes content to a file, creating directories as needed.
+  Takes the same args as File.write!/3.
+
+  Returns :ok | raises on error
+  """
+  def write_p!(filepath, content, modes \\ []) do
+    filepath
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.write!(filepath, content, modes)
+  end
+
   @doc """
   Fetches the file size of a media item and saves it to the database.
 
@@ -35,4 +48,36 @@ defmodule Pinchflat.Filesystem.FilesystemHelpers do
         err
     end
   end
+
+  @doc """
+  Deletes a file and removes any empty directories in the path.
+  Does NOT remove any directories that are not empty.
+
+  Returns :ok | {:error, any()}
+  """
+  def delete_file_and_remove_empty_directories(filepath) do
+    case File.rm(filepath) do
+      :ok ->
+        filepath
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+  end
+
+  defp recursively_delete_empty_directories(directory) do
+    case File.rmdir(directory) do
+      :ok ->
+        directory
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+
+    :ok
+  end
 end

lib/pinchflat/media/media.ex

@@ -7,9 +7,10 @@ defmodule Pinchflat.Media do
 
   alias Pinchflat.Repo
   alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
+  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @doc """
   Returns the list of media_items.
@@ -141,38 +142,6 @@ defmodule Pinchflat.Media do
   """
   def get_media_item!(id), do: Repo.get!(MediaItem, id)
 
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's downloaded files
-
-  Returns [binary()]
-  """
-  def media_filepaths(media_item) do
-    mapped_struct = Map.from_struct(media_item)
-
-    MediaItem.filepath_attributes()
-    |> Enum.map(fn
-      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-      field -> List.wrap(mapped_struct[field])
-    end)
-    |> List.flatten()
-    |> Enum.filter(&is_binary/1)
-  end
-
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's metadata files.
-  Returns an empty list if the media_item has no metadata.
-
-  Returns [binary()] | []
-  """
-  def metadata_filepaths(media_item) do
-    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
-    mapped_struct = Map.from_struct(metadata)
-
-    MediaMetadata.filepath_attributes()
-    |> Enum.map(fn field -> mapped_struct[field] end)
-    |> Enum.filter(&is_binary/1)
-  end
-
   @doc """
   Creates a media_item.
 
@@ -219,19 +188,22 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Deletes a media_item and its associated tasks.
+  Deletes a media_item, its associated tasks, and our internal metadata files.
-  Can optionally delete the media_item's files.
+  Can optionally delete the media_item's media files (media, thumbnail, subtitles, etc).
 
   Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
   """
   def delete_media_item(%MediaItem{} = media_item, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(media_item)
+
     if delete_files do
-      {:ok, _} = delete_all_attachments(media_item)
+      {:ok, _} = delete_media_files(media_item)
     end
 
-    Tasks.delete_tasks_for(media_item)
+    # Should delete these no matter what
+    delete_internal_metadata_files(media_item)
     Repo.delete(media_item)
   end
 
@@ -242,26 +214,31 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_all_attachments(media_item) do
+  defp delete_media_files(media_item) do
-    media_item = Repo.preload(media_item, :metadata)
+    mapped_struct = Map.from_struct(media_item)
 
-    media_item
+    MediaItem.filepath_attributes()
-    |> media_filepaths()
+    |> Enum.map(fn
-    |> Enum.concat(metadata_filepaths(media_item))
+      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-    |> Enum.each(&File.rm/1)
+      field -> List.wrap(mapped_struct[field])
-
+    end)
-    # rmdir will attempt to delete the directory, but only if it is empty
+    |> List.flatten()
-    if media_item.media_filepath do
+    |> Enum.filter(&is_binary/1)
-      File.rmdir(Path.dirname(media_item.media_filepath))
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
-    end
-
-    if media_item.metadata && media_item.metadata.metadata_filepath do
-      File.rmdir(Path.dirname(media_item.metadata.metadata_filepath))
-    end
 
     {:ok, media_item}
   end
 
+  defp delete_internal_metadata_files(media_item) do
+    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    MediaMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp maybe_apply_cutoff_date(source) do
     if source.download_cutoff_date do
       dynamic([mi], mi.upload_date >= ^source.download_cutoff_date)

lib/pinchflat/media/media_item.ex

@@ -27,7 +27,8 @@ defmodule Pinchflat.Media.MediaItem do
     :media_size_bytes,
     :subtitle_filepaths,
     :thumbnail_filepath,
-    :metadata_filepath
+    :metadata_filepath,
+    :nfo_filepath
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -54,6 +55,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :media_size_bytes, :integer
     field :thumbnail_filepath, :string
     field :metadata_filepath, :string
+    field :nfo_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.
@@ -82,6 +84,6 @@ defmodule Pinchflat.Media.MediaItem do
 
   @doc false
   def filepath_attributes do
-    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths)a
+    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
 end

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -9,6 +9,8 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
   needed
   """
 
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
   @doc """
   Compresses and stores metadata for a media item, returning the filepath.
 
@@ -18,8 +20,7 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, "metadata.json.gz")
     {:ok, json} = Phoenix.json_library().encode(metadata_map)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, json, [:compressed])
-    :ok = File.write(filepath, json, [:compressed])
 
     filepath
   end
@@ -45,12 +46,23 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
     thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 
-    File.mkdir_p!(Path.dirname(filepath))
+    :ok = FilesystemHelpers.write_p!(filepath, thumbnail_blob)
-    :ok = File.write(filepath, thumbnail_blob)
 
     filepath
   end
 
+  @doc """
+  Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
+  and returns a Date struct.
+
+  Returns Date.t()
+  """
+  def parse_upload_date(upload_date) do
+    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
+
+    Date.from_iso8601!("#{year}-#{month}-#{day}")
+  end
+
   defp fetch_thumbnail_from_url(url) do
     http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
     {:ok, body} = http_client.get(url, [], body_format: :binary)

lib/pinchflat/metadata/metadata_parser.ex

@@ -54,10 +54,23 @@ defmodule Pinchflat.Metadata.MetadataParser do
       |> Enum.reverse()
       |> Enum.find_value(fn attrs -> attrs["filepath"] end)
 
+    if thumbnail_filepath do
+      # NOTE: whole ordeal needed due to a bug I found in yt-dlp
+      # https://github.com/yt-dlp/yt-dlp/issues/9445
+      # Can be reverted to remove this entire conditional once fixed
+      %{
+        thumbnail_filepath:
+          thumbnail_filepath
+          |> String.split(~r{\.}, include_captures: true)
+          |> List.insert_at(-3, "-thumb")
+          |> Enum.join()
+      }
+    else
       %{
         thumbnail_filepath: thumbnail_filepath
       }
     end
+  end
 
   defp parse_infojson_metadata(metadata) do
     %{

lib/pinchflat/metadata/nfo_builder.ex

@@ -0,0 +1,44 @@
+defmodule Pinchflat.Metadata.NfoBuilder do
+  @moduledoc """
+  Provides methods for building and storing NFO files for
+  use by Kodi/Jellyfin and other media center software.
+  """
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Builds an NFO file for a media item (read: single "episode") and
+  stores it in the same directory as the media file. Has the same name
+  as the media file, but with a .nfo extension.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_media_item(metadata) do
+    filepath = Path.rootname(metadata["filepath"]) <> ".nfo"
+    nfo = build_for_media_item(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  defp build_for_media_item(metadata) do
+    upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
+    # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
+    # WHO NEEDS A FANCY XML PARSER ANYWAY?!
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <episodedetails>
+      <title>#{metadata["title"]}</title>
+      <showtitle>#{metadata["uploader"]}</showtitle>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <plot>#{metadata["description"]}</plot>
+      <premiered>#{upload_date}</premiered>
+      <season>#{upload_date.year}</season>
+      <episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
+      <genre>YouTube</genre>
+    </episodedetails>
+    """
+  end
+end

lib/pinchflat/metadata/source_metadata.ex

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Metadata.SourceMetadata do
+  @moduledoc """
+  The SourceMetadata schema.
+
+  Look. Don't @ me about Metadata vs. Metadatum. I'm very sensitive.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Pinchflat.Sources.Source
+
+  @allowed_fields ~w(metadata_filepath)a
+  @required_fields ~w(metadata_filepath)a
+
+  schema "source_metadata" do
+    field :metadata_filepath, :string
+
+    belongs_to :source, Source
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(source_metadata, attrs) do
+    source_metadata
+    |> cast(attrs, @allowed_fields)
+    |> validate_required(@required_fields)
+    |> unique_constraint([:source_id])
+  end
+
+  @doc false
+  def filepath_attributes do
+    ~w(metadata_filepath)a
+  end
+end

lib/pinchflat/metadata/source_metadata_storage_worker.ex

@@ -0,0 +1,56 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :remote_metadata,
+    tags: ["media_source", "source_metadata", "remote_metadata"],
+    max_attempts: 1,
+    # This is the only thing stopping this job from calling itself
+    # in an infinite loop.
+    unique: [period: 600]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Repo
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.MetadataFileHelpers
+
+  @doc """
+  Starts the source metadata storage worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> SourceMetadataStorageWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches and stores metadata for a source in the secret metadata location.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id}}) do
+    source = Repo.preload(Sources.get_source!(source_id), :metadata)
+    {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url)
+
+    # Since updating a source kicks this job off again, we enforce job uniqueness (above)
+    # to once, per source, per x minutes. This is to prevent a job from calling itself
+    # in an infinite loop.
+    Sources.update_source(source, %{
+      metadata: %{
+        metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, metadata)
+      }
+    })
+
+    :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
+  end
+end

lib/pinchflat/profiles/media_profile.ex

@@ -19,6 +19,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
     embed_thumbnail
     download_metadata
     embed_metadata
+    download_nfo
     shorts_behaviour
     livestream_behaviour
     preferred_resolution
@@ -32,17 +33,18 @@ defmodule Pinchflat.Profiles.MediaProfile do
     field :output_path_template, :string,
       default: "/{{ source_custom_name }}/{{ upload_yyyy_mm_dd }} {{ title }}/{{ title }} [{{ id }}].{{ ext }}"
 
-    field :download_subs, :boolean, default: true
+    field :download_subs, :boolean, default: false
-    field :download_auto_subs, :boolean, default: true
+    field :download_auto_subs, :boolean, default: false
-    field :embed_subs, :boolean, default: true
+    field :embed_subs, :boolean, default: false
     field :sub_langs, :string, default: "en"
 
-    field :download_thumbnail, :boolean, default: true
+    field :download_thumbnail, :boolean, default: false
-    field :embed_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: false
 
-    field :download_metadata, :boolean, default: true
+    field :download_metadata, :boolean, default: false
-    field :embed_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: false
 
+    field :download_nfo, :boolean, default: false
     # NOTE: these do NOT speed up indexing - the indexer still has to go
     # through the entire collection to determine if a media is a short or
     # a livestream.
@@ -65,6 +67,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     media_profile
     |> cast(attrs, @allowed_fields)
     |> validate_required(@required_fields)
+    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
+    |> validate_format(:output_path_template, ext_regex(), message: "must end with .{{ ext }}")
     |> unique_constraint(:name)
   end
+
+  defp ext_regex do
+    ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
+  end
 end

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -6,6 +6,8 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "media_collection_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
@@ -13,7 +15,17 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
-  @impl Oban.Worker
+  @doc """
+  Starts the source slow indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> MediaCollectionIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
   @doc """
   The ID is that of a source _record_, not a YouTube channel/playlist ID. Indexes
   the provided source, kicks off downloads for each new MediaItem, and
@@ -58,6 +70,7 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
 
   Returns :ok | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
@@ -79,6 +92,9 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
         # perform a no-op
         :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -31,10 +31,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
     Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
-    %{id: source.id}
+    MediaCollectionIndexingWorker.kickoff_with_task(source)
-    # Schedule this one immediately, but future ones will be on an interval
-    |> MediaCollectionIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
   end
 
   @doc """
@@ -125,9 +122,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
-          %{id: media_item.id}
+          MediaDownloadWorker.kickoff_with_task(media_item)
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
         end
 
       {:error, changeset} ->

lib/pinchflat/sources/source.ex

@@ -10,6 +10,7 @@ defmodule Pinchflat.Sources.Source do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Metadata.SourceMetadata
 
   @allowed_fields ~w(
     collection_name
@@ -28,7 +29,8 @@ defmodule Pinchflat.Sources.Source do
   # Expensive API calls are made when a source is inserted/updated so
   # we want to ensure that the source is valid before making the call.
   # This way, we check that the other attributes are valid before ensuring
-  # that all fields are valid.
+  # that all fields are valid. This is still only one DB insert but it's
+  # a two-stage validation process to fail fast before the API call.
   @initially_required_fields ~w(
     index_frequency_minutes
     fast_index
@@ -60,6 +62,8 @@ defmodule Pinchflat.Sources.Source do
 
     belongs_to :media_profile, MediaProfile
 
+    has_one :metadata, SourceMetadata, on_replace: :update
+
     has_many :tasks, Task
     has_many :media_items, MediaItem, foreign_key: :source_id
 
@@ -80,6 +84,7 @@ defmodule Pinchflat.Sources.Source do
     |> cast(attrs, @allowed_fields)
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> validate_required(required_fields)
+    |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id])
   end
 

lib/pinchflat/sources/sources.ex

@@ -11,9 +11,12 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Sources.Source
   alias Pinchflat.Profiles.MediaProfile
   alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -54,7 +57,7 @@ defmodule Pinchflat.Sources do
     case change_source(%Source{}, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         %Source{}
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
         |> commit_and_handle_tasks()
 
@@ -82,7 +85,7 @@ defmodule Pinchflat.Sources do
     case change_source(source, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         source
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
         |> commit_and_handle_tasks()
 
@@ -100,13 +103,15 @@ defmodule Pinchflat.Sources do
   def delete_source(%Source{} = source, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(source)
+
     source
     |> Media.list_media_items_for()
     |> Enum.each(fn media_item ->
       Media.delete_media_item(media_item, delete_files: delete_files)
     end)
 
-    Tasks.delete_tasks_for(source)
+    delete_source_metadata_files(source)
     Repo.delete(source)
   end
 
@@ -117,19 +122,9 @@ defmodule Pinchflat.Sources do
     Source.changeset(source, attrs, validation_stage)
   end
 
-  @doc """
+  # NOTE: When operating in the ideal path, this effectively adds an API call
-  Returns an `%Ecto.Changeset{}` for tracking source changes and additionally
+  # to the source creation/update process. Should be used only when needed.
-  fetches source details from the original_url (if provided). If the source
+  defp maybe_change_source_from_url(%Source{} = source, attrs) do
-  details cannot be fetched, an error is added to the changeset.
-
-  Note that this fetches source details as long as the `original_url` is present.
-  This means that it'll go for it even if a changeset is otherwise invalid. This
-  is pretty easy to change, but for MVP I'm not concerned.
-
-  NOTE: When operating in the ideal path, this effectively adds an API call
-  to the source creation/update process. Should be used only when needed.
-  """
-  def change_source_from_url(%Source{} = source, attrs) do
     case change_source(source, attrs) do
       %Ecto.Changeset{changes: %{original_url: _}} = changeset ->
         add_source_details_to_changeset(source, changeset)
@@ -139,6 +134,18 @@ defmodule Pinchflat.Sources do
     end
   end
 
+  defp delete_source_metadata_files(source) do
+    metadata = Repo.preload(source, :metadata).metadata || %SourceMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    filepaths =
+      SourceMetadata.filepath_attributes()
+      |> Enum.map(fn field -> mapped_struct[field] end)
+      |> Enum.filter(&is_binary/1)
+
+    Enum.each(filepaths, &FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp add_source_details_to_changeset(source, changeset) do
     %Ecto.Changeset{changes: changes} = changeset
 
@@ -196,6 +203,9 @@ defmodule Pinchflat.Sources do
       {:ok, %Source{} = source} ->
         maybe_handle_media_tasks(changeset, source)
         maybe_run_indexing_task(changeset, source)
+        run_metadata_storage_task(source)
+
+        {:ok, source}
 
       err ->
         err
@@ -215,8 +225,6 @@ defmodule Pinchflat.Sources do
       _ ->
         :ok
     end
-
-    {:ok, source}
   end
 
   defp maybe_run_indexing_task(changeset, source) do
@@ -231,8 +239,11 @@ defmodule Pinchflat.Sources do
         maybe_update_slow_indexing_task(changeset, source)
         maybe_update_fast_indexing_task(changeset, source)
     end
+  end
 
-    {:ok, source}
+  # This runs every time to pick up any changes to the metadata
+  defp run_metadata_storage_task(source) do
+    SourceMetadataStorageWorker.kickoff_with_task(source)
   end
 
   defp maybe_update_slow_indexing_task(changeset, source) do
@@ -253,7 +264,8 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        FastIndexingHelpers.kickoff_fast_indexing_task(source)
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+        FastIndexingWorker.kickoff_with_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/tasks.ex

@@ -22,9 +22,15 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil, job_states \\ Oban.Job.states()) do
+  def list_tasks_for(record, worker_name \\ nil, job_states \\ Oban.Job.states()) do
     stringified_states = Enum.map(job_states, &to_string/1)
 
+    record_type =
+      case record do
+        %Source{} -> :source_id
+        %MediaItem{} -> :media_item_id
+      end
+
     worker_name_finder =
       if worker_name do
         # Workers are the full module name - we want to match on the string ENDING with
@@ -41,7 +47,7 @@ defmodule Pinchflat.Tasks do
     Repo.all(
       from t in Task,
         join: j in assoc(t, :job),
-        where: field(t, ^attached_record_type) == ^attached_record_id,
+        where: field(t, ^record_type) == ^record.id,
         where: ^worker_name_finder,
         where: j.state in ^stringified_states
     )
@@ -53,10 +59,9 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_pending_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil) do
+  def list_pending_tasks_for(record, worker_name \\ nil) do
     list_tasks_for(
-      attached_record_type,
+      record,
-      attached_record_id,
       worker_name,
       [:available, :scheduled, :retryable]
     )
@@ -126,14 +131,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_tasks_for(worker_name)
-        %Source{} = source -> list_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """
@@ -142,14 +143,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_pending_tasks_for(attached_record, worker_name \\ nil) do
+  def delete_pending_tasks_for(record, worker_name \\ nil) do
-    tasks =
+    record
-      case attached_record do
+    |> list_pending_tasks_for(worker_name)
-        %Source{} = source -> list_pending_tasks_for(:source_id, source.id, worker_name)
+    |> Enum.each(&delete_task/1)
-        %MediaItem{} = media_item -> list_pending_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
   end
 
   @doc """

lib/pinchflat/yt_dlp/command_runner.ex

@@ -37,7 +37,8 @@ defmodule Pinchflat.YtDlp.CommandRunner do
 
     case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
-        # IDEA: consider deleting the file after reading it
+        # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
+        # a huge deal, but it's still a good idea to clean up after ourselves.
         # (even on error? especially on error?)
         File.read(output_filepath)
 

lib/pinchflat/yt_dlp/media.ex

@@ -25,6 +25,7 @@ defmodule Pinchflat.YtDlp.Media do
 
   alias __MODULE__
   alias Pinchflat.Utils.FunctionUtils
+  alias Pinchflat.Metadata.MetadataFileHelpers
 
   @doc """
   Downloads a single piece of media (and possibly its metadata) directly to its
@@ -86,7 +87,7 @@ defmodule Pinchflat.YtDlp.Media do
       original_url: response["webpage_url"],
       livestream: response["was_live"],
       short_form_content: response["webpage_url"] && short_form_content?(response),
-      upload_date: response["upload_date"] && parse_upload_date(response["upload_date"])
+      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
     }
   end
 
@@ -106,12 +107,6 @@ defmodule Pinchflat.YtDlp.Media do
     end
   end
 
-  defp parse_upload_date(upload_date) do
-    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
-
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
-  end
-
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

lib/pinchflat/yt_dlp/media_collection.ex

@@ -6,7 +6,6 @@ defmodule Pinchflat.YtDlp.MediaCollection do
 
   require Logger
 
-  alias Pinchflat.Utils.FunctionUtils
   alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
@@ -36,11 +35,20 @@ defmodule Pinchflat.YtDlp.MediaCollection do
 
     case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
       {:ok, output} ->
+        parsed_lines =
           output
           |> String.split("\n", trim: true)
-        |> Enum.map(&Phoenix.json_library().decode!/1)
+          |> Enum.map(fn line ->
-        |> Enum.map(&YtDlpMedia.response_to_struct/1)
+            case Phoenix.json_library().decode(line) do
-        |> FunctionUtils.wrap_ok()
+              {:ok, parsed_json} ->
+                YtDlpMedia.response_to_struct(parsed_json)
+
+              _ ->
+                nil
+            end
+          end)
+
+        {:ok, Enum.filter(parsed_lines, &(&1 != nil))}
 
       res ->
         res
@@ -50,8 +58,8 @@ defmodule Pinchflat.YtDlp.MediaCollection do
   @doc """
   Gets a source's ID and name from its URL.
 
-  yt-dlp does not _really_ have source-specific functions, so
+  yt-dlp does not _really_ have source-specific functions that return what
-  instead we're fetching just the first video (using playlist_end: 1)
+  we need, so instead we're fetching just the first video (using playlist_end: 1)
   and parsing the source ID and name from _its_ metadata
 
   Returns {:ok, map()} | {:error, any, ...}.
@@ -71,7 +79,35 @@ defmodule Pinchflat.YtDlp.MediaCollection do
     end
   end
 
+  @doc """
+  Gets a source's metadata from its URL.
+
+  This is mostly for things like getting the source's avatar and banner image
+  (if applicable). However, this yt-dlp call doesn't have enough overlap with
+  `get_source_details/1` to allow combining them - this one has _almost_ everything
+  we need, but it doesn't contain enough information to tell 100% if the url is a channel
+  or a playlist.
+
+  The main purpose of this (past using as a fetcher for _other_ metadata) is to live
+  as a compressed blob for possible future use. That's why it's not getting formatted like
+  `get_source_details/1`
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_metadata(source_url) do
+    opts = [playlist_items: 0]
+    output_template = "playlist:%()j"
+
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, parsed_json}
+    else
+      err -> err
+    end
+  end
+
   defp format_source_details(response) do
+    # NOTE: I should probably make this a struct some day
     %{
       channel_id: response["channel_id"],
       channel_name: response["channel"],

lib/pinchflat_web/components/core_components.ex

@@ -317,17 +317,7 @@ defmodule PinchflatWeb.CoreComponents do
         <span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
       <div class="relative">
-        <input type="hidden" name={@name} value="false" />
+        <input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
-        <input
-          type="checkbox"
-          id={@id}
-          name={@name}
-          value="true"
-          x-bind:checked="enabled"
-          class="sr-only"
-          @change="enabled = !enabled"
-          {@rest}
-        />
         <div class="inline-block cursor-pointer" @click="enabled = !enabled">
           <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
           <div
@@ -349,9 +339,10 @@ defmodule PinchflatWeb.CoreComponents do
   def input(%{type: "select"} = assigns) do
     ~H"""
     <div phx-feedback-for={@name}>
-      <.label for={@id}>
+      <.label :if={@label} for={@id}>
         <%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
+      <div class="flex">
         <select
           id={@id}
           name={@name}
@@ -366,6 +357,8 @@ defmodule PinchflatWeb.CoreComponents do
           <option :if={@prompt} value=""><%= @prompt %></option>
           <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
         </select>
+        <%= render_slot(@inner_block) %>
+      </div>
       <.help :if={@help}><%= @help %></.help>
       <.error :for={msg <- @errors}><%= msg %></.error>
     </div>

lib/pinchflat_web/components/custom_components/button_components.ex

@@ -28,9 +28,10 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
         "#{@rounding} inline-flex items-center justify-center px-8 py-4",
         "#{@color}",
         "hover:bg-opacity-90 lg:px-8 xl:px-10",
-        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-gray-2",
+        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-grey-5",
         @class
       ]}
+      type={@type}
       disabled={@disabled}
       {@rest}
     >

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -1,6 +1,8 @@
 defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   use PinchflatWeb, :html
 
+  alias Pinchflat.Profiles.MediaProfile
+
   embed_templates "media_profile_html/*"
 
   @doc """
@@ -53,4 +55,25 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       duration_string
     )a
   end
+
+  def preset_options do
+    [
+      {"Default", "default"},
+      {"Media Center (Plex, Jellyfin, Kodi, etc.)", "media_center"},
+      {"Music", "audio"},
+      {"Archiving", "archiving"}
+    ]
+  end
+
+  defp default_output_template do
+    %MediaProfile{}.output_path_template
+  end
+
+  defp media_center_output_template do
+    "/shows/{{ source_custom_name }}/Season {{ season_from_date }}/{{ season_episode_from_date }} - {{ title }}.{{ ext }}"
+  end
+
+  defp audio_output_template do
+    "/music/{{ source_custom_name }}/{{ title }}.{{ ext }}"
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -3,116 +3,218 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
+  <section x-data="{ selectedPreset: null }">
     <h3 class="my-4 text-2xl text-black dark:text-white">
+      Use a Preset
+    </h3>
+    <section x-data="{ selection: null }">
+      <.input
+        prompt="Select preset"
+        name="media_profile_preset"
+        value=""
+        options={preset_options()}
+        type="select"
+        x-model="selection"
+        inputclass="w-full"
+        help="You can further customize the settings after selecting a preset. This is just a starting point"
+      >
+        <.button
+          class="h-13 w-2/5 lg:w-1/5 ml-2 md:ml-4"
+          rounding="rounded"
+          type="button"
+          x-on:click="selectedPreset = selection; selection = null"
+          x-bind:disabled="!selection"
+        >
+          <span x-text="selection ? 'Load' : 'Select'">Select</span><span class="hidden lg:inline ml-1">Preset</span>
+        </.button>
+      </.input>
+    </section>
+
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
       General Options
     </h3>
+
+    <section x-data="{ 
+        presets: { 
+          default: 'Default',
+          media_center: 'TV Shows',
+          audio: 'Audio',
+          archiving: 'Archiving'
+        }
+    }">
       <.input
         field={f[:name]}
         type="text"
         label="Name"
         placeholder="New Profile"
         help="Something descriptive. Does not impact indexing or downloading (required)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
+    <section x-data={"{ 
+        presets: { 
+          default: '#{default_output_template()}',
+          media_center: '#{media_center_output_template()}',
+          audio: '#{audio_output_template()}',
+          archiving: '#{default_output_template()}'
+        }
+    }"}>
       <.input
         field={f[:output_path_template]}
         type="text"
         inputclass="font-mono"
         label="Output path template"
         help="Must end with .{{ ext }}. See below for more details. The default is good for most cases (required)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Subtitle Options
     </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:download_subs]}
         type="toggle"
         label="Download Subtitles"
         help="Downloads subtitle files alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: false } }">
       <.input
         field={f[:download_auto_subs]}
         type="toggle"
         label="Download Autogenerated Subtitles"
-    help="Prefers normal subs but will download autogenerated if needed"
+        help="Prefers normal subs but will download autogenerated if needed. Requires 'Download Subtitles' to be enabled"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:embed_subs]}
         type="toggle"
         label="Embed Subtitles"
-    help="Embeds subtitles in the video file itself, if supported (recommended)"
+        help="Downloads and embeds subtitles in the media file itself, if supported. Uneffected by 'Download Subtitles' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: 'en', media_center: 'en', audio: '', archiving: 'all' } }">
       <.input
         field={f[:sub_langs]}
         type="text"
         label="Subtitle Languages"
         help="Use commas for multiple languages (eg: en,de)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Thumbnail Options
     </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:download_thumbnail]}
         type="toggle"
         label="Download Thumbnail"
         help="Downloads thumbnail alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
       <.input
         field={f[:embed_thumbnail]}
         type="toggle"
         label="Embed Thumbnail"
-    help="Embeds thumbnail in the video file itself, if supported (recommended)"
+        help="Downloads and embeds thumbnail in the media file itself, if supported. Uneffected by 'Download Thumbnail' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Metadata Options
     </h3>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: true } }">
       <.input
         field={f[:download_metadata]}
         type="toggle"
         label="Download Metadata"
         help="Downloads metadata file alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
       <.input
         field={f[:embed_metadata]}
         type="toggle"
         label="Embed Metadata"
-    help="Embeds metadata in the video file itself, if supported (recommended)"
+        help="Downloads and embeds metadata in the media file itself, if supported. Uneffected by 'Download Metadata' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <section x-data="{ presets: { default: false, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_nfo]}
+        type="toggle"
+        label="Download NFO data"
+        help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc."
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Release Format Options
     </h3>
 
+    <section x-data="{ presets: { default: 'include', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
       <.input
         field={f[:shorts_behaviour]}
         options={friendly_format_type_options()}
         type="select"
         label="Include Shorts?"
         help="Experimental. Please report any issues on GitHub"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: 'exclude', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
       <.input
         field={f[:livestream_behaviour]}
         options={friendly_format_type_options()}
         type="select"
         label="Include Livestreams?"
+        help="Excludes media that comes from a past livestream"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Quality Options
     </h3>
 
+    <section x-data="{ presets: { default: '1080p', media_center: '1080p', audio: 'audio', archiving: '2160p' } }">
       <.input
         field={f[:preferred_resolution]}
         options={friendly_resolution_options()}
         type="select"
         label="Preferred Resolution"
-    help="Will grab the closest available resolution if your preferred is not available"
+        help="Will grab the closest available resolution if your preferred is not available. 'Audio Only' grabs the highest quality m4a"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
     <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+  </section>
 
   <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
     <.output_template_help />

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -46,7 +46,7 @@ defmodule PinchflatWeb.Sources.SourceController do
   def show(conn, %{"id" => id}) do
     source = Repo.preload(Sources.get_source!(id), :media_profile)
 
-    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(:source_id, source.id), :job)
+    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(source), :job)
     pending_media = Media.list_pending_media_items_for(source, limit: 100)
     downloaded_media = Media.list_downloaded_media_items_for(source, limit: 100)
 

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -14,7 +14,7 @@ defmodule PinchflatWeb.Sources.SourceHTML do
 
   def friendly_index_frequencies do
     [
-      {"On Create", -1},
+      {"Only once when first created", -1},
       {"1 Hour", 60},
       {"3 Hours", 3 * 60},
       {"6 Hours", 6 * 60},

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -21,6 +21,7 @@
     options={Enum.map(@media_profiles, &{&1.name, &1.id})}
     type="select"
     label="Media Profile"
+    help="Sets your preferences for what media to look for and how to store it"
   />
 
   <h3 class="mt-8 text-2xl text-black dark:text-white">
@@ -32,7 +33,7 @@
     options={friendly_index_frequencies()}
     type="select"
     label="Index Frequency"
-    help="Time between one index of this source finishing and the next one starting. Setting to 'On Create' will still run an initial index but no subsequent ones"
+    help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
   />
 
   <%!-- TODO: use Alpine to disable the index frequency when fast indexing is enabled --%>

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.0-alpha.4",
+      version: "0.1.0-alpha.5",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -13,6 +13,16 @@ defmodule Pinchflat.MixProject do
       preferred_cli_env: [
         check: :test,
         credo: :test
+      ],
+      test_coverage: [
+        ignore_modules: [
+          Pinchflat.HTTP.HTTPClient,
+          PinchflatWeb.Layouts,
+          Pinchflat.DataCase,
+          Pinchflat.Release,
+          ~r/Fixtures/,
+          ~r/HTML$/
+        ]
       ]
     ]
   end

priv/repo/migrations/20240313175638_add_nfo_filepath_to_media_item.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddNfoFilepathToMediaItem do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :nfo_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240313175744_add_download_nfo_to_media_profile.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddDownloadNfoToMediaProfile do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_nfo, :boolean, default: false, null: false
+    end
+  end
+end

priv/repo/migrations/20240314174731_create_source_metadata.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.CreateSourceMetadata do
+  use Ecto.Migration
+
+  def change do
+    create table(:source_metadata) do
+      add :metadata_filepath, :string, null: false
+      add :source_id, references(:sources, on_delete: :delete_all), null: false
+
+      timestamps(type: :utc_datetime)
+    end
+
+    create unique_index(:source_metadata, [:source_id])
+  end
+end

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.Boot.DataBackfillWorkerTest do
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Filesystem.FilesystemDataWorker
+  alias Pinchflat.JobFixtures.TestJobWorker
 
   describe "cancel_pending_backfill_jobs/0" do
     test "cancels all pending backfill jobs" do
@@ -21,14 +21,14 @@ defmodule Pinchflat.Boot.DataBackfillWorkerTest do
 
     test "does not cancel jobs for other workers" do
       %{id: 0}
-      |> FilesystemDataWorker.new()
+      |> TestJobWorker.new()
       |> Repo.insert_unique_job()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
 
       DataBackfillWorker.cancel_pending_backfill_jobs()
 
-      assert_enqueued(worker: FilesystemDataWorker)
+      assert_enqueued(worker: TestJobWorker)
     end
   end
 

test/pinchflat/downloading/download_option_builder_test.exs

@@ -125,7 +125,15 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
       assert :write_thumbnail in res
     end
 
-    test "convertes thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
+    test "appends -thumb to the thumbnail name when download_thumbnail is true", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:output, "thumbnail:/tmp/test/media/%(title)S-thumb.%(ext)s"} in res
+    end
+
+    test "converts thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
 
       assert {:ok, res} = DownloadOptionBuilder.build(media_item)
@@ -189,12 +197,19 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   end
 
   describe "build/1 when testing quality options" do
-    test "it includes quality options", %{media_item: media_item} do
+    test "it includes quality options" do
-      media_item = update_media_profile_attribute(media_item, %{preferred_resolution: :"1080p"})
+      resolutions = ["360", "480", "720", "1080", "2160"]
+
+      Enum.each(resolutions, fn resolution ->
+        resolution_atom = String.to_existing_atom(resolution <> "p")
+
+        media_profile = media_profile_fixture(%{preferred_resolution: resolution_atom})
+        source = source_fixture(%{media_profile_id: media_profile.id})
+        media_item = Repo.preload(media_item_fixture(source_id: source.id), source: :media_profile)
 
         assert {:ok, res} = DownloadOptionBuilder.build(media_item)
-
+        assert {:format_sort, "res:#{resolution},+codec:avc:m4a"} in res
-      assert {:format_sort, "res:1080,+codec:avc:m4a"} in res
+      end)
     end
 
     test "it includes quality options for audio only", %{media_item: media_item} do
@@ -210,10 +225,10 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   defp update_media_profile_attribute(media_item_with_preloads, attrs) do
     media_item_with_preloads.source.media_profile
     |> Profiles.change_media_profile(attrs)
-    |> Repo.update!()
+    |> Repo.update()
 
     media_item_with_preloads
     |> Repo.reload()
-    |> Repo.preload(source: :media_profile)
+    |> Repo.preload([source: :media_profile], force: true)
   end
 end

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -34,11 +34,11 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [] = Tasks.list_tasks_for(media_item)
 
       assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
 
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [_] = Tasks.list_tasks_for(media_item)
     end
 
     test "it does not create a job if the source is set to not download" do
@@ -54,7 +54,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
       assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [] = Tasks.list_tasks_for(media_item)
     end
   end
 
@@ -69,7 +69,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       refute_enqueued(worker: MediaDownloadWorker)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [] = Tasks.list_tasks_for(media_item)
     end
   end
 end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -5,25 +5,37 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Sources
+  alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   setup :verify_on_exit!
 
   setup do
-    media_item =
-      Repo.preload(
-        media_item_fixture(%{media_filepath: nil}),
-        [:metadata, source: :media_profile]
-      )
-
     stub(HTTPClientMock, :get, fn _url, _headers, _opts ->
       {:ok, ""}
     end)
 
+    media_item =
+      %{media_filepath: nil}
+      |> media_item_fixture()
+      |> Repo.preload([:metadata, source: :media_profile])
+
     {:ok, %{media_item: media_item}}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{media_item: media_item} do
+      assert [] = all_enqueued(worker: MediaDownloadWorker)
+      assert {:ok, _} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "attaches a task", %{media_item: media_item} do
+      assert {:ok, task} = MediaDownloadWorker.kickoff_with_task(media_item)
+      assert task.media_item_id == media_item.id
+    end
+  end
+
   describe "perform/1" do
     test "it saves attributes to the media_item", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -70,16 +82,22 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       perform_job(MediaDownloadWorker, %{id: media_item.id})
     end
 
-    test "it schedules a filesystem data worker", %{media_item: media_item} do
+    test "it saves the file's size to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, render_metadata(:media_metadata)}
+        metadata = render_parsed_metadata(:media_metadata)
+        FilesystemHelpers.write_p!(metadata["filepath"], "test")
+
+        {:ok, Phoenix.json_library().encode!(metadata)}
       end)
 
-      assert [] = all_enqueued(worker: FilesystemDataWorker)
-
       perform_job(MediaDownloadWorker, %{id: media_item.id})
+      media_item = Repo.reload(media_item)
 
-      assert [_] = all_enqueued(worker: FilesystemDataWorker)
+      assert media_item.media_size_bytes > 0
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0})
     end
   end
 end

test/pinchflat/downloading/media_downloader_test.exs

@@ -2,6 +2,8 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
 
   alias Pinchflat.Downloading.MediaDownloader
 
@@ -103,4 +105,37 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
       assert String.ends_with?(updated_media_item.metadata_filepath, ".info.json")
     end
   end
+
+  describe "download_for_media_item/3 when testing NFO generation" do
+    setup do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      :ok
+    end
+
+    test "it generates an NFO file if the source is set to download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: true})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert String.ends_with?(updated_media_item.nfo_filepath, ".nfo")
+      assert File.exists?(updated_media_item.nfo_filepath)
+
+      File.rm!(updated_media_item.nfo_filepath)
+    end
+
+    test "it does not generate an NFO file if the source is set to not download NFOs" do
+      profile = media_profile_fixture(%{download_nfo: false})
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_fixture(%{source_id: source.id})
+
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+
+      assert updated_media_item.nfo_filepath == nil
+    end
+  end
 end

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -2,51 +2,20 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
   use Pinchflat.DataCase
 
   import Mox
-  import Pinchflat.TasksFixtures
   import Pinchflat.MediaFixtures
   import Pinchflat.SourcesFixtures
   import Pinchflat.ProfilesFixtures
 
   alias Pinchflat.Tasks
-  alias Pinchflat.Tasks.Task
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Downloading.MediaDownloadWorker
   alias Pinchflat.FastIndexing.MediaIndexingWorker
   alias Pinchflat.FastIndexing.FastIndexingHelpers
-  alias Pinchflat.FastIndexing.FastIndexingWorker
 
   setup :verify_on_exit!
 
   @media_url "https://www.youtube.com/watch?v=test_1"
 
-  describe "kickoff_fast_indexing_task/1" do
-    test "it schedules a job" do
-      source = source_fixture()
-
-      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
-
-      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it creates and attaches a task" do
-      source = source_fixture()
-
-      assert {:ok, %Task{} = task} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
-
-      assert task.source_id == source.id
-    end
-
-    test "it deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-  end
-
   describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
     setup do
       {:ok, [source: source_fixture()]}
@@ -104,7 +73,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
     test "creates a download task record", %{source: source} do
       assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
 
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
+      assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker")
     end
 
     test "does not enqueue a download job if the source does not allow it" do

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -9,18 +9,35 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
 
   setup :verify_on_exit!
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker" do
+      source = source_fixture(fast_index: true)
+
+      assert [] = all_enqueued(worker: FastIndexingWorker)
+      assert {:ok, _} = FastIndexingWorker.kickoff_with_task(source)
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "attaches a task" do
+      source = source_fixture(fast_index: true)
+
+      assert {:ok, task} = FastIndexingWorker.kickoff_with_task(source)
+      assert task.source_id == source.id
+    end
+  end
+
   describe "perform/1" do
     test "calls out to Youtube RSS if enabled" do
       expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
       source = source_fixture(fast_index: true)
 
-      perform_job(FastIndexingWorker, %{"id" => source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
     end
 
     test "reschedules itself if fast indexing is enabled" do
       expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
       source = source_fixture(fast_index: true)
-      perform_job(FastIndexingWorker, %{"id" => source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
 
       assert_enqueued(
         worker: FastIndexingWorker,
@@ -29,18 +46,32 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
       )
     end
 
+    test "does not reschedule if that would create a duplicate job" do
+      stub(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{id: source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
+
+      assert [_] = all_enqueued(worker: FastIndexingWorker)
+    end
+
     test "does not call out to Youtube RSS if disabled" do
       expect(HTTPClientMock, :get, 0, fn _url -> {:ok, ""} end)
       source = source_fixture(fast_index: false)
 
-      perform_job(FastIndexingWorker, %{"id" => source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
     end
 
     test "does not reschedule itself if fast indexing is disabled" do
       source = source_fixture(fast_index: false)
-      perform_job(FastIndexingWorker, %{"id" => source.id})
+      perform_job(FastIndexingWorker, %{id: source.id})
 
       refute_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(FastIndexingWorker, %{id: 0})
+    end
   end
 end

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -6,8 +6,8 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.FastIndexing.MediaIndexingWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
 
   @media_url "https://www.youtube.com/watch?v=1234567890"
 
@@ -19,6 +19,19 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
     {:ok, source: source}
   end
 
+  describe "kickoff_with_task/2" do
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: MediaIndexingWorker)
+      assert {:ok, _} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert [_] = all_enqueued(worker: MediaIndexingWorker)
+    end
+
+    test "attaches a task", %{source: source} do
+      assert {:ok, task} = MediaIndexingWorker.kickoff_with_task(source, @media_url)
+      assert task.source_id == source.id
+    end
+  end
+
   describe "perform/1" do
     test "indexes the media item and saves it to the database", %{source: source} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
@@ -40,5 +53,9 @@ defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
 
       assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaDownloadWorker, %{id: 0, media_url: @media_url})
+    end
   end
 end

test/pinchflat/filesystem/filesystem_data_worker_test.exs

@@ -1,19 +0,0 @@
-defmodule Pinchflat.Filesystem.FilesystemDataWorkerTest do
-  use Pinchflat.DataCase
-
-  import Pinchflat.MediaFixtures
-
-  alias Pinchflat.Filesystem.FilesystemDataWorker
-
-  describe "perform/1" do
-    test "Computes and stores the media file size" do
-      media_item = media_item_with_attachments()
-
-      refute media_item.media_size_bytes
-
-      perform_job(FilesystemDataWorker, %{id: media_item.id})
-
-      assert Repo.reload!(media_item).media_size_bytes
-    end
-  end
-end

test/pinchflat/filesystem/filesystem_helpers_test.exs

@@ -33,4 +33,78 @@ defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
       assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
     end
   end
+
+  describe "write_p!/3" do
+    test "writes content to a file" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+
+    test "creates directories as needed" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "file.json"])
+      content = "{}"
+
+      assert :ok = FilesystemHelpers.write_p!(filepath, content)
+      assert File.read!(filepath) == content
+
+      File.rm!(filepath)
+    end
+  end
+
+  describe "delete_file_and_remove_empty_directories/1" do
+    test "deletes file at the provided filepath" do
+      filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert File.exists?(filepath)
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+    end
+
+    test "deletes empty directories" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      FilesystemHelpers.write_p!(filepath, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+
+      refute File.exists?(filepath)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo"]))
+    end
+
+    test "does not delete directories with other files in them" do
+      tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+      filepath_1 = Path.join([tmpfile_directory, "foo", "bar", "baz", "qux.json"])
+      filepath_2 = Path.join([tmpfile_directory, "foo", "baz.json"])
+      FilesystemHelpers.write_p!(filepath_1, "")
+      FilesystemHelpers.write_p!(filepath_2, "")
+
+      assert :ok = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_1)
+
+      refute File.exists?(filepath_1)
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar", "baz"]))
+      refute File.exists?(Path.join([tmpfile_directory, "foo", "bar"]))
+
+      assert File.exists?(filepath_2)
+      assert File.exists?(Path.join([tmpfile_directory, "foo"]))
+
+      # cleanup
+      FilesystemHelpers.delete_file_and_remove_empty_directories(filepath_2)
+    end
+
+    test "returns an error if file could not be deleted" do
+      filepath = "/nonexistent/file.json"
+
+      assert {:error, _} = FilesystemHelpers.delete_file_and_remove_empty_directories(filepath)
+    end
+  end
 end

test/pinchflat/media_test.exs

@@ -360,57 +360,6 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
-  describe "media_filepaths/1" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: "/video/test.jpg",
-        subtitle_filepaths: [["en", "video/test.srt"]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == [
-               "/video/test.mp4",
-               "/video/test.jpg",
-               "video/test.srt"
-             ]
-    end
-
-    test "strips out nil values" do
-      filepaths = %{
-        media_filepath: "/video/test.mp4",
-        thumbnail_filepath: nil,
-        subtitle_filepaths: [["en", nil]]
-      }
-
-      media_item = media_item_fixture(filepaths)
-
-      assert Media.media_filepaths(media_item) == ["/video/test.mp4"]
-    end
-  end
-
-  describe "metadata_filepaths" do
-    test "returns filepaths in a flat list" do
-      filepaths = %{
-        metadata_filepath: "/metadata.json.gz",
-        thumbnail_filepath: "/thumbnail.jpg"
-      }
-
-      media_item = media_item_fixture(%{metadata: filepaths})
-
-      assert Media.metadata_filepaths(media_item) == [
-               "/metadata.json.gz",
-               "/thumbnail.jpg"
-             ]
-    end
-
-    test "returns an empty list when there is no metadata" do
-      media_item = media_item_fixture()
-      assert Media.metadata_filepaths(media_item) == []
-    end
-  end
-
   describe "create_media_item/1" do
     test "creating with valid data creates a media_item" do
       valid_attrs = %{
@@ -515,6 +464,26 @@ defmodule Pinchflat.MediaTest do
       assert {:ok, _} = Media.delete_media_item(media_item)
       assert File.exists?(media_item.media_filepath)
     end
+
+    test "does delete the media item's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      media_item = Repo.preload(media_item_with_attachments(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
+          thumbnail_filepath:
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
+        }
+      }
+
+      {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs)
+
+      assert {:ok, _} = Media.delete_media_item(updated_media_item)
+      refute File.exists?(updated_media_item.metadata.metadata_filepath)
+    end
   end
 
   describe "delete_media_item/2 when testing file deletion" do

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -84,4 +84,12 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
       assert Path.basename(filepath) == "maxres.webp"
     end
   end
+
+  describe "parse_upload_date/1" do
+    test "returns a date from the given metadata upload date" do
+      upload_date = "20210101"
+
+      assert Helpers.parse_upload_date(upload_date) == ~D[2021-01-01]
+    end
+  end
 end

test/pinchflat/metadata/metadata_parser_test.exs

@@ -4,22 +4,7 @@ defmodule Pinchflat.Metadata.MetadataParserTest do
   alias Pinchflat.Metadata.MetadataParser, as: Parser
 
   setup do
-    json_filepath =
+    {:ok, %{metadata: render_parsed_metadata(:media_metadata)}}
-      Path.join([
-        File.cwd!(),
-        "test",
-        "support",
-        "files",
-        "media_metadata.json"
-      ])
-
-    {:ok, file_body} = File.read(json_filepath)
-    {:ok, parsed_json} = Phoenix.json_library().decode(file_body)
-
-    {:ok,
-     %{
-       metadata: parsed_json
-     }}
   end
 
   describe "parse_for_media_item/1 when testing media metadata" do
@@ -109,6 +94,15 @@ defmodule Pinchflat.Metadata.MetadataParserTest do
       assert String.ends_with?(result.thumbnail_filepath, ".webp")
     end
 
+    # NOTE: this can be removed once this bug is fixed
+    # https://github.com/yt-dlp/yt-dlp/issues/9445
+    # and the associated conditional in the parser is removed
+    test "automatically appends `-thumb` to the thumbnail filename", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert String.contains?(result.thumbnail_filepath, "-thumb.webp")
+    end
+
     test "doesn't freak out if the media has no thumbnails", %{metadata: metadata} do
       metadata = Map.put(metadata, "thumbnails", %{})
 

test/pinchflat/metadata/nfo_builder_test.exs

@@ -0,0 +1,38 @@
+defmodule Pinchflat.Metadata.NfoBuilderTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Metadata.NfoBuilder
+
+  setup do
+    {:ok, %{metadata: render_parsed_metadata(:media_metadata)}}
+  end
+
+  describe "build_and_store_for_media_item/1" do
+    test "returns the filepath", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+
+      assert File.exists?(result)
+
+      File.rm!(result)
+    end
+
+    test "builds filepath based on media location", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+
+      assert String.contains?(result, Path.rootname(metadata["filepath"]))
+      assert String.ends_with?(result, ".nfo")
+
+      File.rm!(result)
+    end
+
+    test "builds an NFO file", %{metadata: metadata} do
+      result = NfoBuilder.build_and_store_for_media_item(metadata)
+      nfo = File.read!(result)
+
+      assert String.contains?(nfo, ~S(<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>))
+      assert String.contains?(nfo, "<title>#{metadata["title"]}</title>")
+
+      File.rm!(result)
+    end
+  end
+end

test/pinchflat/metadata/source_metadata_storage_worker_test.exs

@@ -0,0 +1,82 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
+
+  setup :verify_on_exit!
+
+  describe "kickoff_with_task/1" do
+    test "enqueues a new worker for the source" do
+      source = source_fixture()
+
+      assert {:ok, _} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
+
+    test "creates a new task for the source" do
+      source = source_fixture()
+
+      assert {:ok, task} = SourceMetadataStorageWorker.kickoff_with_task(source)
+
+      assert task.source_id == source.id
+    end
+  end
+
+  describe "perform/1" do
+    test "sets metadata location for source" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source = Repo.preload(source_fixture(), :metadata)
+
+      refute source.metadata
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+
+      assert source.metadata.metadata_filepath
+
+      File.rm!(source.metadata.metadata_filepath)
+    end
+
+    test "fetches and stores returned metadata for source" do
+      source = source_fixture()
+      file_contents = Phoenix.json_library().encode!(%{"title" => "test"})
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, file_contents} end)
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      source = Repo.preload(Repo.reload(source), :metadata)
+      {:ok, metadata} = MetadataFileHelpers.read_compressed_metadata(source.metadata.metadata_filepath)
+
+      assert metadata == %{"title" => "test"}
+    end
+
+    test "won't call itself in an infinite loop" do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source = source_fixture()
+
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source.id})
+
+      assert [_] = all_enqueued(worker: SourceMetadataStorageWorker)
+    end
+
+    test "doesn't prevent over source jobs from running" do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "{}"} end)
+      source_1 = source_fixture()
+      source_2 = source_fixture()
+
+      perform_job(SourceMetadataStorageWorker, %{id: source_1.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_1.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_2.id})
+      perform_job(SourceMetadataStorageWorker, %{id: source_2.id})
+
+      assert [_, _] = all_enqueued(worker: SourceMetadataStorageWorker)
+    end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(SourceMetadataStorageWorker, %{id: 0})
+    end
+  end
+end

test/pinchflat/profiles_test.exs

@@ -26,11 +26,11 @@ defmodule Pinchflat.ProfilesTest do
 
   describe "create_media_profile/1" do
     test "creation with valid data creates a media_profile" do
-      valid_attrs = %{name: "some name", output_path_template: "some output_path_template"}
+      valid_attrs = %{name: "some name", output_path_template: "output_template.{{ ext }}"}
 
       assert {:ok, %MediaProfile{} = media_profile} = Profiles.create_media_profile(valid_attrs)
       assert media_profile.name == "some name"
-      assert media_profile.output_path_template == "some output_path_template"
+      assert media_profile.output_path_template == "output_template.{{ ext }}"
     end
 
     test "creation with invalid data returns error changeset" do
@@ -44,14 +44,14 @@ defmodule Pinchflat.ProfilesTest do
 
       update_attrs = %{
         name: "some updated name",
-        output_path_template: "some updated output_path_template"
+        output_path_template: "new_output_template.{{ ext }}"
       }
 
       assert {:ok, %MediaProfile{} = media_profile} =
                Profiles.update_media_profile(media_profile, update_attrs)
 
       assert media_profile.name == "some updated name"
-      assert media_profile.output_path_template == "some updated output_path_template"
+      assert media_profile.output_path_template == "new_output_template.{{ ext }}"
     end
 
     test "updating with invalid data returns error changeset" do
@@ -132,5 +132,41 @@ defmodule Pinchflat.ProfilesTest do
       media_profile = media_profile_fixture()
       assert %Ecto.Changeset{} = Profiles.change_media_profile(media_profile)
     end
+
+    test "it ensures the media profile's output template ends with an extension" do
+      valid_templates = [
+        "output_template.{{ ext }}",
+        "output_template.{{ext}}",
+        "output_template.%(ext)s",
+        "output_template.%(ext)S",
+        "output_template.%( ext )s",
+        "output_template.%( ext )S"
+      ]
+
+      for template <- valid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        assert cs.valid?
+      end
+    end
+
+    test "it does not allow invalid output templates" do
+      invalid_templates = [
+        "output_template.{{ ext }}.something",
+        "output_template.{{   ext   }}",
+        "output_template{{ ext }}",
+        "output_template.%(ext)s.something",
+        "output_template.txt",
+        "output_template%(ext)s",
+        "output_template.%(nope)s",
+        "output_template"
+      ]
+
+      for template <- invalid_templates do
+        cs = Profiles.change_media_profile(%MediaProfile{}, %{name: "a", output_path_template: template})
+
+        refute cs.valid?
+      end
+    end
   end
 end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -101,7 +101,10 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
 
       source = source_fixture(index_frequency_minutes: 10)
-      task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
+
+      task_count_fetcher = fn ->
+        Enum.count(Tasks.list_tasks_for(source, "MediaCollectionIndexingWorker"))
+      end
 
       assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
@@ -158,5 +161,9 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
         perform_job(MediaCollectionIndexingWorker, %{id: source.id})
       end)
     end
+
+    test "does not blow up if the record doesn't exist" do
+      assert :ok = perform_job(MediaCollectionIndexingWorker, %{id: 0})
+    end
   end
 end

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -149,7 +149,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
 
       SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+      assert [] = Tasks.list_tasks_for(media_item)
     end
 
     test "it doesn't blow up if a media item cannot be coerced into a struct", %{source: source} do
@@ -290,5 +290,22 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
       assert Repo.aggregate(MediaItem, :count, :id) == 3
       assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
     end
+
+    test "does not blow up if the file returns invalid json", %{source: source} do
+      watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
+        filepath = Keyword.get(addl_opts, :output_filepath)
+        File.write(filepath, "INVALID")
+
+        # Need to add a delay to ensure the file watcher has time to read the file
+        :timer.sleep(watcher_poll_interval * 2)
+        # We know we're testing the file watcher since the syncronous call will only
+        # return an empty string (creating no records)
+        {:ok, ""}
+      end)
+
+      assert [] = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+    end
   end
 end

test/pinchflat/sources_test.exs

@@ -8,16 +8,32 @@ defmodule Pinchflat.SourcesTest do
 
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
+  alias Pinchflat.Metadata.MetadataFileHelpers
   alias Pinchflat.Downloading.DownloadingHelpers
   alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
   alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
   alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   @invalid_source_attrs %{name: nil, collection_id: nil}
 
   setup :verify_on_exit!
 
+  describe "schema" do
+    test "source_metadata is deleted when the source is deleted" do
+      source =
+        source_fixture(%{metadata: %{metadata_filepath: "/metadata.json.gz"}})
+
+      metadata = source.metadata
+      assert {:ok, %Source{}} = Sources.delete_source(source)
+
+      assert_raise Ecto.NoResultsError, fn ->
+        Repo.reload!(metadata)
+      end
+    end
+  end
+
   describe "list_sources/0" do
     test "it returns all sources" do
       source = source_fixture()
@@ -220,6 +236,21 @@ defmodule Pinchflat.SourcesTest do
 
       assert source.index_frequency_minutes == 0
     end
+
+    test "creating will kickoff a metadata storage worker" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: false,
+        index_frequency_minutes: 0
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
   end
 
   describe "update_source/2" do
@@ -384,6 +415,15 @@ defmodule Pinchflat.SourcesTest do
 
       assert source.index_frequency_minutes == 0
     end
+
+    test "updating will kickoff a metadata storage worker" do
+      source = source_fixture()
+      update_attrs = %{name: "some updated name"}
+
+      assert {:ok, %Source{} = source} = Sources.update_source(source, update_attrs)
+
+      assert_enqueued(worker: SourceMetadataStorageWorker, args: %{"id" => source.id})
+    end
   end
 
   describe "delete_source/2" do
@@ -421,6 +461,22 @@ defmodule Pinchflat.SourcesTest do
       assert {:ok, %Source{}} = Sources.delete_source(source)
       assert File.exists?(media_item.media_filepath)
     end
+
+    test "deletes the source's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      source = Repo.preload(source_fixture(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, %{})
+        }
+      }
+
+      {:ok, updated_source} = Sources.update_source(source, update_attrs)
+
+      assert {:ok, _} = Sources.delete_source(updated_source, delete_files: true)
+      refute File.exists?(updated_source.metadata.metadata_filepath)
+    end
   end
 
   describe "delete_source/2 when deleting files" do
@@ -452,70 +508,6 @@ defmodule Pinchflat.SourcesTest do
     end
   end
 
-  describe "change_source_from_url/2" do
-    test "it returns a changeset" do
-      stub(YtDlpRunnerMock, :run, &channel_mock/3)
-      source = source_fixture()
-
-      assert %Ecto.Changeset{} = Sources.change_source_from_url(source, %{})
-    end
-
-    test "it does not fetch source details if the original_url isn't in the changeset" do
-      expect(YtDlpRunnerMock, :run, 0, &channel_mock/3)
-
-      changeset = Sources.change_source_from_url(%Source{}, %{name: "some updated name"})
-
-      assert %Ecto.Changeset{} = changeset
-    end
-
-    test "it fetches source details if the original_url is in the changeset" do
-      expect(YtDlpRunnerMock, :run, &channel_mock/3)
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123"
-        })
-
-      assert %Ecto.Changeset{} = changeset
-    end
-
-    test "it adds source details to the changeset, keeping the orignal details" do
-      expect(YtDlpRunnerMock, :run, &channel_mock/3)
-
-      media_profile = media_profile_fixture()
-      media_profile_id = media_profile.id
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123",
-          media_profile_id: media_profile.id
-        })
-
-      assert %Ecto.Changeset{} = changeset
-      assert String.starts_with?(changeset.changes.collection_id, "some_channel_id_")
-
-      assert %{
-               collection_name: "some channel name",
-               media_profile_id: ^media_profile_id,
-               original_url: "https://www.youtube.com/channel/abc123"
-             } = changeset.changes
-    end
-
-    test "it adds an error to the changeset if the runner fails" do
-      expect(YtDlpRunnerMock, :run, 1, fn _url, _opts, _ot ->
-        {:error, "some error", 1}
-      end)
-
-      changeset =
-        Sources.change_source_from_url(%Source{}, %{
-          original_url: "https://www.youtube.com/channel/abc123"
-        })
-
-      assert %Ecto.Changeset{} = changeset
-      assert errors_on(changeset).original_url == ["could not fetch source details from URL"]
-    end
-  end
-
   defp playlist_mock(_url, _opts, _ot) do
     {
       :ok,

test/pinchflat/tasks_test.exs

@@ -36,53 +36,59 @@ defmodule Pinchflat.TasksTest do
     end
   end
 
-  describe "list_tasks_for/4" do
+  describe "list_tasks_for/3" do
     test "it lets you specify which record type/ID to join on" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id) == [task]
+      assert Tasks.list_tasks_for(source, nil, [:available]) == [task]
     end
 
     test "it lets you specify which job states to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil, [:available]) == [task]
+      assert Tasks.list_tasks_for(source, nil, [:available]) == [task]
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil, [:cancelled]) == []
+      assert Tasks.list_tasks_for(source, nil, [:cancelled]) == []
     end
 
     test "it lets you specify which worker to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, "TestJobWorker") == [task]
+      assert Tasks.list_tasks_for(source, "TestJobWorker") == [task]
-      assert Tasks.list_tasks_for(:source_id, task.source_id, "FooBarWorker") == []
+      assert Tasks.list_tasks_for(source, "FooBarWorker") == []
     end
 
     test "it includes all workers if no worker is specified" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_tasks_for(:source_id, task.source_id, nil) == [task]
+      assert Tasks.list_tasks_for(source, nil) == [task]
     end
   end
 
   describe "list_pending_tasks_for/3" do
     test "it lists pending tasks" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id) == [task]
+      assert Tasks.list_pending_tasks_for(source) == [task]
     end
 
     test "it does not list non-pending tasks" do
-      task = Repo.preload(task_fixture(), :job)
+      task = Repo.preload(task_fixture(), [:job, :source])
       :ok = Oban.cancel_job(task.job)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id) == []
+      assert Tasks.list_pending_tasks_for(task.source) == []
     end
 
     test "it lets you specify which worker to include" do
-      task = task_fixture()
+      source = source_fixture()
+      task = task_fixture(source_id: source.id)
 
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id, "TestJobWorker") == [task]
+      assert Tasks.list_pending_tasks_for(source, "TestJobWorker") == [task]
-      assert Tasks.list_pending_tasks_for(:source_id, task.source_id, "FooBarWorker") == []
+      assert Tasks.list_pending_tasks_for(source, "FooBarWorker") == []
     end
   end
 

test/pinchflat/yt_dlp/media_collection_test.exs

@@ -62,6 +62,15 @@ defmodule Pinchflat.YtDlp.MediaCollectionTest do
       assert_receive {:handler, filename}
       assert String.ends_with?(filename, ".json")
     end
+
+    test "gracefully handles partially failed responses" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        {:ok, "INVALID\n\n" <> source_attributes_return_fixture() <> "\nINVALID\n"}
+      end)
+
+      assert {:ok, [%Media{media_id: "video1"}, %Media{media_id: "video2"}, %Media{media_id: "video3"}]} =
+               MediaCollection.get_media_attributes_for_collection(@channel_url)
+    end
   end
 
   describe "get_source_details/1" do
@@ -108,4 +117,39 @@ defmodule Pinchflat.YtDlp.MediaCollectionTest do
       assert {:error, %Jason.DecodeError{}} = MediaCollection.get_source_details(@channel_url)
     end
   end
+
+  describe "get_source_metadata/1" do
+    test "it returns a map with data on success" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        Phoenix.json_library().encode(%{channel: "TheUselessTrials"})
+      end)
+
+      assert {:ok, res} = MediaCollection.get_source_metadata(@channel_url)
+
+      assert %{"channel" => "TheUselessTrials"} = res
+    end
+
+    test "it passes the expected args to the backend runner" do
+      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
+        assert opts == [playlist_items: 0]
+        assert ot == "playlist:%()j"
+
+        {:ok, "{}"}
+      end)
+
+      assert {:ok, _} = MediaCollection.get_source_metadata(@channel_url)
+    end
+
+    test "it returns an error if the runner returns an error" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
+
+      assert {:error, "Big issue", 1} = MediaCollection.get_source_metadata(@channel_url)
+    end
+
+    test "it returns an error if the output is not JSON" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "Not JSON"} end)
+
+      assert {:error, %Jason.DecodeError{}} = MediaCollection.get_source_metadata(@channel_url)
+    end
+  end
 end

test/pinchflat_web/controllers/media_profile_controller_test.exs

@@ -8,10 +8,10 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
   alias Pinchflat.Repo
   alias Pinchflat.Settings
 
-  @create_attrs %{name: "some name", output_path_template: "some output_path_template"}
+  @create_attrs %{name: "some name", output_path_template: "output_template.{{ ext }}"}
   @update_attrs %{
     name: "some updated name",
-    output_path_template: "some updated output_path_template"
+    output_path_template: "new_output_template.{{ ext }}"
   }
   @invalid_attrs %{name: nil, output_path_template: nil}
 

test/support/fixtures/sources_fixtures.ex

@@ -34,6 +34,20 @@ defmodule Pinchflat.SourcesFixtures do
     source
   end
 
+  @doc """
+  Generate a source with metadata.
+  """
+  def source_with_metadata(attrs \\ %{}) do
+    merged_attrs =
+      Map.merge(attrs, %{
+        metadata: %{
+          metadata_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/metadata.json.gz"
+        }
+      })
+
+    source_fixture(merged_attrs)
+  end
+
   def source_attributes_return_fixture do
     source_attributes = [
       %{

test/support/testing_helper_methods.ex

@@ -48,4 +48,10 @@ defmodule Pinchflat.TestingHelperMethods do
 
     File.read!(json_filepath)
   end
+
+  def render_parsed_metadata(metadata_name) do
+    metadata_name
+    |> render_metadata()
+    |> Phoenix.json_library().decode!()
+  end
 end