Bumped version

* Placeholder commit

* Added new logo to app

* added logo to header of small screens

* Updated favicon

* Added logo to README

* Updated README

.credo.exs

@@ -128,7 +128,7 @@
           {Credo.Check.Refactor.MatchInCondition, []},
           {Credo.Check.Refactor.NegatedConditionsInUnless, []},
           {Credo.Check.Refactor.NegatedConditionsWithElse, []},
-          {Credo.Check.Refactor.Nesting, []},
+          {Credo.Check.Refactor.Nesting, [max_nesting: 4]},
           {Credo.Check.Refactor.RedundantWithClauseResult, []},
           {Credo.Check.Refactor.RejectReject, []},
           {Credo.Check.Refactor.UnlessWithElse, []},
@@ -157,7 +157,16 @@
           {Credo.Check.Warning.UnusedRegexOperation, []},
           {Credo.Check.Warning.UnusedStringOperation, []},
           {Credo.Check.Warning.UnusedTupleOperation, []},
-          {Credo.Check.Warning.WrongTestFileExtension, []}
+          {Credo.Check.Warning.WrongTestFileExtension, []},
+
+          #
+          ## Naming Checks
+          #
+          {CredoNaming.Check.Consistency.ModuleFilename,
+           [
+             priority: :normal,
+             excluded_paths: [~r/test\/support/, ~r/priv/, ~r/lib\/pinchflat_web/, ~r/test\/pinchflat_web/]
+           ]}
         ],
         disabled: [
           #

.iex.exs

@@ -4,8 +4,7 @@ alias Pinchflat.Repo
 alias Pinchflat.Tasks.Task
 alias Pinchflat.Sources.Source
 alias Pinchflat.Media.MediaItem
-alias Pinchflat.Tasks.SourceTasks
-alias Pinchflat.Media.MediaMetadata
+alias Pinchflat.Metadata.MediaMetadata
 alias Pinchflat.Profiles.MediaProfile
 
 alias Pinchflat.Tasks
@@ -14,14 +13,14 @@ alias Pinchflat.Profiles
 alias Pinchflat.Sources
 alias Pinchflat.Settings
 
-alias Pinchflat.MediaClient.MediaDownloader
-alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
+alias Pinchflat.Downloading.MediaDownloader
+alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+alias Pinchflat.YtDlp.MediaCollection, as: YtDlpCollection
 
-alias Pinchflat.Api.YoutubeRss
+alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
-alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+alias Pinchflat.SlowIndexing.FileFollowerServer
 
 defmodule IexHelpers do
   def playlist_url do

README.md

@@ -1,7 +1,27 @@
-# Pinchflat (Alpha)
+<p align="center">  
+  <img 
+    src="priv/static/images/originals/logo-white-wordmark-with-background.png" 
+    alt="Pinchflat Logo by @hernandito"
+    width="700" 
+  />
+</p>
+
+<p align="center">  
+  <sup>
+    <em>logo by <a href="https://github.com/hernandito" target="_blank">@hernandito</a></em>
+  </sup>
+</p>
+
+# Your next YouTube media manager
+
+## Disclaimer
 
 This is alpha software and anything can break at any time. I make not guarantees about the stability of this software, forward-compatibility of updates, or integrity (both related to and independent of Pinchflat). Essentially, use at your own risk and expect there will be rough edges.
 
+## EFF Donation Receipts
+
+A portion of all donations to Pinchflat will be donated to the Electronic Frontier Foundation. [See here](https://github.com/kieraneglin/pinchflat/wiki/EFF-Donation-Receipts) for a list of donation receipts.
+
 ## What is Pinchflat?
 
 TODO: expand on this.

config/config.exs

@@ -12,7 +12,7 @@ config :pinchflat,
   generators: [timestamp_type: :utc_datetime],
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
-  yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
+  yt_dlp_runner: Pinchflat.YtDlp.CommandRunner,
   media_directory: "/downloads",
   # The user may or may not store metadata for their needs, but the app will always store its copy
   metadata_directory: "/config/metadata",
@@ -48,7 +48,8 @@ config :pinchflat, Oban,
     media_indexing: 2,
     media_collection_indexing: 2,
     media_fetching: 2,
-    media_local_metadata: 8
+    local_metadata: 8,
+    remote_metadata: 4
   ]
 
 # Configures the mailer

config/runtime.exs

@@ -25,6 +25,21 @@ config :pinchflat,
   basic_auth_username: System.get_env("BASIC_AUTH_USERNAME"),
   basic_auth_password: System.get_env("BASIC_AUTH_PASSWORD")
 
+arch_string = to_string(:erlang.system_info(:system_architecture))
+
+system_arch =
+  cond do
+    String.contains?(arch_string, "arm") -> "arm"
+    String.contains?(arch_string, "aarch") -> "arm"
+    String.contains?(arch_string, "x86") -> "x86"
+    true -> "unknown"
+  end
+
+config :pinchflat, Pinchflat.Repo,
+  load_extensions: [
+    Path.join([:code.priv_dir(:pinchflat), "repo", "extensions", "sqlean-linux-#{system_arch}", "sqlean"])
+  ]
+
 if config_env() == :prod do
   config_path =
     System.get_env("CONFIG_PATH") ||

lib/pinchflat/application.ex

@@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
       # Must be before startup tasks
+      Pinchflat.Boot.PreJobStartupTasks,
       {Oban, Application.fetch_env!(:pinchflat, Oban)},
-      Pinchflat.StartupTasks,
+      Pinchflat.Boot.PostJobStartupTasks,
       {DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: Pinchflat.PubSub},
       # Start the Finch HTTP client for sending emails

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Workers.DataBackfillWorker do
+defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
-    queue: :media_local_metadata,
+    queue: :local_metadata,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
 
@@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
   """
   def cancel_pending_backfill_jobs do
     Oban.Job
-    |> where(worker: "Pinchflat.Workers.DataBackfillWorker")
+    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
     |> Oban.cancel_all_jobs()
   end
 

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,6 +1,7 @@
-defmodule Pinchflat.StartupTasks do
+defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
-  This module is responsible for running startup tasks on app boot.
+  This module is responsible for running startup tasks on app boot
+  AFTER the job runner has initiallized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Settings
-  alias Pinchflat.Workers.DataBackfillWorker
+  alias Pinchflat.Boot.DataBackfillWorker
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
   """
   @impl true
   def init(state) do
-    apply_default_settings()
     enqueue_backfill_worker()
 
     {:ok, state}
   end
 
-  defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
-    Settings.fetch!(:pro_enabled, false)
-  end
-
   defp enqueue_backfill_worker do
     DataBackfillWorker.cancel_pending_backfill_jobs()
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -0,0 +1,89 @@
+defmodule Pinchflat.Boot.PreJobStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  BEFORE the job runner has initiallized.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Settings
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{}, opts)
+  end
+
+  @doc """
+  Runs application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(state) do
+    apply_default_settings()
+    ensure_directories_are_writeable()
+    rename_old_job_workers()
+
+    {:ok, state}
+  end
+
+  defp apply_default_settings do
+    Settings.fetch!(:onboarding, true)
+    Settings.fetch!(:pro_enabled, false)
+  end
+
+  defp ensure_directories_are_writeable do
+    directories = [
+      Application.get_env(:pinchflat, :media_directory),
+      Application.get_env(:pinchflat, :tmpfile_directory),
+      Application.get_env(:pinchflat, :metadata_directory)
+    ]
+
+    Enum.each(directories, fn dir ->
+      file = Path.join([dir, ".keep"])
+
+      # This will fail if the directory is not writeable, stopping boot
+      FilesystemHelpers.write_p!(file, "")
+    end)
+  end
+
+  # As part of a large refactor, I ended up moving a bunch of workers around. This
+  # is a problem because the workers are stored in the database and the runner
+  # will try to run the OLD jobs. This is also why these tasks run before the job
+  # runner starts up.
+  #
+  # Can be removed after a few months (created: 2024-03-12)
+  defp rename_old_job_workers do
+    # [ [old_name, new_name], ...]
+    rename_map = [
+      ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
+      ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
+      ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
+      ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
+      ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
+    ]
+
+    jobs_renamed =
+      Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
+        {count, _} =
+          Oban.Job
+          |> where(worker: ^old_name)
+          |> Repo.update_all(set: [worker: new_name])
+
+        acc + count
+      end)
+
+    Logger.info("Renamed #{jobs_renamed} old job workers")
+  end
+end

lib/pinchflat/downloading/download_option_builder.ex

@@ -1,10 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
+  alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   @doc """
   Builds the options for yt-dlp to download media based on the given media's profile.
@@ -18,7 +19,7 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
-        thumbnail_options(media_profile) ++
+        thumbnail_options(media_item_with_preloads) ++
         metadata_options(media_profile) ++
         quality_options(media_profile) ++
         output_options(media_item_with_preloads)
@@ -26,6 +27,18 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     {:ok, built_options}
   end
 
+  @doc """
+  Builds the output path for yt-dlp to download media based on the given source's
+  media profile.
+
+  Returns binary()
+  """
+  def build_output_path_for(%Source{} = source_with_preloads) do
+    output_path_template = source_with_preloads.media_profile.output_path_template
+
+    build_output_path(output_path_template, source_with_preloads)
+  end
+
   defp default_options do
     [:no_progress, :windows_filenames]
   end
@@ -57,16 +70,19 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
     end)
   end
 
-  defp thumbnail_options(media_profile) do
+  defp thumbnail_options(media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
     mapped_struct = Map.from_struct(media_profile)
 
     Enum.reduce(mapped_struct, [], fn attr, acc ->
       case attr do
         {:download_thumbnail, true} ->
-          acc ++ [:write_thumbnail, convert_thumbnail: "jpg"]
+          thumbnail_save_location = determine_thumbnail_location(media_item_with_preloads)
+
+          acc ++ [:write_thumbnail, convert_thumbnail: "jpg", output: "thumbnail:#{thumbnail_save_location}"]
 
         {:embed_thumbnail, true} ->
-          acc ++ [:embed_thumbnail]
+          acc ++ [:embed_thumbnail, convert_thumbnail: "jpg"]
 
         _ ->
           acc
@@ -96,30 +112,43 @@ defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
       :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
       :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
       :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
-      :"1440p" -> [format_sort: "res:1440,#{video_codec_options}"]
       :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
     end
   end
 
   defp output_options(media_item_with_preloads) do
-    media_profile = media_item_with_preloads.source.media_profile
-    additional_options_map = output_options_map(media_item_with_preloads)
-    {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
-
     [
-      output: Path.join(base_directory(), output_path)
+      output: build_output_path_for(media_item_with_preloads.source)
     ]
   end
 
-  defp output_options_map(media_item_with_preloads) do
-    source = media_item_with_preloads.source
+  defp build_output_path(string, source) do
+    additional_options_map = output_options_map(source)
+    {:ok, output_path} = OutputPathBuilder.build(string, additional_options_map)
 
+    Path.join(base_directory(), output_path)
+  end
+
+  defp output_options_map(source) do
     %{
       "source_custom_name" => source.custom_name,
       "source_collection_type" => source.collection_type
     }
   end
 
+  # I don't love the string manipulation here, but what can ya' do.
+  # It's dependent on the output_path_template being a string ending `.{{ ext }}`
+  # (or equivalent), but that's validated by the MediaProfile schema.
+  defp determine_thumbnail_location(media_item_with_preloads) do
+    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
+
+    output_path_template
+    |> String.split(~r{\.}, include_captures: true)
+    |> List.insert_at(-3, "-thumb")
+    |> Enum.join()
+    |> build_output_path(media_item_with_preloads.source)
+  end
+
   defp base_directory do
     Application.get_env(:pinchflat, :media_directory)
   end

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&MediaDownloadWorker.kickoff_with_task/1)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaDownloadWorker do
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,19 +6,32 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Repo
   alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.MediaClient.MediaDownloader
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Downloading.MediaDownloader
+
+  @doc """
+  Starts the media_item media download worker and creates a task for the media_item.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(media_item, opts \\ []) do
+    %{id: media_item.id}
+    |> MediaDownloadWorker.new(opts)
+    |> Tasks.create_job_with_task(media_item)
+  end
 
-  @impl Oban.Worker
   @doc """
   For a given media item, download the media alongside any options.
   Does not download media if its source is set to not download media.
 
   Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item =
       media_item_id
@@ -31,26 +44,30 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
   defp download_media_and_schedule_jobs(media_item) do
     case MediaDownloader.download_for_media_item(media_item) do
-      {:ok, _} ->
-        schedule_filesystem_data_worker(media_item)
-        {:ok, media_item}
+      {:ok, updated_media_item} ->
+        compute_and_save_media_filesize(updated_media_item)
+
+        {:ok, updated_media_item}
 
       err ->
         err
     end
   end
 
-  defp schedule_filesystem_data_worker(media_item) do
-    %{id: media_item.id}
-    |> FilesystemDataWorker.new()
-    |> Tasks.create_job_with_task(media_item)
-    |> case do
-      {:ok, task} -> {:ok, task}
-      {:error, :duplicate_job} -> {:ok, :job_exists}
+  defp compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      _ ->
+        :ok
     end
   end
 end

lib/pinchflat/downloading/media_downloader.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.MediaDownloader do
+defmodule Pinchflat.Downloading.MediaDownloader do
   @moduledoc """
   This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
@@ -8,11 +8,12 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.Metadata.MetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
-  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
-  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Downloads media for a media item, updating the media item based on the metadata
@@ -30,16 +31,17 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
 
     case download_with_options(media_item.original_url, item_with_preloads) do
       {:ok, parsed_json} ->
-        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
-
         parsed_attrs =
           parsed_json
-          |> parser.parse_for_media_item()
+          |> MetadataParser.parse_for_media_item()
           |> Map.merge(%{
             media_downloaded_at: DateTime.utc_now(),
+            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
             metadata: %{
-              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
-              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+              # IDEA: might be worth kicking off a job for this since thumbnail fetching
+              # could fail and I want to handle that in isolation
+              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
             }
           })
 
@@ -52,13 +54,18 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
     end
   end
 
-  # def download_for_source(source, url) do
-  #   # Create MI from source and URL
-  #   media_item = nil
-  # end
+  defp determine_nfo_filepath(media_item, parsed_json) do
+    if media_item.source.media_profile.download_nfo do
+      filepath = Path.rootname(parsed_json["filepath"]) <> ".nfo"
+
+      NfoBuilder.build_and_store_for_media_item(filepath, parsed_json)
+    else
+      nil
+    end
+  end
 
   defp download_with_options(url, item_with_preloads) do
-    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
+    {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
     YtDlpMedia.download(url, options)
   end

lib/pinchflat/downloading/output_path/base.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.RenderedString.Base do
+defmodule Pinchflat.Downloading.OutputPath.Base do
   @moduledoc """
   A base module for parsing rendered strings, designed as a macro to be used
   in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
   NOTE: if the needs here get any more complicated, look into using a Liquid
   template parser. No need to reinvent the wheel any more than I already have.
 
-  NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
   """
 
   defmacro __using__(_opts) do

lib/pinchflat/downloading/output_path/parser.ex

@@ -1,11 +1,11 @@
-defmodule Pinchflat.RenderedString.Parser do
+defmodule Pinchflat.Downloading.OutputPath.Parser do
   @moduledoc """
   Parses liquid-ish-style strings into a rendered string
 
   Used for turning filepath templates into real filepaths
   """
 
-  use Pinchflat.RenderedString.Base
+  use Pinchflat.Downloading.OutputPath.Base
 
   @doc """
   Parses a string into a rendered string, using the provided variables. Optionally

lib/pinchflat/downloading/output_path_builder.ex

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Profiles.OutputPathBuilder do
+defmodule Pinchflat.Downloading.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
   """
 
-  alias Pinchflat.RenderedString.Parser, as: TemplateParser
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
 
   @doc """
   Builds the actual final filepath from a given template. Optionally, you can pass in
@@ -40,7 +40,9 @@ defmodule Pinchflat.Profiles.OutputPathBuilder do
       "upload_year" => "%(upload_date>%Y)S",
       "upload_month" => "%(upload_date>%m)S",
       "upload_day" => "%(upload_date>%d)S",
-      "upload_yyyy_mm_dd" => "%(upload_date>%Y)S-%(upload_date>%m)S-%(upload_date>%d)S"
+      "upload_yyyy_mm_dd" => "%(upload_date>%Y-%m-%d)S",
+      "season_from_date" => "%(upload_date>%Y)S",
+      "season_episode_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S"
     }
   end
 end

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  @moduledoc """
+  Methods for performing fast indexing tasks and managing the fast indexing process.
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  alias Pinchflat.Media
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      MediaIndexingWorker.kickoff_with_task(source, url)
+    end)
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          MediaDownloadWorker.kickoff_with_task(media_item)
+        end
+
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,13 +6,25 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "fast_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the source fast indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> FastIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Kicks off the fast indexing process for a source, reschedules the job to run again
   once complete. See `MediaCollectionIndexingWorker` and `MediaIndexingWorker` comments
@@ -20,25 +32,26 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
 
   Returns :ok | {:ok, :job_exists} | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else
       :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do
     next_run_in = Source.fast_index_frequency() * 60
 
-    %{id: source.id}
-    |> FastIndexingWorker.new(schedule_in: next_run_in)
-    |> Tasks.create_job_with_task(source)
-    |> case do
+    case kickoff_with_task(source, schedule_in: next_run_in) do
       {:ok, task} -> {:ok, task}
       {:error, :duplicate_job} -> {:ok, :job_exists}
     end

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorker do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -8,10 +8,22 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   require Logger
 
+  alias __MODULE__
+  alias Pinchflat.Tasks
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+
+  @doc """
+  Starts the fast media indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, media_url, opts \\ []) do
+    %{id: source.id, media_url: media_url}
+    |> MediaIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   Similar to `MediaCollectionIndexingWorker`, but for individual media items.
   Does not reschedule or check anything to do with a source's indexing
@@ -23,7 +35,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item
@@ -37,10 +49,11 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
 
   Returns :ok
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
-    case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
       {:ok, media_item} ->
         Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
 
@@ -49,5 +62,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
     end
 
     :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 end

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -0,0 +1,96 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpers do
+  @moduledoc """
+  Utility methods for working with the filesystem
+  """
+  alias Pinchflat.Media
+  alias Pinchflat.Utils.StringUtils
+
+  @doc """
+  Generates a temporary file and returns its path. The file is empty and has the given type.
+  Generates all the directories in the path if they don't exist.
+
+  Returns binary()
+  """
+  def generate_metadata_tmpfile(type) do
+    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
+
+    :ok = write_p!(filepath, "")
+
+    filepath
+  end
+
+  @doc """
+  Writes content to a file, creating directories as needed.
+  Takes the same args as File.write!/3.
+
+  Returns :ok | raises on error
+  """
+  def write_p!(filepath, content, modes \\ []) do
+    filepath
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.write!(filepath, content, modes)
+  end
+
+  @doc """
+  Copies a file from source to destination, creating directories as needed.
+
+  Returns :ok | raises on error
+  """
+  def cp_p!(source, destination) do
+    destination
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    File.cp!(source, destination)
+  end
+
+  @doc """
+  Fetches the file size of a media item and saves it to the database.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      err ->
+        err
+    end
+  end
+
+  @doc """
+  Deletes a file and removes any empty directories in the path.
+  Does NOT remove any directories that are not empty.
+
+  Returns :ok | {:error, any()}
+  """
+  def delete_file_and_remove_empty_directories(filepath) do
+    case File.rm(filepath) do
+      :ok ->
+        filepath
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+  end
+
+  defp recursively_delete_empty_directories(directory) do
+    case File.rmdir(directory) do
+      :ok ->
+        directory
+        |> Path.dirname()
+        |> recursively_delete_empty_directories()
+
+      err ->
+        err
+    end
+
+    :ok
+  end
+end

lib/pinchflat/media/media.ex

@@ -7,9 +7,10 @@ defmodule Pinchflat.Media do
 
   alias Pinchflat.Repo
   alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @doc """
   Returns the list of media_items.
@@ -65,6 +66,7 @@ defmodule Pinchflat.Media do
     |> where([mi], mi.source_id == ^source.id and is_nil(mi.media_filepath))
     |> where(^build_format_clauses(media_profile))
     |> where(^maybe_apply_cutoff_date(source))
+    |> where(^maybe_apply_title_regex(source))
     |> Repo.maybe_limit(limit)
     |> Repo.all()
   end
@@ -141,38 +143,6 @@ defmodule Pinchflat.Media do
   """
   def get_media_item!(id), do: Repo.get!(MediaItem, id)
 
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's downloaded files
-
-  Returns [binary()]
-  """
-  def media_filepaths(media_item) do
-    mapped_struct = Map.from_struct(media_item)
-
-    MediaItem.filepath_attributes()
-    |> Enum.map(fn
-      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
-      field -> List.wrap(mapped_struct[field])
-    end)
-    |> List.flatten()
-    |> Enum.filter(&is_binary/1)
-  end
-
-  @doc """
-  Produces a flat list of the filesystem paths for a media_item's metadata files.
-  Returns an empty list if the media_item has no metadata.
-
-  Returns [binary()] | []
-  """
-  def metadata_filepaths(media_item) do
-    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
-    mapped_struct = Map.from_struct(metadata)
-
-    MediaMetadata.filepath_attributes()
-    |> Enum.map(fn field -> mapped_struct[field] end)
-    |> Enum.filter(&is_binary/1)
-  end
-
   @doc """
   Creates a media_item.
 
@@ -219,19 +189,22 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Deletes a media_item and its associated tasks.
-  Can optionally delete the media_item's files.
+  Deletes a media_item, its associated tasks, and our internal metadata files.
+  Can optionally delete the media_item's media files (media, thumbnail, subtitles, etc).
 
   Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
   """
   def delete_media_item(%MediaItem{} = media_item, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
 
+    Tasks.delete_tasks_for(media_item)
+
     if delete_files do
-      {:ok, _} = delete_all_attachments(media_item)
+      {:ok, _} = delete_media_files(media_item)
     end
 
-    Tasks.delete_tasks_for(media_item)
+    # Should delete these no matter what
+    delete_internal_metadata_files(media_item)
     Repo.delete(media_item)
   end
 
@@ -242,26 +215,31 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_all_attachments(media_item) do
-    media_item = Repo.preload(media_item, :metadata)
+  defp delete_media_files(media_item) do
+    mapped_struct = Map.from_struct(media_item)
 
-    media_item
-    |> media_filepaths()
-    |> Enum.concat(metadata_filepaths(media_item))
-    |> Enum.each(&File.rm/1)
-
-    # rmdir will attempt to delete the directory, but only if it is empty
-    if media_item.media_filepath do
-      File.rmdir(Path.dirname(media_item.media_filepath))
-    end
-
-    if media_item.metadata && media_item.metadata.metadata_filepath do
-      File.rmdir(Path.dirname(media_item.metadata.metadata_filepath))
-    end
+    MediaItem.filepath_attributes()
+    |> Enum.map(fn
+      :subtitle_filepaths = field -> Enum.map(mapped_struct[field], fn [_, filepath] -> filepath end)
+      field -> List.wrap(mapped_struct[field])
+    end)
+    |> List.flatten()
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
 
     {:ok, media_item}
   end
 
+  defp delete_internal_metadata_files(media_item) do
+    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    MediaMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
   defp maybe_apply_cutoff_date(source) do
     if source.download_cutoff_date do
       dynamic([mi], mi.upload_date >= ^source.download_cutoff_date)
@@ -270,6 +248,14 @@ defmodule Pinchflat.Media do
     end
   end
 
+  defp maybe_apply_title_regex(source) do
+    if source.title_filter_regex do
+      dynamic([mi], fragment("regexp_like(?, ?)", mi.title, ^source.title_filter_regex))
+    else
+      dynamic(true)
+    end
+  end
+
   defp build_format_clauses(media_profile) do
     mapped_struct = Map.from_struct(media_profile)
 

lib/pinchflat/media/media_item.ex

@@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
-  alias Pinchflat.Media.MediaItemSearchIndex
+  alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Media.MediaItemsSearchIndex
 
   @allowed_fields [
     # these fields are captured on indexing (and again on download)
@@ -27,7 +27,8 @@ defmodule Pinchflat.Media.MediaItem do
     :media_size_bytes,
     :subtitle_filepaths,
     :thumbnail_filepath,
-    :metadata_filepath
+    :metadata_filepath,
+    :nfo_filepath
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -54,6 +55,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :media_size_bytes, :integer
     field :thumbnail_filepath, :string
     field :metadata_filepath, :string
+    field :nfo_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.
@@ -64,7 +66,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
-    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
+    has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 
@@ -82,6 +84,6 @@ defmodule Pinchflat.Media.MediaItem do
 
   @doc false
   def filepath_attributes do
-    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths)a
+    ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
 end

lib/pinchflat/media/media_items_search_index.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaItemSearchIndex do
+defmodule Pinchflat.Media.MediaItemsSearchIndex do
   @moduledoc """
   The MediaItem fts5 search index. Not made to be directly interacted with,
   but I figured it'd be better to have it in-app so it's not a mystery.

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -9,6 +9,8 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
   needed
   """
 
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
   @doc """
   Compresses and stores metadata for a media item, returning the filepath.
 
@@ -18,8 +20,7 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, "metadata.json.gz")
     {:ok, json} = Phoenix.json_library().encode(metadata_map)
 
-    File.mkdir_p!(Path.dirname(filepath))
-    :ok = File.write(filepath, json, [:compressed])
+    :ok = FilesystemHelpers.write_p!(filepath, json, [:compressed])
 
     filepath
   end
@@ -45,12 +46,60 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
     filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
     thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
 
-    File.mkdir_p!(Path.dirname(filepath))
-    :ok = File.write(filepath, thumbnail_blob)
+    :ok = FilesystemHelpers.write_p!(filepath, thumbnail_blob)
 
     filepath
   end
 
+  @doc """
+  Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
+  and returns a Date struct.
+
+  Returns Date.t()
+  """
+  def parse_upload_date(upload_date) do
+    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
+
+    Date.from_iso8601!("#{year}-#{month}-#{day}")
+  end
+
+  @doc """
+  Attempts to determine the series directory from a media filepath.
+  The series directory is the "root" directory for a given source
+  which should contain all the season-level folders of that source.
+
+  Used for determining where to store things like NFO data and banners
+  for media center apps. Not useful without a media center app.
+
+  Returns {:ok, binary()} | {:error, :indeterminable}
+  """
+  def series_directory_from_media_filepath(media_filepath) do
+    # Matches "s" or "season" (case-insensitive)
+    # followed by an optional non-word character (. or _ or <space>, etc)
+    # followed by at least one digit
+    # followed immediately by the end of the string
+    # Example matches: s1, s.1, s01 season 1, Season.01, Season_1, Season 1, Season1
+    # Example non-matches: s01e01, season, series 1,
+    season_regex = ~r/^s(eason)?(\W|_)?\d{1,}$/i
+
+    {series_directory, found_series_directory} =
+      media_filepath
+      |> Path.split()
+      |> Enum.reduce_while({[], false}, fn part, {directory_acc, _} ->
+        if String.match?(part, season_regex) do
+          {:halt, {directory_acc, true}}
+        else
+          {:cont, {directory_acc ++ [part], false}}
+        end
+      end)
+
+    if found_series_directory do
+      {:ok, Path.join(series_directory)}
+    else
+      {:error, :indeterminable}
+    end
+  end
+
   defp fetch_thumbnail_from_url(url) do
     http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
     {:ok, body} = http_client.get(url, [], body_format: :binary)

lib/pinchflat/metadata/metadata_parser.ex

@@ -54,10 +54,23 @@ defmodule Pinchflat.Metadata.MetadataParser do
       |> Enum.reverse()
       |> Enum.find_value(fn attrs -> attrs["filepath"] end)
 
+    if thumbnail_filepath do
+      # NOTE: whole ordeal needed due to a bug I found in yt-dlp
+      # https://github.com/yt-dlp/yt-dlp/issues/9445
+      # Can be reverted to remove this entire conditional once fixed
+      %{
+        thumbnail_filepath:
+          thumbnail_filepath
+          |> String.split(~r{\.}, include_captures: true)
+          |> List.insert_at(-3, "-thumb")
+          |> Enum.join()
+      }
+    else
       %{
         thumbnail_filepath: thumbnail_filepath
       }
     end
+  end
 
   defp parse_infojson_metadata(metadata) do
     %{

lib/pinchflat/metadata/nfo_builder.ex

@@ -0,0 +1,68 @@
+defmodule Pinchflat.Metadata.NfoBuilder do
+  @moduledoc """
+  Provides methods for building and storing NFO files for
+  use by Kodi/Jellyfin and other media center software.
+  """
+
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Builds an NFO file for a media item (read: single "episode") and
+  stores it at the specified location.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_media_item(filepath, metadata) do
+    nfo = build_for_media_item(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  @doc """
+  Builds an NFO file for a souce and stores it at the specified location.
+  Technically works for playlists, but it's really made for channels.
+
+  Returns the filepath of the NFO file.
+  """
+  def build_and_store_for_source(filepath, metadata) do
+    nfo = build_for_source(metadata)
+
+    FilesystemHelpers.write_p!(filepath, nfo)
+
+    filepath
+  end
+
+  defp build_for_media_item(metadata) do
+    upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
+    # Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
+    # WHO NEEDS A FANCY XML PARSER ANYWAY?!
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <episodedetails>
+      <title>#{metadata["title"]}</title>
+      <showtitle>#{metadata["uploader"]}</showtitle>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <plot>#{metadata["description"]}</plot>
+      <aired>#{upload_date}</aired>
+      <season>#{upload_date.year}</season>
+      <episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
+      <genre>YouTube</genre>
+    </episodedetails>
+    """
+  end
+
+  defp build_for_source(metadata) do
+    """
+    <?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+    <tvshow>
+      <title>#{metadata["title"]}</title>
+      <plot>#{metadata["description"]}</plot>
+      <uniqueid type="youtube" default="true">#{metadata["id"]}</uniqueid>
+      <genre>YouTube</genre>
+    </tvshow>
+    """
+  end
+end

lib/pinchflat/metadata/source_image_parser.ex

@@ -0,0 +1,69 @@
+defmodule Pinchflat.Metadata.SourceImageParser do
+  @moduledoc """
+  Functions for parsing and storing source images.
+  """
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  @doc """
+  Given a base directory and source metadata, look for the appropriate images
+  and move them to the base directory. Returns a map of image types and their
+  filepaths (specifically for a source). If a given image type is not found,
+  the key will not be present in the map.
+
+  The metadata is expected to contain the `filepath` key for images, implying
+  that the images have already been downloaded by yt-dlp. This method will NOT
+  download images from the internet - it just copys them around.
+
+  Returns a map with the possible keys :poster_filepath, :fanart_filepath, and
+  :banner_filepath.
+  """
+  def store_source_images(base_directory, source_metadata) do
+    (source_metadata["thumbnails"] || [])
+    |> Enum.filter(&(&1["filepath"] != nil))
+    |> select_useful_images()
+    |> Enum.map(&move_image(&1, base_directory))
+    |> Enum.into(%{})
+  end
+
+  defp select_useful_images(images) do
+    labelled_images =
+      Enum.reduce(images, [], fn image_map, acc ->
+        case image_map do
+          %{"id" => "avatar_uncropped"} ->
+            acc ++ [{:poster, :poster_filepath, image_map["filepath"]}]
+
+          %{"id" => "banner_uncropped"} ->
+            acc ++ [{:fanart, :fanart_filepath, image_map["filepath"]}]
+
+          _ ->
+            acc
+        end
+      end)
+
+    labelled_images
+    |> Enum.concat([{:banner, :banner_filepath, determine_best_banner(images)}])
+    |> Enum.filter(fn {_, _, tmp_filepath} -> tmp_filepath end)
+  end
+
+  defp determine_best_banner(images) do
+    best_candidate =
+      images
+      # Filter out images that don't have a width and height attribute
+      |> Enum.filter(&(&1["width"] && &1["height"]))
+      # Sort images with the highest width first
+      |> Enum.sort(&(&1["width"] >= &2["width"]))
+      # Find the first image where the ratio of width to height is greater than 3
+      |> Enum.find(&(&1["width"] / &1["height"] > 3))
+
+    Map.get(best_candidate || %{}, "filepath")
+  end
+
+  defp move_image({filename, source_attr_name, tmp_filepath}, base_directory) do
+    extension = Path.extname(tmp_filepath)
+    final_filepath = Path.join([base_directory, "#{filename}#{extension}"])
+
+    FilesystemHelpers.cp_p!(tmp_filepath, final_filepath)
+
+    {source_attr_name, final_filepath}
+  end
+end

lib/pinchflat/metadata/source_metadata.ex

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Metadata.SourceMetadata do
+  @moduledoc """
+  The SourceMetadata schema.
+
+  Look. Don't @ me about Metadata vs. Metadatum. I'm very sensitive.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Pinchflat.Sources.Source
+
+  @allowed_fields ~w(metadata_filepath)a
+  @required_fields ~w(metadata_filepath)a
+
+  schema "source_metadata" do
+    field :metadata_filepath, :string
+
+    belongs_to :source, Source
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(source_metadata, attrs) do
+    source_metadata
+    |> cast(attrs, @allowed_fields)
+    |> validate_required(@required_fields)
+    |> unique_constraint([:source_id])
+  end
+
+  @doc false
+  def filepath_attributes do
+    ~w(metadata_filepath)a
+  end
+end

lib/pinchflat/metadata/source_metadata_storage_worker.ex

@@ -0,0 +1,111 @@
+defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :remote_metadata,
+    tags: ["media_source", "source_metadata", "remote_metadata"],
+    max_attempts: 3
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Repo
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources
+  alias Pinchflat.Utils.StringUtils
+  alias Pinchflat.Metadata.NfoBuilder
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceImageParser
+  alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Downloading.DownloadOptionBuilder
+
+  @doc """
+  Starts the source metadata storage worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> SourceMetadataStorageWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches and stores various forms of metadata for a source:
+    - JSON metadata for internal use
+    - The series directory for the source
+    - The NFO file for the source (if specified)
+    - Downloads and stores source images (if specified)
+
+  The worker is kicked off after a source is inserted/updated - this can
+  take an unknown amount of time so don't rely on this data being here
+  before, say, the first indexing or downloading task is complete.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id}}) do
+    source = Repo.preload(Sources.get_source!(source_id), [:metadata, :media_profile])
+    series_directory = determine_series_directory(source)
+    {source_metadata, image_filepath_attrs} = fetch_source_metadata_and_images(series_directory, source)
+
+    # `run_post_commit_tasks: false` prevents this from running in an infinite loop
+    Sources.update_source(
+      source,
+      Map.merge(
+        %{
+          series_directory: series_directory,
+          nfo_filepath: store_source_nfo(source, series_directory, source_metadata),
+          metadata: %{
+            metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(source, source_metadata)
+          }
+        },
+        image_filepath_attrs
+      ),
+      run_post_commit_tasks: false
+    )
+
+    :ok
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
+  end
+
+  defp fetch_source_metadata_and_images(series_directory, source) do
+    if source.media_profile.download_source_images && series_directory do
+      output_path = "#{tmp_directory()}/#{StringUtils.random_string(16)}/source_image.%(ext)S"
+      opts = [:write_all_thumbnails, convert_thumbnails: "jpg", output: output_path]
+
+      {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url, opts)
+      image_attrs = SourceImageParser.store_source_images(series_directory, metadata)
+
+      {metadata, image_attrs}
+    else
+      {:ok, metadata} = MediaCollection.get_source_metadata(source.original_url, [])
+
+      {metadata, %{}}
+    end
+  end
+
+  defp determine_series_directory(source) do
+    output_path = DownloadOptionBuilder.build_output_path_for(source)
+    {:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, output: output_path)
+
+    case MetadataFileHelpers.series_directory_from_media_filepath(filepath) do
+      {:ok, series_directory} -> series_directory
+      {:error, _} -> nil
+    end
+  end
+
+  defp store_source_nfo(source, series_directory, metadata) do
+    if source.media_profile.download_nfo && series_directory do
+      nfo_filepath = Path.join(series_directory, "tvshow.nfo")
+
+      NfoBuilder.build_and_store_for_source(nfo_filepath, metadata)
+    end
+  end
+
+  defp tmp_directory do
+    Application.get_env(:pinchflat, :tmpfile_directory)
+  end
+end

lib/pinchflat/profiles/media_profile.ex

@@ -17,8 +17,10 @@ defmodule Pinchflat.Profiles.MediaProfile do
     sub_langs
     download_thumbnail
     embed_thumbnail
+    download_source_images
     download_metadata
     embed_metadata
+    download_nfo
     shorts_behaviour
     livestream_behaviour
     preferred_resolution
@@ -32,17 +34,19 @@ defmodule Pinchflat.Profiles.MediaProfile do
     field :output_path_template, :string,
       default: "/{{ source_custom_name }}/{{ upload_yyyy_mm_dd }} {{ title }}/{{ title }} [{{ id }}].{{ ext }}"
 
-    field :download_subs, :boolean, default: true
-    field :download_auto_subs, :boolean, default: true
-    field :embed_subs, :boolean, default: true
+    field :download_subs, :boolean, default: false
+    field :download_auto_subs, :boolean, default: false
+    field :embed_subs, :boolean, default: false
     field :sub_langs, :string, default: "en"
 
-    field :download_thumbnail, :boolean, default: true
-    field :embed_thumbnail, :boolean, default: true
+    field :download_thumbnail, :boolean, default: false
+    field :embed_thumbnail, :boolean, default: false
+    field :download_source_images, :boolean, default: false
 
-    field :download_metadata, :boolean, default: true
-    field :embed_metadata, :boolean, default: true
+    field :download_metadata, :boolean, default: false
+    field :embed_metadata, :boolean, default: false
 
+    field :download_nfo, :boolean, default: false
     # NOTE: these do NOT speed up indexing - the indexer still has to go
     # through the entire collection to determine if a media is a short or
     # a livestream.
@@ -65,6 +69,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     media_profile
     |> cast(attrs, @allowed_fields)
     |> validate_required(@required_fields)
+    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
+    |> validate_format(:output_path_template, ext_regex(), message: "must end with .{{ ext }}")
     |> unique_constraint(:name)
   end
+
+  defp ext_regex do
+    ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
+  end
 end

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
+defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   @moduledoc """
   A GenServer that watches a file for new lines and processes them as they come in.
   This is useful for tailing log files and other similar tasks. If there's no activity
@@ -9,7 +9,7 @@ defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
   require Logger
 
   @poll_interval_ms Application.compile_env(:pinchflat, :file_watcher_poll_interval)
-  @activity_timeout_ms 60_000
+  @activity_timeout_ms 600_000
 
   # Client API
   @doc """

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -6,14 +6,26 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
     tags: ["media_source", "media_collection_indexing"]
 
+  require Logger
+
   alias __MODULE__
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+
+  @doc """
+  Starts the source slow indexing worker and creates a task for the source.
+
+  Returns {:ok, %Task{}} | {:error, :duplicate_job} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff_with_task(source, opts \\ []) do
+    %{id: source.id}
+    |> MediaCollectionIndexingWorker.new(opts)
+    |> Tasks.create_job_with_task(source)
+  end
 
-  @impl Oban.Worker
   @doc """
   The ID is that of a source _record_, not a YouTube channel/playlist ID. Indexes
   the provided source, kicks off downloads for each new MediaItem, and
@@ -40,7 +52,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
        by the `download_media` field on the source as well as the profile's
        shorts/livestream behaviour. At this step we also attach a file reader
        to the `yt-dlp` output file so we can create media items as they come in
-       for a little speedup (see SourceTasks comments for more)
+       for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
     4. If this job is meant to reschedule (ie: has an index frequency > 0),
        it reschedules itself. If not, it runs once and does not reschedule
     5. If the source uses fast indexing, that job is kicked off as well. It
@@ -56,24 +68,23 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   NOTE: Since indexing can take a LONG time, I should check what happens if an
   application restart occurs while a job is running. Will the job be lost?
 
-  IDEA: Should I use paging and do indexing in chunks? Is that even faster?
-
   Returns :ok | {:ok, %Task{}}
   """
+  @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
     source = Sources.get_source!(source_id)
 
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         :ok
 
       _ ->
@@ -81,6 +92,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
         # perform a no-op
         :ok
     end
+  rescue
+    Ecto.NoResultsError -> Logger.info("#{__MODULE__} discarded: source #{source_id} not found")
+    Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
   end
 
   defp reschedule_indexing(source) do

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -1,27 +1,25 @@
-defmodule Pinchflat.Tasks.SourceTasks do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   @moduledoc """
-  Contains methods used by OR used to create/manage tasks for sources.
+  Methods for performing slow indexing tasks and managing the indexing process.
 
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
+  Many of these methods are made to be kickoff or be consumed by workers.
   """
 
   require Logger
 
+  alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
-  alias Pinchflat.YtDlp.Backend.MediaCollection
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.SlowIndexing.FileFollowerServer
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Starts tasks for indexing a source's media regardless of the source's indexing
@@ -34,56 +32,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
     Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
 
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> MediaCollectionIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
+    MediaCollectionIndexingWorker.kickoff_with_task(source)
   end
 
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
   item belonging to the source. You can't tell me the method name isn't descriptive!
+  Returns a list of media items or changesets (if the media item couldn't be created).
 
   Indexing is slow and usually returns a list of all media data at once for record creation.
   To help with this, we use a file follower to watch the file that yt-dlp writes to
@@ -99,56 +55,30 @@ defmodule Pinchflat.Tasks.SourceTasks do
   Since indexing returns all media data EVERY TIME, we that that opportunity to update
   indexing metadata for media items that have already been created.
 
-  Returns [%MediaItem{}, ...]
+  Returns [%MediaItem{} | %Ecto.Changeset{}]
   """
   def index_and_enqueue_download_for_media_items(%Source{} = source) do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
-    result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
+    # Reload because the source may have been updated during the (long-running) indexing process
+    # and important settings like `download_media` may have changed.
+    source = Repo.reload!(source)
+
+    result =
+      Enum.map(media_attributes, fn media_attrs ->
+        case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+          {:ok, media_item} -> media_item
+          {:error, changeset} -> changeset
+        end
+      end)
 
     Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
-    enqueue_pending_media_tasks(source)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
 
     result
   end
 
-  @doc """
-  Starts tasks for downloading media for any of a sources _pending_ media items.
-  Jobs are not enqueued if the source is set to not download media. This will return :ok.
-
-  NOTE: this starts a download for each media item that is pending,
-  not just the ones that were indexed in this job run. This should ensure
-  that any stragglers are caught if, for some reason, they weren't enqueued
-  or somehow got de-queued.
-
-  Returns :ok
-  """
-  def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
-
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
-    :ok
-  end
-
-  @doc """
-  Deletes ALL pending tasks for a source's media items.
-
-  Returns :ok
-  """
-  def dequeue_pending_media_tasks(%Source{} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -191,27 +121,20 @@ defmodule Pinchflat.Tasks.SourceTasks do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
-    maybe_media_item = create_media_item_from_attributes(source, media_attrs)
+    # Reload because the source may have been updated during the (long-running) indexing process
+    # and important settings like `download_media` may have changed.
+    source = Repo.reload!(source)
 
-    case maybe_media_item do
-      %MediaItem{} = media_item ->
+    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+      {:ok, %MediaItem{} = media_item} ->
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
+          MediaDownloadWorker.kickoff_with_task(media_item)
         end
 
-      changeset ->
+      {:error, changeset} ->
         changeset
     end
   end
-
-  defp create_media_item_from_attributes(source, media_attrs) do
-    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-      {:ok, media_item} -> media_item
-      {:error, changeset} -> changeset
-    end
-  end
 end

lib/pinchflat/sources/source.ex

@@ -10,25 +10,33 @@ defmodule Pinchflat.Sources.Source do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Metadata.SourceMetadata
 
   @allowed_fields ~w(
     collection_name
     collection_id
     collection_type
     custom_name
+    nfo_filepath
+    poster_filepath
+    fanart_filepath
+    banner_filepath
+    series_directory
     index_frequency_minutes
     fast_index
     download_media
     last_indexed_at
     original_url
     download_cutoff_date
+    title_filter_regex
     media_profile_id
   )a
 
   # Expensive API calls are made when a source is inserted/updated so
   # we want to ensure that the source is valid before making the call.
   # This way, we check that the other attributes are valid before ensuring
-  # that all fields are valid.
+  # that all fields are valid. This is still only one DB insert but it's
+  # a two-stage validation process to fail fast before the API call.
   @initially_required_fields ~w(
     index_frequency_minutes
     fast_index
@@ -57,9 +65,18 @@ defmodule Pinchflat.Sources.Source do
     # Only download media items that were published after this date
     field :download_cutoff_date, :date
     field :original_url, :string
+    field :title_filter_regex, :string
+
+    field :series_directory, :string
+    field :nfo_filepath, :string
+    field :poster_filepath, :string
+    field :fanart_filepath, :string
+    field :banner_filepath, :string
 
     belongs_to :media_profile, MediaProfile
 
+    has_one :metadata, SourceMetadata, on_replace: :update
+
     has_many :tasks, Task
     has_many :media_items, MediaItem, foreign_key: :source_id
 
@@ -80,7 +97,7 @@ defmodule Pinchflat.Sources.Source do
     |> cast(attrs, @allowed_fields)
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> validate_required(required_fields)
-    |> unique_constraint([:collection_id, :media_profile_id])
+    |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
   end
 
   @doc false
@@ -94,4 +111,9 @@ defmodule Pinchflat.Sources.Source do
     # minutes
     15
   end
+
+  @doc false
+  def filepath_attributes do
+    ~w(nfo_filepath fanart_filepath poster_filepath banner_filepath)a
+  end
 end

lib/pinchflat/sources/sources.ex

@@ -9,9 +9,14 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Metadata.SourceMetadata
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -46,15 +51,19 @@ defmodule Pinchflat.Sources do
   though we know it's going to fail so it picks up any addl. database errors
   and fulfills our return contract.
 
+  You can pass options to control the behavior of the function:
+    - `run_post_commit_tasks` (default: true) - If false, the function will not
+      enqueue any tasks in `commit_and_handle_tasks`.
+
   Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}}
   """
-  def create_source(attrs) do
+  def create_source(attrs, opts \\ []) do
     case change_source(%Source{}, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         %Source{}
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
-        |> commit_and_handle_tasks()
+        |> commit_and_handle_tasks(opts)
 
       changeset ->
         Repo.insert(changeset)
@@ -67,22 +76,26 @@ defmodule Pinchflat.Sources do
   media if the indexing frequency has been changed.
 
   Existing indexing tasks will be cancelled if the indexing frequency has been
-  changed (logic in `SourceTasks.kickoff_indexing_task`)
+  changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
 
   Runs an initial `change_source` check to ensure most of the source is valid
   before making an expensive API call. Runs it through `Repo.update` even
   though we know it's going to fail so it picks up any addl. database errors
   and fulfills our return contract.
 
+  You can pass options to control the behavior of the function:
+    - `run_post_commit_tasks` (default: true) - If false, the function will not
+      enqueue any tasks in `commit_and_handle_tasks`.
+
   Returns {:ok, %Source{}} | {:error, %Ecto.Changeset{}}
   """
-  def update_source(%Source{} = source, attrs) do
+  def update_source(%Source{} = source, attrs, opts \\ []) do
     case change_source(source, attrs, :initial) do
       %Ecto.Changeset{valid?: true} ->
         source
-        |> change_source_from_url(attrs)
+        |> maybe_change_source_from_url(attrs)
         |> maybe_change_indexing_frequency()
-        |> commit_and_handle_tasks()
+        |> commit_and_handle_tasks(opts)
 
       changeset ->
         Repo.update(changeset)
@@ -97,6 +110,7 @@ defmodule Pinchflat.Sources do
   """
   def delete_source(%Source{} = source, opts \\ []) do
     delete_files = Keyword.get(opts, :delete_files, false)
+    Tasks.delete_tasks_for(source)
 
     source
     |> Media.list_media_items_for()
@@ -104,7 +118,11 @@ defmodule Pinchflat.Sources do
       Media.delete_media_item(media_item, delete_files: delete_files)
     end)
 
-    Tasks.delete_tasks_for(source)
+    if delete_files do
+      delete_source_files(source)
+    end
+
+    delete_internal_metadata_files(source)
     Repo.delete(source)
   end
 
@@ -115,19 +133,9 @@ defmodule Pinchflat.Sources do
     Source.changeset(source, attrs, validation_stage)
   end
 
-  @doc """
-  Returns an `%Ecto.Changeset{}` for tracking source changes and additionally
-  fetches source details from the original_url (if provided). If the source
-  details cannot be fetched, an error is added to the changeset.
-
-  Note that this fetches source details as long as the `original_url` is present.
-  This means that it'll go for it even if a changeset is otherwise invalid. This
-  is pretty easy to change, but for MVP I'm not concerned.
-
-  NOTE: When operating in the ideal path, this effectively adds an API call
-  to the source creation/update process. Should be used only when needed.
-  """
-  def change_source_from_url(%Source{} = source, attrs) do
+  # NOTE: When operating in the ideal path, this effectively adds an API call
+  # to the source creation/update process. Should be used only when needed.
+  defp maybe_change_source_from_url(%Source{} = source, attrs) do
     case change_source(source, attrs) do
       %Ecto.Changeset{changes: %{original_url: _}} = changeset ->
         add_source_details_to_changeset(source, changeset)
@@ -137,10 +145,27 @@ defmodule Pinchflat.Sources do
     end
   end
 
-  defp add_source_details_to_changeset(source, changeset) do
-    %Ecto.Changeset{changes: changes} = changeset
+  defp delete_source_files(source) do
+    mapped_struct = Map.from_struct(source)
 
-    case MediaCollection.get_source_details(changes.original_url) do
+    Source.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
+  defp delete_internal_metadata_files(source) do
+    metadata = Repo.preload(source, :metadata).metadata || %SourceMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    SourceMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+    |> Enum.each(&FilesystemHelpers.delete_file_and_remove_empty_directories/1)
+  end
+
+  defp add_source_details_to_changeset(source, changeset) do
+    case MediaCollection.get_source_details(changeset.changes.original_url) do
       {:ok, source_details} ->
         add_source_details_by_collection_type(source, changeset, source_details)
 
@@ -189,11 +214,18 @@ defmodule Pinchflat.Sources do
     end
   end
 
-  defp commit_and_handle_tasks(changeset) do
+  defp commit_and_handle_tasks(changeset, opts) do
+    run_post_commit_tasks = Keyword.get(opts, :run_post_commit_tasks, true)
+
     case Repo.insert_or_update(changeset) do
       {:ok, %Source{} = source} ->
+        if run_post_commit_tasks do
           maybe_handle_media_tasks(changeset, source)
           maybe_run_indexing_task(changeset, source)
+          run_metadata_storage_task(source)
+        end
+
+        {:ok, source}
 
       err ->
         err
@@ -205,23 +237,21 @@ defmodule Pinchflat.Sources do
   defp maybe_handle_media_tasks(changeset, source) do
     case {changeset.data, changeset.changes} do
       {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
-        SourceTasks.enqueue_pending_media_tasks(source)
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
-        SourceTasks.dequeue_pending_media_tasks(source)
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
         :ok
     end
-
-    {:ok, source}
   end
 
   defp maybe_run_indexing_task(changeset, source) do
     case changeset.data do
       # If the changeset is new (not persisted), attempt indexing no matter what
       %{__meta__: %{state: :built}} ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
@@ -229,14 +259,17 @@ defmodule Pinchflat.Sources do
         maybe_update_slow_indexing_task(changeset, source)
         maybe_update_fast_indexing_task(changeset, source)
     end
+  end
 
-    {:ok, source}
+  # This runs every time to pick up any changes to the metadata
+  defp run_metadata_storage_task(source) do
+    SourceMetadataStorageWorker.kickoff_with_task(source)
   end
 
   defp maybe_update_slow_indexing_task(changeset, source) do
     case changeset.changes do
       %{index_frequency_minutes: mins} when mins > 0 ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@@ -251,7 +284,8 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+        FastIndexingWorker.kickoff_with_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/media_item_tasks.ex

@@ -1,61 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasks do
-  @moduledoc """
-  Contains methods used by OR used to create/manage tasks for media items.
-
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
-  """
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Fetches the file size of a media item and saves it to the database.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def compute_and_save_media_filesize(media_item) do
-    case File.stat(media_item.media_filepath) do
-      {:ok, %{size: size}} ->
-        Media.update_media_item(media_item, %{media_size_bytes: size})
-
-      err ->
-        err
-    end
-  end
-
-  @doc """
-  Indexes a single media item for a source and enqueues a download job if the
-  media should be downloaded. This method creates the media item record so it's
-  the one-stop-shop for adding a media item (and possibly downloading it) just
-  by a URL and source.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
-    maybe_media_item = create_media_item_from_url(source, url)
-
-    case maybe_media_item do
-      {:ok, media_item} ->
-        if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
-        end
-
-        {:ok, media_item}
-
-      err ->
-        err
-    end
-  end
-
-  defp create_media_item_from_url(source, url) do
-    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
-
-    Media.create_media_item_from_backend_attrs(source, media_attrs)
-  end
-end

lib/pinchflat/tasks/tasks.ex

@@ -22,9 +22,15 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil, job_states \\ Oban.Job.states()) do
+  def list_tasks_for(record, worker_name \\ nil, job_states \\ Oban.Job.states()) do
     stringified_states = Enum.map(job_states, &to_string/1)
 
+    record_type =
+      case record do
+        %Source{} -> :source_id
+        %MediaItem{} -> :media_item_id
+      end
+
     worker_name_finder =
       if worker_name do
         # Workers are the full module name - we want to match on the string ENDING with
@@ -41,7 +47,7 @@ defmodule Pinchflat.Tasks do
     Repo.all(
       from t in Task,
         join: j in assoc(t, :job),
-        where: field(t, ^attached_record_type) == ^attached_record_id,
+        where: field(t, ^record_type) == ^record.id,
         where: ^worker_name_finder,
         where: j.state in ^stringified_states
     )
@@ -53,10 +59,9 @@ defmodule Pinchflat.Tasks do
 
   Returns [%Task{}, ...]
   """
-  def list_pending_tasks_for(attached_record_type, attached_record_id, worker_name \\ nil) do
+  def list_pending_tasks_for(record, worker_name \\ nil) do
     list_tasks_for(
-      attached_record_type,
-      attached_record_id,
+      record,
       worker_name,
       [:available, :scheduled, :retryable]
     )
@@ -126,14 +131,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_tasks_for(attached_record, worker_name \\ nil) do
-    tasks =
-      case attached_record do
-        %Source{} = source -> list_tasks_for(:source_id, source.id, worker_name)
-        %MediaItem{} = media_item -> list_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
+  def delete_tasks_for(record, worker_name \\ nil) do
+    record
+    |> list_tasks_for(worker_name)
+    |> Enum.each(&delete_task/1)
   end
 
   @doc """
@@ -142,14 +143,10 @@ defmodule Pinchflat.Tasks do
 
   Returns :ok
   """
-  def delete_pending_tasks_for(attached_record, worker_name \\ nil) do
-    tasks =
-      case attached_record do
-        %Source{} = source -> list_pending_tasks_for(:source_id, source.id, worker_name)
-        %MediaItem{} = media_item -> list_pending_tasks_for(:media_item_id, media_item.id, worker_name)
-      end
-
-    Enum.each(tasks, &delete_task/1)
+  def delete_pending_tasks_for(record, worker_name \\ nil) do
+    record
+    |> list_pending_tasks_for(worker_name)
+    |> Enum.each(&delete_task/1)
   end
 
   @doc """

lib/pinchflat/utils/filesystem_utils.ex

@@ -1,23 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtils do
-  @moduledoc """
-  Utility methods for working with the filesystem
-  """
-
-  alias Pinchflat.Utils.StringUtils
-
-  @doc """
-  Generates a temporary file and returns its path. The file is empty and has the given type.
-  Generates all the directories in the path if they don't exist.
-
-  Returns binary()
-  """
-  def generate_metadata_tmpfile(type) do
-    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
-    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
-
-    :ok = File.mkdir_p!(Path.dirname(filepath))
-    :ok = File.write(filepath, "")
-
-    filepath
-  end
-end

lib/pinchflat/workers/filesystem_data_worker.ex

@@ -1,27 +0,0 @@
-defmodule Pinchflat.Workers.FilesystemDataWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :media_local_metadata,
-    tags: ["media_item", "media_metadata", "local_metadata"],
-    max_attempts: 1
-
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks.MediaItemTasks
-
-  @impl Oban.Worker
-  @doc """
-  For a given media item, compute and save metadata about the file on-disk.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
-
-    MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-    # Don't retry on failure - if it didn't work immediately there's no
-    # reason to believe it will work later.
-    :ok
-  end
-end

lib/pinchflat/yt_dlp/backend/media_collection.ex

@@ -1,81 +0,0 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollection do
-  @moduledoc """
-  Contains utilities for working with collections of
-  media (aka: a source [ie: channels, playlists]).
-  """
-
-  require Logger
-
-  alias Pinchflat.Utils.FunctionUtils
-  alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Returns a list of maps representing the media in the collection.
-
-  Options:
-    - :file_listener_handler - a function that will be called with the path to the
-      file that will be written to when yt-dlp is done. This is useful for
-      setting up a file watcher to know when the file is ready to be read.
-
-  Returns {:ok, [map()]} | {:error, any, ...}.
-  """
-  def get_media_attributes_for_collection(url, addl_opts \\ []) do
-    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
-    command_opts = [:simulate, :skip_download]
-    output_template = YtDlpMedia.indexing_output_template()
-    output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
-    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
-
-    if file_listener_handler do
-      file_listener_handler.(output_filepath)
-    end
-
-    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
-      {:ok, output} ->
-        output
-        |> String.split("\n", trim: true)
-        |> Enum.map(&Phoenix.json_library().decode!/1)
-        |> Enum.map(&YtDlpMedia.response_to_struct/1)
-        |> FunctionUtils.wrap_ok()
-
-      res ->
-        res
-    end
-  end
-
-  @doc """
-  Gets a source's ID and name from its URL.
-
-  yt-dlp does not _really_ have source-specific functions, so
-  instead we're fetching just the first video (using playlist_end: 1)
-  and parsing the source ID and name from _its_ metadata
-
-  Returns {:ok, map()} | {:error, any, ...}.
-  """
-  def get_source_details(source_url) do
-    opts = [:simulate, :skip_download, playlist_end: 1]
-    output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
-
-    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
-         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
-      {:ok, format_source_details(parsed_json)}
-    else
-      err -> err
-    end
-  end
-
-  defp format_source_details(response) do
-    %{
-      channel_id: response["channel_id"],
-      channel_name: response["channel"],
-      playlist_id: response["playlist_id"],
-      playlist_name: response["playlist_title"]
-    }
-  end
-
-  defp backend_runner do
-    # This approach lets us mock the command for testing
-    Application.get_env(:pinchflat, :yt_dlp_runner)
-  end
-end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.BackendCommandRunner do
   @moduledoc """
   A behaviour for running CLI commands against a downloader backend (yt-dlp).
 

lib/pinchflat/yt_dlp/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunner do
+defmodule Pinchflat.YtDlp.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   require Logger
 
   alias Pinchflat.Utils.StringUtils
-  alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
-  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
+  alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
+  alias Pinchflat.YtDlp.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 
@@ -37,7 +37,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
 
     case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
-        # IDEA: consider deleting the file after reading it
+        # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
+        # a huge deal, but it's still a good idea to clean up after ourselves.
         # (even on error? especially on error?)
         File.read(output_filepath)
 

lib/pinchflat/yt_dlp/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.Media do
+defmodule Pinchflat.YtDlp.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """
@@ -25,6 +25,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do
 
   alias __MODULE__
   alias Pinchflat.Utils.FunctionUtils
+  alias Pinchflat.Metadata.MetadataFileHelpers
 
   @doc """
   Downloads a single piece of media (and possibly its metadata) directly to its
@@ -85,8 +86,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       description: response["description"],
       original_url: response["webpage_url"],
       livestream: response["was_live"],
-      short_form_content: short_form_content?(response),
-      upload_date: parse_upload_date(response["upload_date"])
+      short_form_content: response["webpage_url"] && short_form_content?(response),
+      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
     }
   end
 
@@ -99,16 +100,13 @@ defmodule Pinchflat.YtDlp.Backend.Media do
       # WILL returns false positives, but it's a best-effort approach
       # that should work for most cases. The aspect_ratio check is
       # based on a gut feeling and may need to be tweaked.
+      #
+      # These don't fail if duration or aspect_ratio are missing
+      # due to Elixir's comparison semantics
       response["duration"] <= 60 && response["aspect_ratio"] < 0.8
     end
   end
 
-  defp parse_upload_date(upload_date) do
-    <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
-
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
-  end
-
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

lib/pinchflat/yt_dlp/media_collection.ex

@@ -0,0 +1,127 @@
+defmodule Pinchflat.YtDlp.MediaCollection do
+  @moduledoc """
+  Contains utilities for working with collections of
+  media (aka: a source [ie: channels, playlists]).
+  """
+
+  require Logger
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Returns a list of maps representing the media in the collection.
+
+  Options:
+    - :file_listener_handler - a function that will be called with the path to the
+      file that will be written to when yt-dlp is done. This is useful for
+      setting up a file watcher to know when the file is ready to be read.
+
+  Returns {:ok, [map()]} | {:error, any, ...}.
+  """
+  def get_media_attributes_for_collection(url, addl_opts \\ []) do
+    runner = Application.get_env(:pinchflat, :yt_dlp_runner)
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the media details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error]
+    output_template = YtDlpMedia.indexing_output_template()
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+    file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
+
+    if file_listener_handler do
+      file_listener_handler.(output_filepath)
+    end
+
+    case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
+      {:ok, output} ->
+        parsed_lines =
+          output
+          |> String.split("\n", trim: true)
+          |> Enum.map(fn line ->
+            case Phoenix.json_library().decode(line) do
+              {:ok, parsed_json} ->
+                YtDlpMedia.response_to_struct(parsed_json)
+
+              _ ->
+                nil
+            end
+          end)
+
+        {:ok, Enum.filter(parsed_lines, &(&1 != nil))}
+
+      res ->
+        res
+    end
+  end
+
+  @doc """
+  Gets a source's ID and name from its URL.
+
+  yt-dlp does not _really_ have source-specific functions that return what
+  we need, so instead we're fetching just the first video (using playlist_end: 1)
+  and parsing the source ID and name from _its_ metadata
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_details(source_url, addl_opts \\ []) do
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the source details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1] ++ addl_opts
+    output_template = "%(.{channel,channel_id,playlist_id,playlist_title,filename})j"
+
+    with {:ok, output} <- backend_runner().run(source_url, command_opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, format_source_details(parsed_json)}
+    else
+      err -> err
+    end
+  end
+
+  @doc """
+  Gets a source's metadata from its URL.
+
+  This is mostly for things like getting the source's avatar and banner image
+  (if applicable). However, this yt-dlp call doesn't have enough overlap with
+  `get_source_details/1` to allow combining them - this one has _almost_ everything
+  we need, but it doesn't contain enough information to tell 100% if the url is a channel
+  or a playlist.
+
+  The main purpose of this (past using as a fetcher for _other_ metadata) is to live
+  as a compressed blob for possible future use. That's why it's not getting formatted like
+  `get_source_details/1`
+
+  Returns {:ok, map()} | {:error, any, ...}.
+  """
+  def get_source_metadata(source_url, addl_opts \\ []) do
+    opts = [playlist_items: 0] ++ addl_opts
+    output_template = "playlist:%()j"
+
+    with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
+      {:ok, parsed_json}
+    else
+      err -> err
+    end
+  end
+
+  defp format_source_details(response) do
+    # NOTE: I should probably make this a struct some day
+    %{
+      channel_id: response["channel_id"],
+      channel_name: response["channel"],
+      playlist_id: response["playlist_id"],
+      playlist_name: response["playlist_title"],
+      # It's not a name, it's a path dammit!
+      # This actually isn't used for the inital response - it's
+      # used later to update a source's metadata
+      filepath: response["filename"]
+    }
+  end
+
+  defp backend_runner do
+    # This approach lets us mock the command for testing
+    Application.get_env(:pinchflat, :yt_dlp_runner)
+  end
+end

lib/pinchflat_web/components/core_components.ex

@@ -317,17 +317,7 @@ defmodule PinchflatWeb.CoreComponents do
         <span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
       <div class="relative">
-        <input type="hidden" name={@name} value="false" />
-        <input
-          type="checkbox"
-          id={@id}
-          name={@name}
-          value="true"
-          x-bind:checked="enabled"
-          class="sr-only"
-          @change="enabled = !enabled"
-          {@rest}
-        />
+        <input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
         <div class="inline-block cursor-pointer" @click="enabled = !enabled">
           <div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
           <div
@@ -349,9 +339,10 @@ defmodule PinchflatWeb.CoreComponents do
   def input(%{type: "select"} = assigns) do
     ~H"""
     <div phx-feedback-for={@name}>
-      <.label for={@id}>
+      <.label :if={@label} for={@id}>
         <%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
       </.label>
+      <div class="flex">
         <select
           id={@id}
           name={@name}
@@ -366,6 +357,8 @@ defmodule PinchflatWeb.CoreComponents do
           <option :if={@prompt} value=""><%= @prompt %></option>
           <%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
         </select>
+        <%= render_slot(@inner_block) %>
+      </div>
       <.help :if={@help}><%= @help %></.help>
       <.error :for={msg <- @errors}><%= msg %></.error>
     </div>

lib/pinchflat_web/components/custom_components/button_components.ex

@@ -28,9 +28,10 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
         "#{@rounding} inline-flex items-center justify-center px-8 py-4",
         "#{@color}",
         "hover:bg-opacity-90 lg:px-8 xl:px-10",
-        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-gray-2",
+        "disabled:bg-opacity-50 disabled:cursor-not-allowed disabled:text-grey-5",
         @class
       ]}
+      type={@type}
       disabled={@disabled}
       {@rest}
     >

lib/pinchflat_web/components/custom_components/text_components.ex

@@ -31,6 +31,20 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
     """
   end
 
+  @doc """
+  Renders a subtle link with the given href and content.
+  """
+  attr :href, :string, required: true
+  slot :inner_block
+
+  def subtle_link(assigns) do
+    ~H"""
+    <.link href={@href} class="underline decoration-bodydark decoration-1 hover:decoration-white">
+      <%= render_slot(@inner_block) %>
+    </.link>
+    """
+  end
+
   @doc """
   Renders an icon as a link with the given href.
   """

lib/pinchflat_web/components/layouts/onboarding.html.heex

@@ -1,10 +1,9 @@
 <div class="flex h-screen overflow-hidden">
   <div class="relative flex flex-1 flex-col overflow-y-auto overflow-x-hidden">
     <header class="sticky top-0 z-999 flex w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
-      <div class="flex flex-grow items-center px-4 py-4 shadow-2 md:px-6 2xl:px-11">
-        <div class="flex items-center">
-          <img src={~p"/images/logo.png?cachebust=2024-02-29"} alt="Pinchflat" class="w-9 h-9" />
-          <h2 class="text-xl font-bold text-white pl-2">Pinchflat</h2>
+      <div class="w-65 px-4 py-2 shadow-2 md:px-6">
+        <div class="flex items-center gap-2 py-2">
+          <img src={~p"/images/logo.png?cachebust=2024-03-20"} alt="Pinchflat" class="w-auto" />
         </div>
       </div>
     </header>

lib/pinchflat_web/components/layouts/partials/header.html.heex

@@ -1,18 +1,18 @@
-<header class="sticky top-0 z-999 flex w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
+<header class="sticky top-0 z-999 flex h-20 w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
   <div class="flex flex-grow items-center justify-between lg:justify-end px-4 py-4 shadow-2 md:px-6 2xl:px-11">
-    <div class="flex items-center gap-2 sm:gap-4 lg:hidden">
+    <div class="flex items-center gap-2 sm:gap-4 lg:hidden w-2/6">
+      <section class="pr-1">
+        <img src={~p"/images/icon.png?cachebust=2024-03-20"} alt="Pinchflat" class="w-10" />
+      </section>
+
       <button
-        class="z-99999 block rounded-sm border border-stroke bg-white p-1.5 shadow-sm dark:border-strokedark dark:bg-boxdark lg:hidden"
+        class="z-99999 block mx-2 rounded-sm border border-stroke bg-white p-1.5 shadow-sm dark:border-strokedark dark:bg-boxdark lg:hidden"
         @click.stop="sidebarVisible = !sidebarVisible"
       >
         <.icon name="hero-bars-3" />
       </button>
-      <a class="hidden sm:flex items-center lg:hidden" href="/">
-        <img src={~p"/images/logo.png?cachebust=2024-02-29"} alt="Pinchflat" class="w-9 h-9" />
-        <h2 class="text-xl font-bold text-white pl-2">Pinchflat</h2>
-      </a>
     </div>
-    <div class="bg-meta-4 rounded-md">
+    <div class="bg-meta-4 rounded-md w-4/6 lg:w-3/6 xl:w-2/6">
       <div class="relative">
         <span class="absolute left-2 top-1/2 -translate-y-1/2 flex">
           <.icon name="hero-magnifying-glass" />
@@ -23,7 +23,7 @@
             name="q"
             value={@params["q"]}
             placeholder="Type to search..."
-            class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none lg:w-125"
+            class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none"
           />
         </form>
       </div>

lib/pinchflat_web/components/layouts/partials/sidebar.html.heex

@@ -1,24 +1,23 @@
 <aside
   x-bind:class="sidebarVisible ? 'translate-x-0' : '-translate-x-full'"
   class={[
-    "-translate-x-full absolute left-0 top-0 z-9999 flex h-screen w-60 flex-col overflow-y-hidden justify-between",
+    "-translate-x-full absolute left-0 top-0 z-9999 flex h-screen w-65 flex-col overflow-y-hidden justify-between",
     "bg-black duration-300 ease-linear shadow-lg sm:shadow-none dark:bg-boxdark lg:static lg:translate-x-0"
   ]}
   @click.outside="sidebarVisible = false"
 >
   <section>
-    <div class="flex items-center justify-between gap-2 px-6 py-5.5 lg:py-6.5">
+    <div class="flex items-center justify-between gap-2 px-6 py-4">
       <a href="/" class="flex items-center">
-        <img src={~p"/images/logo.png?cachebust=2024-02-29"} alt="Pinchflat" class="w-9 h-9" />
-        <h2 class="text-xl font-bold text-white pl-2">Pinchflat</h2>
+        <img src={~p"/images/logo.png?cachebust=2024-03-20"} alt="Pinchflat" class="w-auto" />
       </a>
 
-      <button class="block lg:hidden" @click.stop="sidebarVisible = !sidebarVisible">
+      <button class="block mt-3 lg:hidden" @click.stop="sidebarVisible = !sidebarVisible">
         <.icon name="hero-arrow-left" class="fill-current" />
       </button>
     </div>
     <div class="no-scrollbar flex flex-col overflow-y-auto duration-300 ease-linear">
-      <nav class="mt-5 px-4 py-4 lg:mt-9 lg:px-6">
+      <nav class="mt-3 px-4 py-4 lg:px-6">
         <h3 class="mb-4 ml-4 text-sm font-medium text-bodydark2">MENU</h3>
         <div class="flex flex-col justify-between">
           <ul class="mb-6 flex flex-col gap-1.5">
@@ -31,7 +30,7 @@
     </div>
   </section>
   <section>
-    <nav class="mt-5 px-4 py-4 lg:mt-9 lg:px-6">
+    <nav class="px-4 py-4 lg:px-6">
       <ul class="mb-6 flex flex-col gap-1.5">
         <.sidebar_item
           icon="hero-code-bracket"

lib/pinchflat_web/components/layouts/root.html.heex

@@ -8,7 +8,7 @@
       <%= assigns[:page_title] || "Pinchflat" %>
     </.live_title>
     <link phx-track-static rel="stylesheet" href={~p"/assets/app.css"} />
-    <link rel="icon" type="image/x-icon" href={~p"/favicon.ico?cachebust=2024-02-29"} />
+    <link rel="icon" type="image/x-icon" href={~p"/favicon.ico?cachebust=2024-03-20"} />
     <script defer phx-track-static type="text/javascript" src={~p"/assets/app.js"}>
     </script>
   </head>

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -30,7 +30,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this record and all associated files on disk? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -1,6 +1,8 @@
 defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   use PinchflatWeb, :html
 
+  alias Pinchflat.Profiles.MediaProfile
+
   embed_templates "media_profile_html/*"
 
   @doc """
@@ -53,4 +55,25 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
       duration_string
     )a
   end
+
+  def preset_options do
+    [
+      {"Default", "default"},
+      {"Media Center (Plex, Jellyfin, Kodi, etc.)", "media_center"},
+      {"Music", "audio"},
+      {"Archiving", "archiving"}
+    ]
+  end
+
+  defp default_output_template do
+    %MediaProfile{}.output_path_template
+  end
+
+  defp media_center_output_template do
+    "/shows/{{ source_custom_name }}/Season {{ season_from_date }}/{{ season_episode_from_date }} - {{ title }}.{{ ext }}"
+  end
+
+  defp audio_output_template do
+    "/music/{{ source_custom_name }}/{{ title }}.{{ ext }}"
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/edit.html.heex

@@ -2,7 +2,9 @@
   <.link href={~p"/media_profiles"}>
     <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
   </.link>
-  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">Edit Media Profile</h2>
+  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
+    Editing "<%= @media_profile.name %>"
+  </h2>
 </div>
 
 <div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">

lib/pinchflat_web/controllers/media_profiles/media_profile_html/index.html.heex

@@ -4,7 +4,7 @@
   </h2>
   <nav>
     <.link href={~p"/media_profiles/new"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <span class="font-bold text-xl mx-2">+</span> New <span class="hidden sm:inline pl-1">Media Profile</span>
       </.button>
     </.link>
@@ -16,10 +16,9 @@
     <div class="flex flex-col gap-10 min-w-max">
       <.table rows={@media_profiles} table_class="text-black dark:text-white">
         <:col :let={media_profile} label="Name">
+          <.subtle_link href={~p"/media_profiles/#{media_profile.id}"}>
             <%= media_profile.name %>
-        </:col>
-        <:col :let={media_profile} label="Output Template">
-          <code class="text-sm"><%= media_profile.output_path_template %></code>
+          </.subtle_link>
         </:col>
         <:col :let={media_profile} label="Preferred Resolution">
           <%= media_profile.preferred_resolution %>

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -3,116 +3,243 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
+  <section x-data="{ selectedPreset: null }">
     <h3 class="my-4 text-2xl text-black dark:text-white">
+      Use a Preset
+    </h3>
+    <section x-data="{ selection: null }">
+      <.input
+        prompt="Select preset"
+        name="media_profile_preset"
+        value=""
+        options={preset_options()}
+        type="select"
+        x-model="selection"
+        inputclass="w-full"
+        help="You can further customize the settings after selecting a preset. This is just a starting point"
+      >
+        <.button
+          class="h-13 w-2/5 lg:w-1/5 ml-2 md:ml-4"
+          rounding="rounded"
+          type="button"
+          x-on:click="selectedPreset = selection; selection = null"
+          x-bind:disabled="!selection"
+        >
+          <span x-text="selection ? 'Load' : 'Select'">Select</span><span class="hidden lg:inline ml-1">Preset</span>
+        </.button>
+      </.input>
+    </section>
+
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
       General Options
     </h3>
+
+    <section x-data="{ 
+        presets: { 
+          default: 'Default',
+          media_center: 'TV Shows',
+          audio: 'Audio',
+          archiving: 'Archiving'
+        }
+    }">
       <.input
         field={f[:name]}
         type="text"
         label="Name"
         placeholder="New Profile"
         help="Something descriptive. Does not impact indexing or downloading (required)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
+    <section x-data={"{ 
+        presets: { 
+          default: '#{default_output_template()}',
+          media_center: '#{media_center_output_template()}',
+          audio: '#{audio_output_template()}',
+          archiving: '#{default_output_template()}'
+        }
+    }"}>
       <.input
         field={f[:output_path_template]}
         type="text"
         inputclass="font-mono"
         label="Output path template"
         help="Must end with .{{ ext }}. See below for more details. The default is good for most cases (required)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Subtitle Options
     </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:download_subs]}
         type="toggle"
         label="Download Subtitles"
         help="Downloads subtitle files alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: false } }">
       <.input
         field={f[:download_auto_subs]}
         type="toggle"
         label="Download Autogenerated Subtitles"
-    help="Prefers normal subs but will download autogenerated if needed"
+        help="Prefers normal subs but will download autogenerated if needed. Requires 'Download Subtitles' to be enabled"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:embed_subs]}
         type="toggle"
         label="Embed Subtitles"
-    help="Embeds subtitles in the video file itself, if supported (recommended)"
+        help="Downloads and embeds subtitles in the media file itself, if supported. Uneffected by 'Download Subtitles' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: 'en', media_center: 'en', audio: '', archiving: 'all' } }">
       <.input
         field={f[:sub_langs]}
         type="text"
         label="Subtitle Languages"
         help="Use commas for multiple languages (eg: en,de)"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Thumbnail Options
     </h3>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: false, archiving: true } }">
       <.input
         field={f[:download_thumbnail]}
         type="toggle"
         label="Download Thumbnail"
         help="Downloads thumbnail alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
       <.input
         field={f[:embed_thumbnail]}
         type="toggle"
         label="Embed Thumbnail"
-    help="Embeds thumbnail in the video file itself, if supported (recommended)"
+        help="Downloads and embeds thumbnail in the media file itself, if supported. Uneffected by 'Download Thumbnail' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Metadata Options
     </h3>
+
+    <section x-data="{ presets: { default: false, media_center: false, audio: false, archiving: true } }">
       <.input
         field={f[:download_metadata]}
         type="toggle"
         label="Download Metadata"
         help="Downloads metadata file alongside media file"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: true, media_center: true, audio: true, archiving: true } }">
       <.input
         field={f[:embed_metadata]}
         type="toggle"
         label="Embed Metadata"
-    help="Embeds metadata in the video file itself, if supported (recommended)"
+        help="Downloads and embeds metadata in the media file itself, if supported. Uneffected by 'Download Metadata' (recommended)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Release Format Options
     </h3>
 
+    <section x-data="{ presets: { default: 'include', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
       <.input
         field={f[:shorts_behaviour]}
         options={friendly_format_type_options()}
         type="select"
-    label="Include Shorts?"
+        label="Include Shorts"
         help="Experimental. Please report any issues on GitHub"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
+
+    <section x-data="{ presets: { default: 'exclude', media_center: 'exclude', audio: 'exclude', archiving: 'include' } }">
       <.input
         field={f[:livestream_behaviour]}
         options={friendly_format_type_options()}
         type="select"
-    label="Include Livestreams?"
+        label="Include Livestreams"
+        help="Excludes media that comes from a past livestream"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+    <h3 class="mt-10 text-2xl text-black dark:text-white">
       Quality Options
     </h3>
 
+    <section x-data="{ presets: { default: '1080p', media_center: '1080p', audio: 'audio', archiving: '2160p' } }">
       <.input
         field={f[:preferred_resolution]}
         options={friendly_resolution_options()}
         type="select"
         label="Preferred Resolution"
-    help="Will grab the closest available resolution if your preferred is not available"
+        help="Will grab the closest available resolution if your preferred is not available. 'Audio Only' grabs the highest quality m4a"
+        x-init="$watch('selectedPreset', p => p && ($el.value = presets[p]))"
       />
+    </section>
 
-  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Media profile</.button>
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
+      Media Center Options
+    </h3>
+    <p class="text-sm mt-2 max-w-prose">
+      Everything in this section is experimental - please open a GitHub issue if you see something odd.
+      <strong>
+        These options only work if this Media Profile's output template is set to split media into seasons.
+      </strong>
+      Try the "Media Center" preset if you're not sure.
+    </p>
+
+    <section
+      phx-click={show_modal("upgrade-modal")}
+      x-data="{ presets: { default: false, media_center: true, audio: false, archiving: false } }"
+    >
+      <.input
+        field={f[:download_nfo]}
+        type="toggle"
+        label="Download NFO data"
+        label_suffix="(pro)"
+        help="Downloads NFO data alongside media file for use with Jellyfin, Kodi, etc."
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <section x-data="{ presets: { default: false, media_center: true, audio: false, archiving: true } }">
+      <.input
+        field={f[:download_source_images]}
+        type="toggle"
+        label="Download Series Images"
+        help="Downloads poster and banner images for use with Jellyfin, Kodi, etc. Only works for full channels (not playlists)"
+        x-init="$watch('selectedPreset', p => p && (enabled = presets[p]))"
+      />
+    </section>
+
+    <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Media profile</.button>
+  </section>
 
   <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
     <.output_template_help />

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -4,13 +4,13 @@
       <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
     </.link>
     <h2 class="text-title-md2 font-bold text-black dark:text-white ml-2">
-      Media Profile <span class="hidden sm:inline">#<%= @media_profile.id %></span>
+      <%= @media_profile.name %>
     </h2>
   </div>
 
   <nav>
     <.link href={~p"/media_profiles/#{@media_profile}/edit"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <.icon name="hero-pencil-square" class="mr-2" />Edit <span class="hidden sm:inline pl-1">Media Profile</span>
       </.button>
     </.link>
@@ -31,7 +31,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this profile and all its sources (leaving files in place)? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Profile and its Sources
             </.button>
           </.link>
@@ -41,7 +41,7 @@
             data-confirm="Are you sure you want to delete this profile, all its sources, and its files on disk? This cannot be undone."
             class="mt-5 md:mt-0"
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Profile, Sources, and Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/pages/page_html/onboarding_checklist.html.heex

@@ -11,7 +11,7 @@
         <p class="text-md text-bodydark">Don't worry, you can create more Media Profiles later!</p>
         <div class="mt-8">
           <.link href={~p"/media_profiles/new"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={@media_profiles_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={@media_profiles_exist}>
               <span class="font-bold mx-2">+</span> New Media Profile
             </.button>
           </.link>
@@ -25,7 +25,7 @@
         </p>
         <div class="mt-8">
           <.link href={~p"/sources/new"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={not @media_profiles_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={not @media_profiles_exist}>
               <span class="font-bold mx-2">+</span> New Source
             </.button>
           </.link>
@@ -40,7 +40,7 @@
         <p class="text-md text-bodydark">Feel free to add more Media Profiles or Sources in the meantime!</p>
         <div class="mt-8">
           <.link href={~p"/?onboarding=0"}>
-            <.button color="bg-primary" rounding="rounded-full" disabled={not @sources_exist}>
+            <.button color="bg-primary" rounding="rounded-lg" disabled={not @sources_exist}>
               Let's Go <span class="font-bold mx-2">🚀</span>
             </.button>
           </.link>

lib/pinchflat_web/controllers/searches/search_html/show.html.heex

@@ -1,6 +1,6 @@
 <div class="mb-6 flex gap-3 flex-row items-center justify-between">
   <h2 class="text-title-md2 font-bold text-black dark:text-white">
-    <span class="hidden sm:inline">Search </span>Results for "<%= StringUtils.truncate(@search_term, 50) %>"
+    Results for "<%= StringUtils.truncate(@search_term, 50) %>"
   </h2>
 </div>
 
@@ -10,7 +10,7 @@
       <%= if match?([_|_], @search_results) do %>
         <.table rows={@search_results} table_class="text-black dark:text-white">
           <:col :let={result} label="Title">
-            <%= StringUtils.truncate(result.title, 40) %>
+            <%= result.title %>
           </:col>
           <:col :let={result} label="Excerpt">
             <.highlight_search_terms text={result.matching_search_term} />

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -46,7 +46,7 @@ defmodule PinchflatWeb.Sources.SourceController do
   def show(conn, %{"id" => id}) do
     source = Repo.preload(Sources.get_source!(id), :media_profile)
 
-    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(:source_id, source.id), :job)
+    pending_tasks = Repo.preload(Tasks.list_pending_tasks_for(source), :job)
     pending_media = Media.list_pending_media_items_for(source, limit: 100)
     downloaded_media = Media.list_downloaded_media_items_for(source, limit: 100)
 

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -14,7 +14,7 @@ defmodule PinchflatWeb.Sources.SourceHTML do
 
   def friendly_index_frequencies do
     [
-      {"On Create", -1},
+      {"Only once when first created", -1},
       {"1 Hour", 60},
       {"3 Hours", 3 * 60},
       {"6 Hours", 6 * 60},

lib/pinchflat_web/controllers/sources/source_html/edit.html.heex

@@ -2,7 +2,9 @@
   <.link href={~p"/sources"}>
     <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
   </.link>
-  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">Edit Source</h2>
+  <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
+    Editing "<%= @source.custom_name %>"
+  </h2>
 </div>
 
 <div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">

lib/pinchflat_web/controllers/sources/source_html/index.html.heex

@@ -2,7 +2,7 @@
   <h2 class="text-title-md2 font-bold text-black dark:text-white">Sources</h2>
   <nav>
     <.link href={~p"/sources/new"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <span class="font-bold mx-2">+</span> New <span class="hidden sm:inline pl-1">Source</span>
       </.button>
     </.link>
@@ -14,19 +14,18 @@
     <div class="flex flex-col gap-10 min-w-max">
       <.table rows={@sources} table_class="text-black dark:text-white">
         <:col :let={source} label="Name">
+          <.subtle_link href={~p"/sources/#{source.id}"}>
             <%= source.custom_name || source.collection_name %>
+          </.subtle_link>
         </:col>
         <:col :let={source} label="Type"><%= source.collection_type %></:col>
         <:col :let={source} label="Should Download?">
           <.icon name={if source.download_media, do: "hero-check", else: "hero-x-mark"} />
         </:col>
         <:col :let={source} label="Media Profile">
-          <.link
-            href={~p"/media_profiles/#{source.media_profile_id}"}
-            class="hover:text-secondary duration-200 ease-in-out"
-          >
+          <.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
             <%= source.media_profile.name %>
-          </.link>
+          </.subtle_link>
         </:col>
         <:col :let={source} label="" class="flex place-content-evenly">
           <.icon_link href={~p"/sources/#{source.id}"} icon="hero-eye" class="mx-1" />

lib/pinchflat_web/controllers/sources/source_html/show.html.heex

@@ -4,13 +4,13 @@
       <.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
     </.link>
     <h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
-      Source <span class="hidden sm:inline">#<%= @source.id %></span>
+      <%= @source.custom_name %>
     </h2>
   </div>
 
   <nav>
     <.link href={~p"/sources/#{@source}/edit"}>
-      <.button color="bg-primary" rounding="rounded-full">
+      <.button color="bg-primary" rounding="rounded-lg">
         <.icon name="hero-pencil-square" class="mr-2" /> Edit <span class="hidden sm:inline pl-1">Source</span>
       </.button>
     </.link>
@@ -38,7 +38,7 @@
             method="delete"
             data-confirm="Are you sure you want to delete this source (leaving files in place)? This cannot be undone."
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Source
             </.button>
           </.link>
@@ -48,7 +48,7 @@
             data-confirm="Are you sure you want to delete this source and it's files on disk? This cannot be undone."
             class="mt-5 md:mt-0"
           >
-            <.button color="bg-meta-1" rounding="rounded-full">
+            <.button color="bg-meta-1" rounding="rounded-lg">
               Delete Source and Files
             </.button>
           </.link>

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -1,11 +1,22 @@
-<.simple_form :let={f} for={@changeset} action={@action}>
+<.simple_form
+  :let={f}
+  for={@changeset}
+  action={@action}
+  x-data="{ advancedMode: !!JSON.parse(localStorage.getItem('advancedMode')) }"
+  x-init="$watch('advancedMode', value => localStorage.setItem('advancedMode', JSON.stringify(value)))"
+>
   <.error :if={@changeset.action}>
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
+  <section class="flex justify-between items-center mt-8">
+    <h3 class=" text-2xl text-black dark:text-white">
       General Options
     </h3>
+    <span class="cursor-pointer hover:underline" x-on:click="advancedMode = !advancedMode">
+      Editing Mode: <span x-text="advancedMode ? 'Advanced' : 'Basic'"></span>
+    </span>
+  </section>
 
   <.input
     field={f[:custom_name]}
@@ -21,6 +32,7 @@
     options={Enum.map(@media_profiles, &{&1.name, &1.id})}
     type="select"
     label="Media Profile"
+    help="Sets your preferences for what media to look for and how to store it"
   />
 
   <h3 class="mt-8 text-2xl text-black dark:text-white">
@@ -32,7 +44,7 @@
     options={friendly_index_frequencies()}
     type="select"
     label="Index Frequency"
-    help="Time between one index of this source finishing and the next one starting. Setting to 'On Create' will still run an initial index but no subsequent ones"
+    help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
   />
 
   <%!-- TODO: use Alpine to disable the index frequency when fast indexing is enabled --%>
@@ -40,7 +52,7 @@
     <.input
       field={f[:fast_index]}
       type="toggle"
-      label="Use Fast Indexing?"
+      label="Use Fast Indexing"
       label_suffix="(pro)"
       help="Experimental. Ignores 'Index Frequency'. Recommended for large channels that upload frequently. See below for more info"
     />
@@ -53,7 +65,7 @@
   <.input
     field={f[:download_media]}
     type="toggle"
-    label="Download Media?"
+    label="Download Media"
     help="Unchecking still indexes media but it won't be downloaded until you enable this option"
   />
 
@@ -62,10 +74,30 @@
     type="text"
     label="Download Cutoff Date"
     placeholder="YYYY-MM-DD"
+    maxlength="10"
+    pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
+    title="YYYY-MM-DD"
     help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
   />
 
-  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto">Save Source</.button>
+  <section x-show="advancedMode">
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
+      Advanced Options
+    </h3>
+    <p class="text-sm mt-2">
+      Tread carefully
+    </p>
+
+    <.input
+      field={f[:title_filter_regex]}
+      type="text"
+      label="Title Filter Regex"
+      placeholder="(?i)^How to Bike$"
+      help="A PCRE-compatible regex. Only media with titles that match this regex will be downloaded. Look up 'SQLean Regex docs' for more"
+    />
+  </section>
+
+  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Source</.button>
 
   <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
     <.fast_indexing_help />

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.0-alpha.2",
+      version: "0.1.0-alpha.7",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -13,6 +13,16 @@ defmodule Pinchflat.MixProject do
       preferred_cli_env: [
         check: :test,
         credo: :test
+      ],
+      test_coverage: [
+        ignore_modules: [
+          Pinchflat.HTTP.HTTPClient,
+          PinchflatWeb.Layouts,
+          Pinchflat.DataCase,
+          Pinchflat.Release,
+          ~r/Fixtures/,
+          ~r/HTML$/
+        ]
       ]
     ]
   end
@@ -59,6 +69,7 @@ defmodule Pinchflat.MixProject do
       {:nimble_parsec, "~> 1.4"},
       {:mox, "~> 1.0", only: :test},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
+      {:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
       {:ex_check, "~> 0.14.0", only: [:dev, :test], runtime: false},
       {:faker, "~> 0.17", only: :test},
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false}

mix.lock

@@ -6,6 +6,7 @@
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
+  "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
   "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},

priv/repo/migrations/20240313175638_add_nfo_filepath_to_media_item.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddNfoFilepathToMediaItem do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_items) do
+      add :nfo_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240313175744_add_download_nfo_to_media_profile.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddDownloadNfoToMediaProfile do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_nfo, :boolean, default: false, null: false
+    end
+  end
+end

priv/repo/migrations/20240314174731_create_source_metadata.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.CreateSourceMetadata do
+  use Ecto.Migration
+
+  def change do
+    create table(:source_metadata) do
+      add :metadata_filepath, :string, null: false
+      add :source_id, references(:sources, on_delete: :delete_all), null: false
+
+      timestamps(type: :utc_datetime)
+    end
+
+    create unique_index(:source_metadata, [:source_id])
+  end
+end