Media downloading indexing options (round 2) (#14)

* Adds options + option builder + metadata parsing for media thumbnails

* Added release-type options to media profile; built option parser for indexing operations

* Added new media_profile options to creation form; made show helper for rendering database items

* Added options for downloading/embedding metadata

* Adds option on sources to not download media (index only)

* reformatted docs

.gitignore

@@ -39,3 +39,4 @@ npm-debug.log
 /.elixir_ls
 .env
 .DS_Store
+scratchpad.md

lib/pinchflat/media/media_item.ex

@@ -10,13 +10,23 @@ defmodule Pinchflat.Media.MediaItem do
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Media.MediaMetadata
 
+  @allowed_fields ~w(
+    title
+    media_id
+    media_filepath
+    source_id
+    subtitle_filepaths
+    thumbnail_filepath
+    metadata_filepath
+  )a
   @required_fields ~w(media_id source_id)a
-  @allowed_fields ~w(title media_id media_filepath source_id subtitle_filepaths)a
 
   schema "media_items" do
     field :title, :string
     field :media_id, :string
     field :media_filepath, :string
+    field :thumbnail_filepath, :string
+    field :metadata_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.

lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex

@@ -25,6 +25,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
     metadata_attrs
     |> Map.merge(parse_media_metadata(metadata))
     |> Map.merge(parse_subtitle_metadata(metadata))
+    |> Map.merge(parse_thumbnail_metadata(metadata))
+    |> Map.merge(parse_infojson_metadata(metadata))
   end
 
   defp parse_media_metadata(metadata) do
@@ -35,10 +37,9 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
   end
 
   defp parse_subtitle_metadata(metadata) do
-    subtitle_map = metadata["requested_subtitles"] || %{}
     # IDEA: if needed, consider filtering out subtitles that don't exist on-disk
     subtitle_filepaths =
-      subtitle_map
+      (metadata["requested_subtitles"] || %{})
       |> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end)
       |> Enum.sort(fn [lang_a, _], [lang_b, _] -> lang_a < lang_b end)
 
@@ -46,4 +47,24 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
       subtitle_filepaths: subtitle_filepaths
     }
   end
+
+  defp parse_thumbnail_metadata(metadata) do
+    thumbnail_filepath =
+      (metadata["thumbnails"] || %{})
+      # Reverse so that higher resolution thumbnails come first.
+      # This _shouldn't_ matter yet, but I'd rather default to the best
+      # in case I'm wrong.
+      |> Enum.reverse()
+      |> Enum.find_value(fn attrs -> attrs["filepath"] end)
+
+    %{
+      thumbnail_filepath: thumbnail_filepath
+    }
+  end
+
+  defp parse_infojson_metadata(metadata) do
+    %{
+      metadata_filepath: metadata["infojson_filename"]
+    }
+  end
 end

lib/pinchflat/media_client/source_details.ex

@@ -6,10 +6,14 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   it open-ish for future expansion (just in case).
   """
 
+  alias Pinchflat.Repo
+  alias Pinchflat.MediaSource.Source
+
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
+  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
 
   @doc """
-  Gets a source's ID and name from its URL, using the given backend.
+  Gets a source's ID and name from its URL using the given backend.
 
   Returns {:ok, map()} | {:error, any, ...}.
   """
@@ -18,11 +22,23 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   end
 
   @doc """
-  Returns a list of video IDs for the given source URL, using the given backend.
+  Returns a list of video IDs for the given source URL OR source record using the given backend.
+
+  If passing a source record, the call to the backend may have custom options applied based on
+  the `option_builder`.
 
   Returns {:ok, list(binary())} | {:error, any, ...}.
   """
-  def get_video_ids(source_url, backend \\ :yt_dlp) do
+  def get_video_ids(sourceable, backend \\ :yt_dlp)
+
+  def get_video_ids(%Source{} = source, backend) do
+    media_profile = Repo.preload(source, :media_profile).media_profile
+    {:ok, options} = option_builder(backend).build(media_profile)
+
+    source_module(backend).get_video_ids(source.collection_id, options)
+  end
+
+  def get_video_ids(source_url, backend) when is_binary(source_url) do
     source_module(backend).get_video_ids(source_url)
   end
 
@@ -31,4 +47,10 @@ defmodule Pinchflat.MediaClient.SourceDetails do
       :yt_dlp -> YtDlpSource
     end
   end
+
+  defp option_builder(backend) do
+    case backend do
+      :yt_dlp -> YtDlpIndexOptionBuilder
+    end
+  end
 end

lib/pinchflat/media_client/video_downloader.ex

@@ -14,7 +14,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   alias Pinchflat.Profiles.MediaProfile
 
   alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
-  alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder
+  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
 
   @doc """
@@ -22,6 +22,10 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   returned by the backend. Also saves the entire metadata response to the associated
   media_metadata record.
 
+  NOTE: related methods (like the download worker) won't download if the media item's source
+  is set to not download media. However, I'm not enforcing that here since I need this for testing.
+  This may change in the future but I'm not stressed.
+
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
@@ -52,7 +56,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
 
   defp option_builder(backend) do
     case backend do
-      :yt_dlp -> YtDlpOptionBuilder
+      :yt_dlp -> YtDlpDownloadOptionBuilder
     end
   end
 

lib/pinchflat/media_source/source.ex

@@ -15,6 +15,7 @@ defmodule Pinchflat.MediaSource.Source do
     collection_type
     friendly_name
     index_frequency_minutes
+    download_media
     original_url
     media_profile_id
   )a
@@ -27,6 +28,7 @@ defmodule Pinchflat.MediaSource.Source do
     field :collection_id, :string
     field :collection_type, Ecto.Enum, values: [:channel, :playlist]
     field :index_frequency_minutes, :integer, default: 60 * 24
+    field :download_media, :boolean, default: true
     # This should only be used for user reference going forward
     # as the collection_id should be used for all API calls
     field :original_url, :string

lib/pinchflat/profiles/media_profile.ex

@@ -15,6 +15,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     download_auto_subs
     embed_subs
     sub_langs
+    download_thumbnail
+    embed_thumbnail
+    download_metadata
+    embed_metadata
+    shorts_behaviour
+    livestream_behaviour
   )a
 
   @required_fields ~w(name output_path_template)a
@@ -22,11 +28,26 @@ defmodule Pinchflat.Profiles.MediaProfile do
   schema "media_profiles" do
     field :name, :string
     field :output_path_template, :string
+
     field :download_subs, :boolean, default: true
     field :download_auto_subs, :boolean, default: true
     field :embed_subs, :boolean, default: true
     field :sub_langs, :string, default: "en"
 
+    field :download_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: true
+
+    field :download_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: true
+
+    # NOTE: these do NOT speed up indexing - the indexer still has to go
+    # through the entire collection to determine if a video is a short or
+    # a livestream.
+    # NOTE: these can BOTH be set to :only which will download shorts and
+    # livestreams _only_ and ignore regular videos.
+    field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+    field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+
     has_many :sources, Source
 
     timestamps(type: :utc_datetime)

lib/pinchflat/profiles/options/yt_dlp/download_option_builder.ex

@@ -1,6 +1,6 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
+defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
   @moduledoc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider making this a behaviour so I can add other backends later
   """
@@ -9,7 +9,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
   alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
 
   @doc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider adding the ability to pass in a second argument to override
         these options
@@ -24,6 +24,8 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
+        thumbnail_options(media_profile) ++
+        metadata_options(media_profile) ++
         output_options(media_profile)
 
     {:ok, built_options}
@@ -31,11 +33,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
 
   # This will be updated a lot as I add new options to profiles
   defp default_options do
-    [
+    [:no_progress]
-      :embed_metadata,
-      :embed_thumbnail,
-      :no_progress
-    ]
   end
 
   defp subtitle_options(media_profile) do
@@ -65,6 +63,30 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     end)
   end
 
+  defp thumbnail_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} -> acc ++ [:write_thumbnail]
+        {:embed_thumbnail, true} -> acc ++ [:embed_thumbnail]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
   defp output_options(media_profile) do
     {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)
 

lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+
+  alias Pinchflat.Profiles.MediaProfile
+
+  @doc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+  def build(%MediaProfile{} = media_profile) do
+    built_options = release_type_options(media_profile)
+
+    {:ok, built_options}
+  end
+
+  defp release_type_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    # Appending multiple match filters treats them as an OR condition,
+    # so we have to be careful around combining `only` and `exclude` options.
+    # eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
+    # which will return all shorts AND normal videos.
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:shorts_behaviour, :only}, _} ->
+          acc ++ [match_filter: "original_url*=/shorts/"]
+
+        {{:livestream_behaviour, :only}, _} ->
+          acc ++ [match_filter: "was_live"]
+
+        # Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
+        # other type is set to `only`. There is also special behaviour if they're both excludes,
+        # hence why these check against `:include` alone.
+        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
+          acc ++ [match_filter: "original_url!*=/shorts/"]
+
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
+          acc ++ [match_filter: "!was_live"]
+
+        # Again, since it's an OR, there's a special syntax if they're both excluded
+        # to make it an AND. Note that I'm not checking for the other permutation of
+        # both excluding since this MUST get hit so adding the other version would double up.
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
+          acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+end

lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex

@@ -12,6 +12,11 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
 
   Translates liquid-style templates into yt-dlp-style templates,
   leaving yt-dlp syntax intact.
+
+  IDEA: apart from any custom options I've defined, I can support any yt-dlp
+  option by assuming `{{ identifier }}` should transform to `%(identifier)S`.
+  It's not doing anything huge, but it's nicer to type and more approachable IMO.
+  IDEA: set a default for `MediaProfile`'s `output_path_template` field
   """
   def build(template_string) do
     TemplateParser.parse(template_string, full_yt_dlp_options_map())

lib/pinchflat/tasks/source_tasks.ex

@@ -35,6 +35,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   @doc """
   Starts tasks for downloading videos for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
 
   NOTE: this starts a download for each media item that is pending,
   not just the ones that were indexed in this job run. This should ensure
@@ -45,7 +46,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   Returns :ok
   """
-  def enqueue_pending_media_downloads(%Source{} = source) do
+  def enqueue_pending_media_downloads(%Source{download_media: true} = source) do
     source
     |> Media.list_pending_media_items_for()
     |> Enum.each(fn media_item ->
@@ -55,4 +56,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
       |> Tasks.create_job_with_task(media_item)
     end)
   end
+
+  def enqueue_pending_media_downloads(%Source{download_media: false} = _source) do
+    :ok
+  end
 end

lib/pinchflat/workers/video_download_worker.ex

@@ -6,18 +6,32 @@ defmodule Pinchflat.Workers.VideoDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.MediaClient.VideoDownloader
 
   @impl Oban.Worker
   @doc """
-  For a given media item, download the video and save the metadata.
+  For a given media item, download the media alongside any options.
+  Does not download media if its source is set to not download media.
 
-  Returns {:ok, %MediaItem{}} | {:error, any, ...any}
+  Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
+    media_item =
+      media_item_id
+      |> Media.get_media_item!()
+      |> Repo.preload(:source)
 
+    # If the source is set to not download media, perform a no-op
+    if media_item.source.download_media do
+      download_media(media_item)
+    else
+      :ok
+    end
+  end
+
+  defp download_media(media_item) do
     case VideoDownloader.download_for_media_item(media_item) do
       {:ok, _} -> {:ok, media_item}
       err -> err

lib/pinchflat_web/components/core_components.ex

@@ -535,7 +535,7 @@ defmodule PinchflatWeb.CoreComponents do
 
   def list(assigns) do
     ~H"""
-    <div class="mt-14">
+    <div class="mt-2 mb-14">
       <dl class="-my-4 divide-y divide-zinc-100">
         <div :for={item <- @item} class="flex gap-4 py-4 text-sm leading-6 sm:gap-8">
           <dt class="w-1/4 flex-none text-zinc-500"><%= item.title %></dt>
@@ -546,6 +546,30 @@ defmodule PinchflatWeb.CoreComponents do
     """
   end
 
+  @doc """
+  Renders a data list from a given map. Used in development to
+  quickly show the attributes of a database record.
+  """
+  attr :map, :map, required: true
+
+  def list_items_from_map(assigns) do
+    attrs =
+      Enum.filter(assigns.map, fn
+        {_, %{__struct__: _}} -> false
+        _ -> true
+      end)
+
+    assigns = assign(assigns, iterable_attributes: attrs)
+
+    ~H"""
+    <.list>
+      <:item :for={{k, v} <- @iterable_attributes} title={k}>
+        <%= v %>
+      </:item>
+    </.list>
+    """
+  end
+
   @doc """
   Renders a back navigation link.
 

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -10,4 +10,12 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   attr :action, :string, required: true
 
   def media_profile_form(assigns)
+
+  def friendly_format_type_options do
+    [
+      {"Include (default)", :include},
+      {"Exclude", :exclude},
+      {"Only", :only}
+    ]
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -4,10 +4,35 @@
   </.error>
   <.input field={f[:name]} type="text" label="Name" />
   <.input field={f[:output_path_template]} type="text" label="Output path template" />
+
+  <h3>Subtitle Options</h3>
   <.input field={f[:download_subs]} type="checkbox" label="Download Subs" />
   <.input field={f[:download_auto_subs]} type="checkbox" label="Download Autogenerated Subs" />
   <.input field={f[:embed_subs]} type="checkbox" label="Embed Subs" />
   <.input field={f[:sub_langs]} type="text" label="Sub Langs" />
+
+  <h3>Thumbnail Options</h3>
+  <.input field={f[:download_thumbnail]} type="checkbox" label="Download Thumbnail" />
+  <.input field={f[:embed_thumbnail]} type="checkbox" label="Embed Thumbnail" />
+
+  <h3>Metadata Options</h3>
+  <.input field={f[:download_metadata]} type="checkbox" label="Download Metadata" />
+  <.input field={f[:embed_metadata]} type="checkbox" label="Embed Metadata" />
+
+  <h3>Release Format Options</h3>
+  <.input
+    field={f[:shorts_behaviour]}
+    options={friendly_format_type_options()}
+    type="select"
+    label="Include Shorts?"
+  />
+  <.input
+    field={f[:livestream_behaviour]}
+    options={friendly_format_type_options()}
+    type="select"
+    label="Include Livestreams?"
+  />
+
   <:actions>
     <.button>Save Media profile</.button>
   </:actions>

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -8,15 +8,6 @@
   </:actions>
 </.header>
 
-<.list>
+<.list_items_from_map map={Map.from_struct(@media_profile)} />
-  <:item
-    :for={
-      attr <- ~w(name output_path_template download_subs download_auto_subs embed_subs sub_langs)a
-    }
-    title={attr}
-  >
-    <%= Map.get(@media_profile, attr) %>
-  </:item>
-</.list>
 
 <.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

lib/pinchflat_web/controllers/media_sources/source_html/show.html.heex

@@ -8,19 +8,16 @@
   </:actions>
 </.header>
 
+<h3 class="mt-14">Relationships</h3>
 <.list>
   <:item title="media_profile">
     <.link href={~p"/media_profiles/#{@source.media_profile}"}>
       <%= @source.media_profile.name %>
     </.link>
   </:item>
-
-  <:item
-    :for={attr <- ~w(collection_type collection_name collection_id original_url friendly_name)a}
-    title={attr}
-  >
-    <%= Map.get(@source, attr) %>
-  </:item>
 </.list>
 
+<h3>Attributes</h3>
+<.list_items_from_map map={Map.from_struct(@source)} />
+
 <.back navigate={~p"/media_sources/sources"}>Back to sources</.back>

lib/pinchflat_web/controllers/media_sources/source_html/source_form.html.heex

@@ -28,6 +28,8 @@
     label="Index Frequency"
   />
 
+  <.input field={f[:download_media]} type="checkbox" label="Download Media?" />
+
   <:actions>
     <.button>Save Source</.button>
   </:actions>

priv/repo/migrations/20240205194558_add_thumbnail_options_to_media_profiles.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.AddThumbnailOptionsToMediaProfiles do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_thumbnail, :boolean, default: true, null: false
+      add :embed_thumbnail, :boolean, default: true, null: false
+    end
+
+    alter table(:media_items) do
+      add :thumbnail_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240205215634_add_release_type_behaviour_to_media_profiles.exs

@@ -0,0 +1,10 @@
+defmodule Pinchflat.Repo.Migrations.AddReleaseTypeBehaviourToMediaProfiles do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :shorts_behaviour, :string, null: false, default: "include"
+      add :livestream_behaviour, :string, null: false, default: "include"
+    end
+  end
+end

priv/repo/migrations/20240206201218_add_metadata_options_to_media_profiles.exs

@@ -0,0 +1,14 @@
+defmodule Pinchflat.Repo.Migrations.AddMetadataOptionsToMediaProfiles do
+  use Ecto.Migration
+
+  def change do
+    alter table(:media_profiles) do
+      add :download_metadata, :boolean, default: true, null: false
+      add :embed_metadata, :boolean, default: true, null: false
+    end
+
+    alter table(:media_items) do
+      add :metadata_filepath, :string
+    end
+  end
+end

priv/repo/migrations/20240206204740_add_download_media_to_sources.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddDownloadMediaToSources do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :download_media, :boolean, default: true, null: false
+    end
+  end
+end

test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs

@@ -47,7 +47,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
     test "extracts the subtitle filepaths", %{metadata: metadata} do
       result = Parser.parse_for_media_item(metadata)
 
-      assert [["de", german_filepath], ["en", english_filepath]] = result.subtitle_filepaths
+      assert [["de", german_filepath], ["en", english_filepath] | _rest] = result.subtitle_filepaths
 
       assert String.ends_with?(english_filepath, ".en.srt")
       assert String.ends_with?(german_filepath, ".de.srt")
@@ -83,4 +83,44 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
       assert result.subtitle_filepaths == []
     end
   end
+
+  describe "parse_for_media_item/1 when testing thumbnail metadata" do
+    test "extracts the thumbnail filepath", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert String.ends_with?(result.thumbnail_filepath, ".webp")
+    end
+
+    test "doesn't freak out if the video has no thumbnails", %{metadata: metadata} do
+      metadata = Map.put(metadata, "thumbnails", %{})
+
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.thumbnail_filepath == nil
+    end
+
+    test "doesn't freak out if the thumbnails key is missing", %{metadata: metadata} do
+      metadata = Map.delete(metadata, "thumbnails")
+
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.thumbnail_filepath == nil
+    end
+  end
+
+  describe "parse_for_media_item/1 when testing infojson metadata" do
+    test "extracts the metadata filepath", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert String.ends_with?(result.metadata_filepath, ".info.json")
+    end
+
+    test "doesn't freak out if the video has no infojson", %{metadata: metadata} do
+      metadata = Map.put(metadata, "infojson_filename", nil)
+
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.metadata_filepath == nil
+    end
+  end
 end

test/pinchflat/media_client/source_details_test.exs

@@ -1,6 +1,8 @@
 defmodule Pinchflat.MediaClient.SourceDetailsTest do
-  use ExUnit.Case, async: true
+  use Pinchflat.DataCase
   import Mox
+  import Pinchflat.ProfilesFixtures
+  import Pinchflat.MediaSourceFixtures
 
   alias Pinchflat.MediaClient.SourceDetails
 
@@ -41,7 +43,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
     end
   end
 
-  describe "get_video_ids/2" do
+  describe "get_video_ids/2 when passed a string" do
     test "it passes the expected arguments to the backend" do
       expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
         assert opts == [:simulate, :skip_download]
@@ -61,4 +63,33 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
       assert {:ok, ["video1", "video2", "video3"]} = SourceDetails.get_video_ids(@channel_url)
     end
   end
+
+  describe "get_video_ids/2 when passed a Source record" do
+    test "it calls the backend with the source's collection ID" do
+      source = source_fixture()
+
+      expect(YtDlpRunnerMock, :run, fn url, _opts, _ot ->
+        assert source.collection_id == url
+        {:ok, "video1\nvideo2\nvideo3"}
+      end)
+
+      assert {:ok, _} = SourceDetails.get_video_ids(source)
+    end
+
+    test "it builds options based on the source's media profile" do
+      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
+        assert opts == [{:match_filter, "!was_live"}, :simulate, :skip_download]
+        {:ok, ""}
+      end)
+
+      media_profile =
+        media_profile_fixture(
+          shorts_behaviour: :include,
+          livestream_behaviour: :exclude
+        )
+
+      source = source_fixture(media_profile_id: media_profile.id)
+      assert {:ok, _} = SourceDetails.get_video_ids(source)
+    end
+  end
 end

test/pinchflat/media_client/video_downloader_test.exs

@@ -28,18 +28,6 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
       assert {:ok, _} = VideoDownloader.download_for_media_item(media_item)
     end
 
-    test "it writes attributes to the media item", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, render_metadata(:media_metadata)}
-      end)
-
-      assert %{media_filepath: nil, title: nil, subtitle_filepaths: []} = media_item
-      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
-      assert updated_media_item.media_filepath
-      assert updated_media_item.title
-      assert length(updated_media_item.subtitle_filepaths) > 0
-    end
-
     test "it saves the metadata to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
         {:ok, render_metadata(:media_metadata)}
@@ -59,4 +47,44 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
       assert {:error, :some_error} = VideoDownloader.download_for_media_item(media_item)
     end
   end
+
+  describe "download_for_media_item/3 when testing media_item attributes" do
+    setup do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      :ok
+    end
+
+    test "it extracts the title", %{media_item: media_item} do
+      assert media_item.title == nil
+      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
+      assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
+    end
+
+    test "it extracts the media_filepath", %{media_item: media_item} do
+      assert media_item.media_filepath == nil
+      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
+      assert String.ends_with?(updated_media_item.media_filepath, ".mkv")
+    end
+
+    test "it extracts the subtitle_filepaths", %{media_item: media_item} do
+      assert media_item.subtitle_filepaths == []
+      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
+      assert [["de", _], ["en", _] | _rest] = updated_media_item.subtitle_filepaths
+    end
+
+    test "it extracts the thumbnail_filepath", %{media_item: media_item} do
+      assert media_item.thumbnail_filepath == nil
+      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
+      assert String.ends_with?(updated_media_item.thumbnail_filepath, ".webp")
+    end
+
+    test "it extracts the metadata_filepath", %{media_item: media_item} do
+      assert media_item.metadata_filepath == nil
+      assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
+      assert String.ends_with?(updated_media_item.metadata_filepath, ".info.json")
+    end
+  end
 end

test/pinchflat/profiles/options/yt_dlp/download_option_builder_test.exs

@@ -0,0 +1,161 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilderTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder
+
+  @media_profile %MediaProfile{
+    output_path_template: "{{ title }}.%(ext)s"
+  }
+
+  describe "build/1" do
+    test "it generates an expanded output path based on the given template" do
+      assert {:ok, res} = DownloadOptionBuilder.build(@media_profile)
+
+      assert {:output, "/tmp/videos/%(title)S.%(ext)s"} in res
+    end
+  end
+
+  describe "build/1 when testing subtitle options" do
+    test "includes :write_subs option when specified" do
+      media_profile = %MediaProfile{@media_profile | download_subs: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :write_subs in res
+    end
+
+    test "forces SRT format when download_subs is true" do
+      media_profile = %MediaProfile{@media_profile | download_subs: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert {:convert_subs, "srt"} in res
+    end
+
+    test "includes :write_auto_subs option when specified" do
+      media_profile = %MediaProfile{@media_profile | download_subs: true, download_auto_subs: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :write_auto_subs in res
+    end
+
+    test "doesn't include :write_auto_subs option when download_subs is false" do
+      media_profile = %MediaProfile{@media_profile | download_subs: false, download_auto_subs: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      refute :write_auto_subs in res
+    end
+
+    test "includes :embed_subs option when specified" do
+      media_profile = %MediaProfile{@media_profile | embed_subs: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :embed_subs in res
+    end
+
+    test "includes sub_langs option when download_subs is true" do
+      media_profile = %MediaProfile{@media_profile | download_subs: true, sub_langs: "en"}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert {:sub_langs, "en"} in res
+    end
+
+    test "includes sub_langs option when embed_subs is true" do
+      media_profile = %MediaProfile{@media_profile | embed_subs: true, sub_langs: "en"}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert {:sub_langs, "en"} in res
+    end
+
+    test "doesn't include sub_langs option when neither downloading nor embedding" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | embed_subs: false,
+          download_subs: false,
+          sub_langs: "en"
+      }
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      refute {:sub_langs, "en"} in res
+    end
+
+    test "other struct attributes are ignored" do
+      media_profile = %MediaProfile{@media_profile | id: -1}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      refute {:id, -1} in res
+    end
+  end
+
+  describe "build/1 when testing thumbnail options" do
+    test "includes :write_thumbnail option when specified" do
+      media_profile = %MediaProfile{@media_profile | download_thumbnail: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :write_thumbnail in res
+    end
+
+    test "includes :embed_thumbnail option when specified" do
+      media_profile = %MediaProfile{@media_profile | embed_thumbnail: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :embed_thumbnail in res
+    end
+
+    test "doesn't include these options when not specified" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | embed_thumbnail: false,
+          download_thumbnail: false
+      }
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      refute :write_thumbnail in res
+      refute :embed_thumbnail in res
+    end
+  end
+
+  describe "build/1 when testing metadata options" do
+    test "includes :write_info_json option when specified" do
+      media_profile = %MediaProfile{@media_profile | download_metadata: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :write_info_json in res
+      assert :clean_info_json in res
+    end
+
+    test "includes :embed_metadata option when specified" do
+      media_profile = %MediaProfile{@media_profile | embed_metadata: true}
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      assert :embed_metadata in res
+    end
+
+    test "doesn't include these options when not specified" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | embed_metadata: false,
+          download_metadata: false
+      }
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_profile)
+
+      refute :write_info_json in res
+      refute :clean_info_json in res
+      refute :embed_metadata in res
+    end
+  end
+end

test/pinchflat/profiles/options/yt_dlp/index_option_builder_test.exs

@@ -0,0 +1,104 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilderTest do
+  use ExUnit.Case, async: true
+
+  alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder
+
+  @media_profile %MediaProfile{
+    output_path_template: "{{ title }}.%(ext)s",
+    shorts_behaviour: :include,
+    livestream_behaviour: :include
+  }
+
+  describe "build/1 when testing release type options" do
+    test "adds correct filter when shorts_behaviour is :only" do
+      media_profile = %MediaProfile{@media_profile | shorts_behaviour: :only}
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "original_url*=/shorts/"} in res
+      refute {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "!was_live"} in res
+      refute {:match_filter, "was_live"} in res
+    end
+
+    test "adds correct filter when livestream_behaviour is :only" do
+      media_profile = %MediaProfile{@media_profile | livestream_behaviour: :only}
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "was_live"} in res
+      refute {:match_filter, "!was_live"} in res
+      refute {:match_filter, "!original_url*=/shorts/"} in res
+      refute {:match_filter, "original_url*=/shorts/"} in res
+    end
+
+    test "adds correct filter when both livestreams and shorts are :only" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | shorts_behaviour: :only,
+          livestream_behaviour: :only
+      }
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "original_url*=/shorts/"} in res
+      assert {:match_filter, "was_live"} in res
+      refute {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "!was_live"} in res
+    end
+
+    test "adds correct filter when shorts_behaviour is :exclude" do
+      media_profile = %MediaProfile{@media_profile | shorts_behaviour: :exclude}
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "original_url*=/shorts/"} in res
+      refute {:match_filter, "was_live"} in res
+      refute {:match_filter, "!was_live"} in res
+    end
+
+    test "adds correct filter when livestream_behaviour is :exclude" do
+      media_profile = %MediaProfile{@media_profile | livestream_behaviour: :exclude}
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "!was_live"} in res
+      refute {:match_filter, "was_live"} in res
+      refute {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "original_url*=/shorts/"} in res
+    end
+
+    test "adds correct filter when shorts and livestreams are both exclude" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | shorts_behaviour: :exclude,
+          livestream_behaviour: :exclude
+      }
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "!was_live & original_url!*=/shorts/"} in res
+      refute {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "!was_live"} in res
+      refute {:match_filter, "original_url*=/shorts/"} in res
+      refute {:match_filter, "was_live"} in res
+    end
+
+    test "does not add exclusion filter if one is excluded and the other is only" do
+      media_profile = %MediaProfile{
+        @media_profile
+        | shorts_behaviour: :exclude,
+          livestream_behaviour: :only
+      }
+
+      assert {:ok, res} = IndexOptionBuilder.build(media_profile)
+
+      assert {:match_filter, "was_live"} in res
+      refute {:match_filter, "original_url!*=/shorts/"} in res
+      refute {:match_filter, "original_url*=/shorts/"} in res
+      refute {:match_filter, "!was_live"} in res
+    end
+  end
+end

test/pinchflat/profiles/options/yt_dlp/option_builder_test.exs

@@ -1,97 +0,0 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilderTest do
-  use ExUnit.Case, async: true
-
-  alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder
-
-  @media_profile %MediaProfile{
-    output_path_template: "{{ title }}.%(ext)s"
-  }
-
-  describe "build/1" do
-    test "it generates an expanded output path based on the given template" do
-      assert {:ok, res} = OptionBuilder.build(@media_profile)
-
-      assert {:output, "/tmp/videos/%(title)S.%(ext)s"} in res
-    end
-  end
-
-  describe "build/1 when testing subtitle options" do
-    test "includes :write_subs option when specified" do
-      media_profile = %MediaProfile{@media_profile | download_subs: true}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert :write_subs in res
-    end
-
-    test "forces SRT format when download_subs is true" do
-      media_profile = %MediaProfile{@media_profile | download_subs: true}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert {:convert_subs, "srt"} in res
-    end
-
-    test "includes :write_auto_subs option when specified" do
-      media_profile = %MediaProfile{@media_profile | download_subs: true, download_auto_subs: true}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert :write_auto_subs in res
-    end
-
-    test "doesn't include :write_auto_subs option when download_subs is false" do
-      media_profile = %MediaProfile{@media_profile | download_subs: false, download_auto_subs: true}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      refute :write_auto_subs in res
-    end
-
-    test "includes :embed_subs option when specified" do
-      media_profile = %MediaProfile{@media_profile | embed_subs: true}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert :embed_subs in res
-    end
-
-    test "includes sub_langs option when download_subs is true" do
-      media_profile = %MediaProfile{@media_profile | download_subs: true, sub_langs: "en"}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert {:sub_langs, "en"} in res
-    end
-
-    test "includes sub_langs option when embed_subs is true" do
-      media_profile = %MediaProfile{@media_profile | embed_subs: true, sub_langs: "en"}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      assert {:sub_langs, "en"} in res
-    end
-
-    test "doesn't include sub_langs option when neither downloading nor embedding" do
-      media_profile = %MediaProfile{
-        @media_profile
-        | embed_subs: false,
-          download_subs: false,
-          sub_langs: "en"
-      }
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      refute {:sub_langs, "en"} in res
-    end
-
-    test "other struct attributes are ignored" do
-      media_profile = %MediaProfile{@media_profile | id: -1}
-
-      assert {:ok, res} = OptionBuilder.build(media_profile)
-
-      refute {:id, -1} in res
-    end
-  end
-end

test/pinchflat/tasks/source_tasks_test.exs

@@ -75,5 +75,21 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
       assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
     end
+
+    test "it does not create a job if the source is set to not download" do
+      source = source_fixture(download_media: false)
+
+      assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
+
+      refute_enqueued(worker: VideoDownloadWorker)
+    end
+
+    test "it does not attach tasks if the source is set to not download" do
+      source = source_fixture(download_media: false)
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = SourceTasks.enqueue_pending_media_downloads(source)
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
   end
 end

test/pinchflat/workers/video_download_worker_test.exs

@@ -4,6 +4,7 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
   import Mox
   import Pinchflat.MediaFixtures
 
+  alias Pinchflat.MediaSource
   alias Pinchflat.Workers.VideoDownloadWorker
 
   setup :verify_on_exit!
@@ -55,5 +56,13 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
         assert job.state == "retryable"
       end)
     end
+
+    test "it does not download if the source is set to not download", %{media_item: media_item} do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot -> :ok end)
+
+      MediaSource.update_source(media_item.source, %{download_media: false})
+
+      perform_job(VideoDownloadWorker, %{id: media_item.id})
+    end
   end
 end