Media downloading indexing options (round 2) (#14)

* Adds options + option builder + metadata parsing for media thumbnails

* Added release-type options to media profile; built option parser for indexing operations

* Added new media_profile options to creation form; made show helper for rendering database items

* Added options for downloading/embedding metadata

* Adds option on sources to not download media (index only)

* reformatted docs

lib/pinchflat/media/media_item.ex

@@ -10,13 +10,23 @@ defmodule Pinchflat.Media.MediaItem do
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Media.MediaMetadata
 
+  @allowed_fields ~w(
+    title
+    media_id
+    media_filepath
+    source_id
+    subtitle_filepaths
+    thumbnail_filepath
+    metadata_filepath
+  )a
   @required_fields ~w(media_id source_id)a
-  @allowed_fields ~w(title media_id media_filepath source_id subtitle_filepaths)a
 
   schema "media_items" do
     field :title, :string
     field :media_id, :string
     field :media_filepath, :string
+    field :thumbnail_filepath, :string
+    field :metadata_filepath, :string
     # This is an array of [iso-2 language, filepath] pairs. Probably could
     # be an associated record, but I don't see the benefit right now.
     # Will very likely revisit because I can't leave well-enough alone.

lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex

@@ -25,6 +25,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
     metadata_attrs
     |> Map.merge(parse_media_metadata(metadata))
     |> Map.merge(parse_subtitle_metadata(metadata))
+    |> Map.merge(parse_thumbnail_metadata(metadata))
+    |> Map.merge(parse_infojson_metadata(metadata))
   end
 
   defp parse_media_metadata(metadata) do
@@ -35,10 +37,9 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
   end
 
   defp parse_subtitle_metadata(metadata) do
-    subtitle_map = metadata["requested_subtitles"] || %{}
     # IDEA: if needed, consider filtering out subtitles that don't exist on-disk
     subtitle_filepaths =
-      subtitle_map
+      (metadata["requested_subtitles"] || %{})
       |> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end)
       |> Enum.sort(fn [lang_a, _], [lang_b, _] -> lang_a < lang_b end)
 
@@ -46,4 +47,24 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
       subtitle_filepaths: subtitle_filepaths
     }
   end
+
+  defp parse_thumbnail_metadata(metadata) do
+    thumbnail_filepath =
+      (metadata["thumbnails"] || %{})
+      # Reverse so that higher resolution thumbnails come first.
+      # This _shouldn't_ matter yet, but I'd rather default to the best
+      # in case I'm wrong.
+      |> Enum.reverse()
+      |> Enum.find_value(fn attrs -> attrs["filepath"] end)
+
+    %{
+      thumbnail_filepath: thumbnail_filepath
+    }
+  end
+
+  defp parse_infojson_metadata(metadata) do
+    %{
+      metadata_filepath: metadata["infojson_filename"]
+    }
+  end
 end

lib/pinchflat/media_client/source_details.ex

@@ -6,10 +6,14 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   it open-ish for future expansion (just in case).
   """
 
+  alias Pinchflat.Repo
+  alias Pinchflat.MediaSource.Source
+
   alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection, as: YtDlpSource
+  alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder, as: YtDlpIndexOptionBuilder
 
   @doc """
-  Gets a source's ID and name from its URL, using the given backend.
+  Gets a source's ID and name from its URL using the given backend.
 
   Returns {:ok, map()} | {:error, any, ...}.
   """
@@ -18,11 +22,23 @@ defmodule Pinchflat.MediaClient.SourceDetails do
   end
 
   @doc """
-  Returns a list of video IDs for the given source URL, using the given backend.
+  Returns a list of video IDs for the given source URL OR source record using the given backend.
+
+  If passing a source record, the call to the backend may have custom options applied based on
+  the `option_builder`.
 
   Returns {:ok, list(binary())} | {:error, any, ...}.
   """
-  def get_video_ids(source_url, backend \\ :yt_dlp) do
+  def get_video_ids(sourceable, backend \\ :yt_dlp)
+
+  def get_video_ids(%Source{} = source, backend) do
+    media_profile = Repo.preload(source, :media_profile).media_profile
+    {:ok, options} = option_builder(backend).build(media_profile)
+
+    source_module(backend).get_video_ids(source.collection_id, options)
+  end
+
+  def get_video_ids(source_url, backend) when is_binary(source_url) do
     source_module(backend).get_video_ids(source_url)
   end
 
@@ -31,4 +47,10 @@ defmodule Pinchflat.MediaClient.SourceDetails do
       :yt_dlp -> YtDlpSource
     end
   end
+
+  defp option_builder(backend) do
+    case backend do
+      :yt_dlp -> YtDlpIndexOptionBuilder
+    end
+  end
 end

lib/pinchflat/media_client/video_downloader.ex

@@ -14,7 +14,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   alias Pinchflat.Profiles.MediaProfile
 
   alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
-  alias Pinchflat.Profiles.Options.YtDlp.OptionBuilder, as: YtDlpOptionBuilder
+  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
 
   @doc """
@@ -22,6 +22,10 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   returned by the backend. Also saves the entire metadata response to the associated
   media_metadata record.
 
+  NOTE: related methods (like the download worker) won't download if the media item's source
+  is set to not download media. However, I'm not enforcing that here since I need this for testing.
+  This may change in the future but I'm not stressed.
+
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def download_for_media_item(%MediaItem{} = media_item, backend \\ :yt_dlp) do
@@ -52,7 +56,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
 
   defp option_builder(backend) do
     case backend do
-      :yt_dlp -> YtDlpOptionBuilder
+      :yt_dlp -> YtDlpDownloadOptionBuilder
     end
   end
 

lib/pinchflat/media_source/source.ex

@@ -15,6 +15,7 @@ defmodule Pinchflat.MediaSource.Source do
     collection_type
     friendly_name
     index_frequency_minutes
+    download_media
     original_url
     media_profile_id
   )a
@@ -27,6 +28,7 @@ defmodule Pinchflat.MediaSource.Source do
     field :collection_id, :string
     field :collection_type, Ecto.Enum, values: [:channel, :playlist]
     field :index_frequency_minutes, :integer, default: 60 * 24
+    field :download_media, :boolean, default: true
     # This should only be used for user reference going forward
     # as the collection_id should be used for all API calls
     field :original_url, :string

lib/pinchflat/profiles/media_profile.ex

@@ -15,6 +15,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     download_auto_subs
     embed_subs
     sub_langs
+    download_thumbnail
+    embed_thumbnail
+    download_metadata
+    embed_metadata
+    shorts_behaviour
+    livestream_behaviour
   )a
 
   @required_fields ~w(name output_path_template)a
@@ -22,11 +28,26 @@ defmodule Pinchflat.Profiles.MediaProfile do
   schema "media_profiles" do
     field :name, :string
     field :output_path_template, :string
+
     field :download_subs, :boolean, default: true
     field :download_auto_subs, :boolean, default: true
     field :embed_subs, :boolean, default: true
     field :sub_langs, :string, default: "en"
 
+    field :download_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: true
+
+    field :download_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: true
+
+    # NOTE: these do NOT speed up indexing - the indexer still has to go
+    # through the entire collection to determine if a video is a short or
+    # a livestream.
+    # NOTE: these can BOTH be set to :only which will download shorts and
+    # livestreams _only_ and ignore regular videos.
+    field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+    field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+
     has_many :sources, Source
 
     timestamps(type: :utc_datetime)

lib/pinchflat/profiles/options/yt_dlp/download_option_builder.ex

@@ -1,6 +1,6 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
+defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
   @moduledoc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider making this a behaviour so I can add other backends later
   """
@@ -9,7 +9,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
   alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
 
   @doc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider adding the ability to pass in a second argument to override
         these options
@@ -24,6 +24,8 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
+        thumbnail_options(media_profile) ++
+        metadata_options(media_profile) ++
         output_options(media_profile)
 
     {:ok, built_options}
@@ -31,11 +33,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
 
   # This will be updated a lot as I add new options to profiles
   defp default_options do
-    [
-      :embed_metadata,
-      :embed_thumbnail,
-      :no_progress
-    ]
+    [:no_progress]
   end
 
   defp subtitle_options(media_profile) do
@@ -65,6 +63,30 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     end)
   end
 
+  defp thumbnail_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} -> acc ++ [:write_thumbnail]
+        {:embed_thumbnail, true} -> acc ++ [:embed_thumbnail]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
   defp output_options(media_profile) do
     {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)
 

lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+
+  alias Pinchflat.Profiles.MediaProfile
+
+  @doc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+  def build(%MediaProfile{} = media_profile) do
+    built_options = release_type_options(media_profile)
+
+    {:ok, built_options}
+  end
+
+  defp release_type_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    # Appending multiple match filters treats them as an OR condition,
+    # so we have to be careful around combining `only` and `exclude` options.
+    # eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
+    # which will return all shorts AND normal videos.
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:shorts_behaviour, :only}, _} ->
+          acc ++ [match_filter: "original_url*=/shorts/"]
+
+        {{:livestream_behaviour, :only}, _} ->
+          acc ++ [match_filter: "was_live"]
+
+        # Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
+        # other type is set to `only`. There is also special behaviour if they're both excludes,
+        # hence why these check against `:include` alone.
+        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
+          acc ++ [match_filter: "original_url!*=/shorts/"]
+
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
+          acc ++ [match_filter: "!was_live"]
+
+        # Again, since it's an OR, there's a special syntax if they're both excluded
+        # to make it an AND. Note that I'm not checking for the other permutation of
+        # both excluding since this MUST get hit so adding the other version would double up.
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
+          acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+end

lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex

@@ -12,6 +12,11 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
 
   Translates liquid-style templates into yt-dlp-style templates,
   leaving yt-dlp syntax intact.
+
+  IDEA: apart from any custom options I've defined, I can support any yt-dlp
+  option by assuming `{{ identifier }}` should transform to `%(identifier)S`.
+  It's not doing anything huge, but it's nicer to type and more approachable IMO.
+  IDEA: set a default for `MediaProfile`'s `output_path_template` field
   """
   def build(template_string) do
     TemplateParser.parse(template_string, full_yt_dlp_options_map())

lib/pinchflat/tasks/source_tasks.ex

@@ -35,6 +35,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   @doc """
   Starts tasks for downloading videos for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
 
   NOTE: this starts a download for each media item that is pending,
   not just the ones that were indexed in this job run. This should ensure
@@ -45,7 +46,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
 
   Returns :ok
   """
-  def enqueue_pending_media_downloads(%Source{} = source) do
+  def enqueue_pending_media_downloads(%Source{download_media: true} = source) do
     source
     |> Media.list_pending_media_items_for()
     |> Enum.each(fn media_item ->
@@ -55,4 +56,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
       |> Tasks.create_job_with_task(media_item)
     end)
   end
+
+  def enqueue_pending_media_downloads(%Source{download_media: false} = _source) do
+    :ok
+  end
 end

lib/pinchflat/workers/video_download_worker.ex

@@ -6,18 +6,32 @@ defmodule Pinchflat.Workers.VideoDownloadWorker do
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
     tags: ["media_item", "media_fetching"]
 
+  alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.MediaClient.VideoDownloader
 
   @impl Oban.Worker
   @doc """
-  For a given media item, download the video and save the metadata.
+  For a given media item, download the media alongside any options.
+  Does not download media if its source is set to not download media.
 
-  Returns {:ok, %MediaItem{}} | {:error, any, ...any}
+  Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
-    media_item = Media.get_media_item!(media_item_id)
+    media_item =
+      media_item_id
+      |> Media.get_media_item!()
+      |> Repo.preload(:source)
 
+    # If the source is set to not download media, perform a no-op
+    if media_item.source.download_media do
+      download_media(media_item)
+    else
+      :ok
+    end
+  end
+
+  defp download_media(media_item) do
     case VideoDownloader.download_for_media_item(media_item) do
       {:ok, _} -> {:ok, media_item}
       err -> err

lib/pinchflat_web/components/core_components.ex

@@ -535,7 +535,7 @@ defmodule PinchflatWeb.CoreComponents do
 
   def list(assigns) do
     ~H"""
-    <div class="mt-14">
+    <div class="mt-2 mb-14">
       <dl class="-my-4 divide-y divide-zinc-100">
         <div :for={item <- @item} class="flex gap-4 py-4 text-sm leading-6 sm:gap-8">
           <dt class="w-1/4 flex-none text-zinc-500"><%= item.title %></dt>
@@ -546,6 +546,30 @@ defmodule PinchflatWeb.CoreComponents do
     """
   end
 
+  @doc """
+  Renders a data list from a given map. Used in development to
+  quickly show the attributes of a database record.
+  """
+  attr :map, :map, required: true
+
+  def list_items_from_map(assigns) do
+    attrs =
+      Enum.filter(assigns.map, fn
+        {_, %{__struct__: _}} -> false
+        _ -> true
+      end)
+
+    assigns = assign(assigns, iterable_attributes: attrs)
+
+    ~H"""
+    <.list>
+      <:item :for={{k, v} <- @iterable_attributes} title={k}>
+        <%= v %>
+      </:item>
+    </.list>
+    """
+  end
+
   @doc """
   Renders a back navigation link.
 

lib/pinchflat_web/controllers/media_profiles/media_profile_html.ex

@@ -10,4 +10,12 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileHTML do
   attr :action, :string, required: true
 
   def media_profile_form(assigns)
+
+  def friendly_format_type_options do
+    [
+      {"Include (default)", :include},
+      {"Exclude", :exclude},
+      {"Only", :only}
+    ]
+  end
 end

lib/pinchflat_web/controllers/media_profiles/media_profile_html/media_profile_form.html.heex

@@ -4,10 +4,35 @@
   </.error>
   <.input field={f[:name]} type="text" label="Name" />
   <.input field={f[:output_path_template]} type="text" label="Output path template" />
+
+  <h3>Subtitle Options</h3>
   <.input field={f[:download_subs]} type="checkbox" label="Download Subs" />
   <.input field={f[:download_auto_subs]} type="checkbox" label="Download Autogenerated Subs" />
   <.input field={f[:embed_subs]} type="checkbox" label="Embed Subs" />
   <.input field={f[:sub_langs]} type="text" label="Sub Langs" />
+
+  <h3>Thumbnail Options</h3>
+  <.input field={f[:download_thumbnail]} type="checkbox" label="Download Thumbnail" />
+  <.input field={f[:embed_thumbnail]} type="checkbox" label="Embed Thumbnail" />
+
+  <h3>Metadata Options</h3>
+  <.input field={f[:download_metadata]} type="checkbox" label="Download Metadata" />
+  <.input field={f[:embed_metadata]} type="checkbox" label="Embed Metadata" />
+
+  <h3>Release Format Options</h3>
+  <.input
+    field={f[:shorts_behaviour]}
+    options={friendly_format_type_options()}
+    type="select"
+    label="Include Shorts?"
+  />
+  <.input
+    field={f[:livestream_behaviour]}
+    options={friendly_format_type_options()}
+    type="select"
+    label="Include Livestreams?"
+  />
+
   <:actions>
     <.button>Save Media profile</.button>
   </:actions>

lib/pinchflat_web/controllers/media_profiles/media_profile_html/show.html.heex

@@ -8,15 +8,6 @@
   </:actions>
 </.header>
 
-<.list>
-  <:item
-    :for={
-      attr <- ~w(name output_path_template download_subs download_auto_subs embed_subs sub_langs)a
-    }
-    title={attr}
-  >
-    <%= Map.get(@media_profile, attr) %>
-  </:item>
-</.list>
+<.list_items_from_map map={Map.from_struct(@media_profile)} />
 
 <.back navigate={~p"/media_profiles"}>Back to media_profiles</.back>

lib/pinchflat_web/controllers/media_sources/source_html/show.html.heex

@@ -8,19 +8,16 @@
   </:actions>
 </.header>
 
+<h3 class="mt-14">Relationships</h3>
 <.list>
   <:item title="media_profile">
     <.link href={~p"/media_profiles/#{@source.media_profile}"}>
       <%= @source.media_profile.name %>
     </.link>
   </:item>
-
-  <:item
-    :for={attr <- ~w(collection_type collection_name collection_id original_url friendly_name)a}
-    title={attr}
-  >
-    <%= Map.get(@source, attr) %>
-  </:item>
 </.list>
 
+<h3>Attributes</h3>
+<.list_items_from_map map={Map.from_struct(@source)} />
+
 <.back navigate={~p"/media_sources/sources"}>Back to sources</.back>

lib/pinchflat_web/controllers/media_sources/source_html/source_form.html.heex

@@ -28,6 +28,8 @@
     label="Index Frequency"
   />
 
+  <.input field={f[:download_media]} type="checkbox" label="Download Media?" />
+
   <:actions>
     <.button>Save Source</.button>
   </:actions>