Media downloading indexing options (round 2) (#14)

* Adds options + option builder + metadata parsing for media thumbnails

* Added release-type options to media profile; built option parser for indexing operations

* Added new media_profile options to creation form; made show helper for rendering database items

* Added options for downloading/embedding metadata

* Adds option on sources to not download media (index only)

* reformatted docs

lib/pinchflat/profiles/media_profile.ex

@@ -15,6 +15,12 @@ defmodule Pinchflat.Profiles.MediaProfile do
     download_auto_subs
     embed_subs
     sub_langs
+    download_thumbnail
+    embed_thumbnail
+    download_metadata
+    embed_metadata
+    shorts_behaviour
+    livestream_behaviour
   )a
 
   @required_fields ~w(name output_path_template)a
@@ -22,11 +28,26 @@ defmodule Pinchflat.Profiles.MediaProfile do
   schema "media_profiles" do
     field :name, :string
     field :output_path_template, :string
+
     field :download_subs, :boolean, default: true
     field :download_auto_subs, :boolean, default: true
     field :embed_subs, :boolean, default: true
     field :sub_langs, :string, default: "en"
 
+    field :download_thumbnail, :boolean, default: true
+    field :embed_thumbnail, :boolean, default: true
+
+    field :download_metadata, :boolean, default: true
+    field :embed_metadata, :boolean, default: true
+
+    # NOTE: these do NOT speed up indexing - the indexer still has to go
+    # through the entire collection to determine if a video is a short or
+    # a livestream.
+    # NOTE: these can BOTH be set to :only which will download shorts and
+    # livestreams _only_ and ignore regular videos.
+    field :shorts_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+    field :livestream_behaviour, Ecto.Enum, values: [:include, :exclude, :only], default: :include
+
     has_many :sources, Source
 
     timestamps(type: :utc_datetime)

lib/pinchflat/profiles/options/yt_dlp/download_option_builder.ex

@@ -1,6 +1,6 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
+defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder do
   @moduledoc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider making this a behaviour so I can add other backends later
   """
@@ -9,7 +9,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
   alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
 
   @doc """
-  Builds the options for yt-dlp based on the given media profile.
+  Builds the options for yt-dlp to download media based on the given media profile.
 
   IDEA: consider adding the ability to pass in a second argument to override
         these options
@@ -24,6 +24,8 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     built_options =
       default_options() ++
         subtitle_options(media_profile) ++
+        thumbnail_options(media_profile) ++
+        metadata_options(media_profile) ++
         output_options(media_profile)
 
     {:ok, built_options}
@@ -31,11 +33,7 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
 
   # This will be updated a lot as I add new options to profiles
   defp default_options do
-    [
-      :embed_metadata,
-      :embed_thumbnail,
-      :no_progress
-    ]
+    [:no_progress]
   end
 
   defp subtitle_options(media_profile) do
@@ -65,6 +63,30 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OptionBuilder do
     end)
   end
 
+  defp thumbnail_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} -> acc ++ [:write_thumbnail]
+        {:embed_thumbnail, true} -> acc ++ [:embed_thumbnail]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
   defp output_options(media_profile) do
     {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template)
 

lib/pinchflat/profiles/options/yt_dlp/index_option_builder.ex

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+
+  alias Pinchflat.Profiles.MediaProfile
+
+  @doc """
+  Builds the options for yt-dlp to index a media source based on the given media profile.
+  """
+  def build(%MediaProfile{} = media_profile) do
+    built_options = release_type_options(media_profile)
+
+    {:ok, built_options}
+  end
+
+  defp release_type_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    # Appending multiple match filters treats them as an OR condition,
+    # so we have to be careful around combining `only` and `exclude` options.
+    # eg: only shorts + exclude livestreams = "any video that is a short OR is not a livestream"
+    # which will return all shorts AND normal videos.
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:shorts_behaviour, :only}, _} ->
+          acc ++ [match_filter: "original_url*=/shorts/"]
+
+        {{:livestream_behaviour, :only}, _} ->
+          acc ++ [match_filter: "was_live"]
+
+        # Since match_filter is an OR (see above), `exclude`s must be ignored entirely if the
+        # other type is set to `only`. There is also special behaviour if they're both excludes,
+        # hence why these check against `:include` alone.
+        {{:shorts_behaviour, :exclude}, %{livestream_behaviour: :include}} ->
+          acc ++ [match_filter: "original_url!*=/shorts/"]
+
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :include}} ->
+          acc ++ [match_filter: "!was_live"]
+
+        # Again, since it's an OR, there's a special syntax if they're both excluded
+        # to make it an AND. Note that I'm not checking for the other permutation of
+        # both excluding since this MUST get hit so adding the other version would double up.
+        {{:livestream_behaviour, :exclude}, %{shorts_behaviour: :exclude}} ->
+          acc ++ [match_filter: "!was_live & original_url!*=/shorts/"]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+end

lib/pinchflat/profiles/options/yt_dlp/output_path_builder.ex

@@ -12,6 +12,11 @@ defmodule Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder do
 
   Translates liquid-style templates into yt-dlp-style templates,
   leaving yt-dlp syntax intact.
+
+  IDEA: apart from any custom options I've defined, I can support any yt-dlp
+  option by assuming `{{ identifier }}` should transform to `%(identifier)S`.
+  It's not doing anything huge, but it's nicer to type and more approachable IMO.
+  IDEA: set a default for `MediaProfile`'s `output_path_template` field
   """
   def build(template_string) do
     TemplateParser.parse(template_string, full_yt_dlp_options_map())