Refactor modules into contexts (#78)

* [WIP] break out a few contexts, start refactoring fast index modules

* [WIP] more contexts, this time around slow indexing and downloads

* [WIP] got all tests passing

* [WIP] Added moduledocs

* Built a genserver to rename old jobs on boot

* Added a module naming check; moved things around

* Fixed specs

lib/pinchflat/downloading/download_option_builder.ex

@@ -0,0 +1,126 @@
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
+  @moduledoc """
+  Builds the options for yt-dlp to download media based on the given media profile.
+  """
+
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.OutputPathBuilder
+
+  @doc """
+  Builds the options for yt-dlp to download media based on the given media's profile.
+
+  IDEA: consider adding the ability to pass in a second argument to override
+        these options
+  """
+  def build(%MediaItem{} = media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
+
+    built_options =
+      default_options() ++
+        subtitle_options(media_profile) ++
+        thumbnail_options(media_profile) ++
+        metadata_options(media_profile) ++
+        quality_options(media_profile) ++
+        output_options(media_item_with_preloads)
+
+    {:ok, built_options}
+  end
+
+  defp default_options do
+    [:no_progress, :windows_filenames]
+  end
+
+  defp subtitle_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case {attr, media_profile} do
+        {{:download_subs, true}, _} ->
+          # Force SRT for now - MAY provide as an option in the future
+          acc ++ [:write_subs, convert_subs: "srt"]
+
+        {{:download_auto_subs, true}, %{download_subs: true}} ->
+          acc ++ [:write_auto_subs]
+
+        {{:embed_subs, true}, %{preferred_resolution: pr}} when pr != :audio ->
+          acc ++ [:embed_subs]
+
+        {{:sub_langs, sub_langs}, %{download_subs: true}} ->
+          acc ++ [sub_langs: sub_langs]
+
+        {{:sub_langs, sub_langs}, %{embed_subs: true}} ->
+          acc ++ [sub_langs: sub_langs]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
+  defp thumbnail_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_thumbnail, true} ->
+          acc ++ [:write_thumbnail, convert_thumbnail: "jpg"]
+
+        {:embed_thumbnail, true} ->
+          acc ++ [:embed_thumbnail]
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
+  defp metadata_options(media_profile) do
+    mapped_struct = Map.from_struct(media_profile)
+
+    Enum.reduce(mapped_struct, [], fn attr, acc ->
+      case attr do
+        {:download_metadata, true} -> acc ++ [:write_info_json, :clean_info_json]
+        {:embed_metadata, true} -> acc ++ [:embed_metadata]
+        _ -> acc
+      end
+    end)
+  end
+
+  defp quality_options(media_profile) do
+    video_codec_options = "+codec:avc:m4a"
+
+    case media_profile.preferred_resolution do
+      # Also be aware that :audio disabled all embedding options for subtitles
+      :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
+      :"360p" -> [format_sort: "res:360,#{video_codec_options}"]
+      :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
+      :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
+      :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
+      :"1440p" -> [format_sort: "res:1440,#{video_codec_options}"]
+      :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
+    end
+  end
+
+  defp output_options(media_item_with_preloads) do
+    media_profile = media_item_with_preloads.source.media_profile
+    additional_options_map = output_options_map(media_item_with_preloads)
+    {:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
+
+    [
+      output: Path.join(base_directory(), output_path)
+    ]
+  end
+
+  defp output_options_map(media_item_with_preloads) do
+    source = media_item_with_preloads.source
+
+    %{
+      "source_custom_name" => source.custom_name,
+      "source_collection_type" => source.collection_type
+    }
+  end
+
+  defp base_directory do
+    Application.get_env(:pinchflat, :media_directory)
+  end
+end

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,50 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(fn media_item ->
+      %{id: media_item.id}
+      |> MediaDownloadWorker.new()
+      |> Tasks.create_job_with_task(media_item)
+    end)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -0,0 +1,56 @@
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :media_fetching,
+    unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
+    tags: ["media_item", "media_fetching"]
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.MediaDownloader
+  alias Pinchflat.Filesystem.FilesystemDataWorker
+
+  @impl Oban.Worker
+  @doc """
+  For a given media item, download the media alongside any options.
+  Does not download media if its source is set to not download media.
+
+  Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
+  """
+  def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
+    media_item =
+      media_item_id
+      |> Media.get_media_item!()
+      |> Repo.preload(:source)
+
+    # If the source is set to not download media, perform a no-op
+    if media_item.source.download_media do
+      download_media_and_schedule_jobs(media_item)
+    else
+      :ok
+    end
+  end
+
+  defp download_media_and_schedule_jobs(media_item) do
+    case MediaDownloader.download_for_media_item(media_item) do
+      {:ok, _} ->
+        schedule_filesystem_data_worker(media_item)
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp schedule_filesystem_data_worker(media_item) do
+    %{id: media_item.id}
+    |> FilesystemDataWorker.new()
+    |> Tasks.create_job_with_task(media_item)
+    |> case do
+      {:ok, task} -> {:ok, task}
+      {:error, :duplicate_job} -> {:ok, :job_exists}
+    end
+  end
+end

lib/pinchflat/downloading/media_downloader.ex

@@ -0,0 +1,65 @@
+defmodule Pinchflat.Downloading.MediaDownloader do
+  @moduledoc """
+  This is the integration layer for actually downloading media.
+  It takes into account the media profile's settings in order
+  to download the media with the desired options.
+  """
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Media
+  alias Pinchflat.Media.MediaItem
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+  alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
+  alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
+  alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
+
+  @doc """
+  Downloads media for a media item, updating the media item based on the metadata
+  returned by yt-dlp. Also saves the entire metadata response to the associated
+  media_metadata record.
+
+  NOTE: related methods (like the download worker) won't download if the media item's source
+  is set to not download media. However, I'm not enforcing that here since I need this for testing.
+  This may change in the future but I'm not stressed.
+
+  Returns {:ok, %MediaItem{}} | {:error, any, ...any}
+  """
+  def download_for_media_item(%MediaItem{} = media_item) do
+    item_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
+
+    case download_with_options(media_item.original_url, item_with_preloads) do
+      {:ok, parsed_json} ->
+        {parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
+
+        parsed_attrs =
+          parsed_json
+          |> parser.parse_for_media_item()
+          |> Map.merge(%{
+            media_downloaded_at: DateTime.utc_now(),
+            metadata: %{
+              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+            }
+          })
+
+        # Don't forgor to use preloaded associations or updates to
+        # associations won't work!
+        Media.update_media_item(item_with_preloads, parsed_attrs)
+
+      err ->
+        err
+    end
+  end
+
+  # def download_for_source(source, url) do
+  #   # Create MI from source and URL
+  #   media_item = nil
+  # end
+
+  defp download_with_options(url, item_with_preloads) do
+    {:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
+
+    YtDlpMedia.download(url, options)
+  end
+end

lib/pinchflat/downloading/output_path/base.ex

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Downloading.OutputPath.Base do
+  @moduledoc """
+  A base module for parsing rendered strings, designed as a macro to be used
+  in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
+
+  NOTE: if the needs here get any more complicated, look into using a Liquid
+  template parser. No need to reinvent the wheel any more than I already have.
+
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
+  """
+
+  defmacro __using__(_opts) do
+    quote location: :keep do
+      import NimbleParsec
+
+      opening_tag = string("{{")
+      closing_tag = string("}}")
+      optional_whitespaces = ascii_string(~c[ \t\n\r], min: 0)
+
+      # Capture everything up to the opening object
+      text =
+        lookahead_not(opening_tag)
+        # ... as long as it's a character
+        |> utf8_char([])
+        # ... and there's at least one character
+        |> times(min: 1)
+        # ... and then convert it to a string
+        |> reduce({List, :to_string, []})
+        # ... finally bag it and tag it
+        |> unwrap_and_tag(:text)
+
+      identifier =
+        utf8_string([?a..?z, ?A..?Z, ?_, ?0..?9], min: 1)
+        |> reduce({Enum, :join, []})
+        |> unwrap_and_tag(:identifier)
+
+      defparsecp(:expression, identifier)
+
+      # when spotting interpolation, ignore the opening tag and any whitespace
+      interpolation =
+        ignore(concat(opening_tag, optional_whitespaces))
+        # ... then parse the expression (identifier)
+        |> parsec(:expression)
+        # ... then ignore any whitespace and the closing tags after the expression
+        |> ignore(concat(optional_whitespaces, closing_tag))
+        # ... once again we bag it and tag it
+        |> unwrap_and_tag(:interpolation)
+
+      defparsec(:do_parse, choice([interpolation, text]) |> repeat() |> eos())
+    end
+  end
+end

lib/pinchflat/downloading/output_path/parser.ex

@@ -0,0 +1,42 @@
+defmodule Pinchflat.Downloading.OutputPath.Parser do
+  @moduledoc """
+  Parses liquid-ish-style strings into a rendered string
+
+  Used for turning filepath templates into real filepaths
+  """
+
+  use Pinchflat.Downloading.OutputPath.Base
+
+  @doc """
+  Parses a string into a rendered string, using the provided variables. Optionally
+  takes a custom fetcher function for handling missing variables.
+
+  Variable identifiers are surrounded by {{ and }}. The variable keys MUST be strings.
+  If an identifier is not found in the provided variables, it will be removed from the string.
+
+  Returns `{:ok, binary()}` or `{:error, binary()}`.
+  """
+  def parse(string, variables, value_fetch_fn \\ &default_fetcher/2) do
+    # `do_parse` comes from `RenderedString.Base`
+    case do_parse(string) do
+      {:ok, parsed, _, _, _, _} ->
+        {:ok, build_string(parsed, variables, value_fetch_fn)}
+
+      {:error, message, _, _, _, _} ->
+        {:error, message}
+    end
+  end
+
+  defp build_string(parsed, variables, value_fetch_fn) do
+    Enum.reduce(parsed, "", fn element, acc ->
+      case element do
+        {:text, text} -> acc <> text
+        {:interpolation, {:identifier, identifier}} -> acc <> value_fetch_fn.(identifier, variables)
+      end
+    end)
+  end
+
+  def default_fetcher(identifier, variables) do
+    Map.get(variables, identifier, "")
+  end
+end

lib/pinchflat/downloading/output_path_builder.ex

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Downloading.OutputPathBuilder do
+  @moduledoc """
+  Builds yt-dlp-friendly output paths for downloaded media
+  """
+
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
+
+  @doc """
+  Builds the actual final filepath from a given template. Optionally, you can pass in
+  a map of additional options to be used in the template.
+
+  Translates liquid-style templates into yt-dlp-style templates,
+  leaving yt-dlp syntax intact.
+  """
+  def build(template_string, additional_template_options \\ %{}) do
+    combined_options = Map.merge(custom_yt_dlp_option_map(), additional_template_options)
+
+    TemplateParser.parse(template_string, combined_options, &identifier_fn/2)
+  end
+
+  # The `nil` case simply wraps the identifier in yt-dlp-style syntax. This assumes that
+  # the identifier is a valid yt-dlp option. The upside is that this gives the user
+  # access to ALL single-word yt-dlp options in the (imo) more friendly/forgiving liquid-style syntax.
+  #
+  # For all "custom" variables, we use the `Map.get/3` function to look up the value in the provided.
+  # See `custom_yt_dlp_option_map` for a list of those.
+  defp identifier_fn(identifier, variables) do
+    case Map.get(variables, identifier) do
+      nil -> "%(#{identifier})S"
+      value -> value
+    end
+  end
+
+  # This isn't the only source for custom options, since they can be passed in my the caller.
+  # `download_option_builder` is the most likely place for other custom options to be added,
+  # but if in doubt just search the codebase for `OutputPathBuilder.build`.
+  defp custom_yt_dlp_option_map do
+    %{
+      # Individual parts of the upload date
+      "upload_year" => "%(upload_date>%Y)S",
+      "upload_month" => "%(upload_date>%m)S",
+      "upload_day" => "%(upload_date>%d)S",
+      "upload_yyyy_mm_dd" => "%(upload_date>%Y)S-%(upload_date>%m)S-%(upload_date>%d)S"
+    }
+  end
+end