Refactor modules into contexts (#78)

* [WIP] break out a few contexts, start refactoring fast index modules

* [WIP] more contexts, this time around slow indexing and downloads

* [WIP] got all tests passing

* [WIP] Added moduledocs

* Built a genserver to rename old jobs on boot

* Added a module naming check; moved things around

* Fixed specs

lib/pinchflat/application.ex

@@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
       # Must be before startup tasks
+      Pinchflat.Boot.PreJobStartupTasks,
       {Oban, Application.fetch_env!(:pinchflat, Oban)},
-      Pinchflat.StartupTasks,
+      Pinchflat.Boot.PostJobStartupTasks,
       {DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: Pinchflat.PubSub},
       # Start the Finch HTTP client for sending emails

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.DataBackfillWorker do
+defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
   """
   def cancel_pending_backfill_jobs do
     Oban.Job
-    |> where(worker: "Pinchflat.Workers.DataBackfillWorker")
+    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
     |> Oban.cancel_all_jobs()
   end
 

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,6 +1,7 @@
-defmodule Pinchflat.StartupTasks do
+defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
-  This module is responsible for running startup tasks on app boot.
+  This module is responsible for running startup tasks on app boot
+  AFTER the job runner has initiallized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Settings
-  alias Pinchflat.Workers.DataBackfillWorker
+  alias Pinchflat.Boot.DataBackfillWorker
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
   """
   @impl true
   def init(state) do
-    apply_default_settings()
     enqueue_backfill_worker()
 
     {:ok, state}
   end
 
-  defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
-    Settings.fetch!(:pro_enabled, false)
-  end
-
   defp enqueue_backfill_worker do
     DataBackfillWorker.cancel_pending_backfill_jobs()
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -0,0 +1,73 @@
+defmodule Pinchflat.Boot.PreJobStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  BEFORE the job runner has initiallized.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Settings
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{}, opts)
+  end
+
+  @doc """
+  Runs application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(state) do
+    apply_default_settings()
+    rename_old_job_workers()
+
+    {:ok, state}
+  end
+
+  defp apply_default_settings do
+    Settings.fetch!(:onboarding, true)
+    Settings.fetch!(:pro_enabled, false)
+  end
+
+  # As part of a large refactor, I ended up moving a bunch of workers around. This
+  # is a problem because the workers are stored in the database and the runner
+  # will try to run the OLD jobs. This is also why these tasks run before the job
+  # runner starts up.
+  #
+  # Can be removed after a few months (created: 2024-03-12)
+  defp rename_old_job_workers do
+    # [ [old_name, new_name], ...]
+    rename_map = [
+      ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
+      ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
+      ["Pinchflat.Workers.FilesystemDataWorker", "Pinchflat.Filesystem.FilesystemDataWorker"],
+      ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
+      ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
+      ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
+    ]
+
+    jobs_renamed =
+      Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
+        {count, _} =
+          Oban.Job
+          |> where(worker: ^old_name)
+          |> Repo.update_all(set: [worker: new_name])
+
+        acc + count
+      end)
+
+    Logger.info("Renamed #{jobs_renamed} old job workers")
+  end
+end

lib/pinchflat/downloading/download_option_builder.ex

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   @doc """
   Builds the options for yt-dlp to download media based on the given media's profile.

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,50 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(fn media_item ->
+      %{id: media_item.id}
+      |> MediaDownloadWorker.new()
+      |> Tasks.create_job_with_task(media_item)
+    end)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaDownloadWorker do
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -9,8 +9,8 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Tasks
-  alias Pinchflat.MediaClient.MediaDownloader
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Downloading.MediaDownloader
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   @impl Oban.Worker
   @doc """

lib/pinchflat/downloading/media_downloader.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.MediaDownloader do
+defmodule Pinchflat.Downloading.MediaDownloader do
   @moduledoc """
   This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
@@ -9,8 +9,8 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+  alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 

lib/pinchflat/downloading/output_path/base.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.RenderedString.Base do
+defmodule Pinchflat.Downloading.OutputPath.Base do
   @moduledoc """
   A base module for parsing rendered strings, designed as a macro to be used
   in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
   NOTE: if the needs here get any more complicated, look into using a Liquid
   template parser. No need to reinvent the wheel any more than I already have.
 
-  NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
   """
 
   defmacro __using__(_opts) do

lib/pinchflat/downloading/output_path/parser.ex

@@ -1,11 +1,11 @@
-defmodule Pinchflat.RenderedString.Parser do
+defmodule Pinchflat.Downloading.OutputPath.Parser do
   @moduledoc """
   Parses liquid-ish-style strings into a rendered string
 
   Used for turning filepath templates into real filepaths
   """
 
-  use Pinchflat.RenderedString.Base
+  use Pinchflat.Downloading.OutputPath.Base
 
   @doc """
   Parses a string into a rendered string, using the provided variables. Optionally

lib/pinchflat/downloading/output_path_builder.ex

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Profiles.OutputPathBuilder do
+defmodule Pinchflat.Downloading.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
   """
 
-  alias Pinchflat.RenderedString.Parser, as: TemplateParser
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
 
   @doc """
   Builds the actual final filepath from a given template. Optionally, you can pass in

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,94 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  @moduledoc """
+  Methods for performing fast indexing tasks and managing the fast indexing process.
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Starts tasks for running a fast indexing task for a source's media
+  regardless of the source's fast_index state. It's assumed the
+  caller will check for fast_index.
+
+  This is used for running fast index tasks on update. On creation, the
+  fast index is enqueued after the slow index is complete.
+
+  Returns {:ok, %Task{}}.
+  """
+  def kickoff_fast_indexing_task(%Source{} = source) do
+    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+
+    %{id: source.id}
+    # Schedule this one immediately, but future ones will be on an interval
+    |> FastIndexingWorker.new()
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      %{id: source.id, media_url: url}
+      |> MediaIndexingWorker.new()
+      |> Tasks.create_job_with_task(source)
+    end)
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          %{id: media_item.id}
+          |> MediaDownloadWorker.new()
+          |> Tasks.create_job_with_task(media_item)
+        end
+
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -10,7 +10,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -24,7 +24,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorker do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -9,7 +9,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   require Logger
 
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -23,7 +23,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item
@@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
-    case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
       {:ok, media_item} ->
         Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
 

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/filesystem/filesystem_data_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FilesystemDataWorker do
+defmodule Pinchflat.Filesystem.FilesystemDataWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -7,7 +7,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
     max_attempts: 1
 
   alias Pinchflat.Media
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @impl Oban.Worker
   @doc """
@@ -18,7 +18,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item = Media.get_media_item!(media_item_id)
 
-    MediaItemTasks.compute_and_save_media_filesize(media_item)
+    FilesystemHelpers.compute_and_save_media_filesize(media_item)
 
     # Don't retry on failure - if it didn't work immediately there's no
     # reason to believe it will work later.

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Utils.FilesystemUtils do
+defmodule Pinchflat.Filesystem.FilesystemHelpers do
   @moduledoc """
   Utility methods for working with the filesystem
   """
-
+  alias Pinchflat.Media
   alias Pinchflat.Utils.StringUtils
 
   @doc """
@@ -20,4 +20,19 @@ defmodule Pinchflat.Utils.FilesystemUtils do
 
     filepath
   end
+
+  @doc """
+  Fetches the file size of a media item and saves it to the database.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      err ->
+        err
+    end
+  end
 end

lib/pinchflat/media/media.ex

@@ -9,7 +9,7 @@ defmodule Pinchflat.Media do
   alias Pinchflat.Tasks
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
 
   @doc """
   Returns the list of media_items.

lib/pinchflat/media/media_item.ex

@@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
-  alias Pinchflat.Media.MediaItemSearchIndex
+  alias Pinchflat.Metadata.MediaMetadata
+  alias Pinchflat.Media.MediaItemsSearchIndex
 
   @allowed_fields [
     # these fields are captured on indexing (and again on download)
@@ -64,7 +64,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
-    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
+    has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 

lib/pinchflat/media/media_items_search_index.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaItemSearchIndex do
+defmodule Pinchflat.Media.MediaItemsSearchIndex do
   @moduledoc """
   The MediaItem fts5 search index. Not made to be directly interacted with,
   but I figured it'd be better to have it in-app so it's not a mystery.

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
+defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   @moduledoc """
   A GenServer that watches a file for new lines and processes them as they come in.
   This is useful for tailing log files and other similar tasks. If there's no activity

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -10,8 +10,8 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
        by the `download_media` field on the source as well as the profile's
        shorts/livestream behaviour. At this step we also attach a file reader
        to the `yt-dlp` output file so we can create media items as they come in
-       for a little speedup (see SourceTasks comments for more)
+       for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
     4. If this job is meant to reschedule (ie: has an index frequency > 0),
        it reschedules itself. If not, it runs once and does not reschedule
     5. If the source uses fast indexing, that job is kicked off as well. It
@@ -56,8 +56,6 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   NOTE: Since indexing can take a LONG time, I should check what happens if an
   application restart occurs while a job is running. Will the job be lost?
 
-  IDEA: Should I use paging and do indexing in chunks? Is that even faster?
-
   Returns :ok | {:ok, %Task{}}
   """
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
@@ -66,14 +64,14 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         :ok
 
       _ ->

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -1,9 +1,8 @@
-defmodule Pinchflat.Tasks.SourceTasks do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   @moduledoc """
-  Contains methods used by OR used to create/manage tasks for sources.
+  Methods for performing slow indexing tasks and managing the indexing process.
 
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
+  Many of these methods are made to be kickoff or be consumed by workers.
   """
 
   require Logger
@@ -12,16 +11,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
-  alias Pinchflat.YtDlp.Backend.MediaCollection
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.SlowIndexing.FileFollowerServer
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Starts tasks for indexing a source's media regardless of the source's indexing
@@ -40,46 +37,6 @@ defmodule Pinchflat.Tasks.SourceTasks do
     |> Tasks.create_job_with_task(source)
   end
 
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
-  end
-
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
@@ -105,50 +62,21 @@ defmodule Pinchflat.Tasks.SourceTasks do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
-    result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
+
+    result =
+      Enum.map(media_attributes, fn media_attrs ->
+        case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+          {:ok, media_item} -> media_item
+          {:error, changeset} -> changeset
+        end
+      end)
 
     Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
-    enqueue_pending_media_tasks(source)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
 
     result
   end
 
-  @doc """
-  Starts tasks for downloading media for any of a sources _pending_ media items.
-  Jobs are not enqueued if the source is set to not download media. This will return :ok.
-
-  NOTE: this starts a download for each media item that is pending,
-  not just the ones that were indexed in this job run. This should ensure
-  that any stragglers are caught if, for some reason, they weren't enqueued
-  or somehow got de-queued.
-
-  Returns :ok
-  """
-  def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
-
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
-    :ok
-  end
-
-  @doc """
-  Deletes ALL pending tasks for a source's media items.
-
-  Returns :ok
-  """
-  def dequeue_pending_media_tasks(%Source{} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -191,10 +119,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
-    maybe_media_item = create_media_item_from_attributes(source, media_attrs)
-
-    case maybe_media_item do
-      %MediaItem{} = media_item ->
+    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+      {:ok, %MediaItem{} = media_item} ->
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
@@ -203,15 +129,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
           |> Tasks.create_job_with_task(media_item)
         end
 
-      changeset ->
+      {:error, changeset} ->
         changeset
     end
   end
-
-  defp create_media_item_from_attributes(source, media_attrs) do
-    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-      {:ok, media_item} -> media_item
-      {:error, changeset} -> changeset
-    end
-  end
 end

lib/pinchflat/sources/sources.ex

@@ -9,9 +9,11 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -67,7 +69,7 @@ defmodule Pinchflat.Sources do
   media if the indexing frequency has been changed.
 
   Existing indexing tasks will be cancelled if the indexing frequency has been
-  changed (logic in `SourceTasks.kickoff_indexing_task`)
+  changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
 
   Runs an initial `change_source` check to ensure most of the source is valid
   before making an expensive API call. Runs it through `Repo.update` even
@@ -205,10 +207,10 @@ defmodule Pinchflat.Sources do
   defp maybe_handle_media_tasks(changeset, source) do
     case {changeset.data, changeset.changes} do
       {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
-        SourceTasks.enqueue_pending_media_tasks(source)
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
-        SourceTasks.dequeue_pending_media_tasks(source)
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
         :ok
@@ -221,7 +223,7 @@ defmodule Pinchflat.Sources do
     case changeset.data do
       # If the changeset is new (not persisted), attempt indexing no matter what
       %{__meta__: %{state: :built}} ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
@@ -236,7 +238,7 @@ defmodule Pinchflat.Sources do
   defp maybe_update_slow_indexing_task(changeset, source) do
     case changeset.changes do
       %{index_frequency_minutes: mins} when mins > 0 ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@@ -251,7 +253,7 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        FastIndexingHelpers.kickoff_fast_indexing_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/media_item_tasks.ex

@@ -1,61 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasks do
-  @moduledoc """
-  Contains methods used by OR used to create/manage tasks for media items.
-
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
-  """
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Fetches the file size of a media item and saves it to the database.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def compute_and_save_media_filesize(media_item) do
-    case File.stat(media_item.media_filepath) do
-      {:ok, %{size: size}} ->
-        Media.update_media_item(media_item, %{media_size_bytes: size})
-
-      err ->
-        err
-    end
-  end
-
-  @doc """
-  Indexes a single media item for a source and enqueues a download job if the
-  media should be downloaded. This method creates the media item record so it's
-  the one-stop-shop for adding a media item (and possibly downloading it) just
-  by a URL and source.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
-    maybe_media_item = create_media_item_from_url(source, url)
-
-    case maybe_media_item do
-      {:ok, media_item} ->
-        if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
-        end
-
-        {:ok, media_item}
-
-      err ->
-        err
-    end
-  end
-
-  defp create_media_item_from_url(source, url) do
-    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
-
-    Media.create_media_item_from_backend_attrs(source, media_attrs)
-  end
-end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.BackendCommandRunner do
   @moduledoc """
   A behaviour for running CLI commands against a downloader backend (yt-dlp).
 

lib/pinchflat/yt_dlp/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunner do
+defmodule Pinchflat.YtDlp.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   require Logger
 
   alias Pinchflat.Utils.StringUtils
-  alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
-  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
+  alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
+  alias Pinchflat.YtDlp.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 

lib/pinchflat/yt_dlp/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.Media do
+defmodule Pinchflat.YtDlp.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """

lib/pinchflat/yt_dlp/media_collection.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollection do
+defmodule Pinchflat.YtDlp.MediaCollection do
   @moduledoc """
   Contains utilities for working with collections of
   media (aka: a source [ie: channels, playlists]).
@@ -7,8 +7,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
   require Logger
 
   alias Pinchflat.Utils.FunctionUtils
-  alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Returns a list of maps representing the media in the collection.
@@ -24,7 +24,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
     runner = Application.get_env(:pinchflat, :yt_dlp_runner)
     command_opts = [:simulate, :skip_download]
     output_template = YtDlpMedia.indexing_output_template()
-    output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
     file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
 
     if file_listener_handler do