Bumped version

* Fixed issue with source details when first video is a premier

* Updated other occurance

.credo.exs

@@ -157,7 +157,16 @@
           {Credo.Check.Warning.UnusedRegexOperation, []},
           {Credo.Check.Warning.UnusedStringOperation, []},
           {Credo.Check.Warning.UnusedTupleOperation, []},
-          {Credo.Check.Warning.WrongTestFileExtension, []}
+          {Credo.Check.Warning.WrongTestFileExtension, []},
+
+          #
+          ## Naming Checks
+          #
+          {CredoNaming.Check.Consistency.ModuleFilename,
+           [
+             priority: :normal,
+             excluded_paths: [~r/test\/support/, ~r/priv/, ~r/lib\/pinchflat_web/, ~r/test\/pinchflat_web/]
+           ]}
         ],
         disabled: [
           #

.iex.exs

@@ -4,8 +4,7 @@ alias Pinchflat.Repo
 alias Pinchflat.Tasks.Task
 alias Pinchflat.Sources.Source
 alias Pinchflat.Media.MediaItem
-alias Pinchflat.Tasks.SourceTasks
+alias Pinchflat.Metadata.MediaMetadata
-alias Pinchflat.Media.MediaMetadata
 alias Pinchflat.Profiles.MediaProfile
 
 alias Pinchflat.Tasks
@@ -14,14 +13,14 @@ alias Pinchflat.Profiles
 alias Pinchflat.Sources
 alias Pinchflat.Settings
 
-alias Pinchflat.MediaClient.MediaDownloader
+alias Pinchflat.Downloading.MediaDownloader
-alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-alias Pinchflat.YtDlp.Backend.MediaCollection, as: YtDlpCollection
+alias Pinchflat.YtDlp.MediaCollection, as: YtDlpCollection
 
-alias Pinchflat.Api.YoutubeRss
+alias Pinchflat.FastIndexing.YoutubeRss
 alias Pinchflat.Metadata.MetadataFileHelpers
 
-alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+alias Pinchflat.SlowIndexing.FileFollowerServer
 
 defmodule IexHelpers do
   def playlist_url do

config/config.exs

@@ -12,7 +12,7 @@ config :pinchflat,
   generators: [timestamp_type: :utc_datetime],
   # Specifying backend data here makes mocking and local testing SUPER easy
   yt_dlp_executable: System.find_executable("yt-dlp"),
-  yt_dlp_runner: Pinchflat.YtDlp.Backend.CommandRunner,
+  yt_dlp_runner: Pinchflat.YtDlp.CommandRunner,
   media_directory: "/downloads",
   # The user may or may not store metadata for their needs, but the app will always store its copy
   metadata_directory: "/config/metadata",

lib/pinchflat/application.ex

@@ -11,8 +11,9 @@ defmodule Pinchflat.Application do
       PinchflatWeb.Telemetry,
       Pinchflat.Repo,
       # Must be before startup tasks
+      Pinchflat.Boot.PreJobStartupTasks,
       {Oban, Application.fetch_env!(:pinchflat, Oban)},
-      Pinchflat.StartupTasks,
+      Pinchflat.Boot.PostJobStartupTasks,
       {DNSCluster, query: Application.get_env(:pinchflat, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: Pinchflat.PubSub},
       # Start the Finch HTTP client for sending emails

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.DataBackfillWorker do
+defmodule Pinchflat.Boot.DataBackfillWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -28,7 +28,7 @@ defmodule Pinchflat.Workers.DataBackfillWorker do
   """
   def cancel_pending_backfill_jobs do
     Oban.Job
-    |> where(worker: "Pinchflat.Workers.DataBackfillWorker")
+    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
     |> Oban.cancel_all_jobs()
   end
 

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -1,6 +1,7 @@
-defmodule Pinchflat.StartupTasks do
+defmodule Pinchflat.Boot.PostJobStartupTasks do
   @moduledoc """
-  This module is responsible for running startup tasks on app boot.
+  This module is responsible for running startup tasks on app boot
+  AFTER the job runner has initiallized.
 
   It's a GenServer because that plays REALLY nicely with the existing
   Phoenix supervision tree.
@@ -11,8 +12,7 @@ defmodule Pinchflat.StartupTasks do
   import Ecto.Query, warn: false
 
   alias Pinchflat.Repo
-  alias Pinchflat.Settings
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.DataBackfillWorker
 
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
@@ -29,17 +29,11 @@ defmodule Pinchflat.StartupTasks do
   """
   @impl true
   def init(state) do
-    apply_default_settings()
     enqueue_backfill_worker()
 
     {:ok, state}
   end
 
-  defp apply_default_settings do
-    Settings.fetch!(:onboarding, true)
-    Settings.fetch!(:pro_enabled, false)
-  end
-
   defp enqueue_backfill_worker do
     DataBackfillWorker.cancel_pending_backfill_jobs()
 

lib/pinchflat/boot/pre_job_startup_tasks.ex

@@ -0,0 +1,73 @@
+defmodule Pinchflat.Boot.PreJobStartupTasks do
+  @moduledoc """
+  This module is responsible for running startup tasks on app boot
+  BEFORE the job runner has initiallized.
+
+  It's a GenServer because that plays REALLY nicely with the existing
+  Phoenix supervision tree.
+  """
+
+  # restart: :temporary means that this process will never be restarted (ie: will run once and then die)
+  use GenServer, restart: :temporary
+  import Ecto.Query, warn: false
+  require Logger
+
+  alias Pinchflat.Repo
+  alias Pinchflat.Settings
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, %{}, opts)
+  end
+
+  @doc """
+  Runs application startup tasks.
+
+  Any code defined here will run every time the application starts. You must
+  make sure that the code is idempotent and safe to run multiple times.
+
+  This is a good place to set up default settings, create initial records, stuff like that.
+  Should be fast - anything with the potential to be slow should be kicked off as a job instead.
+  """
+  @impl true
+  def init(state) do
+    apply_default_settings()
+    rename_old_job_workers()
+
+    {:ok, state}
+  end
+
+  defp apply_default_settings do
+    Settings.fetch!(:onboarding, true)
+    Settings.fetch!(:pro_enabled, false)
+  end
+
+  # As part of a large refactor, I ended up moving a bunch of workers around. This
+  # is a problem because the workers are stored in the database and the runner
+  # will try to run the OLD jobs. This is also why these tasks run before the job
+  # runner starts up.
+  #
+  # Can be removed after a few months (created: 2024-03-12)
+  defp rename_old_job_workers do
+    # [ [old_name, new_name], ...]
+    rename_map = [
+      ["Pinchflat.Workers.MediaIndexingWorker", "Pinchflat.FastIndexing.MediaIndexingWorker"],
+      ["Pinchflat.Workers.MediaDownloadWorker", "Pinchflat.Downloading.MediaDownloadWorker"],
+      ["Pinchflat.Workers.FilesystemDataWorker", "Pinchflat.Filesystem.FilesystemDataWorker"],
+      ["Pinchflat.Workers.FastIndexingWorker", "Pinchflat.FastIndexing.FastIndexingWorker"],
+      ["Pinchflat.Workers.MediaCollectionIndexingWorker", "Pinchflat.SlowIndexing.MediaCollectionIndexingWorker"],
+      ["Pinchflat.Workers.DataBackfillWorker", "Pinchflat.Boot.DataBackfillWorker"]
+    ]
+
+    jobs_renamed =
+      Enum.reduce(rename_map, 0, fn [old_name, new_name], acc ->
+        {count, _} =
+          Oban.Job
+          |> where(worker: ^old_name)
+          |> Repo.update_all(set: [worker: new_name])
+
+        acc + count
+      end)
+
+    Logger.info("Renamed #{jobs_renamed} old job workers")
+  end
+end

lib/pinchflat/downloading/download_option_builder.ex

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
+defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   @moduledoc """
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   @doc """
   Builds the options for yt-dlp to download media based on the given media's profile.

lib/pinchflat/downloading/downloading_helpers.ex

@@ -0,0 +1,50 @@
+defmodule Pinchflat.Downloading.DownloadingHelpers do
+  @moduledoc """
+  Methods for helping download media
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  require Logger
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  @doc """
+  Starts tasks for downloading media for any of a sources _pending_ media items.
+  Jobs are not enqueued if the source is set to not download media. This will return :ok.
+
+  NOTE: this starts a download for each media item that is pending,
+  not just the ones that were indexed in this job run. This should ensure
+  that any stragglers are caught if, for some reason, they weren't enqueued
+  or somehow got de-queued.
+
+  Returns :ok
+  """
+  def enqueue_pending_download_tasks(%Source{download_media: true} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(fn media_item ->
+      %{id: media_item.id}
+      |> MediaDownloadWorker.new()
+      |> Tasks.create_job_with_task(media_item)
+    end)
+  end
+
+  def enqueue_pending_download_tasks(%Source{download_media: false}) do
+    :ok
+  end
+
+  @doc """
+  Deletes ALL pending tasks for a source's media items.
+
+  Returns :ok
+  """
+  def dequeue_pending_download_tasks(%Source{} = source) do
+    source
+    |> Media.list_pending_media_items_for()
+    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
+  end
+end

lib/pinchflat/downloading/media_download_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaDownloadWorker do
+defmodule Pinchflat.Downloading.MediaDownloadWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -9,8 +9,8 @@ defmodule Pinchflat.Workers.MediaDownloadWorker do
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Tasks
-  alias Pinchflat.MediaClient.MediaDownloader
+  alias Pinchflat.Downloading.MediaDownloader
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   @impl Oban.Worker
   @doc """

lib/pinchflat/downloading/media_downloader.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.MediaClient.MediaDownloader do
+defmodule Pinchflat.Downloading.MediaDownloader do
   @moduledoc """
   This is the integration layer for actually downloading media.
   It takes into account the media profile's settings in order
@@ -9,8 +9,8 @@ defmodule Pinchflat.MediaClient.MediaDownloader do
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
-  alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
 

lib/pinchflat/downloading/output_path/base.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.RenderedString.Base do
+defmodule Pinchflat.Downloading.OutputPath.Base do
   @moduledoc """
   A base module for parsing rendered strings, designed as a macro to be used
   in other modules. See https://elixirforum.com/t/help-to-parse-a-template-with-nimbleparsec/47980
@@ -6,7 +6,7 @@ defmodule Pinchflat.RenderedString.Base do
   NOTE: if the needs here get any more complicated, look into using a Liquid
   template parser. No need to reinvent the wheel any more than I already have.
 
-  NOTE: this is effectively tested by the `Pinchflat.RenderedString.Parser`'s tests
+  NOTE: this is effectively tested by the `Pinchflat.Downloading.OutputPath.Parser`'s tests
   """
 
   defmacro __using__(_opts) do

lib/pinchflat/downloading/output_path/parser.ex

@@ -1,11 +1,11 @@
-defmodule Pinchflat.RenderedString.Parser do
+defmodule Pinchflat.Downloading.OutputPath.Parser do
   @moduledoc """
   Parses liquid-ish-style strings into a rendered string
 
   Used for turning filepath templates into real filepaths
   """
 
-  use Pinchflat.RenderedString.Base
+  use Pinchflat.Downloading.OutputPath.Base
 
   @doc """
   Parses a string into a rendered string, using the provided variables. Optionally

lib/pinchflat/downloading/output_path_builder.ex

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Profiles.OutputPathBuilder do
+defmodule Pinchflat.Downloading.OutputPathBuilder do
   @moduledoc """
   Builds yt-dlp-friendly output paths for downloaded media
   """
 
-  alias Pinchflat.RenderedString.Parser, as: TemplateParser
+  alias Pinchflat.Downloading.OutputPath.Parser, as: TemplateParser
 
   @doc """
   Builds the actual final filepath from a given template. Optionally, you can pass in

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -0,0 +1,94 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
+  @moduledoc """
+  Methods for performing fast indexing tasks and managing the fast indexing process.
+
+  Many of these methods are made to be kickoff or be consumed by workers.
+  """
+
+  alias Pinchflat.Media
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.FastIndexing.YoutubeRss
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
+  @doc """
+  Starts tasks for running a fast indexing task for a source's media
+  regardless of the source's fast_index state. It's assumed the
+  caller will check for fast_index.
+
+  This is used for running fast index tasks on update. On creation, the
+  fast index is enqueued after the slow index is complete.
+
+  Returns {:ok, %Task{}}.
+  """
+  def kickoff_fast_indexing_task(%Source{} = source) do
+    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+
+    %{id: source.id}
+    # Schedule this one immediately, but future ones will be on an interval
+    |> FastIndexingWorker.new()
+    |> Tasks.create_job_with_task(source)
+  end
+
+  @doc """
+  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
+  for any new media items. See comments in `MediaIndexingWorker` for more info on the
+  order of operations and how this fits into the indexing process.
+
+  Despite the similar name to `kickoff_fast_indexing_task`, this does work differently.
+  `kickoff_fast_indexing_task` starts a task that _calls_ this function whereas this
+  function starts individual indexing tasks for each new media item. I think it does
+  make sense grammatically, but I could see how that's confusing.
+
+  Returns :ok
+  """
+  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
+    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
+    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
+
+    Enum.each(new_media_ids, fn media_id ->
+      url = "https://www.youtube.com/watch?v=#{media_id}"
+
+      %{id: source.id, media_url: url}
+      |> MediaIndexingWorker.new()
+      |> Tasks.create_job_with_task(source)
+    end)
+  end
+
+  @doc """
+  Indexes a single media item for a source and enqueues a download job if the
+  media should be downloaded. This method creates the media item record so it's
+  the one-stop-shop for adding a media item (and possibly downloading it) just
+  by a URL and source.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
+    maybe_media_item = create_media_item_from_url(source, url)
+
+    case maybe_media_item do
+      {:ok, media_item} ->
+        if source.download_media && Media.pending_download?(media_item) do
+          %{id: media_item.id}
+          |> MediaDownloadWorker.new()
+          |> Tasks.create_job_with_task(media_item)
+        end
+
+        {:ok, media_item}
+
+      err ->
+        err
+    end
+  end
+
+  defp create_media_item_from_url(source, url) do
+    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
+
+    Media.create_media_item_from_backend_attrs(source, media_attrs)
+  end
+end

lib/pinchflat/fast_indexing/fast_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FastIndexingWorker do
+defmodule Pinchflat.FastIndexing.FastIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -10,7 +10,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -24,7 +24,7 @@ defmodule Pinchflat.Workers.FastIndexingWorker do
     source = Sources.get_source!(source_id)
 
     if source.fast_index do
-      SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+      FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
 
       reschedule_indexing(source)
     else

lib/pinchflat/fast_indexing/media_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorker do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -9,7 +9,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   require Logger
 
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -23,7 +23,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   and the media matches the profile's format preferences)
 
   Order of operations:
-    1. SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
+    1. FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed/1 (which is running
        in its own worker) periodically checks the YouTube RSS feed for new media
     2. If new media is found, it enqueues a MediaIndexingWorker (this module) for each new media
        item
@@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorker do
   def perform(%Oban.Job{args: %{"id" => source_id, "media_url" => media_url}}) do
     source = Sources.get_source!(source_id)
 
-    case MediaItemTasks.index_and_enqueue_download_for_media_item(source, media_url) do
+    case FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, media_url) do
       {:ok, media_item} ->
         Logger.debug("Indexed and enqueued download for url: #{media_url} (media item: #{media_item.id})")
 

lib/pinchflat/fast_indexing/youtube_rss.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Api.YoutubeRss do
+defmodule Pinchflat.FastIndexing.YoutubeRss do
   @moduledoc """
   Methods for interacting with YouTube RSS feeds
   """

lib/pinchflat/filesystem/filesystem_data_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.FilesystemDataWorker do
+defmodule Pinchflat.Filesystem.FilesystemDataWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -7,7 +7,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
     max_attempts: 1
 
   alias Pinchflat.Media
-  alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.Filesystem.FilesystemHelpers
 
   @impl Oban.Worker
   @doc """
@@ -18,7 +18,7 @@ defmodule Pinchflat.Workers.FilesystemDataWorker do
   def perform(%Oban.Job{args: %{"id" => media_item_id}}) do
     media_item = Media.get_media_item!(media_item_id)
 
-    MediaItemTasks.compute_and_save_media_filesize(media_item)
+    FilesystemHelpers.compute_and_save_media_filesize(media_item)
 
     # Don't retry on failure - if it didn't work immediately there's no
     # reason to believe it will work later.

lib/pinchflat/filesystem/filesystem_helpers.ex

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Utils.FilesystemUtils do
+defmodule Pinchflat.Filesystem.FilesystemHelpers do
   @moduledoc """
   Utility methods for working with the filesystem
   """
-
+  alias Pinchflat.Media
   alias Pinchflat.Utils.StringUtils
 
   @doc """
@@ -20,4 +20,19 @@ defmodule Pinchflat.Utils.FilesystemUtils do
 
     filepath
   end
+
+  @doc """
+  Fetches the file size of a media item and saves it to the database.
+
+  Returns {:ok, media_item} | {:error, any()}
+  """
+  def compute_and_save_media_filesize(media_item) do
+    case File.stat(media_item.media_filepath) do
+      {:ok, %{size: size}} ->
+        Media.update_media_item(media_item, %{media_size_bytes: size})
+
+      err ->
+        err
+    end
+  end
 end

lib/pinchflat/media/media.ex

@@ -9,7 +9,7 @@ defmodule Pinchflat.Media do
   alias Pinchflat.Tasks
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
 
   @doc """
   Returns the list of media_items.

lib/pinchflat/media/media_item.ex

@@ -8,8 +8,8 @@ defmodule Pinchflat.Media.MediaItem do
 
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Metadata.MediaMetadata
-  alias Pinchflat.Media.MediaItemSearchIndex
+  alias Pinchflat.Media.MediaItemsSearchIndex
 
   @allowed_fields [
     # these fields are captured on indexing (and again on download)
@@ -64,7 +64,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
-    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
+    has_one :media_items_search_index, MediaItemsSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 

lib/pinchflat/media/media_items_search_index.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaItemSearchIndex do
+defmodule Pinchflat.Media.MediaItemsSearchIndex do
   @moduledoc """
   The MediaItem fts5 search index. Not made to be directly interacted with,
   but I figured it'd be better to have it in-app so it's not a mystery.

lib/pinchflat/metadata/media_metadata.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Media.MediaMetadata do
+defmodule Pinchflat.Metadata.MediaMetadata do
   @moduledoc """
   The MediaMetadata schema.
 

lib/pinchflat/slow_indexing/file_follower_server.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServer do
+defmodule Pinchflat.SlowIndexing.FileFollowerServer do
   @moduledoc """
   A GenServer that watches a file for new lines and processes them as they come in.
   This is useful for tailing log files and other similar tasks. If there's no activity

lib/pinchflat/slow_indexing/media_collection_indexing_worker.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
   @moduledoc false
 
   use Oban.Worker,
@@ -10,8 +10,8 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
   @impl Oban.Worker
   @doc """
@@ -40,7 +40,7 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
        by the `download_media` field on the source as well as the profile's
        shorts/livestream behaviour. At this step we also attach a file reader
        to the `yt-dlp` output file so we can create media items as they come in
-       for a little speedup (see SourceTasks comments for more)
+       for a little speedup (see {Fast,Slow}IndexingHelpers comments for more)
     4. If this job is meant to reschedule (ie: has an index frequency > 0),
        it reschedules itself. If not, it runs once and does not reschedule
     5. If the source uses fast indexing, that job is kicked off as well. It
@@ -56,8 +56,6 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
   NOTE: Since indexing can take a LONG time, I should check what happens if an
   application restart occurs while a job is running. Will the job be lost?
 
-  IDEA: Should I use paging and do indexing in chunks? Is that even faster?
-
   Returns :ok | {:ok, %Task{}}
   """
   def perform(%Oban.Job{args: %{"id" => source_id}}) do
@@ -66,14 +64,14 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorker do
     case {source.index_frequency_minutes, source.last_indexed_at} do
       {index_freq, _} when index_freq > 0 ->
         # If the indexing is on a schedule simply run indexing and reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         maybe_enqueue_fast_indexing_task(source)
         reschedule_indexing(source)
 
       {_, nil} ->
         # If the source has never been indexed, index it once
         # even if it's not meant to reschedule
-        SourceTasks.index_and_enqueue_download_for_media_items(source)
+        SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
         :ok
 
       _ ->

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -1,9 +1,8 @@
-defmodule Pinchflat.Tasks.SourceTasks do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   @moduledoc """
-  Contains methods used by OR used to create/manage tasks for sources.
+  Methods for performing slow indexing tasks and managing the indexing process.
 
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
+  Many of these methods are made to be kickoff or be consumed by workers.
-  do is also defined here. Essentially, a one-stop-shop for source-related tasks/workers.
   """
 
   require Logger
@@ -12,16 +11,14 @@ defmodule Pinchflat.Tasks.SourceTasks do
   alias Pinchflat.Tasks
   alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Api.YoutubeRss
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.YtDlp.MediaCollection
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.FileFollowerServer
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Starts tasks for indexing a source's media regardless of the source's indexing
@@ -40,46 +37,6 @@ defmodule Pinchflat.Tasks.SourceTasks do
     |> Tasks.create_job_with_task(source)
   end
 
-  @doc """
-  Starts tasks for running a fast indexing task for a source's media
-  regardless of the source's fast_index state. It's assumed the
-  caller will check for fast_index.
-
-  This is used for running fast index tasks on update. On creation, the
-  fast index is enqueued after the slow index is complete.
-
-  Returns {:ok, %Task{}}.
-  """
-  def kickoff_fast_indexing_task(%Source{} = source) do
-    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-
-    %{id: source.id}
-    # Schedule this one immediately, but future ones will be on an interval
-    |> FastIndexingWorker.new()
-    |> Tasks.create_job_with_task(source)
-  end
-
-  @doc """
-  Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
-  for any new media items. See comments in `MediaIndexingWorker` for more info on the
-  order of operations and how this fits into the indexing process.
-
-  Returns :ok
-  """
-  def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
-    {:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
-    existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
-    new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
-
-    Enum.each(new_media_ids, fn media_id ->
-      url = "https://www.youtube.com/watch?v=#{media_id}"
-
-      %{id: source.id, media_url: url}
-      |> MediaIndexingWorker.new()
-      |> Tasks.create_job_with_task(source)
-    end)
-  end
-
   @doc """
   Given a media source, creates (indexes) the media by creating media_items for each
   media ID in the source. Afterward, kicks off a download task for each pending media
@@ -105,50 +62,21 @@ defmodule Pinchflat.Tasks.SourceTasks do
     # See the method definition below for more info on how file watchers work
     # (important reading if you're not familiar with it)
     {:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
-    result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
+
+    result =
+      Enum.map(media_attributes, fn media_attrs ->
+        case Media.create_media_item_from_backend_attrs(source, media_attrs) do
+          {:ok, media_item} -> media_item
+          {:error, changeset} -> changeset
+        end
+      end)
 
     Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
-    enqueue_pending_media_tasks(source)
+    DownloadingHelpers.enqueue_pending_download_tasks(source)
 
     result
   end
 
-  @doc """
-  Starts tasks for downloading media for any of a sources _pending_ media items.
-  Jobs are not enqueued if the source is set to not download media. This will return :ok.
-
-  NOTE: this starts a download for each media item that is pending,
-  not just the ones that were indexed in this job run. This should ensure
-  that any stragglers are caught if, for some reason, they weren't enqueued
-  or somehow got de-queued.
-
-  Returns :ok
-  """
-  def enqueue_pending_media_tasks(%Source{download_media: true} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(fn media_item ->
-      %{id: media_item.id}
-      |> MediaDownloadWorker.new()
-      |> Tasks.create_job_with_task(media_item)
-    end)
-  end
-
-  def enqueue_pending_media_tasks(%Source{download_media: false} = _source) do
-    :ok
-  end
-
-  @doc """
-  Deletes ALL pending tasks for a source's media items.
-
-  Returns :ok
-  """
-  def dequeue_pending_media_tasks(%Source{} = source) do
-    source
-    |> Media.list_pending_media_items_for()
-    |> Enum.each(&Tasks.delete_pending_tasks_for/1)
-  end
-
   # The file follower is a GenServer that watches a file for new lines and
   # processes them. This works well, but we have to be resilliant to partially-written
   # lines (ie: you should gracefully fail if you can't parse a line).
@@ -191,10 +119,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
   end
 
   defp create_media_item_and_enqueue_download(source, media_attrs) do
-    maybe_media_item = create_media_item_from_attributes(source, media_attrs)
+    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-
+      {:ok, %MediaItem{} = media_item} ->
-    case maybe_media_item do
-      %MediaItem{} = media_item ->
         if source.download_media && Media.pending_download?(media_item) do
           Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
 
@@ -203,15 +129,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
           |> Tasks.create_job_with_task(media_item)
         end
 
-      changeset ->
+      {:error, changeset} ->
         changeset
     end
   end
-
-  defp create_media_item_from_attributes(source, media_attrs) do
-    case Media.create_media_item_from_backend_attrs(source, media_attrs) do
-      {:ok, media_item} -> media_item
-      {:error, changeset} -> changeset
-    end
-  end
 end

lib/pinchflat/sources/sources.ex

@@ -9,9 +9,11 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Media
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Profiles.MediaProfile
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
 
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
@@ -67,7 +69,7 @@ defmodule Pinchflat.Sources do
   media if the indexing frequency has been changed.
 
   Existing indexing tasks will be cancelled if the indexing frequency has been
-  changed (logic in `SourceTasks.kickoff_indexing_task`)
+  changed (logic in `SlowIndexingHelpers.kickoff_indexing_task`)
 
   Runs an initial `change_source` check to ensure most of the source is valid
   before making an expensive API call. Runs it through `Repo.update` even
@@ -205,10 +207,10 @@ defmodule Pinchflat.Sources do
   defp maybe_handle_media_tasks(changeset, source) do
     case {changeset.data, changeset.changes} do
       {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
-        SourceTasks.enqueue_pending_media_tasks(source)
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
-        SourceTasks.dequeue_pending_media_tasks(source)
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
         :ok
@@ -221,7 +223,7 @@ defmodule Pinchflat.Sources do
     case changeset.data do
       # If the changeset is new (not persisted), attempt indexing no matter what
       %{__meta__: %{state: :built}} ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       # If the record has been persisted, only run indexing if the
       # indexing frequency has been changed and is now greater than 0
@@ -236,7 +238,7 @@ defmodule Pinchflat.Sources do
   defp maybe_update_slow_indexing_task(changeset, source) do
     case changeset.changes do
       %{index_frequency_minutes: mins} when mins > 0 ->
-        SourceTasks.kickoff_indexing_task(source)
+        SlowIndexingHelpers.kickoff_indexing_task(source)
 
       %{index_frequency_minutes: _} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
@@ -251,7 +253,7 @@ defmodule Pinchflat.Sources do
   defp maybe_update_fast_indexing_task(changeset, source) do
     case changeset.changes do
       %{fast_index: true} ->
-        SourceTasks.kickoff_fast_indexing_task(source)
+        FastIndexingHelpers.kickoff_fast_indexing_task(source)
 
       %{fast_index: false} ->
         Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")

lib/pinchflat/tasks/media_item_tasks.ex

@@ -1,61 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasks do
-  @moduledoc """
-  Contains methods used by OR used to create/manage tasks for media items.
-
-  Tasks/workers are meant to be thin wrappers so most of the actual work they
-  do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
-  """
-  alias Pinchflat.Media
-  alias Pinchflat.Tasks
-  alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
-
-  @doc """
-  Fetches the file size of a media item and saves it to the database.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def compute_and_save_media_filesize(media_item) do
-    case File.stat(media_item.media_filepath) do
-      {:ok, %{size: size}} ->
-        Media.update_media_item(media_item, %{media_size_bytes: size})
-
-      err ->
-        err
-    end
-  end
-
-  @doc """
-  Indexes a single media item for a source and enqueues a download job if the
-  media should be downloaded. This method creates the media item record so it's
-  the one-stop-shop for adding a media item (and possibly downloading it) just
-  by a URL and source.
-
-  Returns {:ok, media_item} | {:error, any()}
-  """
-  def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
-    maybe_media_item = create_media_item_from_url(source, url)
-
-    case maybe_media_item do
-      {:ok, media_item} ->
-        if source.download_media && Media.pending_download?(media_item) do
-          %{id: media_item.id}
-          |> MediaDownloadWorker.new()
-          |> Tasks.create_job_with_task(media_item)
-        end
-
-        {:ok, media_item}
-
-      err ->
-        err
-    end
-  end
-
-  defp create_media_item_from_url(source, url) do
-    {:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
-
-    Media.create_media_item_from_backend_attrs(source, media_attrs)
-  end
-end

lib/pinchflat/yt_dlp/backend_command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
+defmodule Pinchflat.YtDlp.BackendCommandRunner do
   @moduledoc """
   A behaviour for running CLI commands against a downloader backend (yt-dlp).
 

lib/pinchflat/yt_dlp/command_runner.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunner do
+defmodule Pinchflat.YtDlp.CommandRunner do
   @moduledoc """
   Runs yt-dlp commands using the `System.cmd/3` function
   """
@@ -6,8 +6,8 @@ defmodule Pinchflat.YtDlp.Backend.CommandRunner do
   require Logger
 
   alias Pinchflat.Utils.StringUtils
-  alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
+  alias Pinchflat.Filesystem.FilesystemHelpers, as: FSUtils
-  alias Pinchflat.YtDlp.Backend.BackendCommandRunner
+  alias Pinchflat.YtDlp.BackendCommandRunner
 
   @behaviour BackendCommandRunner
 

lib/pinchflat/yt_dlp/media.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.Media do
+defmodule Pinchflat.YtDlp.Media do
   @moduledoc """
   Contains utilities for working with singular pieces of media
   """

lib/pinchflat/yt_dlp/media_collection.ex

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollection do
+defmodule Pinchflat.YtDlp.MediaCollection do
   @moduledoc """
   Contains utilities for working with collections of
   media (aka: a source [ie: channels, playlists]).
@@ -7,8 +7,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
   require Logger
 
   alias Pinchflat.Utils.FunctionUtils
-  alias Pinchflat.Utils.FilesystemUtils
+  alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   @doc """
   Returns a list of maps representing the media in the collection.
@@ -22,9 +22,12 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
   """
   def get_media_attributes_for_collection(url, addl_opts \\ []) do
     runner = Application.get_env(:pinchflat, :yt_dlp_runner)
-    command_opts = [:simulate, :skip_download]
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the media details
+    command_opts = [:simulate, :skip_download, :ignore_no_formats_error]
     output_template = YtDlpMedia.indexing_output_template()
-    output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
     file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
 
     if file_listener_handler do
@@ -54,7 +57,10 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollection do
   Returns {:ok, map()} | {:error, any, ...}.
   """
   def get_source_details(source_url) do
-    opts = [:simulate, :skip_download, playlist_end: 1]
+    # `ignore_no_formats_error` is necessary because yt-dlp will error out if
+    # the first video has not released yet (ie: is a premier). We don't care about
+    # available formats since we're just getting the source details
+    opts = [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1]
     output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
 
     with {:ok, output} <- backend_runner().run(source_url, opts, output_template),

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.0-alpha.2",
+      version: "0.1.0-alpha.3",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -59,6 +59,7 @@ defmodule Pinchflat.MixProject do
       {:nimble_parsec, "~> 1.4"},
       {:mox, "~> 1.0", only: :test},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
+      {:credo_naming, "~> 2.1", only: [:dev, :test], runtime: false},
       {:ex_check, "~> 0.14.0", only: [:dev, :test], runtime: false},
       {:faker, "~> 0.17", only: :test},
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false}

mix.lock

@@ -6,6 +6,7 @@
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "credo": {:hex, :credo, "1.7.3", "05bb11eaf2f2b8db370ecaa6a6bda2ec49b2acd5e0418bc106b73b07128c0436", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "35ea675a094c934c22fb1dca3696f3c31f2728ae6ef5a53b5d648c11180a4535"},
+  "credo_naming": {:hex, :credo_naming, "2.1.0", "d44ad58890d4db552e141ce64756a74ac1573665af766d1ac64931aa90d47744", [:make, :mix], [{:credo, "~> 1.6", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "830e23b3fba972e2fccec49c0c089fe78c1e64bc16782a2682d78082351a2909"},
   "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.Workers.DataBackfillWorkerTest do
+defmodule Pinchflat.Boot.DataBackfillWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.DataBackfillWorker
+  alias Pinchflat.Boot.DataBackfillWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   describe "cancel_pending_backfill_jobs/0" do
     test "cancels all pending backfill jobs" do

test/pinchflat/boot/pre_job_startup_tasks_test.exs

@@ -0,0 +1,25 @@
+defmodule Pinchflat.Boot.PreJobStartupTasksTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Settings
+  alias Pinchflat.Settings.Setting
+  alias Pinchflat.Boot.PreJobStartupTasks
+
+  describe "apply_default_settings" do
+    setup do
+      Repo.delete_all(Setting)
+
+      :ok
+    end
+
+    test "sets default settings" do
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
+
+      PreJobStartupTasks.start_link()
+
+      assert Settings.get!(:onboarding)
+      refute Settings.get!(:pro_enabled)
+    end
+  end
+end

test/pinchflat/downloading/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -0,0 +1,75 @@
+defmodule Pinchflat.Downloading.DownloadingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  setup :verify_on_exit!
+
+  describe "enqueue_pending_download_tasks/1" do
+    test "it enqueues a job for each pending media item" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "it does not enqueue a job for media items with a filepath" do
+      source = source_fixture()
+      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it attaches a task to each enqueued job" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+
+    test "it does not create a job if the source is set to not download" do
+      source = source_fixture(download_media: false)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it does not attach tasks if the source is set to not download" do
+      source = source_fixture(download_media: false)
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+  end
+
+  describe "dequeue_pending_download_tasks/1" do
+    test "it deletes all pending tasks for a source's media items" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+
+      assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+  end
+end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
+defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   setup :verify_on_exit!
 

test/pinchflat/downloading/media_downloader_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.MediaClient.MediaDownloaderTest do
+defmodule Pinchflat.Downloading.MediaDownloaderTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.MediaClient.MediaDownloader
+  alias Pinchflat.Downloading.MediaDownloader
 
   setup :verify_on_exit!
 

test/pinchflat/downloading/output_path/parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.RenderedString.ParserTest do
+defmodule Pinchflat.Downloading.OutputPath.ParserTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.RenderedString.Parser
+  alias Pinchflat.Downloading.OutputPath.Parser
 
   describe "parse/3" do
     test "it returns the rendered string when the string is valid" do

test/pinchflat/downloading/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.Profiles.OutputPathBuilderTest do
+defmodule Pinchflat.Downloading.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -0,0 +1,142 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.TasksFixtures
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Tasks.Task
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+
+  setup :verify_on_exit!
+
+  @media_url "https://www.youtube.com/watch?v=test_1"
+
+  describe "kickoff_fast_indexing_task/1" do
+    test "it schedules a job" do
+      source = source_fixture()
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "it creates and attaches a task" do
+      source = source_fixture()
+
+      assert {:ok, %Task{} = task} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert task.source_id == source.id
+    end
+
+    test "it deletes any fast indexing tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+  end
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+  end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert Repo.aggregate(MediaItem, :count) == 1
+      assert mi_1.id == mi_2.id
+    end
+
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "creates a download task record", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
+    end
+
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
+
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
+      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
+      source = source_fixture(%{media_profile_id: profile.id})
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        output =
+          Phoenix.json_library().encode!(%{
+            id: "video2",
+            title: "Video 2",
+            webpage_url: "https://example.com/shorts/video2",
+            was_live: true,
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67,
+            upload_date: "20210101"
+          })
+
+        {:ok, output}
+      end)
+
+      assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+  end
+end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.Workers.FastIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
 
   setup :verify_on_exit!
 

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -6,8 +6,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   @media_url "https://www.youtube.com/watch?v=1234567890"
 

test/pinchflat/fast_indexing/youtube_rss_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Api.YoutubeRssTest do
+defmodule Pinchflat.FastIndexing.YoutubeRssTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
 
   setup :verify_on_exit!
 

test/pinchflat/filesystem/filesystem_data_worker_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Workers.FilesystemDataWorkerTest do
+defmodule Pinchflat.Filesystem.FilesystemDataWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   describe "perform/1" do
     test "Computes and stores the media file size" do

test/pinchflat/filesystem/filesystem_helpers_test.exs

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  describe "generate_metadata_tmpfile/1" do
+    test "creates a tmpfile and returns its path" do
+      res = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert String.ends_with?(res, ".json")
+      assert File.exists?(res)
+
+      File.rm!(res)
+    end
+  end
+
+  describe "compute_and_save_media_filesize/1" do
+    test "updates the media item with the file size" do
+      media_item = media_item_with_attachments()
+
+      refute media_item.media_size_bytes
+
+      assert {:ok, media_item} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+
+      assert Repo.reload!(media_item).media_size_bytes
+    end
+
+    test "returns the error if operation fails" do
+      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
+
+      assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+    end
+  end
+end

test/pinchflat/media_test.exs

@@ -11,7 +11,7 @@ defmodule Pinchflat.MediaTest do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.MetadataFileHelpers
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   setup :verify_on_exit!
 

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
+defmodule Pinchflat.Metadata.MetadataParserTest do
   use Pinchflat.DataCase
 
   alias Pinchflat.Metadata.MetadataParser, as: Parser

test/pinchflat/slow_indexing/file_follower_server_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServerTest do
+defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
   use ExUnit.Case, async: true
 
-  alias alias Pinchflat.Utils.FilesystemUtils
+  alias alias Pinchflat.Filesystem.FilesystemHelpers
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias Pinchflat.SlowIndexing.FileFollowerServer
 
   setup do
     {:ok, pid} = FileFollowerServer.start_link()
-    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
+    tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
 
     {:ok, %{pid: pid, tmpfile: tmpfile}}
   end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -8,9 +8,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Tasks.SourceTasksTest do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -9,12 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Tasks.Task
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -22,7 +22,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it schedules a job" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
@@ -30,7 +30,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it creates and attaches a task" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, %Task{} = task} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert task.source_id == source.id
     end
@@ -40,7 +40,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -50,7 +50,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -60,65 +60,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
   end
 
-  describe "kickoff_fast_indexing_task/1" do
-    test "it schedules a job" do
-      source = source_fixture()
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it creates and attaches a task" do
-      source = source_fixture()
-
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert task.source_id == source.id
-    end
-
-    test "it deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-  end
-
-  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
-    setup do
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
-      assert worker.args["id"] == source.id
-      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
-    end
-
-    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      media_item_fixture(source_id: source.id, media_id: "test_1")
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaIndexingWorker)
-    end
-  end
-
   describe "index_and_enqueue_download_for_media_items/1" do
     setup do
       stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
@@ -129,7 +76,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it creates a media_item record for each media ID returned", %{source: source} do
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
@@ -141,15 +88,15 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
     test "it attaches all media_items to the given source", %{source: source} do
       source_id = source.id
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
     end
 
     test "it won't duplicate media_items based on media_id and source", %{source: source} do
-      _first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      _first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      _duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      _duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       media_items = Repo.preload(source, :media_items).media_items
       assert Enum.count(media_items) == 3
@@ -158,8 +105,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it can duplicate media_ids for different sources", %{source: source} do
       other_source = source_fixture()
 
-      media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      media_items_other_source = SourceTasks.index_and_enqueue_download_for_media_items(other_source)
+      media_items_other_source = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(other_source)
 
       assert Enum.count(media_items) == 3
       assert Enum.count(media_items_other_source) == 3
@@ -169,8 +116,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it returns a list of media_items", %{source: source} do
-      first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
-      duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       first_ids = Enum.map(first_run, & &1.id)
       duplicate_ids = Enum.map(duplicate_run, & &1.id)
@@ -181,7 +128,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it updates the source's last_indexed_at field", %{source: source} do
       assert source.last_indexed_at == nil
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       source = Repo.reload!(source)
 
       assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
@@ -191,7 +138,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
     end
@@ -200,7 +147,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture(download_media: false)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
     end
@@ -226,7 +173,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       assert Repo.aggregate(MediaItem, :count, :id) == 0
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
     end
 
@@ -245,7 +192,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       refute_enqueued(worker: MediaDownloadWorker)
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -264,7 +211,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -297,7 +244,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -315,71 +262,9 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, source_attributes_return_fixture()}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
       assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
     end
   end
-
-  describe "enqueue_pending_media_tasks/1" do
-    test "it enqueues a job for each pending media item" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "it does not enqueue a job for media items with a filepath" do
-      source = source_fixture()
-      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it attaches a task to each enqueued job" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-
-    test "it does not create a job if the source is set to not download" do
-      source = source_fixture(download_media: false)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it does not attach tasks if the source is set to not download" do
-      source = source_fixture(download_media: false)
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-  end
-
-  describe "dequeue_pending_media_tasks/1" do
-    test "it deletes all pending tasks for a source's media items" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      SourceTasks.enqueue_pending_media_tasks(source)
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-
-      assert :ok = SourceTasks.dequeue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-  end
 end

test/pinchflat/sources_test.exs

@@ -7,12 +7,12 @@ defmodule Pinchflat.SourcesTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   @invalid_source_attrs %{name: nil, collection_id: nil}
 
@@ -331,7 +331,7 @@ defmodule Pinchflat.SourcesTest do
       source = source_fixture(download_media: true)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
       update_attrs = %{download_media: false}
-      SourceTasks.enqueue_pending_media_tasks(source)
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)

test/pinchflat/startup_tasks_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.StartupTasksTest do
-  use Pinchflat.DataCase
-
-  alias Pinchflat.Settings
-
-  # Since this runs on app boot (even in the test env),
-  # any actions in the `init/1` function will already have
-  # run. So we can only test the side effects of those actions,
-  # rather than the actions themselves.
-
-  describe "apply_default_settings" do
-    test "sets default settings" do
-      assert Settings.get!(:onboarding) == true
-    end
-  end
-end

test/pinchflat/tasks/media_items_tasks_test.exs

@@ -1,103 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasksTest do
-  use Pinchflat.DataCase
-
-  import Mox
-  import Pinchflat.MediaFixtures
-  import Pinchflat.SourcesFixtures
-  import Pinchflat.ProfilesFixtures
-
-  alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Tasks.MediaItemTasks
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  setup :verify_on_exit!
-
-  @media_url "https://www.youtube.com/watch?v=1234"
-
-  describe "compute_and_save_media_filesize/1" do
-    test "updates the media item with the file size" do
-      media_item = media_item_with_attachments()
-
-      refute media_item.media_size_bytes
-
-      assert {:ok, media_item} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-      assert Repo.reload!(media_item).media_size_bytes
-    end
-
-    test "returns the error if operation fails" do
-      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
-
-      assert {:error, _} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-    end
-  end
-
-  describe "index_and_enqueue_download_for_media_item/2" do
-    setup do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, media_attributes_return_fixture()}
-      end)
-
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "creates a new media item based on the URL", %{source: source} do
-      assert Repo.aggregate(MediaItem, :count) == 0
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert Repo.aggregate(MediaItem, :count) == 1
-    end
-
-    test "won't duplicate media_items based on media_id and source", %{source: source} do
-      assert {:ok, mi_1} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert {:ok, mi_2} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert Repo.aggregate(MediaItem, :count) == 1
-      assert mi_1.id == mi_2.id
-    end
-
-    test "enqueues a download job", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "creates a download task record", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
-    end
-
-    test "does not enqueue a download job if the source does not allow it" do
-      source = source_fixture(%{download_media: false})
-
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "does not enqueue a download job if the media item does not match the format rules" do
-      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
-      source = source_fixture(%{media_profile_id: profile.id})
-
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        output =
-          Phoenix.json_library().encode!(%{
-            id: "video2",
-            title: "Video 2",
-            webpage_url: "https://example.com/shorts/video2",
-            was_live: true,
-            description: "desc2",
-            aspect_ratio: 1.67,
-            duration: 345.67,
-            upload_date: "20210101"
-          })
-
-        {:ok, output}
-      end)
-
-      assert {:ok, _media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-  end
-end

test/pinchflat/utils/filesystem_utils_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtilsTest do
-  use ExUnit.Case, async: true
-
-  alias Pinchflat.Utils.FilesystemUtils
-
-  describe "generate_metadata_tmpfile/1" do
-    test "creates a tmpfile and returns its path" do
-      res = FilesystemUtils.generate_metadata_tmpfile(:json)
-
-      assert String.ends_with?(res, ".json")
-      assert File.exists?(res)
-
-      File.rm!(res)
-    end
-  end
-end

test/pinchflat/yt_dlp/command_runner_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
+defmodule Pinchflat.YtDlp.CommandRunnerTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
+  alias Pinchflat.YtDlp.CommandRunner, as: Runner
 
   @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

test/pinchflat/yt_dlp/media_collection_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
+defmodule Pinchflat.YtDlp.MediaCollectionTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Media
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.MediaCollection
 
   @channel_url "https://www.youtube.com/c/TheUselessTrials"
 
@@ -22,7 +22,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
 
     test "it passes the expected default args" do
       expect(YtDlpRunnerMock, :run, fn _url, opts, ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
+        assert opts == [:simulate, :skip_download, :ignore_no_formats_error]
         assert ot == Media.indexing_output_template()
 
         {:ok, ""}
@@ -87,7 +87,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
 
     test "it passes the expected args to the backend runner" do
       expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
-        assert opts == [:simulate, :skip_download, playlist_end: 1]
+        assert opts == [:simulate, :skip_download, :ignore_no_formats_error, playlist_end: 1]
         assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
 
         {:ok, "{}"}

test/pinchflat/yt_dlp/media_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.YtDlp.Backend.MediaTest do
+defmodule Pinchflat.YtDlp.MediaTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Media
 
   @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
 

test/test_helper.exs

@@ -1,4 +1,4 @@
-Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.Backend.BackendCommandRunner)
+Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.BackendCommandRunner)
 Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
 
 Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)