Refactor modules into contexts (#78)

* [WIP] break out a few contexts, start refactoring fast index modules

* [WIP] more contexts, this time around slow indexing and downloads

* [WIP] got all tests passing

* [WIP] Added moduledocs

* Built a genserver to rename old jobs on boot

* Added a module naming check; moved things around

* Fixed specs

test/pinchflat/boot/data_backfill_worker_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.Workers.DataBackfillWorkerTest do
+defmodule Pinchflat.Boot.DataBackfillWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.DataBackfillWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Boot.DataBackfillWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   describe "cancel_pending_backfill_jobs/0" do
     test "cancels all pending backfill jobs" do

test/pinchflat/boot/pre_job_startup_tasks_test.exs

@@ -0,0 +1,25 @@
+defmodule Pinchflat.Boot.PreJobStartupTasksTest do
+  use Pinchflat.DataCase
+
+  alias Pinchflat.Settings
+  alias Pinchflat.Settings.Setting
+  alias Pinchflat.Boot.PreJobStartupTasks
+
+  describe "apply_default_settings" do
+    setup do
+      Repo.delete_all(Setting)
+
+      :ok
+    end
+
+    test "sets default settings" do
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:onboarding) end
+      assert_raise Ecto.NoResultsError, fn -> Settings.get!(:pro_enabled) end
+
+      PreJobStartupTasks.start_link()
+
+      assert Settings.get!(:onboarding)
+      refute Settings.get!(:pro_enabled)
+    end
+  end
+end

test/pinchflat/downloading/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.Downloading.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -0,0 +1,75 @@
+defmodule Pinchflat.Downloading.DownloadingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.Downloading.MediaDownloadWorker
+
+  setup :verify_on_exit!
+
+  describe "enqueue_pending_download_tasks/1" do
+    test "it enqueues a job for each pending media item" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "it does not enqueue a job for media items with a filepath" do
+      source = source_fixture()
+      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it attaches a task to each enqueued job" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+
+    test "it does not create a job if the source is set to not download" do
+      source = source_fixture(download_media: false)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "it does not attach tasks if the source is set to not download" do
+      source = source_fixture(download_media: false)
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      assert :ok = DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+  end
+
+  describe "dequeue_pending_download_tasks/1" do
+    test "it deletes all pending tasks for a source's media items" do
+      source = source_fixture()
+      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+
+      assert :ok = DownloadingHelpers.dequeue_pending_download_tasks(source)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
+    end
+  end
+end

test/pinchflat/downloading/media_download_worker_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Workers.MediaDownloadWorkerTest do
+defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.MediaFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   setup :verify_on_exit!
 

test/pinchflat/downloading/media_downloader_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.MediaClient.MediaDownloaderTest do
+defmodule Pinchflat.Downloading.MediaDownloaderTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.MediaClient.MediaDownloader
+  alias Pinchflat.Downloading.MediaDownloader
 
   setup :verify_on_exit!
 

test/pinchflat/downloading/output_path/parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.RenderedString.ParserTest do
+defmodule Pinchflat.Downloading.OutputPath.ParserTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.RenderedString.Parser
+  alias Pinchflat.Downloading.OutputPath.Parser
 
   describe "parse/3" do
     test "it returns the rendered string when the string is valid" do

test/pinchflat/downloading/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.Profiles.OutputPathBuilderTest do
+defmodule Pinchflat.Downloading.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.OutputPathBuilder
+  alias Pinchflat.Downloading.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs

@@ -0,0 +1,142 @@
+defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.TasksFixtures
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Tasks.Task
+  alias Pinchflat.Media.MediaItem
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+
+  setup :verify_on_exit!
+
+  @media_url "https://www.youtube.com/watch?v=test_1"
+
+  describe "kickoff_fast_indexing_task/1" do
+    test "it schedules a job" do
+      source = source_fixture()
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "it creates and attaches a task" do
+      source = source_fixture()
+
+      assert {:ok, %Task{} = task} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert task.source_id == source.id
+    end
+
+    test "it deletes any fast indexing tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = FastIndexingHelpers.kickoff_fast_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+  end
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = FastIndexingHelpers.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
+  end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, mi_1} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:ok, mi_2} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert Repo.aggregate(MediaItem, :count) == 1
+      assert mi_1.id == mi_2.id
+    end
+
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "creates a download task record", %{source: source} do
+      assert {:ok, media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
+    end
+
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
+
+      assert {:ok, _} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
+      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
+      source = source_fixture(%{media_profile_id: profile.id})
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        output =
+          Phoenix.json_library().encode!(%{
+            id: "video2",
+            title: "Video 2",
+            webpage_url: "https://example.com/shorts/video2",
+            was_live: true,
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67,
+            upload_date: "20210101"
+          })
+
+        {:ok, output}
+      end)
+
+      assert {:ok, _media_item} = FastIndexingHelpers.index_and_enqueue_download_for_media_item(source, @media_url)
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+  end
+end

test/pinchflat/fast_indexing/fast_indexing_worker_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.Workers.FastIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.FastIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
 
   setup :verify_on_exit!
 

test/pinchflat/fast_indexing/media_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
+defmodule Pinchflat.FastIndexing.MediaIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -6,8 +6,8 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
 
   @media_url "https://www.youtube.com/watch?v=1234567890"
 

test/pinchflat/fast_indexing/youtube_rss_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Api.YoutubeRssTest do
+defmodule Pinchflat.FastIndexing.YoutubeRssTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Api.YoutubeRss
+  alias Pinchflat.FastIndexing.YoutubeRss
 
   setup :verify_on_exit!
 

test/pinchflat/filesystem/filesystem_data_worker_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.Workers.FilesystemDataWorkerTest do
+defmodule Pinchflat.Filesystem.FilesystemDataWorkerTest do
   use Pinchflat.DataCase
 
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.Workers.FilesystemDataWorker
+  alias Pinchflat.Filesystem.FilesystemDataWorker
 
   describe "perform/1" do
     test "Computes and stores the media file size" do

test/pinchflat/filesystem/filesystem_helpers_test.exs

@@ -0,0 +1,36 @@
+defmodule Pinchflat.Filesystem.FilesystemHelpersTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.Filesystem.FilesystemHelpers
+
+  describe "generate_metadata_tmpfile/1" do
+    test "creates a tmpfile and returns its path" do
+      res = FilesystemHelpers.generate_metadata_tmpfile(:json)
+
+      assert String.ends_with?(res, ".json")
+      assert File.exists?(res)
+
+      File.rm!(res)
+    end
+  end
+
+  describe "compute_and_save_media_filesize/1" do
+    test "updates the media item with the file size" do
+      media_item = media_item_with_attachments()
+
+      refute media_item.media_size_bytes
+
+      assert {:ok, media_item} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+
+      assert Repo.reload!(media_item).media_size_bytes
+    end
+
+    test "returns the error if operation fails" do
+      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
+
+      assert {:error, _} = FilesystemHelpers.compute_and_save_media_filesize(media_item)
+    end
+  end
+end

test/pinchflat/media_test.exs

@@ -11,7 +11,7 @@ defmodule Pinchflat.MediaTest do
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.MetadataFileHelpers
 
-  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
   setup :verify_on_exit!
 

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
+defmodule Pinchflat.Metadata.MetadataParserTest do
   use Pinchflat.DataCase
 
   alias Pinchflat.Metadata.MetadataParser, as: Parser

test/pinchflat/slow_indexing/file_follower_server_test.exs

@@ -1,12 +1,12 @@
-defmodule Pinchflat.Utils.FilesystemUtils.FileFollowerServerTest do
+defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
   use ExUnit.Case, async: true
 
-  alias alias Pinchflat.Utils.FilesystemUtils
-  alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
+  alias alias Pinchflat.Filesystem.FilesystemHelpers
+  alias Pinchflat.SlowIndexing.FileFollowerServer
 
   setup do
     {:ok, pid} = FileFollowerServer.start_link()
-    tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
+    tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
 
     {:ok, %{pid: pid, tmpfile: tmpfile}}
   end

test/pinchflat/slow_indexing/media_collection_indexing_worker_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
+defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -8,9 +8,9 @@ defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 

test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs

@@ -1,4 +1,4 @@
-defmodule Pinchflat.Tasks.SourceTasksTest do
+defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
   use Pinchflat.DataCase
 
   import Mox
@@ -9,12 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
   alias Pinchflat.Tasks
   alias Pinchflat.Tasks.Task
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -22,7 +22,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it schedules a job" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
@@ -30,7 +30,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it creates and attaches a task" do
       source = source_fixture(index_frequency_minutes: 1)
 
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, %Task{} = task} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert task.source_id == source.id
     end
@@ -40,7 +40,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -50,7 +50,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
@@ -60,65 +60,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
 
-      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+      assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
   end
 
-  describe "kickoff_fast_indexing_task/1" do
-    test "it schedules a job" do
-      source = source_fixture()
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it creates and attaches a task" do
-      source = source_fixture()
-
-      assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert task.source_id == source.id
-    end
-
-    test "it deletes any fast indexing tasks for the source" do
-      source = source_fixture()
-      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
-
-      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
-    end
-  end
-
-  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
-    setup do
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
-      assert worker.args["id"] == source.id
-      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
-    end
-
-    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
-      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
-      media_item_fixture(source_id: source.id, media_id: "test_1")
-
-      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
-
-      refute_enqueued(worker: MediaIndexingWorker)
-    end
-  end
-
   describe "index_and_enqueue_download_for_media_items/1" do
     setup do
       stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
@@ -129,7 +76,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it creates a media_item record for each media ID returned", %{source: source} do
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
@@ -141,15 +88,15 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
     test "it attaches all media_items to the given source", %{source: source} do
       source_id = source.id
-      assert media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert Enum.count(media_items) == 3
       assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
     end
 
     test "it won't duplicate media_items based on media_id and source", %{source: source} do
-      _first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
-      _duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      _first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+      _duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       media_items = Repo.preload(source, :media_items).media_items
       assert Enum.count(media_items) == 3
@@ -158,8 +105,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it can duplicate media_ids for different sources", %{source: source} do
       other_source = source_fixture()
 
-      media_items = SourceTasks.index_and_enqueue_download_for_media_items(source)
-      media_items_other_source = SourceTasks.index_and_enqueue_download_for_media_items(other_source)
+      media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+      media_items_other_source = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(other_source)
 
       assert Enum.count(media_items) == 3
       assert Enum.count(media_items_other_source) == 3
@@ -169,8 +116,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     end
 
     test "it returns a list of media_items", %{source: source} do
-      first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
-      duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
+      first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
+      duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       first_ids = Enum.map(first_run, & &1.id)
       duplicate_ids = Enum.map(duplicate_run, & &1.id)
@@ -181,7 +128,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
     test "it updates the source's last_indexed_at field", %{source: source} do
       assert source.last_indexed_at == nil
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       source = Repo.reload!(source)
 
       assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
@@ -191,7 +138,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture()
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
     end
@@ -200,7 +147,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       source = source_fixture(download_media: false)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
 
       assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
     end
@@ -226,7 +173,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       assert Repo.aggregate(MediaItem, :count, :id) == 0
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
     end
 
@@ -245,7 +192,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       end)
 
       refute_enqueued(worker: MediaDownloadWorker)
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -264,7 +211,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -297,7 +244,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, ""}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
@@ -315,71 +262,9 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
         {:ok, source_attributes_return_fixture()}
       end)
 
-      SourceTasks.index_and_enqueue_download_for_media_items(source)
+      SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
       assert Repo.aggregate(MediaItem, :count, :id) == 3
       assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
     end
   end
-
-  describe "enqueue_pending_media_tasks/1" do
-    test "it enqueues a job for each pending media item" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "it does not enqueue a job for media items with a filepath" do
-      source = source_fixture()
-      _media_item = media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4")
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it attaches a task to each enqueued job" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-
-    test "it does not create a job if the source is set to not download" do
-      source = source_fixture(download_media: false)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "it does not attach tasks if the source is set to not download" do
-      source = source_fixture(download_media: false)
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      assert :ok = SourceTasks.enqueue_pending_media_tasks(source)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-  end
-
-  describe "dequeue_pending_media_tasks/1" do
-    test "it deletes all pending tasks for a source's media items" do
-      source = source_fixture()
-      media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-
-      SourceTasks.enqueue_pending_media_tasks(source)
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-
-      assert :ok = SourceTasks.dequeue_pending_media_tasks(source)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-      assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
-    end
-  end
 end

test/pinchflat/sources_test.exs

@@ -7,12 +7,12 @@ defmodule Pinchflat.SourcesTest do
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Sources
-  alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.FastIndexingWorker
-  alias Pinchflat.Workers.MediaDownloadWorker
-  alias Pinchflat.Workers.MediaIndexingWorker
-  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+  alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
+  alias Pinchflat.Downloading.MediaDownloadWorker
+  alias Pinchflat.FastIndexing.MediaIndexingWorker
+  alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
 
   @invalid_source_attrs %{name: nil, collection_id: nil}
 
@@ -331,7 +331,7 @@ defmodule Pinchflat.SourcesTest do
       source = source_fixture(download_media: true)
       media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
       update_attrs = %{download_media: false}
-      SourceTasks.enqueue_pending_media_tasks(source)
+      DownloadingHelpers.enqueue_pending_download_tasks(source)
 
       assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)

test/pinchflat/startup_tasks_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.StartupTasksTest do
-  use Pinchflat.DataCase
-
-  alias Pinchflat.Settings
-
-  # Since this runs on app boot (even in the test env),
-  # any actions in the `init/1` function will already have
-  # run. So we can only test the side effects of those actions,
-  # rather than the actions themselves.
-
-  describe "apply_default_settings" do
-    test "sets default settings" do
-      assert Settings.get!(:onboarding) == true
-    end
-  end
-end

test/pinchflat/tasks/media_items_tasks_test.exs

@@ -1,103 +0,0 @@
-defmodule Pinchflat.Tasks.MediaItemTasksTest do
-  use Pinchflat.DataCase
-
-  import Mox
-  import Pinchflat.MediaFixtures
-  import Pinchflat.SourcesFixtures
-  import Pinchflat.ProfilesFixtures
-
-  alias Pinchflat.Tasks
-  alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Tasks.MediaItemTasks
-  alias Pinchflat.Workers.MediaDownloadWorker
-
-  setup :verify_on_exit!
-
-  @media_url "https://www.youtube.com/watch?v=1234"
-
-  describe "compute_and_save_media_filesize/1" do
-    test "updates the media item with the file size" do
-      media_item = media_item_with_attachments()
-
-      refute media_item.media_size_bytes
-
-      assert {:ok, media_item} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-
-      assert Repo.reload!(media_item).media_size_bytes
-    end
-
-    test "returns the error if operation fails" do
-      media_item = media_item_fixture(%{media_filepath: "/nonexistent/file.mkv"})
-
-      assert {:error, _} = MediaItemTasks.compute_and_save_media_filesize(media_item)
-    end
-  end
-
-  describe "index_and_enqueue_download_for_media_item/2" do
-    setup do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, media_attributes_return_fixture()}
-      end)
-
-      {:ok, [source: source_fixture()]}
-    end
-
-    test "creates a new media item based on the URL", %{source: source} do
-      assert Repo.aggregate(MediaItem, :count) == 0
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert Repo.aggregate(MediaItem, :count) == 1
-    end
-
-    test "won't duplicate media_items based on media_id and source", %{source: source} do
-      assert {:ok, mi_1} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      assert {:ok, mi_2} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert Repo.aggregate(MediaItem, :count) == 1
-      assert mi_1.id == mi_2.id
-    end
-
-    test "enqueues a download job", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
-    end
-
-    test "creates a download task record", %{source: source} do
-      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
-    end
-
-    test "does not enqueue a download job if the source does not allow it" do
-      source = source_fixture(%{download_media: false})
-
-      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-
-    test "does not enqueue a download job if the media item does not match the format rules" do
-      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
-      source = source_fixture(%{media_profile_id: profile.id})
-
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        output =
-          Phoenix.json_library().encode!(%{
-            id: "video2",
-            title: "Video 2",
-            webpage_url: "https://example.com/shorts/video2",
-            was_live: true,
-            description: "desc2",
-            aspect_ratio: 1.67,
-            duration: 345.67,
-            upload_date: "20210101"
-          })
-
-        {:ok, output}
-      end)
-
-      assert {:ok, _media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
-      refute_enqueued(worker: MediaDownloadWorker)
-    end
-  end
-end

test/pinchflat/utils/filesystem_utils_test.exs

@@ -1,16 +0,0 @@
-defmodule Pinchflat.Utils.FilesystemUtilsTest do
-  use ExUnit.Case, async: true
-
-  alias Pinchflat.Utils.FilesystemUtils
-
-  describe "generate_metadata_tmpfile/1" do
-    test "creates a tmpfile and returns its path" do
-      res = FilesystemUtils.generate_metadata_tmpfile(:json)
-
-      assert String.ends_with?(res, ".json")
-      assert File.exists?(res)
-
-      File.rm!(res)
-    end
-  end
-end

test/pinchflat/yt_dlp/command_runner_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
+defmodule Pinchflat.YtDlp.CommandRunnerTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
+  alias Pinchflat.YtDlp.CommandRunner, as: Runner
 
   @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

test/pinchflat/yt_dlp/media_collection_test.exs

@@ -1,10 +1,10 @@
-defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
+defmodule Pinchflat.YtDlp.MediaCollectionTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
-  alias Pinchflat.YtDlp.Backend.MediaCollection
+  alias Pinchflat.YtDlp.Media
+  alias Pinchflat.YtDlp.MediaCollection
 
   @channel_url "https://www.youtube.com/c/TheUselessTrials"
 

test/pinchflat/yt_dlp/media_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.YtDlp.Backend.MediaTest do
+defmodule Pinchflat.YtDlp.MediaTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Media
 
   @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
 

test/test_helper.exs

@@ -1,4 +1,4 @@
-Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.Backend.BackendCommandRunner)
+Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.BackendCommandRunner)
 Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
 
 Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)