Fast indexing (#58)

* Made method to getting singular media details; Renamed other related method

* Takes a fun and flirty digression to remove abstractions around yt-dlp since I'm 100% committed to using it exclusively

* Removed commented test code

* Lays the groundwork for fast indexing

* Added module for working with youtube RSS feed

* Added methods to kick off indexing workers from RSS response

* Improve short detection (#59)

* Made media attribute-related yt-dlp calls return a struct

* Added shorts attribute to media items

* Added ability to discern a short from yt-dlp response

* Updated search to use new shorts attribute

* Fast index UI (#63)

* Added fast_index field and adds it to source form

* Added fast indexing to source changeset operations

* Added fast indexing worker and updated other modules to start using it

* Handled fast index worker on source update

* Add support modals (#65)

* Added fast indexing upgrade modal

* Improved modal on smaller screens

* Updated links to work again

* Added donation modal

* Reverted source fast index to 15 minutes

* Removed unneeded HTML attributes from old alpine approach

test/pinchflat/api/youtube_rss_test.exs

@@ -0,0 +1,79 @@
+defmodule Pinchflat.Api.YoutubeRssTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Api.YoutubeRss
+
+  setup :verify_on_exit!
+
+  setup do
+    source = source_fixture()
+
+    {:ok, source: source}
+  end
+
+  describe "get_recent_media_ids_from_rss/1" do
+    test "calls the expected URL for channel sources" do
+      source = source_fixture(collection_type: :channel, collection_id: "channel_id")
+
+      expect(HTTPClientMock, :get, fn url ->
+        assert url =~ "https://www.youtube.com/feeds/videos.xml?channel_id=#{source.collection_id}"
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "calls the expected URL for playlist sources" do
+      source = source_fixture(collection_type: :playlist, collection_id: "playlist_id")
+
+      expect(HTTPClientMock, :get, fn url ->
+        assert url =~ "https://www.youtube.com/feeds/videos.xml?playlist_id=#{source.collection_id}"
+
+        {:ok, ""}
+      end)
+
+      assert {:ok, _} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "returns an error if the HTTP request fails", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:error, ""} end)
+
+      assert {:error, "Failed to fetch RSS feed"} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "returns the media IDs from the RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url ->
+        {:ok, "<yt:videoId>test_1</yt:videoId><yt:videoId>test_2</yt:videoId>"}
+      end)
+
+      assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "strips whitespace from media IDs", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url ->
+        {:ok, "<yt:videoId> test_1 </yt:videoId><yt:videoId> test_2 </yt:videoId>"}
+      end)
+
+      assert {:ok, ["test_1", "test_2"]} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "removes empty media IDs", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url ->
+        {:ok, "<yt:videoId>test_1</yt:videoId><yt:videoId></yt:videoId>"}
+      end)
+
+      assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+
+    test "removes duplicate media IDs", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url ->
+        {:ok, "<yt:videoId>test_1</yt:videoId><yt:videoId>test_1</yt:videoId>"}
+      end)
+
+      assert {:ok, ["test_1"]} = YoutubeRss.get_recent_media_ids_from_rss(source)
+    end
+  end
+end

test/pinchflat/media_client/backends/yt_dlp/media_test.exs

@@ -1,56 +0,0 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaTest do
-  use Pinchflat.DataCase
-  import Mox
-
-  alias Pinchflat.MediaClient.Backends.YtDlp.Media
-
-  @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
-
-  setup :verify_on_exit!
-
-  # expect(YtDlpRunnerMock, :run, fn _url, [_, _, json_output_path | _] ->
-  # copy_metadata(json_output_path)
-
-  #   {:ok, ""}
-  # end)
-
-  describe "download/2" do
-    test "it calls the backend runner with the expected arguments" do
-      expect(YtDlpRunnerMock, :run, fn @media_url, opts, ot ->
-        assert [:no_simulate] = opts
-        assert "after_move:%()j" = ot
-
-        {:ok, render_metadata(:media_metadata)}
-      end)
-
-      assert {:ok, _} = Media.download(@media_url)
-    end
-
-    test "it passes along additional options" do
-      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
-        assert [:no_simulate, :custom_arg] = opts
-
-        {:ok, "{}"}
-      end)
-
-      assert {:ok, _} = Media.download(@media_url, [:custom_arg])
-    end
-
-    test "it parses and returns the generated file as JSON" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        {:ok, render_metadata(:media_metadata)}
-      end)
-
-      assert {:ok, %{"title" => "Trying to Wheelie Without the Rear Brake"}} =
-               Media.download(@media_url)
-    end
-
-    test "it returns errors" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opt, _ot ->
-        {:error, "something"}
-      end)
-
-      assert {:error, "something"} = Media.download(@media_url)
-    end
-  end
-end

test/pinchflat/media_client/source_details_test.exs

@@ -1,112 +0,0 @@
-defmodule Pinchflat.MediaClient.SourceDetailsTest do
-  use Pinchflat.DataCase
-  import Mox
-  import Pinchflat.ProfilesFixtures
-  import Pinchflat.SourcesFixtures
-
-  alias Pinchflat.MediaClient.SourceDetails
-
-  @channel_url "https://www.youtube.com/c/TheUselessTrials"
-
-  setup :verify_on_exit!
-
-  describe "get_source_details/2" do
-    test "it passes the expected arguments to the backend" do
-      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
-        assert opts == [:simulate, :skip_download, playlist_end: 1]
-        assert ot == "%(.{channel,channel_id,playlist_id,playlist_title})j"
-
-        {:ok, "{}"}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
-    end
-
-    test "it returns a map composed of the returned data" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
-        Phoenix.json_library().encode(%{
-          channel: "TheUselessTrials",
-          channel_id: "UCQH2",
-          playlist_id: "PLQH2",
-          playlist_title: "TheUselessTrials - Videos"
-        })
-      end)
-
-      assert {:ok, res} = SourceDetails.get_source_details(@channel_url)
-
-      assert %{
-               channel_id: "UCQH2",
-               channel_name: "TheUselessTrials",
-               playlist_id: "PLQH2",
-               playlist_name: "TheUselessTrials - Videos"
-             } = res
-    end
-  end
-
-  describe "get_media_attributes/2 when passed a string" do
-    test "it passes the expected arguments to the backend" do
-      expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
-        assert ot == "%(.{id,title,was_live,original_url,description})j"
-
-        {:ok, ""}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_media_attributes(@channel_url)
-    end
-
-    test "it returns a list of maps" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes(@channel_url)
-    end
-  end
-
-  describe "get_media_attributes/2 when passed a Source record" do
-    test "it calls the backend with the source's collection ID" do
-      source = source_fixture()
-
-      expect(YtDlpRunnerMock, :run, fn url, _opts, _ot, _addl_opts ->
-        assert source.collection_id == url
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      assert {:ok, _} = SourceDetails.get_media_attributes(source)
-    end
-
-    test "it builds options based on the source's media profile" do
-      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl_opts ->
-        assert opts == [:simulate, :skip_download]
-        {:ok, ""}
-      end)
-
-      media_profile =
-        media_profile_fixture(
-          shorts_behaviour: :include,
-          livestream_behaviour: :exclude
-        )
-
-      source = source_fixture(media_profile_id: media_profile.id)
-      assert {:ok, _} = SourceDetails.get_media_attributes(source)
-    end
-
-    test "lets you pass through an optional file_listener_handler" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      source = source_fixture()
-      current_self = self()
-
-      handler = fn filename ->
-        send(current_self, {:handler, filename})
-      end
-
-      assert {:ok, _} = SourceDetails.get_media_attributes(source, file_listener_handler: handler)
-
-      assert_receive {:handler, _}
-    end
-  end
-end

test/pinchflat/media_test.exs

@@ -9,7 +9,9 @@ defmodule Pinchflat.MediaTest do
 
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers
+  alias Pinchflat.Metadata.MetadataFileHelpers
+
+  alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
 
   setup :verify_on_exit!
 
@@ -45,6 +47,23 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "list_media_items_by_media_id_for/2" do
+    test "returns media_items for a given source and media_ids" do
+      source = source_fixture()
+      media_item = media_item_fixture(%{source_id: source.id, media_id: "123"})
+
+      assert Media.list_media_items_by_media_id_for(source, ["123"]) == [media_item]
+    end
+
+    test "does not return matching media_ids for a different source" do
+      source = source_fixture()
+      other_source = source_fixture()
+      _media_item = media_item_fixture(%{source_id: other_source.id, media_id: "123"})
+
+      assert Media.list_media_items_by_media_id_for(source, ["123"]) == []
+    end
+  end
+
   describe "list_pending_media_items_for/1" do
     test "it returns pending without a filepath for a given source" do
       source = source_fixture()
@@ -78,7 +97,7 @@ defmodule Pinchflat.MediaTest do
     test "returns shorts and normal media when shorts_behaviour is :include" do
       source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :include}).id})
       normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
-      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [normal, short]
     end
@@ -86,7 +105,7 @@ defmodule Pinchflat.MediaTest do
     test "returns only shorts when shorts_behaviour is :only" do
       source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :only}).id})
       _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
-      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [short]
     end
@@ -94,7 +113,7 @@ defmodule Pinchflat.MediaTest do
     test "returns only normal media when shorts_behaviour is :exclude" do
       source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :exclude}).id})
       normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
-      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [normal]
     end
@@ -139,7 +158,7 @@ defmodule Pinchflat.MediaTest do
 
       normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
       livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
-      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [normal, livestream, short]
     end
@@ -156,7 +175,7 @@ defmodule Pinchflat.MediaTest do
 
       _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
       livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
-      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [livestream, short]
     end
@@ -173,7 +192,7 @@ defmodule Pinchflat.MediaTest do
 
       normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
       _livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
-      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      _short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [normal]
     end
@@ -190,7 +209,7 @@ defmodule Pinchflat.MediaTest do
 
       _normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
       _livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
-      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      short = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       assert Media.list_pending_media_items_for(source) == [short]
     end
@@ -230,7 +249,7 @@ defmodule Pinchflat.MediaTest do
 
     test "returns false if the media hasn't been downloaded but the profile doesn't DL shorts" do
       source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :exclude}).id})
-      media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
+      media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, short_form_content: true})
 
       refute Media.pending_download?(media_item)
     end
@@ -368,6 +387,24 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "create_media_item_from_backend_attrs/2" do
+    test "creates a media item for a given source and attributes" do
+      source = source_fixture()
+
+      media_attrs =
+        media_attributes_return_fixture()
+        |> Phoenix.json_library().decode!()
+        |> YtDlpMedia.response_to_struct()
+
+      assert {:ok, %MediaItem{} = media_item} = Media.create_media_item_from_backend_attrs(source, media_attrs)
+      assert media_item.source_id == source.id
+      assert media_item.title == media_attrs.title
+      assert media_item.media_id == media_attrs.media_id
+      assert media_item.original_url == media_attrs.original_url
+      assert media_item.description == media_attrs.description
+    end
+  end
+
   describe "update_media_item/2" do
     test "updating with valid data updates the media_item" do
       media_item = media_item_fixture()

test/pinchflat/metadata/metadata_file_helpers_test.exs

@@ -1,9 +1,9 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpersTest do
+defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.MediaFixtures
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: Helpers
+  alias Pinchflat.Metadata.MetadataFileHelpers, as: Helpers
 
   setup do
     media_item = media_item_fixture()

test/pinchflat/metadata/metadata_parser_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
+defmodule Pinchflat.YtDlp.Backend.MediaParserTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: Parser
+  alias Pinchflat.Metadata.MetadataParser, as: Parser
 
   setup do
     json_filepath =
@@ -33,13 +33,31 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
     test "it extracts the title", %{metadata: metadata} do
       result = Parser.parse_for_media_item(metadata)
 
-      assert result.title == "Trying to Wheelie Without the Rear Brake"
+      assert result.title == metadata["title"]
     end
 
     test "it extracts the description", %{metadata: metadata} do
       result = Parser.parse_for_media_item(metadata)
 
-      assert is_binary(result.description)
+      assert result.description == metadata["description"]
+    end
+
+    test "it extracts the original_url", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.original_url == metadata["original_url"]
+    end
+
+    test "it extracts the media_id", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.media_id == metadata["id"]
+    end
+
+    test "it extracts the livestream flag", %{metadata: metadata} do
+      result = Parser.parse_for_media_item(metadata)
+
+      assert result.livestream == metadata["was_live"]
     end
   end
 

test/pinchflat/sources_test.exs

@@ -9,8 +9,10 @@ defmodule Pinchflat.SourcesTest do
   alias Pinchflat.Sources
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Sources.Source
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Workers.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Workers.MediaCollectionIndexingWorker
 
   @invalid_source_attrs %{name: nil, collection_id: nil}
 
@@ -166,7 +168,7 @@ defmodule Pinchflat.SourcesTest do
 
       assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
 
-      assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
     test "creation schedules an index test even if the index frequency is 0" do
@@ -180,7 +182,37 @@ defmodule Pinchflat.SourcesTest do
 
       assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
 
-      assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "fast_index forces the index frequency to be a default value" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: true,
+        index_frequency_minutes: 0
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+
+      assert source.index_frequency_minutes == Source.index_frequency_when_fast_indexing()
+    end
+
+    test "disabling fast index will not change the index frequency" do
+      expect(YtDlpRunnerMock, :run, &channel_mock/3)
+
+      valid_attrs = %{
+        media_profile_id: media_profile_fixture().id,
+        original_url: "https://www.youtube.com/channel/abc123",
+        fast_index: false,
+        index_frequency_minutes: 0
+      }
+
+      assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
+
+      assert source.index_frequency_minutes == 0
     end
   end
 
@@ -230,7 +262,7 @@ defmodule Pinchflat.SourcesTest do
 
       assert {:ok, %Source{} = source} = Sources.update_source(source, update_attrs)
       assert source.index_frequency_minutes == 123
-      assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
     test "updating the index frequency to 0 will not re-schedule the indexing task" do
@@ -239,18 +271,25 @@ defmodule Pinchflat.SourcesTest do
 
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
 
-      refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      refute_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
     test "updating the index frequency to 0 will delete any pending tasks" do
       source = source_fixture()
-      {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
-      task = task_fixture(source_id: source.id, job_id: job.id)
       update_attrs = %{index_frequency_minutes: 0}
 
+      {:ok, job_1} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task_1 = task_fixture(source_id: source.id, job_id: job_1.id)
+      {:ok, job_2} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
+      task_2 = task_fixture(source_id: source.id, job_id: job_2.id)
+      {:ok, job_3} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
+      task_3 = task_fixture(source_id: source.id, job_id: job_3.id)
+
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
 
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_1) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_2) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task_3) end
     end
 
     test "not updating the index frequency will not re-schedule the indexing task or delete tasks" do
@@ -261,7 +300,7 @@ defmodule Pinchflat.SourcesTest do
       assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
 
       assert Repo.reload!(task)
-      refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      refute_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
     test "enabling the download_media attribute will schedule a download task" do
@@ -285,6 +324,26 @@ defmodule Pinchflat.SourcesTest do
       refute_enqueued(worker: MediaDownloadWorker)
     end
 
+    test "enabling fast_index will schedule a fast indexing task" do
+      source = source_fixture(fast_index: false)
+      update_attrs = %{fast_index: true}
+
+      refute_enqueued(worker: FastIndexingWorker)
+      assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "disabling fast_index will cancel the fast indexing task" do
+      source = source_fixture(fast_index: true)
+      update_attrs = %{fast_index: false}
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task_fixture(source_id: source.id, job_id: job.id)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+      assert {:ok, %Source{}} = Sources.update_source(source, update_attrs)
+      refute_enqueued(worker: FastIndexingWorker)
+    end
+
     test "updates with invalid data returns error changeset" do
       source = source_fixture()
 
@@ -293,6 +352,24 @@ defmodule Pinchflat.SourcesTest do
 
       assert source == Sources.get_source!(source.id)
     end
+
+    test "fast_index forces the index frequency to be a default value" do
+      source = source_fixture(%{fast_index: true})
+      update_attrs = %{index_frequency_minutes: 0}
+
+      assert {:ok, source} = Sources.update_source(source, update_attrs)
+
+      assert source.index_frequency_minutes == Source.index_frequency_when_fast_indexing()
+    end
+
+    test "disabling fast index will not change the index frequency" do
+      source = source_fixture(%{fast_index: false})
+      update_attrs = %{index_frequency_minutes: 0}
+
+      assert {:ok, source} = Sources.update_source(source, update_attrs)
+
+      assert source.index_frequency_minutes == 0
+    end
   end
 
   describe "delete_source/2" do

test/pinchflat/tasks/media_items_tasks_test.exs

@@ -1,9 +1,19 @@
 defmodule Pinchflat.Tasks.MediaItemTasksTest do
   use Pinchflat.DataCase
 
+  import Mox
   import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
 
+  alias Pinchflat.Tasks
+  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Tasks.MediaItemTasks
+  alias Pinchflat.Workers.MediaDownloadWorker
+
+  setup :verify_on_exit!
+
+  @media_url "https://www.youtube.com/watch?v=1234"
 
   describe "compute_and_save_media_filesize/1" do
     test "updates the media item with the file size" do
@@ -22,4 +32,70 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do
       assert {:error, _} = MediaItemTasks.compute_and_save_media_filesize(media_item)
     end
   end
+
+  describe "index_and_enqueue_download_for_media_item/2" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "creates a new media item based on the URL", %{source: source} do
+      assert Repo.aggregate(MediaItem, :count) == 0
+      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "won't duplicate media_items based on media_id and source", %{source: source} do
+      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+      assert {:error, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert Repo.aggregate(MediaItem, :count) == 1
+    end
+
+    test "enqueues a download job", %{source: source} do
+      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
+    end
+
+    test "creates a download task record", %{source: source} do
+      assert {:ok, media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      assert [_] = Tasks.list_tasks_for(:media_item_id, media_item.id, "MediaDownloadWorker")
+    end
+
+    test "does not enqueue a download job if the source does not allow it" do
+      source = source_fixture(%{download_media: false})
+
+      assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+
+    test "does not enqueue a download job if the media item does not match the format rules" do
+      profile = media_profile_fixture(%{shorts_behaviour: :exclude})
+      source = source_fixture(%{media_profile_id: profile.id})
+
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        output =
+          Phoenix.json_library().encode!(%{
+            id: "video2",
+            title: "Video 2",
+            webpage_url: "https://example.com/shorts/video2",
+            was_live: true,
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67
+          })
+
+        {:ok, output}
+      end)
+
+      assert {:ok, _media_item} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
+      refute_enqueued(worker: MediaDownloadWorker)
+    end
+  end
 end

test/pinchflat/tasks/source_tasks_test.exs

@@ -11,8 +11,10 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.Tasks.SourceTasks
   alias Pinchflat.Media.MediaItem
-  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Workers.FastIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Workers.MediaIndexingWorker
+  alias Pinchflat.Workers.MediaCollectionIndexingWorker
 
   setup :verify_on_exit!
 
@@ -22,7 +24,7 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
       assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
 
-      assert_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
+      assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
     end
 
     test "it creates and attaches a task" do
@@ -33,7 +35,17 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
       assert task.source_id == source.id
     end
 
-    test "it deletes any pending tasks for the source" do
+    test "it deletes any pending media collection tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+
+    test "it deletes any pending media tasks for the source" do
       source = source_fixture()
       {:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
       task = task_fixture(source_id: source.id, job_id: job.id)
@@ -42,6 +54,69 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
 
       assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
     end
+
+    test "it deletes any fast indexing tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = SourceTasks.kickoff_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+  end
+
+  describe "kickoff_fast_indexing_task/1" do
+    test "it schedules a job" do
+      source = source_fixture()
+
+      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
+
+      assert_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "it creates and attaches a task" do
+      source = source_fixture()
+
+      assert {:ok, %Task{} = task} = SourceTasks.kickoff_fast_indexing_task(source)
+
+      assert task.source_id == source.id
+    end
+
+    test "it deletes any fast indexing tasks for the source" do
+      source = source_fixture()
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      assert {:ok, _} = SourceTasks.kickoff_fast_indexing_task(source)
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+  end
+
+  describe "kickoff_indexing_tasks_from_youtube_rss_feed/1" do
+    setup do
+      {:ok, [source: source_fixture()]}
+    end
+
+    test "enqueues a new worker for each new media_id in the source's RSS feed", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+
+      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      assert [worker] = all_enqueued(worker: MediaIndexingWorker)
+      assert worker.args["id"] == source.id
+      assert worker.args["media_url"] == "https://www.youtube.com/watch?v=test_1"
+    end
+
+    test "does not enqueue a new worker for the source's media IDs we already know about", %{source: source} do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, "<yt:videoId>test_1</yt:videoId>"} end)
+      media_item_fixture(source_id: source.id, media_id: "test_1")
+
+      assert :ok = SourceTasks.kickoff_indexing_tasks_from_youtube_rss_feed(source)
+
+      refute_enqueued(worker: MediaIndexingWorker)
+    end
   end
 
   describe "index_and_enqueue_download_for_media_items/1" do
@@ -203,9 +278,11 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
           Phoenix.json_library().encode!(%{
             id: "video2",
             title: "Video 2",
-            original_url: "https://example.com/shorts/video2",
+            webpage_url: "https://example.com/shorts/video2",
             was_live: true,
-            description: "desc2"
+            description: "desc2",
+            aspect_ratio: 1.67,
+            duration: 345.67
           })
 
         File.write(filepath, contents)

test/pinchflat/workers/fast_indexing_worker_test.exs

@@ -0,0 +1,46 @@
+defmodule Pinchflat.Workers.FastIndexingWorkerTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Workers.FastIndexingWorker
+
+  setup :verify_on_exit!
+
+  describe "perform/1" do
+    test "calls out to Youtube RSS if enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+
+      perform_job(FastIndexingWorker, %{"id" => source.id})
+    end
+
+    test "reschedules itself if fast indexing is enabled" do
+      expect(HTTPClientMock, :get, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: true)
+      perform_job(FastIndexingWorker, %{"id" => source.id})
+
+      assert_enqueued(
+        worker: FastIndexingWorker,
+        args: %{"id" => source.id},
+        scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
+      )
+    end
+
+    test "does not call out to Youtube RSS if disabled" do
+      expect(HTTPClientMock, :get, 0, fn _url -> {:ok, ""} end)
+      source = source_fixture(fast_index: false)
+
+      perform_job(FastIndexingWorker, %{"id" => source.id})
+    end
+
+    test "does not reschedule itself if fast indexing is disabled" do
+      source = source_fixture(fast_index: false)
+      perform_job(FastIndexingWorker, %{"id" => source.id})
+
+      refute_enqueued(worker: FastIndexingWorker, args: %{"id" => source.id})
+    end
+  end
+end

test/pinchflat/workers/media_collection_indexing_worker_test.exs

@@ -0,0 +1,162 @@
+defmodule Pinchflat.Workers.MediaCollectionIndexingWorkerTest do
+  use Pinchflat.DataCase
+
+  import Mox
+  import Pinchflat.TasksFixtures
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Tasks
+  alias Pinchflat.Sources.Source
+  alias Pinchflat.Workers.FastIndexingWorker
+  alias Pinchflat.Workers.MediaDownloadWorker
+  alias Pinchflat.Workers.MediaCollectionIndexingWorker
+
+  setup :verify_on_exit!
+
+  describe "perform/1" do
+    test "it indexes the source if it should be indexed" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+    end
+
+    test "it indexes the source no matter what if the source has never been indexed before" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 0, last_indexed_at: nil)
+
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+    end
+
+    test "it does not do any indexing if the source has been indexed and shouldn't be rescheduled" do
+      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: -1, last_indexed_at: DateTime.utc_now())
+
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+    end
+
+    test "it does not reschedule if the source shouldn't be indexed" do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: -1)
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      refute_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
+    end
+
+    test "it kicks off a download job for each pending media item" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        {:ok, source_attributes_return_fixture()}
+      end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      assert length(all_enqueued(worker: MediaDownloadWorker)) == 3
+    end
+
+    test "it starts a job for any pending media item even if it's from another run" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        {:ok, source_attributes_return_fixture()}
+      end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+      media_item_fixture(%{source_id: source.id, media_filepath: nil})
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      assert length(all_enqueued(worker: MediaDownloadWorker)) == 4
+    end
+
+    test "it does not kick off a job for media items that could not be saved" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
+        {:ok, source_attributes_return_fixture()}
+      end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+      media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      # Only 3 jobs should be enqueued, since the first video is a duplicate
+      assert length(all_enqueued(worker: MediaDownloadWorker))
+    end
+
+    test "it reschedules the job based on the index frequency" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      assert_enqueued(
+        worker: MediaCollectionIndexingWorker,
+        args: %{"id" => source.id},
+        scheduled_at: now_plus(source.index_frequency_minutes, :minutes)
+      )
+    end
+
+    test "it creates a task for the rescheduled job" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+      task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
+
+      assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
+        perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+      end)
+    end
+
+    test "it creates a future task for fast indexing if appropriate" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10, fast_index: true)
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      assert_enqueued(
+        worker: FastIndexingWorker,
+        args: %{"id" => source.id},
+        scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
+      )
+    end
+
+    test "it deletes existing fast indexing tasks if a new one is created" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10, fast_index: true)
+      {:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
+      task = task_fixture(source_id: source.id, job_id: job.id)
+
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
+    end
+
+    test "it does not create a task for fast indexing otherwise" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
+
+      source = source_fixture(index_frequency_minutes: 10, fast_index: false)
+      perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+
+      refute_enqueued(worker: FastIndexingWorker)
+    end
+
+    test "it creates the basic media_item records" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, source_attributes_return_fixture()} end)
+
+      source = source_fixture(index_frequency_minutes: 10)
+
+      media_item_fetcher = fn ->
+        source
+        |> Repo.preload(:media_items)
+        |> Map.get(:media_items)
+        |> Enum.map(fn media_item -> media_item.media_id end)
+      end
+
+      assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
+        perform_job(MediaCollectionIndexingWorker, %{id: source.id})
+      end)
+    end
+  end
+end

test/pinchflat/workers/media_indexing_worker_test.exs

@@ -5,121 +5,40 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
   import Pinchflat.MediaFixtures
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.Tasks
+  alias Pinchflat.Media.MediaItem
   alias Pinchflat.Workers.MediaIndexingWorker
   alias Pinchflat.Workers.MediaDownloadWorker
 
+  @media_url "https://www.youtube.com/watch?v=1234567890"
+
   setup :verify_on_exit!
 
+  setup do
+    source = source_fixture()
+
+    {:ok, source: source}
+  end
+
   describe "perform/1" do
-    test "it indexes the source if it should be indexed" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: 10)
-
-      perform_job(MediaIndexingWorker, %{id: source.id})
-    end
-
-    test "it indexes the source no matter what if the source has never been indexed before" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: 0, last_indexed_at: nil)
-
-      perform_job(MediaIndexingWorker, %{id: source.id})
-    end
-
-    test "it does not do any indexing if the source has been indexed and shouldn't be rescheduled" do
-      expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: -1, last_indexed_at: DateTime.utc_now())
-
-      perform_job(MediaIndexingWorker, %{id: source.id})
-    end
-
-    test "it does not reschedule if the source shouldn't be indexed" do
-      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: -1)
-      perform_job(MediaIndexingWorker, %{id: source.id})
-
-      refute_enqueued(worker: MediaIndexingWorker, args: %{"id" => source.id})
-    end
-
-    test "it kicks off a download job for each pending media item" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
+    test "indexes the media item and saves it to the database", %{source: source} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
       end)
 
-      source = source_fixture(index_frequency_minutes: 10)
-      perform_job(MediaIndexingWorker, %{id: source.id})
+      before = Repo.aggregate(MediaItem, :count, :id)
+      perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
 
-      assert length(all_enqueued(worker: MediaDownloadWorker)) == 3
+      assert Repo.aggregate(MediaItem, :count, :id) == before + 1
     end
 
-    test "it starts a job for any pending media item even if it's from another run" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
+    test "enqueues a download job for the media item", %{source: source} do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
       end)
 
-      source = source_fixture(index_frequency_minutes: 10)
-      media_item_fixture(%{source_id: source.id, media_filepath: nil})
-      perform_job(MediaIndexingWorker, %{id: source.id})
+      perform_job(MediaIndexingWorker, %{id: source.id, media_url: @media_url})
 
-      assert length(all_enqueued(worker: MediaDownloadWorker)) == 4
-    end
-
-    test "it does not kick off a job for media items that could not be saved" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
-        {:ok, source_attributes_return_fixture()}
-      end)
-
-      source = source_fixture(index_frequency_minutes: 10)
-      media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
-      perform_job(MediaIndexingWorker, %{id: source.id})
-
-      # Only 3 jobs should be enqueued, since the first video is a duplicate
-      assert length(all_enqueued(worker: MediaDownloadWorker))
-    end
-
-    test "it reschedules the job based on the index frequency" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: 10)
-      perform_job(MediaIndexingWorker, %{id: source.id})
-
-      assert_enqueued(
-        worker: MediaIndexingWorker,
-        args: %{"id" => source.id},
-        scheduled_at: now_plus(source.index_frequency_minutes, :minutes)
-      )
-    end
-
-    test "it creates a task for the rescheduled job" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
-
-      source = source_fixture(index_frequency_minutes: 10)
-      task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
-
-      assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
-        perform_job(MediaIndexingWorker, %{id: source.id})
-      end)
-    end
-
-    test "it creates the basic media_item records" do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, source_attributes_return_fixture()} end)
-
-      source = source_fixture(index_frequency_minutes: 10)
-
-      media_item_fetcher = fn ->
-        source
-        |> Repo.preload(:media_items)
-        |> Map.get(:media_items)
-        |> Enum.map(fn media_item -> media_item.media_id end)
-      end
-
-      assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
-        perform_job(MediaIndexingWorker, %{id: source.id})
-      end)
+      assert [_] = all_enqueued(worker: MediaDownloadWorker)
     end
   end
 end

test/pinchflat/yt_dlp/backend/command_runner_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunnerTest do
+defmodule Pinchflat.YtDlp.Backend.CommandRunnerTest do
   use ExUnit.Case, async: true
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, as: Runner
+  alias Pinchflat.YtDlp.Backend.CommandRunner, as: Runner
 
   @original_executable Application.compile_env(:pinchflat, :yt_dlp_executable)
   @media_url "https://www.youtube.com/watch?v=-LHXuyzpex0"

test/pinchflat/yt_dlp/backend/media_collection_test.exs

@@ -1,39 +1,40 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do
+defmodule Pinchflat.YtDlp.Backend.MediaCollectionTest do
   use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 
-  alias Pinchflat.MediaClient.Backends.YtDlp.MediaCollection
+  alias Pinchflat.YtDlp.Backend.Media
+  alias Pinchflat.YtDlp.Backend.MediaCollection
 
   @channel_url "https://www.youtube.com/c/TheUselessTrials"
 
   setup :verify_on_exit!
 
-  describe "get_media_attributes/2" do
+  describe "get_media_attributes_for_collection/2" do
     test "returns a list of video attributes with no blank elements" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
         {:ok, source_attributes_return_fixture() <> "\n\n"}
       end)
 
-      assert {:ok, [%{"id" => "video1"}, %{"id" => "video2"}, %{"id" => "video3"}]} =
-               MediaCollection.get_media_attributes(@channel_url)
+      assert {:ok, [%Media{media_id: "video1"}, %Media{media_id: "video2"}, %Media{media_id: "video3"}]} =
+               MediaCollection.get_media_attributes_for_collection(@channel_url)
     end
 
     test "it passes the expected default args" do
       expect(YtDlpRunnerMock, :run, fn _url, opts, ot, _addl_opts ->
         assert opts == [:simulate, :skip_download]
-        assert ot == "%(.{id,title,was_live,original_url,description})j"
+        assert ot == Media.indexing_output_template()
 
         {:ok, ""}
       end)
 
-      assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url)
+      assert {:ok, _} = MediaCollection.get_media_attributes_for_collection(@channel_url)
     end
 
     test "returns the error straight through when the command fails" do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:error, "Big issue", 1} end)
 
-      assert {:error, "Big issue", 1} = MediaCollection.get_media_attributes(@channel_url)
+      assert {:error, "Big issue", 1} = MediaCollection.get_media_attributes_for_collection(@channel_url)
     end
 
     test "passes the explict tmpfile path to runner" do
@@ -44,7 +45,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do
         {:ok, ""}
       end)
 
-      assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url)
+      assert {:ok, _} = MediaCollection.get_media_attributes_for_collection(@channel_url)
     end
 
     test "supports an optional file_listener_handler that gets passed a filename" do
@@ -55,7 +56,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaCollectionTest do
         send(current_self, {:handler, filename})
       end
 
-      assert {:ok, _} = MediaCollection.get_media_attributes(@channel_url, file_listener_handler: handler)
+      assert {:ok, _} =
+               MediaCollection.get_media_attributes_for_collection(@channel_url, file_listener_handler: handler)
 
       assert_receive {:handler, filename}
       assert String.ends_with?(filename, ".json")

test/pinchflat/yt_dlp/backend/media_test.exs

@@ -0,0 +1,139 @@
+defmodule Pinchflat.YtDlp.Backend.MediaTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.YtDlp.Backend.Media
+
+  @media_url "https://www.youtube.com/watch?v=TiZPUDkDYbk"
+
+  setup :verify_on_exit!
+
+  describe "download/2" do
+    test "it calls the backend runner with the expected arguments" do
+      expect(YtDlpRunnerMock, :run, fn @media_url, opts, ot ->
+        assert [:no_simulate] = opts
+        assert "after_move:%()j" = ot
+
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      assert {:ok, _} = Media.download(@media_url)
+    end
+
+    test "it passes along additional options" do
+      expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
+        assert [:no_simulate, :custom_arg] = opts
+
+        {:ok, "{}"}
+      end)
+
+      assert {:ok, _} = Media.download(@media_url, [:custom_arg])
+    end
+
+    test "it parses and returns the generated file as JSON" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      assert {:ok, %{"title" => "Trying to Wheelie Without the Rear Brake"}} =
+               Media.download(@media_url)
+    end
+
+    test "it returns errors" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opt, _ot ->
+        {:error, "something"}
+      end)
+
+      assert {:error, "something"} = Media.download(@media_url)
+    end
+  end
+
+  describe "get_media_attributes/1" do
+    test "returns a list of video attributes" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      assert {:ok, %{description: _, media_id: _, original_url: _, title: _, livestream: _}} =
+               Media.get_media_attributes(@media_url)
+    end
+
+    test "it passes the expected default args" do
+      expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
+        assert opts == [:simulate, :skip_download]
+        assert ot == Media.indexing_output_template()
+
+        {:ok, media_attributes_return_fixture()}
+      end)
+
+      assert {:ok, _} = Media.get_media_attributes(@media_url)
+    end
+
+    test "returns the error straight through when the command fails" do
+      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
+
+      assert {:error, "Big issue", 1} = Media.get_media_attributes(@media_url)
+    end
+  end
+
+  describe "indexing_output_template/0" do
+    test "contains all the greatest hits" do
+      assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" ==
+               Media.indexing_output_template()
+    end
+  end
+
+  describe "response_to_struct/1" do
+    test "transforms a response into a struct" do
+      response = %{
+        "id" => "TiZPUDkDYbk",
+        "title" => "Trying to Wheelie Without the Rear Brake",
+        "description" => "I'm not sure what I expected.",
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "was_live" => false,
+        "aspect_ratio" => 1.0,
+        "duration" => 60
+      }
+
+      assert %Media{
+               media_id: "TiZPUDkDYbk",
+               title: "Trying to Wheelie Without the Rear Brake",
+               description: "I'm not sure what I expected.",
+               original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+               livestream: false,
+               short_form_content: false
+             } = Media.response_to_struct(response)
+    end
+
+    test "sets short_form_content to true if the URL contains /shorts/" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61
+      }
+
+      assert %Media{short_form_content: true} = Media.response_to_struct(response)
+    end
+
+    test "sets short_form_content to true if the aspect ratio are duration are right" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 0.5,
+        "duration" => 59
+      }
+
+      assert %Media{short_form_content: true} = Media.response_to_struct(response)
+    end
+
+    test "sets short_form_content to false otherwise" do
+      response = %{
+        "webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
+        "aspect_ratio" => 1.0,
+        "duration" => 61
+      }
+
+      assert %Media{short_form_content: false} = Media.response_to_struct(response)
+    end
+  end
+end

test/pinchflat/yt_dlp/download_option_builder_test.exs

@@ -1,11 +1,11 @@
-defmodule Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilderTest do
+defmodule Pinchflat.YtDlp.DownloadOptionBuilderTest do
   use Pinchflat.DataCase
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
   import Pinchflat.SourcesFixtures
 
   alias Pinchflat.Profiles
-  alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder
+  alias Pinchflat.YtDlp.DownloadOptionBuilder
 
   setup do
     media_profile = media_profile_fixture(%{output_path_template: "{{ title }}.%(ext)s"})

test/profiles/output_path_builder_test.exs

@@ -1,7 +1,7 @@
-defmodule Pinchflat.MediaClient.Backends.YtDlp.OutputPathBuilderTest do
+defmodule Pinchflat.Profiles.OutputPathBuilderTest do
   use Pinchflat.DataCase
 
-  alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
+  alias Pinchflat.Profiles.OutputPathBuilder
 
   describe "build/2" do
     test "it expands 'standard' curly brace variables in the template" do

test/support/fixtures/media_fixtures.ex

@@ -19,6 +19,7 @@ defmodule Pinchflat.MediaFixtures do
         title: Faker.Commerce.product_name(),
         original_url: "https://www.youtube.com/watch?v=#{media_id}",
         livestream: false,
+        short_form_content: false,
         media_filepath: "/video/#{Faker.File.file_name(:video)}",
         source_id: SourcesFixtures.source_fixture().id
       })
@@ -65,4 +66,18 @@ defmodule Pinchflat.MediaFixtures do
     merged_attrs = Map.merge(attrs, %{media_filepath: stored_media_filepath})
     media_item_fixture(merged_attrs)
   end
+
+  def media_attributes_return_fixture do
+    media_attributes = %{
+      id: "video1",
+      title: "Video 1",
+      webpage_url: "https://example.com/video1",
+      was_live: false,
+      description: "desc1",
+      aspect_ratio: 1.67,
+      duration: 123.45
+    }
+
+    Phoenix.json_library().encode!(media_attributes)
+  end
 end

test/support/fixtures/sources_fixtures.ex

@@ -35,23 +35,29 @@ defmodule Pinchflat.SourcesFixtures do
       %{
         id: "video1",
         title: "Video 1",
-        original_url: "https://example.com/video1",
+        webpage_url: "https://example.com/video1",
         was_live: false,
-        description: "desc1"
+        description: "desc1",
+        aspect_ratio: 1.67,
+        duration: 12.34
       },
       %{
         id: "video2",
         title: "Video 2",
-        original_url: "https://example.com/video2",
+        webpage_url: "https://example.com/video2",
         was_live: true,
-        description: "desc2"
+        description: "desc2",
+        aspect_ratio: 1.67,
+        duration: 345.67
       },
       %{
         id: "video3",
         title: "Video 3",
-        original_url: "https://example.com/video3",
+        webpage_url: "https://example.com/video3",
         was_live: false,
-        description: "desc3"
+        description: "desc3",
+        aspect_ratio: 1.0,
+        duration: 678.90
       }
     ]
 

test/test_helper.exs

@@ -1,4 +1,4 @@
-Mox.defmock(YtDlpRunnerMock, for: Pinchflat.MediaClient.Backends.BackendCommandRunner)
+Mox.defmock(YtDlpRunnerMock, for: Pinchflat.YtDlp.Backend.BackendCommandRunner)
 Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
 
 Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)