hermes-agent/pinchflat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Redo indexing mechanism (#16)

Browse Source 
* Bumped up the line length because I fear no man

* Refactored indexing

Previously, indexing worked by collecting the video IDs of only videos
that matched indexing criteria. This new model instead stores ALL videos
for a given source, but will only _download_ videos that meet that criteria.
This lets us backfill without indexing, makes it easier to add in other
backends, lets us download one-off videos for a source that don't quite
meet criteria, you name it.

* Updated media finders to respect format filters; Added credo file
This commit is contained in:
Kieran
2024-02-09 13:23:37 -08:00
committed by GitHub
parent 89497c4c6b
commit dabec62e99
31 changed files with 646 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs
+10 -8
View File
@@ -1,6 +1,7 @@
defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
use ExUnit.Case, async: true
import Mox
import Pinchflat.MediaSourceFixtures
alias Pinchflat.MediaClient.Backends.YtDlp.VideoCollection
@@ -8,22 +9,23 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
setup :verify_on_exit!
describe "get_video_ids/2" do
test "returns a list of video ids with no blank elements" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "id1\nid2\n\nid3\n"} end)
describe "get_media_attributes/2" do
test "returns a list of video attributes with no blank elements" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture() <> "\n\n"} end)
assert {:ok, ["id1", "id2", "id3"]} = VideoCollection.get_video_ids(@channel_url)
assert {:ok, [%{"id" => "video1"}, %{"id" => "video2"}, %{"id" => "video3"}]} =
VideoCollection.get_media_attributes(@channel_url)
end
test "it passes the expected default args" do
expect(YtDlpRunnerMock, :run, fn _url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(id)s"
assert ot == "%(.{id,title,was_live,original_url})j"
{:ok, ""}
end)
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url)
assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url)
end
test "it passes the expected custom args" do
@@ -33,13 +35,13 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
{:ok, ""}
end)
assert {:ok, _} = VideoCollection.get_video_ids(@channel_url, [:custom_arg])
assert {:ok, _} = VideoCollection.get_media_attributes(@channel_url, [:custom_arg])
end
test "returns the error straight through when the command fails" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:error, "Big issue", 1} end)
assert {:error, "Big issue", 1} = VideoCollection.get_video_ids(@channel_url)
assert {:error, "Big issue", 1} = VideoCollection.get_media_attributes(@channel_url)
end
end
test/pinchflat/media_client/source_details_test.exs
+12 -12
View File
@@ -22,7 +22,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
assert {:ok, _} = SourceDetails.get_source_details(@channel_url)
end
test "it returns a struct composed of the returned data" do
test "it returns a map composed of the returned data" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
Phoenix.json_library().encode(%{
channel: "TheUselessTrials",
@@ -43,42 +43,42 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
end
end
describe "get_video_ids/2 when passed a string" do
describe "get_media_attributes/2 when passed a string" do
test "it passes the expected arguments to the backend" do
expect(YtDlpRunnerMock, :run, fn @channel_url, opts, ot ->
assert opts == [:simulate, :skip_download]
assert ot == "%(id)s"
assert ot == "%(.{id,title,was_live,original_url})j"
{:ok, ""}
end)
assert {:ok, _} = SourceDetails.get_video_ids(@channel_url)
assert {:ok, _} = SourceDetails.get_media_attributes(@channel_url)
end
test "it returns a list of strings" do
test "it returns a list of maps" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
{:ok, "video1\nvideo2\nvideo3"}
{:ok, source_attributes_return_fixture()}
end)
assert {:ok, ["video1", "video2", "video3"]} = SourceDetails.get_video_ids(@channel_url)
assert {:ok, [%{}, %{}, %{}]} = SourceDetails.get_media_attributes(@channel_url)
end
end
describe "get_video_ids/2 when passed a Source record" do
describe "get_media_attributes/2 when passed a Source record" do
test "it calls the backend with the source's collection ID" do
source = source_fixture()
expect(YtDlpRunnerMock, :run, fn url, _opts, _ot ->
assert source.collection_id == url
{:ok, "video1\nvideo2\nvideo3"}
{:ok, source_attributes_return_fixture()}
end)
assert {:ok, _} = SourceDetails.get_video_ids(source)
assert {:ok, _} = SourceDetails.get_media_attributes(source)
end
test "it builds options based on the source's media profile" do
expect(YtDlpRunnerMock, :run, fn _url, opts, _ot ->
assert opts == [{:match_filter, "!was_live"}, :simulate, :skip_download]
assert opts == [:simulate, :skip_download]
{:ok, ""}
end)
@@ -89,7 +89,7 @@ defmodule Pinchflat.MediaClient.SourceDetailsTest do
)
source = source_fixture(media_profile_id: media_profile.id)
assert {:ok, _} = SourceDetails.get_video_ids(source)
assert {:ok, _} = SourceDetails.get_media_attributes(source)
end
end
end
test/pinchflat/media_client/video_downloader_test.exs
+1 -2
View File
@@ -10,7 +10,7 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
setup do
media_item =
Repo.preload(
media_item_fixture(%{title: nil, media_filepath: nil}),
media_item_fixture(%{title: "Something", media_filepath: nil}),
[:metadata, source: :media_profile]
)
@@ -58,7 +58,6 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
end
test "it extracts the title", %{media_item: media_item} do
assert media_item.title == nil
assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
assert updated_media_item.title == "Trying to Wheelie Without the Rear Brake"
end
test/pinchflat/media_source_test.exs
-54
View File
@@ -6,7 +6,6 @@ defmodule Pinchflat.MediaSourceTest do
import Pinchflat.MediaSourceFixtures
alias Pinchflat.MediaSource
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaSource.Source
alias Pinchflat.Workers.MediaIndexingWorker
@@ -117,59 +116,6 @@ defmodule Pinchflat.MediaSourceTest do
end
end
describe "index_media_items/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2\nvideo3"} end)
{:ok, [source: source_fixture()]}
end
test "it creates a media_item record for each media ID returned", %{source: source} do
assert media_items = MediaSource.index_media_items(source)
assert Enum.count(media_items) == 3
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
end
test "it attaches all media_items to the given source", %{source: source} do
source_id = source.id
assert media_items = MediaSource.index_media_items(source)
assert Enum.count(media_items) == 3
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
end
test "it won't duplicate media_items based on media_id and source", %{source: source} do
_first_run = MediaSource.index_media_items(source)
_duplicate_run = MediaSource.index_media_items(source)
media_items = Repo.preload(source, :media_items).media_items
assert Enum.count(media_items) == 3
end
test "it can duplicate media_ids for different sources", %{source: source} do
other_source = source_fixture()
media_items = MediaSource.index_media_items(source)
media_items_other_source = MediaSource.index_media_items(other_source)
assert Enum.count(media_items) == 3
assert Enum.count(media_items_other_source) == 3
assert Enum.map(media_items, & &1.media_id) ==
Enum.map(media_items_other_source, & &1.media_id)
end
test "it returns a list of media_items or changesets", %{source: source} do
first_run = MediaSource.index_media_items(source)
duplicate_run = MediaSource.index_media_items(source)
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
end
end
describe "update_source/2" do
test "updates with valid data updates the source" do
source = source_fixture()
test/pinchflat/media_test.exs
+136 -2
View File
@@ -3,6 +3,7 @@ defmodule Pinchflat.MediaTest do
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.ProfilesFixtures
import Pinchflat.MediaSourceFixtures
alias Pinchflat.Media
@@ -30,7 +31,7 @@ defmodule Pinchflat.MediaTest do
end
describe "list_pending_media_items_for/1" do
test "it returns pending media_items for a given source" do
test "it returns pending without a filepath for a given source" do
source = source_fixture()
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
@@ -50,6 +51,138 @@ defmodule Pinchflat.MediaTest do
end
end
describe "list_pending_media_items_for/1 when testing shorts" do
test "returns shorts and normal media when shorts_behaviour is :include" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :include}).id})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [normal, short]
end
test "returns only shorts when shorts_behaviour is :only" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :only}).id})
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [short]
end
test "returns only normal media when shorts_behaviour is :exclude" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{shorts_behaviour: :exclude}).id})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
_short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [normal]
end
end
describe "list_pending_media_items_for/1 when testing livestreams" do
test "returns livestreams and normal media when livestream_behaviour is :include" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :include}).id})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
assert Media.list_pending_media_items_for(source) == [normal, livestream]
end
test "returns only livestreams when livestream_behaviour is :only" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :only}).id})
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
assert Media.list_pending_media_items_for(source) == [livestream]
end
test "returns only normal media when livestream_behaviour is :exclude" do
source = source_fixture(%{media_profile_id: media_profile_fixture(%{livestream_behaviour: :exclude}).id})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
assert Media.list_pending_media_items_for(source) == [normal]
end
end
describe "list_pending_media_items_for/1 when testing all format options" do
test "returns livestreams, shorts, and normal media when behaviour is :include" do
source =
source_fixture(%{
media_profile_id:
media_profile_fixture(%{
shorts_behaviour: :include,
livestream_behaviour: :include
}).id
})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [normal, livestream, short]
end
test "returns only livestreams and shorts when behaviour is :only" do
source =
source_fixture(%{
media_profile_id:
media_profile_fixture(%{
shorts_behaviour: :only,
livestream_behaviour: :only
}).id
})
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [livestream, short]
end
test "returns only normal media when behaviour is :exclude" do
source =
source_fixture(%{
media_profile_id:
media_profile_fixture(%{
shorts_behaviour: :exclude,
livestream_behaviour: :exclude
}).id
})
normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
_short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [normal]
end
test ":only and :exclude return the expected results" do
source =
source_fixture(%{
media_profile_id:
media_profile_fixture(%{
shorts_behaviour: :only,
livestream_behaviour: :exclude
}).id
})
_normal = media_item_fixture(%{source_id: source.id, media_filepath: nil})
_livestream = media_item_fixture(%{source_id: source.id, media_filepath: nil, livestream: true})
short = media_item_fixture(%{source_id: source.id, media_filepath: nil, original_url: "/shorts/"})
assert Media.list_pending_media_items_for(source) == [short]
end
end
describe "list_downloaded_media_items_for/1" do
test "returns only media items with a media_filepath" do
source = source_fixture()
_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: "/video/#{Faker.File.file_name(:video)}"})
assert Media.list_downloaded_media_items_for(source) == [media_item]
end
end
describe "get_media_item!/1" do
test "it returns the media_item with given id" do
media_item = media_item_fixture()
@@ -63,7 +196,8 @@ defmodule Pinchflat.MediaTest do
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
test/pinchflat/profiles/options/yt_dlp/index_option_builder_test.exs
-104
View File
@@ -1,104 +0,0 @@
defmodule Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilderTest do
use ExUnit.Case, async: true
alias Pinchflat.Profiles.MediaProfile
alias Pinchflat.Profiles.Options.YtDlp.IndexOptionBuilder
@media_profile %MediaProfile{
output_path_template: "{{ title }}.%(ext)s",
shorts_behaviour: :include,
livestream_behaviour: :include
}
describe "build/1 when testing release type options" do
test "adds correct filter when shorts_behaviour is :only" do
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :only}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "was_live"} in res
end
test "adds correct filter when livestream_behaviour is :only" do
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :only}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "was_live"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "!original_url*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
end
test "adds correct filter when both livestreams and shorts are :only" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :only,
livestream_behaviour: :only
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url*=/shorts/"} in res
assert {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
end
test "adds correct filter when shorts_behaviour is :exclude" do
media_profile = %MediaProfile{@media_profile | shorts_behaviour: :exclude}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "was_live"} in res
refute {:match_filter, "!was_live"} in res
end
test "adds correct filter when livestream_behaviour is :exclude" do
media_profile = %MediaProfile{@media_profile | livestream_behaviour: :exclude}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "!was_live"} in res
refute {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
end
test "adds correct filter when shorts and livestreams are both exclude" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :exclude,
livestream_behaviour: :exclude
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "!was_live & original_url!*=/shorts/"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "was_live"} in res
end
test "does not add exclusion filter if one is excluded and the other is only" do
media_profile = %MediaProfile{
@media_profile
| shorts_behaviour: :exclude,
livestream_behaviour: :only
}
assert {:ok, res} = IndexOptionBuilder.build(media_profile)
assert {:match_filter, "was_live"} in res
refute {:match_filter, "original_url!*=/shorts/"} in res
refute {:match_filter, "original_url*=/shorts/"} in res
refute {:match_filter, "!was_live"} in res
end
end
end
test/pinchflat/tasks/source_tasks_test.exs
+59
View File
@@ -1,6 +1,7 @@
defmodule Pinchflat.Tasks.SourceTasksTest do
use Pinchflat.DataCase
import Mox
import Pinchflat.TasksFixtures
import Pinchflat.MediaFixtures
import Pinchflat.MediaSourceFixtures
@@ -8,9 +9,12 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
alias Pinchflat.Tasks
alias Pinchflat.Tasks.Task
alias Pinchflat.Tasks.SourceTasks
alias Pinchflat.Media.MediaItem
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.VideoDownloadWorker
setup :verify_on_exit!
describe "kickoff_indexing_task/1" do
test "it does not schedule a job if the interval is <= 0" do
source = source_fixture(index_frequency_minutes: -1)
@@ -46,6 +50,61 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
end
end
describe "index_media_items/1" do
setup do
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
{:ok, [source: source_fixture()]}
end
test "it creates a media_item record for each media ID returned", %{source: source} do
assert media_items = SourceTasks.index_media_items(source)
assert Enum.count(media_items) == 3
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
assert ["Video 1", "Video 2", "Video 3"] == Enum.map(media_items, & &1.title)
assert Enum.all?(media_items, fn mi -> mi.original_url end)
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
end
test "it attaches all media_items to the given source", %{source: source} do
source_id = source.id
assert media_items = SourceTasks.index_media_items(source)
assert Enum.count(media_items) == 3
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
end
test "it won't duplicate media_items based on media_id and source", %{source: source} do
_first_run = SourceTasks.index_media_items(source)
_duplicate_run = SourceTasks.index_media_items(source)
media_items = Repo.preload(source, :media_items).media_items
assert Enum.count(media_items) == 3
end
test "it can duplicate media_ids for different sources", %{source: source} do
other_source = source_fixture()
media_items = SourceTasks.index_media_items(source)
media_items_other_source = SourceTasks.index_media_items(other_source)
assert Enum.count(media_items) == 3
assert Enum.count(media_items_other_source) == 3
assert Enum.map(media_items, & &1.media_id) ==
Enum.map(media_items_other_source, & &1.media_id)
end
test "it returns a list of media_items or changesets", %{source: source} do
first_run = SourceTasks.index_media_items(source)
duplicate_run = SourceTasks.index_media_items(source)
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
end
end
describe "enqueue_pending_media_downloads/1" do
test "it enqueues a job for each pending media item" do
source = source_fixture()
test/pinchflat/utils/function_utils_test.exs
+11
View File
@@ -0,0 +1,11 @@
defmodule Pinchflat.Utils.FunctionUtilsTest do
use ExUnit.Case, async: true
alias Pinchflat.Utils.FunctionUtils
describe "wrap_ok/1" do
test "wraps the provided term in an :ok tuple" do
assert FunctionUtils.wrap_ok("hello") == {:ok, "hello"}
end
end
end
test/pinchflat/workers/media_indexing_worker_test.exs
+10 -9
View File
@@ -38,32 +38,33 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
end
test "it kicks off a download job for each pending media item" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
source = source_fixture(index_frequency_minutes: 10)
perform_job(MediaIndexingWorker, %{id: source.id})
assert [_] = all_enqueued(worker: VideoDownloadWorker)
assert length(all_enqueued(worker: VideoDownloadWorker)) == 3
end
test "it starts a job for any pending media item even if it's from another run" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
source = source_fixture(index_frequency_minutes: 10)
media_item_fixture(%{source_id: source.id, media_filepath: nil})
perform_job(MediaIndexingWorker, %{id: source.id})
assert [_, _] = all_enqueued(worker: VideoDownloadWorker)
assert length(all_enqueued(worker: VideoDownloadWorker)) == 4
end
test "it does not kick off a job for media items that could not be saved" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo1"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
source = source_fixture(index_frequency_minutes: 10)
media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
perform_job(MediaIndexingWorker, %{id: source.id})
# Only one job should be enqueued, since the second video is a duplicate
assert [_] = all_enqueued(worker: VideoDownloadWorker)
# Only 3 jobs should be enqueued, since the first video is a duplicate
assert length(all_enqueued(worker: VideoDownloadWorker))
end
test "it reschedules the job based on the index frequency" do
@@ -91,7 +92,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
end
test "it creates the basic media_item records" do
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, "video1\nvideo2"} end)
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, source_attributes_return_fixture()} end)
source = source_fixture(index_frequency_minutes: 10)
@@ -102,7 +103,7 @@ defmodule Pinchflat.Workers.MediaIndexingWorkerTest do
|> Enum.map(fn media_item -> media_item.media_id end)
end
assert_changed([from: [], to: ["video1", "video2"]], media_item_fetcher, fn ->
assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
perform_job(MediaIndexingWorker, %{id: source.id})
end)
end
test/support/fixtures/media_fixtures.ex
+2
View File
@@ -15,6 +15,8 @@ defmodule Pinchflat.MediaFixtures do
|> Enum.into(%{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
livestream: false,
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: MediaSourceFixtures.source_fixture().id
})
test/support/fixtures/media_source_fixtures.ex
+26
View File
@@ -29,4 +29,30 @@ defmodule Pinchflat.MediaSourceFixtures do
source
end
def source_attributes_return_fixture do
source_attributes = [
%{
id: "video1",
title: "Video 1",
original_url: "https://example.com/video1",
was_live: false
},
%{
id: "video2",
title: "Video 2",
original_url: "https://example.com/video2",
was_live: true
},
%{
id: "video3",
title: "Video 3",
original_url: "https://example.com/video3",
was_live: false
}
]
source_attributes
|> Enum.map_join("\n", &Phoenix.json_library().encode!(&1))
end
end