Refactor modules into contexts (#78)
* [WIP] break out a few contexts, start refactoring fast index modules * [WIP] more contexts, this time around slow indexing and downloads * [WIP] got all tests passing * [WIP] Added moduledocs * Built a genserver to rename old jobs on boot * Added a module naming check; moved things around * Fixed specs
This commit is contained in:
Kieran authored and 
GitHub committed
1 parent
7fa5c4da63
commit
33baa99aae
66 files changed
+1092
-568
No files matched your search
@@ -0,0 +1,52 @@
|
||||
defmodule Pinchflat.SlowIndexing.FileFollowerServerTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias alias Pinchflat.Filesystem.FilesystemHelpers
|
||||
alias Pinchflat.SlowIndexing.FileFollowerServer
|
||||
|
||||
setup do
|
||||
{:ok, pid} = FileFollowerServer.start_link()
|
||||
tmpfile = FilesystemHelpers.generate_metadata_tmpfile(:txt)
|
||||
|
||||
{:ok, %{pid: pid, tmpfile: tmpfile}}
|
||||
end
|
||||
|
||||
describe "watch_file" do
|
||||
test "calls the handler for each existing line in the file", %{pid: pid, tmpfile: tmpfile} do
|
||||
File.write!(tmpfile, "line1\nline2")
|
||||
parent = self()
|
||||
|
||||
handler = fn line -> send(parent, line) end
|
||||
FileFollowerServer.watch_file(pid, tmpfile, handler)
|
||||
|
||||
assert_receive "line1\n"
|
||||
assert_receive "line2"
|
||||
end
|
||||
|
||||
test "calls the handler for each new line in the file", %{pid: pid, tmpfile: tmpfile} do
|
||||
parent = self()
|
||||
file = File.open!(tmpfile, [:append])
|
||||
handler = fn line -> send(parent, line) end
|
||||
|
||||
FileFollowerServer.watch_file(pid, tmpfile, handler)
|
||||
|
||||
IO.binwrite(file, "line1\n")
|
||||
assert_receive "line1\n"
|
||||
IO.binwrite(file, "line2")
|
||||
assert_receive "line2"
|
||||
end
|
||||
end
|
||||
|
||||
describe "stop" do
|
||||
test "stops the watcher", %{pid: pid, tmpfile: tmpfile} do
|
||||
handler = fn _line -> :noop end
|
||||
FileFollowerServer.watch_file(pid, tmpfile, handler)
|
||||
|
||||
refute is_nil(Process.info(pid))
|
||||
FileFollowerServer.stop(pid)
|
||||
# Gotta wait for the server to stop async
|
||||
:timer.sleep(50)
|
||||
assert is_nil(Process.info(pid))
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,162 @@
|
||||
defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorkerTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
import Mox
|
||||
import Pinchflat.TasksFixtures
|
||||
import Pinchflat.MediaFixtures
|
||||
import Pinchflat.SourcesFixtures
|
||||
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.FastIndexing.FastIndexingWorker
|
||||
alias Pinchflat.Downloading.MediaDownloadWorker
|
||||
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
|
||||
|
||||
setup :verify_on_exit!
|
||||
|
||||
describe "perform/1" do
|
||||
test "it indexes the source if it should be indexed" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
end
|
||||
|
||||
test "it indexes the source no matter what if the source has never been indexed before" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 0, last_indexed_at: nil)
|
||||
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
end
|
||||
|
||||
test "it does not do any indexing if the source has been indexed and shouldn't be rescheduled" do
|
||||
expect(YtDlpRunnerMock, :run, 0, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: -1, last_indexed_at: DateTime.utc_now())
|
||||
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
end
|
||||
|
||||
test "it does not reschedule if the source shouldn't be indexed" do
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: -1)
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
refute_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
|
||||
end
|
||||
|
||||
test "it kicks off a download job for each pending media item" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
assert length(all_enqueued(worker: MediaDownloadWorker)) == 3
|
||||
end
|
||||
|
||||
test "it starts a job for any pending media item even if it's from another run" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
media_item_fixture(%{source_id: source.id, media_filepath: nil})
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
assert length(all_enqueued(worker: MediaDownloadWorker)) == 4
|
||||
end
|
||||
|
||||
test "it does not kick off a job for media items that could not be saved" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
media_item_fixture(%{source_id: source.id, media_filepath: nil, media_id: "video1"})
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
# Only 3 jobs should be enqueued, since the first video is a duplicate
|
||||
assert length(all_enqueued(worker: MediaDownloadWorker))
|
||||
end
|
||||
|
||||
test "it reschedules the job based on the index frequency" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
assert_enqueued(
|
||||
worker: MediaCollectionIndexingWorker,
|
||||
args: %{"id" => source.id},
|
||||
scheduled_at: now_plus(source.index_frequency_minutes, :minutes)
|
||||
)
|
||||
end
|
||||
|
||||
test "it creates a task for the rescheduled job" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
task_count_fetcher = fn -> Enum.count(Tasks.list_tasks()) end
|
||||
|
||||
assert_changed([from: 0, to: 1], task_count_fetcher, fn ->
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
end)
|
||||
end
|
||||
|
||||
test "it creates a future task for fast indexing if appropriate" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10, fast_index: true)
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
assert_enqueued(
|
||||
worker: FastIndexingWorker,
|
||||
args: %{"id" => source.id},
|
||||
scheduled_at: now_plus(Source.fast_index_frequency(), :minutes)
|
||||
)
|
||||
end
|
||||
|
||||
test "it deletes existing fast indexing tasks if a new one is created" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10, fast_index: true)
|
||||
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "it does not create a task for fast indexing otherwise" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, ""} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10, fast_index: false)
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
|
||||
refute_enqueued(worker: FastIndexingWorker)
|
||||
end
|
||||
|
||||
test "it creates the basic media_item records" do
|
||||
expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts -> {:ok, source_attributes_return_fixture()} end)
|
||||
|
||||
source = source_fixture(index_frequency_minutes: 10)
|
||||
|
||||
media_item_fetcher = fn ->
|
||||
source
|
||||
|> Repo.preload(:media_items)
|
||||
|> Map.get(:media_items)
|
||||
|> Enum.map(fn media_item -> media_item.media_id end)
|
||||
end
|
||||
|
||||
assert_changed([from: [], to: ["video1", "video2", "video3"]], media_item_fetcher, fn ->
|
||||
perform_job(MediaCollectionIndexingWorker, %{id: source.id})
|
||||
end)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,270 @@
|
||||
defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
|
||||
use Pinchflat.DataCase
|
||||
|
||||
import Mox
|
||||
import Pinchflat.TasksFixtures
|
||||
import Pinchflat.MediaFixtures
|
||||
import Pinchflat.SourcesFixtures
|
||||
import Pinchflat.ProfilesFixtures
|
||||
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Tasks.Task
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.FastIndexing.FastIndexingWorker
|
||||
alias Pinchflat.Downloading.MediaDownloadWorker
|
||||
alias Pinchflat.FastIndexing.MediaIndexingWorker
|
||||
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
|
||||
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
|
||||
|
||||
setup :verify_on_exit!
|
||||
|
||||
describe "kickoff_indexing_task/1" do
|
||||
test "it schedules a job" do
|
||||
source = source_fixture(index_frequency_minutes: 1)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_enqueued(worker: MediaCollectionIndexingWorker, args: %{"id" => source.id})
|
||||
end
|
||||
|
||||
test "it creates and attaches a task" do
|
||||
source = source_fixture(index_frequency_minutes: 1)
|
||||
|
||||
assert {:ok, %Task{} = task} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert task.source_id == source.id
|
||||
end
|
||||
|
||||
test "it deletes any pending media collection tasks for the source" do
|
||||
source = source_fixture()
|
||||
{:ok, job} = Oban.insert(MediaCollectionIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "it deletes any pending media tasks for the source" do
|
||||
source = source_fixture()
|
||||
{:ok, job} = Oban.insert(MediaIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
|
||||
test "it deletes any fast indexing tasks for the source" do
|
||||
source = source_fixture()
|
||||
{:ok, job} = Oban.insert(FastIndexingWorker.new(%{"id" => source.id}))
|
||||
task = task_fixture(source_id: source.id, job_id: job.id)
|
||||
|
||||
assert {:ok, _} = SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
assert_raise Ecto.NoResultsError, fn -> Repo.reload!(task) end
|
||||
end
|
||||
end
|
||||
|
||||
describe "index_and_enqueue_download_for_media_items/1" do
|
||||
setup do
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl_opts ->
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
{:ok, [source: source_fixture()]}
|
||||
end
|
||||
|
||||
test "it creates a media_item record for each media ID returned", %{source: source} do
|
||||
assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert ["video1", "video2", "video3"] == Enum.map(media_items, & &1.media_id)
|
||||
assert ["Video 1", "Video 2", "Video 3"] == Enum.map(media_items, & &1.title)
|
||||
assert ["desc1", "desc2", "desc3"] == Enum.map(media_items, & &1.description)
|
||||
assert Enum.all?(media_items, fn mi -> mi.original_url end)
|
||||
assert Enum.all?(media_items, fn %MediaItem{} -> true end)
|
||||
end
|
||||
|
||||
test "it attaches all media_items to the given source", %{source: source} do
|
||||
source_id = source.id
|
||||
assert media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.all?(media_items, fn %MediaItem{source_id: ^source_id} -> true end)
|
||||
end
|
||||
|
||||
test "it won't duplicate media_items based on media_id and source", %{source: source} do
|
||||
_first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
_duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
media_items = Repo.preload(source, :media_items).media_items
|
||||
assert Enum.count(media_items) == 3
|
||||
end
|
||||
|
||||
test "it can duplicate media_ids for different sources", %{source: source} do
|
||||
other_source = source_fixture()
|
||||
|
||||
media_items = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
media_items_other_source = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(other_source)
|
||||
|
||||
assert Enum.count(media_items) == 3
|
||||
assert Enum.count(media_items_other_source) == 3
|
||||
|
||||
assert Enum.map(media_items, & &1.media_id) ==
|
||||
Enum.map(media_items_other_source, & &1.media_id)
|
||||
end
|
||||
|
||||
test "it returns a list of media_items", %{source: source} do
|
||||
first_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
duplicate_run = SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
first_ids = Enum.map(first_run, & &1.id)
|
||||
duplicate_ids = Enum.map(duplicate_run, & &1.id)
|
||||
|
||||
assert first_ids == duplicate_ids
|
||||
end
|
||||
|
||||
test "it updates the source's last_indexed_at field", %{source: source} do
|
||||
assert source.last_indexed_at == nil
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
source = Repo.reload!(source)
|
||||
|
||||
assert DateTime.diff(DateTime.utc_now(), source.last_indexed_at) < 2
|
||||
end
|
||||
|
||||
test "it enqueues a job for each pending media item" do
|
||||
source = source_fixture()
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
assert_enqueued(worker: MediaDownloadWorker, args: %{"id" => media_item.id})
|
||||
end
|
||||
|
||||
test "it does not attach tasks if the source is set to not download" do
|
||||
source = source_fixture(download_media: false)
|
||||
media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
|
||||
assert [] = Tasks.list_tasks_for(:media_item_id, media_item.id)
|
||||
end
|
||||
end
|
||||
|
||||
describe "index_and_enqueue_download_for_media_items/1 when testing file watcher" do
|
||||
setup do
|
||||
{:ok, [source: source_fixture()]}
|
||||
end
|
||||
|
||||
test "creates a new media item for everything already in the file", %{source: source} do
|
||||
watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
|
||||
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
|
||||
filepath = Keyword.get(addl_opts, :output_filepath)
|
||||
File.write(filepath, source_attributes_return_fixture())
|
||||
|
||||
# Need to add a delay to ensure the file watcher has time to read the file
|
||||
:timer.sleep(watcher_poll_interval * 2)
|
||||
# We know we're testing the file watcher since the syncronous call will only
|
||||
# return an empty string (creating no records)
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
assert Repo.aggregate(MediaItem, :count, :id) == 0
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
assert Repo.aggregate(MediaItem, :count, :id) == 3
|
||||
end
|
||||
|
||||
test "enqueues a download for everything already in the file", %{source: source} do
|
||||
watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
|
||||
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
|
||||
filepath = Keyword.get(addl_opts, :output_filepath)
|
||||
File.write(filepath, source_attributes_return_fixture())
|
||||
|
||||
# Need to add a delay to ensure the file watcher has time to read the file
|
||||
:timer.sleep(watcher_poll_interval * 2)
|
||||
# We know we're testing the file watcher since the syncronous call will only
|
||||
# return an empty string (creating no records)
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
assert_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "does not enqueue downloads if the source is set to not download" do
|
||||
watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
|
||||
source = source_fixture(download_media: false)
|
||||
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
|
||||
filepath = Keyword.get(addl_opts, :output_filepath)
|
||||
File.write(filepath, source_attributes_return_fixture())
|
||||
|
||||
# Need to add a delay to ensure the file watcher has time to read the file
|
||||
:timer.sleep(watcher_poll_interval * 2)
|
||||
# We know we're testing the file watcher since the syncronous call will only
|
||||
# return an empty string (creating no records)
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "does not enqueue downloads for media that doesn't match the profile's format options" do
|
||||
watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
|
||||
profile = media_profile_fixture(%{shorts_behaviour: :exclude})
|
||||
source = source_fixture(%{media_profile_id: profile.id})
|
||||
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
|
||||
filepath = Keyword.get(addl_opts, :output_filepath)
|
||||
|
||||
contents =
|
||||
Phoenix.json_library().encode!(%{
|
||||
id: "video2",
|
||||
title: "Video 2",
|
||||
webpage_url: "https://example.com/shorts/video2",
|
||||
was_live: true,
|
||||
description: "desc2",
|
||||
aspect_ratio: 1.67,
|
||||
duration: 345.67,
|
||||
upload_date: "20210101"
|
||||
})
|
||||
|
||||
File.write(filepath, contents)
|
||||
|
||||
# Need to add a delay to ensure the file watcher has time to read the file
|
||||
:timer.sleep(watcher_poll_interval * 2)
|
||||
# We know we're testing the file watcher since the syncronous call will only
|
||||
# return an empty string (creating no records)
|
||||
{:ok, ""}
|
||||
end)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
refute_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
|
||||
test "does not enqueue multiple download jobs for the same media items", %{source: source} do
|
||||
watcher_poll_interval = Application.get_env(:pinchflat, :file_watcher_poll_interval)
|
||||
|
||||
stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts ->
|
||||
filepath = Keyword.get(addl_opts, :output_filepath)
|
||||
File.write(filepath, source_attributes_return_fixture())
|
||||
|
||||
# Need to add a delay to ensure the file watcher has time to read the file
|
||||
:timer.sleep(watcher_poll_interval * 2)
|
||||
# This also returns the final result to the yt-dlp call (like the real usage actually would do)
|
||||
# so it'll attempt to create the media items and enqueue the download jobs based on this as well
|
||||
{:ok, source_attributes_return_fixture()}
|
||||
end)
|
||||
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
assert Repo.aggregate(MediaItem, :count, :id) == 3
|
||||
assert [_, _, _] = all_enqueued(worker: MediaDownloadWorker)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user