hermes-agent/pinchflat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Download cutoff date for sources (#69)

Browse Source 
* Added new uploaded_at column to media items

* Updated indexer to pull upload date

* Updates media item creation to update on conflict

* Added download cutoff date to sources

* Applies cutoff date logic to pending media logic

* Updated docs
This commit is contained in:
Kieran
2024-03-10 21:24:01 -07:00
committed by GitHub
parent c0a11e98d9
commit f60ec4f49d
19 changed files with 304 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/pinchflat/media_test.exs
+66 -1
View File
@@ -215,6 +215,30 @@ defmodule Pinchflat.MediaTest do
end
end
describe "list_pending_media_items_for/1 when testing cutoff dates" do
test "does not return media items with an upload date before the cutoff date" do
source = source_fixture(%{download_cutoff_date: now_minus(1, :day)})
_old_media_item =
media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now()})
assert Media.list_pending_media_items_for(source) == [new_media_item]
end
test "does not apply a cutoff if there is no cutoff date" do
source = source_fixture(%{download_cutoff_date: nil})
old_media_item =
media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now()})
assert Media.list_pending_media_items_for(source) == [old_media_item, new_media_item]
end
end
describe "list_downloaded_media_items_for/1" do
test "returns only media items with a media_filepath" do
source = source_fixture()
@@ -260,6 +284,27 @@ defmodule Pinchflat.MediaTest do
refute Media.pending_download?(media_item)
end
test "returns true if there is a cutoff date before the media's upload date" do
source = source_fixture(%{download_cutoff_date: now_minus(2, :days)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(1, :day)})
assert Media.pending_download?(media_item)
end
test "returns false if there is a cutoff date after the media's upload date" do
source = source_fixture(%{download_cutoff_date: now_minus(1, :day)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
refute Media.pending_download?(media_item)
end
test "returns true if there is no cutoff date" do
source = source_fixture(%{download_cutoff_date: nil})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(1, :day)})
assert Media.pending_download?(media_item)
end
end
describe "search/1" do
@@ -373,10 +418,12 @@ defmodule Pinchflat.MediaTest do
title: Faker.Commerce.product_name(),
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today()
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
assert media_item.title == valid_attrs.title
assert media_item.media_id == valid_attrs.media_id
assert media_item.media_filepath == valid_attrs.media_filepath
@@ -397,12 +444,30 @@ defmodule Pinchflat.MediaTest do
|> YtDlpMedia.response_to_struct()
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item_from_backend_attrs(source, media_attrs)
assert media_item.source_id == source.id
assert media_item.title == media_attrs.title
assert media_item.media_id == media_attrs.media_id
assert media_item.original_url == media_attrs.original_url
assert media_item.description == media_attrs.description
end
test "updates the media item if it already exists" do
source = source_fixture()
media_attrs =
media_attributes_return_fixture()
|> Phoenix.json_library().decode!()
|> YtDlpMedia.response_to_struct()
different_attrs = %YtDlpMedia{media_attrs | title: "Different title"}
assert {:ok, %MediaItem{} = media_item_1} = Media.create_media_item_from_backend_attrs(source, media_attrs)
assert {:ok, %MediaItem{} = media_item_2} = Media.create_media_item_from_backend_attrs(source, different_attrs)
assert media_item_1.id == media_item_2.id
assert media_item_2.title == different_attrs.title
end
end
describe "update_media_item/2" do
test/pinchflat/sources_test.exs
+15 -1
View File
@@ -115,6 +115,12 @@ defmodule Pinchflat.SourcesTest do
assert {:error, %Ecto.Changeset{}} = Sources.create_source(@invalid_source_attrs)
end
test "creation with invalid data fails fast and does not call the runner" do
expect(YtDlpRunnerMock, :run, 0, &channel_mock/3)
assert {:error, %Ecto.Changeset{}} = Sources.create_source(@invalid_source_attrs)
end
test "creation enforces uniqueness of collection_id scoped to the media_profile" do
expect(YtDlpRunnerMock, :run, 2, fn _url, _opts, _ot ->
{:ok,
@@ -225,6 +231,14 @@ defmodule Pinchflat.SourcesTest do
assert source.collection_name == "some updated name"
end
test "updates with invalid data fails fast and does not call the runner" do
expect(YtDlpRunnerMock, :run, 0, &channel_mock/3)
source = source_fixture()
assert {:error, %Ecto.Changeset{}} = Sources.update_source(source, @invalid_source_attrs)
end
test "updating the original_url will re-fetch the source details for channels" do
expect(YtDlpRunnerMock, :run, &channel_mock/3)
@@ -430,7 +444,7 @@ defmodule Pinchflat.SourcesTest do
end
end
describe "change_source/2" do
describe "change_source/3" do
test "it returns a changeset" do
source = source_fixture()
test/pinchflat/tasks/media_items_tasks_test.exs
+5 -3
View File
@@ -49,10 +49,11 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do
end
test "won't duplicate media_items based on media_id and source", %{source: source} do
assert {:ok, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:error, _} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:ok, mi_1} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert {:ok, mi_2} = MediaItemTasks.index_and_enqueue_download_for_media_item(source, @media_url)
assert Repo.aggregate(MediaItem, :count) == 1
assert mi_1.id == mi_2.id
end
test "enqueues a download job", %{source: source} do
@@ -88,7 +89,8 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20210101"
})
{:ok, output}
test/pinchflat/tasks/source_tasks_test.exs
+7 -4
View File
@@ -168,12 +168,14 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
Enum.map(media_items_other_source, & &1.media_id)
end
test "it returns a list of media_items or changesets", %{source: source} do
test "it returns a list of media_items", %{source: source} do
first_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
duplicate_run = SourceTasks.index_and_enqueue_download_for_media_items(source)
assert Enum.all?(first_run, fn %MediaItem{} -> true end)
assert Enum.all?(duplicate_run, fn %Ecto.Changeset{} -> true end)
first_ids = Enum.map(first_run, & &1.id)
duplicate_ids = Enum.map(duplicate_run, & &1.id)
assert first_ids == duplicate_ids
end
test "it updates the source's last_indexed_at field", %{source: source} do
@@ -282,7 +284,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20210101"
})
File.write(filepath, contents)
test/pinchflat/yt_dlp/backend/media_test.exs
+25 -7
View File
@@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
describe "indexing_output_template/0" do
test "contains all the greatest hits" do
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" ==
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" ==
Media.indexing_output_template()
end
end
@@ -93,7 +93,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"was_live" => false,
"aspect_ratio" => 1.0,
"duration" => 60
"duration" => 60,
"upload_date" => "20210101"
}
assert %Media{
@@ -102,15 +103,17 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
description: "I'm not sure what I expected.",
original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
livestream: false,
short_form_content: false
} = Media.response_to_struct(response)
short_form_content: false,
upload_date: Date.from_iso8601!("2021-01-01")
} == Media.response_to_struct(response)
end
test "sets short_form_content to true if the URL contains /shorts/" do
response = %{
"webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61
"duration" => 61,
"upload_date" => "20210101"
}
assert %Media{short_form_content: true} = Media.response_to_struct(response)
@@ -120,7 +123,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 0.5,
"duration" => 59
"duration" => 59,
"upload_date" => "20210101"
}
assert %Media{short_form_content: true} = Media.response_to_struct(response)
@@ -130,10 +134,24 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61
"duration" => 61,
"upload_date" => "20210101"
}
assert %Media{short_form_content: false} = Media.response_to_struct(response)
end
test "parses the upload date" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => "20210101"
}
expected_date = Date.from_iso8601!("2021-01-01")
assert %Media{upload_date: ^expected_date} = Media.response_to_struct(response)
end
end
end
test/support/fixtures/media_fixtures.ex
+4 -2
View File
@@ -21,7 +21,8 @@ defmodule Pinchflat.MediaFixtures do
livestream: false,
short_form_content: false,
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: SourcesFixtures.source_fixture().id
source_id: SourcesFixtures.source_fixture().id,
upload_date: DateTime.utc_now()
})
|> Pinchflat.Media.create_media_item()
@@ -75,7 +76,8 @@ defmodule Pinchflat.MediaFixtures do
was_live: false,
description: "desc1",
aspect_ratio: 1.67,
duration: 123.45
duration: 123.45,
upload_date: "20210101"
}
Phoenix.json_library().encode!(media_attributes)
test/support/fixtures/sources_fixtures.ex
+19 -12
View File
@@ -15,15 +15,19 @@ defmodule Pinchflat.SourcesFixtures do
{:ok, source} =
%Source{}
|> Source.changeset(
Enum.into(attrs, %{
collection_name: "Source ##{:rand.uniform(1_000_000)}",
collection_id: Base.encode16(:crypto.hash(:md5, "#{:rand.uniform(1_000_000)}")),
collection_type: "channel",
custom_name: "Cool and good internal name!",
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
media_profile_id: ProfilesFixtures.media_profile_fixture().id,
index_frequency_minutes: 60
})
Enum.into(
attrs,
%{
collection_name: "Source ##{:rand.uniform(1_000_000)}",
collection_id: Base.encode16(:crypto.hash(:md5, "#{:rand.uniform(1_000_000)}")),
collection_type: "channel",
custom_name: "Cool and good internal name!",
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
media_profile_id: ProfilesFixtures.media_profile_fixture().id,
index_frequency_minutes: 60
}
),
:pre_insert
)
|> Repo.insert()
@@ -39,7 +43,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: false,
description: "desc1",
aspect_ratio: 1.67,
duration: 12.34
duration: 12.34,
upload_date: "20210101"
},
%{
id: "video2",
@@ -48,7 +53,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20220202"
},
%{
id: "video3",
@@ -57,7 +63,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: false,
description: "desc3",
aspect_ratio: 1.0,
duration: 678.90
duration: 678.90,
upload_date: "20230303"
}
]
test/support/testing_helper_methods.ex
+8
View File
@@ -11,6 +11,14 @@ defmodule Pinchflat.TestingHelperMethods do
DateTime.add(now(), offset, :minute)
end
def now_minus(offset, unit) when unit in [:minute, :minutes] do
DateTime.add(now(), -offset, :minute)
end
def now_minus(offset, unit) when unit in [:day, :days] do
DateTime.add(now(), -offset, :day)
end
def assert_changed(checker_fun, action_fn) do
before_res = checker_fun.()
action_fn.()