hermes-agent/pinchflat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Went back to using original_url over webpage_url to improve short detection (#447)

Browse Source
This commit is contained in:
Kieran
2024-10-30 14:09:28 -07:00
committed by GitHub
parent 2f83fe03d2
commit a5afcf168b
6 changed files with 27 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/pinchflat/yt_dlp/media.ex
+6 -4
View File
@@ -87,9 +87,11 @@ defmodule Pinchflat.YtDlp.Media do
Returns the output template for yt-dlp's indexing command. Returns the output template for yt-dlp's indexing command.
NOTE: playlist_index is really only useful for playlists that will never change their order. NOTE: playlist_index is really only useful for playlists that will never change their order.
NOTE: I've switched back to `original_url` (from `webpage_url`) since it's started indicating
if something is a short via the URL again
""" """
def indexing_output_template do def indexing_output_template do
"%(.{id,title,live_status,webpage_url,description,aspect_ratio,duration,upload_date,timestamp,playlist_index})j" "%(.{id,title,live_status,original_url,description,aspect_ratio,duration,upload_date,timestamp,playlist_index})j"
end end
@doc """ @doc """
@@ -103,17 +105,17 @@ defmodule Pinchflat.YtDlp.Media do
media_id: response["id"], media_id: response["id"],
title: response["title"], title: response["title"],
description: response["description"], description: response["description"],
original_url: response["webpage_url"], original_url: response["original_url"],
livestream: !!response["live_status"] && response["live_status"] != "not_live", livestream: !!response["live_status"] && response["live_status"] != "not_live",
duration_seconds: response["duration"] && round(response["duration"]), duration_seconds: response["duration"] && round(response["duration"]),
short_form_content: response["webpage_url"] && short_form_content?(response), short_form_content: response["original_url"] && short_form_content?(response),
uploaded_at: response["upload_date"] && parse_uploaded_at(response), uploaded_at: response["upload_date"] && parse_uploaded_at(response),
playlist_index: response["playlist_index"] || 0 playlist_index: response["playlist_index"] || 0
} }
end end
defp short_form_content?(response) do defp short_form_content?(response) do
if String.contains?(response["webpage_url"], "/shorts/") do if String.contains?(response["original_url"], "/shorts/") do
true true
else else
# Sometimes shorts are returned without /shorts/ in the URL, # Sometimes shorts are returned without /shorts/ in the URL,
test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs
+1 -1
View File
@@ -100,7 +100,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do
Phoenix.json_library().encode!(%{ Phoenix.json_library().encode!(%{
id: "video2", id: "video2",
title: "Video 2", title: "Video 2",
webpage_url: "https://example.com/shorts/video2", original_url: "https://example.com/shorts/video2",
live_status: "is_live", live_status: "is_live",
description: "desc2", description: "desc2",
aspect_ratio: 1.67, aspect_ratio: 1.67,
test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs
+2 -2
View File
@@ -188,7 +188,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
description: "desc3", description: "desc3",
# Only focusing on these because these are passed to functions that # Only focusing on these because these are passed to functions that
# could fail if they're not present # could fail if they're not present
webpage_url: nil, original_url: nil,
aspect_ratio: nil, aspect_ratio: nil,
duration: nil, duration: nil,
upload_date: nil upload_date: nil
@@ -299,7 +299,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do
Phoenix.json_library().encode!(%{ Phoenix.json_library().encode!(%{
id: "video2", id: "video2",
title: "Video 2", title: "Video 2",
webpage_url: "https://example.com/shorts/video2", original_url: "https://example.com/shorts/video2",
live_status: "is_live", live_status: "is_live",
description: "desc2", description: "desc2",
aspect_ratio: 1.67, aspect_ratio: 1.67,
test/pinchflat/yt_dlp/media_test.exs
+14 -14
View File
@@ -139,7 +139,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
describe "indexing_output_template/0" do describe "indexing_output_template/0" do
test "contains all the greatest hits" do test "contains all the greatest hits" do
attrs = attrs =
~w(id title live_status webpage_url description aspect_ratio duration upload_date timestamp playlist_index)a ~w(id title live_status original_url description aspect_ratio duration upload_date timestamp playlist_index)a
formatted_attrs = "%(.{#{Enum.join(attrs, ",")}})j" formatted_attrs = "%(.{#{Enum.join(attrs, ",")}})j"
@@ -153,7 +153,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
"id" => "TiZPUDkDYbk", "id" => "TiZPUDkDYbk",
"title" => "Trying to Wheelie Without the Rear Brake", "title" => "Trying to Wheelie Without the Rear Brake",
"description" => "I'm not sure what I expected.", "description" => "I'm not sure what I expected.",
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"live_status" => "not_live", "live_status" => "not_live",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 60, "duration" => 60,
@@ -177,7 +177,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "sets short_form_content to true if the URL contains /shorts/" do test "sets short_form_content to true if the URL contains /shorts/" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk", "original_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -188,7 +188,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "sets short_form_content to true if the aspect ratio are duration are right" do test "sets short_form_content to true if the aspect ratio are duration are right" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 0.5, "aspect_ratio" => 0.5,
"duration" => 59, "duration" => 59,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -199,7 +199,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "sets short_form_content to false otherwise" do test "sets short_form_content to false otherwise" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -210,7 +210,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "doesn't blow up if short form content-related fields are missing" do test "doesn't blow up if short form content-related fields are missing" do
response = %{ response = %{
"webpage_url" => nil, "original_url" => nil,
"aspect_ratio" => nil, "aspect_ratio" => nil,
"duration" => nil, "duration" => nil,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -221,7 +221,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "parses the duration" do test "parses the duration" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 60.4, "duration" => 60.4,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -232,7 +232,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "doesn't blow up if duration is missing" do test "doesn't blow up if duration is missing" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => nil, "duration" => nil,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -243,7 +243,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "sets livestream to false if the live_status field isn't present" do test "sets livestream to false if the live_status field isn't present" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 60, "duration" => 60,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -254,7 +254,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "doesn't blow up if playlist_index is missing" do test "doesn't blow up if playlist_index is missing" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => nil, "duration" => nil,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -267,7 +267,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
describe "response_to_struct/1 when testing uploaded_at" do describe "response_to_struct/1 when testing uploaded_at" do
test "parses the upload date from the timestamp if present" do test "parses the upload date from the timestamp if present" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => "20210101", "upload_date" => "20210101",
@@ -281,7 +281,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "parses the upload date from the uploaded_at if timestamp is present but nil" do test "parses the upload date from the uploaded_at if timestamp is present but nil" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => "20210101", "upload_date" => "20210101",
@@ -295,7 +295,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "parses the upload date from the uploaded_at if timestamp absent" do test "parses the upload date from the uploaded_at if timestamp absent" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => "20210101" "upload_date" => "20210101"
@@ -308,7 +308,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
test "doesn't blow up if upload date is missing" do test "doesn't blow up if upload date is missing" do
response = %{ response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk", "original_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0, "aspect_ratio" => 1.0,
"duration" => 61, "duration" => 61,
"upload_date" => nil "upload_date" => nil
test/support/fixtures/media_fixtures.ex
+1 -1
View File
@@ -95,7 +95,7 @@ defmodule Pinchflat.MediaFixtures do
media_attributes = %{ media_attributes = %{
id: "video1", id: "video1",
title: "Video 1", title: "Video 1",
webpage_url: "https://example.com/video1", original_url: "https://example.com/video1",
live_status: "not_live", live_status: "not_live",
description: "desc1", description: "desc1",
aspect_ratio: 1.67, aspect_ratio: 1.67,
test/support/fixtures/sources_fixtures.ex
+3 -3
View File
@@ -80,7 +80,7 @@ defmodule Pinchflat.SourcesFixtures do
%{ %{
id: "video1", id: "video1",
title: "Video 1", title: "Video 1",
webpage_url: "https://example.com/video1", original_url: "https://example.com/video1",
live_status: "not_live", live_status: "not_live",
description: "desc1", description: "desc1",
aspect_ratio: 1.67, aspect_ratio: 1.67,
@@ -90,7 +90,7 @@ defmodule Pinchflat.SourcesFixtures do
%{ %{
id: "video2", id: "video2",
title: "Video 2", title: "Video 2",
webpage_url: "https://example.com/video2", original_url: "https://example.com/video2",
live_status: "is_live", live_status: "is_live",
description: "desc2", description: "desc2",
aspect_ratio: 1.67, aspect_ratio: 1.67,
@@ -100,7 +100,7 @@ defmodule Pinchflat.SourcesFixtures do
%{ %{
id: "video3", id: "video3",
title: "Video 3", title: "Video 3",
webpage_url: "https://example.com/video3", original_url: "https://example.com/video3",
live_status: "not_live", live_status: "not_live",
description: "desc3", description: "desc3",
aspect_ratio: 1.0, aspect_ratio: 1.0,