RSS feed for sources (#110)

* Add media streaming (#108)

* [WIP] set up streaming route

* Added UUID to sources and media items

* Added media preview to MI show page

* Added plug to strip file extensions

* [VERY WIP] basic podcast RSS setup

* [WIP] got basic podcast RSS working

* [WIP] more expanding on RSS

* Comment

* [WIP] Working on refactoring feed

* Added UUID backfill to a migration

* [WIP] Moar refactoring

* [WIP] Adding UI for getting RSS feed

* Many tests

* Added conditional routing for feed URLs

* Removed the need for url_base to be set

* Updated preset name

* Rendered certain fields HTML-safe; Added logging to confirm range request support

* Fixed incorrect scheme issue

* Updated env var

* Updated other UI to use dropdown

* removed commented code

* Generate rss feeds (#123)

* Added plug to strip file extensions

* [VERY WIP] basic podcast RSS setup

* [WIP] got basic podcast RSS working

* [WIP] more expanding on RSS

* [WIP] Working on refactoring feed

* Added UUID backfill to a migration

* [WIP] Moar refactoring

* [WIP] Adding UI for getting RSS feed

* Many tests

* Added conditional routing for feed URLs

* Removed the need for url_base to be set

* Updated preset name

* Rendered certain fields HTML-safe; Added logging to confirm range request support

* Fixed incorrect scheme issue

* Updated env var

* Updated other UI to use dropdown

* removed commented code

* docs

* Added unique index to UUID fields
This commit is contained in:
Kieran
2024-03-27 18:11:25 -07:00
committed by GitHub
parent d043e5574b
commit 320b25f5b6
38 changed files with 1237 additions and 169 deletions
+31
View File
@@ -398,6 +398,37 @@ defmodule Pinchflat.MediaTest do
assert media_item.media_filepath == valid_attrs.media_filepath
end
test "automatically sets the UUID" do
valid_attrs = %{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today()
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
assert String.length(media_item.uuid) == 36
end
test "UUID is not writable by the user" do
valid_attrs = %{
media_id: Faker.String.base64(12),
title: Faker.Commerce.product_name(),
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today(),
uuid: "some-uuid"
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
assert String.length(media_item.uuid) == 36
end
test "creating with invalid data returns error changeset" do
assert {:error, %Ecto.Changeset{}} = Media.create_media_item(@invalid_attrs)
end
@@ -57,7 +57,7 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorkerTest do
_url, _opts, ot when ot == @metadata_ot -> {:ok, render_metadata(:channel_source_metadata)}
end)
source = source_fixture()
source = source_fixture(%{description: nil})
refute source.description
perform_job(SourceMetadataStorageWorker, %{id: source.id})
@@ -0,0 +1,64 @@
defmodule Pinchflat.Podcasts.PodcastHelpersTest do
use Pinchflat.DataCase
import Pinchflat.SourcesFixtures
import Pinchflat.MediaFixtures
alias Pinchflat.Podcasts.PodcastHelpers
describe "persisted_media_items_for/2" do
test "returns media items with files that exist on-disk" do
source = source_fixture()
good_media = media_item_with_attachments(%{source_id: source.id})
_bad_media = media_item_fixture(%{source_id: source.id, media_filepath: "/tmp/existing_file.mp3"})
assert [persisted_media] = PodcastHelpers.persisted_media_items_for(source)
assert persisted_media.id == good_media.id
end
test "lets you specify a limit" do
source = source_fixture()
_good_media = media_item_with_attachments(%{source_id: source.id})
assert [] = PodcastHelpers.persisted_media_items_for(source, limit: 0)
end
end
describe "select_cover_image/2" do
test "returns a source's poster, if present" do
source = source_with_metadata_attachments()
{:ok, res} = PodcastHelpers.select_cover_image(source, [])
assert res == source.metadata.poster_filepath
end
test "falls back to a source's fanart, if present" do
source = source_with_metadata_attachments()
File.rm(source.metadata.poster_filepath)
{:ok, res} = PodcastHelpers.select_cover_image(source, [])
assert res == source.metadata.fanart_filepath
end
test "falls back to a media item's thumbnail, if present" do
source = source_with_metadata_attachments()
media_item = media_item_with_metadata_attachments(%{source_id: source.id})
File.rm(source.metadata.poster_filepath)
File.rm(source.metadata.fanart_filepath)
{:ok, res} = PodcastHelpers.select_cover_image(source, [media_item])
assert res == media_item.metadata.thumbnail_filepath
end
test "returns error if no artwork can be found" do
source = source_fixture()
assert PodcastHelpers.select_cover_image(source, []) == {:error, :no_suitable_image}
end
end
end
@@ -0,0 +1,145 @@
defmodule Pinchflat.Podcasts.RssFeedBuilderTest do
use Pinchflat.DataCase
import Pinchflat.MediaFixtures
import Pinchflat.SourcesFixtures
alias Pinchflat.Podcasts.RssFeedBuilder
@datetime_format "%a, %d %b %Y %H:%M:%S %z"
setup do
source = source_fixture()
{:ok, source: source}
end
describe "build/2" do
test "returns an XML document", %{source: source} do
res = RssFeedBuilder.build(source)
assert String.contains?(res, ~s(<?xml version="1.0" encoding="UTF-8"?>))
end
test "escapes illegal characters" do
source = source_fixture(%{custom_name: "A & B"})
res = RssFeedBuilder.build(source)
assert String.contains?(res, ~s(<title>A &amp; B</title>))
end
test "can optionally apply a limit to media items", %{source: source} do
good_media = media_item_with_attachments(%{source_id: source.id})
res = RssFeedBuilder.build(source, limit: 0)
refute String.contains?(res, ~s(<title>#{good_media.title}</title>))
end
test "can optionally specify a URL base", %{source: source} do
res = RssFeedBuilder.build(source, url_base: "http://example.com")
assert String.contains?(res, ~s(http://example.com/sources/#{source.uuid}/feed.xml))
end
end
describe "build/2 when testing source XML" do
test "returns XML for static source attributes", %{source: source} do
res = RssFeedBuilder.build(source)
assert String.contains?(res, ~s(<title>#{source.custom_name}</title>))
assert String.contains?(res, ~s(<link>#{source.original_url}</link>))
assert String.contains?(res, ~s(<description>#{source.description}</description>))
assert String.contains?(res, ~s(<itunes:author>#{source.custom_name}</itunes:author>))
assert String.contains?(res, ~s(<itunes:subtitle>#{source.custom_name}</itunes:subtitle>))
assert String.contains?(res, ~s(<description>#{source.description}</description>))
assert String.contains?(res, ~s(<podcast:guid>#{source.uuid}</podcast:guid>))
end
test "returns the lastBuildDate and pubDate based off the source's timestamps", %{source: source} do
res = RssFeedBuilder.build(source)
assert String.contains?(res, ~s(<lastBuildDate>#{format_date(source.updated_at)}</lastBuildDate>))
assert String.contains?(res, ~s(<pubDate>#{format_date(source.inserted_at)}</pubDate>))
end
test "returns a self-link", %{source: source} do
res = RssFeedBuilder.build(source)
assert String.contains?(
res,
~s(<atom:link href="http://localhost:8945/sources/#{source.uuid}/feed.xml" rel="self" type="application/rss+xml" />)
)
end
test "returns a link to the feed image" do
source = source_with_metadata_attachments()
res = RssFeedBuilder.build(source)
[_before, image_block, _after] = String.split(res, ~r(</?image>))
assert String.contains?(image_block, ~s(<url>http://localhost:8945/sources/#{source.uuid}/feed_image.jpg</url>))
assert String.contains?(image_block, ~s(<title>#{source.custom_name}</title>))
assert String.contains?(image_block, ~s(<link>#{source.original_url}</link>))
assert String.contains?(
res,
~s(<itunes:image href="http://localhost:8945/sources/#{source.uuid}/feed_image.jpg"></itunes:image>)
)
end
end
describe "build/2 when testing media XML" do
test "only includes media persisted to disk", %{source: source} do
good_media = media_item_with_attachments(%{source_id: source.id})
bad_media = media_item_fixture(%{source_id: source.id, media_filepath: "/tmp/existing_file.mp3"})
pending_media = media_item_fixture(%{source_id: source.id, media_filepath: nil})
res = RssFeedBuilder.build(source)
assert String.contains?(res, ~s(<title>#{good_media.title}</title>))
refute String.contains?(res, ~s(<title>#{bad_media.title}</title>))
refute String.contains?(res, ~s(<title>#{pending_media.title}</title>))
end
test "returns XML for static media attributes", %{source: source} do
media_item = media_item_with_attachments(%{source_id: source.id})
res = RssFeedBuilder.build(source)
[_before, item_xml, _after] = String.split(res, ~r(</?item>))
assert String.contains?(item_xml, ~s(<guid isPermaLink="false">#{media_item.uuid}</guid>))
assert String.contains?(item_xml, ~s(<title>#{media_item.title}</title>))
assert String.contains?(item_xml, ~s(<link>#{media_item.original_url}</link>))
assert String.contains?(item_xml, ~s(<description>#{media_item.description}</description>))
assert String.contains?(item_xml, ~s(<itunes:author>#{source.custom_name}</itunes:author>))
assert String.contains?(item_xml, ~s(<itunes:subtitle>#{media_item.title}</itunes:subtitle>))
assert String.contains?(item_xml, ~s(<itunes:summary><![CDATA[#{media_item.description}]]></itunes:summary>))
end
test "returns pubDate based off the media's upload_date", %{source: source} do
media_item_with_attachments(%{source_id: source.id, upload_date: ~D[2020-01-01]})
res = RssFeedBuilder.build(source)
[_before, item_xml, _after] = String.split(res, ~r(</?item>))
assert String.contains?(item_xml, ~s(<pubDate>Wed, 01 Jan 2020 00:00:00 +0000</pubDate>))
end
test "returns an enclosure tag with the media's stream URL", %{source: source} do
media_item = media_item_with_attachments(%{source_id: source.id, media_size_bytes: 1234})
res = RssFeedBuilder.build(source)
[_before, item_xml, _after] = String.split(res, ~r(</?item>))
assert String.contains?(item_xml, ~s(<enclosure))
assert String.contains?(item_xml, ~s(url="http://localhost:8945/media/#{media_item.uuid}/stream.mp4"))
assert String.contains?(item_xml, ~s(length="1234"))
assert String.contains?(item_xml, ~s(type="video/mp4"))
end
end
defp format_date(date) do
Calendar.strftime(date, @datetime_format)
end
end
+25
View File
@@ -59,6 +59,31 @@ defmodule Pinchflat.SourcesTest do
end
describe "create_source/2" do
test "automatically sets the UUID" do
expect(YtDlpRunnerMock, :run, &channel_mock/3)
valid_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/channel/abc123"
}
assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
assert String.length(source.uuid) == 36
end
test "UUID is not writable by the user" do
expect(YtDlpRunnerMock, :run, &channel_mock/3)
valid_attrs = %{
media_profile_id: media_profile_fixture().id,
original_url: "https://www.youtube.com/channel/abc123",
uuid: "some_uuid"
}
assert {:ok, %Source{} = source} = Sources.create_source(valid_attrs)
assert String.length(source.uuid) == 36
end
test "creates a source and adds name + ID from runner response for channels" do
expect(YtDlpRunnerMock, :run, &channel_mock/3)
@@ -0,0 +1,14 @@
defmodule Pinchflat.Utils.DatetimeUtilsTest do
use ExUnit.Case, async: true
alias Pinchflat.Utils.DatetimeUtils
describe "date_to_datetime/1" do
test "converts a Date to a DateTime" do
date = ~D[2022-01-01]
datetime = DatetimeUtils.date_to_datetime(date)
assert datetime == ~U[2022-01-01 00:00:00Z]
end
end
end
@@ -66,6 +66,147 @@ defmodule PinchflatWeb.MediaItemControllerTest do
end
end
describe "streaming media" do
test "returns 404 if the media isn't found", %{conn: conn} do
media_item = media_item_fixture()
conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
assert conn.status == 404
end
test "automatically sets the content type", %{conn: conn} do
media_item = media_item_with_attachments()
conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
assert {"content-type", "video/mp4; charset=utf-8"} in conn.resp_headers
end
test "sets the content length", %{conn: conn} do
media_item = media_item_with_attachments()
filesize = File.stat!(media_item.media_filepath).size
conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
assert {"content-length", to_string(filesize)} in conn.resp_headers
end
end
describe "streaming media when range is valid" do
setup do
media_item = media_item_with_attachments()
%{media_item: media_item}
end
test "sets the correct status and headers", %{conn: conn, media_item: media_item} do
filesize = File.stat!(media_item.media_filepath).size
conn =
conn
|> put_req_header("range", "bytes=0-100")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.status == 206
assert {"content-range", "bytes 0-100/#{filesize}"} in conn.resp_headers
assert {"content-length", "101"} in conn.resp_headers
end
test "streams the specified range", %{conn: conn, media_item: media_item} do
conn =
conn
|> put_req_header("range", "bytes=0-100")
|> get(~p"/media/#{media_item.uuid}/stream")
assert byte_size(conn.resp_body) == 101
end
test "supports range offsets", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
expected = String.slice(contents, 100..200)
conn =
conn
|> put_req_header("range", "bytes=100-200")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.resp_body == expected
end
test "returns as expected if the requested range is larger than the file", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
filesize = File.stat!(media_item.media_filepath).size
conn =
conn
|> put_req_header("range", "bytes=0-#{filesize * 10}")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.resp_body == contents
assert {"content-range", "bytes 0-#{filesize - 1}/#{filesize}"} in conn.resp_headers
assert {"content-length", to_string(filesize)} in conn.resp_headers
end
test "supports endless ranges", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
conn =
conn
|> put_req_header("range", "bytes=0-")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.resp_body == contents
end
test "supports endless ranges with offsets", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
{_, expected} = String.split_at(contents, 100)
conn =
conn
|> put_req_header("range", "bytes=100-")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.resp_body == expected
end
end
describe "streaming media when range is invalid or not present" do
setup do
media_item = media_item_with_attachments()
%{media_item: media_item}
end
test "sets the correct status and headers", %{conn: conn, media_item: media_item} do
filesize = File.stat!(media_item.media_filepath).size
conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
assert conn.status == 200
assert {"content-length", to_string(filesize)} in conn.resp_headers
end
test "streams the entire file", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
conn = get(conn, ~p"/media/#{media_item.uuid}/stream")
assert conn.resp_body == contents
end
test "doesn't blow up if the range header is invalid", %{conn: conn, media_item: media_item} do
contents = File.read!(media_item.media_filepath)
conn =
conn
|> put_req_header("range", "bytes=-")
|> get(~p"/media/#{media_item.uuid}/stream")
assert conn.status == 200
assert conn.resp_body == contents
end
end
defp create_media_item(_) do
media_item = media_item_fixture()
%{media_item: media_item}
@@ -0,0 +1,38 @@
defmodule PinchflatWeb.PodcastControllerTest do
use PinchflatWeb.ConnCase
import Pinchflat.SourcesFixtures
describe "rss_feed" do
test "renders the XML document", %{conn: conn} do
source = source_fixture()
conn = get(conn, ~p"/sources/#{source.uuid}/feed" <> ".xml")
assert conn.status == 200
assert {"content-type", "application/rss+xml; charset=utf-8"} in conn.resp_headers
assert {"content-disposition", "inline"} in conn.resp_headers
end
end
describe "feed_image" do
test "returns a feed image if one can be found", %{conn: conn} do
source = source_with_metadata_attachments()
conn = get(conn, ~p"/sources/#{source.uuid}/feed_image" <> ".jpg")
assert conn.status == 200
assert {"content-type", "image/jpeg; charset=utf-8"} in conn.resp_headers
assert conn.resp_body == File.read!(source.metadata.poster_filepath)
end
test "returns 404 if an image cannot be found", %{conn: conn} do
source = source_fixture()
conn = get(conn, ~p"/sources/#{source.uuid}/feed_image" <> ".jpg")
assert conn.status == 404
assert conn.resp_body == "Image not found"
end
end
end
+51
View File
@@ -1,6 +1,8 @@
defmodule PinchflatWeb.RoutingTest do
use PinchflatWeb.ConnCase
import Pinchflat.SourcesFixtures
describe "basic_auth plug" do
setup do
old_username = Application.get_env(:pinchflat, :basic_auth_username)
@@ -54,4 +56,53 @@ defmodule PinchflatWeb.RoutingTest do
assert conn.status == 200
end
end
describe "maybe_basic_auth plug" do
setup do
old_username = Application.get_env(:pinchflat, :basic_auth_username)
old_password = Application.get_env(:pinchflat, :basic_auth_password)
old_expose_feed_endpoints = Application.get_env(:pinchflat, :expose_feed_endpoints)
source = source_fixture()
on_exit(fn ->
Application.put_env(:pinchflat, :basic_auth_username, old_username)
Application.put_env(:pinchflat, :basic_auth_password, old_password)
Application.put_env(:pinchflat, :expose_feed_endpoints, old_expose_feed_endpoints)
end)
{:ok, source: source}
end
test "uses basic auth when expose_feed_endpoints is false", %{source: source} do
Application.put_env(:pinchflat, :basic_auth_username, "user")
Application.put_env(:pinchflat, :basic_auth_password, "pass")
Application.put_env(:pinchflat, :expose_feed_endpoints, false)
conn = get(build_conn(), "/sources/#{source.uuid}/feed")
assert conn.status == 401
assert {"www-authenticate", "Basic realm=\"Pinchflat\""} in conn.resp_headers
end
test "does not use basic auth when expose_feed_endpoints is true", %{source: source} do
Application.put_env(:pinchflat, :basic_auth_username, "user")
Application.put_env(:pinchflat, :basic_auth_password, "pass")
Application.put_env(:pinchflat, :expose_feed_endpoints, true)
conn = get(build_conn(), "/sources/#{source.uuid}/feed")
assert conn.status == 200
end
test "does not use basic auth when username/password aren't set", %{source: source} do
Application.put_env(:pinchflat, :basic_auth_username, nil)
Application.put_env(:pinchflat, :basic_auth_password, nil)
Application.put_env(:pinchflat, :expose_feed_endpoints, false)
conn = get(build_conn(), "/sources/#{source.uuid}/feed")
assert conn.status == 200
end
end
end
+34 -5
View File
@@ -5,6 +5,7 @@ defmodule Pinchflat.MediaFixtures do
"""
alias Pinchflat.SourcesFixtures
alias Pinchflat.Filesystem.FilesystemHelpers
@doc """
Generate a media_item.
@@ -44,18 +45,36 @@ defmodule Pinchflat.MediaFixtures do
media_item_fixture(merged_attrs)
end
def media_item_with_metadata_attachments(attrs \\ %{}) do
metadata_dir =
Path.join(Application.get_env(:pinchflat, :metadata_directory), "#{:rand.uniform(1_000_000)}")
json_gz_filepath = Path.join(metadata_dir, "metadata.json.gz")
thumbnail_filepath = Path.join(metadata_dir, "thumbnail.jpg")
FilesystemHelpers.cp_p!(media_metadata_filepath_fixture(), json_gz_filepath)
FilesystemHelpers.cp_p!(thumbnail_filepath_fixture(), thumbnail_filepath)
merged_attrs =
Map.merge(attrs, %{
metadata: %{
metadata_filepath: json_gz_filepath,
thumbnail_filepath: thumbnail_filepath
}
})
media_item_with_attachments(merged_attrs)
end
def media_item_with_attachments(attrs \\ %{}) do
stored_media_filepath =
Path.join([
Application.get_env(:pinchflat, :media_directory),
"#{:rand.uniform(1_000_000)}",
"#{:rand.uniform(1_000_000)}_media.mkv"
"#{:rand.uniform(1_000_000)}_media.mp4"
])
fixture_media_filepath = media_filepath_fixture()
:ok = File.mkdir_p(Path.dirname(stored_media_filepath))
:ok = File.cp(fixture_media_filepath, stored_media_filepath)
FilesystemHelpers.cp_p!(media_filepath_fixture(), stored_media_filepath)
merged_attrs = Map.merge(attrs, %{media_filepath: stored_media_filepath})
media_item_fixture(merged_attrs)
@@ -105,4 +124,14 @@ defmodule Pinchflat.MediaFixtures do
"example.info.json"
])
end
def media_metadata_filepath_fixture do
Path.join([
File.cwd!(),
"test",
"support",
"files",
"media_metadata.json"
])
end
end
+28 -1
View File
@@ -5,8 +5,10 @@ defmodule Pinchflat.SourcesFixtures do
"""
alias Pinchflat.Repo
alias Pinchflat.ProfilesFixtures
alias Pinchflat.MediaFixtures
alias Pinchflat.Sources.Source
alias Pinchflat.ProfilesFixtures
alias Pinchflat.Filesystem.FilesystemHelpers
@doc """
Generate a source.
@@ -22,6 +24,7 @@ defmodule Pinchflat.SourcesFixtures do
collection_id: Base.encode16(:crypto.hash(:md5, "#{:rand.uniform(1_000_000)}")),
collection_type: "channel",
custom_name: "Cool and good internal name!",
description: "This is a description",
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
media_profile_id: ProfilesFixtures.media_profile_fixture().id,
index_frequency_minutes: 60
@@ -48,6 +51,30 @@ defmodule Pinchflat.SourcesFixtures do
source_fixture(merged_attrs)
end
def source_with_metadata_attachments(attrs \\ %{}) do
metadata_dir =
Path.join(Application.get_env(:pinchflat, :metadata_directory), "#{:rand.uniform(1_000_000)}")
json_gz_filepath = Path.join(metadata_dir, "metadata.json.gz")
poster_filepath = Path.join(metadata_dir, "poster.jpg")
fanart_filepath = Path.join(metadata_dir, "fanart.jpg")
FilesystemHelpers.cp_p!(MediaFixtures.media_metadata_filepath_fixture(), json_gz_filepath)
FilesystemHelpers.cp_p!(MediaFixtures.thumbnail_filepath_fixture(), poster_filepath)
FilesystemHelpers.cp_p!(MediaFixtures.thumbnail_filepath_fixture(), fanart_filepath)
merged_attrs =
Map.merge(attrs, %{
metadata: %{
metadata_filepath: json_gz_filepath,
poster_filepath: poster_filepath,
fanart_filepath: fanart_filepath
}
})
source_fixture(merged_attrs)
end
def source_attributes_return_fixture do
source_attributes = [
%{