Fast indexing (#58)
* Made method to getting singular media details; Renamed other related method * Takes a fun and flirty digression to remove abstractions around yt-dlp since I'm 100% committed to using it exclusively * Removed commented test code * Lays the groundwork for fast indexing * Added module for working with youtube RSS feed * Added methods to kick off indexing workers from RSS response * Improve short detection (#59) * Made media attribute-related yt-dlp calls return a struct * Added shorts attribute to media items * Added ability to discern a short from yt-dlp response * Updated search to use new shorts attribute * Fast index UI (#63) * Added fast_index field and adds it to source form * Added fast indexing to source changeset operations * Added fast indexing worker and updated other modules to start using it * Handled fast index worker on source update * Add support modals (#65) * Added fast indexing upgrade modal * Improved modal on smaller screens * Updated links to work again * Added donation modal * Reverted source fast index to 15 minutes * Removed unneeded HTML attributes from old alpine approach
This commit is contained in:
Kieran
GitHub
@@ -0,0 +1,11 @@
|
||||
defmodule Pinchflat.YtDlp.Backend.BackendCommandRunner do
|
||||
@moduledoc """
|
||||
A behaviour for running CLI commands against a downloader backend (yt-dlp).
|
||||
|
||||
Used so we can implement Mox for testing without actually running the
|
||||
yt-dlp command.
|
||||
"""
|
||||
|
||||
@callback run(binary(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback run(binary(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
end
|
||||
@@ -0,0 +1,82 @@
|
||||
defmodule Pinchflat.YtDlp.Backend.CommandRunner do
|
||||
@moduledoc """
|
||||
Runs yt-dlp commands using the `System.cmd/3` function
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Utils.StringUtils
|
||||
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
|
||||
alias Pinchflat.YtDlp.Backend.BackendCommandRunner
|
||||
|
||||
@behaviour BackendCommandRunner
|
||||
|
||||
@doc """
|
||||
Runs a yt-dlp command and returns the string output. Saves the output to
|
||||
a file and then returns its contents because yt-dlp will return warnings
|
||||
to stdout even if the command is successful, but these will break JSON parsing.
|
||||
|
||||
Additional Opts:
|
||||
- :output_filepath - the path to save the output to. If not provided, a temporary
|
||||
file will be created and used. Useful for if you need a reference to the file
|
||||
for a file watcher.
|
||||
|
||||
Returns {:ok, binary()} | {:error, output, status}.
|
||||
"""
|
||||
@impl BackendCommandRunner
|
||||
def run(url, command_opts, output_template, addl_opts \\ []) do
|
||||
# This approach lets us mock the command for testing
|
||||
command = backend_executable()
|
||||
# These must stay in exactly this order, hence why I'm giving it its own variable.
|
||||
# Also, can't use RAM file since yt-dlp needs a concrete filepath.
|
||||
output_filepath = Keyword.get(addl_opts, :output_filepath, FSUtils.generate_metadata_tmpfile(:json))
|
||||
print_to_file_opts = [{:print_to_file, output_template}, output_filepath]
|
||||
formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts)
|
||||
|
||||
Logger.info("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
|
||||
|
||||
case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
|
||||
{_, 0} ->
|
||||
# IDEA: consider deleting the file after reading it
|
||||
# (even on error? especially on error?)
|
||||
File.read(output_filepath)
|
||||
|
||||
{output, status} ->
|
||||
{:error, output, status}
|
||||
end
|
||||
end
|
||||
|
||||
# We want to satisfy the following behaviours:
|
||||
#
|
||||
# 1. If the key is an atom, convert it to a string and convert it to kebab case (for convenience)
|
||||
# 2. If the key is a string, assume we want it as-is and don't convert it
|
||||
# 3. If the key is accompanied by a value, append the value to the list
|
||||
# 4. If the key is not accompanied by a value, assume it's a flag and PREpend it to the list
|
||||
defp parse_options(command_opts) do
|
||||
Enum.reduce(command_opts, [], &parse_option/2)
|
||||
end
|
||||
|
||||
defp parse_option({k, v}, acc) when is_atom(k) do
|
||||
stringified_key = StringUtils.to_kebab_case(Atom.to_string(k))
|
||||
|
||||
parse_option({"--#{stringified_key}", v}, acc)
|
||||
end
|
||||
|
||||
defp parse_option({k, v}, acc) when is_binary(k) do
|
||||
acc ++ [k, to_string(v)]
|
||||
end
|
||||
|
||||
defp parse_option(arg, acc) when is_atom(arg) do
|
||||
stringified_arg = StringUtils.to_kebab_case(Atom.to_string(arg))
|
||||
|
||||
parse_option("--#{stringified_arg}", acc)
|
||||
end
|
||||
|
||||
defp parse_option(arg, acc) when is_binary(arg) do
|
||||
acc ++ [arg]
|
||||
end
|
||||
|
||||
defp backend_executable do
|
||||
Application.get_env(:pinchflat, :yt_dlp_executable)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,107 @@
|
||||
defmodule Pinchflat.YtDlp.Backend.Media do
|
||||
@moduledoc """
|
||||
Contains utilities for working with singular pieces of media
|
||||
"""
|
||||
|
||||
@enforce_keys [
|
||||
:media_id,
|
||||
:title,
|
||||
:description,
|
||||
:original_url,
|
||||
:livestream,
|
||||
:short_form_content
|
||||
]
|
||||
|
||||
defstruct [
|
||||
:media_id,
|
||||
:title,
|
||||
:description,
|
||||
:original_url,
|
||||
:livestream,
|
||||
:short_form_content
|
||||
]
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Utils.FunctionUtils
|
||||
|
||||
@doc """
|
||||
Downloads a single piece of media (and possibly its metadata) directly to its
|
||||
final destination. Returns the parsed JSON output from yt-dlp.
|
||||
|
||||
Returns {:ok, map()} | {:error, any, ...}.
|
||||
"""
|
||||
def download(url, command_opts \\ []) do
|
||||
opts = [:no_simulate] ++ command_opts
|
||||
|
||||
with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j"),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
|
||||
{:ok, parsed_json}
|
||||
else
|
||||
err -> err
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a map representing the media at the given URL.
|
||||
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_media_attributes(url) do
|
||||
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
command_opts = [:simulate, :skip_download]
|
||||
output_template = indexing_output_template()
|
||||
|
||||
case runner.run(url, command_opts, output_template) do
|
||||
{:ok, output} ->
|
||||
output
|
||||
|> Phoenix.json_library().decode!()
|
||||
|> response_to_struct()
|
||||
|> FunctionUtils.wrap_ok()
|
||||
|
||||
res ->
|
||||
res
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the output template for yt-dlp's indexing command.
|
||||
"""
|
||||
def indexing_output_template do
|
||||
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j"
|
||||
end
|
||||
|
||||
@doc """
|
||||
Transforms a response from yt-dlp into a struct. Interprets the response to
|
||||
determine if the media is short-form content.
|
||||
|
||||
Returns %Media{}.
|
||||
"""
|
||||
def response_to_struct(response) do
|
||||
%Media{
|
||||
media_id: response["id"],
|
||||
title: response["title"],
|
||||
description: response["description"],
|
||||
original_url: response["webpage_url"],
|
||||
livestream: response["was_live"],
|
||||
short_form_content: short_form_content?(response)
|
||||
}
|
||||
end
|
||||
|
||||
defp short_form_content?(response) do
|
||||
if String.contains?(response["webpage_url"], "/shorts/") do
|
||||
true
|
||||
else
|
||||
# Sometimes shorts are returned without /shorts/ in the URL,
|
||||
# so we need to do our best to determine if it's a short. This
|
||||
# WILL returns false positives, but it's a best-effort approach
|
||||
# that should work for most cases. The aspect_ratio check is
|
||||
# based on a gut feeling and may need to be tweaked.
|
||||
response["duration"] <= 60 && response["aspect_ratio"] < 0.8
|
||||
end
|
||||
end
|
||||
|
||||
defp backend_runner do
|
||||
# This approach lets us mock the command for testing
|
||||
Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,81 @@
|
||||
defmodule Pinchflat.YtDlp.Backend.MediaCollection do
|
||||
@moduledoc """
|
||||
Contains utilities for working with collections of
|
||||
media (aka: a source [ie: channels, playlists]).
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Utils.FunctionUtils
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
|
||||
|
||||
@doc """
|
||||
Returns a list of maps representing the media in the collection.
|
||||
|
||||
Options:
|
||||
- :file_listener_handler - a function that will be called with the path to the
|
||||
file that will be written to when yt-dlp is done. This is useful for
|
||||
setting up a file watcher to know when the file is ready to be read.
|
||||
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_media_attributes_for_collection(url, addl_opts \\ []) do
|
||||
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
command_opts = [:simulate, :skip_download]
|
||||
output_template = YtDlpMedia.indexing_output_template()
|
||||
output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
|
||||
file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
|
||||
|
||||
if file_listener_handler do
|
||||
file_listener_handler.(output_filepath)
|
||||
end
|
||||
|
||||
case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
|
||||
{:ok, output} ->
|
||||
output
|
||||
|> String.split("\n", trim: true)
|
||||
|> Enum.map(&Phoenix.json_library().decode!/1)
|
||||
|> Enum.map(&YtDlpMedia.response_to_struct/1)
|
||||
|> FunctionUtils.wrap_ok()
|
||||
|
||||
res ->
|
||||
res
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Gets a source's ID and name from its URL.
|
||||
|
||||
yt-dlp does not _really_ have source-specific functions, so
|
||||
instead we're fetching just the first video (using playlist_end: 1)
|
||||
and parsing the source ID and name from _its_ metadata
|
||||
|
||||
Returns {:ok, map()} | {:error, any, ...}.
|
||||
"""
|
||||
def get_source_details(source_url) do
|
||||
opts = [:simulate, :skip_download, playlist_end: 1]
|
||||
output_template = "%(.{channel,channel_id,playlist_id,playlist_title})j"
|
||||
|
||||
with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
|
||||
{:ok, format_source_details(parsed_json)}
|
||||
else
|
||||
err -> err
|
||||
end
|
||||
end
|
||||
|
||||
defp format_source_details(response) do
|
||||
%{
|
||||
channel_id: response["channel_id"],
|
||||
channel_name: response["channel"],
|
||||
playlist_id: response["playlist_id"],
|
||||
playlist_name: response["playlist_title"]
|
||||
}
|
||||
end
|
||||
|
||||
defp backend_runner do
|
||||
# This approach lets us mock the command for testing
|
||||
Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,131 @@
|
||||
defmodule Pinchflat.YtDlp.DownloadOptionBuilder do
|
||||
@moduledoc """
|
||||
Builds the options for yt-dlp to download media based on the given media profile.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Profiles.OutputPathBuilder
|
||||
|
||||
@doc """
|
||||
Builds the options for yt-dlp to download media based on the given media's profile.
|
||||
|
||||
IDEA: consider adding the ability to pass in a second argument to override
|
||||
these options
|
||||
"""
|
||||
def build(%MediaItem{} = media_item_with_preloads) do
|
||||
media_profile = media_item_with_preloads.source.media_profile
|
||||
|
||||
built_options =
|
||||
default_options() ++
|
||||
subtitle_options(media_profile) ++
|
||||
thumbnail_options(media_profile) ++
|
||||
metadata_options(media_profile) ++
|
||||
quality_options(media_profile) ++
|
||||
output_options(media_item_with_preloads)
|
||||
|
||||
{:ok, built_options}
|
||||
end
|
||||
|
||||
defp default_options do
|
||||
[:no_progress, :windows_filenames]
|
||||
end
|
||||
|
||||
defp subtitle_options(media_profile) do
|
||||
mapped_struct = Map.from_struct(media_profile)
|
||||
|
||||
Enum.reduce(mapped_struct, [], fn attr, acc ->
|
||||
case {attr, media_profile} do
|
||||
{{:download_subs, true}, _} ->
|
||||
# Force SRT for now - MAY provide as an option in the future
|
||||
acc ++ [:write_subs, convert_subs: "srt"]
|
||||
|
||||
{{:download_auto_subs, true}, %{download_subs: true}} ->
|
||||
acc ++ [:write_auto_subs]
|
||||
|
||||
{{:embed_subs, true}, %{preferred_resolution: pr}} when pr != :audio ->
|
||||
acc ++ [:embed_subs]
|
||||
|
||||
{{:sub_langs, sub_langs}, %{download_subs: true}} ->
|
||||
acc ++ [sub_langs: sub_langs]
|
||||
|
||||
{{:sub_langs, sub_langs}, %{embed_subs: true}} ->
|
||||
acc ++ [sub_langs: sub_langs]
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp thumbnail_options(media_profile) do
|
||||
mapped_struct = Map.from_struct(media_profile)
|
||||
|
||||
Enum.reduce(mapped_struct, [], fn attr, acc ->
|
||||
case {attr, media_profile} do
|
||||
{{:download_thumbnail, true}, _} ->
|
||||
acc ++ [:write_thumbnail]
|
||||
|
||||
{{:embed_thumbnail, true}, %{preferred_resolution: pr}} when pr != :audio ->
|
||||
acc ++ [:embed_thumbnail]
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp metadata_options(media_profile) do
|
||||
mapped_struct = Map.from_struct(media_profile)
|
||||
|
||||
Enum.reduce(mapped_struct, [], fn attr, acc ->
|
||||
case {attr, media_profile} do
|
||||
{{:download_metadata, true}, _} ->
|
||||
acc ++ [:write_info_json, :clean_info_json]
|
||||
|
||||
{{:embed_metadata, true}, %{preferred_resolution: pr}} when pr != :audio ->
|
||||
acc ++ [:embed_metadata]
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp quality_options(media_profile) do
|
||||
codec_options = "+codec:avc:m4a"
|
||||
|
||||
case media_profile.preferred_resolution do
|
||||
# Also be aware that :audio disabled all embedding options for thumbnails, subtitles, and metadata
|
||||
:audio -> [format_sort: "ext", format: "bestaudio"]
|
||||
:"360p" -> [format_sort: "res:360,#{codec_options}"]
|
||||
:"480p" -> [format_sort: "res:480,#{codec_options}"]
|
||||
:"720p" -> [format_sort: "res:720,#{codec_options}"]
|
||||
:"1080p" -> [format_sort: "res:1080,#{codec_options}"]
|
||||
:"1440p" -> [format_sort: "res:1440,#{codec_options}"]
|
||||
:"2160p" -> [format_sort: "res:2160,#{codec_options}"]
|
||||
end
|
||||
end
|
||||
|
||||
defp output_options(media_item_with_preloads) do
|
||||
media_profile = media_item_with_preloads.source.media_profile
|
||||
additional_options_map = output_options_map(media_item_with_preloads)
|
||||
{:ok, output_path} = OutputPathBuilder.build(media_profile.output_path_template, additional_options_map)
|
||||
|
||||
[
|
||||
output: Path.join(base_directory(), output_path)
|
||||
]
|
||||
end
|
||||
|
||||
defp output_options_map(media_item_with_preloads) do
|
||||
source = media_item_with_preloads.source
|
||||
|
||||
%{
|
||||
"source_custom_name" => source.custom_name,
|
||||
"source_collection_type" => source.collection_type
|
||||
}
|
||||
end
|
||||
|
||||
defp base_directory do
|
||||
Application.get_env(:pinchflat, :media_directory)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,65 @@
|
||||
defmodule Pinchflat.MediaClient.MediaDownloader do
|
||||
@moduledoc """
|
||||
This is the integration layer for actually downloading media.
|
||||
It takes into account the media profile's settings in order
|
||||
to download the media with the desired options.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Media.MediaItem
|
||||
|
||||
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
|
||||
alias Pinchflat.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
|
||||
alias Pinchflat.Metadata.MetadataParser, as: YtDlpMetadataParser
|
||||
alias Pinchflat.Metadata.MetadataFileHelpers, as: YtDlpMetadataHelpers
|
||||
|
||||
@doc """
|
||||
Downloads media for a media item, updating the media item based on the metadata
|
||||
returned by yt-dlp. Also saves the entire metadata response to the associated
|
||||
media_metadata record.
|
||||
|
||||
NOTE: related methods (like the download worker) won't download if the media item's source
|
||||
is set to not download media. However, I'm not enforcing that here since I need this for testing.
|
||||
This may change in the future but I'm not stressed.
|
||||
|
||||
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
|
||||
"""
|
||||
def download_for_media_item(%MediaItem{} = media_item) do
|
||||
item_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
|
||||
|
||||
case download_with_options(media_item.original_url, item_with_preloads) do
|
||||
{:ok, parsed_json} ->
|
||||
{parser, helpers} = {YtDlpMetadataParser, YtDlpMetadataHelpers}
|
||||
|
||||
parsed_attrs =
|
||||
parsed_json
|
||||
|> parser.parse_for_media_item()
|
||||
|> Map.merge(%{
|
||||
media_downloaded_at: DateTime.utc_now(),
|
||||
metadata: %{
|
||||
metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
|
||||
thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
|
||||
}
|
||||
})
|
||||
|
||||
# Don't forgor to use preloaded associations or updates to
|
||||
# associations won't work!
|
||||
Media.update_media_item(item_with_preloads, parsed_attrs)
|
||||
|
||||
err ->
|
||||
err
|
||||
end
|
||||
end
|
||||
|
||||
# def download_for_source(source, url) do
|
||||
# # Create MI from source and URL
|
||||
# media_item = nil
|
||||
# end
|
||||
|
||||
defp download_with_options(url, item_with_preloads) do
|
||||
{:ok, options} = YtDlpDownloadOptionBuilder.build(item_with_preloads)
|
||||
|
||||
YtDlpMedia.download(url, options)
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user