hermes-agent/pinchflat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fast indexing (#58)

Browse Source 
* Made method to getting singular media details; Renamed other related method

* Takes a fun and flirty digression to remove abstractions around yt-dlp since I'm 100% committed to using it exclusively

* Removed commented test code

* Lays the groundwork for fast indexing

* Added module for working with youtube RSS feed

* Added methods to kick off indexing workers from RSS response

* Improve short detection (#59)

* Made media attribute-related yt-dlp calls return a struct

* Added shorts attribute to media items

* Added ability to discern a short from yt-dlp response

* Updated search to use new shorts attribute

* Fast index UI (#63)

* Added fast_index field and adds it to source form

* Added fast indexing to source changeset operations

* Added fast indexing worker and updated other modules to start using it

* Handled fast index worker on source update

* Add support modals (#65)

* Added fast indexing upgrade modal

* Improved modal on smaller screens

* Updated links to work again

* Added donation modal

* Reverted source fast index to 15 minutes

* Removed unneeded HTML attributes from old alpine approach
This commit is contained in:
Kieran
2024-03-10 14:36:34 -07:00
committed by GitHub
parent 128485c30a
commit e1f8e94686
72 changed files with 1698 additions and 626 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/pinchflat/tasks/media_item_tasks.ex
+37
View File
@@ -6,6 +6,11 @@ defmodule Pinchflat.Tasks.MediaItemTasks do
do is also defined here. Essentially, a one-stop-shop for media-related tasks/workers.
"""
alias Pinchflat.Media
alias Pinchflat.Tasks
alias Pinchflat.Sources.Source
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
@doc """
Fetches the file size of a media item and saves it to the database.
@@ -21,4 +26,36 @@ defmodule Pinchflat.Tasks.MediaItemTasks do
err
end
end
@doc """
Indexes a single media item for a source and enqueues a download job if the
media should be downloaded. This method creates the media item record so it's
the one-stop-shop for adding a media item (and possibly downloading it) just
by a URL and source.
Returns {:ok, media_item} | {:error, any()}
"""
def index_and_enqueue_download_for_media_item(%Source{} = source, url) do
maybe_media_item = create_media_item_from_url(source, url)
case maybe_media_item do
{:ok, media_item} ->
if source.download_media && Media.pending_download?(media_item) do
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end
{:ok, media_item}
err ->
err
end
end
defp create_media_item_from_url(source, url) do
{:ok, media_attrs} = YtDlpMedia.get_media_attributes(url)
Media.create_media_item_from_backend_attrs(source, media_attrs)
end
end
lib/pinchflat/tasks/source_tasks.ex
+62 -31
View File
@@ -12,31 +12,72 @@ defmodule Pinchflat.Tasks.SourceTasks do
alias Pinchflat.Tasks
alias Pinchflat.Sources
alias Pinchflat.Sources.Source
alias Pinchflat.Api.YoutubeRss
alias Pinchflat.Media.MediaItem
alias Pinchflat.MediaClient.SourceDetails
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.Workers.FastIndexingWorker
alias Pinchflat.Workers.MediaDownloadWorker
alias Pinchflat.Workers.MediaIndexingWorker
alias Pinchflat.YtDlp.Backend.MediaCollection
alias Pinchflat.Workers.MediaCollectionIndexingWorker
alias Pinchflat.Utils.FilesystemUtils.FileFollowerServer
alias Pinchflat.YtDlp.Backend.Media, as: YtDlpMedia
@doc """
Starts tasks for indexing a source's media regardless of the source's indexing
frequency. It's assumed the caller will check for that.
frequency. It's assumed the caller will check for indexing frequency.
Returns {:ok, %Task{}}.
"""
def kickoff_indexing_task(%Source{} = source) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaIndexingWorker")
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
source
|> Map.take([:id])
%{id: source.id}
# Schedule this one immediately, but future ones will be on an interval
|> MediaIndexingWorker.new()
|> MediaCollectionIndexingWorker.new()
|> Tasks.create_job_with_task(source)
|> case do
# This should never return {:error, :duplicate_job} since we just deleted
# any pending tasks. I'm being assertive about it so it's obvious if I'm wrong
{:ok, task} -> {:ok, task}
end
end
@doc """
Starts tasks for running a fast indexing task for a source's media
regardless of the source's fast_index state. It's assumed the
caller will check for fast_index.
This is used for running fast index tasks on update. On creation, the
fast index is enqueued after the slow index is complete.
Returns {:ok, %Task{}}.
"""
def kickoff_fast_indexing_task(%Source{} = source) do
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
%{id: source.id}
# Schedule this one immediately, but future ones will be on an interval
|> FastIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end
@doc """
Fetches new media IDs from a source's YouTube RSS feed and kicks off indexing tasks
for any new media items. See comments in `MediaIndexingWorker` for more info on the
order of operations and how this fits into the indexing process.
Returns :ok
"""
def kickoff_indexing_tasks_from_youtube_rss_feed(%Source{} = source) do
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
existing_media_items = Media.list_media_items_by_media_id_for(source, media_ids)
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
Enum.each(new_media_ids, fn media_id ->
url = "https://www.youtube.com/watch?v=#{media_id}"
%{id: source.id, media_url: url}
|> MediaIndexingWorker.new()
|> Tasks.create_job_with_task(source)
end)
end
@doc """
@@ -65,9 +106,9 @@ defmodule Pinchflat.Tasks.SourceTasks do
def index_and_enqueue_download_for_media_items(%Source{} = source) do
# See the method definition below for more info on how file watchers work
# (important reading if you're not familiar with it)
{:ok, media_attributes} = get_media_attributes_and_setup_file_watcher(source)
{:ok, media_attributes} = get_media_attributes_for_collection_and_setup_file_watcher(source)
result = Enum.map(media_attributes, fn media_attrs -> create_media_item_from_attributes(source, media_attrs) end)
Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
enqueue_pending_media_tasks(source)
@@ -89,8 +130,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
source
|> Media.list_pending_media_items_for()
|> Enum.each(fn media_item ->
media_item
|> Map.take([:id])
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end)
@@ -116,7 +156,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
# lines (ie: you should gracefully fail if you can't parse a line).
#
# This works in-tandem with the normal (blocking) media indexing behaviour. When
# the `get_media_attributes` method completes it'll return the FULL result to
# the `get_media_attributes_for_collection` method completes it'll return the FULL result to
# the caller for parsing. Ideally, every item in the list will have already
# been processed by the file follower, but if not, the caller handles creation
# of any media items that were missed/initially failed.
@@ -124,11 +164,11 @@ defmodule Pinchflat.Tasks.SourceTasks do
# It attempts a graceful shutdown of the file follower after the indexing is done,
# but the FileFollowerServer will also stop itself if it doesn't see any activity
# for a sufficiently long time.
defp get_media_attributes_and_setup_file_watcher(source) do
defp get_media_attributes_for_collection_and_setup_file_watcher(source) do
{:ok, pid} = FileFollowerServer.start_link()
handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end
result = SourceDetails.get_media_attributes(source.original_url, file_listener_handler: handler)
result = MediaCollection.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler)
FileFollowerServer.stop(pid)
@@ -141,7 +181,8 @@ defmodule Pinchflat.Tasks.SourceTasks do
{:ok, media_attrs} ->
Logger.debug("FileFollowerServer Handler: Got media attributes: #{inspect(media_attrs)}")
create_media_item_and_enqueue_download(source, media_attrs)
media_struct = YtDlpMedia.response_to_struct(media_attrs)
create_media_item_and_enqueue_download(source, media_struct)
err ->
Logger.debug("FileFollowerServer Handler: Error decoding JSON: #{inspect(err)}")
@@ -159,8 +200,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
if source.download_media && Media.pending_download?(media_item) do
Logger.debug("FileFollowerServer Handler: Enqueuing download task for #{inspect(media_attrs)}")
media_item
|> Map.take([:id])
%{id: media_item.id}
|> MediaDownloadWorker.new()
|> Tasks.create_job_with_task(media_item)
end
@@ -171,16 +211,7 @@ defmodule Pinchflat.Tasks.SourceTasks do
end
defp create_media_item_from_attributes(source, media_attrs) do
attrs = %{
source_id: source.id,
title: media_attrs["title"],
media_id: media_attrs["id"],
original_url: media_attrs["original_url"],
livestream: media_attrs["was_live"],
description: media_attrs["description"]
}
case Media.create_media_item(attrs) do
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
{:ok, media_item} -> media_item
{:error, changeset} -> changeset
end