hermes-agent/pinchflat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Enhancement] Delete media after "x" days (#160)

Browse Source 
* [Enhancement] Adds ability to stop media from re-downloading (#159)

* Added column

* Added methods for ignoring media items from future download

* Added new deletion options to controller and UI

* Added controller actions and UI for editing a media item

* Added column to sources

* Added retention period to form

* [WIP] getting retention methods in place

* Hooked up retention worker

* Added column and UI to prevent automatic deletion

* Docs

* Removed unused backfill worker

* Added edit links to media item tabs on source view

* Clarified form wording

* Form wording (again)
This commit is contained in:
Kieran
2024-04-03 10:44:11 -07:00
committed by GitHub
parent f9c2f7b8f2
commit 79c61bca4f
25 changed files with 576 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/pinchflat/boot/data_backfill_worker.ex
-62
View File
@@ -1,62 +0,0 @@
defmodule Pinchflat.Boot.DataBackfillWorker do
@moduledoc false
use Oban.Worker,
queue: :local_metadata,
unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
# This one is going to be a little more self-contained
# instead of relying on outside modules for the methods.
# That's because, for now, these methods are not intended
# to be used elsewhere.
#
# I'm just trying out that pattern and seeing if I like it better
# so this may change.
import Ecto.Query, warn: false
require Logger
alias __MODULE__
alias Pinchflat.Repo
@doc """
Cancels all pending backfill jobs. Useful for ensuring worker runs immediately
on app boot.
Returns {:ok, integer()}
"""
def cancel_pending_backfill_jobs do
Oban.Job
|> where(worker: "Pinchflat.Boot.DataBackfillWorker")
|> Oban.cancel_all_jobs()
end
@impl Oban.Worker
@doc """
Performs one-off tasks to get data in the right shape.
This can be needed when we add new features or change the way
we store data. Must be idempotent. All new data should already
conform to the expected schema so this should only be needed
for existing data. Still runs periodically to be safe.
Returns :ok
"""
def perform(%Oban.Job{}) do
Logger.info("Running data backfill worker")
# Nothing to do for now - just reschedule
# Keeping in-place because we _will_ need it in the future
reschedule_backfill()
:ok
end
defp reschedule_backfill do
# Run hourly
next_run_in = 60 * 60
%{}
|> DataBackfillWorker.new(schedule_in: next_run_in)
|> Repo.insert_unique_job()
end
end
lib/pinchflat/boot/post_job_startup_tasks.ex
+1 -13
View File
@@ -11,9 +11,6 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
use GenServer, restart: :temporary
import Ecto.Query, warn: false
alias Pinchflat.Repo
alias Pinchflat.Boot.DataBackfillWorker
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, %{}, opts)
end
@@ -29,16 +26,7 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
"""
@impl true
def init(state) do
enqueue_backfill_worker()
# Empty for now, keeping because tasks _will_ be added in future
{:ok, state}
end
defp enqueue_backfill_worker do
DataBackfillWorker.cancel_pending_backfill_jobs()
%{}
|> DataBackfillWorker.new()
|> Repo.insert_unique_job()
end
end
lib/pinchflat/downloading/media_retention_worker.ex
+33
View File
@@ -0,0 +1,33 @@
defmodule Pinchflat.Downloading.MediaRetentionWorker do
@moduledoc false
use Oban.Worker,
queue: :local_metadata,
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
tags: ["media_item", "local_metadata"]
require Logger
alias Pinchflat.Media
@doc """
Deletes media items that are past their retention date and prevents
them from being re-downloaded.
This worker is scheduled to run daily via the Oban Cron plugin.
Returns :ok
"""
@impl Oban.Worker
def perform(%Oban.Job{}) do
cullable_media = Media.list_cullable_media_items()
Logger.info("Culling #{length(cullable_media)} media items past their retention date")
Enum.each(cullable_media, fn media_item ->
Media.delete_media_files(media_item, %{
prevent_download: true,
culled_at: DateTime.utc_now()
})
end)
end
end
lib/pinchflat/media/media.ex
+39 -4
View File
@@ -22,13 +22,28 @@ defmodule Pinchflat.Media do
Repo.all(MediaItem)
end
@doc """
Returns a list of media_items that are cullable based on the retention period
of the source they belong to.
Returns [%MediaItem{}, ...]
"""
def list_cullable_media_items do
MediaQuery.new()
|> MediaQuery.join_sources()
|> MediaQuery.with_media_filepath()
|> MediaQuery.with_passed_retention_period()
|> MediaQuery.with_no_culling_prevention()
|> Repo.all()
end
@doc """
Returns a list of pending media_items for a given source, where
pending means the `media_filepath` is `nil` AND the media_item
matches the format selection rules of the parent media_profile.
See `build_format_clauses` but tl;dr is it _may_ filter based
on shorts or livestreams depending on the media_profile settings.
See `matching_download_criteria_for` but tl;dr is it _may_ filter based
on shorts livestreams depending on the media_profile settings.
Returns [%MediaItem{}, ...].
"""
@@ -161,7 +176,7 @@ defmodule Pinchflat.Media do
Tasks.delete_tasks_for(media_item)
if delete_files do
{:ok, _} = delete_media_files(media_item)
{:ok, _} = do_delete_media_files(media_item)
end
# Should delete these no matter what
@@ -169,6 +184,25 @@ defmodule Pinchflat.Media do
Repo.delete(media_item)
end
@doc """
Deletes the tasks and media files associated with a media_item but leaves the
media_item in the database. Does not delete anything to do with associated metadata.
Optionally accepts a second argument `addl_attrs` which will be merged into the
media_item before it is updated. Useful for setting things like `prevent_download`
and `culled_at`, if wanted
Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
"""
def delete_media_files(%MediaItem{} = media_item, addl_attrs \\ %{}) do
filepath_attrs = MediaItem.filepath_attribute_defaults()
Tasks.delete_tasks_for(media_item)
{:ok, _} = do_delete_media_files(media_item)
update_media_item(media_item, Map.merge(filepath_attrs, addl_attrs))
end
@doc """
Returns an `%Ecto.Changeset{}` for tracking media_item changes.
"""
@@ -176,7 +210,7 @@ defmodule Pinchflat.Media do
MediaItem.changeset(media_item, attrs)
end
defp delete_media_files(media_item) do
defp do_delete_media_files(media_item) do
mapped_struct = Map.from_struct(media_item)
MediaItem.filepath_attributes()
@@ -203,6 +237,7 @@ defmodule Pinchflat.Media do
defp matching_download_criteria_for(query, source_with_preloads) do
query
|> MediaQuery.with_no_prevented_download()
|> MediaQuery.with_no_media_filepath()
|> MediaQuery.with_upload_date_after(source_with_preloads.download_cutoff_date)
|> MediaQuery.with_format_preference(source_with_preloads.media_profile)
lib/pinchflat/media/media_item.ex
+20 -2
View File
@@ -30,7 +30,11 @@ defmodule Pinchflat.Media.MediaItem do
:subtitle_filepaths,
:thumbnail_filepath,
:metadata_filepath,
:nfo_filepath
:nfo_filepath,
# These are user or system controlled fields
:prevent_download,
:prevent_culling,
:culled_at
]
# Pretty much all the fields captured at index are required.
@required_fields ~w(
@@ -42,7 +46,7 @@ defmodule Pinchflat.Media.MediaItem do
source_id
upload_date
short_form_content
)a
)a
schema "media_items" do
# This is _not_ used as the primary key or internally in the database
@@ -70,6 +74,10 @@ defmodule Pinchflat.Media.MediaItem do
# Will very likely revisit because I can't leave well-enough alone.
field :subtitle_filepaths, {:array, {:array, :string}}, default: []
field :prevent_download, :boolean, default: false
field :prevent_culling, :boolean, default: false
field :culled_at, :utc_datetime
field :matching_search_term, :string, virtual: true
belongs_to :source, Source
@@ -96,4 +104,14 @@ defmodule Pinchflat.Media.MediaItem do
def filepath_attributes do
~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
end
@doc false
def filepath_attribute_defaults do
filepath_attributes()
|> Enum.map(fn
:subtitle_filepaths -> {:subtitle_filepaths, []}
field -> {field, nil}
end)
|> Enum.into(%{})
end
end
lib/pinchflat/media/media_query.ex
+26
View File
@@ -17,6 +17,7 @@ defmodule Pinchflat.Media.MediaQuery do
# Prefixes:
# - for_* - belonging to a certain record
# - join_* - for joining on a certain record
# - with_* - for filtering based on full, concrete attributes
# - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search)
#
@@ -31,6 +32,27 @@ defmodule Pinchflat.Media.MediaQuery do
where(query, [mi], mi.source_id == ^source.id)
end
def join_sources(query) do
from(mi in query, join: s in assoc(mi, :source), as: :sources)
end
def with_passed_retention_period(query) do
where(
query,
[mi, sources],
fragment(
"IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?",
sources.retention_period_days,
sources.retention_period_days,
mi.media_downloaded_at
)
)
end
def with_no_culling_prevention(query) do
where(query, [mi], mi.prevent_culling == false)
end
def with_id(query, id) do
where(query, [mi], mi.id == ^id)
end
@@ -53,6 +75,10 @@ defmodule Pinchflat.Media.MediaQuery do
where(query, [mi], mi.upload_date >= ^date)
end
def with_no_prevented_download(query) do
where(query, [mi], mi.prevent_download == false)
end
def matching_title_regex(query, nil), do: query
def matching_title_regex(query, regex) do
lib/pinchflat/sources/source.ex
+3
View File
@@ -29,6 +29,7 @@ defmodule Pinchflat.Sources.Source do
last_indexed_at
original_url
download_cutoff_date
retention_period_days
title_filter_regex
media_profile_id
)a
@@ -72,6 +73,7 @@ defmodule Pinchflat.Sources.Source do
field :last_indexed_at, :utc_datetime
# Only download media items that were published after this date
field :download_cutoff_date, :date
field :retention_period_days, :integer
field :original_url, :string
field :title_filter_regex, :string
@@ -106,6 +108,7 @@ defmodule Pinchflat.Sources.Source do
|> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
|> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
|> validate_required(required_fields)
|> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
|> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
|> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
end
lib/pinchflat_web/controllers/media_items/media_item_controller.ex
+24 -10
View File
@@ -16,20 +16,34 @@ defmodule PinchflatWeb.MediaItems.MediaItemController do
render(conn, :show, media_item: media_item)
end
def delete(conn, %{"id" => id} = params) do
delete_files = Map.get(params, "delete_files", false)
def edit(conn, %{"id" => id}) do
media_item = Media.get_media_item!(id)
{:ok, _} = Media.delete_media_item(media_item, delete_files: delete_files)
changeset = Media.change_media_item(media_item)
flash_message =
if delete_files do
"Record and files deleted successfully."
else
"Record deleted successfully. Files were not deleted."
end
render(conn, :edit, media_item: media_item, changeset: changeset)
end
def update(conn, %{"id" => id, "media_item" => params}) do
media_item = Media.get_media_item!(id)
case Media.update_media_item(media_item, params) do
{:ok, media_item} ->
conn
|> put_flash(:info, "Media Item updated successfully.")
|> redirect(to: ~p"/sources/#{media_item.source_id}/media/#{media_item}")
{:error, %Ecto.Changeset{} = changeset} ->
render(conn, :edit, media_item: media_item, changeset: changeset)
end
end
def delete(conn, %{"id" => id} = params) do
prevent_download = Map.get(params, "prevent_download", false)
media_item = Media.get_media_item!(id)
{:ok, _} = Media.delete_media_files(media_item, %{prevent_download: prevent_download})
conn
|> put_flash(:info, flash_message)
|> put_flash(:info, "Files deleted successfully.")
|> redirect(to: ~p"/sources/#{media_item.source_id}")
end
lib/pinchflat_web/controllers/media_items/media_item_html.ex
+8
View File
@@ -3,6 +3,14 @@ defmodule PinchflatWeb.MediaItems.MediaItemHTML do
embed_templates "media_item_html/*"
@doc """
Renders a media item form.
"""
attr :changeset, Ecto.Changeset, required: true
attr :action, :string, required: true
def media_item_form(assigns)
def media_file_exists?(media_item) do
!!media_item.media_filepath and File.exists?(media_item.media_filepath)
end
lib/pinchflat_web/controllers/media_items/media_item_html/edit.html.heex
+13
View File
@@ -0,0 +1,13 @@
<div class="mb-6 flex gap-3 flex-row items-center">
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
Editing "<%= StringUtils.truncate(@media_item.title, 35) %>"
</h2>
</div>
<div class="rounded-sm border border-stroke bg-white px-5 pb-2.5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark sm:px-7.5 xl:pb-1">
<div class="max-w-full overflow-x-auto">
<div class="flex flex-col gap-10">
<.media_item_form changeset={@changeset} action={~p"/sources/#{@media_item.source_id}/media/#{@media_item}"} />
</div>
</div>
</div>
lib/pinchflat_web/controllers/media_items/media_item_html/media_item_form.html.heex
+31
View File
@@ -0,0 +1,31 @@
<.simple_form
:let={f}
for={@changeset}
action={@action}
x-data="{ advancedMode: !!JSON.parse(localStorage.getItem('advancedMode')) }"
x-init="$watch('advancedMode', value => localStorage.setItem('advancedMode', JSON.stringify(value)))"
>
<.error :if={@changeset.action}>
Oops, something went wrong! Please check the errors below.
</.error>
<h3 class=" text-2xl text-black dark:text-white">
General Options
</h3>
<.input
field={f[:prevent_download]}
type="toggle"
label="Prevent Download"
help="Checking excludes this media item from being downloaded"
/>
<.input
field={f[:prevent_culling]}
type="toggle"
label="Prevent Automatic Deletion"
help="Checking excludes media from being automatically deleted based on media retention rules"
/>
<.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Media Item</.button>
</.simple_form>
lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex
+21 -3
View File
@@ -4,9 +4,17 @@
<.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
</.link>
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
Media Item #<%= @media_item.id %>
<%= StringUtils.truncate(@media_item.title, 35) %>
</h2>
</div>
<nav>
<.link href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}/edit"}>
<.button color="bg-primary" rounding="rounded-lg">
<.icon name="hero-pencil-square" class="mr-2" /> Edit
</.button>
</.link>
</nav>
</div>
<div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark px-7.5">
<div class="max-w-full overflow-x-auto">
@@ -15,13 +23,22 @@
<.button_dropdown text="Actions" class="justify-center w-full sm:w-50">
<:option>
<.link
href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}?delete_files=true"}
href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}"}
method="delete"
data-confirm="Are you sure you want to delete this record and all associated files on disk? This cannot be undone."
data-confirm="Are you sure you want to delete all files for this media item? This cannot be undone."
>
Delete Files
</.link>
</:option>
<:option>
<.link
href={~p"/sources/#{@media_item.source_id}/media/#{@media_item}?prevent_download=true"}
method="delete"
data-confirm="Are you sure you want to delete all files for this media item and prevent it from re-downloading in the future? This cannot be undone."
>
Delete and Ignore
</.link>
</:option>
</.button_dropdown>
</:tab_append>
@@ -32,6 +49,7 @@
<.media_preview media_item={@media_item} />
<% end %>
<h2 class="font-bold text-2xl"><%= @media_item.title %></h2>
<h3 class="font-bold text-xl">Attributes</h3>
<section>
<strong>Source:</strong>
lib/pinchflat_web/controllers/sources/source_html/index.html.heex
+7
View File
@@ -22,6 +22,13 @@
<:col :let={source} label="Should Download?">
<.icon name={if source.download_media, do: "hero-check", else: "hero-x-mark"} />
</:col>
<:col :let={source} label="Retention">
<%= if source.retention_period_days && source.retention_period_days > 0 do %>
<%= source.retention_period_days %> day(s)
<% else %>
<span class="text-lg">∞</span>
<% end %>
</:col>
<:col :let={source} label="Media Profile">
<.subtle_link href={~p"/media_profiles/#{source.media_profile_id}"}>
<%= source.media_profile.name %>
lib/pinchflat_web/controllers/sources/source_html/show.html.heex
+18 -4
View File
@@ -77,10 +77,17 @@
<h4 class="text-white text-lg mb-6">Shows a maximum of 100 media items</h4>
<.table rows={@pending_media} table_class="text-black dark:text-white">
<:col :let={media_item} label="Title">
<%= StringUtils.truncate(media_item.title, 50) %>
<.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
<%= StringUtils.truncate(media_item.title, 50) %>
</.subtle_link>
</:col>
<:col :let={media_item} label="" class="flex place-content-evenly">
<.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" />
<.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" class="mx-1" />
<.icon_link
href={~p"/sources/#{@source.id}/media/#{media_item.id}/edit"}
icon="hero-pencil-square"
class="mx-1"
/>
</:col>
</.table>
<% else %>
@@ -92,10 +99,17 @@
<h4 class="text-white text-lg mb-6">Shows a maximum of 100 media items (<%= @total_downloaded %> total)</h4>
<.table rows={@downloaded_media} table_class="text-black dark:text-white">
<:col :let={media_item} label="Title">
<%= StringUtils.truncate(media_item.title, 50) %>
<.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
<%= StringUtils.truncate(media_item.title, 50) %>
</.subtle_link>
</:col>
<:col :let={media_item} label="" class="flex place-content-evenly">
<.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" />
<.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"} icon="hero-eye" class="mx-1" />
<.icon_link
href={~p"/sources/#{@source.id}/media/#{media_item.id}/edit"}
icon="hero-pencil-square"
class="mx-1"
/>
</:col>
</.table>
<% else %>
lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex
+8
View File
@@ -88,6 +88,14 @@
help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
/>
<.input
field={f[:retention_period_days]}
type="number"
label="Retention Period (days)"
min="0"
help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
/>
<section x-show="advancedMode">
<h3 class="mt-8 text-2xl text-black dark:text-white">
Advanced Options
lib/pinchflat_web/router.ex
+1 -1
View File
@@ -32,7 +32,7 @@ defmodule PinchflatWeb.Router do
resources "/search", Searches.SearchController, only: [:show], singleton: true
resources "/sources", Sources.SourceController do
resources "/media", MediaItems.MediaItemController, only: [:show, :delete]
resources "/media", MediaItems.MediaItemController, only: [:show, :edit, :update, :delete]
end
end