[Enhancement] Delete media after "x" days (#160)

* [Enhancement] Adds ability to stop media from re-downloading (#159)

* Added column

* Added methods for ignoring media items from future download

* Added new deletion options to controller and UI

* Added controller actions and UI for editing a media item

* Added column to sources

* Added retention period to form

* [WIP] getting retention methods in place

* Hooked up retention worker

* Added column and UI to prevent automatic deletion

* Docs

* Removed unused backfill worker

* Added edit links to media item tabs on source view

* Clarified form wording

* Form wording (again)

lib/pinchflat/boot/data_backfill_worker.ex

@@ -1,62 +0,0 @@
-defmodule Pinchflat.Boot.DataBackfillWorker do
-  @moduledoc false
-
-  use Oban.Worker,
-    queue: :local_metadata,
-    unique: [period: :infinity, states: [:available, :scheduled, :retryable]],
-    tags: ["media_item", "media_metadata", "local_metadata", "data_backfill"]
-
-  # This one is going to be a little more self-contained
-  # instead of relying on outside modules for the methods.
-  # That's because, for now, these methods are not intended
-  # to be used elsewhere.
-  #
-  # I'm just trying out that pattern and seeing if I like it better
-  # so this may change.
-  import Ecto.Query, warn: false
-  require Logger
-
-  alias __MODULE__
-  alias Pinchflat.Repo
-
-  @doc """
-  Cancels all pending backfill jobs. Useful for ensuring worker runs immediately
-  on app boot.
-
-  Returns {:ok, integer()}
-  """
-  def cancel_pending_backfill_jobs do
-    Oban.Job
-    |> where(worker: "Pinchflat.Boot.DataBackfillWorker")
-    |> Oban.cancel_all_jobs()
-  end
-
-  @impl Oban.Worker
-  @doc """
-  Performs one-off tasks to get data in the right shape.
-  This can be needed when we add new features or change the way
-  we store data. Must be idempotent. All new data should already
-  conform to the expected schema so this should only be needed
-  for existing data. Still runs periodically to be safe.
-
-  Returns :ok
-  """
-  def perform(%Oban.Job{}) do
-    Logger.info("Running data backfill worker")
-    # Nothing to do for now - just reschedule
-    # Keeping in-place because we _will_ need it in the future
-
-    reschedule_backfill()
-
-    :ok
-  end
-
-  defp reschedule_backfill do
-    # Run hourly
-    next_run_in = 60 * 60
-
-    %{}
-    |> DataBackfillWorker.new(schedule_in: next_run_in)
-    |> Repo.insert_unique_job()
-  end
-end

lib/pinchflat/boot/post_job_startup_tasks.ex

@@ -11,9 +11,6 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   use GenServer, restart: :temporary
   import Ecto.Query, warn: false
 
-  alias Pinchflat.Repo
-  alias Pinchflat.Boot.DataBackfillWorker
-
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, %{}, opts)
   end
@@ -29,16 +26,7 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
   """
   @impl true
   def init(state) do
-    enqueue_backfill_worker()
-
+    # Empty for now, keeping because tasks _will_ be added in future
     {:ok, state}
   end
-
-  defp enqueue_backfill_worker do
-    DataBackfillWorker.cancel_pending_backfill_jobs()
-
-    %{}
-    |> DataBackfillWorker.new()
-    |> Repo.insert_unique_job()
-  end
 end

lib/pinchflat/downloading/media_retention_worker.ex

@@ -0,0 +1,33 @@
+defmodule Pinchflat.Downloading.MediaRetentionWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_metadata,
+    unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
+    tags: ["media_item", "local_metadata"]
+
+  require Logger
+
+  alias Pinchflat.Media
+
+  @doc """
+  Deletes media items that are past their retention date and prevents
+  them from being re-downloaded.
+
+  This worker is scheduled to run daily via the Oban Cron plugin.
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    cullable_media = Media.list_cullable_media_items()
+    Logger.info("Culling #{length(cullable_media)} media items past their retention date")
+
+    Enum.each(cullable_media, fn media_item ->
+      Media.delete_media_files(media_item, %{
+        prevent_download: true,
+        culled_at: DateTime.utc_now()
+      })
+    end)
+  end
+end

lib/pinchflat/media/media.ex

@@ -22,13 +22,28 @@ defmodule Pinchflat.Media do
     Repo.all(MediaItem)
   end
 
+  @doc """
+  Returns a list of media_items that are cullable based on the retention period
+  of the source they belong to.
+
+  Returns [%MediaItem{}, ...]
+  """
+  def list_cullable_media_items do
+    MediaQuery.new()
+    |> MediaQuery.join_sources()
+    |> MediaQuery.with_media_filepath()
+    |> MediaQuery.with_passed_retention_period()
+    |> MediaQuery.with_no_culling_prevention()
+    |> Repo.all()
+  end
+
   @doc """
   Returns a list of pending media_items for a given source, where
   pending means the `media_filepath` is `nil` AND the media_item
   matches the format selection rules of the parent media_profile.
 
-  See `build_format_clauses` but tl;dr is it _may_ filter based
-  on shorts or livestreams depending on the media_profile settings.
+  See `matching_download_criteria_for` but tl;dr is it _may_ filter based
+  on shorts livestreams depending on the media_profile settings.
 
   Returns [%MediaItem{}, ...].
   """
@@ -161,7 +176,7 @@ defmodule Pinchflat.Media do
     Tasks.delete_tasks_for(media_item)
 
     if delete_files do
-      {:ok, _} = delete_media_files(media_item)
+      {:ok, _} = do_delete_media_files(media_item)
     end
 
     # Should delete these no matter what
@@ -169,6 +184,25 @@ defmodule Pinchflat.Media do
     Repo.delete(media_item)
   end
 
+  @doc """
+  Deletes the tasks and media files associated with a media_item but leaves the
+  media_item in the database. Does not delete anything to do with associated metadata.
+
+  Optionally accepts a second argument `addl_attrs` which will be merged into the
+  media_item before it is updated. Useful for setting things like `prevent_download`
+  and `culled_at`, if wanted
+
+  Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
+  """
+  def delete_media_files(%MediaItem{} = media_item, addl_attrs \\ %{}) do
+    filepath_attrs = MediaItem.filepath_attribute_defaults()
+
+    Tasks.delete_tasks_for(media_item)
+    {:ok, _} = do_delete_media_files(media_item)
+
+    update_media_item(media_item, Map.merge(filepath_attrs, addl_attrs))
+  end
+
   @doc """
   Returns an `%Ecto.Changeset{}` for tracking media_item changes.
   """
@@ -176,7 +210,7 @@ defmodule Pinchflat.Media do
     MediaItem.changeset(media_item, attrs)
   end
 
-  defp delete_media_files(media_item) do
+  defp do_delete_media_files(media_item) do
     mapped_struct = Map.from_struct(media_item)
 
     MediaItem.filepath_attributes()
@@ -203,6 +237,7 @@ defmodule Pinchflat.Media do
 
   defp matching_download_criteria_for(query, source_with_preloads) do
     query
+    |> MediaQuery.with_no_prevented_download()
     |> MediaQuery.with_no_media_filepath()
     |> MediaQuery.with_upload_date_after(source_with_preloads.download_cutoff_date)
     |> MediaQuery.with_format_preference(source_with_preloads.media_profile)

lib/pinchflat/media/media_item.ex

@@ -30,7 +30,11 @@ defmodule Pinchflat.Media.MediaItem do
     :subtitle_filepaths,
     :thumbnail_filepath,
     :metadata_filepath,
-    :nfo_filepath
+    :nfo_filepath,
+    # These are user or system controlled fields
+    :prevent_download,
+    :prevent_culling,
+    :culled_at
   ]
   # Pretty much all the fields captured at index are required.
   @required_fields ~w(
@@ -42,7 +46,7 @@ defmodule Pinchflat.Media.MediaItem do
     source_id
     upload_date
     short_form_content
-    )a
+  )a
 
   schema "media_items" do
     # This is _not_ used as the primary key or internally in the database
@@ -70,6 +74,10 @@ defmodule Pinchflat.Media.MediaItem do
     # Will very likely revisit because I can't leave well-enough alone.
     field :subtitle_filepaths, {:array, {:array, :string}}, default: []
 
+    field :prevent_download, :boolean, default: false
+    field :prevent_culling, :boolean, default: false
+    field :culled_at, :utc_datetime
+
     field :matching_search_term, :string, virtual: true
 
     belongs_to :source, Source
@@ -96,4 +104,14 @@ defmodule Pinchflat.Media.MediaItem do
   def filepath_attributes do
     ~w(media_filepath thumbnail_filepath metadata_filepath subtitle_filepaths nfo_filepath)a
   end
+
+  @doc false
+  def filepath_attribute_defaults do
+    filepath_attributes()
+    |> Enum.map(fn
+      :subtitle_filepaths -> {:subtitle_filepaths, []}
+      field -> {field, nil}
+    end)
+    |> Enum.into(%{})
+  end
 end

lib/pinchflat/media/media_query.ex

@@ -17,6 +17,7 @@ defmodule Pinchflat.Media.MediaQuery do
 
   # Prefixes:
   # - for_* - belonging to a certain record
+  # - join_* - for joining on a certain record
   # - with_* - for filtering based on full, concrete attributes
   # - matching_* - for filtering based on partial attributes (e.g. LIKE, regex, full-text search)
   #
@@ -31,6 +32,27 @@ defmodule Pinchflat.Media.MediaQuery do
     where(query, [mi], mi.source_id == ^source.id)
   end
 
+  def join_sources(query) do
+    from(mi in query, join: s in assoc(mi, :source), as: :sources)
+  end
+
+  def with_passed_retention_period(query) do
+    where(
+      query,
+      [mi, sources],
+      fragment(
+        "IFNULL(?, 0) > 0 AND DATETIME('now', '-' || ? || ' day') > ?",
+        sources.retention_period_days,
+        sources.retention_period_days,
+        mi.media_downloaded_at
+      )
+    )
+  end
+
+  def with_no_culling_prevention(query) do
+    where(query, [mi], mi.prevent_culling == false)
+  end
+
   def with_id(query, id) do
     where(query, [mi], mi.id == ^id)
   end
@@ -53,6 +75,10 @@ defmodule Pinchflat.Media.MediaQuery do
     where(query, [mi], mi.upload_date >= ^date)
   end
 
+  def with_no_prevented_download(query) do
+    where(query, [mi], mi.prevent_download == false)
+  end
+
   def matching_title_regex(query, nil), do: query
 
   def matching_title_regex(query, regex) do

lib/pinchflat/sources/source.ex

@@ -29,6 +29,7 @@ defmodule Pinchflat.Sources.Source do
     last_indexed_at
     original_url
     download_cutoff_date
+    retention_period_days
     title_filter_regex
     media_profile_id
   )a
@@ -72,6 +73,7 @@ defmodule Pinchflat.Sources.Source do
     field :last_indexed_at, :utc_datetime
     # Only download media items that were published after this date
     field :download_cutoff_date, :date
+    field :retention_period_days, :integer
     field :original_url, :string
     field :title_filter_regex, :string
 
@@ -106,6 +108,7 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
     |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
+    |> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
   end