Bumped version

* [WIP] started on source deletion

* Removed unneeded test blocks

* Added marked_for_deletion_at to sources and media profiles

* Hooked up async deletion to media profiles as well

config/config.exs

@@ -65,7 +65,7 @@ config :pinchflat, Oban,
     media_indexing: 2,
     media_collection_indexing: 2,
     media_fetching: 2,
-    local_metadata: 8,
+    local_data: 8,
     remote_metadata: 4
   ]
 

docker/dev.Dockerfile

@@ -12,7 +12,7 @@ RUN echo "Building for ${TARGETPLATFORM:?}"
 RUN apt-get update -qq && \
   apt-get install -y inotify-tools curl git openssh-client jq \
     python3 python3-setuptools python3-wheel python3-dev pipx \
-    python3-mutagen locales procps build-essential graphviz
+    python3-mutagen locales procps build-essential graphviz zsh
 
 # Install ffmpeg
 RUN export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
@@ -38,7 +38,10 @@ RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh && \
   # Install Apprise
   export PIPX_HOME=/opt/pipx && \
   export PIPX_BIN_DIR=/usr/local/bin && \
-  pipx install apprise
+  pipx install apprise && \
+  # Set up ZSH tools
+  chsh -s $(which zsh) && \
+  sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
 
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen

lib/pinchflat/downloading/downloading_helpers.ex

@@ -91,8 +91,7 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
         [m, s, mp],
         ^MediaQuery.for_source(source) and
           ^MediaQuery.downloaded() and
-          not (^MediaQuery.download_prevented()) and
+          not (^MediaQuery.download_prevented())
-          not (^MediaQuery.culled())
       )
     )
     |> Repo.all()

lib/pinchflat/downloading/media_download_worker.ex

@@ -39,17 +39,14 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
     - `quality_upgrade?`: re-downloads media, including the video. Does not force download
       if the source is set to not download media
 
-  Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
+  Returns :ok | {:error, any, ...any}
   """
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => media_item_id} = args}) do
     should_force = Map.get(args, "force", false)
     is_quality_upgrade = Map.get(args, "quality_upgrade?", false)
 
-    media_item =
+    media_item = fetch_and_run_prevent_download_user_script(media_item_id)
-      media_item_id
-      |> Media.get_media_item!()
-      |> Repo.preload(:source)
 
     # If the source or media item is set to not download media, perform a no-op unless forced
     if (media_item.source.download_media && !media_item.prevent_download) || should_force do
@@ -62,6 +59,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
     Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
   end
 
+  # If a user script exists and, when run, returns a non-zero exit code, prevent this and all future downloads
+  # of the media item.
+  defp fetch_and_run_prevent_download_user_script(media_item_id) do
+    media_item = Media.get_media_item!(media_item_id)
+
+    {:ok, media_item} =
+      case run_user_script(:media_pre_download, media_item) do
+        {:ok, _, exit_code} when exit_code != 0 -> Media.update_media_item(media_item, %{prevent_download: true})
+        _ -> {:ok, media_item}
+      end
+
+    Repo.preload(media_item, :source)
+  end
+
   defp download_media_and_schedule_jobs(media_item, is_quality_upgrade, should_force) do
     overwrite_behaviour = if should_force || is_quality_upgrade, do: :force_overwrites, else: :no_force_overwrites
     override_opts = [overwrite_behaviour: overwrite_behaviour]
@@ -74,9 +85,9 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
             media_redownloaded_at: get_redownloaded_at(is_quality_upgrade)
           })
 
-        :ok = run_user_script(updated_media_item)
+        run_user_script(:media_downloaded, updated_media_item)
 
-        {:ok, updated_media_item}
+        :ok
 
       {:recovered, _} ->
         {:error, :retry}
@@ -112,9 +123,9 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
   # NOTE: I like this pattern of using the default value so that I don't have to
   # define it in config.exs (and friends). Consider using this elsewhere.
-  defp run_user_script(media_item) do
+  defp run_user_script(event, media_item) do
     runner = Application.get_env(:pinchflat, :user_script_runner, UserScriptRunner)
 
-    runner.run(:media_downloaded, media_item)
+    runner.run(event, media_item)
   end
 end

lib/pinchflat/downloading/media_downloader.ex

@@ -79,6 +79,7 @@ defmodule Pinchflat.Downloading.MediaDownloader do
       |> MetadataParser.parse_for_media_item()
       |> Map.merge(%{
         media_downloaded_at: DateTime.utc_now(),
+        culled_at: nil,
         nfo_filepath: determine_nfo_filepath(media_with_preloads, parsed_json),
         metadata: %{
           # IDEA: might be worth kicking off a job for this since thumbnail fetching

lib/pinchflat/downloading/media_quality_upgrade_worker.ex

@@ -23,10 +23,10 @@ defmodule Pinchflat.Downloading.MediaQualityUpgradeWorker do
   """
   @impl Oban.Worker
   def perform(%Oban.Job{}) do
-    redownloadable_media = Media.list_redownloadable_media_items()
+    upgradable_media = Media.list_upgradeable_media_items()
-    Logger.info("Redownloading #{length(redownloadable_media)} media items")
+    Logger.info("Redownloading #{length(upgradable_media)} media items")
 
-    Enum.each(redownloadable_media, fn media_item ->
+    Enum.each(upgradable_media, fn media_item ->
       MediaDownloadWorker.kickoff_with_task(media_item, %{quality_upgrade?: true})
     end)
   end

lib/pinchflat/downloading/media_retention_worker.ex

@@ -2,12 +2,15 @@ defmodule Pinchflat.Downloading.MediaRetentionWorker do
   @moduledoc false
 
   use Oban.Worker,
-    queue: :local_metadata,
+    queue: :local_data,
     unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
-    tags: ["media_item", "local_metadata"]
+    tags: ["media_item", "local_data"]
+
+  use Pinchflat.Media.MediaQuery
 
   require Logger
 
+  alias Pinchflat.Repo
   alias Pinchflat.Media
 
   @doc """
@@ -20,14 +23,51 @@ defmodule Pinchflat.Downloading.MediaRetentionWorker do
   """
   @impl Oban.Worker
   def perform(%Oban.Job{}) do
-    cullable_media = Media.list_cullable_media_items()
+    cull_cullable_media_items()
+    delete_media_items_from_before_cutoff()
+
+    :ok
+  end
+
+  defp cull_cullable_media_items do
+    cullable_media =
+      MediaQuery.new()
+      |> MediaQuery.require_assoc(:source)
+      |> where(^MediaQuery.cullable())
+      |> Repo.all()
+
     Logger.info("Culling #{length(cullable_media)} media items past their retention date")
 
     Enum.each(cullable_media, fn media_item ->
+      # Setting `prevent_download` does what it says on the tin, but `culled_at` is purely informational.
+      # We don't actually do anything with that in terms of queries and it gets set to nil if the media item
+      # gets re-downloaded.
       Media.delete_media_files(media_item, %{
         prevent_download: true,
         culled_at: DateTime.utc_now()
       })
     end)
   end
+
+  defp delete_media_items_from_before_cutoff do
+    deletable_media =
+      MediaQuery.new()
+      |> MediaQuery.require_assoc(:source)
+      |> where(^MediaQuery.deletable_based_on_source_cutoff())
+      |> Repo.all()
+
+    Logger.info("Deleting #{length(deletable_media)} media items that are from before the source cutoff")
+
+    Enum.each(deletable_media, fn media_item ->
+      # Note that I'm not setting `prevent_download` on the media_item here.
+      # That's because cutoff_date can easily change and it's a valid behavior to re-download older
+      # media items if the cutoff_date changes.
+      # Download is ultimately prevented because `MediaQuery.pending()` only returns media items
+      # from after the cutoff date (among other things), so it's not like the media will just immediately
+      # be re-downloaded.
+      Media.delete_media_files(media_item, %{
+        culled_at: DateTime.utc_now()
+      })
+    end)
+  end
 end

lib/pinchflat/lifecycle/user_scripts/command_runner.ex

@@ -12,6 +12,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
   @behaviour UserScriptCommandRunner
 
   @event_types [
+    :media_pre_download,
     :media_downloaded,
     :media_deleted
   ]
@@ -22,24 +23,25 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
 
   This function will succeed in almost all cases, even if the user script command
   failed - this is because I don't want bad scripts to stop the whole process.
-  If something fails, it'll be logged.
+  If something fails, it'll be logged and returned BUT the tuple will always
+  start with {:ok, ...}.
 
   The only things that can cause a true failure are passing in an invalid event
   type or if the passed data cannot be encoded into JSON - both indicative of
   failures in the development process.
 
-  Returns :ok
+  Returns {:ok, :no_executable} | {:ok, output, exit_code}
   """
   @impl UserScriptCommandRunner
   def run(event_type, encodable_data) when event_type in @event_types do
     case backend_executable() do
       {:ok, :no_executable} ->
-        :ok
+        {:ok, :no_executable}
 
       {:ok, executable_path} ->
         {:ok, encoded_data} = Phoenix.json_library().encode(encodable_data)
 
-        {_output, _exit_code} =
+        {output, exit_code} =
           CliUtils.wrap_cmd(
             executable_path,
             [to_string(event_type), encoded_data],
@@ -47,7 +49,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
             logging_arg_override: "[suppressed]"
           )
 
-        :ok
+        {:ok, output, exit_code}
     end
   end
 
@@ -62,7 +64,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
     if FilesystemUtils.exists_and_nonempty?(filepath) do
       {:ok, filepath}
     else
-      Logger.warning("User scripts lifecyle file either not present or is empty. Skipping.")
+      Logger.info("User scripts lifecyle file either not present or is empty. Skipping.")
 
       {:ok, :no_executable}
     end

lib/pinchflat/media/media.ex

@@ -25,21 +25,10 @@ defmodule Pinchflat.Media do
   end
 
   @doc """
-  Returns a list of media_items that are cullable based on the retention period
+  Returns a list of media_items that are upgradeable based on the redownload delay
-  of the source they belong to.
+  of the media_profile their source belongs to. In this context, upgradeable means
-
+  that it's been long enough since upload that the video may be in a higher quality
-  Returns [%MediaItem{}, ...]
+  or have better sponsorblock segments (or similar).
-  """
-  def list_cullable_media_items do
-    MediaQuery.new()
-    |> MediaQuery.require_assoc(:source)
-    |> where(^MediaQuery.cullable())
-    |> Repo.all()
-  end
-
-  @doc """
-  Returns a list of media_items that are redownloadable based on the redownload delay
-  of the media_profile their source belongs to.
 
   The logic is that a media_item is past_redownload_delay if the media_item's uploaded_at is
   at least redownload_delay_days ago AND `media_downloaded_at` - `redownload_delay_days`
@@ -52,10 +41,10 @@ defmodule Pinchflat.Media do
 
   Returns [%MediaItem{}, ...]
   """
-  def list_redownloadable_media_items do
+  def list_upgradeable_media_items do
     MediaQuery.new()
     |> MediaQuery.require_assoc(:media_profile)
-    |> where(^MediaQuery.redownloadable())
+    |> where(^MediaQuery.upgradeable())
     |> Repo.all()
   end
 
@@ -182,7 +171,7 @@ defmodule Pinchflat.Media do
 
     if delete_files do
       {:ok, _} = do_delete_media_files(media_item)
-      :ok = run_user_script(:media_deleted, media_item)
+      run_user_script(:media_deleted, media_item)
     end
 
     # Should delete these no matter what
@@ -205,7 +194,7 @@ defmodule Pinchflat.Media do
 
     Tasks.delete_tasks_for(media_item)
     {:ok, _} = do_delete_media_files(media_item)
-    :ok = run_user_script(:media_deleted, media_item)
+    run_user_script(:media_deleted, media_item)
 
     update_media_item(media_item, Map.merge(filepath_attrs, addl_attrs))
   end

lib/pinchflat/media/media_query.ex

@@ -33,7 +33,6 @@ defmodule Pinchflat.Media.MediaQuery do
   def downloaded, do: dynamic([mi], not is_nil(mi.media_filepath))
   def download_prevented, do: dynamic([mi], mi.prevent_download == true)
   def culling_prevented, do: dynamic([mi], mi.prevent_culling == true)
-  def culled, do: dynamic([mi], not is_nil(mi.culled_at))
   def redownloaded, do: dynamic([mi], not is_nil(mi.media_redownloaded_at))
   def upload_date_matches(other_date), do: dynamic([mi], fragment("date(?) = date(?)", mi.uploaded_at, ^other_date))
 
@@ -108,6 +107,15 @@ defmodule Pinchflat.Media.MediaQuery do
     )
   end
 
+  def deletable_based_on_source_cutoff do
+    dynamic(
+      [mi, source],
+      ^downloaded() and
+        not (^upload_date_after_source_cutoff()) and
+        not (^culling_prevented())
+    )
+  end
+
   def pending do
     dynamic(
       [mi],
@@ -119,12 +127,11 @@ defmodule Pinchflat.Media.MediaQuery do
     )
   end
 
-  def redownloadable do
+  def upgradeable do
     dynamic(
       [mi, source],
       ^downloaded() and
         not (^download_prevented()) and
-        not (^culled()) and
         not (^redownloaded()) and
         ^past_redownload_delay()
     )

lib/pinchflat/profiles/media_profile.ex

@@ -28,6 +28,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
     livestream_behaviour
     preferred_resolution
     redownload_delay_days
+    marked_for_deletion_at
   )a
 
   @required_fields ~w(name output_path_template)a
@@ -65,6 +66,8 @@ defmodule Pinchflat.Profiles.MediaProfile do
     field :livestream_behaviour, Ecto.Enum, values: ~w(include exclude only)a, default: :include
     field :preferred_resolution, Ecto.Enum, values: ~w(4320p 2160p 1080p 720p 480p 360p audio)a, default: :"1080p"
 
+    field :marked_for_deletion_at, :utc_datetime
+
     has_many :sources, Source
 
     timestamps(type: :utc_datetime)

lib/pinchflat/profiles/media_profile_deletion_worker.ex

@@ -0,0 +1,38 @@
+defmodule Pinchflat.Profiles.MediaProfileDeletionWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_data,
+    tags: ["media_profiles", "local_data"]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Profiles
+
+  @doc """
+  Starts the profile deletion worker. Does not attach it to a task like `kickoff_with_task/2`
+  since deletion also cancels all tasks for the profile
+
+  Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff(profile, job_args \\ %{}, job_opts \\ []) do
+    %{id: profile.id}
+    |> Map.merge(job_args)
+    |> MediaProfileDeletionWorker.new(job_opts)
+    |> Oban.insert()
+  end
+
+  @doc """
+  Deletes a profile and optionally deletes its files
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => profile_id} = args}) do
+    delete_files = Map.get(args, "delete_files", false)
+    profile = Profiles.get_media_profile!(profile_id)
+
+    Profiles.delete_media_profile(profile, delete_files: delete_files)
+  end
+end

lib/pinchflat/sources/source.ex

@@ -35,6 +35,7 @@ defmodule Pinchflat.Sources.Source do
     title_filter_regex
     media_profile_id
     output_path_template_override
+    marked_for_deletion_at
   )a
 
   # Expensive API calls are made when a source is inserted/updated so
@@ -87,6 +88,8 @@ defmodule Pinchflat.Sources.Source do
     field :fanart_filepath, :string
     field :banner_filepath, :string
 
+    field :marked_for_deletion_at, :utc_datetime
+
     belongs_to :media_profile, MediaProfile
 
     has_one :metadata, SourceMetadata, on_replace: :update

lib/pinchflat/sources/source_deletion_worker.ex

@@ -0,0 +1,38 @@
+defmodule Pinchflat.Sources.SourceDeletionWorker do
+  @moduledoc false
+
+  use Oban.Worker,
+    queue: :local_data,
+    tags: ["sources", "local_data"]
+
+  require Logger
+
+  alias __MODULE__
+  alias Pinchflat.Sources
+
+  @doc """
+  Starts the source deletion worker. Does not attach it to a task like `kickoff_with_task/2`
+  since deletion also cancels all tasks for the source
+
+  Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
+  """
+  def kickoff(source, job_args \\ %{}, job_opts \\ []) do
+    %{id: source.id}
+    |> Map.merge(job_args)
+    |> SourceDeletionWorker.new(job_opts)
+    |> Oban.insert()
+  end
+
+  @doc """
+  Deletes a source and optionally deletes its files
+
+  Returns :ok
+  """
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => source_id} = args}) do
+    delete_files = Map.get(args, "delete_files", false)
+    source = Sources.get_source!(source_id)
+
+    Sources.delete_source(source, delete_files: delete_files)
+  end
+end

lib/pinchflat_web/controllers/media_profiles/media_profile_controller.ex

@@ -5,10 +5,12 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
   alias Pinchflat.Repo
   alias Pinchflat.Profiles
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Profiles.MediaProfileDeletionWorker
 
   def index(conn, _params) do
     media_profiles =
       MediaProfile
+      |> where([mp], is_nil(mp.marked_for_deletion_at))
       |> order_by(asc: :name)
       |> Repo.all()
 
@@ -70,19 +72,15 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
   end
 
   def delete(conn, %{"id" => id} = params) do
-    delete_files = Map.get(params, "delete_files", false)
+    # This awkward comparison converts the string to a boolean
+    delete_files = Map.get(params, "delete_files", "") == "true"
     media_profile = Profiles.get_media_profile!(id)
-    {:ok, _media_profile} = Profiles.delete_media_profile(media_profile, delete_files: delete_files)
 
-    flash_message =
+    {:ok, _} = Profiles.update_media_profile(media_profile, %{marked_for_deletion_at: DateTime.utc_now()})
-      if delete_files do
+    MediaProfileDeletionWorker.kickoff(media_profile, %{delete_files: delete_files})
-        "Media profile, its sources, and its files deleted successfully."
-      else
-        "Media profile and its sources deleted successfully. Files were not deleted."
-      end
 
     conn
-    |> put_flash(:info, flash_message)
+    |> put_flash(:info, "Media Profile deletion started. This may take a while to complete.")
     |> redirect(to: ~p"/media_profiles")
   end
 

lib/pinchflat_web/controllers/sources/source_controller.ex

@@ -8,6 +8,7 @@ defmodule PinchflatWeb.Sources.SourceController do
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Profiles.MediaProfile
+  alias Pinchflat.Sources.SourceDeletionWorker
   alias Pinchflat.Downloading.DownloadingHelpers
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
@@ -17,6 +18,7 @@ defmodule PinchflatWeb.Sources.SourceController do
       from s in Source,
         as: :source,
         inner_join: mp in assoc(s, :media_profile),
+        where: is_nil(s.marked_for_deletion_at) and is_nil(mp.marked_for_deletion_at),
         preload: [media_profile: mp],
         order_by: [asc: s.custom_name],
         select: map(s, ^Source.__schema__(:fields)),
@@ -124,19 +126,15 @@ defmodule PinchflatWeb.Sources.SourceController do
   end
 
   def delete(conn, %{"id" => id} = params) do
-    delete_files = Map.get(params, "delete_files", false)
+    # This awkward comparison converts the string to a boolean
+    delete_files = Map.get(params, "delete_files", "") == "true"
     source = Sources.get_source!(id)
-    {:ok, _source} = Sources.delete_source(source, delete_files: delete_files)
 
-    flash_message =
+    {:ok, _} = Sources.update_source(source, %{marked_for_deletion_at: DateTime.utc_now()})
-      if delete_files do
+    SourceDeletionWorker.kickoff(source, %{delete_files: delete_files})
-        "Source and files deleted successfully."
-      else
-        "Source deleted successfully. Files were not deleted."
-      end
 
     conn
-    |> put_flash(:info, flash_message)
+    |> put_flash(:info, "Source deletion started. This may take a while to complete.")
     |> redirect(to: ~p"/sources")
   end
 

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -94,7 +94,7 @@
       maxlength="10"
       pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
       title="YYYY-MM-DD"
-      help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
+      help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format. Old media may be deleted or downloaded if you change this date"
     />
 
     <.input

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "2024.7.18",
+      version: "2024.7.30",
       elixir: "~> 1.17",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
@@ -103,7 +103,8 @@ defmodule Pinchflat.MixProject do
       "ecto.rollback": [
         "ecto.rollback",
         ~s(cmd [ -z "$MIX_ENV" ] && yarn run create-erd || echo "No ERD generated")
-      ]
+      ],
+      "version.bump": "cmd ./tooling/version_bump.sh"
     ]
   end
 end

priv/repo/migrations/20240722183656_add_marked_for_deletion_at_to_sources_and_profiles.exs

@@ -0,0 +1,13 @@
+defmodule Pinchflat.Repo.Migrations.AddMarkedForDeletionAtToSources do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :marked_for_deletion_at, :utc_datetime
+    end
+
+    alter table(:media_profiles) do
+      add :marked_for_deletion_at, :utc_datetime
+    end
+  end
+end

test/pinchflat/downloading/downloading_helpers_test.exs

@@ -130,9 +130,6 @@ defmodule Pinchflat.Downloading.DownloadingHelpersTest do
       _download_prevented =
         media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4", prevent_download: true)
 
-      _culled =
-        media_item_fixture(source_id: source.id, media_filepath: "some/filepath.mp4", culled_at: now())
-
       assert [] = DownloadingHelpers.kickoff_redownload_for_existing_media(source)
 
       refute_enqueued(worker: MediaDownloadWorker)

test/pinchflat/downloading/media_download_worker_test.exs

@@ -10,7 +10,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
 
   setup do
     stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot -> {:ok, ""} end)
-    stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+    stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
     stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
 
     media_item =
@@ -162,20 +162,6 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       assert media_item.media_redownloaded_at == nil
     end
 
-    test "calls the user script runner", %{media_item: media_item} do
-      expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
-        {:ok, render_metadata(:media_metadata)}
-      end)
-
-      expect(UserScriptRunnerMock, :run, fn :media_downloaded, data ->
-        assert data.id == media_item.id
-
-        :ok
-      end)
-
-      perform_job(MediaDownloadWorker, %{id: media_item.id})
-    end
-
     test "does not blow up if the record doesn't exist" do
       assert :ok = perform_job(MediaDownloadWorker, %{id: 0})
     end
@@ -237,4 +223,59 @@ defmodule Pinchflat.Downloading.MediaDownloadWorkerTest do
       perform_job(MediaDownloadWorker, %{id: media_item.id, force: true})
     end
   end
+
+  describe "perform/1 when testing user script callbacks" do
+    setup do
+      stub(YtDlpRunnerMock, :run, fn _url, _opts, _ot, _addl ->
+        {:ok, render_metadata(:media_metadata)}
+      end)
+
+      :ok
+    end
+
+    test "calls the media_pre_download user script runner", %{media_item: media_item} do
+      expect(UserScriptRunnerMock, :run, fn :media_pre_download, data ->
+        assert data.id == media_item.id
+
+        {:ok, "", 0}
+      end)
+
+      expect(UserScriptRunnerMock, :run, fn :media_downloaded, _ -> {:ok, "", 0} end)
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id})
+    end
+
+    test "does not download the media if the pre-download script returns an error", %{media_item: media_item} do
+      expect(UserScriptRunnerMock, :run, fn :media_pre_download, _ -> {:ok, "", 1} end)
+
+      assert :ok = perform_job(MediaDownloadWorker, %{id: media_item.id})
+      media_item = Repo.reload!(media_item)
+
+      refute media_item.media_filepath
+      assert media_item.prevent_download
+    end
+
+    test "downloads media if the pre-download script is not present", %{media_item: media_item} do
+      expect(UserScriptRunnerMock, :run, fn :media_pre_download, _ -> {:ok, :no_executable} end)
+      expect(UserScriptRunnerMock, :run, fn :media_downloaded, _ -> {:ok, :no_executable} end)
+
+      assert :ok = perform_job(MediaDownloadWorker, %{id: media_item.id})
+      media_item = Repo.reload!(media_item)
+
+      assert media_item.media_filepath
+      refute media_item.prevent_download
+    end
+
+    test "calls the media_downloaded user script runner", %{media_item: media_item} do
+      expect(UserScriptRunnerMock, :run, fn :media_pre_download, _ -> {:ok, "", 0} end)
+
+      expect(UserScriptRunnerMock, :run, fn :media_downloaded, data ->
+        assert data.id == media_item.id
+
+        {:ok, "", 0}
+      end)
+
+      perform_job(MediaDownloadWorker, %{id: media_item.id})
+    end
+  end
 end

test/pinchflat/downloading/media_downloader_test.exs

@@ -5,6 +5,7 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
   import Pinchflat.SourcesFixtures
   import Pinchflat.ProfilesFixtures
 
+  alias Pinchflat.Media
   alias Pinchflat.Downloading.MediaDownloader
 
   setup do
@@ -123,6 +124,12 @@ defmodule Pinchflat.Downloading.MediaDownloaderTest do
       assert DateTime.diff(DateTime.utc_now(), updated_media_item.media_downloaded_at) < 2
     end
 
+    test "it sets the culled_at to nil", %{media_item: media_item} do
+      Media.update_media_item(media_item, %{culled_at: DateTime.utc_now()})
+      assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
+      assert updated_media_item.culled_at == nil
+    end
+
     test "it extracts the title", %{media_item: media_item} do
       assert {:ok, updated_media_item} = MediaDownloader.download_for_media_item(media_item)
       assert updated_media_item.title == "Pinchflat Example Video"

test/pinchflat/downloading/media_retention_worker_test.exs

@@ -7,15 +7,36 @@ defmodule Pinchflat.Downloading.MediaRetentionWorkerTest do
   alias Pinchflat.Media
   alias Pinchflat.Downloading.MediaRetentionWorker
 
-  describe "perform/1" do
+  setup do
-    setup do
+    stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
 
-      :ok
+    :ok
+  end
+
+  describe "perform/1" do
+    test "sets deleted media to not re-download" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_retention_date()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(new_media_item).prevent_download
+      assert Repo.reload!(old_media_item).prevent_download
     end
 
+    test "sets culled_at timestamp on deleted media" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_retention_date()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(new_media_item).culled_at
+      assert Repo.reload!(old_media_item).culled_at
+      assert DateTime.diff(now(), Repo.reload!(old_media_item).culled_at) < 1
+    end
+  end
+
+  describe "perform/1 when testing retention_period-based culling" do
     test "deletes media files that are past their retention date" do
-      {_source, old_media_item, new_media_item} = prepare_records()
+      {_source, old_media_item, new_media_item} = prepare_records_for_retention_date()
 
       perform_job(MediaRetentionWorker, %{})
 
@@ -25,27 +46,33 @@ defmodule Pinchflat.Downloading.MediaRetentionWorkerTest do
       refute Repo.reload!(old_media_item).media_filepath
     end
 
-    test "sets deleted media to not re-download" do
+    test "sets culled_at and prevent_download" do
-      {_source, old_media_item, new_media_item} = prepare_records()
+      {_source, old_media_item, new_media_item} = prepare_records_for_retention_date()
-
-      perform_job(MediaRetentionWorker, %{})
-
-      refute Repo.reload!(new_media_item).prevent_download
-      assert Repo.reload!(old_media_item).prevent_download
-    end
-
-    test "sets culled_at timestamp on deleted media" do
-      {_source, old_media_item, new_media_item} = prepare_records()
 
       perform_job(MediaRetentionWorker, %{})
 
       refute Repo.reload!(new_media_item).culled_at
       assert Repo.reload!(old_media_item).culled_at
-      assert DateTime.diff(now(), Repo.reload!(old_media_item).culled_at) < 1
+      refute Repo.reload!(new_media_item).prevent_download
+      assert Repo.reload!(old_media_item).prevent_download
+    end
+
+    test "doesn't cull if the source doesn't have a retention period" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_retention_date(nil)
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(new_media_item.media_filepath)
+      assert File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(new_media_item).media_filepath
+      assert Repo.reload!(old_media_item).media_filepath
+
+      refute Repo.reload!(new_media_item).culled_at
+      refute Repo.reload!(old_media_item).culled_at
     end
 
     test "doesn't cull media items that have prevent_culling set" do
-      {_source, old_media_item, _new_media_item} = prepare_records()
+      {_source, old_media_item, _new_media_item} = prepare_records_for_retention_date()
 
       Media.update_media_item(old_media_item, %{prevent_culling: true})
 
@@ -53,11 +80,99 @@ defmodule Pinchflat.Downloading.MediaRetentionWorkerTest do
 
       assert File.exists?(old_media_item.media_filepath)
       assert Repo.reload!(old_media_item).media_filepath
+      refute Repo.reload!(old_media_item).culled_at
+    end
+
+    test "doesn't cull if the media item has no media_filepath" do
+      {_source, old_media_item, _new_media_item} = prepare_records_for_retention_date()
+
+      Media.update_media_item(old_media_item, %{media_filepath: nil})
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(old_media_item).culled_at
     end
   end
 
-  defp prepare_records do
+  describe "perform/1 when testing source cutoff-based culling" do
-    source = source_fixture(%{retention_period_days: 2})
+    test "culls media from before the cutoff date" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_source_cutoff_date()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(new_media_item.media_filepath)
+      refute File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(new_media_item).media_filepath
+      refute Repo.reload!(old_media_item).media_filepath
+    end
+
+    test "sets culled_at but not prevent_download" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_source_cutoff_date()
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(new_media_item).culled_at
+      assert Repo.reload!(old_media_item).culled_at
+      refute Repo.reload!(new_media_item).prevent_download
+      refute Repo.reload!(old_media_item).prevent_download
+    end
+
+    test "doesn't cull media if the source doesn't have a cutoff date" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_source_cutoff_date(nil)
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(new_media_item.media_filepath)
+      assert File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(new_media_item).media_filepath
+      assert Repo.reload!(old_media_item).media_filepath
+
+      refute Repo.reload!(new_media_item).culled_at
+      refute Repo.reload!(old_media_item).culled_at
+    end
+
+    test "doesn't cull media from on or after the cutoff date" do
+      {_source, old_media_item, new_media_item} = prepare_records_for_source_cutoff_date(2)
+
+      Media.update_media_item(old_media_item, %{uploaded_at: now_minus(2, :days)})
+      Media.update_media_item(new_media_item, %{uploaded_at: now_minus(1, :day)})
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(new_media_item.media_filepath)
+      assert File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(new_media_item).media_filepath
+      assert Repo.reload!(old_media_item).media_filepath
+
+      refute Repo.reload!(new_media_item).culled_at
+      refute Repo.reload!(old_media_item).culled_at
+    end
+
+    test "doesn't cull media items that have prevent_culling set" do
+      {_source, old_media_item, _new_media_item} = prepare_records_for_source_cutoff_date()
+
+      Media.update_media_item(old_media_item, %{prevent_culling: true})
+
+      perform_job(MediaRetentionWorker, %{})
+
+      assert File.exists?(old_media_item.media_filepath)
+      assert Repo.reload!(old_media_item).media_filepath
+      refute Repo.reload!(old_media_item).culled_at
+    end
+
+    test "doesn't cull if the media item has no media_filepath" do
+      {_source, old_media_item, _new_media_item} = prepare_records_for_source_cutoff_date()
+
+      Media.update_media_item(old_media_item, %{media_filepath: nil})
+
+      perform_job(MediaRetentionWorker, %{})
+
+      refute Repo.reload!(old_media_item).culled_at
+    end
+  end
+
+  defp prepare_records_for_retention_date(retention_period_days \\ 2) do
+    source = source_fixture(%{retention_period_days: retention_period_days})
 
     old_media_item =
       media_item_with_attachments(%{
@@ -73,4 +188,23 @@ defmodule Pinchflat.Downloading.MediaRetentionWorkerTest do
 
     {source, old_media_item, new_media_item}
   end
+
+  defp prepare_records_for_source_cutoff_date(download_cutoff_date_days_ago \\ 2) do
+    cutoff_date = if download_cutoff_date_days_ago, do: now_minus(download_cutoff_date_days_ago, :days), else: nil
+    source = source_fixture(%{download_cutoff_date: cutoff_date})
+
+    old_media_item =
+      media_item_with_attachments(%{
+        source_id: source.id,
+        uploaded_at: now_minus(3, :days)
+      })
+
+    new_media_item =
+      media_item_with_attachments(%{
+        source_id: source.id,
+        uploaded_at: now_minus(1, :day)
+      })
+
+    {source, old_media_item, new_media_item}
+  end
 end

test/pinchflat/lifecycle/user_scripts/command_runner_test.exs

@@ -19,7 +19,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunnerTest do
       File.write(filepath(), "#!/bin/bash\ntouch #{filename}\n")
 
       refute File.exists?(filename)
-      assert :ok = Runner.run(:media_downloaded, %{})
+      assert {:ok, _, _} = Runner.run(:media_downloaded, %{})
       assert File.exists?(filename)
     end
 
@@ -27,7 +27,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunnerTest do
       tmp_dir = Application.get_env(:pinchflat, :tmpfile_directory)
       File.write(filepath(), "#!/bin/bash\necho $1 > #{tmp_dir}/event_name\n")
 
-      assert :ok = Runner.run(:media_downloaded, %{})
+      assert {:ok, _, _} = Runner.run(:media_downloaded, %{})
       assert File.read!("#{tmp_dir}/event_name") == "media_downloaded\n"
     end
 
@@ -35,26 +35,32 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunnerTest do
       tmp_dir = Application.get_env(:pinchflat, :tmpfile_directory)
       File.write(filepath(), "#!/bin/bash\necho $2 > #{tmp_dir}/encoded_data\n")
 
-      assert :ok = Runner.run(:media_downloaded, %{foo: "bar"})
+      assert {:ok, _, _} = Runner.run(:media_downloaded, %{foo: "bar"})
       assert File.read!("#{tmp_dir}/encoded_data") == "{\"foo\":\"bar\"}\n"
     end
 
     test "does nothing if the lifecycle file is not present" do
       :ok = File.rm(filepath())
 
-      assert :ok = Runner.run(:media_downloaded, %{})
+      assert {:ok, :no_executable} = Runner.run(:media_downloaded, %{})
     end
 
     test "does nothing if the lifecycle file is empty" do
       File.write(filepath(), "")
 
-      assert :ok = Runner.run(:media_downloaded, %{})
+      assert {:ok, :no_executable} = Runner.run(:media_downloaded, %{})
     end
 
     test "returns :ok if the command exits with a non-zero status" do
       File.write(filepath(), "#!/bin/bash\nexit 1\n")
 
-      assert :ok = Runner.run(:media_downloaded, %{})
+      assert {:ok, _, 1} = Runner.run(:media_downloaded, %{})
+    end
+
+    test "returns the output of the command" do
+      File.write(filepath(), "#!/bin/bash\necho 'hello'\n")
+
+      assert {:ok, "hello\n", 0} = Runner.run(:media_downloaded, %{})
     end
 
     test "gets upset if you pass an invalid event type" do

test/pinchflat/media_test.exs

@@ -41,99 +41,7 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
-  describe "list_cullable_media_items/0" do
+  describe "list_upgradeable_media_items/0" do
-    test "returns media items where the source has a retention period" do
-      source_one = source_fixture(%{retention_period_days: 2})
-      source_two = source_fixture(%{retention_period_days: 0})
-      source_three = source_fixture(%{retention_period_days: nil})
-
-      _media_item =
-        media_item_fixture(%{
-          source_id: source_two.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      _media_item =
-        media_item_fixture(%{
-          source_id: source_three.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      expected_media_item =
-        media_item_fixture(%{
-          source_id: source_one.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      assert Media.list_cullable_media_items() == [expected_media_item]
-    end
-
-    test "returns media_items with a media_filepath" do
-      source = source_fixture(%{retention_period_days: 2})
-
-      _media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: nil,
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      expected_media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      assert Media.list_cullable_media_items() == [expected_media_item]
-    end
-
-    test "returns items that have passed their retention period" do
-      source = source_fixture(%{retention_period_days: 2})
-
-      _media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(2, :days)
-        })
-
-      expected_media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      assert Media.list_cullable_media_items() == [expected_media_item]
-    end
-
-    test "doesn't return items that are set to prevent culling" do
-      source = source_fixture(%{retention_period_days: 2})
-
-      _media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days),
-          prevent_culling: true
-        })
-
-      expected_media_item =
-        media_item_fixture(%{
-          source_id: source.id,
-          media_filepath: "/video/#{Faker.File.file_name(:video)}",
-          media_downloaded_at: now_minus(3, :days)
-        })
-
-      assert Media.list_cullable_media_items() == [expected_media_item]
-    end
-  end
-
-  describe "list_redownloadable_media_items/0" do
     setup do
       media_profile = media_profile_fixture(%{redownload_delay_days: 4})
       source = source_fixture(%{media_profile_id: media_profile.id, inserted_at: now_minus(10, :days)})
@@ -149,7 +57,7 @@ defmodule Pinchflat.MediaTest do
           media_downloaded_at: now_minus(5, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == [media_item]
+      assert Media.list_upgradeable_media_items() == [media_item]
     end
 
     test "returns media items that were downloaded in past but still meet redownload delay", %{source: source} do
@@ -160,7 +68,7 @@ defmodule Pinchflat.MediaTest do
           media_downloaded_at: now_minus(19, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == [media_item]
+      assert Media.list_upgradeable_media_items() == [media_item]
     end
 
     test "does not return media items without a media_downloaded_at", %{source: source} do
@@ -171,7 +79,7 @@ defmodule Pinchflat.MediaTest do
           media_downloaded_at: nil
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items that are set to prevent download", %{source: source} do
@@ -183,7 +91,7 @@ defmodule Pinchflat.MediaTest do
           prevent_download: true
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items that have been culled", %{source: source} do
@@ -195,7 +103,7 @@ defmodule Pinchflat.MediaTest do
           culled_at: now()
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items before the download delay", %{source: source} do
@@ -206,7 +114,7 @@ defmodule Pinchflat.MediaTest do
           media_downloaded_at: now_minus(3, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items that have already been redownloaded", %{source: source} do
@@ -218,7 +126,7 @@ defmodule Pinchflat.MediaTest do
           media_redownloaded_at: now()
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items that were first downloaded well after the uploaded_at", %{source: source} do
@@ -229,7 +137,7 @@ defmodule Pinchflat.MediaTest do
           uploaded_at: now_minus(20, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items that were recently uploaded", %{source: source} do
@@ -240,7 +148,7 @@ defmodule Pinchflat.MediaTest do
           uploaded_at: now_minus(2, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
 
     test "does not return media items without a redownload delay" do
@@ -254,7 +162,7 @@ defmodule Pinchflat.MediaTest do
           media_downloaded_at: now_minus(5, :days)
         })
 
-      assert Media.list_redownloadable_media_items() == []
+      assert Media.list_upgradeable_media_items() == []
     end
   end
 
@@ -776,7 +684,7 @@ defmodule Pinchflat.MediaTest do
 
   describe "delete_media_item/2 when testing file deletion" do
     setup do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       :ok
     end
@@ -837,7 +745,7 @@ defmodule Pinchflat.MediaTest do
       expect(UserScriptRunnerMock, :run, fn :media_deleted, data ->
         assert data.id == media_item.id
 
-        :ok
+        {:ok, "", 0}
       end)
 
       assert {:ok, _} = Media.delete_media_item(media_item, delete_files: true)
@@ -846,7 +754,7 @@ defmodule Pinchflat.MediaTest do
 
   describe "delete_media_files/2" do
     setup do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       :ok
     end
@@ -909,7 +817,7 @@ defmodule Pinchflat.MediaTest do
       expect(UserScriptRunnerMock, :run, fn :media_deleted, data ->
         assert data.id == media_item.id
 
-        :ok
+        {:ok, "", 0}
       end)
 
       assert {:ok, _} = Media.delete_media_files(media_item)

test/pinchflat/profiles/media_profile_deletion_worker_test.exs

@@ -0,0 +1,57 @@
+defmodule Pinchflat.Profiles.MediaProfileDeletionWorkerTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
+
+  alias Pinchflat.Profiles.MediaProfileDeletionWorker
+
+  setup do
+    stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+
+    {:ok, %{profile: media_profile_fixture()}}
+  end
+
+  describe "kickoff/3" do
+    test "starts the worker", %{profile: profile} do
+      assert [] = all_enqueued(worker: MediaProfileDeletionWorker)
+      assert {:ok, _} = MediaProfileDeletionWorker.kickoff(profile)
+      assert [_] = all_enqueued(worker: MediaProfileDeletionWorker)
+    end
+
+    test "can be called with additional job arguments", %{profile: profile} do
+      job_args = %{"delete_files" => true}
+
+      assert {:ok, _} = MediaProfileDeletionWorker.kickoff(profile, job_args)
+
+      assert_enqueued(worker: MediaProfileDeletionWorker, args: %{"id" => profile.id, "delete_files" => true})
+    end
+  end
+
+  describe "perform/1" do
+    test "deletes the profile, sources, and media but leaves the files", %{profile: profile} do
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_with_attachments(%{source_id: source.id})
+
+      perform_job(MediaProfileDeletionWorker, %{"id" => profile.id})
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(profile) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      assert File.exists?(media_item.media_filepath)
+    end
+
+    test "deletes the profile, sources, and media and files if specified", %{profile: profile} do
+      source = source_fixture(%{media_profile_id: profile.id})
+      media_item = media_item_with_attachments(%{source_id: source.id})
+
+      perform_job(MediaProfileDeletionWorker, %{"id" => profile.id, "delete_files" => true})
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(profile) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      refute File.exists?(media_item.media_filepath)
+    end
+  end
+end

test/pinchflat/profiles_test.exs

@@ -113,7 +113,7 @@ defmodule Pinchflat.ProfilesTest do
 
   describe "delete_media_profile/2 when deleting files" do
     setup do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       :ok
     end

test/pinchflat/sources/source_deletion_worker_test.exs

@@ -0,0 +1,52 @@
+defmodule Pinchflat.Sources.SourceDeletionWorkerTest do
+  use Pinchflat.DataCase
+
+  import Pinchflat.MediaFixtures
+  import Pinchflat.SourcesFixtures
+
+  alias Pinchflat.Sources.SourceDeletionWorker
+
+  setup do
+    stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+
+    {:ok, %{source: source_fixture()}}
+  end
+
+  describe "kickoff/3" do
+    test "starts the worker", %{source: source} do
+      assert [] = all_enqueued(worker: SourceDeletionWorker)
+      assert {:ok, _} = SourceDeletionWorker.kickoff(source)
+      assert [_] = all_enqueued(worker: SourceDeletionWorker)
+    end
+
+    test "can be called with additional job arguments", %{source: source} do
+      job_args = %{"delete_files" => true}
+
+      assert {:ok, _} = SourceDeletionWorker.kickoff(source, job_args)
+
+      assert_enqueued(worker: SourceDeletionWorker, args: %{"id" => source.id, "delete_files" => true})
+    end
+  end
+
+  describe "perform/1" do
+    test "deletes the source but leaves the files", %{source: source} do
+      media_item = media_item_with_attachments(%{source_id: source.id})
+
+      perform_job(SourceDeletionWorker, %{"id" => source.id})
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      assert File.exists?(media_item.media_filepath)
+    end
+
+    test "deletes the source and files if specified", %{source: source} do
+      media_item = media_item_with_attachments(%{source_id: source.id})
+
+      perform_job(SourceDeletionWorker, %{"id" => source.id, "delete_files" => true})
+
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
+      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
+      refute File.exists?(media_item.media_filepath)
+    end
+  end
+end

test/pinchflat/sources_test.exs

@@ -617,7 +617,7 @@ defmodule Pinchflat.SourcesTest do
 
   describe "delete_source/2 when deleting files" do
     setup do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       :ok
     end

test/pinchflat_web/controllers/media_item_controller_test.exs

@@ -56,7 +56,7 @@ defmodule PinchflatWeb.MediaItemControllerTest do
   describe "delete media" do
     setup do
       media_item = media_item_with_attachments()
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       %{media_item: media_item}
     end

test/pinchflat_web/controllers/media_profile_controller_test.exs

@@ -1,12 +1,11 @@
 defmodule PinchflatWeb.MediaProfileControllerTest do
   use PinchflatWeb.ConnCase
 
-  import Pinchflat.MediaFixtures
-  import Pinchflat.SourcesFixtures
   import Pinchflat.ProfilesFixtures
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.Profiles.MediaProfileDeletionWorker
 
   @create_attrs %{name: "some name", output_path_template: "output_template.{{ ext }}"}
   @update_attrs %{
@@ -23,8 +22,17 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
 
   describe "index" do
     test "lists all media_profiles", %{conn: conn} do
+      profile = media_profile_fixture()
       conn = get(conn, ~p"/media_profiles")
+
       assert html_response(conn, 200) =~ "Media Profiles"
+      assert html_response(conn, 200) =~ profile.name
+    end
+
+    test "omits profiles that have marked_for_deletion_at set", %{conn: conn} do
+      profile = media_profile_fixture(marked_for_deletion_at: DateTime.utc_now())
+      conn = get(conn, ~p"/media_profiles")
+      refute html_response(conn, 200) =~ profile.name
     end
   end
 
@@ -102,34 +110,28 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
     end
   end
 
-  describe "delete media_profile when just deleting the records" do
+  describe "delete media_profile in all cases" do
     setup [:create_media_profile]
 
-    test "deletes chosen media_profile and its associations", %{conn: conn, media_profile: media_profile} do
-      source = source_fixture(media_profile_id: media_profile.id)
-      media_item = media_item_with_attachments(%{source_id: source.id})
-
-      conn = delete(conn, ~p"/media_profiles/#{media_profile}")
-      assert redirected_to(conn) == ~p"/media_profiles"
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_profile) end
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
-    end
-
     test "redirects to the media_profiles page", %{conn: conn, media_profile: media_profile} do
       conn = delete(conn, ~p"/media_profiles/#{media_profile}")
 
       assert redirected_to(conn) == ~p"/media_profiles"
     end
 
-    test "doesn't delete any files", %{conn: conn, media_profile: media_profile} do
+    test "sets marked_for_deletion_at", %{conn: conn, media_profile: media_profile} do
-      source = source_fixture(media_profile_id: media_profile.id)
+      delete(conn, ~p"/media_profiles/#{media_profile}")
-      media_item = media_item_with_attachments(%{source_id: source.id})
+      assert Repo.reload!(media_profile).marked_for_deletion_at
+    end
+  end
 
+  describe "delete media_profile when just deleting the records" do
+    setup [:create_media_profile]
+
+    test "enqueues a job without the delete_files arg", %{conn: conn, media_profile: media_profile} do
       delete(conn, ~p"/media_profiles/#{media_profile}")
 
-      assert File.exists?(media_item.media_filepath)
+      assert [%{args: %{"delete_files" => false}}] = all_enqueued(worker: MediaProfileDeletionWorker)
     end
   end
 
@@ -137,36 +139,15 @@ defmodule PinchflatWeb.MediaProfileControllerTest do
     setup [:create_media_profile]
 
     setup do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
+      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> {:ok, "", 0} end)
 
       :ok
     end
 
-    test "deletes chosen media_profile and its associations", %{conn: conn, media_profile: media_profile} do
+    test "enqueues a job with the delete_files arg", %{conn: conn, media_profile: media_profile} do
-      source = source_fixture(media_profile_id: media_profile.id)
-      media_item = media_item_with_attachments(%{source_id: source.id})
-
-      conn = delete(conn, ~p"/media_profiles/#{media_profile}?delete_files=true")
-      assert redirected_to(conn) == ~p"/media_profiles"
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_profile) end
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
-    end
-
-    test "redirects to the media_profiles page", %{conn: conn, media_profile: media_profile} do
-      conn = delete(conn, ~p"/media_profiles/#{media_profile}?delete_files=true")
-
-      assert redirected_to(conn) == ~p"/media_profiles"
-    end
-
-    test "deletes the files", %{conn: conn, media_profile: media_profile} do
-      source = source_fixture(media_profile_id: media_profile.id)
-      media_item = media_item_with_attachments(%{source_id: source.id})
-
       delete(conn, ~p"/media_profiles/#{media_profile}?delete_files=true")
 
-      refute File.exists?(media_item.media_filepath)
+      assert [%{args: %{"delete_files" => true}}] = all_enqueued(worker: MediaProfileDeletionWorker)
     end
   end
 

test/pinchflat_web/controllers/source_controller_test.exs

@@ -7,6 +7,7 @@ defmodule PinchflatWeb.SourceControllerTest do
 
   alias Pinchflat.Repo
   alias Pinchflat.Settings
+  alias Pinchflat.Sources.SourceDeletionWorker
   alias Pinchflat.Downloading.MediaDownloadWorker
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
   alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
@@ -33,8 +34,26 @@ defmodule PinchflatWeb.SourceControllerTest do
 
   describe "index" do
     test "lists all sources", %{conn: conn} do
+      source = source_fixture()
       conn = get(conn, ~p"/sources")
+
       assert html_response(conn, 200) =~ "Sources"
+      assert html_response(conn, 200) =~ source.custom_name
+    end
+
+    test "omits sources that have marked_for_deletion_at set", %{conn: conn} do
+      source = source_fixture(marked_for_deletion_at: DateTime.utc_now())
+      conn = get(conn, ~p"/sources")
+
+      refute html_response(conn, 200) =~ source.custom_name
+    end
+
+    test "omits sources who's media profile has marked_for_deletion_at set", %{conn: conn} do
+      media_profile = media_profile_fixture(marked_for_deletion_at: DateTime.utc_now())
+      source = source_fixture(media_profile_id: media_profile.id)
+      conn = get(conn, ~p"/sources")
+
+      refute html_response(conn, 200) =~ source.custom_name
     end
   end
 
@@ -127,51 +146,37 @@ defmodule PinchflatWeb.SourceControllerTest do
     end
   end
 
-  describe "delete source when just deleting the records" do
+  describe "delete source in all cases" do
     setup [:create_source]
 
-    test "deletes chosen source and media_items", %{conn: conn, source: source, media_item: media_item} do
-      delete(conn, ~p"/sources/#{source}")
-
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
-    end
-
     test "redirects to the sources page", %{conn: conn, source: source} do
       conn = delete(conn, ~p"/sources/#{source}")
       assert redirected_to(conn) == ~p"/sources"
     end
 
-    test "does not delete the files", %{conn: conn, source: source, media_item: media_item} do
+    test "sets marked_for_deletion_at", %{conn: conn, source: source} do
       delete(conn, ~p"/sources/#{source}")
-      assert File.exists?(media_item.media_filepath)
+      assert Repo.reload!(source).marked_for_deletion_at
+    end
+  end
+
+  describe "delete source when just deleting the records" do
+    setup [:create_source]
+
+    test "enqueues a job without the delete_files arg", %{conn: conn, source: source} do
+      delete(conn, ~p"/sources/#{source}")
+
+      assert [%{args: %{"delete_files" => false}}] = all_enqueued(worker: SourceDeletionWorker)
     end
   end
 
   describe "delete source when deleting the records and files" do
     setup [:create_source]
 
-    setup do
+    test "enqueues a job without the delete_files arg", %{conn: conn, source: source} do
-      stub(UserScriptRunnerMock, :run, fn _event_type, _data -> :ok end)
-
-      :ok
-    end
-
-    test "deletes chosen source and media_items", %{conn: conn, source: source, media_item: media_item} do
       delete(conn, ~p"/sources/#{source}?delete_files=true")
 
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(source) end
+      assert [%{args: %{"delete_files" => true}}] = all_enqueued(worker: SourceDeletionWorker)
-      assert_raise Ecto.NoResultsError, fn -> Repo.reload!(media_item) end
-    end
-
-    test "redirects to the sources page", %{conn: conn, source: source} do
-      conn = delete(conn, ~p"/sources/#{source}?delete_files=true")
-      assert redirected_to(conn) == ~p"/sources"
-    end
-
-    test "deletes the files", %{conn: conn, source: source, media_item: media_item} do
-      delete(conn, ~p"/sources/#{source}?delete_files=true")
-      refute File.exists?(media_item.media_filepath)
     end
   end
 

tooling/version_bump.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Get the current date in the format YYYY.MM.DD (stripping leading zeros)
+DATE=$(date +"%Y.%-m.%-d")
+
+# Get the current version from mix.exs
+VERSION=$(grep "version: " mix.exs | cut -d '"' -f2)
+
+echo "Bumping version from $VERSION to $DATE"
+# Replace the version in mix.exs with the new version
+sed -i "s/version: \"$VERSION\"/version: \"$DATE\"/g" mix.exs
+
+# Run checks to ensure it's a valid mix.exs file
+mix check