[Enhancement] Adds ability to enable/disable sources (#481)

* [Unrelated] updated module name for existing liveview module

* Updated toggle component and moved MP index table to a liveview

* [WIP] reverted MP index table; added source count to MP index

* Moved new live table to sources index

* Added 'enabled' boolean to sources

* Got 'enabled' logic working re: downloading pending media

* Updated sources context to do the right thing when a source is updated

* Docs and tests

* Updated slow indexing to maintain its old schedule if re-enabled

* Hooked up the enabled toggle to the sources page

* [Unrelated] added direct links to various tabs on the sources table

* More tests

* Removed unneeded guard in

* Removed outdated comment

lib/pinchflat/fast_indexing/fast_indexing_helpers.ex

@@ -11,14 +11,27 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
 
   alias Pinchflat.Repo
   alias Pinchflat.Media
+  alias Pinchflat.Tasks
   alias Pinchflat.Sources.Source
   alias Pinchflat.FastIndexing.YoutubeRss
   alias Pinchflat.FastIndexing.YoutubeApi
   alias Pinchflat.Downloading.DownloadingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.Downloading.DownloadOptionBuilder
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
 
+  @doc """
+  Kicks off a new fast indexing task for a source. This will delete any existing fast indexing
+  tasks for the source before starting a new one.
+
+  Returns {:ok, %Task{}}
+  """
+  def kickoff_indexing_task(%Source{} = source) do
+    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
+    FastIndexingWorker.kickoff_with_task(source)
+  end
+
   @doc """
   Fetches new media IDs for a source from YT's API or RSS, indexes them, and kicks off downloading
   tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the

lib/pinchflat/profiles/profiles_query.ex

@@ -0,0 +1,29 @@
+defmodule Pinchflat.Profiles.ProfilesQuery do
+  @moduledoc """
+  Query helpers for the Profiles context.
+
+  These methods are made to be one-ish liners used
+  to compose queries. Each method should strive to do
+  _one_ thing. These don't need to be tested as
+  they are just building blocks for other functionality
+  which, itself, will be tested.
+  """
+  import Ecto.Query, warn: false
+
+  alias Pinchflat.Profiles.MediaProfile
+
+  # This allows the module to be aliased and query methods to be used
+  # all in one go
+  # usage: use Pinchflat.Profiles.ProfilesQuery
+  defmacro __using__(_opts) do
+    quote do
+      import Ecto.Query, warn: false
+
+      alias unquote(__MODULE__)
+    end
+  end
+
+  def new do
+    MediaProfile
+  end
+end

lib/pinchflat/slow_indexing/slow_indexing_helpers.ex

@@ -25,13 +25,28 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
   Starts tasks for indexing a source's media regardless of the source's indexing
   frequency. It's assumed the caller will check for indexing frequency.
 
-  Returns {:ok, %Task{}}.
+  Returns {:ok, %Task{}}
   """
   def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
+    job_offset_seconds = calculate_job_offset_seconds(source)
+
     Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
     Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
 
-    MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
+    MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts ++ [schedule_in: job_offset_seconds])
+  end
+
+  @doc """
+  A helper method to delete all indexing-related tasks for a source.
+  Optionally, you can include executing tasks in the deletion process.
+
+  Returns :ok
+  """
+  def delete_indexing_tasks(%Source{} = source, opts \\ []) do
+    include_executing = Keyword.get(opts, :include_executing, false)
+
+    Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: include_executing)
+    Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: include_executing)
   end
 
   @doc """
@@ -141,4 +156,14 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
         changeset
     end
   end
+
+  # Find the difference between the current time and the last time the source was indexed
+  defp calculate_job_offset_seconds(%Source{last_indexed_at: nil}), do: 0
+
+  defp calculate_job_offset_seconds(source) do
+    offset_seconds = DateTime.diff(DateTime.utc_now(), source.last_indexed_at, :second)
+    index_frequency_seconds = source.index_frequency_minutes * 60
+
+    max(0, index_frequency_seconds - offset_seconds)
+  end
 end

lib/pinchflat/sources/source.ex

@@ -15,6 +15,7 @@ defmodule Pinchflat.Sources.Source do
   alias Pinchflat.Metadata.SourceMetadata
 
   @allowed_fields ~w(
+    enabled
     collection_name
     collection_id
     collection_type
@@ -64,6 +65,7 @@ defmodule Pinchflat.Sources.Source do
                                 )a
 
   schema "sources" do
+    field :enabled, :boolean, default: true
     # This is _not_ used as the primary key or internally in the database
     # relations. This is only used to prevent an enumeration attack on the streaming
     # and RSS feed endpoints since those _must_ be public (ie: no basic auth)

lib/pinchflat/sources/sources.ex

@@ -15,8 +15,8 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.Metadata.SourceMetadata
   alias Pinchflat.Utils.FilesystemUtils
   alias Pinchflat.Downloading.DownloadingHelpers
-  alias Pinchflat.FastIndexing.FastIndexingWorker
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
+  alias Pinchflat.FastIndexing.FastIndexingHelpers
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
   @doc """
@@ -255,19 +255,40 @@ defmodule Pinchflat.Sources do
     end
   end
 
-  # If the source is NOT new (ie: updated) and the download_media flag has changed,
+  # If the source is new (ie: not persisted), do nothing
+  defp maybe_handle_media_tasks(%{data: %{__meta__: %{state: state}}}, _source) when state != :loaded do
+    :ok
+  end
+
+  # If the source is NOT new (ie: updated),
   # enqueue or dequeue media download tasks as necessary.
   defp maybe_handle_media_tasks(changeset, source) do
-    case {changeset.data, changeset.changes} do
-      {%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
+    current_changes = changeset.changes
+    applied_changes = Ecto.Changeset.apply_changes(changeset)
+
+    # We need both current_changes and applied_changes to determine
+    # the course of action to take. For example, we only care if a source is supposed
+    # to be `enabled` or not - we don't care if that information comes from the
+    # current changes or if that's how it already was in the database.
+    # Rephrased, we're essentially using it in place of `get_field/2`
+    case {current_changes, applied_changes} do
+      {%{download_media: true}, %{enabled: true}} ->
         DownloadingHelpers.enqueue_pending_download_tasks(source)
 
-      {%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
+      {%{enabled: true}, %{download_media: true}} ->
+        DownloadingHelpers.enqueue_pending_download_tasks(source)
+
+      {%{download_media: false}, _} ->
+        DownloadingHelpers.dequeue_pending_download_tasks(source)
+
+      {%{enabled: false}, _} ->
         DownloadingHelpers.dequeue_pending_download_tasks(source)
 
       _ ->
-        :ok
+        nil
     end
+
+    :ok
   end
 
   defp maybe_run_indexing_task(changeset, source) do
@@ -301,13 +322,22 @@ defmodule Pinchflat.Sources do
   end
 
   defp maybe_update_slow_indexing_task(changeset, source) do
-    case changeset.changes do
-      %{index_frequency_minutes: mins} when mins > 0 ->
+    # See comment in `maybe_handle_media_tasks` as to why we need these
+    current_changes = changeset.changes
+    applied_changes = Ecto.Changeset.apply_changes(changeset)
+
+    case {current_changes, applied_changes} do
+      {%{index_frequency_minutes: mins}, %{enabled: true}} when mins > 0 ->
         SlowIndexingHelpers.kickoff_indexing_task(source)
 
-      %{index_frequency_minutes: _} ->
-        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-        Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
+      {%{enabled: true}, %{index_frequency_minutes: mins}} when mins > 0 ->
+        SlowIndexingHelpers.kickoff_indexing_task(source)
+
+      {%{index_frequency_minutes: _}, _} ->
+        SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
+
+      {%{enabled: false}, _} ->
+        SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
 
       _ ->
         :ok
@@ -315,13 +345,25 @@ defmodule Pinchflat.Sources do
   end
 
   defp maybe_update_fast_indexing_task(changeset, source) do
-    case changeset.changes do
-      %{fast_index: true} ->
-        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
-        FastIndexingWorker.kickoff_with_task(source)
+    # See comment in `maybe_handle_media_tasks` as to why we need these
+    current_changes = changeset.changes
+    applied_changes = Ecto.Changeset.apply_changes(changeset)
 
-      %{fast_index: false} ->
-        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
+    # This technically could be simplified since `maybe_update_slow_indexing_task`
+    # has some overlap re: deleting pending tasks, but I'm keeping it separate
+    # for clarity and explicitness.
+    case {current_changes, applied_changes} do
+      {%{fast_index: true}, %{enabled: true}} ->
+        FastIndexingHelpers.kickoff_indexing_task(source)
+
+      {%{enabled: true}, %{fast_index: true}} ->
+        FastIndexingHelpers.kickoff_indexing_task(source)
+
+      {%{fast_index: false}, _} ->
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
+
+      {%{enabled: false}, _} ->
+        Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
 
       _ ->
         :ok