[Ehnacement] More gracefully handle Sponsorblock failures (#169)

* Updated downloader and runner to handle sponsorblock failures more gracefully

* stopped download worker from running if a media item is preventing download

lib/pinchflat/downloading/media_download_worker.ex

@@ -40,8 +40,8 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
       |> Media.get_media_item!()
       |> Repo.preload(:source)
 
-    # If the source is set to not download media, perform a no-op
-    if media_item.source.download_media || args["force"] do
+    # If the source or media item is set to not download media, perform a no-op unless forced
+    if (media_item.source.download_media && !media_item.prevent_download) || args["force"] do
       download_media_and_schedule_jobs(media_item)
     else
       :ok
@@ -58,9 +58,10 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
 
         {:ok, updated_media_item}
 
-      err ->
-        Logger.error("Failed to download media for media item #{media_item.id}: #{inspect(err)}")
+      {:recovered, _} ->
+        {:error, :retry}
 
+      {:error, _message} ->
         {:error, :download_failed}
     end
   end

lib/pinchflat/downloading/media_downloader.ex

@@ -5,12 +5,15 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   to download the media with the desired options.
   """
 
+  require Logger
+
   alias Pinchflat.Repo
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Metadata.NfoBuilder
   alias Pinchflat.Metadata.MetadataParser
   alias Pinchflat.Metadata.MetadataFileHelpers
+  alias Pinchflat.Filesystem.FilesystemHelpers
   alias Pinchflat.Downloading.DownloadOptionBuilder
 
   alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@@ -27,33 +30,69 @@ defmodule Pinchflat.Downloading.MediaDownloader do
   Returns {:ok, %MediaItem{}} | {:error, any, ...any}
   """
   def download_for_media_item(%MediaItem{} = media_item) do
-    item_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
+    output_filepath = FilesystemHelpers.generate_metadata_tmpfile(:json)
+    media_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
 
-    case download_with_options(media_item.original_url, item_with_preloads) do
+    case download_with_options(media_item.original_url, media_with_preloads, output_filepath) do
       {:ok, parsed_json} ->
-        parsed_attrs =
-          parsed_json
-          |> MetadataParser.parse_for_media_item()
-          |> Map.merge(%{
-            media_downloaded_at: DateTime.utc_now(),
-            nfo_filepath: determine_nfo_filepath(item_with_preloads, parsed_json),
-            metadata: %{
-              # IDEA: might be worth kicking off a job for this since thumbnail fetching
-              # could fail and I want to handle that in isolation
-              metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, parsed_json),
-              thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_item, parsed_json)
-            }
-          })
+        update_media_item_from_parsed_json(media_with_preloads, parsed_json)
 
-        # Don't forgor to use preloaded associations or updates to
-        # associations won't work!
-        Media.update_media_item(item_with_preloads, parsed_attrs)
+      {:error, message, _exit_code} ->
+        Logger.error("yt-dlp download error for media item ##{media_with_preloads.id}: #{inspect(message)}")
+
+        if String.contains?(to_string(message), recoverable_errors()) do
+          attempt_update_media_item(media_with_preloads, output_filepath)
+
+          {:recovered, message}
+        else
+          {:error, message}
+        end
 
       err ->
-        err
+        Logger.error("Unknown error downloading media item ##{media_with_preloads.id}: #{inspect(err)}")
+
+        {:error, "Unknown error: #{inspect(err)}"}
     end
   end
 
+  defp attempt_update_media_item(media_with_preloads, output_filepath) do
+    with {:ok, contents} <- File.read(output_filepath),
+         {:ok, parsed_json} <- Phoenix.json_library().decode(contents) do
+      Logger.info("""
+      Recovery from yt-dlp error seems possible. Updating media item ##{media_with_preloads.id}
+      with parsed JSON from partial download attempt. Full download will be re-attemted in future
+      anyway
+      """)
+
+      update_media_item_from_parsed_json(media_with_preloads, parsed_json)
+    else
+      err ->
+        Logger.error("Unable to recover error for media item ##{media_with_preloads.id}: #{inspect(err)}")
+
+        {:error, :retry_failed}
+    end
+  end
+
+  defp update_media_item_from_parsed_json(media_with_preloads, parsed_json) do
+    parsed_attrs =
+      parsed_json
+      |> MetadataParser.parse_for_media_item()
+      |> Map.merge(%{
+        media_downloaded_at: DateTime.utc_now(),
+        nfo_filepath: determine_nfo_filepath(media_with_preloads, parsed_json),
+        metadata: %{
+          # IDEA: might be worth kicking off a job for this since thumbnail fetching
+          # could fail and I want to handle that in isolation
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_with_preloads, parsed_json),
+          thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_with_preloads, parsed_json)
+        }
+      })
+
+    # Don't forgor to use preloaded associations or updates to
+    # associations won't work!
+    Media.update_media_item(media_with_preloads, parsed_attrs)
+  end
+
   defp determine_nfo_filepath(media_item, parsed_json) do
     if media_item.source.media_profile.download_nfo do
       filepath = Path.rootname(parsed_json["filepath"]) <> ".nfo"
@@ -64,9 +103,15 @@ defmodule Pinchflat.Downloading.MediaDownloader do
     end
   end
 
-  defp download_with_options(url, item_with_preloads) do
+  defp download_with_options(url, item_with_preloads, output_filepath) do
     {:ok, options} = DownloadOptionBuilder.build(item_with_preloads)
 
-    YtDlpMedia.download(url, options)
+    YtDlpMedia.download(url, options, output_filepath: output_filepath)
+  end
+
+  defp recoverable_errors do
+    [
+      "Unable to communicate with SponsorBlock"
+    ]
   end
 end

lib/pinchflat/yt_dlp/command_runner.ex

@@ -29,7 +29,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     command = backend_executable()
     # These must stay in exactly this order, hence why I'm giving it its own variable.
     # Also, can't use RAM file since yt-dlp needs a concrete filepath.
-    output_filepath = Keyword.get(addl_opts, :output_filepath, FSUtils.generate_metadata_tmpfile(:json))
+    output_filepath = generate_output_filepath(addl_opts)
     print_to_file_opts = [{:print_to_file, output_template}, output_filepath]
     cookie_opts = build_cookie_options()
     formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts ++ cookie_opts)
@@ -61,6 +61,13 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     end
   end
 
+  defp generate_output_filepath(addl_opts) do
+    case Keyword.get(addl_opts, :output_filepath) do
+      nil -> FSUtils.generate_metadata_tmpfile(:json)
+      path -> path
+    end
+  end
+
   defp build_cookie_options do
     base_dir = Application.get_env(:pinchflat, :extras_directory)
     cookie_file = Path.join(base_dir, "cookies.txt")

lib/pinchflat/yt_dlp/media.ex

@@ -35,10 +35,10 @@ defmodule Pinchflat.YtDlp.Media do
 
   Returns {:ok, map()} | {:error, any, ...}.
   """
-  def download(url, command_opts \\ []) do
+  def download(url, command_opts \\ [], addl_opts \\ []) do
     opts = [:no_simulate] ++ command_opts
 
-    with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j"),
+    with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j", addl_opts),
          {:ok, parsed_json} <- Phoenix.json_library().decode(output) do
       {:ok, parsed_json}
     else