Improve metadata storage (#37)

* Updated config path to specify metadata storage path

* Removed metadata column from DB, replacing it with filepath columns

* Updated app to store compressed metadata; automatically download thumbnails

* Ensured metadata is deleted when other files are deleted

* Updated docs

* Added inets to application start so http calls work in prod

lib/pinchflat/http/http_behaviour.ex

@@ -0,0 +1,8 @@
+defmodule Pinchflat.HTTP.HTTPBehaviour do
+  @moduledoc """
+  This module defines the behaviour for HTTP clients. Literally just
+  so I can use Mox to create an HTTP mock
+  """
+
+  @callback get(String.t(), Keyword.t(), Keyword.t()) :: {:ok, String.t()} | {:error, String.t()}
+end

lib/pinchflat/http/http_client.ex

@@ -0,0 +1,35 @@
+defmodule Pinchflat.HTTP.HTTPClient do
+  @moduledoc """
+  This module provides a simple interface for making HTTP requests.
+
+  Made to be easily swappable with other HTTP clients. If you need more complexity
+  or security, check out HTTPoison or Mint.
+  """
+
+  alias Pinchflat.HTTP.HTTPBehaviour
+
+  @behaviour HTTPBehaviour
+
+  @doc """
+  Makes a GET request to the given URL and returns the response.
+
+  NOTE: I can't really test this with Mox and I can't think of a way to test this
+  that isn't ultimately redundant. I'm just going to leave it untested for now and
+  focus more on testing the consumers of this module.
+
+  Returns {:ok, String.t()} | {:error, String.t()}
+  """
+  @impl HTTPBehaviour
+  def get(url, headers \\ [], opts \\ []) do
+    case :httpc.request(:get, {url, headers}, [], opts) do
+      {:ok, {{_version, 200, _reason_phrase}, _headers, body}} ->
+        {:ok, body}
+
+      {:ok, {{_version, status_code, reason_phrase}, _headers, _body}} ->
+        {:error, "HTTP request failed with status code #{status_code}: #{reason_phrase}"}
+
+      {:error, reason} ->
+        {:error, "HTTP request failed: #{reason}"}
+    end
+  end
+end

lib/pinchflat/media.ex

@@ -9,6 +9,7 @@ defmodule Pinchflat.Media do
   alias Pinchflat.Tasks
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Sources.Source
+  alias Pinchflat.Media.MediaMetadata
 
   @doc """
   Returns the list of media_items. Returns [%MediaItem{}, ...].
@@ -103,6 +104,21 @@ defmodule Pinchflat.Media do
     |> Enum.filter(&is_binary/1)
   end
 
+  @doc """
+  Produces a flat list of the filesystem paths for a media_item's metadata files.
+  Returns an empty list if the media_item has no metadata.
+
+  Returns [binary()] | []
+  """
+  def metadata_filepaths(media_item) do
+    metadata = Repo.preload(media_item, :metadata).metadata || %MediaMetadata{}
+    mapped_struct = Map.from_struct(metadata)
+
+    MediaMetadata.filepath_attributes()
+    |> Enum.map(fn field -> mapped_struct[field] end)
+    |> Enum.filter(&is_binary/1)
+  end
+
   @doc """
   Creates a media_item. Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}.
   """
@@ -134,18 +150,31 @@ defmodule Pinchflat.Media do
   @doc """
   Deletes the media_item's associated files. Will leave the media_item in the database.
 
+  NOTE: this deletes the metadata files as well, but maybe it shouldn't? I'm wondering if
+  the metadata is more a concern of the DB record itself and should be lumped in with those
+  delete operations. But the metadata does come from the download operation of the file.
+  Food for thought but not a priority at the moment.
+
   Returns {:ok, %MediaItem{}}
   """
   def delete_attachments(media_item) do
+    media_item = Repo.preload(media_item, :metadata)
+
     media_item
     |> media_filepaths()
+    |> Enum.concat(metadata_filepaths(media_item))
     |> Enum.each(&File.rm/1)
 
-    # Fails if the directory is not empty
-    case File.rmdir(Path.dirname(media_item.media_filepath)) do
-      :ok -> {:ok, media_item}
-      {:error, :eexist} -> {:ok, media_item}
+    # rmdir will attempt to delete the directory, but only if it is empty
+    if media_item.media_filepath do
+      File.rmdir(Path.dirname(media_item.media_filepath))
     end
+
+    if media_item.metadata && media_item.metadata.metadata_filepath do
+      File.rmdir(Path.dirname(media_item.metadata.metadata_filepath))
+    end
+
+    {:ok, media_item}
   end
 
   @doc """

lib/pinchflat/media/media_metadata.ex

@@ -10,8 +10,12 @@ defmodule Pinchflat.Media.MediaMetadata do
 
   alias Pinchflat.Media.MediaItem
 
+  @allowed_fields ~w(metadata_filepath thumbnail_filepath)a
+  @required_fields ~w(metadata_filepath thumbnail_filepath)a
+
   schema "media_metadata" do
-    field :client_response, :map
+    field :metadata_filepath, :string
+    field :thumbnail_filepath, :string
 
     belongs_to :media_item, MediaItem
 
@@ -21,8 +25,13 @@ defmodule Pinchflat.Media.MediaMetadata do
   @doc false
   def changeset(media_metadata, attrs) do
     media_metadata
-    |> cast(attrs, [:client_response])
-    |> validate_required([:client_response])
+    |> cast(attrs, @allowed_fields)
+    |> validate_required(@required_fields)
     |> unique_constraint([:media_item_id])
   end
+
+  @doc false
+  def filepath_attributes do
+    ~w(metadata_filepath thumbnail_filepath)a
+  end
 end

lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex

@@ -44,8 +44,8 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do
   end
 
   defp generate_json_output_path do
-    metadata_directory = Application.get_env(:pinchflat, :metadata_directory)
-    filepath = Path.join([metadata_directory, "#{StringUtils.random_string(64)}.json"])
+    tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
+    filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.json"])
 
     # Ensure the file can be created and written to BEFORE we run the `yt-dlp` command
     :ok = File.mkdir_p!(Path.dirname(filepath))

lib/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers.ex

@@ -0,0 +1,72 @@
+defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers do
+  @moduledoc """
+  Provides methods for creating/downloading/storing related metadata
+  out-of-band of the normal yt-dlp backend process.
+
+  The idea is that I don't want to craft a complicated yt-dlp command,
+  instead focusing on downloading the video as the user wants it then
+  I can use the result of that here to grab the additional information
+  needed
+  """
+
+  @doc """
+  Compresses and stores metadata for a media item, returning the filepath.
+
+  Returns binary()
+  """
+  def compress_and_store_metadata_for(database_record, metadata_map) do
+    filepath = generate_filepath_for(database_record, "metadata.json.gz")
+    {:ok, json} = Phoenix.json_library().encode(metadata_map)
+
+    File.mkdir_p!(Path.dirname(filepath))
+    :ok = File.write(filepath, json, [:compressed])
+
+    filepath
+  end
+
+  @doc """
+  Reads and decodes compressed metadata from a filepath.
+
+  Returns {:ok, map()} | {:error, any}
+  """
+  def read_compressed_metadata(filepath) do
+    {:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :all))
+
+    Phoenix.json_library().decode(json)
+  end
+
+  @doc """
+  Downloads and stores a thumbnail for a media item, returning the filepath.
+
+  Returns binary()
+  """
+  def download_and_store_thumbnail_for(database_record, metadata_map) do
+    thumbnail_url = metadata_map["thumbnail"]
+    filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
+    thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
+
+    File.mkdir_p!(Path.dirname(filepath))
+    :ok = File.write(filepath, thumbnail_blob)
+
+    filepath
+  end
+
+  defp fetch_thumbnail_from_url(url) do
+    http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
+    {:ok, body} = http_client.get(url, [], body_format: :binary)
+
+    body
+  end
+
+  defp generate_filepath_for(database_record, filename) do
+    metadata_directory = Application.get_env(:pinchflat, :metadata_directory)
+    record_table_name = database_record.__meta__.source
+
+    Path.join([
+      metadata_directory,
+      record_table_name,
+      to_string(database_record.id),
+      filename
+    ])
+  end
+end

lib/pinchflat/media_client/backends/yt_dlp/metadata_parser.ex

@@ -16,13 +16,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
   Returns map()
   """
   def parse_for_media_item(metadata) do
-    metadata_attrs = %{
-      metadata: %{
-        client_response: metadata
-      }
-    }
-
-    metadata_attrs
+    Map.new()
     |> Map.merge(parse_media_metadata(metadata))
     |> Map.merge(parse_subtitle_metadata(metadata))
     |> Map.merge(parse_thumbnail_metadata(metadata))
@@ -38,7 +32,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataParser do
   end
 
   defp parse_subtitle_metadata(metadata) do
-    # IDEA: if needed, consider filtering out subtitles that don't exist on-disk
     subtitle_filepaths =
       (metadata["requested_subtitles"] || %{})
       |> Enum.map(fn {lang, attrs} -> [lang, attrs["filepath"]] end)

lib/pinchflat/media_client/video_downloader.ex

@@ -16,6 +16,7 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
   alias Pinchflat.MediaClient.Backends.YtDlp.Video, as: YtDlpVideo
   alias Pinchflat.Profiles.Options.YtDlp.DownloadOptionBuilder, as: YtDlpDownloadOptionBuilder
   alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: YtDlpMetadataParser
+  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: YtDlpMetadataHelpers
 
   @doc """
   Downloads a video for a media item, updating the media item based on the metadata
@@ -34,12 +35,18 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
 
     case download_for_media_profile(media_item.original_url, media_profile, backend) do
       {:ok, parsed_json} ->
-        parser = metadata_parser(backend)
+        {parser, helpers} = metadata_parsers(backend)
 
         parsed_attrs =
           parsed_json
           |> parser.parse_for_media_item()
-          |> Map.merge(%{media_downloaded_at: DateTime.utc_now()})
+          |> Map.merge(%{
+            media_downloaded_at: DateTime.utc_now(),
+            metadata: %{
+              metadata_filepath: helpers.compress_and_store_metadata_for(media_item, parsed_json),
+              thumbnail_filepath: helpers.download_and_store_thumbnail_for(media_item, parsed_json)
+            }
+          })
 
         # Don't forgor to use preloaded associations or updates to
         # associations won't work!
@@ -70,9 +77,9 @@ defmodule Pinchflat.MediaClient.VideoDownloader do
     end
   end
 
-  defp metadata_parser(backend) do
+  defp metadata_parsers(backend) do
     case backend do
-      :yt_dlp -> YtDlpMetadataParser
+      :yt_dlp -> {YtDlpMetadataParser, YtDlpMetadataHelpers}
     end
   end
 end