Improve metadata storage (#37)

* Updated config path to specify metadata storage path

* Removed metadata column from DB, replacing it with filepath columns

* Updated app to store compressed metadata; automatically download thumbnails

* Ensured metadata is deleted when other files are deleted

* Updated docs

* Added inets to application start so http calls work in prod

test/pinchflat/media_client/backends/yt_dlp/metadata_file_helpers_test.exs

@@ -0,0 +1,87 @@
+defmodule Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpersTest do
+  use Pinchflat.DataCase
+  import Mox
+  import Pinchflat.MediaFixtures
+
+  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers, as: Helpers
+
+  setup do
+    media_item = media_item_fixture()
+
+    {:ok, %{media_item: media_item}}
+  end
+
+  setup :verify_on_exit!
+
+  describe "compress_and_store_metadata_for/2" do
+    test "returns the filepath", %{media_item: media_item} do
+      metadata_map = %{"foo" => "bar"}
+
+      filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map)
+
+      assert filepath =~ ~r{/media_items/#{media_item.id}/metadata.json.gz}
+    end
+
+    test "creates folder structure based on passed record", %{media_item: media_item} do
+      metadata_map = %{"foo" => "bar"}
+
+      filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map)
+
+      assert File.exists?(Path.dirname(filepath))
+    end
+
+    test "stores it as compressed JSON", %{media_item: media_item} do
+      metadata_map = %{"foo" => "bar"}
+
+      filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map)
+      {:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :all))
+
+      assert json == Phoenix.json_library().encode!(metadata_map)
+    end
+  end
+
+  describe "read_compressed_metadata/1" do
+    test "returns the compressed and decoded metadata", %{media_item: media_item} do
+      metadata_map = %{"foo" => "bar"}
+
+      filepath = Helpers.compress_and_store_metadata_for(media_item, metadata_map)
+      {:ok, decoded_json} = Helpers.read_compressed_metadata(filepath)
+
+      assert decoded_json == metadata_map
+    end
+  end
+
+  describe "download_and_store_thumbnail_for/2" do
+    setup do
+      # This tests that the HTTP endpoint is being called with every test
+      expect(HTTPClientMock, :get, fn url, _headers, _opts ->
+        assert url =~ "example.com"
+
+        {:ok, "thumbnail data"}
+      end)
+
+      metadata = %{"thumbnail" => "example.com/thumbnail.jpg"}
+
+      {:ok, %{metadata: metadata}}
+    end
+
+    test "returns the filepath", %{media_item: media_item, metadata: metadata} do
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert filepath =~ ~r{/media_items/#{media_item.id}/thumbnail.jpg}
+    end
+
+    test "creates folder structure based on passed record", %{media_item: media_item, metadata: metadata} do
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert File.exists?(Path.dirname(filepath))
+    end
+
+    test "the filename and extension is based on the URL", %{media_item: media_item} do
+      metadata = %{"thumbnail" => "example.com/maxres.webp"}
+      filepath = Helpers.download_and_store_thumbnail_for(media_item, metadata)
+
+      assert Path.basename(filepath) == "maxres.webp"
+    end
+  end
+end

test/pinchflat/media_client/backends/yt_dlp/metadata_parser_test.exs

@@ -1,5 +1,5 @@
 defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
-  use ExUnit.Case, async: true
+  use Pinchflat.DataCase
 
   alias Pinchflat.MediaClient.Backends.YtDlp.MetadataParser, as: Parser
 
@@ -41,12 +41,6 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.MediaParserTest do
 
       assert is_binary(result.description)
     end
-
-    test "it returns the metadata as a map", %{metadata: metadata} do
-      result = Parser.parse_for_media_item(metadata)
-
-      assert result.metadata.client_response == metadata
-    end
   end
 
   describe "parse_for_media_item/1 when testing subtitle metadata" do

test/pinchflat/media_client/backends/yt_dlp/output_path_builder_test.exs

@@ -1,5 +1,5 @@
 defmodule Pinchflat.MediaClient.Backends.YtDlp.OutputPathBuilderTest do
-  use ExUnit.Case, async: true
+  use Pinchflat.DataCase
 
   alias Pinchflat.Profiles.Options.YtDlp.OutputPathBuilder
 

test/pinchflat/media_client/backends/yt_dlp/video_collection_test.exs

@@ -1,5 +1,5 @@
 defmodule Pinchflat.MediaClient.Backends.YtDlp.VideoCollectionTest do
-  use ExUnit.Case, async: true
+  use Pinchflat.DataCase
   import Mox
   import Pinchflat.SourcesFixtures
 

test/pinchflat/media_client/video_downloader_test.exs

@@ -14,6 +14,10 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
         [:metadata, source: :media_profile]
       )
 
+    stub(HTTPClientMock, :get, fn _url, _headers, _opts ->
+      {:ok, ""}
+    end)
+
     {:ok, %{media_item: media_item}}
   end
 
@@ -29,15 +33,16 @@ defmodule Pinchflat.MediaClient.VideoDownloaderTest do
       assert {:ok, _} = VideoDownloader.download_for_media_item(media_item)
     end
 
-    test "it saves the metadata to the database", %{media_item: media_item} do
+    test "it saves the metadata filepatha to the database", %{media_item: media_item} do
       expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot ->
         {:ok, render_metadata(:media_metadata)}
       end)
 
       assert is_nil(media_item.metadata)
       assert {:ok, updated_media_item} = VideoDownloader.download_for_media_item(media_item)
-      assert updated_media_item.metadata
-      assert is_map(updated_media_item.metadata.client_response)
+
+      assert updated_media_item.metadata.metadata_filepath =~ "media_items/#{media_item.id}/metadata.json.gz"
+      assert updated_media_item.metadata.thumbnail_filepath =~ "media_items/#{media_item.id}/maxresdefault.jpg"
     end
 
     test "errors are passed through", %{media_item: media_item} do

test/pinchflat/media_test.exs

@@ -1,6 +1,7 @@
 defmodule Pinchflat.MediaTest do
   use Pinchflat.DataCase
 
+  import Mox
   import Pinchflat.TasksFixtures
   import Pinchflat.MediaFixtures
   import Pinchflat.ProfilesFixtures
@@ -8,12 +9,17 @@ defmodule Pinchflat.MediaTest do
 
   alias Pinchflat.Media
   alias Pinchflat.Media.MediaItem
+  alias Pinchflat.MediaClient.Backends.YtDlp.MetadataFileHelpers
+
+  setup :verify_on_exit!
 
   @invalid_attrs %{title: nil, media_id: nil, media_filepath: nil}
 
   describe "schema" do
     test "media_metadata is deleted when media_item is deleted" do
-      media_item = media_item_fixture(%{metadata: %{client_response: %{foo: "bar"}}})
+      media_item =
+        media_item_fixture(%{metadata: %{metadata_filepath: "/metadata.json.gz", thumbnail_filepath: "/thumbnail.jpg"}})
+
       metadata = media_item.metadata
       assert {:ok, %MediaItem{}} = Media.delete_media_item(media_item)
 
@@ -259,6 +265,27 @@ defmodule Pinchflat.MediaTest do
     end
   end
 
+  describe "metadata_filepaths" do
+    test "returns filepaths in a flat list" do
+      filepaths = %{
+        metadata_filepath: "/metadata.json.gz",
+        thumbnail_filepath: "/thumbnail.jpg"
+      }
+
+      media_item = media_item_fixture(%{metadata: filepaths})
+
+      assert Media.metadata_filepaths(media_item) == [
+               "/metadata.json.gz",
+               "/thumbnail.jpg"
+             ]
+    end
+
+    test "returns an empty list when there is no metadata" do
+      media_item = media_item_fixture()
+      assert Media.metadata_filepaths(media_item) == []
+    end
+  end
+
   describe "create_media_item/1" do
     test "creating with valid data creates a media_item" do
       valid_attrs = %{
@@ -328,6 +355,26 @@ defmodule Pinchflat.MediaTest do
       refute File.exists?(media_item.media_filepath)
     end
 
+    test "deletes the media item's metadata files" do
+      stub(HTTPClientMock, :get, fn _url, _headers, _opts -> {:ok, ""} end)
+      media_item = Repo.preload(media_item_with_attachments(), :metadata)
+
+      update_attrs = %{
+        metadata: %{
+          metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_item, %{}),
+          thumbnail_filepath:
+            MetadataFileHelpers.download_and_store_thumbnail_for(media_item, %{
+              "thumbnail" => "https://example.com/thumbnail.jpg"
+            })
+        }
+      }
+
+      {:ok, updated_media_item} = Media.update_media_item(media_item, update_attrs)
+
+      assert {:ok, _} = Media.delete_attachments(updated_media_item)
+      refute File.exists?(updated_media_item.metadata.metadata_filepath)
+    end
+
     test "does not delete the media item" do
       media_item = media_item_with_attachments()
 

test/pinchflat/workers/video_download_worker_test.exs

@@ -16,6 +16,10 @@ defmodule Pinchflat.Workers.VideoDownloadWorkerTest do
         [:metadata, source: :media_profile]
       )
 
+    stub(HTTPClientMock, :get, fn _url, _headers, _opts ->
+      {:ok, ""}
+    end)
+
     {:ok, %{media_item: media_item}}
   end
 

test/support/fixtures/media_fixtures.ex

@@ -31,18 +31,13 @@ defmodule Pinchflat.MediaFixtures do
   Generate a media_item with metadata.
   """
   def media_item_with_metadata(attrs \\ %{}) do
-    json_filepath =
-      Path.join([
-        File.cwd!(),
-        "test",
-        "support",
-        "files",
-        "media_metadata.json"
-      ])
-
-    {:ok, file_body} = File.read(json_filepath)
-    {:ok, parsed_json} = Phoenix.json_library().decode(file_body)
-    merged_attrs = Map.merge(attrs, %{metadata: %{client_respinse: parsed_json}})
+    merged_attrs =
+      Map.merge(attrs, %{
+        metadata: %{
+          metadata_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/metadata.json.gz",
+          thumbnail_filepath: Application.get_env(:pinchflat, :metadata_directory) <> "/thumbnail.jpg"
+        }
+      })
 
     media_item_fixture(merged_attrs)
   end

test/test_helper.exs

@@ -1,6 +1,9 @@
 Mox.defmock(YtDlpRunnerMock, for: Pinchflat.MediaClient.Backends.BackendCommandRunner)
 Application.put_env(:pinchflat, :yt_dlp_runner, YtDlpRunnerMock)
 
+Mox.defmock(HTTPClientMock, for: Pinchflat.HTTP.HTTPBehaviour)
+Application.put_env(:pinchflat, :http_client, HTTPClientMock)
+
 ExUnit.start()
 Ecto.Adapters.SQL.Sandbox.mode(Pinchflat.Repo, :manual)
 Faker.start()
@@ -8,7 +11,9 @@ Faker.start()
 ExUnit.after_suite(fn _ ->
   File.rm_rf!(Application.get_env(:pinchflat, :media_directory))
   File.rm_rf!(Application.get_env(:pinchflat, :metadata_directory))
+  File.rm_rf!(Application.get_env(:pinchflat, :tmpfile_directory))
 
   File.mkdir_p!(Application.get_env(:pinchflat, :media_directory))
   File.mkdir_p!(Application.get_env(:pinchflat, :metadata_directory))
+  File.mkdir_p!(Application.get_env(:pinchflat, :tmpfile_directory))
 end)