Bumped version

* Added WIP 4k MP4 fix [skip ci]

* Added tests for new remux options

dev.Dockerfile

@@ -8,7 +8,7 @@ FROM ${DEV_IMAGE}
 # Install debian packages
 RUN apt-get update -qq
 RUN apt-get install -y inotify-tools ffmpeg curl git openssh-client \
-  python3 python3-pip python3-setuptools python3-wheel python3-dev locales
+  python3 python3-pip python3-setuptools python3-wheel python3-dev locales procps
 
 # Install nodejs
 RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh

lib/pinchflat/downloading/download_option_builder.ex

@@ -3,6 +3,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   Builds the options for yt-dlp to download media based on the given media profile.
   """
 
+  alias Pinchflat.Sources
   alias Pinchflat.Sources.Source
   alias Pinchflat.Media.MediaItem
   alias Pinchflat.Downloading.OutputPathBuilder
@@ -30,12 +31,12 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
 
   @doc """
   Builds the output path for yt-dlp to download media based on the given source's
-  media profile.
+  media profile. Uses the source's override output path template if it exists.
 
   Returns binary()
   """
   def build_output_path_for(%Source{} = source_with_preloads) do
-    output_path_template = source_with_preloads.media_profile.output_path_template
+    output_path_template = Sources.output_path_template(source_with_preloads)
 
     build_output_path(output_path_template, source_with_preloads)
   end
@@ -105,16 +106,18 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   end
 
   defp quality_options(media_profile) do
-    video_codec_options = "+codec:avc:m4a"
+    video_codec_option = fn res ->
+      [format_sort: "res:#{res},+codec:avc:m4a", remux_video: "mp4"]
+    end
 
     case media_profile.preferred_resolution do
       # Also be aware that :audio disabled all embedding options for subtitles
       :audio -> [:extract_audio, format: "bestaudio[ext=m4a]"]
-      :"360p" -> [format_sort: "res:360,#{video_codec_options}"]
-      :"480p" -> [format_sort: "res:480,#{video_codec_options}"]
-      :"720p" -> [format_sort: "res:720,#{video_codec_options}"]
-      :"1080p" -> [format_sort: "res:1080,#{video_codec_options}"]
-      :"2160p" -> [format_sort: "res:2160,#{video_codec_options}"]
+      :"360p" -> video_codec_option.("360")
+      :"480p" -> video_codec_option.("480")
+      :"720p" -> video_codec_option.("720")
+      :"1080p" -> video_codec_option.("1080")
+      :"2160p" -> video_codec_option.("2160")
     end
   end
 
@@ -184,7 +187,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
   # It's dependent on the output_path_template being a string ending `.{{ ext }}`
   # (or equivalent), but that's validated by the MediaProfile schema.
   defp determine_thumbnail_location(media_item_with_preloads) do
-    output_path_template = media_item_with_preloads.source.media_profile.output_path_template
+    output_path_template = Sources.output_path_template(media_item_with_preloads.source)
 
     output_path_template
     |> String.split(~r{\.}, include_captures: true)

lib/pinchflat/notifications/command_runner.ex

@@ -45,7 +45,7 @@ defmodule Pinchflat.Notifications.CommandRunner do
   """
   @impl AppriseCommandRunner
   def version do
-    case System.cmd(backend_executable(), ["--version"]) do
+    case CliUtils.wrap_cmd(backend_executable(), ["--version"]) do
       {output, 0} ->
         output
         |> String.split(~r{\r?\n})

lib/pinchflat/profiles/media_profile.ex

@@ -80,7 +80,8 @@ defmodule Pinchflat.Profiles.MediaProfile do
     |> unique_constraint(:name)
   end
 
-  defp ext_regex do
+  @doc false
+  def ext_regex do
     ~r/\.({{ ?ext ?}}|%\( ?ext ?\)[sS])$/
   end
 end

lib/pinchflat/sources/source.ex

@@ -32,6 +32,7 @@ defmodule Pinchflat.Sources.Source do
     retention_period_days
     title_filter_regex
     media_profile_id
+    output_path_template_override
   )a
 
   # Expensive API calls are made when a source is inserted/updated so
@@ -76,6 +77,7 @@ defmodule Pinchflat.Sources.Source do
     field :retention_period_days, :integer
     field :original_url, :string
     field :title_filter_regex, :string
+    field :output_path_template_override, :string
 
     field :series_directory, :string
     field :nfo_filepath, :string
@@ -109,6 +111,8 @@ defmodule Pinchflat.Sources.Source do
     |> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
     |> validate_required(required_fields)
     |> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
+    # Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
+    |> validate_format(:output_path_template_override, MediaProfile.ext_regex(), message: "must end with .{{ ext }}")
     |> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
     |> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
   end

lib/pinchflat/sources/sources.ex

@@ -19,6 +19,19 @@ defmodule Pinchflat.Sources do
   alias Pinchflat.SlowIndexing.SlowIndexingHelpers
   alias Pinchflat.Metadata.SourceMetadataStorageWorker
 
+  @doc """
+  Returns the relevant output path template for a source.
+  Pulls from the source's override if present, otherwise uses the media profile's.
+
+  Returns binary()
+  """
+  def output_path_template(source) do
+    source = Repo.preload(source, :media_profile)
+    media_profile = source.media_profile
+
+    source.output_path_template_override || media_profile.output_path_template
+  end
+
   @doc """
   Returns the list of sources. Returns [%Source{}, ...]
   """

lib/pinchflat/utils/cli_utils.ex

@@ -5,6 +5,23 @@ defmodule Pinchflat.Utils.CliUtils do
 
   alias Pinchflat.Utils.StringUtils
 
+  @doc """
+  Wraps a command in a shell script that will terminate
+  the command if stdin is closed. Useful for stopping
+  commands if the job runner is cancelled.
+
+  Delegates to `System.cmd/3` and any options/output
+  are passed through.
+
+  Returns {binary(), integer()}
+  """
+  def wrap_cmd(command, args, opts \\ []) do
+    wrapper_command = Path.join(:code.priv_dir(:pinchflat), "cmd_wrapper.sh")
+    actual_command = [command] ++ args
+
+    System.cmd(wrapper_command, actual_command, opts)
+  end
+
   @doc """
   Parses a list of command options into a list of strings suitable for passing to
   `System.cmd/3`.

lib/pinchflat/yt_dlp/command_runner.ex

@@ -37,7 +37,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
     formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts)
     Logger.info("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}")
 
-    case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do
+    case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do
       {_, 0} ->
         # IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
         # a huge deal, but it's still a good idea to clean up after ourselves.

lib/pinchflat_web/controllers/sources/source_html.ex

@@ -28,4 +28,20 @@ defmodule PinchflatWeb.Sources.SourceHTML do
   def rss_feed_url(conn, source) do
     url(conn, ~p"/sources/#{source.uuid}/feed") <> ".xml"
   end
+
+  def output_path_template_override_placeholders(media_profiles) do
+    media_profiles
+    |> Enum.map(&{&1.id, &1.output_path_template})
+    |> Map.new()
+    |> Phoenix.json_library().encode!()
+  end
+
+  def output_path_template_override_help do
+    help_button_classes = "underline decoration-bodydark decoration-1 hover:decoration-white cursor-pointer"
+    help_button = ~s{<span class="#{help_button_classes}" x-on:click="$dispatch('load-template')">Click here</span>}
+
+    """
+    Must end with .{{ ext }}. Same rules as Media Profile output path templates. #{help_button} to load your media profile's output template
+    """
+  end
 end

lib/pinchflat_web/controllers/sources/source_html/source_form.html.heex

@@ -9,113 +9,139 @@
     Oops, something went wrong! Please check the errors below.
   </.error>
 
-  <section class="flex justify-between items-center mt-8">
-    <h3 class=" text-2xl text-black dark:text-white">
-      General Options
-    </h3>
-    <span class="cursor-pointer hover:underline" x-on:click="advancedMode = !advancedMode">
-      Editing Mode: <span x-text="advancedMode ? 'Advanced' : 'Basic'"></span>
-    </span>
-  </section>
+  <section x-data="{ mediaProfileId: null }">
+    <section class="flex justify-between items-center mt-8">
+      <h3 class=" text-2xl text-black dark:text-white">
+        General Options
+      </h3>
+      <span class="cursor-pointer hover:underline" x-on:click="advancedMode = !advancedMode">
+        Editing Mode: <span x-text="advancedMode ? 'Advanced' : 'Basic'"></span>
+      </span>
+    </section>
 
-  <.input
-    field={f[:custom_name]}
-    type="text"
-    label="Custom Name"
-    help="Something descriptive. Does not impact indexing or downloading"
-  />
-
-  <.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
-
-  <.input
-    field={f[:media_profile_id]}
-    options={Enum.map(@media_profiles, &{&1.name, &1.id})}
-    type="select"
-    label="Media Profile"
-    help="Sets your preferences for what media to look for and how to store it"
-  />
-
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
-    Indexing Options
-  </h3>
-
-  <section x-data="{ fastIndexingEnabled: null }">
     <.input
-      field={f[:index_frequency_minutes]}
-      options={friendly_index_frequencies()}
-      type="select"
-      label="Index Frequency"
-      x-bind:disabled="fastIndexingEnabled == true"
-      x-init="$watch('fastIndexingEnabled', v => v && ($el.value = 30 * 24 * 60))"
-      help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
+      field={f[:custom_name]}
+      type="text"
+      label="Custom Name"
+      help="Something descriptive. Does not impact indexing or downloading"
     />
 
-    <div phx-click={show_modal("upgrade-modal")}>
+    <.input field={f[:original_url]} type="text" label="Source URL" help="URL of a channel or playlist (required)" />
+
+    <.input
+      field={f[:media_profile_id]}
+      options={Enum.map(@media_profiles, &{&1.name, &1.id})}
+      type="select"
+      label="Media Profile"
+      help="Sets your preferences for what media to look for and how to store it"
+      x-model.fill="mediaProfileId"
+    />
+
+    <h3 class="mt-8 text-2xl text-black dark:text-white">
+      Indexing Options
+    </h3>
+
+    <section x-data="{ fastIndexingEnabled: null }">
       <.input
-        field={f[:fast_index]}
-        type="toggle"
-        label="Use Fast Indexing"
-        label_suffix="(pro)"
-        help="Experimental. Overrides 'Index Frequency'. Recommended for large channels that upload frequently. Does not work with private playlists. See below for more info"
-        x-init="
+        field={f[:index_frequency_minutes]}
+        options={friendly_index_frequencies()}
+        type="select"
+        label="Index Frequency"
+        x-bind:disabled="fastIndexingEnabled == true"
+        x-init="$watch('fastIndexingEnabled', v => v && ($el.value = 30 * 24 * 60))"
+        help="Indexing is the process of checking for media to download. Sets the time between one index of this source finishing and the next one starting"
+      />
+
+      <div phx-click={show_modal("upgrade-modal")}>
+        <.input
+          field={f[:fast_index]}
+          type="toggle"
+          label="Use Fast Indexing"
+          label_suffix="(pro)"
+          help="Experimental. Overrides 'Index Frequency'. Recommended for large channels that upload frequently. Does not work with private playlists. See below for more info"
+          x-init="
           // `enabled` is the data attribute that the toggle uses internally
           fastIndexingEnabled = enabled
           $watch('enabled', value => fastIndexingEnabled = !!value)
         "
-      />
-    </div>
-  </section>
+        />
+      </div>
+    </section>
 
-  <h3 class="mt-8 text-2xl text-black dark:text-white">
-    Downloading Options
-  </h3>
-
-  <.input
-    field={f[:download_media]}
-    type="toggle"
-    label="Download Media"
-    help="Unchecking still indexes media but it won't be downloaded until you enable this option"
-  />
-
-  <.input
-    field={f[:download_cutoff_date]}
-    type="text"
-    label="Download Cutoff Date"
-    placeholder="YYYY-MM-DD"
-    maxlength="10"
-    pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
-    title="YYYY-MM-DD"
-    help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
-  />
-
-  <.input
-    field={f[:retention_period_days]}
-    type="number"
-    label="Retention Period (days)"
-    min="0"
-    help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
-  />
-
-  <section x-show="advancedMode">
     <h3 class="mt-8 text-2xl text-black dark:text-white">
-      Advanced Options
+      Downloading Options
     </h3>
-    <p class="text-sm mt-2">
-      Tread carefully
-    </p>
 
     <.input
-      field={f[:title_filter_regex]}
-      type="text"
-      label="Title Filter Regex"
-      placeholder="(?i)^How to Bike$"
-      help="A PCRE-compatible regex. Only media with titles that match this regex will be downloaded. Look up 'SQLean Regex docs' for more"
+      field={f[:download_media]}
+      type="toggle"
+      label="Download Media"
+      help="Unchecking still indexes media but it won't be downloaded until you enable this option"
     />
+
+    <.input
+      field={f[:download_cutoff_date]}
+      type="text"
+      label="Download Cutoff Date"
+      placeholder="YYYY-MM-DD"
+      maxlength="10"
+      pattern="((?:19|20)[0-9][0-9])-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])"
+      title="YYYY-MM-DD"
+      help="Only download media uploaded after this date. Leave blank to download all media. Must be in YYYY-MM-DD format"
+    />
+
+    <.input
+      field={f[:retention_period_days]}
+      type="number"
+      label="Retention Period (days)"
+      min="0"
+      help="Days between when media is *downloaded* and when it's deleted. Leave blank to keep media indefinitely"
+    />
+
+    <section x-show="advancedMode">
+      <h3 class="mt-8 text-2xl text-black dark:text-white">
+        Advanced Options
+      </h3>
+      <p class="text-sm mt-2">
+        Tread carefully
+      </p>
+
+      <.input
+        field={f[:title_filter_regex]}
+        type="text"
+        label="Title Filter Regex"
+        placeholder="(?i)^How to Bike$"
+        help="A PCRE-compatible regex. Only media with titles that match this regex will be downloaded. Look up 'SQLean Regex docs' for more"
+      />
+
+      <section
+        x-data={
+          """
+          { 
+            placeholders: JSON.parse('#{output_path_template_override_placeholders(@media_profiles)}'),
+            inputValue: null
+          }
+          """
+        }
+        x-on:load-template="inputValue = placeholders[mediaProfileId]"
+      >
+        <.input
+          field={f[:output_path_template_override]}
+          type="text"
+          inputclass="font-mono"
+          label="Output path template override"
+          help={output_path_template_override_help()}
+          html_help={true}
+          x-bind:placeholder="placeholders[mediaProfileId]"
+          x-model.fill="inputValue"
+        />
+      </section>
+    </section>
+
+    <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Source</.button>
+
+    <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
+      <.fast_indexing_help />
+    </div>
   </section>
-
-  <.button class="my-10 sm:mb-7.5 w-full sm:w-auto" rounding="rounded-lg">Save Source</.button>
-
-  <div class="rounded-sm dark:bg-meta-4 p-4 md:p-6 mb-5">
-    <.fast_indexing_help />
-  </div>
 </.simple_form>

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "0.1.12",
+      version: "0.1.13",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

priv/cmd_wrapper.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# This script is a wrapper for other programs
+# that ensures they are killed when stdin closes
+# (eg: a job terminates)
+
+# Start the program in the background
+exec "$@" &
+pid1=$!
+
+# Silence warnings from here on
+exec >/dev/null 2>&1
+
+# Read from stdin in the background and
+# kill running program when stdin closes
+exec 0<&0 $(
+  while read; do :; done
+  kill -KILL $pid1
+) &
+pid2=$!
+
+# Clean up
+wait $pid1
+ret=$?
+kill -KILL $pid2
+exit $ret

priv/repo/migrations/20240411154245_add_output_template_to_sources.exs

@@ -0,0 +1,9 @@
+defmodule Pinchflat.Repo.Migrations.AddOutputTemplateToSources do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add :output_path_template_override, :string
+    end
+  end
+end

test/pinchflat/downloading/download_option_builder_test.exs

@@ -4,6 +4,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
   import Pinchflat.SourcesFixtures
   import Pinchflat.ProfilesFixtures
 
+  alias Pinchflat.Sources
   alias Pinchflat.Profiles
   alias Pinchflat.Utils.FilesystemUtils
   alias Pinchflat.Downloading.DownloadOptionBuilder
@@ -31,6 +32,20 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert {:output, "/tmp/test/media/#{media_item.source.custom_name}.%(ext)s"} in res
     end
+
+    test "uses source's output override if present", %{media_item: media_item} do
+      source = media_item.source
+      {:ok, _} = Sources.update_source(source, %{output_path_template_override: "override.%(ext)s"})
+
+      media_item =
+        media_item
+        |> Repo.reload()
+        |> Repo.preload(source: :media_profile)
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:output, "/tmp/test/media/override.%(ext)s"} in res
+    end
   end
 
   describe "build/1 when testing default options" do
@@ -135,6 +150,20 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
       assert {:output, "thumbnail:/tmp/test/media/%(title)S-thumb.%(ext)s"} in res
     end
 
+    test "appends -thumb to source's output path override, if present", %{media_item: media_item} do
+      media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
+      {:ok, _} = Sources.update_source(media_item.source, %{output_path_template_override: "override.%(ext)s"})
+
+      media_item =
+        media_item
+        |> Repo.reload()
+        |> Repo.preload(source: :media_profile)
+
+      assert {:ok, res} = DownloadOptionBuilder.build(media_item)
+
+      assert {:output, "thumbnail:/tmp/test/media/override-thumb.%(ext)s"} in res
+    end
+
     test "converts thumbnail to jpg when download_thumbnail is true", %{media_item: media_item} do
       media_item = update_media_profile_attribute(media_item, %{download_thumbnail: true})
 
@@ -211,6 +240,7 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
         assert {:ok, res} = DownloadOptionBuilder.build(media_item)
         assert {:format_sort, "res:#{resolution},+codec:avc:m4a"} in res
+        assert {:remux_video, "mp4"} in res
       end)
     end
 
@@ -221,6 +251,8 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert :extract_audio in res
       assert {:format, "bestaudio[ext=m4a]"} in res
+
+      refute {:remux_video, "mp4"} in res
     end
   end
 
@@ -359,6 +391,15 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do
 
       assert path == "/tmp/test/media/%(title)S.%(ext)s"
     end
+
+    test "uses source's output override if present", %{media_item: media_item} do
+      source = media_item.source
+      {:ok, source} = Sources.update_source(source, %{output_path_template_override: "override.%(ext)s"})
+
+      path = DownloadOptionBuilder.build_output_path_for(source)
+
+      assert path == "/tmp/test/media/override.%(ext)s"
+    end
   end
 
   defp update_media_profile_attribute(media_item_with_preloads, attrs) do

test/pinchflat/sources_test.exs

@@ -35,6 +35,28 @@ defmodule Pinchflat.SourcesTest do
     end
   end
 
+  describe "output_path_template/1" do
+    test "returns the source's override if present" do
+      source = source_fixture(%{output_path_template_override: "/override/{{ title }}.{{ ext }}"})
+
+      assert Sources.output_path_template(source) == "/override/{{ title }}.{{ ext }}"
+    end
+
+    test "returns the media profile's template if no override is present" do
+      media_profile = media_profile_fixture(%{output_path_template: "/profile/{{ title }}.{{ ext }}"})
+      source = source_fixture(%{media_profile_id: media_profile.id})
+
+      assert Sources.output_path_template(source) == "/profile/{{ title }}.{{ ext }}"
+    end
+
+    test "Treats empty strings as being blank" do
+      media_profile = media_profile_fixture(%{output_path_template: "/profile/{{ title }}.{{ ext }}"})
+      source = source_fixture(%{media_profile_id: media_profile.id, output_path_template_override: "  "})
+
+      assert Sources.output_path_template(source) == "/profile/{{ title }}.{{ ext }}"
+    end
+  end
+
   describe "list_sources/0" do
     test "it returns all sources" do
       source = source_fixture()

test/pinchflat/utils/cli_utils_test.exs

@@ -3,6 +3,12 @@ defmodule Pinchflat.Utils.CliUtilsTest do
 
   alias Pinchflat.Utils.CliUtils
 
+  describe "wrap_cmd/3" do
+    test "delegates to System.cmd/3" do
+      assert {"output\n", 0} = CliUtils.wrap_cmd("echo", ["output"])
+    end
+  end
+
   describe "parse_options/1" do
     test "it converts symbol k-v arg keys to kebab case" do
       assert ["--buffer-size", "1024"] = CliUtils.parse_options(buffer_size: 1024)