Bumped version

* Changed sqlite FTS to use a trigram tokenizer

* Added search UI to source tables

* Fix bug with special chars in search form

* improved main search results form

* Improved centering for media table header elements

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: '[Triage] <your title here> '
+labels: triage
+assignees: kieraneglin
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Diagnostic info**
+
+<!-- Go to Config > App Info > Copy Diagnostic Info and paste that here -->
+
+**Additional context**
+
+<!-- Go to Config > App Info > Download Logs and attach them, if applicable -->
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,19 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: '[FR] <your title here>'
+labels: feature request
+assignees: kieraneglin
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. It's too complicated to [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/other.md

@@ -0,0 +1,7 @@
+---
+name: Other
+about: For everything else
+title: ''
+labels: ''
+assignees: kieraneglin
+---

lib/pinchflat/media/media_query.ex

@@ -125,6 +125,18 @@ defmodule Pinchflat.Media.MediaQuery do
     )
   end
 
+  def matches_search_term(nil), do: dynamic([mi], true)
+
+  def matches_search_term(term) do
+    escaped_term = clean_search_term(term)
+
+    # Matching on `term` instead of `escaped_term` because the latter can mangle empty strings
+    case String.trim(term) do
+      "" -> dynamic([mi], true)
+      _ -> dynamic([mi], fragment("media_items_search_index MATCH ?", ^escaped_term))
+    end
+  end
+
   def require_assoc(query, identifier) do
     if has_named_binding?(query, identifier) do
       query
@@ -133,6 +145,10 @@ defmodule Pinchflat.Media.MediaQuery do
     end
   end
 
+  defp do_require_assoc(query, :media_items_search_index) do
+    from(mi in query, join: s in assoc(mi, :media_items_search_index), as: :media_items_search_index)
+  end
+
   defp do_require_assoc(query, :source) do
     from(mi in query, join: s in assoc(mi, :source), as: :source)
   end
@@ -148,9 +164,11 @@ defmodule Pinchflat.Media.MediaQuery do
   def matching_search_term(query, nil), do: query
 
   def matching_search_term(query, term) do
+    escaped_term = clean_search_term(term)
+
     from(mi in query,
       join: mi_search_index in assoc(mi, :media_items_search_index),
-      where: fragment("media_items_search_index MATCH ?", ^term),
+      where: fragment("media_items_search_index MATCH ?", ^escaped_term),
       select_merge: %{
         matching_search_term:
           fragment("""
@@ -162,4 +180,26 @@ defmodule Pinchflat.Media.MediaQuery do
       order_by: [desc: fragment("rank")]
     )
   end
+
+  # SQLite's FTS5 is very picky about what it will accept as a search term.
+  # To that end, we need to clean up the search term before passing it to the
+  # MATCH clause.
+  # This method:
+  #   - Trims leading and trailing whitespace
+  #   - Collapses multiple spaces into a single space
+  #   - Removes quote characters
+  #   - Wraps any word in quotes (must happen after the double quote replacement)
+  #
+  # This allows for works with apostrophes and quotes to be searched for correctly
+  defp clean_search_term(nil), do: ""
+  defp clean_search_term(""), do: ""
+
+  defp clean_search_term(term) do
+    term
+    |> String.trim()
+    |> String.replace(~r/\s+/, " ")
+    |> String.split(~r/\s+/)
+    |> Enum.map(fn str -> String.replace(str, ~s("), "") end)
+    |> Enum.map_join(" ", fn str -> ~s("#{str}") end)
+  end
 end

lib/pinchflat_web/components/layouts.ex

@@ -68,7 +68,7 @@ defmodule PinchflatWeb.Layouts do
         </span>
       </span>
 
-      <ul x-bind:class="selected ? 'block' :'hidden'">
+      <ul x-cloak x-show="selected">
         <li :for={menu <- @submenu} class="text-bodydark2">
           <.sidebar_link icon={menu[:icon]} text={menu[:text]} href={menu[:href]} target={menu[:target]} class="pl-10" />
         </li>

lib/pinchflat_web/components/layouts/partials/header.html.heex

@@ -1,4 +1,4 @@
-<header class="sticky top-0 z-999 flex h-20 w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
+<header class="sticky top-0 z-999 flex min-h-20 w-full bg-white drop-shadow-1 dark:bg-boxdark dark:drop-shadow-none">
   <div class="flex flex-grow items-center justify-between lg:justify-end px-4 py-4 shadow-2 md:px-6 2xl:px-11">
     <div class="flex items-center gap-2 sm:gap-4 lg:hidden w-2/6">
       <section class="pr-1">
@@ -22,7 +22,7 @@
             type="text"
             name="q"
             value={@params["q"]}
-            placeholder="Type to search..."
+            placeholder="Search all media..."
             class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none"
           />
         </form>

lib/pinchflat_web/controllers/searches/search_html/show.html.heex

@@ -10,19 +10,13 @@
       <%= if match?([_|_], @search_results) do %>
         <.table rows={@search_results} table_class="text-black dark:text-white">
           <:col :let={result} label="Title">
-            <%= result.title %>
+            <.subtle_link href={~p"/sources/#{result.source_id}/media/#{result.id}"}>
+              <%= StringUtils.truncate(result.title, 35) %>
+            </.subtle_link>
           </:col>
           <:col :let={result} label="Excerpt">
             <.highlight_search_terms text={result.matching_search_term} />
           </:col>
-          <:col :let={result} label="" class="flex place-content-evenly">
-            <.link
-              href={~p"/sources/#{result.source_id}/media/#{result.id}"}
-              class="hover:text-secondary duration-200 ease-in-out mx-0.5"
-            >
-              <.icon name="hero-eye" />
-            </.link>
-          </:col>
         </.table>
       <% else %>
         <p class="font-bold text-lg text-center text-black dark:text-white">No results found</p>

lib/pinchflat_web/controllers/settings/setting_html.ex

@@ -23,11 +23,11 @@ defmodule PinchflatWeb.Settings.SettingHTML do
 
   def diagnostic_info_string do
     """
-    App Version: #{Application.spec(:pinchflat)[:vsn]}
+    - App Version: #{Application.spec(:pinchflat)[:vsn]}
-    yt-dlp Version: #{Settings.get!(:yt_dlp_version)}
+    - yt-dlp Version: #{Settings.get!(:yt_dlp_version)}
-    Apprise Version: #{Settings.get!(:apprise_version)}
+    - Apprise Version: #{Settings.get!(:apprise_version)}
-    System Architecture: #{to_string(:erlang.system_info(:system_architecture))}
+    - System Architecture: #{to_string(:erlang.system_info(:system_architecture))}
-    Timezone: #{Application.get_env(:pinchflat, :timezone)}
+    - Timezone: #{Application.get_env(:pinchflat, :timezone)}
     """
   end
 end

lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex

@@ -8,7 +8,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
 
   @limit 10
 
-  def render(%{records: []} = assigns) do
+  def render(%{total_record_count: 0} = assigns) do
     ~H"""
     <div class="mb-4 flex items-center">
       <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" />
@@ -20,16 +20,38 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   def render(assigns) do
     ~H"""
     <div>
-      <span class="mb-4 flex items-center">
+      <header class="flex justify-between items-center mb-4">
-        <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" />
+        <span class="flex items-center">
-        <span class="ml-2">Showing <%= length(@records) %> of <%= @total_record_count %></span>
+          <.icon_button icon_name="hero-arrow-path" class="h-10 w-10" phx-click="reload_page" tooltip="Refresh" />
-      </span>
+          <span class="ml-2">Showing <%= length(@records) %> of <%= @filtered_record_count %></span>
+        </span>
+        <div class="bg-meta-4 rounded-md">
+          <div class="relative">
+            <span class="absolute left-2 top-1/2 -translate-y-1/2 flex">
+              <.icon name="hero-magnifying-glass" />
+            </span>
+            <form phx-change="search_term" phx-submit="search_term">
+              <input
+                type="text"
+                name="q"
+                value={@search_term}
+                placeholder="Search in table..."
+                class="w-full bg-transparent pl-9 pr-4 border-0 focus:ring-0 focus:outline-none"
+                phx-debounce="200"
+              />
+            </form>
+          </div>
+        </div>
+      </header>
       <.table rows={@records} table_class="text-white">
         <:col :let={media_item} label="Title">
           <.subtle_link href={~p"/sources/#{@source.id}/media/#{media_item.id}"}>
             <%= StringUtils.truncate(media_item.title, 50) %>
           </.subtle_link>
         </:col>
+        <:col :let={media_item} :if={@media_state == "other"} label="Manually Ignored?">
+          <.icon name={if media_item.prevent_download, do: "hero-check", else: "hero-x-mark"} />
+        </:col>
         <:col :let={media_item} label="" class="flex justify-end">
           <.icon_link href={~p"/sources/#{@source.id}/media/#{media_item.id}/edit"} icon="hero-pencil-square" class="mr-4" />
         </:col>
@@ -42,36 +64,73 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   end
 
   def mount(_params, session, socket) do
+    PinchflatWeb.Endpoint.subscribe("media_table")
+
     page = 1
     media_state = session["media_state"]
     source = Sources.get_source!(session["source_id"])
     base_query = generate_base_query(source, media_state)
-    pagination_attrs = fetch_pagination_attributes(base_query, page)
+    pagination_attrs = fetch_pagination_attributes(base_query, page, nil)
 
-    {:ok, assign(socket, Map.merge(pagination_attrs, %{base_query: base_query, source: source}))}
+    new_assigns =
+      Map.merge(
+        pagination_attrs,
+        %{
+          base_query: base_query,
+          source: source,
+          media_state: media_state
+        }
+      )
+
+    {:ok, assign(socket, new_assigns)}
   end
 
   def handle_event("page_change", %{"direction" => direction}, %{assigns: assigns} = socket) do
     direction = if direction == "inc", do: 1, else: -1
     new_page = assigns.page + direction
-    new_assigns = fetch_pagination_attributes(assigns.base_query, new_page)
+    new_assigns = fetch_pagination_attributes(assigns.base_query, new_page, assigns.search_term)
 
     {:noreply, assign(socket, new_assigns)}
   end
 
-  def handle_event("reload_page", _params, %{assigns: assigns} = socket) do
+  def handle_event("search_term", params, socket) do
-    new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page)
+    search_term = Map.get(params, "q", nil)
+    new_assigns = fetch_pagination_attributes(socket.assigns.base_query, 1, search_term)
 
     {:noreply, assign(socket, new_assigns)}
   end
 
-  defp fetch_pagination_attributes(base_query, page) do
+  # This, along with the handle_info below, is a pattern to reload _all_
+  # tables on page rather than just the one that triggered the reload.
+  def handle_event("reload_page", _params, socket) do
+    PinchflatWeb.Endpoint.broadcast("media_table", "reload", nil)
+
+    {:noreply, socket}
+  end
+
+  def handle_info(%{topic: "media_table", event: "reload"}, %{assigns: assigns} = socket) do
+    new_assigns = fetch_pagination_attributes(assigns.base_query, assigns.page, assigns.search_term)
+
+    {:noreply, assign(socket, new_assigns)}
+  end
+
+  defp fetch_pagination_attributes(base_query, page, search_term) do
+    filtered_base_query = filter_base_query(base_query, search_term)
+
     total_record_count = Repo.aggregate(base_query, :count, :id)
-    total_pages = max(ceil(total_record_count / @limit), 1)
+    filtered_record_count = Repo.aggregate(filtered_base_query, :count, :id)
+    total_pages = max(ceil(filtered_record_count / @limit), 1)
     page = NumberUtils.clamp(page, 1, total_pages)
-    records = fetch_records(base_query, page)
+    records = fetch_records(filtered_base_query, page)
 
-    %{page: page, total_pages: total_pages, records: records, total_record_count: total_record_count}
+    %{
+      page: page,
+      total_pages: total_pages,
+      records: records,
+      search_term: search_term,
+      total_record_count: total_record_count,
+      filtered_record_count: filtered_record_count
+    }
   end
 
   defp fetch_records(base_query, page) do
@@ -86,13 +145,33 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
   defp generate_base_query(source, "pending") do
     MediaQuery.new()
     |> MediaQuery.require_assoc(:media_profile)
+    |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending()))
-    |> order_by(desc: :id)
+    |> order_by(desc: fragment("rank"), desc: :id)
   end
 
   defp generate_base_query(source, "downloaded") do
     MediaQuery.new()
+    |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
-    |> order_by(desc: :id)
+    |> order_by(desc: fragment("rank"), desc: :id)
+  end
+
+  defp generate_base_query(source, "other") do
+    MediaQuery.new()
+    |> MediaQuery.require_assoc(:media_profile)
+    |> MediaQuery.require_assoc(:media_items_search_index)
+    |> where(
+      ^dynamic(
+        ^MediaQuery.for_source(source) and
+          (not (^MediaQuery.downloaded()) and not (^MediaQuery.pending()))
+      )
+    )
+    |> order_by(desc: fragment("rank"), desc: :id)
+  end
+
+  defp filter_base_query(base_query, search_term) do
+    base_query
+    |> where(^MediaQuery.matches_search_term(search_term))
   end
 end

lib/pinchflat_web/controllers/sources/source_html/show.html.heex

@@ -36,21 +36,28 @@
           <.list_items_from_map map={Map.from_struct(@source)} />
         </div>
       </:tab>
-      <:tab title="Pending Media" id="pending">
+      <:tab title="Pending" id="pending">
         <%= live_render(
           @conn,
           Pinchflat.Sources.MediaItemTableLive,
           session: %{"source_id" => @source.id, "media_state" => "pending"}
         ) %>
       </:tab>
-      <:tab title="Downloaded Media" id="downloaded">
+      <:tab title="Downloaded" id="downloaded">
         <%= live_render(
           @conn,
           Pinchflat.Sources.MediaItemTableLive,
           session: %{"source_id" => @source.id, "media_state" => "downloaded"}
         ) %>
       </:tab>
-      <:tab title="Pending Tasks" id="tasks">
+      <:tab title="Other" id="other">
+        <%= live_render(
+          @conn,
+          Pinchflat.Sources.MediaItemTableLive,
+          session: %{"source_id" => @source.id, "media_state" => "other"}
+        ) %>
+      </:tab>
+      <:tab title="Tasks" id="tasks">
         <%= if match?([_|_], @pending_tasks) do %>
           <.table rows={@pending_tasks} table_class="text-black dark:text-white">
             <:col :let={task} label="Worker">

mix.exs

@@ -4,7 +4,7 @@ defmodule Pinchflat.MixProject do
   def project do
     [
       app: :pinchflat,
-      version: "2024.5.22",
+      version: "2024.5.24",
       elixir: "~> 1.16",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

package.json

@@ -2,7 +2,7 @@
   "description": "Prettier is used for linting of all files so this package has to live in the root of the project. Use the other package.json files for dependencies. Also, look into making this global or something to remove the need for this file.",
   "devDependencies": {
     "prettier": "3.2.4",
-    "sqleton": "https://github.com/kieraneglin/sqleton#ke/add-index-support"
+    "sqleton": "^2.2.0"
   },
   "scripts": {
     "create-erd": "sqleton -o priv/repo/erd.png priv/repo/pinchflat_dev.db"

priv/repo/migrations/20240523183003_change_media_items_search_index_tokenizer.exs

@@ -0,0 +1,60 @@
+defmodule Pinchflat.Repo.Migrations.ChangeMediaItemsSearchIndexTokenizer do
+  use Ecto.Migration
+
+  def up do
+    # These all need to run as part of separate `execute` blocks. Do NOT ask me why.
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_insert;"
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_update;"
+    execute "DROP TRIGGER IF EXISTS media_items_search_index_delete;"
+    execute "DROP TABLE IF EXISTS media_items_search_index;"
+
+    execute """
+      CREATE VIRTUAL TABLE media_items_search_index USING fts5(
+        title,
+        description,
+        tokenize=trigram
+      );
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_insert AFTER INSERT ON media_items BEGIN
+        INSERT INTO media_items_search_index(
+          rowid,
+          title,
+          description
+        )
+        VALUES(
+          new.id,
+          new.title,
+          new.description
+        );
+      END;
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_update AFTER UPDATE ON media_items BEGIN
+        UPDATE media_items_search_index SET
+          title = new.title,
+          description = new.description
+        WHERE
+          rowid = old.id;
+      END;
+    """
+
+    execute """
+      CREATE TRIGGER media_items_search_index_delete AFTER DELETE ON media_items BEGIN
+        DELETE FROM media_items_search_index WHERE rowid = old.id;
+      END;
+    """
+
+    # Fully re-index the media_items table
+    execute """
+      INSERT INTO media_items_search_index(rowid, title, description)
+      SELECT id, title, description FROM media_items;
+    """
+  end
+
+  def down do
+    execute "DROP TABLE media_items_search_index;"
+  end
+end

test/pinchflat/media_test.exs

@@ -727,6 +727,17 @@ defmodule Pinchflat.MediaTest do
     test "returns an empty list when the search term is nil" do
       assert [] = Media.search(nil)
     end
+
+    test "doesn't blow up if there's an apostrophe or quotes in the search term" do
+      assert [] = Media.search("don't expl'ode")
+      assert [] = Media.search(~s(dont expl"o"de))
+      assert [] = Media.search(~s(dont explo"de))
+    end
+
+    test "doesn't blow up if there is a trailing operand" do
+      assert [] = Media.search("foo OR")
+      assert [] = Media.search("foo AND")
+    end
   end
 
   describe "get_media_item!/1" do

test/pinchflat_web/controllers/sources/media_item_table_live_test.exs

@@ -4,6 +4,7 @@ defmodule PinchflatWeb.Sources.MediaItemTableLiveTest do
   import Phoenix.LiveViewTest
   import Pinchflat.MediaFixtures
   import Pinchflat.SourcesFixtures
+  import Pinchflat.ProfilesFixtures
 
   alias Pinchflat.Sources.MediaItemTableLive
 
@@ -34,10 +35,8 @@ defmodule PinchflatWeb.Sources.MediaItemTableLiveTest do
 
   describe "media_state" do
     test "shows pending media when pending", %{conn: conn, source: source} do
-      downloaded_media_item = media_item_fixture(source_id: source.id, title: "DL-#{Enum.random(0..9999)}")
+      downloaded_media_item = media_item_fixture(source_id: source.id)
-
+      pending_media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
-      pending_media_item =
-        media_item_fixture(source_id: source.id, media_filepath: nil, title: "P-#{Enum.random(0..9999)}")
 
       {:ok, _view, html} = live_isolated(conn, MediaItemTableLive, session: create_session(source, "pending"))
 
@@ -54,6 +53,29 @@ defmodule PinchflatWeb.Sources.MediaItemTableLiveTest do
       assert html =~ downloaded_media_item.title
       refute html =~ pending_media_item.title
     end
+
+    test "shows records that aren't pending or downloaded when other", %{conn: conn} do
+      media_profile = media_profile_fixture(shorts_behaviour: :exclude)
+      source = source_fixture(media_profile_id: media_profile.id)
+
+      downloaded_media_item = media_item_fixture(source_id: source.id)
+      pending_media_item = media_item_fixture(source_id: source.id, media_filepath: nil)
+      other_media_item = media_item_fixture(source_id: source.id, media_filepath: nil, short_form_content: true)
+
+      {:ok, _view, html} = live_isolated(conn, MediaItemTableLive, session: create_session(source, "other"))
+
+      assert html =~ other_media_item.title
+      refute html =~ downloaded_media_item.title
+      refute html =~ pending_media_item.title
+    end
+
+    test "shows 'Manually Ignored' column when other", %{conn: conn, source: source} do
+      _media_item = media_item_fixture(source_id: source.id, prevent_download: true, media_filepath: nil)
+
+      {:ok, _view, html} = live_isolated(conn, MediaItemTableLive, session: create_session(source, "other"))
+
+      assert html =~ "Manually Ignored?"
+    end
   end
 
   defp create_session(source, media_state \\ "pending") do

yarn.lock

@@ -780,9 +780,10 @@ sprintf-js@^1.1.3:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a"
   integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==
 
-"sqleton@https://github.com/kieraneglin/sqleton#ke/add-index-support":
+sqleton@^2.2.0:
-  version "2.1.0"
+  version "2.2.0"
-  resolved "https://github.com/kieraneglin/sqleton#b066b39fd9f7caff7cc86d1d7d37088e74ae44f2"
+  resolved "https://registry.yarnpkg.com/sqleton/-/sqleton-2.2.0.tgz#d265b625c43ec552b5d3275c25b85dfec240d910"
+  integrity sha512-pfjBQRmrRNi4DEiX5X1akyLIG9z6UiG7Hi+5vwB/dG1ksgJ8yuL5na+3HI5nZqaWBJf91L340x55ZLdake9/yg==
   dependencies:
     sqlite3 "^5.1.4"
     yargs "^17.2.1"