[Housekeeping] Store the upload event as a datetime rather than a date (#269)

* [WIP] renamed and migrated upload_date column

* Refactored yt-dlp media module

* Refactored parse_upload_date

* Refactored media item upload_date_index

* Got media tests running

* Refactored media item table live

* Cleaned up the stragglers

* Fixed old oversight re: original_url

lib/pinchflat/media/media.ex

@@ -41,9 +41,10 @@ defmodule Pinchflat.Media do
   Returns a list of media_items that are redownloadable based on the redownload delay
   of the media_profile their source belongs to.
 
-  The logic is that a media_item is past_redownload_delay if the media_item's
-  upload_date is at least redownload_delay_days ago AND
-  `media_downloaded_at` - `redownload_delay_days` is before the media_item's `upload_date`.
+  The logic is that a media_item is past_redownload_delay if the media_item's uploaded_at is
+  at least redownload_delay_days ago AND `media_downloaded_at` - `redownload_delay_days`
+  is before the media_item's `uploaded_at`.
+
   This logic grabs media that we've recently downloaded AND is recently uploaded, but
   doesn't grab media that we've recently downloaded and was uploaded a long time ago.
   This also makes things work as expected when downloading media from a source for the
@@ -135,7 +136,7 @@ defmodule Pinchflat.Media do
 
   Unlike `create_media_item`, this will attempt an update if the media_item
   already exists. This is so that future indexing can pick up attributes that
-  we may not have asked for in the past (eg: upload_date)
+  we may not have asked for in the past (eg: uploaded_at)
 
   Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
   """

lib/pinchflat/media/media_item.ex

@@ -26,7 +26,7 @@ defmodule Pinchflat.Media.MediaItem do
     :livestream,
     :source_id,
     :short_form_content,
-    :upload_date,
+    :uploaded_at,
     :upload_date_index,
     :duration_seconds,
     # these fields are captured only on download
@@ -51,7 +51,7 @@ defmodule Pinchflat.Media.MediaItem do
     livestream
     media_id
     source_id
-    upload_date
+    uploaded_at
     short_form_content
   )a
 
@@ -69,7 +69,7 @@ defmodule Pinchflat.Media.MediaItem do
     field :short_form_content, :boolean, default: false
     field :media_downloaded_at, :utc_datetime
     field :media_redownloaded_at, :utc_datetime
-    field :upload_date, :date
+    field :uploaded_at, :utc_datetime
     field :upload_date_index, :integer, default: 0
     field :duration_seconds, :integer
 
@@ -130,7 +130,7 @@ defmodule Pinchflat.Media.MediaItem do
     ~w(__meta__ __struct__ metadata tasks media_items_search_index)a
   end
 
-  defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :upload_date) do
+  defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :uploaded_at) do
     source_id = get_field(changeset, :source_id)
     source = Sources.get_source!(source_id)
     # Channels should count down from 99, playlists should count up from 0
@@ -142,7 +142,7 @@ defmodule Pinchflat.Media.MediaItem do
 
     current_max =
       MediaQuery.new()
-      |> where(^dynamic([mi], mi.upload_date == ^changes.upload_date and ^MediaQuery.for_source(source)))
+      |> where(^dynamic([mi], ^MediaQuery.upload_date_matches(changes.uploaded_at) and ^MediaQuery.for_source(source)))
       |> Repo.aggregate(aggregator, :upload_date_index)
 
     case current_max do

lib/pinchflat/media/media_query.ex

@@ -35,9 +35,14 @@ defmodule Pinchflat.Media.MediaQuery do
   def culling_prevented, do: dynamic([mi], mi.prevent_culling == true)
   def culled, do: dynamic([mi], not is_nil(mi.culled_at))
   def redownloaded, do: dynamic([mi], not is_nil(mi.media_redownloaded_at))
+  def upload_date_matches(other_date), do: dynamic([mi], fragment("date(?) = date(?)", mi.uploaded_at, ^other_date))
 
   def upload_date_after_source_cutoff do
-    dynamic([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date)
+    dynamic(
+      [mi, source],
+      is_nil(source.download_cutoff_date) or
+        fragment("date(?) >= ?", mi.uploaded_at, source.download_cutoff_date)
+    )
   end
 
   def format_matching_profile_preference do
@@ -84,12 +89,12 @@ defmodule Pinchflat.Media.MediaQuery do
   def past_redownload_delay do
     dynamic(
       [mi, source, media_profile],
-      # Returns media items where the upload_date is at least redownload_delay_days ago AND
+      # Returns media items where the uploaded_at is at least redownload_delay_days ago AND
       # downloaded_at minus the redownload_delay_days is before the upload date
       fragment("""
         IFNULL(redownload_delay_days, 0) > 0 AND
-        DATETIME('now', '-' || redownload_delay_days || ' day') > upload_date AND
-        DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < upload_date
+        DATETIME('now', '-' || redownload_delay_days || ' day') > uploaded_at AND
+        DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < uploaded_at
       """)
     )
   end

lib/pinchflat/metadata/metadata_file_helpers.ex

@@ -88,14 +88,17 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
 
   @doc """
   Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
-  and returns a Date struct.
+  and returns a DateTime struct, appending a time of 00:00:00Z.
 
-  Returns Date.t()
+  Returns DateTime.t()
   """
   def parse_upload_date(upload_date) do
     <<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
 
-    Date.from_iso8601!("#{year}-#{month}-#{day}")
+    case DateTime.from_iso8601("#{year}-#{month}-#{day}T00:00:00Z") do
+      {:ok, datetime, _} -> datetime
+      _ -> raise "Invalid upload date: #{upload_date}"
+    end
   end
 
   @doc """

lib/pinchflat/metadata/metadata_parser.ex

@@ -8,6 +8,8 @@ defmodule Pinchflat.Metadata.MetadataParser do
   and not have it, ya know?
   """
 
+  alias Pinchflat.YtDlp.Media, as: YtDlpMedia
+
   @doc """
   Parses the given JSON response from yt-dlp and returns a map of
   the needful media_item attributes, along with anything needed for
@@ -24,15 +26,12 @@ defmodule Pinchflat.Metadata.MetadataParser do
   end
 
   defp parse_media_metadata(metadata) do
-    %{
-      media_id: metadata["id"],
-      title: metadata["title"],
-      original_url: metadata["original_url"],
-      description: metadata["description"],
-      media_filepath: metadata["filepath"],
-      livestream: !!metadata["was_live"],
-      duration_seconds: metadata["duration"] && round(metadata["duration"])
-    }
+    Map.merge(
+      Map.from_struct(YtDlpMedia.response_to_struct(metadata)),
+      %{
+        media_filepath: metadata["filepath"]
+      }
+    )
   end
 
   defp parse_subtitle_metadata(metadata) do

lib/pinchflat/podcasts/podcast_helpers.ex

@@ -28,7 +28,7 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
 
     MediaQuery.new()
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
-    |> order_by(desc: :upload_date)
+    |> order_by(desc: :uploaded_at)
     |> Repo.maybe_limit(limit)
     |> Repo.all()
     |> Enum.filter(fn media_item -> File.exists?(media_item.media_filepath) end)

lib/pinchflat/podcasts/rss_feed_builder.ex

@@ -7,7 +7,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
 
   import Pinchflat.Utils.XmlUtils, only: [safe: 1]
 
-  alias Pinchflat.Utils.DatetimeUtils
   alias Pinchflat.Podcasts.PodcastHelpers
   alias PinchflatWeb.Router.Helpers, as: Routes
 
@@ -83,7 +82,7 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
       <title>#{safe(media_item.title)}</title>
       <link>#{safe(media_item.original_url)}</link>
       <description>#{safe(media_item.description)}</description>
-      <pubDate>#{generate_upload_date(media_item)}</pubDate>
+      <pubDate>#{Calendar.strftime(media_item.uploaded_at, @datetime_format)}</pubDate>
       <itunes:duration>#{media_item.duration_seconds}</itunes:duration>
       <enclosure
         url="#{media_stream_path(url_base, media_item)}"
@@ -133,12 +132,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
     end
   end
 
-  defp generate_upload_date(media_item) do
-    media_item.upload_date
-    |> DatetimeUtils.date_to_datetime()
-    |> Calendar.strftime(@datetime_format)
-  end
-
   defp podcast_route(action, params) do
     Routes.podcast_path(PinchflatWeb.Endpoint, action, params)
   end

lib/pinchflat/utils/datetime_utils.ex

@@ -1,17 +0,0 @@
-defmodule Pinchflat.Utils.DatetimeUtils do
-  @moduledoc """
-  Utility methods for working with dates and datetimes
-  """
-
-  @doc """
-  Converts a Date to a DateTime
-
-  Returns %DateTime{}
-  """
-  def date_to_datetime(date) do
-    date
-    |> Date.to_gregorian_days()
-    |> Kernel.*(86_400)
-    |> DateTime.from_gregorian_seconds()
-  end
-end

lib/pinchflat/yt_dlp/media.ex

@@ -10,7 +10,7 @@ defmodule Pinchflat.YtDlp.Media do
     :original_url,
     :livestream,
     :short_form_content,
-    :upload_date,
+    :uploaded_at,
     :duration_seconds
   ]
 
@@ -21,7 +21,7 @@ defmodule Pinchflat.YtDlp.Media do
     :original_url,
     :livestream,
     :short_form_content,
-    :upload_date,
+    :uploaded_at,
     :duration_seconds
   ]
 
@@ -72,7 +72,7 @@ defmodule Pinchflat.YtDlp.Media do
   Returns the output template for yt-dlp's indexing command.
   """
   def indexing_output_template do
-    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
+    "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j"
   end
 
   @doc """
@@ -90,7 +90,7 @@ defmodule Pinchflat.YtDlp.Media do
       livestream: !!response["was_live"],
       duration_seconds: response["duration"] && round(response["duration"]),
       short_form_content: response["webpage_url"] && short_form_content?(response),
-      upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
+      uploaded_at: response["upload_date"] && parse_uploaded_at(response)
     }
   end
 
@@ -110,6 +110,18 @@ defmodule Pinchflat.YtDlp.Media do
     end
   end
 
+  defp parse_uploaded_at(%{"timestamp" => ts} = response) when is_number(ts) do
+    case DateTime.from_unix(ts) do
+      {:ok, datetime} -> datetime
+      _ -> MetadataFileHelpers.parse_upload_date(response["upload_date"])
+    end
+  end
+
+  # This field is needed before inserting into the database, but absence
+  # of this field should fail at insert-time rather than here
+  defp parse_uploaded_at(%{"upload_date" => nil}), do: nil
+  defp parse_uploaded_at(response), do: MetadataFileHelpers.parse_upload_date(response["upload_date"])
+
   defp backend_runner do
     # This approach lets us mock the command for testing
     Application.get_env(:pinchflat, :yt_dlp_runner)

lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex

@@ -31,7 +31,7 @@
                 <.media_preview media_item={@media_item} />
               </div>
               <aside class="mt-4 xl:mt-0">
-                <div>Uploaded: <%= @media_item.upload_date %></div>
+                <div>Uploaded: <%= DateTime.to_date(@media_item.uploaded_at) %></div>
                 <div>
                   <span :if={URI.parse(@media_item.original_url).scheme =~ "http"}>
                     <.subtle_link href={@media_item.original_url} target="_blank">Open Original</.subtle_link>

lib/pinchflat_web/controllers/pages/page_html/history_table_live.ex

@@ -32,7 +32,7 @@ defmodule Pinchflat.Pages.HistoryTableLive do
             </.subtle_link>
           </:col>
           <:col :let={media_item} label="Upload Date">
-            <%= media_item.upload_date %>
+            <%= DateTime.to_date(media_item.uploaded_at) %>
           </:col>
           <:col :let={media_item} label="Indexed At">
             <%= format_datetime(media_item.inserted_at) %>

lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex

@@ -50,7 +50,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
           </.subtle_link>
         </:col>
         <:col :let={media_item} label="Upload Date">
-          <%= media_item.upload_date %>
+          <%= DateTime.to_date(media_item.uploaded_at) %>
         </:col>
         <:col :let={media_item} :if={@media_state == "other"} label="Manually Ignored?">
           <.icon name={if media_item.prevent_download, do: "hero-check", else: "hero-x-mark"} />
@@ -150,14 +150,14 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
     |> MediaQuery.require_assoc(:media_profile)
     |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending()))
-    |> order_by(desc: fragment("rank"), desc: :upload_date)
+    |> order_by(desc: fragment("rank"), desc: :uploaded_at)
   end
 
   defp generate_base_query(source, "downloaded") do
     MediaQuery.new()
     |> MediaQuery.require_assoc(:media_items_search_index)
     |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
-    |> order_by(desc: fragment("rank"), desc: :upload_date)
+    |> order_by(desc: fragment("rank"), desc: :uploaded_at)
   end
 
   defp generate_base_query(source, "other") do
@@ -170,7 +170,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
           (not (^MediaQuery.downloaded()) and not (^MediaQuery.pending()))
       )
     )
-    |> order_by(desc: fragment("rank"), desc: :upload_date)
+    |> order_by(desc: fragment("rank"), desc: :uploaded_at)
   end
 
   defp filter_base_query(base_query, search_term) do