n8n-openai-adapter/lib/elixir/unicode/unicode.ex

defmodule String.Unicode do
  @moduledoc false
  def version, do: {6,3,0}

  to_binary = fn
    "" ->
      nil
    codepoints ->
      codepoints = :binary.split(codepoints, " ", [:global])
      Enum.reduce codepoints, "", fn(codepoint, acc) ->
        acc <> << binary_to_integer(codepoint, 16) :: utf8 >>
      end
  end

  data_path = Path.join(__DIR__, "UnicodeData.txt")

  {codes, whitespace} = Enum.reduce File.stream!(data_path), {[], []}, fn(line, {cacc, wacc}) ->
    [ codepoint, _name, _category,
      _class, bidi, _decomposition,
      _numeric_1, _numeric_2, _numeric_3,
      _bidi_mirror, _unicode_1, _iso,
      upper, lower, title ] = :binary.split(line, ";", [:global])

    title = :binary.part(title, 0, size(title) - 1)

    cond do
      upper != "" or lower != "" or title != "" ->
        {[{to_binary.(codepoint), to_binary.(upper), to_binary.(lower), to_binary.(title)} | cacc], wacc}
      bidi in ["B", "S", "WS"] ->
        {cacc, [to_binary.(codepoint) | wacc]}
      true ->
        {cacc, wacc}
    end
  end

  special_path = Path.join(__DIR__, "SpecialCasing.txt")

  codes = Enum.reduce File.stream!(special_path), codes, fn(line, acc) ->
    [ codepoint, lower, title, upper, _comment ] = :binary.split(line, "; ", [:global])
    key = to_binary.(codepoint)
    :lists.keystore(key, 1, acc, {key, to_binary.(upper), to_binary.(lower), to_binary.(title)})
  end

  # Downcase

  def downcase(string), do: do_downcase(string) |> iodata_to_binary

  for {codepoint, _upper, lower, _title} <- codes, lower && lower != codepoint do
    defp do_downcase(unquote(codepoint) <> rest) do
      unquote(:binary.bin_to_list(lower)) ++ downcase(rest)
    end
  end

  defp do_downcase(<< char, rest :: binary >>) do
    [char|do_downcase(rest)]
  end

  defp do_downcase(""), do: []

  # Upcase

  def upcase(string), do: do_upcase(string) |> iodata_to_binary

  for {codepoint, upper, _lower, _title} <- codes, upper && upper != codepoint do
    defp do_upcase(unquote(codepoint) <> rest) do
      unquote(:binary.bin_to_list(upper)) ++ do_upcase(rest)
    end
  end

  defp do_upcase(<< char, rest :: binary >>) do
    [char|do_upcase(rest)]
  end

  defp do_upcase(""), do: []

  # Titlecase once

  def titlecase_once(""), do: {"", ""}

  for {codepoint, _upper, _lower, title} <- codes, title && title != codepoint do
    def titlecase_once(unquote(codepoint) <> rest) do
      {unquote(title), rest}
    end
  end

  def titlecase_once(<< char, rest :: binary >>) do
    {<< char >>, rest}
  end

  # Strip

  def lstrip(""), do: ""

  for codepoint <- whitespace do
    def lstrip(unquote(codepoint) <> rest) do
      lstrip(rest)
    end
  end

  def lstrip(other) when is_binary(other), do: other

  def rstrip(string) when is_binary(string) do
    do_rstrip(string, [], [])
  end

  for codepoint <- whitespace do
    c = :binary.bin_to_list(codepoint) |> :lists.reverse

    defp do_rstrip(unquote(codepoint) <> rest, acc1, acc2) do
      do_rstrip(rest, unquote(c) ++ (acc1 || acc2), acc2)
    end
  end

  defp do_rstrip(<< char, rest :: binary >>, nil, acc2) do
    do_rstrip(rest, nil, [char|acc2])
  end

  defp do_rstrip(<< char, rest :: binary >>, acc1, _acc2) do
    do_rstrip(rest, nil, [char|acc1])
  end

  defp do_rstrip(<<>>, _acc1, acc2), do: acc2 |> :lists.reverse |> iodata_to_binary

  # Split

  def split(""), do: [""]

  def split(string) when is_binary(string) do
    :lists.reverse do_split(string, "", [])
  end

  for codepoint <- whitespace do
    defp do_split(unquote(codepoint) <> rest, buffer, acc) do
      do_split(rest, "", add_buffer_to_acc(buffer, acc))
    end
  end

  defp do_split(<< char, rest :: binary >>, buffer, acc) do
    do_split(rest, << buffer :: binary, char >>, acc)
  end

  defp do_split(<<>>, buffer, acc) do
    add_buffer_to_acc(buffer, acc)
  end

  @compile {:inline, add_buffer_to_acc: 2}

  defp add_buffer_to_acc("", acc),     do: acc
  defp add_buffer_to_acc(buffer, acc), do: [buffer|acc]

  # Codepoints

  def next_codepoint(<< cp :: utf8, rest :: binary >>) do
    {<<cp :: utf8>>, rest}
  end

  def next_codepoint(<< cp, rest :: binary >>) do
    {<<cp>>, rest}
  end

  def next_codepoint(<<>>) do
    nil
  end

  def codepoints(binary) when is_binary(binary) do
    do_codepoints(next_codepoint(binary))
  end

  defp do_codepoints({c, rest}) do
    [c|do_codepoints(next_codepoint(rest))]
  end

  defp do_codepoints(nil) do
    []
  end
end

defmodule String.Graphemes do
  @moduledoc false

  cluster_path = Path.join(__DIR__, "GraphemeBreakProperty.txt")
  regex = ~r/(?:^([0-9A-F]+)(?:\.\.([0-9A-F]+))?)\s+;\s(\w+)/m

  to_range = fn
    first, ""   ->
      [<< binary_to_integer(first, 16) :: utf8 >>]
    first, last ->
      range = binary_to_integer(first, 16)..binary_to_integer(last, 16)
      Enum.map(range, fn(int) -> << int :: utf8 >> end)
  end

  cluster = Enum.reduce File.stream!(cluster_path), HashDict.new, fn(line, dict) ->
    [ _full, first, last, class ] = Regex.run(regex, line)

    # Skip surrogates
    if first == "D800" and last == "DFFF" do
      dict
    else
      list = to_range.(first, last)
      Dict.update(dict, class, list, &(&1 ++ list))
    end
  end

  # There is no codepoint marked as Prepend by Unicode 6.3.0
  if cluster["Prepend"] do
    raise "It seems this new unicode version has added Prepend items. " <>
          "Please remove this error and uncomment the code below."
  end

  # Don't break CRLF
  def next_grapheme(<< ?\n, ?\r, rest :: binary >>) do
    {"\n\r", rest}
  end

  # Break on control
  for codepoint <- cluster["CR"] ++ cluster["LF"] ++ cluster["Control"] do
    def next_grapheme(<< unquote(codepoint), rest :: binary >> = string) do
      {:binary.part(string, 0, unquote(size(codepoint))), rest}
    end
  end

  # Break on Prepend*
  # for codepoint <- cluster["Prepend"] do
  #   def next_grapheme(<< unquote(codepoint), rest :: binary >> = string) do
  #     next_prepend(rest, string, unquote(size(codepoint)))
  #   end
  # end

  # Handle Hangul L
  for codepoint <- cluster["L"] do
    def next_grapheme(<< unquote(codepoint), rest :: binary >> = string) do
      next_hangul_l(rest, string, unquote(size(codepoint)))
    end
  end

  # Handle Hangul T
  for codepoint <- cluster["T"] do
    def next_grapheme(<< unquote(codepoint), rest :: binary >> = string) do
      next_hangul_t(rest, string, unquote(size(codepoint)))
    end
  end

  # Handle Regional
  for codepoint <- cluster["Regional_Indicator"] do
    def next_grapheme(<< unquote(codepoint), rest :: binary >> = string) do
      next_regional(rest, string, unquote(size(codepoint)))
    end
  end

  # Handle extended entries
  def next_grapheme(<< cp :: utf8, rest :: binary >> = string) do
    next_extend(rest, string, byte_size(<< cp :: utf8 >>))
  end

  def next_grapheme(<< cp, rest :: binary >>) do
    {<<cp>>, rest}
  end

  def next_grapheme(<<>>) do
    nil
  end

  # Handle Hangul L
  for codepoint <- cluster["L"] do
    defp next_hangul_l(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_hangul_l(rest, string, size + unquote(size(codepoint)))
    end
  end

  for codepoint <- cluster["LV"] do
    defp next_hangul_l(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_hangul_v(rest, string, size + unquote(size(codepoint)))
    end
  end

  for codepoint <- cluster["LVT"] do
    defp next_hangul_l(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_hangul_t(rest, string, size + unquote(size(codepoint)))
    end
  end

  defp next_hangul_l(rest, string, size) do
    next_hangul_v(rest, string, size)
  end

  # Handle Hangul V
  for codepoint <- cluster["V"] do
    defp next_hangul_v(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_hangul_v(rest, string, size + unquote(size(codepoint)))
    end
  end

  defp next_hangul_v(rest, string, size) do
    next_hangul_t(rest, string, size)
  end

  # Handle Hangul T
  for codepoint <- cluster["T"] do
    defp next_hangul_t(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_hangul_t(rest, string, size + unquote(size(codepoint)))
    end
  end

  defp next_hangul_t(rest, string, size) do
    next_extend(rest, string, size)
  end

  # Handle regional
  for codepoint <- cluster["Regional_Indicator"] do
    defp next_regional(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_regional(rest, string, size + unquote(size(codepoint)))
    end
  end

  defp next_regional(rest, string, size) do
    next_extend(rest, string, size)
  end

  # Handle Extend+SpacingMark
  for codepoint <- cluster["Extend"] ++ cluster["SpacingMark"]  do
    defp next_extend(<< unquote(codepoint), rest :: binary >>, string, size) do
      next_extend(rest, string, size + unquote(size(codepoint)))
    end
  end

  defp next_extend(rest, string, size) do
    {:binary.part(string, 0, size), rest}
  end

  # Handle Prepend
  # for codepoint <- cluster["Prepend"] do
  #   defp next_prepend(<< unquote(codepoint), rest :: binary >>, string, size) do
  #     next_prepend(rest, string, size + unquote(size(codepoint)))
  #   end
  # end
  #
  # defp next_prepend(rest, string, size) do
  #   {:binary.part(string, 0, size), rest}
  # end

  def graphemes(binary) when is_binary(binary) do
    do_graphemes(next_grapheme(binary))
  end

  defp do_graphemes({c, rest}) do
    [c|do_graphemes(next_grapheme(rest))]
  end

  defp do_graphemes(nil) do
    []
  end
end