488 lines
13 KiB
Elixir
488 lines
13 KiB
Elixir
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: 2021 The Elixir Team
|
||
# SPDX-FileCopyrightText: 2012 Plataformatec
|
||
|
||
# How to update the Unicode files
|
||
#
|
||
# Unicode files can be found in https://www.unicode.org/Public/VERSION_NUMBER/ where
|
||
# VERSION_NUMBER is the current Unicode version.
|
||
#
|
||
# 1. Replace UnicodeData.txt by copying original
|
||
# 2. Replace PropertyValueAliases.txt by copying original
|
||
# 3. Replace PropList.txt by copying original
|
||
# 4. Replace ScriptExtensions.txt by copying original
|
||
# 5. Replace Scripts.txt by copying original
|
||
# 6. Replace SpecialCasing.txt by copying original
|
||
# 7. Replace confusables.txt by copying original
|
||
# (from https://www.unicode.org/Public/security/VERSION_NUMBER/)
|
||
# 8. Replace IdentifierType.txt by copying original
|
||
# (from https://www.unicode.org/Public/security/VERSION_NUMBER/)
|
||
# 9. Update String.Unicode.version/0 and on String module docs (version and link)
|
||
# 10. make unicode
|
||
|
||
data_path = Path.join(__DIR__, "UnicodeData.txt")
|
||
|
||
to_binary = fn
|
||
"" ->
|
||
nil
|
||
|
||
codepoints ->
|
||
codepoints
|
||
|> :binary.split(" ", [:global])
|
||
|> Enum.map(&<<String.to_integer(&1, 16)::utf8>>)
|
||
|> IO.iodata_to_binary()
|
||
end
|
||
|
||
rangify = fn [head | tail] ->
|
||
{first, last, acc} =
|
||
Enum.reduce(tail, {head, head, []}, fn
|
||
number, {first, last, acc} when number == first - 1 ->
|
||
{number, last, acc}
|
||
|
||
number, {first, last, acc} ->
|
||
{number, number, [{first, last} | acc]}
|
||
end)
|
||
|
||
[{first, last} | acc]
|
||
end
|
||
|
||
# A character is case ignorable if:
|
||
#
|
||
# Word_Break(C) = MidLetter or MidNumLet or Single_Quote, or
|
||
# General_Category(C) = Nonspacing_Mark (Mn), Enclosing_Mark (Me), Format (Cf),
|
||
# Modifier_Letter (Lm), or Modifier_Symbol (Sk).
|
||
#
|
||
# Word breaks are defined below based on TR29 (https://unicode.org/reports/tr29/).
|
||
# The categories are computed later.
|
||
case_ignorable = [
|
||
0x0027,
|
||
0x002E,
|
||
0x2018,
|
||
0x2019,
|
||
0x2024,
|
||
0xFE52,
|
||
0xFF07,
|
||
0xFF0E,
|
||
0x00B7,
|
||
0x0387,
|
||
0x05F4,
|
||
0x2027,
|
||
0x003A,
|
||
0xFE13,
|
||
0xFE55,
|
||
0xFF1A
|
||
]
|
||
|
||
acc = {[], [], case_ignorable, [], %{}, %{}}
|
||
cased_letter_categories = :binary.compile_pattern(["Ll", "Lt", "Lu"])
|
||
case_ignorable_categories = :binary.compile_pattern(["Mn", "Me", "Cf", "Lm", "Sk"])
|
||
|
||
{codes, cased_letters, case_ignorable, non_breakable, decompositions, combining_classes} =
|
||
data_path
|
||
|> File.read!()
|
||
|> String.split(["\r\n", "\n"], trim: true)
|
||
|> Enum.reduce(acc, fn line, {cacc, lacc, iacc, wacc, dacc, kacc} ->
|
||
[
|
||
codepoint,
|
||
_name,
|
||
category,
|
||
class,
|
||
_bidi,
|
||
decomposition,
|
||
_numeric_1,
|
||
_numeric_2,
|
||
_numeric_3,
|
||
_bidi_mirror,
|
||
_unicode_1,
|
||
_iso,
|
||
upper,
|
||
lower,
|
||
_title
|
||
] = :binary.split(line, ";", [:global])
|
||
|
||
cacc =
|
||
if upper != "" or lower != "" do
|
||
[{to_binary.(codepoint), to_binary.(upper), to_binary.(lower)} | cacc]
|
||
else
|
||
cacc
|
||
end
|
||
|
||
cased_letter_categories = :binary.compile_pattern(["Ll", "Lt", "Lu"])
|
||
case_ignorable_categories = :binary.compile_pattern(["Mn", "Me", "Cf", "Lm", "Sk"])
|
||
|
||
{lacc, iacc} =
|
||
cond do
|
||
match?({0, _}, :binary.match(category, cased_letter_categories)) ->
|
||
{[String.to_integer(codepoint, 16) | lacc], iacc}
|
||
|
||
match?({0, _}, :binary.match(category, case_ignorable_categories)) ->
|
||
{lacc, [String.to_integer(codepoint, 16) | iacc]}
|
||
|
||
true ->
|
||
{lacc, iacc}
|
||
end
|
||
|
||
wacc =
|
||
case decomposition do
|
||
"<noBreak>" <> _ -> [to_binary.(codepoint) | wacc]
|
||
_ -> wacc
|
||
end
|
||
|
||
dacc =
|
||
case decomposition do
|
||
# Decomposition
|
||
<<h, _::binary>> when h != ?< ->
|
||
decomposition =
|
||
decomposition
|
||
|> :binary.split(" ", [:global])
|
||
|> Enum.map(&String.to_integer(&1, 16))
|
||
|
||
:maps.put(String.to_integer(codepoint, 16), decomposition, dacc)
|
||
|
||
_ ->
|
||
dacc
|
||
end
|
||
|
||
kacc =
|
||
case String.to_integer(class) do
|
||
0 -> kacc
|
||
n -> :maps.put(String.to_integer(codepoint, 16), n, kacc)
|
||
end
|
||
|
||
{cacc, lacc, iacc, wacc, dacc, kacc}
|
||
end)
|
||
|
||
defmodule String.Unicode do
|
||
@moduledoc false
|
||
def version, do: {16, 0, 0}
|
||
|
||
[unconditional_mappings, _conditional_mappings] =
|
||
Path.join(__DIR__, "SpecialCasing.txt")
|
||
|> File.read!()
|
||
|> :binary.split("# Conditional Mappings")
|
||
|
||
codes =
|
||
unconditional_mappings
|
||
|> String.split(["\r\n", "\n"], trim: true)
|
||
|> Enum.reduce(codes, fn
|
||
"", acc ->
|
||
acc
|
||
|
||
"#" <> _, acc ->
|
||
acc
|
||
|
||
line, acc ->
|
||
[codepoint, lower, _title, upper, _] = :binary.split(line, "; ", [:global])
|
||
key = to_binary.(codepoint)
|
||
|
||
:lists.keystore(
|
||
key,
|
||
1,
|
||
acc,
|
||
{key, to_binary.(upper), to_binary.(lower)}
|
||
)
|
||
end)
|
||
|
||
# The function computes byte lookups based on the prefix. For example,
|
||
# Á, É, etc all have the same prefix <<195>>, so they are lumped
|
||
# together for lookup and then we just do a byte lookup later. We
|
||
# tried doing the byte lookup on 64-element tuple (since the byte
|
||
# is always within 0b10000000 and 0b10111111) but that's slower,
|
||
# especially because we need to check the byte range for invalid
|
||
# Unicode, instead the last byte lookup is a case. Grouping the
|
||
# top-level lookup makes the cost of a miss 3x cheaper albeit a
|
||
# hit is 10% more expensive) and reduces bytecode size.
|
||
compute_lookup = fn key_values ->
|
||
prefixes =
|
||
Enum.reduce(key_values, %{}, fn {codepoint, result}, acc ->
|
||
prefix_size = bit_size(codepoint) - 8
|
||
<<prefix::size(^prefix_size)-bits, byte>> = codepoint
|
||
Map.update(acc, prefix, [{byte, result}], &[{byte, result} | &1])
|
||
end)
|
||
|
||
{singles, tables} =
|
||
Enum.reduce(Map.delete(prefixes, ""), {[], []}, fn {prefix, pairs}, {singles, tables} ->
|
||
case pairs do
|
||
[{byte, result}] ->
|
||
{[{prefix <> <<byte>>, result} | singles], tables}
|
||
|
||
_ ->
|
||
clauses =
|
||
Enum.flat_map(pairs, fn {byte, result} ->
|
||
quote do
|
||
unquote(byte) -> unquote(result)
|
||
end
|
||
end)
|
||
|
||
clauses = clauses ++ quote do: (byte -> <<unquote(prefix), byte>>)
|
||
{singles, [{prefix, clauses} | tables]}
|
||
end
|
||
end)
|
||
|
||
{Enum.sort(singles), Enum.sort_by(tables, &(-byte_size(elem(&1, 0))))}
|
||
end
|
||
|
||
# Sigma variants for Greek
|
||
@letter_sigma <<0x03A3::utf8>>
|
||
@letter_small_sigma_final <<0x03C2::utf8>>
|
||
@letter_small_sigma <<0x03C3::utf8>>
|
||
|
||
# Letter I variants for Turkic languages
|
||
@letter_I <<0x0049::utf8>>
|
||
@dotless_letter_i <<0x0131::utf8>>
|
||
@letter_i <<0x0069::utf8>>
|
||
@letter_I_dot_above <<0x0130::utf8>>
|
||
@combining_dot_above <<0x0307::utf8>>
|
||
|
||
# Downcase
|
||
|
||
# Turkic İ -> i
|
||
def downcase(<<unquote(@letter_I_dot_above), rest::bits>>, acc, mode) do
|
||
char = if mode == :turkic, do: @letter_i, else: <<@letter_i, @combining_dot_above>>
|
||
downcase(rest, [char | acc], mode)
|
||
end
|
||
|
||
def downcase(<<@letter_I, @combining_dot_above, rest::bits>>, acc, mode) do
|
||
char = if mode == :turkic, do: @letter_i, else: <<@letter_i, @combining_dot_above>>
|
||
downcase(rest, [char | acc], mode)
|
||
end
|
||
|
||
# Turkic I -> ı
|
||
def downcase(<<@letter_I, rest::bits>>, acc, mode) do
|
||
char = if mode == :turkic, do: @dotless_letter_i, else: @letter_i
|
||
downcase(rest, [char | acc], mode)
|
||
end
|
||
|
||
# Greek sigma
|
||
def downcase(<<@letter_sigma, rest::bits>>, acc, mode) do
|
||
downcased =
|
||
if mode == :greek and cased_letter_list?(acc) and not cased_letter_binary?(rest) do
|
||
@letter_small_sigma_final
|
||
else
|
||
@letter_small_sigma
|
||
end
|
||
|
||
downcase(rest, [downcased | acc], mode)
|
||
end
|
||
|
||
conditional_downcase = [@letter_I, @letter_I_dot_above, @letter_sigma]
|
||
|
||
{singles, tables} =
|
||
compute_lookup.(
|
||
for {codepoint, _upper, lower} <- codes,
|
||
lower && lower != codepoint,
|
||
codepoint not in conditional_downcase,
|
||
do: {codepoint, lower}
|
||
)
|
||
|
||
for {codepoint, lower} <- singles do
|
||
def downcase(<<unquote(codepoint), rest::bits>>, acc, mode) do
|
||
downcase(rest, [unquote(lower) | acc], mode)
|
||
end
|
||
end
|
||
|
||
for {prefix, clauses} <- tables do
|
||
def downcase(<<unquote(prefix), byte, rest::bits>>, acc, mode) do
|
||
value = case byte, do: unquote(clauses)
|
||
downcase(rest, [value | acc], mode)
|
||
end
|
||
end
|
||
|
||
def downcase(<<byte, rest::bits>>, acc, mode) do
|
||
if byte >= ?A and byte <= ?Z do
|
||
downcase(rest, [byte + 32 | acc], mode)
|
||
else
|
||
downcase(rest, [byte | acc], mode)
|
||
end
|
||
end
|
||
|
||
def downcase("", acc, _mode), do: IO.iodata_to_binary(:lists.reverse(acc))
|
||
|
||
# Sigma handling
|
||
|
||
defp cased_letter_binary?(<<codepoint::utf8, rest::bits>>) do
|
||
if case_ignorable?(codepoint) do
|
||
cased_letter_binary?(rest)
|
||
else
|
||
cased_letter?(codepoint)
|
||
end
|
||
end
|
||
|
||
defp cased_letter_binary?(_), do: false
|
||
|
||
defp cased_letter_list?([<<codepoint::utf8>> | rest]) do
|
||
if case_ignorable?(codepoint) do
|
||
cased_letter_list?(rest)
|
||
else
|
||
cased_letter?(codepoint)
|
||
end
|
||
end
|
||
|
||
defp cased_letter_list?(_), do: false
|
||
|
||
for {first, last} <- rangify.(cased_letters) do
|
||
if first == last do
|
||
defp cased_letter?(unquote(first)), do: true
|
||
else
|
||
defp cased_letter?(codepoint)
|
||
when codepoint >= unquote(first) and codepoint <= unquote(last),
|
||
do: true
|
||
end
|
||
end
|
||
|
||
defp cased_letter?(_), do: false
|
||
|
||
for {first, last} <- rangify.(case_ignorable) do
|
||
if first == last do
|
||
defp case_ignorable?(unquote(first)), do: true
|
||
else
|
||
defp case_ignorable?(codepoint)
|
||
when codepoint >= unquote(first) and codepoint <= unquote(last),
|
||
do: true
|
||
end
|
||
end
|
||
|
||
defp case_ignorable?(_), do: false
|
||
|
||
# Upcase
|
||
|
||
# Turkic i -> İ
|
||
def upcase(<<@letter_i, rest::bits>>, acc, mode) do
|
||
char = if mode == :turkic, do: @letter_I_dot_above, else: @letter_I
|
||
upcase(rest, [char | acc], mode)
|
||
end
|
||
|
||
conditional_upcase = [@letter_i]
|
||
|
||
{singles, tables} =
|
||
compute_lookup.(
|
||
for {codepoint, upper, _lower} <- codes,
|
||
upper && upper != codepoint,
|
||
codepoint not in conditional_upcase,
|
||
do: {codepoint, upper}
|
||
)
|
||
|
||
for {codepoint, upper} <- singles do
|
||
def upcase(<<unquote(codepoint), rest::bits>>, acc, mode) do
|
||
upcase(rest, [unquote(upper) | acc], mode)
|
||
end
|
||
end
|
||
|
||
for {prefix, clauses} <- tables do
|
||
def upcase(<<unquote(prefix), byte, rest::bits>>, acc, mode) do
|
||
value = case byte, do: unquote(clauses)
|
||
upcase(rest, [value | acc], mode)
|
||
end
|
||
end
|
||
|
||
def upcase(<<byte, rest::bits>>, acc, mode) do
|
||
if byte >= ?a and byte <= ?z do
|
||
upcase(rest, [byte - 32 | acc], mode)
|
||
else
|
||
upcase(rest, [byte | acc], mode)
|
||
end
|
||
end
|
||
|
||
def upcase("", acc, _mode), do: IO.iodata_to_binary(:lists.reverse(acc))
|
||
end
|
||
|
||
defmodule String.Break do
|
||
@moduledoc false
|
||
@whitespace_max_size 3
|
||
|
||
prop_path = Path.join(__DIR__, "PropList.txt")
|
||
|
||
whitespace =
|
||
prop_path
|
||
|> File.read!()
|
||
|> String.split(["\r\n", "\n"])
|
||
|> Enum.reduce([], fn line, acc ->
|
||
case :binary.split(line, ";") do
|
||
[<<first::4-bytes, "..", last::4-bytes, _::binary>>, <<" White_Space", _::binary>>] ->
|
||
first = String.to_integer(first, 16)
|
||
last = String.to_integer(last, 16)
|
||
Enum.map(first..last, fn int -> <<int::utf8>> end) ++ acc
|
||
|
||
[<<single::4-bytes, _::binary>>, <<" White_Space", _::binary>>] ->
|
||
[<<String.to_integer(single, 16)::utf8>> | acc]
|
||
|
||
_ ->
|
||
acc
|
||
end
|
||
end)
|
||
|
||
IO.puts(:stderr, "[Unicode] Break on #{length(whitespace)} whitespace codepoints")
|
||
|
||
# trim_leading
|
||
|
||
def trim_leading(string) when is_binary(string) do
|
||
do_trim_leading(string)
|
||
end
|
||
|
||
for codepoint <- whitespace do
|
||
def do_trim_leading(<<unquote(codepoint), rest::bits>>), do: do_trim_leading(rest)
|
||
end
|
||
|
||
def do_trim_leading(<<rest::bits>>), do: rest
|
||
|
||
# trim_trailing
|
||
|
||
for cp <- whitespace do
|
||
# We need to increment @whitespace_max_size as well
|
||
# as the small table (_s) if we add a new entry here.
|
||
case byte_size(cp) do
|
||
3 ->
|
||
defp do_trim_trailing_l(unquote(cp)), do: -3
|
||
|
||
2 ->
|
||
defp do_trim_trailing_l(<<_, unquote(cp)>>), do: -2
|
||
defp do_trim_trailing_s(unquote(cp)), do: <<>>
|
||
|
||
1 ->
|
||
defp do_trim_trailing_l(<<unquote(cp), unquote(cp), unquote(cp)>>), do: -3
|
||
defp do_trim_trailing_l(<<_, unquote(cp), unquote(cp)>>), do: -2
|
||
defp do_trim_trailing_l(<<_, _, unquote(cp)>>), do: -1
|
||
|
||
defp do_trim_trailing_s(<<x, unquote(cp)>>), do: do_trim_trailing_s(<<x>>)
|
||
defp do_trim_trailing_s(unquote(cp)), do: <<>>
|
||
end
|
||
end
|
||
|
||
defp do_trim_trailing_l(_), do: 0
|
||
defp do_trim_trailing_s(o), do: o
|
||
|
||
def trim_trailing(string) when is_binary(string) do
|
||
trim_trailing(string, byte_size(string))
|
||
end
|
||
|
||
defp trim_trailing(string, size) when size < @whitespace_max_size do
|
||
do_trim_trailing_s(string)
|
||
end
|
||
|
||
defp trim_trailing(string, size) do
|
||
trail = binary_part(string, size, -@whitespace_max_size)
|
||
|
||
case do_trim_trailing_l(trail) do
|
||
0 -> string
|
||
x -> trim_trailing(binary_part(string, 0, size + x), size + x)
|
||
end
|
||
end
|
||
|
||
# Split
|
||
|
||
def split(string) do
|
||
:binary.split(string, unquote(whitespace -- non_breakable), [:global, :trim_all])
|
||
end
|
||
|
||
# Decompose
|
||
|
||
def decompose(entries, map) do
|
||
for entry <- entries do
|
||
case map do
|
||
%{^entry => match} -> decompose(match, map)
|
||
%{} -> <<entry::utf8>>
|
||
end
|
||
end
|
||
end
|
||
end
|