Add support for named sequences and make String module aware of it so it does correct job at calculating length, picking characters, etc.
This commit is contained in:
Yurii Rashkovskii
+37
-41
@@ -2,7 +2,11 @@ defmodule String do
|
||||
@moduledoc """
|
||||
A string in Elixir is a utf-8 binary. This module
|
||||
contains function to work with utf-8 data and its
|
||||
codepoints.
|
||||
codepoints and graphemes.
|
||||
|
||||
The difference between codepoints is that codepoints represent individual
|
||||
characters, however, graphemes contain multiple characters that
|
||||
are "perceived as a single character" by readers.
|
||||
|
||||
For working with raw binaries, use Erlang's :binary
|
||||
module.
|
||||
@@ -295,8 +299,11 @@ defmodule String do
|
||||
|
||||
defp do_codepoints(:no_codepoint), do: []
|
||||
|
||||
@doc false
|
||||
defdelegate codepoint(string), to: String.Unicode
|
||||
|
||||
@doc """
|
||||
Returns the first codepoint from an utf8 string.
|
||||
Returns the first grapheme from an utf8 string.
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -305,14 +312,14 @@ defmodule String do
|
||||
|
||||
"""
|
||||
def first(string) do
|
||||
case codepoint(string) do
|
||||
case grapheme(string) do
|
||||
{ char, _ } -> char
|
||||
:no_codepoint -> ""
|
||||
:no_sequence -> ""
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the last codepoint from an utf8 string.
|
||||
Returns the last grapheme from an utf8 string.
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -321,17 +328,17 @@ defmodule String do
|
||||
|
||||
"""
|
||||
def last(string) do
|
||||
do_last(codepoint(string), "")
|
||||
do_last(grapheme(string), "")
|
||||
end
|
||||
|
||||
defp do_last({char, rest}, _) do
|
||||
do_last(codepoint(rest), char)
|
||||
do_last(grapheme(rest), char)
|
||||
end
|
||||
|
||||
defp do_last(:no_codepoint, last_char), do: last_char
|
||||
defp do_last(:no_sequence, last_char), do: last_char
|
||||
|
||||
@doc """
|
||||
Returns the number of codepoint in an utf8 string.
|
||||
Returns the number of unicode graphemes in an utf8 string.
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -340,17 +347,28 @@ defmodule String do
|
||||
|
||||
"""
|
||||
def length(string) do
|
||||
do_length(codepoint(string))
|
||||
do_length(grapheme(string))
|
||||
end
|
||||
|
||||
defp do_length({_, rest}) do
|
||||
1 + do_length(codepoint(rest))
|
||||
1 + do_length(grapheme(rest))
|
||||
end
|
||||
|
||||
defp do_length(:no_codepoint), do: 0
|
||||
defp do_length(:no_sequence), do: 0
|
||||
|
||||
@doc """
|
||||
Returns the codepoint in the `position` of the given utf8 `string`.
|
||||
Returns unicode graphemes in the string
|
||||
|
||||
## Examples
|
||||
String.graphemes("Ā̀stute") # => ["Ā̀","s","t","u","t","e"]
|
||||
|
||||
"""
|
||||
defdelegate graphemes(string), to: String.Unicode, as: :sequences
|
||||
|
||||
@doc false
|
||||
defdelegate grapheme(string), to: String.Unicode, as: :sequence
|
||||
|
||||
@doc """
|
||||
Returns the grapheme in the `position` of the given utf8 `string`.
|
||||
If `position` is greater than `string` length, than it returns `nil`.
|
||||
|
||||
## Examples
|
||||
@@ -363,47 +381,25 @@ defmodule String do
|
||||
|
||||
"""
|
||||
def at(string, position) when position >= 0 do
|
||||
do_at(codepoint(string), position, 0)
|
||||
do_at(grapheme(string), position, 0)
|
||||
end
|
||||
|
||||
def at(string, position) when position < 0 do
|
||||
real_pos = do_length(codepoint(string)) - abs(position)
|
||||
real_pos = do_length(grapheme(string)) - abs(position)
|
||||
case real_pos >= 0 do
|
||||
true -> do_at(codepoint(string), real_pos, 0)
|
||||
true -> do_at(grapheme(string), real_pos, 0)
|
||||
false -> ""
|
||||
end
|
||||
end
|
||||
|
||||
defp do_at({_ , rest}, desired_pos, current_pos) when desired_pos > current_pos do
|
||||
do_at(codepoint(rest), desired_pos, current_pos + 1)
|
||||
do_at(grapheme(rest), desired_pos, current_pos + 1)
|
||||
end
|
||||
|
||||
defp do_at({char, _}, desired_pos, current_pos) when desired_pos == current_pos do
|
||||
char
|
||||
end
|
||||
|
||||
defp do_at(:no_codepoint, _, _), do: ""
|
||||
defp do_at(:no_sequence, _, _), do: ""
|
||||
|
||||
# Private implementation which returns the first codepoint
|
||||
# of any given utf8 string and the rest of it
|
||||
# If an empty string is given, :no_codepoint is returned.
|
||||
defp codepoint(<<194, char, rest :: binary>>)
|
||||
when char in 161..191,
|
||||
do: { <<194, char>>, rest }
|
||||
|
||||
defp codepoint(<<first, char, rest :: binary>>)
|
||||
when first in 195..223 and char in 128..191,
|
||||
do: { <<first, char>>, rest }
|
||||
|
||||
defp codepoint(<<first, second, char, rest :: binary>>)
|
||||
when first == 224 and second in 160..191 and char in 128..191,
|
||||
do: { <<first, second, char>>, rest }
|
||||
|
||||
defp codepoint(<<first, second, char, rest :: binary>>)
|
||||
when first in 225..239 and second in 128..191 and char in 128..191,
|
||||
do: { <<first, second, char>>, rest }
|
||||
|
||||
defp codepoint(<<other, rest :: binary>>), do: { <<other>>, rest }
|
||||
|
||||
defp codepoint(<<>>), do: :no_codepoint
|
||||
end
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300
|
||||
LATIN SMALL LETTER A WITH MACRON AND GRAVE;0101 0300
|
||||
LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW;0045 0329
|
||||
LATIN SMALL LETTER E WITH VERTICAL LINE BELOW;0065 0329
|
||||
LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00C8 0329
|
||||
LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00E8 0329
|
||||
LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00C9 0329
|
||||
LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00E9 0329
|
||||
LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND MACRON;00CA 0304
|
||||
LATIN SMALL LETTER E WITH CIRCUMFLEX AND MACRON;00EA 0304
|
||||
LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND CARON;00CA 030C
|
||||
LATIN SMALL LETTER E WITH CIRCUMFLEX AND CARON;00EA 030C
|
||||
LATIN CAPITAL LETTER I WITH MACRON AND GRAVE;012A 0300
|
||||
LATIN SMALL LETTER I WITH MACRON AND GRAVE;012B 0300
|
||||
LATIN SMALL LETTER I WITH DOT ABOVE AND ACUTE;0069 0307 0301
|
||||
LATIN SMALL LETTER NG WITH TILDE ABOVE;006E 0360 0067
|
||||
LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW;004F 0329
|
||||
LATIN SMALL LETTER O WITH VERTICAL LINE BELOW;006F 0329
|
||||
LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00D2 0329
|
||||
LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00F2 0329
|
||||
LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00D3 0329
|
||||
LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00F3 0329
|
||||
LATIN CAPITAL LETTER S WITH VERTICAL LINE BELOW;0053 0329
|
||||
LATIN SMALL LETTER S WITH VERTICAL LINE BELOW;0073 0329
|
||||
LATIN CAPITAL LETTER U WITH MACRON AND GRAVE;016A 0300
|
||||
LATIN SMALL LETTER U WITH MACRON AND GRAVE;016B 0300
|
||||
LATIN CAPITAL LETTER A WITH OGONEK AND ACUTE;0104 0301
|
||||
LATIN SMALL LETTER A WITH OGONEK AND ACUTE;0105 0301
|
||||
LATIN CAPITAL LETTER A WITH OGONEK AND TILDE;0104 0303
|
||||
LATIN SMALL LETTER A WITH OGONEK AND TILDE;0105 0303
|
||||
LATIN CAPITAL LETTER E WITH OGONEK AND ACUTE;0118 0301
|
||||
LATIN SMALL LETTER E WITH OGONEK AND ACUTE;0119 0301
|
||||
LATIN CAPITAL LETTER E WITH OGONEK AND TILDE;0118 0303
|
||||
LATIN SMALL LETTER E WITH OGONEK AND TILDE;0119 0303
|
||||
LATIN CAPITAL LETTER E WITH DOT ABOVE AND ACUTE;0116 0301
|
||||
LATIN SMALL LETTER E WITH DOT ABOVE AND ACUTE;0117 0301
|
||||
LATIN CAPITAL LETTER E WITH DOT ABOVE AND TILDE;0116 0303
|
||||
LATIN SMALL LETTER E WITH DOT ABOVE AND TILDE;0117 0303
|
||||
LATIN SMALL LETTER I WITH DOT ABOVE AND GRAVE;0069 0307 0300
|
||||
LATIN SMALL LETTER I WITH DOT ABOVE AND TILDE;0069 0307 0303
|
||||
LATIN CAPITAL LETTER I WITH OGONEK AND ACUTE;012E 0301
|
||||
LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND ACUTE;012F 0307 0301
|
||||
LATIN CAPITAL LETTER I WITH OGONEK AND TILDE;012E 0303
|
||||
LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND TILDE;012F 0307 0303
|
||||
LATIN CAPITAL LETTER J WITH TILDE;004A 0303
|
||||
LATIN SMALL LETTER J WITH DOT ABOVE AND TILDE;006A 0307 0303
|
||||
LATIN CAPITAL LETTER L WITH TILDE;004C 0303
|
||||
LATIN SMALL LETTER L WITH TILDE;006C 0303
|
||||
LATIN CAPITAL LETTER M WITH TILDE;004D 0303
|
||||
LATIN SMALL LETTER M WITH TILDE;006D 0303
|
||||
LATIN CAPITAL LETTER R WITH TILDE;0052 0303
|
||||
LATIN SMALL LETTER R WITH TILDE;0072 0303
|
||||
LATIN CAPITAL LETTER U WITH OGONEK AND ACUTE;0172 0301
|
||||
LATIN SMALL LETTER U WITH OGONEK AND ACUTE;0173 0301
|
||||
LATIN CAPITAL LETTER U WITH OGONEK AND TILDE;0172 0303
|
||||
LATIN SMALL LETTER U WITH OGONEK AND TILDE;0173 0303
|
||||
LATIN CAPITAL LETTER U WITH MACRON AND ACUTE;016A 0301
|
||||
LATIN SMALL LETTER U WITH MACRON AND ACUTE;016B 0301
|
||||
LATIN CAPITAL LETTER U WITH MACRON AND TILDE;016A 0303
|
||||
LATIN SMALL LETTER U WITH MACRON AND TILDE;016B 0303
|
||||
LATIN SMALL LETTER AE WITH GRAVE;00E6 0300
|
||||
LATIN SMALL LETTER OPEN O WITH GRAVE;0254 0300
|
||||
LATIN SMALL LETTER OPEN O WITH ACUTE;0254 0301
|
||||
LATIN SMALL LETTER TURNED V WITH GRAVE;028C 0300
|
||||
LATIN SMALL LETTER TURNED V WITH ACUTE;028C 0301
|
||||
LATIN SMALL LETTER SCHWA WITH GRAVE;0259 0300
|
||||
LATIN SMALL LETTER SCHWA WITH ACUTE;0259 0301
|
||||
LATIN SMALL LETTER HOOKED SCHWA WITH GRAVE;025A 0300
|
||||
LATIN SMALL LETTER HOOKED SCHWA WITH ACUTE;025A 0301
|
||||
BENGALI LETTER KHINYA;0995 09CD 09B7
|
||||
TAMIL CONSONANT K; 0B95 0BCD
|
||||
TAMIL CONSONANT NG; 0B99 0BCD
|
||||
TAMIL CONSONANT C; 0B9A 0BCD
|
||||
TAMIL CONSONANT NY; 0B9E 0BCD
|
||||
TAMIL CONSONANT TT; 0B9F 0BCD
|
||||
TAMIL CONSONANT NN; 0BA3 0BCD
|
||||
TAMIL CONSONANT T; 0BA4 0BCD
|
||||
TAMIL CONSONANT N; 0BA8 0BCD
|
||||
TAMIL CONSONANT P; 0BAA 0BCD
|
||||
TAMIL CONSONANT M; 0BAE 0BCD
|
||||
TAMIL CONSONANT Y; 0BAF 0BCD
|
||||
TAMIL CONSONANT R; 0BB0 0BCD
|
||||
TAMIL CONSONANT L; 0BB2 0BCD
|
||||
TAMIL CONSONANT V; 0BB5 0BCD
|
||||
TAMIL CONSONANT LLL;0BB4 0BCD
|
||||
TAMIL CONSONANT LL; 0BB3 0BCD
|
||||
TAMIL CONSONANT RR; 0BB1 0BCD
|
||||
TAMIL CONSONANT NNN;0BA9 0BCD
|
||||
TAMIL CONSONANT J; 0B9C 0BCD
|
||||
TAMIL CONSONANT SH; 0BB6 0BCD
|
||||
TAMIL CONSONANT SS; 0BB7 0BCD
|
||||
TAMIL CONSONANT S; 0BB8 0BCD
|
||||
TAMIL CONSONANT H; 0BB9 0BCD
|
||||
TAMIL CONSONANT KSS;0B95 0BCD 0BB7 0BCD
|
||||
TAMIL SYLLABLE KAA; 0B95 0BBE
|
||||
TAMIL SYLLABLE KI; 0B95 0BBF
|
||||
TAMIL SYLLABLE KII; 0B95 0BC0
|
||||
TAMIL SYLLABLE KU; 0B95 0BC1
|
||||
TAMIL SYLLABLE KUU; 0B95 0BC2
|
||||
TAMIL SYLLABLE KE; 0B95 0BC6
|
||||
TAMIL SYLLABLE KEE; 0B95 0BC7
|
||||
TAMIL SYLLABLE KAI; 0B95 0BC8
|
||||
TAMIL SYLLABLE KO; 0B95 0BCA
|
||||
TAMIL SYLLABLE KOO; 0B95 0BCB
|
||||
TAMIL SYLLABLE KAU; 0B95 0BCC
|
||||
TAMIL SYLLABLE NGAA; 0B99 0BBE
|
||||
TAMIL SYLLABLE NGI; 0B99 0BBF
|
||||
TAMIL SYLLABLE NGII; 0B99 0BC0
|
||||
TAMIL SYLLABLE NGU; 0B99 0BC1
|
||||
TAMIL SYLLABLE NGUU; 0B99 0BC2
|
||||
TAMIL SYLLABLE NGE; 0B99 0BC6
|
||||
TAMIL SYLLABLE NGEE; 0B99 0BC7
|
||||
TAMIL SYLLABLE NGAI; 0B99 0BC8
|
||||
TAMIL SYLLABLE NGO; 0B99 0BCA
|
||||
TAMIL SYLLABLE NGOO; 0B99 0BCB
|
||||
TAMIL SYLLABLE NGAU; 0B99 0BCC
|
||||
TAMIL SYLLABLE CI; 0B9A 0BBF
|
||||
TAMIL SYLLABLE CII; 0B9A 0BC0
|
||||
TAMIL SYLLABLE CU; 0B9A 0BC1
|
||||
TAMIL SYLLABLE CUU; 0B9A 0BC2
|
||||
TAMIL SYLLABLE CE; 0B9A 0BC6
|
||||
TAMIL SYLLABLE CEE; 0B9A 0BC7
|
||||
TAMIL SYLLABLE CAI; 0B9A 0BC8
|
||||
TAMIL SYLLABLE CO; 0B9A 0BCA
|
||||
TAMIL SYLLABLE COO; 0B9A 0BCB
|
||||
TAMIL SYLLABLE CAU; 0B9A 0BCC
|
||||
TAMIL SYLLABLE NYAA; 0B9E 0BBE
|
||||
TAMIL SYLLABLE NYI; 0B9E 0BBF
|
||||
TAMIL SYLLABLE NYII; 0B9E 0BC0
|
||||
TAMIL SYLLABLE NYU; 0B9E 0BC1
|
||||
TAMIL SYLLABLE NYUU; 0B9E 0BC2
|
||||
TAMIL SYLLABLE NYE; 0B9E 0BC6
|
||||
TAMIL SYLLABLE NYEE; 0B9E 0BC7
|
||||
TAMIL SYLLABLE NYAI; 0B9E 0BC8
|
||||
TAMIL SYLLABLE NYO; 0B9E 0BCA
|
||||
TAMIL SYLLABLE NYOO; 0B9E 0BCB
|
||||
TAMIL SYLLABLE NYAU; 0B9E 0BCC
|
||||
TAMIL SYLLABLE TTAA; 0B9F 0BBE
|
||||
TAMIL SYLLABLE TTI; 0B9F 0BBF
|
||||
TAMIL SYLLABLE TTII; 0B9F 0BC0
|
||||
TAMIL SYLLABLE TTU; 0B9F 0BC1
|
||||
TAMIL SYLLABLE TTUU; 0B9F 0BC2
|
||||
TAMIL SYLLABLE TTE; 0B9F 0BC6
|
||||
TAMIL SYLLABLE TTEE; 0B9F 0BC7
|
||||
TAMIL SYLLABLE TTAI; 0B9F 0BC8
|
||||
TAMIL SYLLABLE TTO; 0B9F 0BCA
|
||||
TAMIL SYLLABLE TTOO; 0B9F 0BCB
|
||||
TAMIL SYLLABLE TTAU; 0B9F 0BCC
|
||||
TAMIL SYLLABLE NNAA; 0BA3 0BBE
|
||||
TAMIL SYLLABLE NNI; 0BA3 0BBF
|
||||
TAMIL SYLLABLE NNII; 0BA3 0BC0
|
||||
TAMIL SYLLABLE NNU; 0BA3 0BC1
|
||||
TAMIL SYLLABLE NNUU; 0BA3 0BC2
|
||||
TAMIL SYLLABLE NNE; 0BA3 0BC6
|
||||
TAMIL SYLLABLE NNEE; 0BA3 0BC7
|
||||
TAMIL SYLLABLE NNAI; 0BA3 0BC8
|
||||
TAMIL SYLLABLE NNO; 0BA3 0BCA
|
||||
TAMIL SYLLABLE NNOO; 0BA3 0BCB
|
||||
TAMIL SYLLABLE NNAU; 0BA3 0BCC
|
||||
TAMIL SYLLABLE TAA; 0BA4 0BBE
|
||||
TAMIL SYLLABLE TI; 0BA4 0BBF
|
||||
TAMIL SYLLABLE TII; 0BA4 0BC0
|
||||
TAMIL SYLLABLE TU; 0BA4 0BC1
|
||||
TAMIL SYLLABLE TUU; 0BA4 0BC2
|
||||
TAMIL SYLLABLE TE; 0BA4 0BC6
|
||||
TAMIL SYLLABLE TEE; 0BA4 0BC7
|
||||
TAMIL SYLLABLE TAI; 0BA4 0BC8
|
||||
TAMIL SYLLABLE TO; 0BA4 0BCA
|
||||
TAMIL SYLLABLE TOO; 0BA4 0BCB
|
||||
TAMIL SYLLABLE TAU; 0BA4 0BCC
|
||||
TAMIL SYLLABLE NAA; 0BA8 0BBE
|
||||
TAMIL SYLLABLE NI; 0BA8 0BBF
|
||||
TAMIL SYLLABLE NII; 0BA8 0BC0
|
||||
TAMIL SYLLABLE NU; 0BA8 0BC1
|
||||
TAMIL SYLLABLE NUU; 0BA8 0BC2
|
||||
TAMIL SYLLABLE NE; 0BA8 0BC6
|
||||
TAMIL SYLLABLE NEE; 0BA8 0BC7
|
||||
TAMIL SYLLABLE NAI; 0BA8 0BC8
|
||||
TAMIL SYLLABLE NO; 0BA8 0BCA
|
||||
TAMIL SYLLABLE NOO; 0BA8 0BCB
|
||||
TAMIL SYLLABLE NAU; 0BA8 0BCC
|
||||
TAMIL SYLLABLE PAA; 0BAA 0BBE
|
||||
TAMIL SYLLABLE PI; 0BAA 0BBF
|
||||
TAMIL SYLLABLE PII; 0BAA 0BC0
|
||||
TAMIL SYLLABLE PU; 0BAA 0BC1
|
||||
TAMIL SYLLABLE PUU; 0BAA 0BC2
|
||||
TAMIL SYLLABLE PE; 0BAA 0BC6
|
||||
TAMIL SYLLABLE PEE; 0BAA 0BC7
|
||||
TAMIL SYLLABLE PAI; 0BAA 0BC8
|
||||
TAMIL SYLLABLE PO; 0BAA 0BCA
|
||||
TAMIL SYLLABLE POO; 0BAA 0BCB
|
||||
TAMIL SYLLABLE PAU; 0BAA 0BCC
|
||||
TAMIL SYLLABLE MAA; 0BAE 0BBE
|
||||
TAMIL SYLLABLE MI; 0BAE 0BBF
|
||||
TAMIL SYLLABLE MII; 0BAE 0BC0
|
||||
TAMIL SYLLABLE MU; 0BAE 0BC1
|
||||
TAMIL SYLLABLE MUU; 0BAE 0BC2
|
||||
TAMIL SYLLABLE ME; 0BAE 0BC6
|
||||
TAMIL SYLLABLE MEE; 0BAE 0BC7
|
||||
TAMIL SYLLABLE MAI; 0BAE 0BC8
|
||||
TAMIL SYLLABLE MO; 0BAE 0BCA
|
||||
TAMIL SYLLABLE MOO; 0BAE 0BCB
|
||||
TAMIL SYLLABLE MAU; 0BAE 0BCC
|
||||
TAMIL SYLLABLE YAA; 0BAF 0BBE
|
||||
TAMIL SYLLABLE YI; 0BAF 0BBF
|
||||
TAMIL SYLLABLE YII; 0BAF 0BC0
|
||||
TAMIL SYLLABLE YU; 0BAF 0BC1
|
||||
TAMIL SYLLABLE YUU; 0BAF 0BC2
|
||||
TAMIL SYLLABLE YE; 0BAF 0BC6
|
||||
TAMIL SYLLABLE YEE; 0BAF 0BC7
|
||||
TAMIL SYLLABLE YAI; 0BAF 0BC8
|
||||
TAMIL SYLLABLE YO; 0BAF 0BCA
|
||||
TAMIL SYLLABLE YOO; 0BAF 0BCB
|
||||
TAMIL SYLLABLE YAU; 0BAF 0BCC
|
||||
TAMIL SYLLABLE RAA; 0BB0 0BBE
|
||||
TAMIL SYLLABLE RI; 0BB0 0BBF
|
||||
TAMIL SYLLABLE RII; 0BB0 0BC0
|
||||
TAMIL SYLLABLE RU; 0BB0 0BC1
|
||||
TAMIL SYLLABLE RUU; 0BB0 0BC2
|
||||
TAMIL SYLLABLE RE; 0BB0 0BC6
|
||||
TAMIL SYLLABLE REE; 0BB0 0BC7
|
||||
TAMIL SYLLABLE RAI; 0BB0 0BC8
|
||||
TAMIL SYLLABLE RO; 0BB0 0BCA
|
||||
TAMIL SYLLABLE ROO; 0BB0 0BCB
|
||||
TAMIL SYLLABLE RAU; 0BB0 0BCC
|
||||
TAMIL SYLLABLE LAA; 0BB2 0BBE
|
||||
TAMIL SYLLABLE LI; 0BB2 0BBF
|
||||
TAMIL SYLLABLE LII; 0BB2 0BC0
|
||||
TAMIL SYLLABLE LU; 0BB2 0BC1
|
||||
TAMIL SYLLABLE LUU; 0BB2 0BC2
|
||||
TAMIL SYLLABLE LE; 0BB2 0BC6
|
||||
TAMIL SYLLABLE LEE; 0BB2 0BC7
|
||||
TAMIL SYLLABLE LAI; 0BB2 0BC8
|
||||
TAMIL SYLLABLE LO; 0BB2 0BCA
|
||||
TAMIL SYLLABLE LOO; 0BB2 0BCB
|
||||
TAMIL SYLLABLE LAU; 0BB2 0BCC
|
||||
TAMIL SYLLABLE VAA; 0BB5 0BBE
|
||||
TAMIL SYLLABLE VI; 0BB5 0BBF
|
||||
TAMIL SYLLABLE VII; 0BB5 0BC0
|
||||
TAMIL SYLLABLE VU; 0BB5 0BC1
|
||||
TAMIL SYLLABLE VUU; 0BB5 0BC2
|
||||
TAMIL SYLLABLE VE; 0BB5 0BC6
|
||||
TAMIL SYLLABLE VEE; 0BB5 0BC7
|
||||
TAMIL SYLLABLE VAI; 0BB5 0BC8
|
||||
TAMIL SYLLABLE VO; 0BB5 0BCA
|
||||
TAMIL SYLLABLE VOO; 0BB5 0BCB
|
||||
TAMIL SYLLABLE VAU; 0BB5 0BCC
|
||||
TAMIL SYLLABLE LLLAA; 0BB4 0BBE
|
||||
TAMIL SYLLABLE LLLI; 0BB4 0BBF
|
||||
TAMIL SYLLABLE LLLII; 0BB4 0BC0
|
||||
TAMIL SYLLABLE LLLU; 0BB4 0BC1
|
||||
TAMIL SYLLABLE LLLUU; 0BB4 0BC2
|
||||
TAMIL SYLLABLE LLLE; 0BB4 0BC6
|
||||
TAMIL SYLLABLE LLLEE; 0BB4 0BC7
|
||||
TAMIL SYLLABLE LLLAI; 0BB4 0BC8
|
||||
TAMIL SYLLABLE LLLO; 0BB4 0BCA
|
||||
TAMIL SYLLABLE LLLOO; 0BB4 0BCB
|
||||
TAMIL SYLLABLE LLLAU; 0BB4 0BCC
|
||||
TAMIL SYLLABLE LLAA; 0BB3 0BBE
|
||||
TAMIL SYLLABLE LLI; 0BB3 0BBF
|
||||
TAMIL SYLLABLE LLII; 0BB3 0BC0
|
||||
TAMIL SYLLABLE LLU; 0BB3 0BC1
|
||||
TAMIL SYLLABLE LLUU; 0BB3 0BC2
|
||||
TAMIL SYLLABLE LLE; 0BB3 0BC6
|
||||
TAMIL SYLLABLE LLEE; 0BB3 0BC7
|
||||
TAMIL SYLLABLE LLAI; 0BB3 0BC8
|
||||
TAMIL SYLLABLE LLO; 0BB3 0BCA
|
||||
TAMIL SYLLABLE LLOO; 0BB3 0BCB
|
||||
TAMIL SYLLABLE LLAU; 0BB3 0BCC
|
||||
TAMIL SYLLABLE RRAA; 0BB1 0BBE
|
||||
TAMIL SYLLABLE RRI; 0BB1 0BBF
|
||||
TAMIL SYLLABLE RRII; 0BB1 0BC0
|
||||
TAMIL SYLLABLE RRU; 0BB1 0BC1
|
||||
TAMIL SYLLABLE RRUU; 0BB1 0BC2
|
||||
TAMIL SYLLABLE RRE; 0BB1 0BC6
|
||||
TAMIL SYLLABLE RREE; 0BB1 0BC7
|
||||
TAMIL SYLLABLE RRAI; 0BB1 0BC8
|
||||
TAMIL SYLLABLE RRO; 0BB1 0BCA
|
||||
TAMIL SYLLABLE RROO; 0BB1 0BCB
|
||||
TAMIL SYLLABLE RRAU; 0BB1 0BCC
|
||||
TAMIL SYLLABLE NNNAA; 0BA9 0BBE
|
||||
TAMIL SYLLABLE NNNI; 0BA9 0BBF
|
||||
TAMIL SYLLABLE NNNII; 0BA9 0BC0
|
||||
TAMIL SYLLABLE NNNU; 0BA9 0BC1
|
||||
TAMIL SYLLABLE NNNUU; 0BA9 0BC2
|
||||
TAMIL SYLLABLE NNNE; 0BA9 0BC6
|
||||
TAMIL SYLLABLE NNNEE; 0BA9 0BC7
|
||||
TAMIL SYLLABLE NNNAI; 0BA9 0BC8
|
||||
TAMIL SYLLABLE NNNO; 0BA9 0BCA
|
||||
TAMIL SYLLABLE NNNOO; 0BA9 0BCB
|
||||
TAMIL SYLLABLE NNNAU; 0BA9 0BCC
|
||||
TAMIL SYLLABLE JAA; 0B9C 0BBE
|
||||
TAMIL SYLLABLE JI; 0B9C 0BBF
|
||||
TAMIL SYLLABLE JII; 0B9C 0BC0
|
||||
TAMIL SYLLABLE JU; 0B9C 0BC1
|
||||
TAMIL SYLLABLE JUU; 0B9C 0BC2
|
||||
TAMIL SYLLABLE JE; 0B9C 0BC6
|
||||
TAMIL SYLLABLE JEE; 0B9C 0BC7
|
||||
TAMIL SYLLABLE JAI; 0B9C 0BC8
|
||||
TAMIL SYLLABLE JO; 0B9C 0BCA
|
||||
TAMIL SYLLABLE JOO; 0B9C 0BCB
|
||||
TAMIL SYLLABLE JAU; 0B9C 0BCC
|
||||
TAMIL SYLLABLE SHAA; 0BB6 0BBE
|
||||
TAMIL SYLLABLE SHI; 0BB6 0BBF
|
||||
TAMIL SYLLABLE SHII; 0BB6 0BC0
|
||||
TAMIL SYLLABLE SHU; 0BB6 0BC1
|
||||
TAMIL SYLLABLE SHUU; 0BB6 0BC2
|
||||
TAMIL SYLLABLE SHE; 0BB6 0BC6
|
||||
TAMIL SYLLABLE SHEE; 0BB6 0BC7
|
||||
TAMIL SYLLABLE SHAI; 0BB6 0BC8
|
||||
TAMIL SYLLABLE SHO; 0BB6 0BCA
|
||||
TAMIL SYLLABLE SHOO; 0BB6 0BCB
|
||||
TAMIL SYLLABLE SHAU; 0BB6 0BCC
|
||||
TAMIL SYLLABLE SSAA; 0BB7 0BBE
|
||||
TAMIL SYLLABLE SSI; 0BB7 0BBF
|
||||
TAMIL SYLLABLE SSII; 0BB7 0BC0
|
||||
TAMIL SYLLABLE SSU; 0BB7 0BC1
|
||||
TAMIL SYLLABLE SSUU; 0BB7 0BC2
|
||||
TAMIL SYLLABLE SSE; 0BB7 0BC6
|
||||
TAMIL SYLLABLE SSEE; 0BB7 0BC7
|
||||
TAMIL SYLLABLE SSAI; 0BB7 0BC8
|
||||
TAMIL SYLLABLE SSO; 0BB7 0BCA
|
||||
TAMIL SYLLABLE SSOO; 0BB7 0BCB
|
||||
TAMIL SYLLABLE SSAU; 0BB7 0BCC
|
||||
TAMIL SYLLABLE SAA; 0BB8 0BBE
|
||||
TAMIL SYLLABLE SI; 0BB8 0BBF
|
||||
TAMIL SYLLABLE SII; 0BB8 0BC0
|
||||
TAMIL SYLLABLE SU; 0BB8 0BC1
|
||||
TAMIL SYLLABLE SUU; 0BB8 0BC2
|
||||
TAMIL SYLLABLE SE; 0BB8 0BC6
|
||||
TAMIL SYLLABLE SEE; 0BB8 0BC7
|
||||
TAMIL SYLLABLE SAI; 0BB8 0BC8
|
||||
TAMIL SYLLABLE SO; 0BB8 0BCA
|
||||
TAMIL SYLLABLE SOO; 0BB8 0BCB
|
||||
TAMIL SYLLABLE SAU; 0BB8 0BCC
|
||||
TAMIL SYLLABLE HAA; 0BB9 0BBE
|
||||
TAMIL SYLLABLE HI; 0BB9 0BBF
|
||||
TAMIL SYLLABLE HII; 0BB9 0BC0
|
||||
TAMIL SYLLABLE HU; 0BB9 0BC1
|
||||
TAMIL SYLLABLE HUU; 0BB9 0BC2
|
||||
TAMIL SYLLABLE HE; 0BB9 0BC6
|
||||
TAMIL SYLLABLE HEE; 0BB9 0BC7
|
||||
TAMIL SYLLABLE HAI; 0BB9 0BC8
|
||||
TAMIL SYLLABLE HO; 0BB9 0BCA
|
||||
TAMIL SYLLABLE HOO; 0BB9 0BCB
|
||||
TAMIL SYLLABLE HAU; 0BB9 0BCC
|
||||
TAMIL SYLLABLE KSSA; 0B95 0BCD 0BB7
|
||||
TAMIL SYLLABLE KSSAA; 0B95 0BCD 0BB7 0BBE
|
||||
TAMIL SYLLABLE KSSI; 0B95 0BCD 0BB7 0BBF
|
||||
TAMIL SYLLABLE KSSII; 0B95 0BCD 0BB7 0BC0
|
||||
TAMIL SYLLABLE KSSU; 0B95 0BCD 0BB7 0BC1
|
||||
TAMIL SYLLABLE KSSUU; 0B95 0BCD 0BB7 0BC2
|
||||
TAMIL SYLLABLE KSSE; 0B95 0BCD 0BB7 0BC6
|
||||
TAMIL SYLLABLE KSSEE; 0B95 0BCD 0BB7 0BC7
|
||||
TAMIL SYLLABLE KSSAI; 0B95 0BCD 0BB7 0BC8
|
||||
TAMIL SYLLABLE KSSO; 0B95 0BCD 0BB7 0BCA
|
||||
TAMIL SYLLABLE KSSOO; 0B95 0BCD 0BB7 0BCB
|
||||
TAMIL SYLLABLE KSSAU; 0B95 0BCD 0BB7 0BCC
|
||||
TAMIL SYLLABLE SHRII; 0BB6 0BCD 0BB0 0BC0
|
||||
SINHALA CONSONANT SIGN YANSAYA;0DCA 200D 0DBA
|
||||
SINHALA CONSONANT SIGN RAKAARAANSAYA;0DCA 200D 0DBB
|
||||
SINHALA CONSONANT SIGN REPAYA;0DBB 0DCA 200D
|
||||
GEORGIAN LETTER U-BRJGU;10E3 0302
|
||||
KHMER CONSONANT SIGN COENG KA;17D2 1780
|
||||
KHMER CONSONANT SIGN COENG KHA;17D2 1781
|
||||
KHMER CONSONANT SIGN COENG KO;17D2 1782
|
||||
KHMER CONSONANT SIGN COENG KHO;17D2 1783
|
||||
KHMER CONSONANT SIGN COENG NGO;17D2 1784
|
||||
KHMER CONSONANT SIGN COENG CA;17D2 1785
|
||||
KHMER CONSONANT SIGN COENG CHA;17D2 1786
|
||||
KHMER CONSONANT SIGN COENG CO;17D2 1787
|
||||
KHMER CONSONANT SIGN COENG CHO;17D2 1788
|
||||
KHMER CONSONANT SIGN COENG NYO;17D2 1789
|
||||
KHMER CONSONANT SIGN COENG DA;17D2 178A
|
||||
KHMER CONSONANT SIGN COENG TTHA;17D2 178B
|
||||
KHMER CONSONANT SIGN COENG DO;17D2 178C
|
||||
KHMER CONSONANT SIGN COENG TTHO;17D2 178D
|
||||
KHMER CONSONANT SIGN COENG NA;17D2 178E
|
||||
KHMER CONSONANT SIGN COENG TA;17D2 178F
|
||||
KHMER CONSONANT SIGN COENG THA;17D2 1790
|
||||
KHMER CONSONANT SIGN COENG TO;17D2 1791
|
||||
KHMER CONSONANT SIGN COENG THO;17D2 1792
|
||||
KHMER CONSONANT SIGN COENG NO;17D2 1793
|
||||
KHMER CONSONANT SIGN COENG BA;17D2 1794
|
||||
KHMER CONSONANT SIGN COENG PHA;17D2 1795
|
||||
KHMER CONSONANT SIGN COENG PO;17D2 1796
|
||||
KHMER CONSONANT SIGN COENG PHO;17D2 1797
|
||||
KHMER CONSONANT SIGN COENG MO;17D2 1798
|
||||
KHMER CONSONANT SIGN COENG YO;17D2 1799
|
||||
KHMER CONSONANT SIGN COENG RO;17D2 179A
|
||||
KHMER CONSONANT SIGN COENG LO;17D2 179B
|
||||
KHMER CONSONANT SIGN COENG VO;17D2 179C
|
||||
KHMER CONSONANT SIGN COENG SHA;17D2 179D
|
||||
KHMER CONSONANT SIGN COENG SSA;17D2 179E
|
||||
KHMER CONSONANT SIGN COENG SA;17D2 179F
|
||||
KHMER CONSONANT SIGN COENG HA;17D2 17A0
|
||||
KHMER CONSONANT SIGN COENG LA;17D2 17A1
|
||||
KHMER VOWEL SIGN COENG QA;17D2 17A2
|
||||
KHMER INDEPENDENT VOWEL SIGN COENG QU;17D2 17A7
|
||||
KHMER INDEPENDENT VOWEL SIGN COENG RY;17D2 17AB
|
||||
KHMER INDEPENDENT VOWEL SIGN COENG RYY;17D2 17AC
|
||||
KHMER INDEPENDENT VOWEL SIGN COENG QE;17D2 17AF
|
||||
KHMER VOWEL SIGN OM;17BB 17C6
|
||||
KHMER VOWEL SIGN AAM;17B6 17C6
|
||||
HIRAGANA LETTER BIDAKUON NGA;304B 309A
|
||||
HIRAGANA LETTER BIDAKUON NGI;304D 309A
|
||||
HIRAGANA LETTER BIDAKUON NGU;304F 309A
|
||||
HIRAGANA LETTER BIDAKUON NGE;3051 309A
|
||||
HIRAGANA LETTER BIDAKUON NGO;3053 309A
|
||||
KATAKANA LETTER BIDAKUON NGA;30AB 309A
|
||||
KATAKANA LETTER BIDAKUON NGI;30AD 309A
|
||||
KATAKANA LETTER BIDAKUON NGU;30AF 309A
|
||||
KATAKANA LETTER BIDAKUON NGE;30B1 309A
|
||||
KATAKANA LETTER BIDAKUON NGO;30B3 309A
|
||||
KATAKANA LETTER AINU CE;30BB 309A
|
||||
KATAKANA LETTER AINU TU;30C4 309A
|
||||
KATAKANA LETTER AINU TO;30C8 309A
|
||||
KATAKANA LETTER AINU P;31F7 309A
|
||||
MODIFIER LETTER EXTRA-HIGH EXTRA-LOW CONTOUR TONE BAR;02E5 02E9
|
||||
MODIFIER LETTER EXTRA-LOW EXTRA-HIGH CONTOUR TONE BAR;02E9 02E5
|
||||
@@ -10,9 +10,9 @@ defmodule String.Unicode do
|
||||
:unicode.characters_to_binary([binary_to_integer(codepoint, 16)])
|
||||
end
|
||||
|
||||
path = File.expand_path("../UnicodeData.txt", __FILE__)
|
||||
data_path = File.expand_path("../UnicodeData.txt", __FILE__)
|
||||
|
||||
codes = Enum.reduce File.iterator!(path), [], fn(line, acc) ->
|
||||
codes = Enum.reduce File.iterator!(data_path), [], fn(line, acc) ->
|
||||
[ codepoint, _name, _category,
|
||||
_class, _bidi, _decomposition,
|
||||
_numeric_1, _numeric_2, _numeric_3,
|
||||
@@ -26,6 +26,15 @@ defmodule String.Unicode do
|
||||
end
|
||||
end
|
||||
|
||||
seqs_path = File.expand_path("../NamedSequences.txt", __FILE__)
|
||||
|
||||
seqs = Enum.map(File.iterator!(seqs_path), fn(line) ->
|
||||
[ _name, codepoints ] = :binary.split(line, ";", [:global])
|
||||
codepoints = Enum.filter(:binary.split(codepoints, " ", [:global, :trim]),
|
||||
fn(x) -> size(x) > 0 end)
|
||||
Enum.map(codepoints, fn(x) -> to_binary.(x) end)
|
||||
end)
|
||||
|
||||
# Downcase
|
||||
|
||||
lc { codepoint, _upper, lower } inlist codes, lower != "" do
|
||||
@@ -59,4 +68,64 @@ defmodule String.Unicode do
|
||||
def upcase(<< >>) do
|
||||
<< >>
|
||||
end
|
||||
|
||||
# Sequences
|
||||
|
||||
lc codepoints inlist seqs do
|
||||
seq_args = quote do: [<< unquote_splicing(codepoints), t :: binary >>]
|
||||
seq_code = quote do: {<< unquote_splicing(codepoints) >>, t}
|
||||
def :sequence, seq_args, [], do: seq_code
|
||||
end
|
||||
|
||||
def sequence(binary) when is_binary(binary) and byte_size(binary) > 0 do
|
||||
case codepoint(binary) do
|
||||
:no_codepoint -> :no_sequence
|
||||
other -> other
|
||||
end
|
||||
end
|
||||
|
||||
def sequence(<< >>) do
|
||||
:no_sequence
|
||||
end
|
||||
|
||||
def sequences(binary) when is_binary(binary) and byte_size(binary) > 0 do
|
||||
case sequence(binary) do
|
||||
{ c, rest } -> [c|sequences(rest)]
|
||||
:no_sequence -> []
|
||||
end
|
||||
end
|
||||
|
||||
def sequences(<< >>) do
|
||||
[]
|
||||
end
|
||||
|
||||
|
||||
# Private implementation which returns the first codepoint
|
||||
# of any given utf8 string and the rest of it
|
||||
# If an empty string is given, :no_codepoint is returned.
|
||||
@doc false
|
||||
def codepoint(<<194, char, rest :: binary>>)
|
||||
when char in 161..191,
|
||||
do: { <<194, char>>, rest }
|
||||
|
||||
|
||||
def codepoint(<<first, char, rest :: binary>>)
|
||||
when first in 195..223 and char in 128..191,
|
||||
do: { <<first, char>>, rest }
|
||||
|
||||
|
||||
def codepoint(<<first, second, char, rest :: binary>>)
|
||||
when first == 224 and second in 160..191 and char in 128..191,
|
||||
do: { <<first, second, char>>, rest }
|
||||
|
||||
|
||||
def codepoint(<<first, second, char, rest :: binary>>)
|
||||
when first in 225..239 and second in 128..191 and char in 128..191,
|
||||
do: { <<first, second, char>>, rest }
|
||||
|
||||
|
||||
def codepoint(<<other, rest :: binary>>), do: { <<other>>, rest }
|
||||
|
||||
|
||||
def codepoint(<<>>), do: :no_codepoint
|
||||
end
|
||||
@@ -97,6 +97,16 @@ defmodule StringTest do
|
||||
assert String.codepoints("ϖͲϥЫݎߟΈټϘለДШव׆ש؇؊صلټܗݎޥޘ߉ऌ૫ሏᶆ℆ℙℱ ⅚Ⅷ↠∈⌘①ffi") == ["ϖ","Ͳ","ϥ","Ы","ݎ","ߟ","Έ","ټ","Ϙ","ለ","Д","Ш","व","׆","ש","؇","؊","ص","ل","ټ","ܗ","ݎ","ޥ","ޘ","߉","ऌ","૫","ሏ","ᶆ","℆","ℙ","ℱ"," ","⅚","Ⅷ","↠","∈","⌘","①","ffi"]
|
||||
end
|
||||
|
||||
test :graphemes do
|
||||
assert String.graphemes("Ā̀stute") == ["Ā̀","s","t","u","t","e"]
|
||||
end
|
||||
|
||||
|
||||
test :grapheme do
|
||||
assert String.grapheme("Ā̀stute") == {"Ā̀","stute"}
|
||||
assert String.grapheme("") == :no_sequence
|
||||
end
|
||||
|
||||
test :first do
|
||||
assert String.first("elixir") == "e"
|
||||
assert String.first("íelixr") == "í"
|
||||
@@ -105,6 +115,7 @@ defmodule StringTest do
|
||||
assert String.first("ελιξήριο") == "ε"
|
||||
assert String.first("סם חיים") == "ס"
|
||||
assert String.first("がガちゃ") == "が"
|
||||
assert String.first("Ā̀stute") == "Ā̀"
|
||||
assert String.first("") == ""
|
||||
end
|
||||
|
||||
@@ -117,6 +128,7 @@ defmodule StringTest do
|
||||
assert String.last("סם ייםח") == "ח"
|
||||
assert String.last("がガちゃ") == "ゃ"
|
||||
assert String.last("") == ""
|
||||
assert String.last("Ā̀") == "Ā̀"
|
||||
end
|
||||
|
||||
test :length do
|
||||
@@ -127,6 +139,7 @@ defmodule StringTest do
|
||||
assert String.length("ειξήριολ") == 8
|
||||
assert String.length("סם ייםח") == 7
|
||||
assert String.length("がガちゃ") == 4
|
||||
assert String.length("Ā̀stute") == 6
|
||||
assert String.length("") == 0
|
||||
end
|
||||
|
||||
@@ -138,6 +151,7 @@ defmodule StringTest do
|
||||
assert String.at("elixir", -1) == "r"
|
||||
assert String.at("がガちゃ", -2) == "ち"
|
||||
assert String.at("л", -3) == ""
|
||||
assert String.at("Ā̀stute", 1) == "s"
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user