diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex index cb967210f5..9e92567ee3 100644 --- a/lib/eex/lib/eex/tokenizer.ex +++ b/lib/eex/lib/eex/tokenizer.ex @@ -12,7 +12,7 @@ defmodule EEx.Tokenizer do """ def tokenize(bin, line) when is_binary(bin) do - tokenize(binary_to_list(bin), line) + tokenize(:unicode.characters_to_list(bin), line) end def tokenize(list, line) do @@ -154,6 +154,6 @@ defmodule EEx.Tokenizer do end defp tokenize_text(line, buffer, acc) do - [{ :text, line, list_to_binary(Enum.reverse(buffer)) } | acc] + [{ :text, line, :unicode.characters_to_binary(Enum.reverse(buffer)) } | acc] end end diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index a9f7546892..e8a74ce815 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -270,6 +270,15 @@ foo assert_eval expected, string end + test "unicode" do + template = """ + • <%= "•" %> • + <%= "Jößé Vâlìm" %> Jößé Vâlìm + """ + result = EEx.eval_string(template) + assert result == " • • •\n Jößé Vâlìm Jößé Vâlìm\n" + end + test "evaluates the source from a given file" do filename = Path.join(__DIR__, "fixtures/eex_template.eex") result = EEx.eval_file(filename)