From ce39f233885092bf16af12a5b52bee1019b22c83 Mon Sep 17 00:00:00 2001 From: Alexei Sholik Date: Tue, 7 May 2013 02:14:48 +0300 Subject: [PATCH] Add support for unicode to EEx.Tokenizer --- lib/eex/lib/eex/tokenizer.ex | 4 ++-- lib/eex/test/eex_test.exs | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex index cb967210f5..9e92567ee3 100644 --- a/lib/eex/lib/eex/tokenizer.ex +++ b/lib/eex/lib/eex/tokenizer.ex @@ -12,7 +12,7 @@ defmodule EEx.Tokenizer do """ def tokenize(bin, line) when is_binary(bin) do - tokenize(binary_to_list(bin), line) + tokenize(:unicode.characters_to_list(bin), line) end def tokenize(list, line) do @@ -154,6 +154,6 @@ defmodule EEx.Tokenizer do end defp tokenize_text(line, buffer, acc) do - [{ :text, line, list_to_binary(Enum.reverse(buffer)) } | acc] + [{ :text, line, :unicode.characters_to_binary(Enum.reverse(buffer)) } | acc] end end diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index 9d5af81e6f..ec9a379cc5 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -270,6 +270,15 @@ foo assert_eval expected, string end + test "unicode" do + template = """ + • <%= "•" %> • + <%= "Jößé Vâlìm" %> Jößé Vâlìm + """ + result = EEx.eval_string(template) + assert result == " • • •\n Jößé Vâlìm Jößé Vâlìm\n" + end + test "evaluates the source from a given file" do filename = Path.expand("../fixtures/eex_template.eex", __FILE__) result = EEx.eval_file(filename)