Merge pull request #1023 from alco/eex-unicode

Add support for unicode to EEx.Tokenizer

lib/eex/lib/eex/tokenizer.ex

@@ -12,7 +12,7 @@ defmodule EEx.Tokenizer do
 
   """
   def tokenize(bin, line) when is_binary(bin) do
-    tokenize(binary_to_list(bin), line)
+    tokenize(:unicode.characters_to_list(bin), line)
   end
 
   def tokenize(list, line) do
@@ -154,6 +154,6 @@ defmodule EEx.Tokenizer do
   end
 
   defp tokenize_text(line, buffer, acc) do
-    [{ :text, line, list_to_binary(Enum.reverse(buffer)) } | acc]
+    [{ :text, line, :unicode.characters_to_binary(Enum.reverse(buffer)) } | acc]
   end
 end

lib/eex/test/eex_test.exs

@@ -270,6 +270,15 @@ foo
     assert_eval expected, string
   end
 
+  test "unicode" do
+    template = """
+      • <%= "•" %> •
+      <%= "Jößé Vâlìm" %> Jößé Vâlìm
+    """
+    result = EEx.eval_string(template)
+    assert result == "  • • •\n  Jößé Vâlìm Jößé Vâlìm\n"
+  end
+
   test "evaluates the source from a given file" do
     filename = Path.join(__DIR__, "fixtures/eex_template.eex")
     result = EEx.eval_file(filename)