From d9cb9c99f02eaa0fbbc2e128dda0a80b3700a7f2 Mon Sep 17 00:00:00 2001
From: vanstee <patrick@vanstee.me>
Date: Sat, 10 Aug 2013 15:32:36 -0400
Subject: [PATCH 1/3] Separate atomization from tokenization

---
 lib/elixir/src/elixir_tokenizer.erl | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl
index 7ace9a7336..9c7c507aed 100644
--- a/lib/elixir/src/elixir_tokenizer.erl
+++ b/lib/elixir/src/elixir_tokenizer.erl
@@ -306,7 +306,8 @@ tokenize([$:,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   end;
 
 tokenize([$:,T|String], Line, Scope, Tokens) when ?is_atom_start(T) ->
-  { Rest, Atom } = tokenize_atom([T|String], [], Scope),
+  { Rest, Part } = tokenize_atom([T|String], []),
+  Atom = unsafe_to_atom(Part, Scope),
   tokenize(Rest, Line, Scope, [{ atom, Line, Atom }|Tokens]);
 
 % Atom identifiers/operators
@@ -738,14 +739,14 @@ tokenize_comment([])                 -> [].
 %% Atoms
 %% Handle atoms specially since they support @
 
-tokenize_atom([H|T], Acc, Scope) when ?is_atom(H) ->
-  tokenize_atom(T, [H|Acc], Scope);
+tokenize_atom([H|T], Acc) when ?is_atom(H) ->
+  tokenize_atom(T, [H|Acc]);
 
-tokenize_atom([H|T], Acc, Scope) when H == $?; H == $! ->
-  { T, unsafe_to_atom(lists:reverse([H|Acc]), Scope) };
+tokenize_atom([H|T], Acc) when H == $?; H == $! ->
+  { T, lists:reverse([H|Acc]) };
 
-tokenize_atom(Rest, Acc, Scope) ->
-  { Rest, unsafe_to_atom(lists:reverse(Acc), Scope) }.
+tokenize_atom(Rest, Acc) ->
+  { Rest, lists:reverse(Acc) }.
 
 %% Identifiers
 %% At this point, the validity of the first character was already verified.

From 5371101ad04924eb15de8b823630282916d9eefa Mon Sep 17 00:00:00 2001
From: vanstee <patrick@vanstee.me>
Date: Sat, 10 Aug 2013 15:47:03 -0400
Subject: [PATCH 2/3] Include line number in atomization

---
 lib/elixir/src/elixir_tokenizer.erl | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl
index 9c7c507aed..5b9cd55b5e 100644
--- a/lib/elixir/src/elixir_tokenizer.erl
+++ b/lib/elixir/src/elixir_tokenizer.erl
@@ -269,7 +269,7 @@ tokenize([$.,$(|Rest], Line, Scope, Tokens) ->
 tokenize([$.,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   case elixir_interpolation:extract(Line, File, true, T, H) of
     { NewLine, [Part], Rest } when is_binary(Part) ->
-      Atom  = unsafe_to_atom(Part, Scope),
+      Atom  = unsafe_to_atom(Part, Line, Scope),
       Token = check_call_identifier(identifier, Line, Atom, Rest),
       tokenize(Rest, NewLine, Scope, [Token|add_token_with_nl({ '.', Line }, Tokens)]);
     Error ->
@@ -297,7 +297,7 @@ tokenize([$:,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   case elixir_interpolation:extract(Line, File, true, T, H) of
     { NewLine, Parts, Rest } ->
       Token = case unescape_tokens(Parts) of
-        [Part] when is_list(Part) or is_binary(Part) -> { atom, Line, unsafe_to_atom(Part, Scope) };
+        [Part] when is_list(Part) or is_binary(Part) -> { atom, Line, unsafe_to_atom(Part, Line, Scope) };
         Unescaped -> { atom, Line, Unescaped }
       end,
       tokenize(Rest, NewLine, Scope, [Token|Tokens]);
@@ -307,7 +307,7 @@ tokenize([$:,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
 
 tokenize([$:,T|String], Line, Scope, Tokens) when ?is_atom_start(T) ->
   { Rest, Part } = tokenize_atom([T|String], []),
-  Atom = unsafe_to_atom(Part, Scope),
+  Atom = unsafe_to_atom(Part, Line, Scope),
   tokenize(Rest, Line, Scope, [{ atom, Line, Atom }|Tokens]);
 
 % Atom identifiers/operators
@@ -463,7 +463,7 @@ tokenize([H|_] = String, Line, Scope, Tokens) when ?is_digit(H) ->
 
 tokenize([H|_] = String, Line, Scope, Tokens) when ?is_upcase(H) ->
   { Rest, Alias } = tokenize_identifier(String, []),
-  Atom = unsafe_to_atom(Alias, Scope),
+  Atom = unsafe_to_atom(Alias, Line, Scope),
   case Rest of
     [$:|T] when ?is_space(hd(T)) ->
       tokenize(T, Line, Scope, [{ kw_identifier, Line, Atom }|Tokens]);
@@ -526,7 +526,7 @@ handle_strings(T, Line, H, #scope{file=File} = Scope, Tokens) ->
     { NewLine, Parts, [$:|Rest] } when ?is_space(hd(Rest)) ->
       case Parts of
         [Bin] when is_binary(Bin) ->
-          Atom = unsafe_to_atom(unescape_chars(Bin), Scope),
+          Atom = unsafe_to_atom(unescape_chars(Bin), Line, Scope),
           tokenize(Rest, NewLine, Scope, [{ kw_identifier, Line, Atom }|Tokens]);
         _ ->
           { error, { Line, "invalid interpolation in key ", [$"|T] } }
@@ -560,13 +560,13 @@ eol(_Line, _Mod, [{',',_}|_] = Tokens)   -> Tokens;
 eol(_Line, _Mod, [{eol,_,_}|_] = Tokens) -> Tokens;
 eol(Line, Mod, Tokens) -> [{eol,Line,Mod}|Tokens].
 
-unsafe_to_atom(Binary, #scope{existing_atoms_only=true}) when is_binary(Binary) ->
+unsafe_to_atom(Binary, _Line, #scope{existing_atoms_only=true}) when is_binary(Binary) ->
   binary_to_existing_atom(Binary, utf8);
-unsafe_to_atom(Binary, #scope{}) when is_binary(Binary) ->
+unsafe_to_atom(Binary, _Line, #scope{}) when is_binary(Binary) ->
   binary_to_atom(Binary, utf8);
-unsafe_to_atom(List, #scope{existing_atoms_only=true}) when is_list(List) ->
+unsafe_to_atom(List, _Line, #scope{existing_atoms_only=true}) when is_list(List) ->
   list_to_existing_atom(List);
-unsafe_to_atom(List, #scope{}) when is_list(List) ->
+unsafe_to_atom(List, _Line, #scope{}) when is_list(List) ->
   list_to_atom(List).
 
 collect_modifiers([H|T], Buffer) when ?is_downcase(H) ->

From ad67e4935a93c1b9a51c586c000e9edc4fe059f9 Mon Sep 17 00:00:00 2001
From: vanstee <patrick@vanstee.me>
Date: Sat, 10 Aug 2013 16:49:26 -0400
Subject: [PATCH 3/3] Return error when tokenizing oversized atom

When "atomizing" an unsafe string, we need to make sure the atom length
is going to be less than the system limit (255). Any function calling
`unsafe_to_atom` checks for an error and returns it if present.
---
 lib/elixir/src/elixir_tokenizer.erl       | 75 ++++++++++++++++-------
 lib/elixir/test/erlang/tokenizer_test.erl | 11 ++++
 2 files changed, 65 insertions(+), 21 deletions(-)

diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl
index 5b9cd55b5e..50a69d8bfc 100644
--- a/lib/elixir/src/elixir_tokenizer.erl
+++ b/lib/elixir/src/elixir_tokenizer.erl
@@ -269,9 +269,13 @@ tokenize([$.,$(|Rest], Line, Scope, Tokens) ->
 tokenize([$.,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   case elixir_interpolation:extract(Line, File, true, T, H) of
     { NewLine, [Part], Rest } when is_binary(Part) ->
-      Atom  = unsafe_to_atom(Part, Line, Scope),
-      Token = check_call_identifier(identifier, Line, Atom, Rest),
-      tokenize(Rest, NewLine, Scope, [Token|add_token_with_nl({ '.', Line }, Tokens)]);
+      case unsafe_to_atom(Part, Line, Scope) of
+        { error, _ } = Error ->
+          Error;
+        Atom ->
+          Token = check_call_identifier(identifier, Line, Atom, Rest),
+          tokenize(Rest, NewLine, Scope, [Token|add_token_with_nl({ '.', Line }, Tokens)])
+      end;
     Error ->
       interpolation_error(Error, " (for function name starting at line ~B)", [Line])
   end;
@@ -296,19 +300,28 @@ tokenize([$'|T], Line, Scope, Tokens) ->
 tokenize([$:,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   case elixir_interpolation:extract(Line, File, true, T, H) of
     { NewLine, Parts, Rest } ->
-      Token = case unescape_tokens(Parts) of
-        [Part] when is_list(Part) or is_binary(Part) -> { atom, Line, unsafe_to_atom(Part, Line, Scope) };
-        Unescaped -> { atom, Line, Unescaped }
+      SafeAtom = case unescape_tokens(Parts) of
+        [Part] when is_list(Part) or is_binary(Part) -> unsafe_to_atom(Part, Line, Scope);
+        Unescaped -> Unescaped
       end,
-      tokenize(Rest, NewLine, Scope, [Token|Tokens]);
+      case SafeAtom of
+        { error, _ } = Error ->
+          Error;
+        Atom ->
+          tokenize(Rest, NewLine, Scope, [{ atom, Line, Atom }|Tokens])
+      end;
     Error ->
       interpolation_error(Error, " (for atom starting at line ~B)", [Line])
   end;
 
 tokenize([$:,T|String], Line, Scope, Tokens) when ?is_atom_start(T) ->
   { Rest, Part } = tokenize_atom([T|String], []),
-  Atom = unsafe_to_atom(Part, Line, Scope),
-  tokenize(Rest, Line, Scope, [{ atom, Line, Atom }|Tokens]);
+  case unsafe_to_atom(Part, Line, Scope) of
+    { error, _ } = Error ->
+      Error;
+    Atom ->
+      tokenize(Rest, Line, Scope, [{ atom, Line, Atom }|Tokens])
+  end;
 
 % Atom identifiers/operators
 
@@ -463,12 +476,16 @@ tokenize([H|_] = String, Line, Scope, Tokens) when ?is_digit(H) ->
 
 tokenize([H|_] = String, Line, Scope, Tokens) when ?is_upcase(H) ->
   { Rest, Alias } = tokenize_identifier(String, []),
-  Atom = unsafe_to_atom(Alias, Line, Scope),
-  case Rest of
-    [$:|T] when ?is_space(hd(T)) ->
-      tokenize(T, Line, Scope, [{ kw_identifier, Line, Atom }|Tokens]);
-    _ ->
-      tokenize(Rest, Line, Scope, [{ aliases, Line, [Atom] }|Tokens])
+  case unsafe_to_atom(Alias, Line, Scope) of
+    { error, _ } = Error ->
+      Error;
+    Atom ->
+      case Rest of
+        [$:|T] when ?is_space(hd(T)) ->
+          tokenize(T, Line, Scope, [{ kw_identifier, Line, Atom }|Tokens]);
+        _ ->
+          tokenize(Rest, Line, Scope, [{ aliases, Line, [Atom] }|Tokens])
+      end
   end;
 
 % Identifier
@@ -526,8 +543,12 @@ handle_strings(T, Line, H, #scope{file=File} = Scope, Tokens) ->
     { NewLine, Parts, [$:|Rest] } when ?is_space(hd(Rest)) ->
       case Parts of
         [Bin] when is_binary(Bin) ->
-          Atom = unsafe_to_atom(unescape_chars(Bin), Line, Scope),
-          tokenize(Rest, NewLine, Scope, [{ kw_identifier, Line, Atom }|Tokens]);
+          case unsafe_to_atom(unescape_chars(Bin), Line, Scope) of
+            { error, _ } = Error ->
+              Error;
+            Atom ->
+              tokenize(Rest, NewLine, Scope, [{ kw_identifier, Line, Atom }|Tokens])
+          end;
         _ ->
           { error, { Line, "invalid interpolation in key ", [$"|T] } }
       end;
@@ -560,6 +581,10 @@ eol(_Line, _Mod, [{',',_}|_] = Tokens)   -> Tokens;
 eol(_Line, _Mod, [{eol,_,_}|_] = Tokens) -> Tokens;
 eol(Line, Mod, Tokens) -> [{eol,Line,Mod}|Tokens].
 
+unsafe_to_atom(Part, Line, #scope{}) when
+    is_binary(Part) andalso size(Part) > 255;
+    is_list(Part) andalso length(Part) > 255 ->
+  { error, { Line, "atom length must be less than system limit", ":" } };
 unsafe_to_atom(Binary, _Line, #scope{existing_atoms_only=true}) when is_binary(Binary) ->
   binary_to_existing_atom(Binary, utf8);
 unsafe_to_atom(Binary, _Line, #scope{}) when is_binary(Binary) ->
@@ -764,11 +789,19 @@ tokenize_any_identifier(String, Line, Scope, Tokens) ->
 
   case Rest of
     [H|T] when H == $?; H == $! ->
-      Atom = unsafe_to_atom(Identifier ++ [H], Scope),
-      tokenize_kw_or_other(T, punctuated_identifier, Line, Atom, Tokens);
+      case unsafe_to_atom(Identifier ++ [H], Line, Scope) of
+        { error, _ } = Error ->
+          Error;
+        Atom ->
+          tokenize_kw_or_other(T, punctuated_identifier, Line, Atom, Tokens)
+      end;
     _ ->
-      Atom = unsafe_to_atom(Identifier, Scope),
-      tokenize_kw_or_other(Rest, identifier, Line, Atom, Tokens)
+      case unsafe_to_atom(Identifier, Line, Scope) of
+        { error, _ } = Error ->
+          Error;
+        Atom ->
+          tokenize_kw_or_other(Rest, identifier, Line, Atom, Tokens)
+      end
   end.
 
 tokenize_kw_or_other([$:,H|T], _Kind, Line, Atom, _Tokens) when ?is_space(H) ->
diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl
index 0c103fa265..539daeff53 100644
--- a/lib/elixir/test/erlang/tokenizer_test.erl
+++ b/lib/elixir/test/erlang/tokenizer_test.erl
@@ -6,6 +6,10 @@ tokenize(String) ->
   { ok, Result } = elixir_tokenizer:tokenize(String, 1, []),
   Result.
 
+tokenize_error(String) ->
+  { error, Error } = elixir_tokenizer:tokenize(String, 1, []),
+  Error.
+
 type_test() ->
   [{number,1,1},{type_op,1,'::'},{number,1,3}] = tokenize("1 :: 3"),
   [{identifier,1,foo},
@@ -40,6 +44,13 @@ unquoted_atom_test() ->
   [{atom, 1, '='}] = tokenize(":="),
   [{atom, 1, '&&'}] = tokenize(":&&").
 
+quoted_atom_test() ->
+  [{atom, 1, 'foo bar'}] = tokenize(":\"foo bar\"").
+
+oversized_atom_test() ->
+  OversizedAtom = ":\"" ++ string:copies("a", 256) ++ "\"",
+  { 1, "atom length must be less than system limit", ":" } = tokenize_error(OversizedAtom).
+
 op_atom_test() ->
   [{atom,1,f0_1}] = tokenize(":f0_1").