From d9cb9c99f02eaa0fbbc2e128dda0a80b3700a7f2 Mon Sep 17 00:00:00 2001
From: vanstee <patrick@vanstee.me>
Date: Sat, 10 Aug 2013 15:32:36 -0400
Subject: [PATCH] Separate atomization from tokenization

---
 lib/elixir/src/elixir_tokenizer.erl | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl
index 7ace9a7336..9c7c507aed 100644
--- a/lib/elixir/src/elixir_tokenizer.erl
+++ b/lib/elixir/src/elixir_tokenizer.erl
@@ -306,7 +306,8 @@ tokenize([$:,H|T], Line, #scope{file=File} = Scope, Tokens) when ?is_quote(H) ->
   end;
 
 tokenize([$:,T|String], Line, Scope, Tokens) when ?is_atom_start(T) ->
-  { Rest, Atom } = tokenize_atom([T|String], [], Scope),
+  { Rest, Part } = tokenize_atom([T|String], []),
+  Atom = unsafe_to_atom(Part, Scope),
   tokenize(Rest, Line, Scope, [{ atom, Line, Atom }|Tokens]);
 
 % Atom identifiers/operators
@@ -738,14 +739,14 @@ tokenize_comment([])                 -> [].
 %% Atoms
 %% Handle atoms specially since they support @
 
-tokenize_atom([H|T], Acc, Scope) when ?is_atom(H) ->
-  tokenize_atom(T, [H|Acc], Scope);
+tokenize_atom([H|T], Acc) when ?is_atom(H) ->
+  tokenize_atom(T, [H|Acc]);
 
-tokenize_atom([H|T], Acc, Scope) when H == $?; H == $! ->
-  { T, unsafe_to_atom(lists:reverse([H|Acc]), Scope) };
+tokenize_atom([H|T], Acc) when H == $?; H == $! ->
+  { T, lists:reverse([H|Acc]) };
 
-tokenize_atom(Rest, Acc, Scope) ->
-  { Rest, unsafe_to_atom(lists:reverse(Acc), Scope) }.
+tokenize_atom(Rest, Acc) ->
+  { Rest, lists:reverse(Acc) }.
 
 %% Identifiers
 %% At this point, the validity of the first character was already verified.