The AST now allows metadata to be attached to each node

This means the second item in the AST is no longer an integer
(representing the line), but a keywords list. Code that relies
on the line information from AST or that manually generate AST
nodes need to be properly updated.

lib/elixir/src/elixir_parser.yrl

@@ -23,7 +23,7 @@ Nonterminals
 Terminals
   identifier kw_identifier punctuated_identifier
   bracket_identifier paren_identifier do_identifier block_identifier
-  fn 'end' '__aliases__'
+  fn 'end' aliases
   number signed_number atom bin_string list_string sigil
   dot_call_op comp_op op_identifier
   'not' 'and' 'or' 'xor' 'when' 'in' 'inlist' 'inbits' 'do'
@@ -141,13 +141,13 @@ matched_op_expr -> colon_colon_op matched_expr : { '$1', '$2' }.
 matched_op_expr -> comp_expr_op matched_expr : { '$1', '$2' }.
 
 block_expr -> parens_call call_args_parens do_block : build_identifier('$1', '$2' ++ '$3').
-block_expr -> parens_call call_args_parens call_args_parens do_block : { build_identifier('$1', '$2'), ?line('$1'), '$3' ++ '$4' }.
+block_expr -> parens_call call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3' ++ '$4').
 block_expr -> dot_punctuated_identifier call_args_no_parens do_block : build_identifier('$1', '$2' ++ '$3').
 block_expr -> dot_do_identifier do_block : build_identifier('$1', '$2').
 block_expr -> dot_identifier call_args_no_parens do_block : build_identifier('$1', '$2' ++ '$3').
 
 fn_expr -> fn_eol stab_expr_list end_eol : build_fn('$1', build_stab(lists:reverse('$2'))).
-fn_expr -> fn_eol '->' grammar 'end' : build_fn('$1', { '->', ?line('$2'), [{ [], build_block('$3') }] }).
+fn_expr -> fn_eol '->' grammar 'end' : build_fn('$1', { '->', [{line,?line('$2')}], [{ [], build_block('$3') }] }).
 fn_expr -> call_expr : '$1'.
 
 call_expr -> dot_punctuated_identifier call_args_no_parens : build_identifier('$1', '$2').
@@ -160,7 +160,7 @@ call_expr -> max_expr : '$1'.
 
 max_expr -> bracket_expr : '$1'.
 max_expr -> parens_call call_args_parens : build_identifier('$1', '$2').
-max_expr -> parens_call call_args_parens call_args_parens : { build_identifier('$1', '$2'), ?line('$1'), '$3' }.
+max_expr -> parens_call call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3').
 max_expr -> dot_ref : '$1'.
 max_expr -> base_expr : '$1'.
 max_expr -> open_paren stab_expr_list close_paren : build_stab(lists:reverse('$2')).
@@ -173,19 +173,19 @@ bracket_at_expr -> at_op max_expr bracket_access : build_access(build_unary_op('
 bracket_at_expr -> bracket_at_expr bracket_access : build_access('$1', '$2').
 
 base_expr -> number : ?exprs('$1').
-base_expr -> signed_number : { element(4, '$1'), ?line('$1'), ?exprs('$1') }.
+base_expr -> signed_number : { element(4, '$1'), [{line,?line('$1')}], ?exprs('$1') }.
 base_expr -> atom : build_atom('$1').
 base_expr -> list : '$1'.
 base_expr -> tuple : '$1'.
 base_expr -> 'true' : ?op('$1').
 base_expr -> 'false' : ?op('$1').
 base_expr -> 'nil' : ?op('$1').
-base_expr -> '__aliases__' : '$1'.
+base_expr -> aliases : { '__aliases__', [{line,?line('$1')}], ?exprs('$1') }.
 base_expr -> bin_string  : build_bin_string('$1').
 base_expr -> list_string : build_list_string('$1').
 base_expr -> bit_string : '$1'.
-base_expr -> '&' : '$1'.
-base_expr -> '...' : { ?op('$1'), ?line('$1'), [] }.
+base_expr -> '&' : { '&', [{line,?line('$1')}], ?exprs('$1') }.
+base_expr -> '...' : { ?op('$1'), [{line,?line('$1')}], [] }.
 base_expr -> sigil : build_sigil('$1').
 
 %% Blocks
@@ -359,7 +359,7 @@ dot_op -> '.' eol : '$1'.
 dot_identifier -> identifier : '$1'.
 dot_identifier -> matched_expr dot_op identifier : build_dot('$2', '$1', '$3').
 
-dot_ref -> matched_expr dot_op '__aliases__' : build_dot_ref('$2', '$1', '$3').
+dot_ref -> matched_expr dot_op aliases : build_dot_ref('$2', '$1', '$3').
 
 dot_op_identifier -> op_identifier : '$1'.
 dot_op_identifier -> matched_expr dot_op op_identifier : build_dot('$2', '$1', '$3').
@@ -377,7 +377,7 @@ dot_punctuated_identifier -> punctuated_identifier : '$1'.
 dot_punctuated_identifier -> matched_expr dot_op punctuated_identifier : build_dot('$2', '$1', '$3').
 
 parens_call -> dot_paren_identifier : '$1'.
-parens_call -> matched_expr dot_call_op : { '.', ?line('$2'), ['$1'] }. % Fun/local calls
+parens_call -> matched_expr dot_call_op : { '.', [{line,?line('$2')}], ['$1'] }. % Fun/local calls
 
 % Function calls
 
@@ -447,8 +447,8 @@ tuple -> open_curly paren_expr ',' call_args close_curly :  build_tuple('$1', ['
 
 % Bitstrings
 
-bit_string -> open_bit '>>' : { '<<>>', ?line('$1'), [] }.
-bit_string -> open_bit call_args close_bit : { '<<>>', ?line('$1'), '$2' }.
+bit_string -> open_bit '>>' : { '<<>>', [{line,?line('$1')}], [] }.
+bit_string -> open_bit call_args close_bit : { '<<>>', [{line,?line('$1')}], '$2' }.
 
 Erlang code.
 
@@ -466,58 +466,62 @@ Erlang code.
 %% Operators
 
 build_op({ _, _, _ } = Op, Left, Right) ->
-  { ?exprs(Op), ?line(Op), [Left, Right] };
+  { ?exprs(Op), [{line,?line(Op)}], [Left, Right] };
 
 build_op({ BOp, Line }, { UOp, _, [Left] }, Right) when ?rearrange_bop(BOp), ?rearrange_uop(UOp) ->
-  { UOp, Line, [{ BOp, Line, [Left, Right] }] };
+  { UOp, [{line,Line}], [{ BOp, [{line,Line}], [Left, Right] }] };
 
 build_op(Op, Left, Right) ->
-  { ?op(Op), ?line(Op), [Left, Right] }.
+  { ?op(Op), [{line,?line(Op)}], [Left, Right] }.
 
 build_unary_op(Op, Expr) ->
-  { ?op(Op), ?line(Op), [Expr] }.
+  { ?op(Op), [{line,?line(Op)}], [Expr] }.
 
 build_tuple(_Marker, [Left, Right]) ->
   { Left, Right };
 
 build_tuple(Marker, Args) ->
-  { '{}', ?line(Marker), Args }.
+  { '{}', [{line,?line(Marker)}], Args }.
 
 %% Blocks
 
-build_block([nil])                                      -> { '__block__', 0, [nil] };
-build_block([{Op,_,[_]}]=Exprs) when ?rearrange_uop(Op) -> { '__block__', 0, Exprs };
+build_block([nil])                                      -> { '__block__', [], [nil] };
+build_block([{Op,_,[_]}]=Exprs) when ?rearrange_uop(Op) -> { '__block__', [], Exprs };
 build_block([Expr]) when not is_list(Expr)              -> Expr;
-build_block(Exprs)                                      -> { '__block__', 0, Exprs }.
+build_block(Exprs)                                      -> { '__block__', [], Exprs }.
 
 %% Dots
 
-build_dot_ref(Dot, { '__aliases__', _, Left }, { '__aliases__', _, Right }) ->
-  { '__aliases__', ?line(Dot), Left ++ Right };
+build_dot_ref(Dot, { '__aliases__', _, Left }, { 'aliases', _, Right }) ->
+  { '__aliases__', [{line,?line(Dot)}], Left ++ Right };
 
-build_dot_ref(Dot, Other, { '__aliases__', _, Right }) ->
-  { '__aliases__', ?line(Dot), [Other|Right] }.
+build_dot_ref(Dot, Other, { 'aliases', _, Right }) ->
+  { '__aliases__', [{line,?line(Dot)}], [Other|Right] }.
 
 build_dot(Dot, Left, Right) ->
-  { '.', ?line(Dot), [Left, extract_identifier(Right)] }.
+  { '.', [{line,?line(Dot)}], [Left, extract_identifier(Right)] }.
 
 %% Identifiers
 
-build_identifier({ '.', Line, _ } = Dot, Args) ->
+build_nested_parens(Dot, Args1, Args2) ->
+  Identifier = build_identifier(Dot, Args1),
+  { Identifier, ?line(Identifier), Args2 }.
+
+build_identifier({ '.', Meta, _ } = Dot, Args) ->
   FArgs = case Args of
     nil -> [];
     _ -> Args
   end,
-  { Dot, Line, FArgs };
+  { Dot, Meta, FArgs };
 
 build_identifier({ Keyword, Line }, Args) when Keyword == fn ->
-  { fn, Line, Args };
+  { fn, [{line,Line}], Args };
 
 build_identifier({ op_identifier, Line, Identifier }, Args) ->
-  { '__ambiguousop__', Line, [{ Identifier, Line, nil }|Args] };
+  { '__ambiguousop__', [{line,Line}], [{ Identifier, [{line,Line}], nil }|Args] };
 
 build_identifier({ _, Line, Identifier }, Args) ->
-  { Identifier, Line, Args }.
+  { Identifier, [{line,Line}], Args }.
 
 extract_identifier({ Kind, _, Identifier }) when
     Kind == identifier; Kind == punctuated_identifier; Kind == bracket_identifier;
@@ -529,32 +533,37 @@ extract_identifier(Other) -> Other.
 %% Fn
 
 build_fn(Op, Stab) ->
-  { fn, ?line(Op), [[{ do, Stab }]] }.
+  { fn, [{line,?line(Op)}], [[{ do, Stab }]] }.
 
 %% Access
 
 build_access(Expr, Access) ->
-  Line = ?line(Access),
-  { { '.', Line, ['Elixir-Kernel', access] }, ?line(Access), [ Expr, ?op(Access) ] }.
+  Meta = [{line,?line(Access)}],
+  { { '.', Meta, ['Elixir-Kernel', access] }, Meta, [ Expr, ?op(Access) ] }.
 
 %% Interpolation aware
 
 build_sigil({ sigil, Line, Sigil, Parts, Modifiers }) ->
-  { list_to_atom([$_,$_,Sigil,$_,$_]), Line, [ { '<<>>', Line, Parts }, Modifiers ] }.
+  Meta = [{line,Line}],
+  { list_to_atom([$_,$_,Sigil,$_,$_]), Meta, [ { '<<>>', Meta, Parts }, Modifiers ] }.
 
 build_bin_string({ bin_string, _Line, [H] }) when is_binary(H) -> H;
-build_bin_string({ bin_string, Line, Args }) -> { '<<>>', Line, Args }.
+build_bin_string({ bin_string, Line, Args }) -> { '<<>>', [{line,Line}], Args }.
 
 build_list_string({ list_string, _Line, [H] }) when is_binary(H) -> binary_to_list(H);
-build_list_string({ list_string, Line, Args }) -> { { '.', Line, [erlang, binary_to_list] }, Line, [{ '<<>>', Line, Args}] }.
+build_list_string({ list_string, Line, Args }) ->
+  Meta = [{line,Line}],
+  { { '.', Meta, [erlang, binary_to_list] }, Meta, [{ '<<>>', Meta, Args}] }.
 
 build_atom({ atom, _Line, Atom }) when is_atom(Atom) -> Atom;
-build_atom({ atom, Line, Args }) -> { { '.', Line, [erlang, binary_to_atom] }, Line, [{ '<<>>', Line, Args }, utf8] }.
+build_atom({ atom, Line, Args }) ->
+  Meta = [{line,Line}],
+  { { '.', Meta, [erlang, binary_to_atom] }, Meta, [{ '<<>>', Meta, Args }, utf8] }.
 
 %% Keywords
 
-build_stab([{ '->', Line, [Left, Right] }|T]) ->
-  { '->', Line, build_stab(T, Left, [Right], []) };
+build_stab([{ '->', Meta, [Left, Right] }|T]) ->
+  { '->', Meta, build_stab(T, Left, [Right], []) };
 
 build_stab(Else) ->
   build_block(Else).