diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index aec5407f60..91aee577ee 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -2,10 +2,9 @@ Nonterminals grammar expr_list expr paren_expr block_expr fn_expr bracket_expr call_expr bracket_at_expr max_expr base_expr matched_expr matched_op_expr unmatched_expr op_expr - match_op tail_op colon_colon_op - comp_op_eol at_op_eol unary_op_eol and_op_eol or_op_eol - add_op_eol mult_op_eol exp_op_eol two_op_eol - arrow_op_eol range_op_eol than_op_eol default_op_eol + comp_op_eol at_op_eol unary_op_eol and_op_eol or_op_eol tail_op_eol + add_op_eol mult_op_eol exp_op_eol two_op_eol type_op_eol stab_op_eol + arrow_op_eol range_op_eol than_op_eol default_op_eol match_op_eol when_op_eol in_op_eol inc_op_eol open_paren close_paren empty_paren open_bracket close_bracket @@ -14,7 +13,7 @@ Nonterminals base_comma_expr comma_expr optional_comma_expr matched_comma_expr call_args call_args_parens parens_call call_args_no_parens call_args_no_parens_strict call_args_parens_not_one - stab stab_eol stab_op stab_expr stab_maybe_expr + stab stab_eol stab_expr stab_maybe_expr kw_eol kw_expr kw_comma kw_base matched_kw_expr matched_kw_comma matched_kw_base dot_op dot_ref dot_identifier dot_op_identifier dot_do_identifier @@ -29,26 +28,24 @@ Terminals fn 'end' aliases number signed_number atom bin_string list_string sigil dot_call_op op_identifier - comp_op at_op unary_op and_op or_op arrow_op - range_op in_op inc_op when_op than_op default_op - dual_op add_op mult_op exp_op two_op - '=' '::' '|' '->' - 'true' 'false' 'nil' 'do' eol ',' '.' '&' + comp_op at_op unary_op and_op or_op arrow_op match_op + range_op in_op inc_op when_op than_op default_op tail_op + dual_op add_op mult_op exp_op two_op type_op stab_op + 'true' 'false' 'nil' 'do' eol ',' '.' '&' '...' '(' ')' '[' ']' '{' '}' '<<' '>>' - '...' . Rootsymbol grammar. Left 5 do. -Right 10 stab_op. +Right 10 stab_op_eol. %% -> Left 20 ','. -Right 30 colon_colon_op. +Right 30 type_op_eol. %% :: Right 40 when_op_eol. %% when -Right 50 default_op_eol. %% // -Left 60 tail_op. -Left 70 inc_op_eol. %% inlist, inbits -Right 80 match_op. +Left 50 inc_op_eol. %% inlist, inbits +Right 60 default_op_eol. %% // +Left 70 tail_op_eol. %% | +Right 80 match_op_eol. %% = Left 130 or_op_eol. %% ||, |||, or, xor Left 140 and_op_eol. %% &&, &&&, and Left 150 comp_op_eol. %% <, >, <=, >=, ==, !=, =~, ===, !== @@ -98,39 +95,39 @@ unmatched_expr -> unary_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> at_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> block_expr : '$1'. -op_expr -> match_op expr : { '$1', '$2' }. +op_expr -> match_op_eol expr : { '$1', '$2' }. op_expr -> add_op_eol expr : { '$1', '$2' }. op_expr -> mult_op_eol expr : { '$1', '$2' }. op_expr -> exp_op_eol expr : { '$1', '$2' }. op_expr -> two_op_eol expr : { '$1', '$2' }. op_expr -> and_op_eol expr : { '$1', '$2' }. op_expr -> or_op_eol expr : { '$1', '$2' }. -op_expr -> tail_op expr : { '$1', '$2' }. +op_expr -> tail_op_eol expr : { '$1', '$2' }. op_expr -> than_op_eol expr : { '$1', '$2' }. op_expr -> in_op_eol expr : { '$1', '$2' }. op_expr -> inc_op_eol expr : { '$1', '$2' }. op_expr -> when_op_eol expr : { '$1', '$2' }. op_expr -> range_op_eol expr : { '$1', '$2' }. op_expr -> default_op_eol expr : { '$1', '$2' }. -op_expr -> colon_colon_op expr : { '$1', '$2' }. +op_expr -> type_op_eol expr : { '$1', '$2' }. op_expr -> comp_op_eol expr : { '$1', '$2' }. op_expr -> arrow_op_eol expr : { '$1', '$2' }. -matched_op_expr -> match_op matched_expr : { '$1', '$2' }. +matched_op_expr -> match_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> add_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> mult_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> exp_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> two_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> and_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> or_op_eol matched_expr : { '$1', '$2' }. -matched_op_expr -> tail_op matched_expr : { '$1', '$2' }. +matched_op_expr -> tail_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> than_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> in_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> inc_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> when_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> range_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> default_op_eol matched_expr : { '$1', '$2' }. -matched_op_expr -> colon_colon_op matched_expr : { '$1', '$2' }. +matched_op_expr -> type_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> comp_op_eol matched_expr : { '$1', '$2' }. matched_op_expr -> arrow_op_eol matched_expr : { '$1', '$2' }. @@ -173,15 +170,15 @@ base_expr -> signed_number : { element(4, '$1'), [{line,?line('$1')}], ?exprs('$ base_expr -> atom : build_atom('$1'). base_expr -> list : '$1'. base_expr -> tuple : '$1'. -base_expr -> 'true' : ?op('$1'). -base_expr -> 'false' : ?op('$1'). -base_expr -> 'nil' : ?op('$1'). +base_expr -> 'true' : ?id('$1'). +base_expr -> 'false' : ?id('$1'). +base_expr -> 'nil' : ?id('$1'). base_expr -> aliases : { '__aliases__', [{line,?line('$1')}], ?exprs('$1') }. base_expr -> bin_string : build_bin_string('$1'). base_expr -> list_string : build_list_string('$1'). base_expr -> bit_string : '$1'. base_expr -> '&' : { '&', [{line,?line('$1')}], ?exprs('$1') }. -base_expr -> '...' : { ?op('$1'), [{line,?line('$1')}], [] }. +base_expr -> '...' : { ?id('$1'), [{line,?line('$1')}], [] }. base_expr -> sigil : build_sigil('$1'). %% Blocks @@ -210,12 +207,12 @@ stab_eol -> stab : '$1'. stab_eol -> stab eol : '$1'. stab_expr -> expr : '$1'. -stab_expr -> stab_op stab_maybe_expr : build_op('$1', [], '$2'). -stab_expr -> call_args_no_parens stab_op stab_maybe_expr : +stab_expr -> stab_op_eol stab_maybe_expr : build_op('$1', [], '$2'). +stab_expr -> call_args_no_parens stab_op_eol stab_maybe_expr : build_op('$2', unwrap_when(unwrap_splice('$1')), '$3'). -stab_expr -> call_args_parens_not_one stab_op stab_maybe_expr : +stab_expr -> call_args_parens_not_one stab_op_eol stab_maybe_expr : build_op('$2', unwrap_splice('$1'), '$3'). -stab_expr -> call_args_parens_not_one when_op matched_expr stab_op stab_maybe_expr : +stab_expr -> call_args_parens_not_one when_op matched_expr stab_op_eol stab_maybe_expr : build_op('$4', [{ 'when', [{line,?line('$2')}], unwrap_splice('$1') ++ ['$3'] }], '$5'). stab_maybe_expr -> 'expr' : '$1'. @@ -272,16 +269,16 @@ two_op_eol -> two_op eol : '$1'. default_op_eol -> default_op : '$1'. default_op_eol -> default_op eol : '$1'. -colon_colon_op -> '::' : '$1'. -colon_colon_op -> '::' eol : '$1'. +type_op_eol -> type_op : '$1'. +type_op_eol -> type_op eol : '$1'. unary_op_eol -> unary_op : '$1'. unary_op_eol -> unary_op eol : '$1'. unary_op_eol -> dual_op : '$1'. unary_op_eol -> dual_op eol : '$1'. -match_op -> '=' : '$1'. -match_op -> '=' eol : '$1'. +match_op_eol -> match_op : '$1'. +match_op_eol -> match_op eol : '$1'. and_op_eol -> and_op : '$1'. and_op_eol -> and_op eol : '$1'. @@ -289,8 +286,8 @@ and_op_eol -> and_op eol : '$1'. or_op_eol -> or_op : '$1'. or_op_eol -> or_op eol : '$1'. -tail_op -> '|' : '$1'. -tail_op -> '|' eol : '$1'. +tail_op_eol -> tail_op : '$1'. +tail_op_eol -> tail_op eol : '$1'. than_op_eol -> than_op : '$1'. than_op_eol -> than_op eol : '$1'. @@ -304,8 +301,8 @@ inc_op_eol -> inc_op eol : '$1'. when_op_eol -> when_op : '$1'. when_op_eol -> when_op eol : '$1'. -stab_op -> '->' : '$1'. -stab_op -> '->' eol : '$1'. +stab_op_eol -> stab_op : '$1'. +stab_op_eol -> stab_op eol : '$1'. range_op_eol -> range_op : '$1'. range_op_eol -> range_op eol : '$1'. @@ -425,7 +422,7 @@ bit_string -> open_bit call_args close_bit : { '<<>>', [{line,?line('$1')}], '$2 Erlang code. --define(op(Node), element(1, Node)). +-define(id(Node), element(1, Node)). -define(line(Node), element(2, Node)). -define(exprs(Node), element(3, Node)). @@ -438,20 +435,14 @@ Erlang code. %% Operators -build_op({ _, Line, Op }, Left, Right) -> - build_op({ Op, Line }, Left, Right); - -build_op({ BOp, Line }, { UOp, _, [Left] }, Right) when ?rearrange_bop(BOp), ?rearrange_uop(UOp) -> +build_op({ _Kind, Line, BOp }, { UOp, _, [Left] }, Right) when ?rearrange_bop(BOp), ?rearrange_uop(UOp) -> { UOp, [{line,Line}], [{ BOp, [{line,Line}], [Left, Right] }] }; -build_op(Op, Left, Right) -> - { ?op(Op), [{line,?line(Op)}], [Left, Right] }. +build_op({ _Kind, Line, Op }, Left, Right) -> + { Op, [{line,Line}], [Left, Right] }. build_unary_op({ _Kind, Line, Op }, Expr) -> - { Op, [{line,Line}], [Expr] }; - -build_unary_op(Op, Expr) -> - { ?op(Op), [{line,?line(Op)}], [Expr] }. + { Op, [{line,Line}], [Expr] }. build_tuple(_Marker, [Left, Right]) -> { Left, Right }; @@ -517,7 +508,7 @@ build_fn(Op, Stab) -> build_access(Expr, Access) -> Meta = [{line,?line(Access)}], - { { '.', Meta, ['Elixir.Kernel', access] }, Meta, [ Expr, ?op(Access) ] }. + { { '.', Meta, ['Elixir.Kernel', access] }, Meta, [ Expr, ?id(Access) ] }. %% Interpolation aware diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 50f0fba28b..bdc794b417 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -15,19 +15,6 @@ T1 == $[, T2 == $] ). --define(op2(T1, T2), - T1 == $+, T2 == $+; - T1 == $-, T2 == $-; - T1 == $*, T2 == $*; - T1 == $:, T2 == $:; - T1 == $-, T2 == $> -). - --define(op1(T), - T == $=; - T == $| -). - %% New ops table -define(at_op(T), @@ -58,6 +45,17 @@ -define(dual_op(T), T == $+ orelse T == $-). +-define(range_op(T1, T2), + T1 == $., T2 == $.). + +-define(arrow_op3(T1, T2, T3), + T1 == $<, T2 == $<, T3 == $<; + T1 == $>, T2 == $>, T3 == $>). + +-define(arrow_op(T1, T2), + T1 == $<, T2 == $-; + T1 == $|, T2 == $>). + -define(comp_op(T), T == $<; T == $>). @@ -85,20 +83,21 @@ -define(or_op3(T1, T2, T3), T1 == $|, T2 == $|, T3 == $|). --define(arrow_op3(T1, T2, T3), - T1 == $<, T2 == $<, T3 == $<; - T1 == $>, T2 == $>, T3 == $>). +-define(match_op(T), + T == $=). --define(arrow_op(T1, T2), - T1 == $<, T2 == $-; - T1 == $|, T2 == $>). - --define(range_op(T1, T2), - T1 == $., T2 == $.). +-define(tail_op(T), + T == $|). -define(default_op(T1, T2), T1 == $/, T2 == $/). +-define(stab_op(T1, T2), + T1 == $-, T2 == $>). + +-define(type_op(T1, T2), + T1 == $:, T2 == $:). + tokenize(String, Line, Opts) -> File = case lists:keyfind(file, 1, Opts) of { file, V1 } -> V1; @@ -253,18 +252,14 @@ tokenize([$.,T1,T2,T3|Rest], Line, Scope, Tokens) when % ## Two Token Operators tokenize([$.,T1,T2|Rest], Line, Scope, Tokens) when ?comp_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); ?arrow_op(T1, T2); - ?range_op(T1, T2); ?than_op(T1, T2); ?default_op(T1, T2); ?two_op(T1, T2) -> - handle_call_identifier(Rest, Line, list_to_atom([T1, T2]), Scope, Tokens); - -tokenize([$.,T1,T2|Rest], Line, Scope, Tokens) when ?op2(T1, T2) -> + ?range_op(T1, T2); ?than_op(T1, T2); ?default_op(T1, T2); ?two_op(T1, T2); + ?stab_op(T1, T2); ?type_op(T1, T2) -> handle_call_identifier(Rest, Line, list_to_atom([T1, T2]), Scope, Tokens); % ## Single Token Operators tokenize([$.,T|Rest], Line, Scope, Tokens) when - ?at_op(T); ?unary_op(T); ?dual_op(T); ?mult_op(T); ?comp_op(T) -> - handle_call_identifier(Rest, Line, list_to_atom([T]), Scope, Tokens); - -tokenize([$.,T|Rest], Line, Scope, Tokens) when ?op1(T); T == $& -> + ?at_op(T); ?unary_op(T); ?dual_op(T); ?mult_op(T); ?comp_op(T); + ?match_op(T); ?tail_op(T); T == $& -> handle_call_identifier(Rest, Line, list_to_atom([T]), Scope, Tokens); % Dot call @@ -337,18 +332,14 @@ tokenize([$:,T1,T2,T3|Rest], Line, Scope, Tokens) when % ## Two Token Operators tokenize([$:,T1,T2|Rest], Line, Scope, Tokens) when ?comp_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); ?arrow_op(T1, T2); - ?range_op(T1, T2); ?than_op(T1, T2); ?default_op(T1, T2); ?two_op(T1, T2) -> - tokenize(Rest, Line, Scope, [{ atom, Line, list_to_atom([T1,T2]) }|Tokens]); - -tokenize([$:,T1,T2|Rest], Line, Scope, Tokens) when ?op2(T1, T2) -> + ?range_op(T1, T2); ?than_op(T1, T2); ?default_op(T1, T2); ?two_op(T1, T2); + ?stab_op(T1, T2); ?type_op(T1, T2) -> tokenize(Rest, Line, Scope, [{ atom, Line, list_to_atom([T1,T2]) }|Tokens]); % ## Single Token Operators tokenize([$:,T|Rest], Line, Scope, Tokens) when - ?at_op(T); ?unary_op(T); ?dual_op(T); ?mult_op(T); ?comp_op(T) -> - tokenize(Rest, Line, Scope, [{ atom, Line, list_to_atom([T]) }|Tokens]); - -tokenize([$:,T|Rest], Line, Scope, Tokens) when ?op1(T); T == $&; T == $. -> + ?at_op(T); ?unary_op(T); ?dual_op(T); ?mult_op(T); ?comp_op(T); + ?match_op(T); ?tail_op(T); T == $&; T == $. -> tokenize(Rest, Line, Scope, [{ atom, Line, list_to_atom([T]) }|Tokens]); % End of line @@ -379,7 +370,7 @@ tokenize([$&,H|Rest], Line, Scope, Tokens) when ?is_digit(H) -> % ## Three token operators tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?unary_op3(T1, T2, T3) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T1,T2,T3]), Scope, Tokens); + handle_nonl_op(Rest, Line, unary_op, list_to_atom([T1,T2,T3]), Scope, Tokens); tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?comp_op3(T1, T2, T3) -> handle_op(Rest, Line, comp_op, list_to_atom([T1,T2,T3]), Scope, Tokens); @@ -431,27 +422,33 @@ tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?or_op(T1, T2) -> tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?default_op(T1, T2) -> handle_op(Rest, Line, default_op, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?op2(T1, T2) -> - handle_op(Rest, Line, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?stab_op(T1, T2) -> + handle_op(Rest, Line, stab_op, list_to_atom([T1, T2]), Scope, Tokens); + +tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?type_op(T1, T2) -> + handle_op(Rest, Line, type_op, list_to_atom([T1, T2]), Scope, Tokens); % ## Single Token Operators tokenize([T|Rest], Line, Scope, Tokens) when ?at_op(T) -> - handle_unary_op(Rest, Line, at_op, list_to_atom([T]), Scope, Tokens); + handle_nonl_op(Rest, Line, at_op, list_to_atom([T]), Scope, Tokens); tokenize([T|Rest], Line, Scope, Tokens) when ?unary_op(T) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T]), Scope, Tokens); + handle_nonl_op(Rest, Line, unary_op, list_to_atom([T]), Scope, Tokens); tokenize([T|Rest], Line, Scope, Tokens) when ?comp_op(T) -> handle_op(Rest, Line, comp_op, list_to_atom([T]), Scope, Tokens); tokenize([T|Rest], Line, Scope, Tokens) when ?dual_op(T) -> - handle_unary_op(Rest, Line, dual_op, list_to_atom([T]), Scope, Tokens); + handle_nonl_op(Rest, Line, dual_op, list_to_atom([T]), Scope, Tokens); tokenize([T|Rest], Line, Scope, Tokens) when ?mult_op(T) -> handle_op(Rest, Line, mult_op, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?op1(T) -> - handle_op(Rest, Line, list_to_atom([T]), Scope, Tokens); +tokenize([T|Rest], Line, Scope, Tokens) when ?match_op(T) -> + handle_op(Rest, Line, match_op, list_to_atom([T]), Scope, Tokens); + +tokenize([T|Rest], Line, Scope, Tokens) when ?tail_op(T) -> + handle_op(Rest, Line, tail_op, list_to_atom([T]), Scope, Tokens); tokenize([$.|Rest], Line, Scope, Tokens) -> tokenize(Rest, Line, Scope, add_token_with_nl({ '.', Line }, Tokens)); @@ -545,16 +542,10 @@ handle_strings(T, Line, H, #scope{file=File} = Scope, Tokens) -> interpolation_error(Error, " (for string starting at line ~B)", [Line]) end. -handle_op([$:|Rest], Line, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> +handle_nonl_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> tokenize(Rest, Line, Scope, [{ kw_identifier, Line, Op }|Tokens]); -handle_op(Rest, Line, Op, Scope, Tokens) -> - tokenize(Rest, Line, Scope, add_token_with_nl({ Op, Line }, Tokens)). - -handle_unary_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{ kw_identifier, Line, Op }|Tokens]); - -handle_unary_op(Rest, Line, Kind, Op, Scope, Tokens) -> +handle_nonl_op(Rest, Line, Kind, Op, Scope, Tokens) -> tokenize(Rest, Line, Scope, [{ Kind, Line, Op }|Tokens]). handle_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index 8ca67bd98b..0c103fa265 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -6,8 +6,8 @@ tokenize(String) -> { ok, Result } = elixir_tokenizer:tokenize(String, 1, []), Result. -colon_colon_test() -> - [{number,1,1},{'::',1},{number,1,3}] = tokenize("1 :: 3"), +type_test() -> + [{number,1,1},{type_op,1,'::'},{number,1,3}] = tokenize("1 :: 3"), [{identifier,1,foo}, {'.',1}, {paren_identifier,1,'::'},