From 49501ef59e2e668c18746ba2fec42fccb7239a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 20 Jul 2012 13:04:33 +0200 Subject: [PATCH] Improve error messages with terminators --- lib/elixir/src/elixir_errors.erl | 9 ++- lib/elixir/src/elixir_tokenizer.erl | 73 +++++++++++++++---- lib/elixir/test/elixir/kernel/errors_test.exs | 10 ++- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 2d5a893b95..dd3c58cb4e 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -43,15 +43,18 @@ parse_error(Line, File, Error, []) -> end, raise(Line, File, 'Elixir.TokenMissingError', Message); +parse_error(Line, File, "syntax error before: ", "'end'") -> + raise(Line, File, 'Elixir.SyntaxError', <<"unexpected token: end">>); + parse_error(Line, File, Error, Token) -> BinError = if is_atom(Error) -> atom_to_binary(Error, utf8); true -> iolist_to_binary(Error) end, - BinToken = case Token of - [] -> <<>>; - _ -> iolist_to_binary(Token) + BinToken = if + Token == [] -> <<>>; + true -> iolist_to_binary(Token) end, Message = <>, diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 621da6ba05..bd968e326a 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -75,9 +75,14 @@ tokenize(String, Line, File) -> tokenize(String, Line, File, [], []). -tokenize([], _Line, _File, _Terminators, Tokens) -> +tokenize([], _Line, _File, [], Tokens) -> { ok, lists:reverse(Tokens) }; +tokenize([], EndLine, _File, [{ Start, StartLine }|_], _Tokens) -> + End = terminator(Start), + Message = io_lib:format("missing terminator: ~s (for ~s starting at line ~B)", [End, Start, StartLine]), + { error, { EndLine, Message, [] } }; + % Base integers tokenize([$0,X,H|T], Line, File, Terminators, Tokens) when (X == $x orelse X == $X), ?is_hex(H) -> @@ -110,7 +115,7 @@ tokenize([$%,S,H,H,H|T], Line, File, Terminators, Tokens) when ?is_quote(H), ?is end; tokenize([$%,S,H|T], Line, File, Terminators, Tokens) when not(?is_word(H)), ?is_upcase(S) orelse ?is_downcase(S) -> - case elixir_interpolation:extract(Line, File, ?is_downcase(S), T, terminator(H)) of + case elixir_interpolation:extract(Line, File, ?is_downcase(S), T, sigil_terminator(H)) of { NewLine, Parts, Rest } -> { Final, Modifiers } = collect_modifiers(Rest, []), tokenize(Final, NewLine, File, Terminators, [{sigil,Line,S,Parts,Modifiers}|Tokens]); @@ -251,11 +256,19 @@ tokenize([T1,T2,T3|Rest], Line, File, Terminators, Tokens) when ?op3(T1, T2, T3) % ## Containers + punctuation tokens tokenize([T,T|Rest], Line, File, Terminators, Tokens) when T == $<; T == $> -> - tokenize(Rest, Line, File, Terminators, [{list_to_atom([T,T]), Line}|Tokens]); + Token = { list_to_atom([T,T]), Line }, + case handle_terminator(Token, Terminators) of + { error, _ } = Error -> Error; + New -> tokenize(Rest, Line, File, New, [Token|Tokens]) + end; tokenize([T|Rest], Line, File, Terminators, Tokens) when T == $(; - T == ${; T == $}; T == $[; T == $]; T == $); T == $, -> - tokenize(Rest, Line, File, Terminators, [{list_to_atom([T]), Line}|Tokens]); + T == ${; T == $}; T == $[; T == $]; T == $); T == $, -> + Token = { list_to_atom([T]), Line }, + case handle_terminator(Token, Terminators) of + { error, _ } = Error -> Error; + New -> tokenize(Rest, Line, File, New, [Token|Tokens]) + end; % ## Comparison two token operators tokenize([T1,T2|Rest], Line, File, Terminators, Tokens) when ?comp2(T1, T2) -> @@ -295,8 +308,11 @@ tokenize([H|_] = String, Line, File, Terminators, Tokens) when ?is_downcase(H); case keyword(Line, Kind, Identifier, Tokens) of false -> tokenize(Rest, Line, File, Terminators, [{Kind,Line,Identifier}|Tokens]); - Else -> - tokenize(Rest, Line, File, Terminators, Else) + [Check|T] -> + case handle_terminator(Check, Terminators) of + { error, _ } = Error -> Error; + New -> tokenize(Rest, Line, File, New, [Check|T]) + end end; % Ambiguous unary/binary operators tokens @@ -631,16 +647,45 @@ add_token_with_nl(Left, T) -> [Left|T]. interpolation_error({ error, { Line, Message, Token } }, Extension, Args) -> { error, { Line, io_lib:format("~s" ++ Extension, [Message|Args]), Token } }. -% String type +% Terminators + string_type($") -> bin_string; string_type($') -> list_string. -% Terminator -terminator($() -> $); -terminator($[) -> $]; -terminator(${) -> $}; -terminator($<) -> $>; -terminator(O) -> O. +sigil_terminator($() -> $); +sigil_terminator($[) -> $]; +sigil_terminator(${) -> $}; +sigil_terminator($<) -> $>; +sigil_terminator(O) -> O. + +handle_terminator({ S, _ } = New, Terminators) when + S == '('; + S == '['; + S == '{'; + S == '<<' -> + [New|Terminators]; + +handle_terminator({ E, _ }, [{ S, _ }|Terminators]) when + S == '(', E == ')'; + S == '[', E == ']'; + S == '{', E == '}'; + S == '<<', E == '>>' -> + Terminators; + +handle_terminator({ E, Line }, _) when + E == ')'; + E == ']'; + E == '}'; + E == '>>' -> + { error, { Line, "unexpected token: ", atom_to_list(E) } }; + +handle_terminator(_, Terminators) -> + Terminators. + +terminator('(') -> ')'; +terminator('[') -> ']'; +terminator('{') -> '}'; +terminator('<<') -> '>>'. % Keywords check keyword(Line, do_identifier, fn, Tokens) -> diff --git a/lib/elixir/test/elixir/kernel/errors_test.exs b/lib/elixir/test/elixir/kernel/errors_test.exs index 2d829388cd..a6b48275a1 100644 --- a/lib/elixir/test/elixir/kernel/errors_test.exs +++ b/lib/elixir/test/elixir/kernel/errors_test.exs @@ -35,12 +35,16 @@ defmodule Kernel.ErrorsTest do assert "nofile:1: partial variable &2 cannot be defined without &1" == format_rescue '&2 + 3' end + test :unexpected_end do + assert "nofile:1: unexpected token: end" == format_rescue '1 end' + end + test :syntax_error do - assert "nofile:1: syntax error before: '}'" == format_rescue 'case 1 do }' + assert "nofile:1: syntax error before: '.'" == format_rescue '+.foo' end test :syntax_error_with_no_token do - assert "nofile:1: syntax error: expression is incomplete" == format_rescue 'case 1 do' + assert "nofile:1: missing terminator: ) (for ( starting at line 1)" == format_rescue 'case 1 (' end test :bad_form do @@ -137,7 +141,7 @@ defmodule Kernel.ErrorsTest do end test :interpolation_error do - assert "nofile:1: syntax error before: ')'" == format_rescue '"foo\#{case 1 do )}bar"' + assert "nofile:1: unexpected token: )" == format_rescue '"foo\#{case 1 do )}bar"' end test :cant_define_local_due_to_in_erlang_macros_conflict do