diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index 4ace24fde1..63910400c6 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -71,10 +71,10 @@ defmodule EEx.Compiler do {:ok, expr, new_line, new_column, rest} -> {key, expr} = case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do - {:ok, _line, _column, _warnings, tokens} -> + {:ok, _line, _column, _warnings, rev_tokens, []} -> # We ignore warnings because the code will be tokenized # again later with the right line+column info - token_key(tokens, expr) + token_key(rev_tokens, expr) {:error, _, _, _, _} -> {:expr, expr} @@ -164,8 +164,8 @@ defmodule EEx.Compiler do end # Receives tokens and check if it is a start, middle or an end token. - defp token_key(tokens, expr) do - case {tokens, tokens |> Enum.reverse() |> drop_eol()} do + defp token_key(rev_tokens, expr) do + case {Enum.reverse(rev_tokens), drop_eol(rev_tokens)} do {[{:end, _} | _], [{:do, _} | _]} -> {:middle_expr, expr} diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex index d84cfb5b9c..345a8415da 100644 --- a/lib/elixir/lib/code/fragment.ex +++ b/lib/elixir/lib/code/fragment.ex @@ -481,10 +481,10 @@ defmodule Code.Fragment do defp operator(rest, count, acc, _call_op?) do case :elixir_tokenizer.tokenize(acc, 1, 1, []) do - {:ok, _, _, _, [{:atom, _, _}]} -> + {:ok, _, _, _, [{:atom, _, _}], []} -> {{:unquoted_atom, tl(acc)}, count} - {:ok, _, _, _, [{_, _, op}]} -> + {:ok, _, _, _, [{_, _, op}], []} -> {rest, dot_count} = strip_spaces(rest, count) cond do @@ -1088,9 +1088,41 @@ defmodule Code.Fragment do @spec container_cursor_to_quoted(List.Chars.t(), keyword()) :: {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} def container_cursor_to_quoted(fragment, opts \\ []) do - opts = - Keyword.take(opts, [:file, :line, :column, :columns, :token_metadata, :literal_encoder]) + opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder]) + opts = [cursor_completion: true, emit_warnings: false] ++ opts - Code.string_to_quoted(fragment, [cursor_completion: true, emit_warnings: false] ++ opts) + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + case :elixir_tokenizer.tokenize(to_charlist(fragment), line, column, opts) do + {:ok, line, column, _warnings, rev_tokens, rev_terminators} -> + tokens = :lists.reverse(rev_tokens, rev_terminators) + + case :elixir.tokens_to_quoted(tokens, file, opts) do + {:ok, ast} -> + {:ok, ast} + + {:error, error} -> + # In case parsing fails, we give it another shot but handling fn/do/else/catch/rescue/after. + tokens = + :lists.reverse( + rev_tokens, + [{:stab_op, {line, column, nil}, :->}, {nil, {line, column + 2, nil}}] ++ + Enum.map(rev_terminators, fn tuple -> + {line, column, info} = elem(tuple, 1) + put_elem(tuple, 1, {line, column + 5, info}) + end) + ) + + case :elixir.tokens_to_quoted(tokens, file, opts) do + {:ok, ast} -> {:ok, ast} + {:error, _} -> {:error, error} + end + end + + {:error, info, _rest, _warnings, _so_far} -> + {:error, :elixir.format_token_error(info)} + end end end diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index ff73dcc2fa..70ad268fa6 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -7,7 +7,8 @@ -export([ string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5, env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, - eval_quoted/4, eval_local_handler/2, eval_external_handler/3 + eval_quoted/4, eval_local_handler/2, eval_external_handler/3, + format_token_error/1 ]). -include("elixir.hrl"). -define(system, 'Elixir.System'). @@ -459,18 +460,21 @@ quoted_to_erl(Quoted, ErlS, ExS, Env) -> string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) -> case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of - {ok, _Line, _Column, [], Tokens} -> - {ok, Tokens}; - {ok, _Line, _Column, Warnings, Tokens} -> + {ok, _Line, _Column, [], Tokens, Terminators} -> + {ok, lists:reverse(Tokens, Terminators)}; + {ok, _Line, _Column, Warnings, Tokens, Terminators} -> (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)], - {ok, Tokens}; - {error, {Location, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} -> - {error, {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}}; - {error, {Location, Error, Token}, _Rest, _Warnings, _SoFar} -> - {error, {Location, to_binary(Error), to_binary(Token)}} + {ok, lists:reverse(Tokens, Terminators)}; + {error, Info, _Rest, _Warnings, _SoFar} -> + {error, format_token_error(Info)} end. +format_token_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) -> + {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}; +format_token_error({Location, Error, Token}) -> + {Location, to_binary(Error), to_binary(Token)}. + tokens_to_quoted(Tokens, WarningFile, Opts) -> handle_parsing_opts(WarningFile, Opts), diff --git a/lib/elixir/src/elixir_interpolation.erl b/lib/elixir/src/elixir_interpolation.erl index 30665691cb..4edf357b8a 100644 --- a/lib/elixir/src/elixir_interpolation.erl +++ b/lib/elixir/src/elixir_interpolation.erl @@ -62,11 +62,11 @@ extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) -> extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last); {error, Reason, _, _, _} -> {error, Reason}; - {ok, EndLine, EndColumn, Warnings, Tokens} when Scope#elixir_tokenizer.cursor_completion /= false -> + {ok, EndLine, EndColumn, Warnings, Tokens, Terminators} when Scope#elixir_tokenizer.cursor_completion /= false -> NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=noprune}, - Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1), + Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens, Terminators), Output1), extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last); - {ok, _, _, _, _} -> + {ok, _, _, _, _, _} -> {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}} end; diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index e8dd4f0245..25e29165ff 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -138,12 +138,12 @@ tokenize(String, Line, Opts) -> tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false -> #elixir_tokenizer{ascii_identifiers_only=Ascii, terminators=Terminators, warnings=Warnings} = Scope, - {CursorColumn, CursorTerminators, CursorTokens} = + {CursorColumn, CursorTerminators, AccTokens} = add_cursor(Line, Column, Cursor, Terminators, Tokens), AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), - AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens), - {ok, Line, Column, AllWarnings, AccTokens}; + {AccTerminators, _AccColumn} = cursor_complete(Line, CursorColumn, CursorTerminators), + {ok, Line, CursorColumn, AllWarnings, AccTokens, AccTerminators}; tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) -> End = terminator(Start), @@ -163,7 +163,7 @@ tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLi tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) -> #elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings} = Scope, AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), - {ok, Line, Column, AllWarnings, lists:reverse(Tokens)}; + {ok, Line, Column, AllWarnings, Tokens, []}; % VC merge conflict @@ -1728,19 +1728,15 @@ error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) -> %% Cursor handling -cursor_complete(Line, Column, Terminators, Tokens) -> - {AccTokens, _} = - lists:foldl( - fun({Start, _, _}, {NewTokens, NewColumn}) -> - End = terminator(Start), - AccTokens = [{End, {Line, NewColumn, nil}} | NewTokens], - AccColumn = NewColumn + length(erlang:atom_to_list(End)), - {AccTokens, AccColumn} - end, - {Tokens, Column}, - Terminators - ), - lists:reverse(AccTokens). +cursor_complete(Line, Column, Terminators) -> + lists:mapfoldl( + fun({Start, _, _}, AccColumn) -> + End = terminator(Start), + {{End, {Line, AccColumn, nil}}, AccColumn + length(erlang:atom_to_list(End))} + end, + Column, + Terminators + ). add_cursor(_Line, Column, noprune, Terminators, Tokens) -> {Column, Terminators, Tokens}; @@ -1783,8 +1779,6 @@ prune_tokens([{'{', _} | Tokens], ['}' | Opener], Terminators) -> prune_tokens([{'<<', _} | Tokens], ['>>' | Opener], Terminators) -> prune_tokens(Tokens, Opener, Terminators); %%% Handle anonymous functions -prune_tokens(Tokens, [], [{'fn', _, _} | Terminators]) -> - prune_tokens(drop_including(Tokens, 'fn'), [], Terminators); prune_tokens([{'(', _}, {capture_op, _, _} | Tokens], [], [{'(', _, _} | Terminators]) -> prune_tokens(Tokens, [], Terminators); %%% or it is time to stop... @@ -1794,6 +1788,8 @@ prune_tokens([{'eol', _} | _] = Tokens, [], Terminators) -> {Tokens, Terminators}; prune_tokens([{',', _} | _] = Tokens, [], Terminators) -> {Tokens, Terminators}; +prune_tokens([{'fn', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; prune_tokens([{'do', _} | _] = Tokens, [], Terminators) -> {Tokens, Terminators}; prune_tokens([{'(', _} | _] = Tokens, [], Terminators) -> @@ -1827,7 +1823,3 @@ prune_tokens([_ | Tokens], Opener, Terminators) -> prune_tokens(Tokens, Opener, Terminators); prune_tokens([], [], Terminators) -> {[], Terminators}. - -drop_including([{Token, _} | Tokens], Token) -> Tokens; -drop_including([_ | Tokens], Token) -> drop_including(Tokens, Token); -drop_including([], _Token) -> []. diff --git a/lib/elixir/test/elixir/code_fragment_test.exs b/lib/elixir/test/elixir/code_fragment_test.exs index c32fbc34ae..5d25973f9e 100644 --- a/lib/elixir/test/elixir/code_fragment_test.exs +++ b/lib/elixir/test/elixir/code_fragment_test.exs @@ -1158,9 +1158,7 @@ defmodule CodeFragmentTest do assert cc2q!("foo(bar do baz ") == s2q!("foo(bar do baz(__cursor__()) end)") assert cc2q!("foo(bar do baz(") == s2q!("foo(bar do baz(__cursor__()) end)") assert cc2q!("foo(bar do baz bat,") == s2q!("foo(bar do baz(bat, __cursor__()) end)") - - assert {:error, {_, "syntax error before: ", "'end'"}} = - CF.container_cursor_to_quoted("foo(bar do baz, bat") + assert cc2q!("foo(bar do baz, bat") == s2q!("foo(bar do baz, __cursor__() -> nil end)") end test "keyword lists" do @@ -1232,11 +1230,11 @@ defmodule CodeFragmentTest do end test "removes anonymous functions" do - assert cc2q!("(fn") == s2q!("(__cursor__())") - assert cc2q!("(fn x") == s2q!("(__cursor__())") - assert cc2q!("(fn x ->") == s2q!("(__cursor__())") - assert cc2q!("(fn x -> x") == s2q!("(__cursor__())") - assert cc2q!("(fn x, y -> x + y") == s2q!("(__cursor__())") + assert cc2q!("(fn") == s2q!("(fn __cursor__() -> nil end)") + assert cc2q!("(fn x") == s2q!("(fn __cursor__() -> nil end)") + assert cc2q!("(fn x ->") == s2q!("(fn x -> __cursor__() end)") + assert cc2q!("(fn x -> x") == s2q!("(fn x -> __cursor__() end)") + assert cc2q!("(fn x, y -> x + y") == s2q!("(fn x, y -> x + __cursor__() end)") assert cc2q!("(fn x, y -> x + y end") == s2q!("(__cursor__())") end diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index c07b847000..53625785c3 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -5,8 +5,8 @@ tokenize(String) -> tokenize(String, []). tokenize(String, Opts) -> - {ok, _Line, _Column, _Warnings, Result} = elixir_tokenizer:tokenize(String, 1, Opts), - Result. + {ok, _Line, _Column, _Warnings, Result, []} = elixir_tokenizer:tokenize(String, 1, Opts), + lists:reverse(Result). tokenize_error(String) -> {error, Error, _, _, _} = elixir_tokenizer:tokenize(String, 1, []),