hermes-agent/n8n-openai-adapter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle cursor completion inside fn/rescue/catch/else/after

Browse Source
This commit is contained in:
José Valim
2024-05-14 17:04:57 +02:00
parent fb0356372c
commit d7ea2fa2e4
7 changed files with 80 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/eex/lib/eex/compiler.ex
+4 -4
View File
@@ -71,10 +71,10 @@ defmodule EEx.Compiler do
{:ok, expr, new_line, new_column, rest} ->
{key, expr} =
case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do
{:ok, _line, _column, _warnings, tokens} ->
{:ok, _line, _column, _warnings, rev_tokens, []} ->
# We ignore warnings because the code will be tokenized
# again later with the right line+column info
token_key(tokens, expr)
token_key(rev_tokens, expr)
{:error, _, _, _, _} ->
{:expr, expr}
@@ -164,8 +164,8 @@ defmodule EEx.Compiler do
end
# Receives tokens and check if it is a start, middle or an end token.
defp token_key(tokens, expr) do
case {tokens, tokens |> Enum.reverse() |> drop_eol()} do
defp token_key(rev_tokens, expr) do
case {Enum.reverse(rev_tokens), drop_eol(rev_tokens)} do
{[{:end, _} | _], [{:do, _} | _]} ->
{:middle_expr, expr}
lib/elixir/lib/code/fragment.ex
+37 -5
View File
@@ -481,10 +481,10 @@ defmodule Code.Fragment do
defp operator(rest, count, acc, _call_op?) do
case :elixir_tokenizer.tokenize(acc, 1, 1, []) do
{:ok, _, _, _, [{:atom, _, _}]} ->
{:ok, _, _, _, [{:atom, _, _}], []} ->
{{:unquoted_atom, tl(acc)}, count}
{:ok, _, _, _, [{_, _, op}]} ->
{:ok, _, _, _, [{_, _, op}], []} ->
{rest, dot_count} = strip_spaces(rest, count)
cond do
@@ -1088,9 +1088,41 @@ defmodule Code.Fragment do
@spec container_cursor_to_quoted(List.Chars.t(), keyword()) ::
{:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}}
def container_cursor_to_quoted(fragment, opts \\ []) do
opts =
Keyword.take(opts, [:file, :line, :column, :columns, :token_metadata, :literal_encoder])
opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder])
opts = [cursor_completion: true, emit_warnings: false] ++ opts
Code.string_to_quoted(fragment, [cursor_completion: true, emit_warnings: false] ++ opts)
file = Keyword.get(opts, :file, "nofile")
line = Keyword.get(opts, :line, 1)
column = Keyword.get(opts, :column, 1)
case :elixir_tokenizer.tokenize(to_charlist(fragment), line, column, opts) do
{:ok, line, column, _warnings, rev_tokens, rev_terminators} ->
tokens = :lists.reverse(rev_tokens, rev_terminators)
case :elixir.tokens_to_quoted(tokens, file, opts) do
{:ok, ast} ->
{:ok, ast}
{:error, error} ->
# In case parsing fails, we give it another shot but handling fn/do/else/catch/rescue/after.
tokens =
:lists.reverse(
rev_tokens,
[{:stab_op, {line, column, nil}, :->}, {nil, {line, column + 2, nil}}] ++
Enum.map(rev_terminators, fn tuple ->
{line, column, info} = elem(tuple, 1)
put_elem(tuple, 1, {line, column + 5, info})
end)
)
case :elixir.tokens_to_quoted(tokens, file, opts) do
{:ok, ast} -> {:ok, ast}
{:error, _} -> {:error, error}
end
end
{:error, info, _rest, _warnings, _so_far} ->
{:error, :elixir.format_token_error(info)}
end
end
end
lib/elixir/src/elixir.erl
+13 -9
View File
@@ -7,7 +7,8 @@
-export([
string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5,
env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3,
eval_quoted/4, eval_local_handler/2, eval_external_handler/3
eval_quoted/4, eval_local_handler/2, eval_external_handler/3,
format_token_error/1
]).
-include("elixir.hrl").
-define(system, 'Elixir.System').
@@ -459,18 +460,21 @@ quoted_to_erl(Quoted, ErlS, ExS, Env) ->
string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) ->
case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of
{ok, _Line, _Column, [], Tokens} ->
{ok, Tokens};
{ok, _Line, _Column, Warnings, Tokens} ->
{ok, _Line, _Column, [], Tokens, Terminators} ->
{ok, lists:reverse(Tokens, Terminators)};
{ok, _Line, _Column, Warnings, Tokens, Terminators} ->
(lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso
[elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)],
{ok, Tokens};
{error, {Location, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} ->
{error, {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}};
{error, {Location, Error, Token}, _Rest, _Warnings, _SoFar} ->
{error, {Location, to_binary(Error), to_binary(Token)}}
{ok, lists:reverse(Tokens, Terminators)};
{error, Info, _Rest, _Warnings, _SoFar} ->
{error, format_token_error(Info)}
end.
format_token_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) ->
{Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)};
format_token_error({Location, Error, Token}) ->
{Location, to_binary(Error), to_binary(Token)}.
tokens_to_quoted(Tokens, WarningFile, Opts) ->
handle_parsing_opts(WarningFile, Opts),
lib/elixir/src/elixir_interpolation.erl
+3 -3
View File
@@ -62,11 +62,11 @@ extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) ->
extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last);
{error, Reason, _, _, _} ->
{error, Reason};
{ok, EndLine, EndColumn, Warnings, Tokens} when Scope#elixir_tokenizer.cursor_completion /= false ->
{ok, EndLine, EndColumn, Warnings, Tokens, Terminators} when Scope#elixir_tokenizer.cursor_completion /= false ->
NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=noprune},
Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1),
Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens, Terminators), Output1),
extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last);
{ok, _, _, _, _} ->
{ok, _, _, _, _, _} ->
{error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}}
end;
lib/elixir/src/elixir_tokenizer.erl
+15 -23
View File
@@ -138,12 +138,12 @@ tokenize(String, Line, Opts) ->
tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false ->
#elixir_tokenizer{ascii_identifiers_only=Ascii, terminators=Terminators, warnings=Warnings} = Scope,
{CursorColumn, CursorTerminators, CursorTokens} =
{CursorColumn, CursorTerminators, AccTokens} =
add_cursor(Line, Column, Cursor, Terminators, Tokens),
AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings),
AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens),
{ok, Line, Column, AllWarnings, AccTokens};
{AccTerminators, _AccColumn} = cursor_complete(Line, CursorColumn, CursorTerminators),
{ok, Line, CursorColumn, AllWarnings, AccTokens, AccTerminators};
tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) ->
End = terminator(Start),
@@ -163,7 +163,7 @@ tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLi
tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) ->
#elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings} = Scope,
AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings),
{ok, Line, Column, AllWarnings, lists:reverse(Tokens)};
{ok, Line, Column, AllWarnings, Tokens, []};
% VC merge conflict
@@ -1728,19 +1728,15 @@ error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) ->
%% Cursor handling
cursor_complete(Line, Column, Terminators, Tokens) ->
{AccTokens, _} =
lists:foldl(
fun({Start, _, _}, {NewTokens, NewColumn}) ->
End = terminator(Start),
AccTokens = [{End, {Line, NewColumn, nil}} | NewTokens],
AccColumn = NewColumn + length(erlang:atom_to_list(End)),
{AccTokens, AccColumn}
end,
{Tokens, Column},
Terminators
),
lists:reverse(AccTokens).
cursor_complete(Line, Column, Terminators) ->
lists:mapfoldl(
fun({Start, _, _}, AccColumn) ->
End = terminator(Start),
{{End, {Line, AccColumn, nil}}, AccColumn + length(erlang:atom_to_list(End))}
end,
Column,
Terminators
).
add_cursor(_Line, Column, noprune, Terminators, Tokens) ->
{Column, Terminators, Tokens};
@@ -1783,8 +1779,6 @@ prune_tokens([{'{', _} | Tokens], ['}' | Opener], Terminators) ->
prune_tokens([{'<<', _} | Tokens], ['>>' | Opener], Terminators) ->
prune_tokens(Tokens, Opener, Terminators);
%%% Handle anonymous functions
prune_tokens(Tokens, [], [{'fn', _, _} | Terminators]) ->
prune_tokens(drop_including(Tokens, 'fn'), [], Terminators);
prune_tokens([{'(', _}, {capture_op, _, _} | Tokens], [], [{'(', _, _} | Terminators]) ->
prune_tokens(Tokens, [], Terminators);
%%% or it is time to stop...
@@ -1794,6 +1788,8 @@ prune_tokens([{'eol', _} | _] = Tokens, [], Terminators) ->
{Tokens, Terminators};
prune_tokens([{',', _} | _] = Tokens, [], Terminators) ->
{Tokens, Terminators};
prune_tokens([{'fn', _} | _] = Tokens, [], Terminators) ->
{Tokens, Terminators};
prune_tokens([{'do', _} | _] = Tokens, [], Terminators) ->
{Tokens, Terminators};
prune_tokens([{'(', _} | _] = Tokens, [], Terminators) ->
@@ -1827,7 +1823,3 @@ prune_tokens([_ | Tokens], Opener, Terminators) ->
prune_tokens(Tokens, Opener, Terminators);
prune_tokens([], [], Terminators) ->
{[], Terminators}.
drop_including([{Token, _} | Tokens], Token) -> Tokens;
drop_including([_ | Tokens], Token) -> drop_including(Tokens, Token);
drop_including([], _Token) -> [].
lib/elixir/test/elixir/code_fragment_test.exs
+6 -8
View File
@@ -1158,9 +1158,7 @@ defmodule CodeFragmentTest do
assert cc2q!("foo(bar do baz ") == s2q!("foo(bar do baz(__cursor__()) end)")
assert cc2q!("foo(bar do baz(") == s2q!("foo(bar do baz(__cursor__()) end)")
assert cc2q!("foo(bar do baz bat,") == s2q!("foo(bar do baz(bat, __cursor__()) end)")
assert {:error, {_, "syntax error before: ", "'end'"}} =
CF.container_cursor_to_quoted("foo(bar do baz, bat")
assert cc2q!("foo(bar do baz, bat") == s2q!("foo(bar do baz, __cursor__() -> nil end)")
end
test "keyword lists" do
@@ -1232,11 +1230,11 @@ defmodule CodeFragmentTest do
end
test "removes anonymous functions" do
assert cc2q!("(fn") == s2q!("(__cursor__())")
assert cc2q!("(fn x") == s2q!("(__cursor__())")
assert cc2q!("(fn x ->") == s2q!("(__cursor__())")
assert cc2q!("(fn x -> x") == s2q!("(__cursor__())")
assert cc2q!("(fn x, y -> x + y") == s2q!("(__cursor__())")
assert cc2q!("(fn") == s2q!("(fn __cursor__() -> nil end)")
assert cc2q!("(fn x") == s2q!("(fn __cursor__() -> nil end)")
assert cc2q!("(fn x ->") == s2q!("(fn x -> __cursor__() end)")
assert cc2q!("(fn x -> x") == s2q!("(fn x -> __cursor__() end)")
assert cc2q!("(fn x, y -> x + y") == s2q!("(fn x, y -> x + __cursor__() end)")
assert cc2q!("(fn x, y -> x + y end") == s2q!("(__cursor__())")
end
lib/elixir/test/erlang/tokenizer_test.erl
+2 -2
View File
@@ -5,8 +5,8 @@ tokenize(String) ->
tokenize(String, []).
tokenize(String, Opts) ->
{ok, _Line, _Column, _Warnings, Result} = elixir_tokenizer:tokenize(String, 1, Opts),
Result.
{ok, _Line, _Column, _Warnings, Result, []} = elixir_tokenizer:tokenize(String, 1, Opts),
lists:reverse(Result).
tokenize_error(String) ->
{error, Error, _, _, _} = elixir_tokenizer:tokenize(String, 1, []),