From a7edd6ea47e0b65e83e505dbae6357207d64aa3d Mon Sep 17 00:00:00 2001 From: Jean Klingler Date: Sat, 24 Jun 2023 18:29:24 +0900 Subject: [PATCH] Parser honors :static_atoms_encoder for multi-letter sigils (#12706) * Unify sigil token generation * Parser honors :static_atoms_encoder for multi-letter sigils * Remove chars from token, reverse engineer from atom --- lib/elixir/lib/code.ex | 5 ++- lib/elixir/src/elixir_errors.erl | 15 +++++++-- lib/elixir/src/elixir_parser.yrl | 4 +-- lib/elixir/src/elixir_tokenizer.erl | 32 ++++++++++++------- lib/elixir/test/elixir/kernel/parser_test.exs | 30 +++++++++++++++++ lib/elixir/test/erlang/tokenizer_test.erl | 20 ++++++------ 6 files changed, 79 insertions(+), 27 deletions(-) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index f32544c981..17a12945b6 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1138,7 +1138,10 @@ defmodule Code do * syntax keywords (`fn`, `do`, `else`, and so on) * atoms containing interpolation (`:"#{1 + 1} is two"`), as these - atoms are constructed at runtime. + atoms are constructed at runtime + + * atoms used to represent single-letter sigils like `:sigil_X` + (but multi-letter sigils like `:sigil_XYZ` are encoded). """ @spec string_to_quoted(List.Chars.t(), keyword) :: diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 76ff74ba58..05f4ffe044 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -248,13 +248,22 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input) %% Produce a human-readable message for errors before a sigil parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, Input) -> - {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full), + {sigil, _, Atom, [Content | _], _, _, _} = parse_erl_term(Full), Content2 = case is_binary(Content) of true -> Content; false -> <<>> end, - SigilName = list_to_binary(Sigil), - Message = <<"syntax error before: sigil \~", SigilName/binary, " starting with content '", Content2/binary, "'">>, + + % :static_atoms_encoder might encode :sigil_ atoms as arbitrary terms + MaybeSigil = case is_atom(Atom) of + true -> case atom_to_binary(Atom) of + <<"sigil_", Chars/binary>> -> <<"\~", Chars/binary, " ">>; + _ -> <<>> + end; + false -> <<>> + end, + + Message = <<"syntax error before: sigil ", MaybeSigil/binary, "starting with content '", Content2/binary, "'">>, raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message); %% Binaries (and interpolation) are wrapped in [<<...>>] diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index 6e48d87960..d5341acd66 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -939,11 +939,11 @@ build_access(Expr, {List, Meta}) -> %% Interpolation aware -build_sigil({sigil, Location, Sigil, Parts, Modifiers, Indentation, Delimiter}) -> +build_sigil({sigil, Location, Atom, Parts, Modifiers, Indentation, Delimiter}) -> Meta = meta_from_location(Location), MetaWithDelimiter = [{delimiter, Delimiter} | Meta], MetaWithIndentation = meta_with_indentation(Meta, Indentation), - {list_to_atom("sigil_" ++ Sigil), + {Atom, MetaWithDelimiter, [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}. diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 6fa5d6ccf5..d060a781d6 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1550,7 +1550,7 @@ tokenize_sigil_name([S | T], NameAcc, Line, Column, Scope, Tokens) when ?is_upca tokenize_sigil_name(T, [S | NameAcc], Line, Column + 1, Scope, Tokens); % With a lowercase letter and a non-empty NameAcc we return an error. tokenize_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) -> - Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ + Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ " sequence of uppercase letters only, got: ", {error, Message, [$~] ++ lists:reverse(NameAcc) ++ Original}; % We finished the letters, so the name is over. @@ -1561,12 +1561,8 @@ tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Col when ?is_quote(H) -> case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> - {Final, Modifiers} = collect_modifiers(Rest, []), Indentation = NewColumn - 4, - TokenColumn = Column - 1 - length(SigilName), - Token = {sigil, {Line, TokenColumn, nil}, SigilName, Parts, Modifiers, Indentation, <>}, - NewColumnWithModifiers = NewColumn + length(Modifiers), - tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <>); {error, Reason} -> error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens) @@ -1576,12 +1572,8 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S when ?is_sigil(H) -> case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of {NewLine, NewColumn, Parts, Rest, NewScope} -> - {Final, Modifiers} = collect_modifiers(Rest, []), Indentation = nil, - TokenColumn = Column - 1 - length(SigilName), - Token = {sigil, {Line, TokenColumn, nil}, SigilName, tokens_to_binary(Parts), Modifiers, Indentation, <>}, - NewColumnWithModifiers = NewColumn + length(Modifiers), - tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <>); {error, Reason} -> Sigil = [$~, S, H], @@ -1601,6 +1593,24 @@ tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Toke tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) -> tokenize([], Line, Column, Scope, Tokens). +add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) -> + TokenColumn = Column - 1 - length(SigilName), + MaybeEncoded = case SigilName of + % Single-letter sigils present no risk of atom exhaustion (limited possibilities) + [_Char] -> {ok, list_to_atom("sigil_" ++ SigilName)}; + _ -> unsafe_to_atom("sigil_" ++ SigilName, Line, TokenColumn, Scope) + end, + case MaybeEncoded of + {ok, Atom} -> + {Final, Modifiers} = collect_modifiers(Rest, []), + Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter}, + NewColumnWithModifiers = NewColumn + length(Modifiers), + tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, Scope, Tokens) + end. + %% Fail early on invalid do syntax. For example, after %% most keywords, after comma and so on. tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) -> diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index 24e7a733a4..7a0671cb27 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -182,6 +182,9 @@ defmodule Kernel.ParserTest do assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) == {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}} + + assert Code.string_to_quoted("~UNKNOWN'foo bar'", existing_atoms_only: true) == + {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "sigil_UNKNOWN"}} end test "encodes atoms" do @@ -228,6 +231,20 @@ defmodule Kernel.ParserTest do ) end + test "encodes multi-letter sigils" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "sigil_UNKNOWN" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, ref} + end + + assert {:ok, {^ref, [delimiter: "'", line: 1], [{:<<>>, [line: 1], ["abc"]}, []]}} = + Code.string_to_quoted("~UNKNOWN'abc'", static_atoms_encoder: encoder) + end + test "addresses ambiguities" do encoder = fn string, _meta -> {:ok, {:atom, string}} end @@ -254,6 +271,16 @@ defmodule Kernel.ParserTest do Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder) end + test "does not encode one-letter sigils" do + encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end + + assert {:ok, {:sigil_z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~z'foo'", static_atoms_encoder: encoder) + + assert {:ok, {:sigil_Z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~Z'foo'", static_atoms_encoder: encoder) + end + test "returns errors on long atoms even when using static_atoms_encoder" do atom = String.duplicate("a", 256) @@ -271,6 +298,9 @@ defmodule Kernel.ParserTest do assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} = Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + + assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "sigil_UNKNOWN"}} = + Code.string_to_quoted("~UNKNOWN'foo bar'", static_atoms_encoder: encoder) end test "may return tuples" do diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index f3c1058770..a4d0e8f076 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -261,22 +261,22 @@ vc_merge_conflict_test() -> tokenize_error("<<<<<<< HEAD\n[1, 2, 3]"). sigil_terminator_test() -> - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}, + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}, {comp_op, {1, 9, nil}, '=='}, {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "iu", nil, <<"/">>}, + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>}, {comp_op, {1, 11, nil}, '=='}, {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"), - [{sigil, {1, 1, nil}, "M", [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). + [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). sigil_heredoc_test() -> - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), - [{sigil, {1, 1, nil}, "s", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), + [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). invalid_sigil_delimiter_test() -> {1, 1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"),