diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 46c0296da4..5d43cfcbac 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -86,7 +86,7 @@ parse_error(Line, File, <<"syntax error before: ">>, <<"'end'">>) -> %% Produce a human-readable message for errors before a sigil parse_error(Line, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full) -> - {sigil, _, Sigil, [Content | _], _} = parse_erl_term(Full), + {sigil, _, Sigil, [Content | _], _, _} = parse_erl_term(Full), Content2 = case is_binary(Content) of true -> Content; false -> <<>> diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index adff17df95..62843f02a2 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -719,9 +719,10 @@ build_access(Expr, {List, Location}) -> %% Interpolation aware -build_sigil({sigil, Location, Sigil, Parts, Modifiers}) -> +build_sigil({sigil, Location, Sigil, Parts, Modifiers, Terminator}) -> Meta = meta_from_location(Location), - {list_to_atom("sigil_" ++ [Sigil]), Meta, [{'<<>>', Meta, string_parts(Parts)}, Modifiers]}. + MetaWithTerminator = [{terminator, Terminator} | Meta], + {list_to_atom("sigil_" ++ [Sigil]), MetaWithTerminator, [{'<<>>', Meta, string_parts(Parts)}, Modifiers]}. build_bin_string({bin_string, _Location, [H]} = Token) when is_binary(H) -> handle_literal(H, Token); diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 57ca1a8076..9dd0874fcc 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -194,7 +194,7 @@ tokenize([$~, S, H, H, H | T] = Original, Line, Column, Scope, Tokens) when ?is_ case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of {ok, NewLine, NewColumn, Parts, Rest} -> {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, NewColumn, Scope, [{sigil, {Line, Column, NewColumn}, S, Parts, Modifiers} | Tokens]); + tokenize(Final, NewLine, NewColumn, Scope, [{sigil, {Line, Column, NewColumn}, S, Parts, Modifiers, [H, H, H]} | Tokens]); {error, Reason} -> {error, Reason, Original, Tokens} end; @@ -203,7 +203,7 @@ tokenize([$~, S, H | T] = Original, Line, Column, Scope, Tokens) when ?is_sigil( case elixir_interpolation:extract(Line, Column + 3, Scope, ?is_downcase(S), T, sigil_terminator(H)) of {NewLine, NewColumn, Parts, Rest} -> {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, NewColumn, Scope, [{sigil, {Line, Column, NewColumn}, S, Parts, Modifiers} | Tokens]); + tokenize(Final, NewLine, NewColumn, Scope, [{sigil, {Line, Column, NewColumn}, S, Parts, Modifiers, [H]} | Tokens]); {error, Reason} -> Sigil = [$~, S, H], interpolation_error(Reason, Original, Tokens, " (for sigil ~ts starting at line ~B)", [Sigil, Line]) diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 1c064c9cbf..c9bfacba52 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -100,6 +100,19 @@ defmodule CodeTest do assert Code.string_to_quoted("a.1") == {:error, {1, "syntax error before: ", "1"}} end + test "string_to_quoted/1 for presence of sigils terminators" do + assert Code.string_to_quoted("~r/foo/") == + {:ok, {:sigil_r, [terminator: '/', line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}} + assert Code.string_to_quoted("~r[foo]") == + {:ok, {:sigil_r, [terminator: '[', line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}} + assert Code.string_to_quoted("~r\"foo\"") == + {:ok, {:sigil_r, [terminator: '"', line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}} + assert Code.string_to_quoted("~S\"\"\"\nsigil heredoc\n\"\"\"") == + {:ok, {:sigil_S, [terminator: '"""', line: 1], [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}} + assert Code.string_to_quoted("~S'''\nsigil heredoc\n'''") == + {:ok, {:sigil_S, [terminator: '\'\'\'', line: 1], [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}} + end + test "string_to_quoted!/1 works as string_to_quoted/1 but raises on errors" do assert Code.string_to_quoted!("1 + 2") == {:+, [line: 1], [1, 2]} diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index e5584bbf70..60a78b5d6a 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -194,6 +194,13 @@ vc_merge_conflict_test() -> {1, "found an unexpected version control marker, please resolve the conflicts: ", "<<<<<<< HEAD"} = tokenize_error("<<<<<<< HEAD\n[1, 2, 3]"). +sigil_terminator_test() -> + [{sigil, {1, 1, 8}, 114, [<<"foo">>], [], "/"}] = tokenize("~r/foo/"), + [{sigil, {1, 1, 8}, 114, [<<"foo">>], [], "["}] = tokenize("~r[foo]"), + [{sigil, {1, 1, 8}, 114, [<<"foo">>], [], "\""}] = tokenize("~r\"foo\""), + [{sigil, {1, 1, 1}, 83, [<<"sigil heredoc\n">>], [], "\"\"\""}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), + [{sigil, {1, 1, 1}, 83, [<<"sigil heredoc\n">>], [], "'''"}] = tokenize("~S'''\nsigil heredoc\n'''"). + invalid_sigil_delimiter_test() -> {1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"), true = lists:prefix("\"\\\" (column 3, codepoint U+005C)", lists:flatten(Message)).