diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 8f94d4a7df..65014b14e6 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -749,18 +749,13 @@ defmodule Code do unescape: false, warn_on_unnecessary_quotes: false, literal_encoder: &{:ok, {:__block__, &2, [&1]}}, - token_metadata: true + token_metadata: true, + emit_warnings: false ] ++ opts {forms, comments} = string_to_quoted_with_comments!(string, to_quoted_opts) - - to_algebra_opts = - [ - comments: comments - ] ++ opts - + to_algebra_opts = [comments: comments] ++ opts doc = Code.Formatter.to_algebra(forms, to_algebra_opts) - Inspect.Algebra.format(doc, line_length) end diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex index b6e0f59a4f..65604da1d5 100644 --- a/lib/elixir/lib/code/fragment.ex +++ b/lib/elixir/lib/code/fragment.ex @@ -813,20 +813,14 @@ defmodule Code.Fragment do columns = Keyword.get(opts, :columns, false) token_metadata = Keyword.get(opts, :token_metadata, false) - fragment = to_charlist(fragment) - tokenizer_opts = [file: file, cursor_completion: true, columns: columns] - - case :elixir_tokenizer.tokenize(fragment, line, column, tokenizer_opts) do - {:ok, _, _, _warnings, tokens} -> - :elixir.tokens_to_quoted(tokens, nil, columns: columns, token_metadata: token_metadata) - - {:error, {line, column, {prefix, suffix}, token}, _rest, _warnings, _so_far} -> - location = [line: line, column: column] - {:error, {location, {to_string(prefix), to_string(suffix)}, to_string(token)}} - - {:error, {line, column, error, token}, _rest, _warnings, _so_far} -> - location = [line: line, column: column] - {:error, {location, to_string(error), to_string(token)}} - end + Code.string_to_quoted(fragment, + file: file, + line: line, + column: column, + columns: columns, + token_metadata: token_metadata, + cursor_completion: true, + emit_warnings: false + ) end end diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index e744db8c4e..19c89cca8a 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -365,9 +365,12 @@ quoted_to_erl(Quoted, Env, Scope) -> %% Converts a given string (charlist) into quote expression string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) -> - case elixir_tokenizer:tokenize(String, StartLine, StartColumn, [{file, File} | Opts]) of + case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of + {ok, _Line, _Column, [], Tokens} -> + {ok, Tokens}; {ok, _Line, _Column, Warnings, Tokens} -> - [elixir_errors:erl_warn(L, F, M) || {L, F, M} <- lists:reverse(Warnings)], + (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso + [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)], {ok, Tokens}; {error, {Line, Column, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} -> Location = [{line, Line}, {column, Column}], @@ -424,7 +427,12 @@ parser_location(Meta) -> to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List); to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom). -handle_parsing_opts(WarningFile, Opts) -> +handle_parsing_opts(File, Opts) -> + WarningFile = + case lists:keyfind(emit_warnings, 1, Opts) of + {emit_warnings, false} -> nil; + _ -> File + end, LiteralEncoder = case lists:keyfind(literal_encoder, 1, Opts) of {literal_encoder, Fun} -> Fun; diff --git a/lib/elixir/src/elixir.hrl b/lib/elixir/src/elixir.hrl index eceb12ef8e..d238cdee4e 100644 --- a/lib/elixir/src/elixir.hrl +++ b/lib/elixir/src/elixir.hrl @@ -25,7 +25,6 @@ }). -record(elixir_tokenizer, { - file=(<<"nofile">>), terminators=[], unescape=true, cursor_completion=false, diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 8a851d6942..070a3393d4 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -113,8 +113,6 @@ tokenize(String, Line, Column, Opts) -> Acc#elixir_tokenizer{terminators=none}; ({cursor_completion, true}, Acc) -> Acc#elixir_tokenizer{cursor_completion=cursor_and_terminators}; - ({file, File}, Acc) when is_binary(File) -> - Acc#elixir_tokenizer{file=File}; ({existing_atoms_only, ExistingAtomsOnly}, Acc) when is_boolean(ExistingAtomsOnly) -> Acc#elixir_tokenizer{existing_atoms_only=ExistingAtomsOnly}; ({static_atoms_encoder, StaticAtomsEncoder}, Acc) when is_function(StaticAtomsEncoder) -> @@ -135,12 +133,12 @@ tokenize(String, Line, Opts) -> tokenize(String, Line, 1, Opts). tokenize([], Line, Column, #elixir_tokenizer{ascii_identifiers_only=Ascii, cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false -> - #elixir_tokenizer{file=File, terminators=Terminators, warnings=Warnings} = Scope, + #elixir_tokenizer{terminators=Terminators, warnings=Warnings} = Scope, {CursorColumn, CursorTerminators, CursorTokens} = add_cursor(Line, Column, Cursor, Terminators, Tokens), - AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, File, Warnings), + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens), {ok, Line, Column, AllWarnings, AccTokens}; @@ -151,8 +149,8 @@ tokenize([], EndLine, Column, #elixir_tokenizer{terminators=[{Start, StartLine, Formatted = io_lib:format(Message, [End, Start, StartLine]), error({EndLine, Column, [Formatted, Hint], []}, [], Scope, Tokens); -tokenize([], Line, Column, #elixir_tokenizer{ascii_identifiers_only=Ascii, file=File, warnings=Warnings}, Tokens) -> - AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, File, Warnings), +tokenize([], Line, Column, #elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings}, Tokens) -> + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), {ok, Line, Column, AllWarnings, lists:reverse(Tokens)}; % VC merge conflict @@ -244,11 +242,11 @@ tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) -> {Escape, Name} -> Msg = io_lib:format("found ?\\ followed by code point 0x~.16B (~ts), please use ?~ts instead", [Char, Name, Escape]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); false when ?is_downcase(H); ?is_upcase(H) -> Msg = io_lib:format("unknown escape sequence ?\\~tc, use ?~tc instead", [H, H]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); false -> Scope @@ -265,7 +263,7 @@ tokenize([$?, Char | T], Line, Column, Scope, Tokens) -> {Escape, Name} -> Msg = io_lib:format("found ? followed by code point 0x~.16B (~ts), please use ?~ts instead", [Char, Name, Escape]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); false -> Scope end, @@ -332,7 +330,7 @@ tokenize([$:, T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when %% TODO: Remove this deprecation on Elixir v2.0 tokenize([$:, $:, $: | Rest], Line, Column, Scope, Tokens) -> Message = "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity", - NewScope = prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Message, Scope), + NewScope = prepend_warning(Line, Column, Message, Scope), Token = {atom, {Line, Column, nil}, '::'}, tokenize(Rest, Line, Column + 3, NewScope, [Token | Tokens]); @@ -507,7 +505,7 @@ tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H) "do not require quotes", [hd(Parts)] ), - prepend_warning(Line, Column, InterScope#elixir_tokenizer.file, WarnMsg, InterScope); + prepend_warning(Line, Column, WarnMsg, InterScope); false -> InterScope @@ -786,7 +784,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> "letters, numbers, and underscores do not require quotes", [hd(Parts)] ), - prepend_warning(Line, Column, InterScope#elixir_tokenizer.file, WarnMsg, InterScope); + prepend_warning(Line, Column, WarnMsg, InterScope); false -> InterScope @@ -854,15 +852,15 @@ handle_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> case Op of '^^^' -> Msg = "^^^ is deprecated. It is typically used as xor but it has the wrong precedence, use Bitwise.bxor/2 instead", - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); '~~~' -> Msg = "~~~ is deprecated. Use Bitwise.bnot/1 instead for clarity", - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); '<|>' -> Msg = "<|> is deprecated. Use another pipe-like operator", - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); _ -> Scope @@ -906,7 +904,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?i "do not require quotes", [Part] ), - prepend_warning(Line, Column, InterScope#elixir_tokenizer.file, WarnMsg, InterScope); + prepend_warning(Line, Column, WarnMsg, InterScope); false -> InterScope @@ -981,11 +979,10 @@ unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{}) when is_binary(Part) andalso byte_size(Part) > 255; is_list(Part) andalso length(Part) > 255 -> {error, {Line, Column, "atom length must be less than system limit: ", elixir_utils:characters_to_list(Part)}}; -unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder} = Scope) when +unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder}) when is_function(StaticAtomsEncoder) -> - Metadata = [{line, Line}, {column, Column}, {file, Scope#elixir_tokenizer.file}], Value = elixir_utils:characters_to_binary(Part), - case StaticAtomsEncoder(Value, Metadata) of + case StaticAtomsEncoder(Value, [{line, Line}, {column, Column}]) of {ok, Term} -> {ok, Term}; {error, Reason} when is_binary(Reason) -> @@ -1088,7 +1085,7 @@ maybe_heredoc_warn(Line, Column, Scope, Marker) -> " end~n~n" "The current heredoc line is indented too little", [[Marker, Marker, Marker]]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope). + prepend_warning(Line, Column, Msg, Scope). extract_heredoc_head([[$\n|H]|T]) -> [H|T]. @@ -1552,7 +1549,7 @@ maybe_warn_too_many_of_same_char([T | _] = Token, [T | _] = _Rest, Line, Column, _ -> io_lib:format("please use a space between \"~ts\" and the next \"~ts\"", [Token, [T]]) end, Message = io_lib:format("found \"~ts\" followed by \"~ts\", ~ts", [Token, [T], Warning]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Message, Scope); + prepend_warning(Line, Column, Message, Scope); maybe_warn_too_many_of_same_char(_Token, _Rest, _Line, _Column, Scope) -> Scope. @@ -1570,22 +1567,22 @@ maybe_warn_for_ambiguous_bang_before_equals(Kind, Unencoded, [$= | _], Line, Col "It is unclear if you mean \"~ts ~ts=\" or \"~ts =\". Please add " "a space before or after ~ts to remove the ambiguity", [What, Identifier, [Last], lists:droplast(Identifier), [Last], Identifier, [Last]]), - prepend_warning(Line, Column, Scope#elixir_tokenizer.file, Msg, Scope); + prepend_warning(Line, Column, Msg, Scope); _ -> Scope end; maybe_warn_for_ambiguous_bang_before_equals(_Kind, _Atom, _Rest, _Line, _Column, Scope) -> Scope. -prepend_warning(Line, Column, File, Msg, #elixir_tokenizer{warnings=Warnings} = Scope) -> - Scope#elixir_tokenizer{warnings = [{{Line, Column}, File, Msg} | Warnings]}. +prepend_warning(Line, Column, Msg, #elixir_tokenizer{warnings=Warnings} = Scope) -> + Scope#elixir_tokenizer{warnings = [{{Line, Column}, Msg} | Warnings]}. track_ascii(true, Scope) -> Scope; track_ascii(false, Scope) -> Scope#elixir_tokenizer{ascii_identifiers_only=false}. -maybe_unicode_lint_warnings(_Ascii=false, Tokens, File, Warnings) -> - 'Elixir.String.Tokenizer.Security':unicode_lint_warnings(lists:reverse(Tokens), File) ++ Warnings; -maybe_unicode_lint_warnings(_Ascii=true, _Tokens, _File, Warnings) -> +maybe_unicode_lint_warnings(_Ascii=false, Tokens, Warnings) -> + 'Elixir.String.Tokenizer.Security':unicode_lint_warnings(lists:reverse(Tokens)) ++ Warnings; +maybe_unicode_lint_warnings(_Ascii=true, _Tokens, Warnings) -> Warnings. error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) -> diff --git a/lib/elixir/test/elixir/code_formatter/general_test.exs b/lib/elixir/test/elixir/code_formatter/general_test.exs index 27da3ab7e6..1ec03db8aa 100644 --- a/lib/elixir/test/elixir/code_formatter/general_test.exs +++ b/lib/elixir/test/elixir/code_formatter/general_test.exs @@ -7,6 +7,10 @@ defmodule Code.Formatter.GeneralTest do @short_length [line_length: 10] + test "does not emit warnings" do + assert_format "fn -> end", "fn -> nil end" + end + describe "aliases" do test "with atom-only parts" do assert_same "Elixir" diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index b3eea97087..52f62c7d9d 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -131,7 +131,6 @@ defmodule Kernel.ParserTest do assert atom == "there_is_no_such_atom" assert meta[:line] == 1 assert meta[:column] == 1 - assert meta[:file] == "nofile" {:ok, {:my, "atom", ref}} end @@ -146,7 +145,6 @@ defmodule Kernel.ParserTest do assert atom == "there_is_no_such_var" assert meta[:line] == 1 assert meta[:column] == 1 - assert meta[:file] == "nofile" {:ok, {:my, "atom", ref}} end @@ -161,7 +159,6 @@ defmodule Kernel.ParserTest do assert atom == "there is no such key" assert meta[:line] == 1 assert meta[:column] == 2 - assert meta[:file] == "nofile" {:ok, {:my, "atom", ref}} end diff --git a/lib/elixir/test/erlang/string_test.erl b/lib/elixir/test/erlang/string_test.erl index c555b3dd9e..cc54ec11d1 100644 --- a/lib/elixir/test/erlang/string_test.erl +++ b/lib/elixir/test/erlang/string_test.erl @@ -8,8 +8,7 @@ eval(Content) -> {Value, Binding}. extract_interpolations(String) -> - Tokenizer = #elixir_tokenizer{file = <<"nofile">>}, - case elixir_interpolation:extract(1, 1, Tokenizer, true, String ++ [$"], $") of + case elixir_interpolation:extract(1, 1, #elixir_tokenizer{}, true, String ++ [$"], $") of {error, Error} -> Error; {_, _, Parts, _, _} -> @@ -23,8 +22,8 @@ extract_interpolations_without_interpolation_test() -> extract_interpolations_with_escaped_interpolation_test() -> ["f\\#{o}o"] = extract_interpolations("f\\#{o}o"), - {1, 8, ["f\\#{o}o"], [], _} = elixir_interpolation:extract(1, 2, - #elixir_tokenizer{file = <<"nofile">>}, true, "f\\#{o}o\"", $"). + {1, 8, ["f\\#{o}o"], [], _} = + elixir_interpolation:extract(1, 2, #elixir_tokenizer{}, true, "f\\#{o}o\"", $"). extract_interpolations_with_interpolation_test() -> ["f", diff --git a/lib/elixir/unicode/security.ex b/lib/elixir/unicode/security.ex index a5a543a262..e246ca575d 100644 --- a/lib/elixir/unicode/security.ex +++ b/lib/elixir/unicode/security.ex @@ -3,14 +3,14 @@ defmodule String.Tokenizer.Security do # UTS39 security checks that operate on all tokens in a file, # like Confusables. If we add whole-file mixed-script-confusables # checks we can add them to the list of lints here - def unicode_lint_warnings(tokens, file \\ "nofile") do + def unicode_lint_warnings(tokens) do for warning <- confusables(tokens), - do: format_warning(file, warning) + do: format_warning(warning) end - defp format_warning(file, {token, reason}) do + defp format_warning({token, reason}) do {_, {line, col, _}, _} = token - {{line, col}, file, to_charlist(reason)} + {{line, col}, to_charlist(reason)} end ## Confusables