Release v1.15.8

Resolves #13462

When a url string is schema less and the host is followed by a colon
without setting the actual port, `:uri_string.parse/1` returns the
port to be `:undefined`. As long as the url string is parseable,
we should translate `:undefined` to `nil`, in order to ensure we return
a valid URI struct.

CHANGELOG.md

@@ -117,6 +117,131 @@ in the long term.
 See the new `Logger` documentation for more information on the
 new features and on compatibility.
 
+## v1.15.8 (2024-05-21)
+
+### 1. Bug fixes
+
+#### Elixir
+
+  * [bin/elixir] Properly handle the `--dbg` flag in Elixir's CLI
+  * [System] Add a note that arguments are unsafe when invoking .bat/.com scripts on Windows via `System.cmd/3`
+  * [Port] Add a note that arguments are unsafe when invoking .bat/.com scripts on Windows
+  * [URI] Ensure `:undefined` fields are properly converted to `nil` when invoking Erlang's API
+
+#### Logger
+
+  * [Logger] Ensure translators are persisted across logger restarts
+
+#### Mix
+
+  * [mix compile] Ensure compile paths are accessible during compilation
+
+## v1.15.7 (2023-10-14)
+
+### 1. Enhancements
+
+#### Elixir
+
+  * [Elixir] Allow code evaluation across Elixir versions
+
+### 2. Bug fixes
+
+#### EEx
+
+  * [EEx] Do not emit duplicate warnings from tokenizer
+
+#### Mix
+
+  * [mix format] Correctly match file to subdirectory in `Mix.Tasks.Format.formatter_for_file/2`
+
+## v1.15.6 (2023-09-20)
+
+This release also includes fixes to the Windows installer.
+
+### 1. Bug fixes
+
+#### EEx
+
+  * [EEx] Do not crash when printing tokenizer warnings
+
+#### Elixir
+
+  * [Code] Fix formatter for nested `*` in bitstrings
+  * [Code] Improve feedback when an invalid block is given `Code.quoted_to_algebra/2`
+  * [Kernel] Trace functions before they are inlined
+
+#### Mix
+
+  * [mix compile] Ensure `:extra_applications` declare in umbrella projects are loaded
+  * [mix deps.get] Do not check for invalid applications before deps.get
+  * [mix deps.update] Do not check for invalid applications before deps.update
+  * [mix format] Load plugins when invoking the formatter from an IDE
+
+## v1.15.5 (2023-08-28)
+
+### 1. Enhancements
+
+#### IEx
+
+  * [IEx.Autocomplete] Speed up loading of struct suggestions
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [Code.Fragment] Fix `Code.Fragment.surround_context/2` for aliases and submodules of non-aliases
+  * [Kernel] Ensure stacktrace is included when necessary when rescuing multiple exceptions in the same branch
+  * [Kernel] Fix index in error message for unused optional arguments
+
+#### ExUnit
+
+  * [ExUnit.Diff] Fix scenario where diff would not show up due to a timed-out loop
+
+#### IEx
+
+  * [IEx] Force group leader to run as a binary and unicode in IEx
+
+#### Mix
+
+  * [mix compile] Do not assume `blake` is always available
+  * [mix format] Load and compile plugins if specified in subdirectories
+
+## v1.15.4 (2023-07-18)
+
+### 1. Bug fixes
+
+#### Mix
+
+  * [mix archive.build] Disable protocol consolidation when building archives on archive.install
+  * [mix compile] Track removed files per local dependency (this addresses a bug where files depending on modules from path dependencies always recompiled)
+  * [mix release] Do not strip relevant chunks from Erlang/OTP 26
+
+## v1.15.3 (2023-07-15)
+
+### 1. Enhancements
+
+#### Elixir
+
+  * [Kernel] Improve stacktraces when executing unnested Elixir code in a file
+
+#### Mix
+
+  * [Mix] Allow to opt-out of starting apps in `Mix.install/2`
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [Code] Ensure `with_diagnostics` propagate warnings from inner Erlang passes
+
+#### IEx
+
+  * [IEx] Fix `--remsh` on Erlang/OTP 25 and earlier
+
+#### Mix
+
+  * [mix compile.elixir] Ensure `__mix_recompile__?` callbacks are properly invoked
+
 ## v1.15.2 (2023-07-01)
 
 ### 1. Bug fixes

VERSION

@@ -1 +1 @@
-1.15.2
+1.15.8

bin/elixir

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 
-ELIXIR_VERSION=1.15.2
+ELIXIR_VERSION=1.15.8
 
 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then
   cat <<USAGE >&2
@@ -112,10 +112,10 @@ while [ $I -le $LENGTH ]; do
         C=1
         MODE="iex"
         ;;
-    -v|--no-halt|--dbg)
+    -v|--no-halt)
         C=1
         ;;
-    -e|-r|-pr|-pa|-pz|--eval|--remsh|--dot-iex)
+    -e|-r|-pr|-pa|-pz|--eval|--remsh|--dot-iex|--dbg)
         C=2
         ;;
     --rpc-eval)

bin/elixir.bat

@@ -1,6 +1,6 @@
 @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off)
 
-set ELIXIR_VERSION=1.15.2
+set ELIXIR_VERSION=1.15.8
 
 setlocal enabledelayedexpansion
 if    ""%1""==""""                if ""%2""=="""" goto documentation

lib/eex/lib/eex/compiler.ex

@@ -71,11 +71,9 @@ defmodule EEx.Compiler do
       {:ok, expr, new_line, new_column, rest} ->
         {key, expr} =
           case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do
-            {:ok, _line, _column, warnings, tokens} ->
-              Enum.each(Enum.reverse(warnings), fn {location, file, msg} ->
-                :elixir_errors.erl_warn(location, file, msg)
-              end)
-
+            {:ok, _line, _column, _warnings, tokens} ->
+              # We ignore warnings because the code will be tokenized
+              # again later with the right line+column info
               token_key(tokens, expr)
 
             {:error, _, _, _, _} ->

lib/eex/test/eex/tokenizer_test.exs

@@ -5,7 +5,7 @@ defmodule EEx.TokenizerTest do
 
   @opts [indentation: 0, trim: false]
 
-  test "simple chars lists" do
+  test "simple charlists" do
     assert EEx.tokenize(~c"foo", @opts) ==
              {:ok, [{:text, ~c"foo", %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]}
   end

lib/eex/test/eex_test.exs

@@ -463,17 +463,13 @@ defmodule EExTest do
       end
     end
 
-    test "when middle expression has a modifier" do
-      assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
-               EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>")
-             end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"]
-    end
+    test "when trying to use marker '|' without implementation" do
+      msg =
+        ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/
 
-    test "when end expression has a modifier" do
-      assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
-               EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>")
-             end) =~
-               ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"]
+      assert_raise EEx.SyntaxError, msg, fn ->
+        EEx.compile_string("<%| true %>")
+      end
     end
 
     test "when trying to use marker '/' without implementation" do
@@ -485,17 +481,6 @@ defmodule EExTest do
       end
     end
 
-    test "when trying to use marker '|' without implementation" do
-      msg =
-        ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/
-
-      assert_raise EEx.SyntaxError, msg, fn ->
-        EEx.compile_string("<%| true %>")
-      end
-    end
-  end
-
-  describe "error messages" do
     test "honor line numbers" do
       assert_raise EEx.SyntaxError,
                    "nofile:100:6: expected closing '%>' for EEx expression",
@@ -516,18 +501,30 @@ defmodule EExTest do
         EEx.compile_string("foo <%= bar", file: "my_file.eex")
       end
     end
+  end
 
-    test "when <%!-- is not closed" do
-      message = """
-      my_file.eex:1:5: expected closing '--%>' for EEx expression
-        |
-      1 | foo <%!-- bar
-        |     ^\
-      """
+  describe "warnings" do
+    test "when middle expression has a modifier" do
+      assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+               EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>")
+             end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"]
+    end
 
-      assert_raise EEx.SyntaxError, message, fn ->
-        EEx.compile_string("foo <%!-- bar", file: "my_file.eex")
-      end
+    test "when end expression has a modifier" do
+      assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+               EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>")
+             end) =~
+               ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"]
+    end
+
+    test "from tokenizer" do
+      warning =
+        ExUnit.CaptureIO.capture_io(:stderr, fn ->
+          EEx.compile_string(~s'<%= :"foo" %>', file: "tokenizer.ex")
+        end)
+
+      assert warning =~ "found quoted atom \"foo\" but the quotes are not required"
+      assert warning =~ "tokenizer.ex:1:5"
     end
   end
 

lib/elixir/lib/access.ex

@@ -191,9 +191,13 @@ defmodule Access do
         case __STACKTRACE__ do
           [unquote(top) | _] ->
             reason =
-              "#{inspect(unquote(module))} does not implement the Access behaviour. " <>
-                "If you are using get_in/put_in/update_in, you can specify the field " <>
-                "to be accessed using Access.key!/1"
+              """
+              #{inspect(unquote(module))} does not implement the Access behaviour
+
+              You can use the "struct.field" syntax to access struct fields. \
+              You can also use Access.key!/1 to access struct fields dynamically \
+              inside get_in/put_in/update_in\
+              """
 
             %{unquote(exception) | reason: reason}
 
@@ -326,9 +330,23 @@ defmodule Access do
     end
   end
 
+  def get(list, key, _default) when is_list(list) and is_integer(key) do
+    raise ArgumentError, """
+    the Access module does not support accessing lists by index, got: #{inspect(key)}
+
+    Accessing a list by index is typically discouraged in Elixir, \
+    instead we prefer to use the Enum module to manipulate lists \
+    as a whole. If you really must access a list element by index, \
+    you can Enum.at/1 or the functions in the List module\
+    """
+  end
+
   def get(list, key, _default) when is_list(list) do
-    raise ArgumentError,
-          "the Access calls for keywords expect the key to be an atom, got: " <> inspect(key)
+    raise ArgumentError, """
+    the Access module supports only keyword lists (with atom keys), got: #{inspect(key)}
+
+    If you want to search lists of tuples, use List.keyfind/3\
+    """
   end
 
   def get(nil, _key, default) do
@@ -377,10 +395,26 @@ defmodule Access do
     Map.get_and_update(map, key, fun)
   end
 
-  def get_and_update(list, key, fun) when is_list(list) do
+  def get_and_update(list, key, fun) when is_list(list) and is_atom(key) do
     Keyword.get_and_update(list, key, fun)
   end
 
+  def get_and_update(list, key, _fun) when is_list(list) and is_integer(key) do
+    raise ArgumentError, """
+    the Access module does not support accessing lists by index, got: #{inspect(key)}
+
+    Accessing a list by index is typically discouraged in Elixir, \
+    instead we prefer to use the Enum module to manipulate lists \
+    as a whole. If you really must mostify a list element by index, \
+    you can Access.at/1 or the functions in the List module\
+    """
+  end
+
+  def get_and_update(list, key, _fun) when is_list(list) do
+    raise ArgumentError,
+          "the Access module supports only keyword lists (with atom keys), got: " <> inspect(key)
+  end
+
   def get_and_update(nil, key, _fun) do
     raise ArgumentError, "could not put/update key #{inspect(key)} on a nil value"
   end
@@ -507,6 +541,21 @@ defmodule Access do
       iex> get_in(map, [Access.key!(:user), Access.key!(:unknown)])
       ** (KeyError) key :unknown not found in: %{name: \"john\"}
 
+  The examples above could be partially written as:
+
+      iex> map = %{user: %{name: "john"}}
+      iex> map.user.name
+      "john"
+      iex> get_and_update_in(map.user.name, fn prev ->
+      ...>   {prev, String.upcase(prev)}
+      ...> end)
+      {"john", %{user: %{name: "JOHN"}}}
+
+  However, it is not possible to remove fields using the dot notation,
+  as it is implified those fields must also be present. In any case,
+  `Access.key!/1` is useful when the key is not known in advance
+  and must be accessed dynamically.
+
   An error is raised if the accessed structure is not a map/struct:
 
       iex> get_in([], [Access.key!(:foo)])

lib/elixir/lib/code/formatter.ex

@@ -431,7 +431,7 @@ defmodule Code.Formatter do
     {color("nil", nil, state.inspect_opts), state}
   end
 
-  defp quoted_to_algebra({:__block__, meta, _} = block, _context, state) do
+  defp quoted_to_algebra({:__block__, meta, args} = block, _context, state) when is_list(args) do
     {block, state} = block_to_algebra(block, line(meta), closing_line(meta), state)
     {surround("(", block, ")"), state}
   end
@@ -1419,7 +1419,9 @@ defmodule Code.Formatter do
 
   defp bitstring_segment_to_algebra({{:"::", _, [segment, spec]}, i}, state, last) do
     {doc, state} = quoted_to_algebra(segment, :parens_arg, state)
-    {spec, state} = bitstring_spec_to_algebra(spec, state, state.normalize_bitstring_modifiers)
+
+    {spec, state} =
+      bitstring_spec_to_algebra(spec, state, state.normalize_bitstring_modifiers, :"::")
 
     spec = wrap_in_parens_if_inspected_atom(spec)
     spec = if i == last, do: bitstring_wrap_parens(spec, i, last), else: spec
@@ -1438,15 +1440,17 @@ defmodule Code.Formatter do
     {bitstring_wrap_parens(doc, i, last), state}
   end
 
-  defp bitstring_spec_to_algebra({op, _, [left, right]}, state, normalize_modifiers)
+  defp bitstring_spec_to_algebra({op, _, [left, right]}, state, normalize_modifiers, paren_op)
        when op in [:-, :*] do
     normalize_modifiers = normalize_modifiers && op != :*
-    {left, state} = bitstring_spec_to_algebra(left, state, normalize_modifiers)
+    {left, state} = bitstring_spec_to_algebra(left, state, normalize_modifiers, op)
     {right, state} = bitstring_spec_element_to_algebra(right, state, normalize_modifiers)
-    {concat(concat(left, Atom.to_string(op)), right), state}
+    doc = concat(concat(left, Atom.to_string(op)), right)
+    doc = if paren_op == :*, do: wrap_in_parens(doc), else: doc
+    {doc, state}
   end
 
-  defp bitstring_spec_to_algebra(spec, state, normalize_modifiers) do
+  defp bitstring_spec_to_algebra(spec, state, normalize_modifiers, _paren_op) do
     bitstring_spec_element_to_algebra(spec, state, normalize_modifiers)
   end
 

lib/elixir/lib/code/fragment.ex

@@ -677,6 +677,9 @@ defmodule Code.Fragment do
           {{:alias, acc}, offset} ->
             build_surround({:alias, acc}, reversed, line, offset)
 
+          {{:alias, parent, acc}, offset} ->
+            build_surround({:alias, parent, acc}, reversed, line, offset)
+
           {{:struct, acc}, offset} ->
             build_surround({:struct, acc}, reversed, line, offset)
 
@@ -753,7 +756,7 @@ defmodule Code.Fragment do
     end
   end
 
-  defp take_alias([h | t], acc) when h in ?A..?Z or h in ?a..?z or h in ?0..9 or h == ?_,
+  defp take_alias([h | t], acc) when h in ?A..?Z or h in ?a..?z or h in ?0..?9 or h == ?_,
     do: take_alias(t, [h | acc])
 
   defp take_alias(rest, acc) do

lib/elixir/lib/io.ex

@@ -593,11 +593,11 @@ defmodule IO do
 
   Another example where you might want to collect a user input
   every new line and break on an empty line, followed by removing
-  redundant new line characters (`"\n"`):
+  redundant new line characters (`"\\n"`):
 
       IO.stream(:stdio, :line)
-      |> Enum.take_while(&(&1 != "\n"))
-      |> Enum.map(&String.replace(&1, "\n", ""))
+      |> Enum.take_while(&(&1 != "\\n"))
+      |> Enum.map(&String.replace(&1, "\\n", ""))
 
   """
   @spec stream(device, :line | pos_integer) :: Enumerable.t()

lib/elixir/lib/kernel.ex

@@ -4903,17 +4903,6 @@ defmodule Kernel do
           {expanded, nil}
       end
 
-    # We do this so that the block is not tail-call optimized and stacktraces
-    # are not messed up. Basically, we just insert something between the return
-    # value of the block and what is returned by defmodule. Using just ":ok" or
-    # similar doesn't work because it's likely optimized away by the compiler.
-    block =
-      quote do
-        result = unquote(block)
-        :elixir_utils.noop()
-        result
-      end
-
     escaped =
       case env do
         %{function: nil, lexical_tracker: pid} when is_pid(pid) ->

lib/elixir/lib/port.ex

@@ -79,6 +79,27 @@ defmodule Port do
   are for advanced usage within the VM. Also consider using `System.cmd/3`
   if all you want is to execute a program and retrieve its return value.
 
+  > #### Windows argument splitting and untrusted arguments {: .warning}
+  >
+  > On Unix systems, arguments are passed to a new operating system
+  > process as an array of strings but on Windows it is up to the child
+  > process to parse them and some Windows programs may apply their own
+  > rules, which are inconsistent with the standard C runtime `argv` parsing
+  >
+  > This is particularly troublesome when invoking `.bat` or `.com` files
+  > as these run implicitly through `cmd.exe`, whose argument parsing is
+  > vulnerable to malicious input and can be used to run arbitrary shell
+  > commands.
+  >
+  > Therefore, if you are running on Windows and you execute batch
+  > files or `.com` applications, you must not pass untrusted input as
+  > arguments to the program. You may avoid accidentally executing them
+  > by explicitly passing the extension of the program you want to run,
+  > such as `.exe`, and double check the program is indeed not a batch
+  > file or `.com` application.
+  >
+  > This affects both `spawn` and `spawn_executable`.
+
   ### spawn
 
   The `:spawn` tuple receives a binary that is going to be executed as a

lib/elixir/lib/regex.ex

@@ -295,7 +295,7 @@ defmodule Regex do
   end
 
   @doc false
-  @deprecated "Use Kernel.is_struct/2 or pattern match on %Regex{} instead"
+  @deprecated "Use Kernel.is_struct(term, Regex) or pattern match on %Regex{} instead"
   def regex?(term)
   def regex?(%Regex{}), do: true
   def regex?(_), do: false

lib/elixir/lib/system.ex

@@ -996,6 +996,25 @@ defmodule System do
   `Port` module describes this problem and possible solutions under
   the "Zombie processes" section.
 
+  > #### Windows argument splitting and untrusted arguments {: .warning}
+  >
+  > On Unix systems, arguments are passed to a new operating system
+  > process as an array of strings but on Windows it is up to the child
+  > process to parse them and some Windows programs may apply their own
+  > rules, which are inconsistent with the standard C runtime `argv` parsing
+  >
+  > This is particularly troublesome when invoking `.bat` or `.com` files
+  > as these run implicitly through `cmd.exe`, whose argument parsing is
+  > vulnerable to malicious input and can be used to run arbitrary shell
+  > commands.
+  >
+  > Therefore, if you are running on Windows and you execute batch
+  > files or `.com` applications, you must not pass untrusted input as
+  > arguments to the program. You may avoid accidentally executing them
+  > by explicitly passing the extension of the program you want to run,
+  > such as `.exe`, and double check the program is indeed not a batch
+  > file or `.com` application.
+
   ## Examples
 
       iex> System.cmd("echo", ["hello"])

lib/elixir/lib/task/supervisor.ex

@@ -162,6 +162,7 @@ defmodule Task.Supervisor do
 
     * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown
       or an integer indicating the timeout value, defaults to 5000 milliseconds.
+      The tasks must trap exits for the timeout to have an effect.
 
   """
   @spec async(Supervisor.supervisor(), (-> any), Keyword.t()) :: Task.t()
@@ -183,6 +184,7 @@ defmodule Task.Supervisor do
 
     * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown
       or an integer indicating the timeout value, defaults to 5000 milliseconds.
+      The tasks must trap exits for the timeout to have an effect.
 
   """
   @spec async(Supervisor.supervisor(), module, atom, [term], Keyword.t()) :: Task.t()
@@ -208,6 +210,7 @@ defmodule Task.Supervisor do
 
     * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown
       or an integer indicating the timeout value, defaults to 5000 milliseconds.
+      The tasks must trap exits for the timeout to have an effect.
 
   ## Compatibility with OTP behaviours
 
@@ -337,6 +340,7 @@ defmodule Task.Supervisor do
 
     * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown
       or an integer indicating the timeout value. Defaults to `5000` milliseconds.
+      The tasks must trap exits for the timeout to have an effect.
 
   ## Examples
 
@@ -455,6 +459,7 @@ defmodule Task.Supervisor do
 
     * `:shutdown` - `:brutal_kill` if the task must be killed directly on shutdown
       or an integer indicating the timeout value, defaults to 5000 milliseconds.
+      The task must trap exits for the timeout to have an effect.
 
   """
   @spec start_child(Supervisor.supervisor(), (-> any), keyword) ::

lib/elixir/lib/uri.ex

@@ -648,16 +648,16 @@ defmodule URI do
         scheme = String.downcase(scheme, :ascii)
 
         case map do
-          %{port: port} when port != :undefined ->
+          %{port: port} when is_integer(port) ->
             %{uri | scheme: scheme}
 
           %{} ->
-            case default_port(scheme) do
-              nil -> %{uri | scheme: scheme}
-              port -> %{uri | scheme: scheme, port: port}
-            end
+            %{uri | scheme: scheme, port: default_port(scheme)}
         end
 
+      %{port: :undefined} ->
+        %{uri | port: nil}
+
       %{} ->
         uri
     end

lib/elixir/pages/compatibility-and-deprecations.md

@@ -43,7 +43,7 @@ Erlang/OTP versioning is independent from the versioning of Elixir. Erlang relea
 Elixir version | Supported Erlang/OTP versions
 :------------- | :-------------------------------
 1.15           | 24 - 26
-1.14           | 23 - 25
+1.14           | 23 - 25 (and Erlang/OTP 26 from v1.14.5)
 1.13           | 22 - 24 (and Erlang/OTP 25 from v1.13.4)
 1.12           | 22 - 24
 1.11           | 21 - 23 (and Erlang/OTP 24 from v1.11.4)
@@ -81,7 +81,7 @@ Version | Deprecated feature                                  | Replaced by (ava
 :-------| :-------------------------------------------------- | :---------------------------------------------------------------
 [v1.15] | `Calendar.ISO.day_of_week/3`                        | `Calendar.ISO.day_of_week/4` (v1.11)
 [v1.15] | `Exception.exception?/1`                            | `Kernel.is_exception/1` (v1.11)
-[v1.15] | `Regex.regex?/1`                                    | `Kernel.is_struct/2` (v1.11)
+[v1.15] | `Regex.regex?/1`                                    | `Kernel.is_struct/2` (`Kernel.is_struct(term, Regex)`) (v1.11)
 [v1.15] | `Logger.warn/2`                                     | `Logger.warning/2` (v1.11)
 [v1.14] | `use Bitwise`                                       | `import Bitwise` (v1.0)
 [v1.14] | `~~~/1`                                             | `bnot/2` (v1.0)

lib/elixir/scripts/windows_installer/installer.nsi

@@ -1,8 +1,6 @@
 !include "MUI2.nsh"
 !include "StrFunc.nsh"
-${Using:StrFunc} StrStr
 ${Using:StrFunc} UnStrStr
-${Using:StrFunc} UnStrRep
 
 Name "Elixir"
 ManifestDPIAware true
@@ -14,11 +12,18 @@ InstallDir "$PROGRAMFILES64\Elixir"
 
 Page custom CheckOTPPageShow CheckOTPPageLeave
 
-var Dialog
-var DownloadOTPLink
-var InstalledERTSVersion
 var InstalledOTPRelease
 var OTPPath
+
+var Dialog
+var NoOTPLabel
+var NoOTPLabelCreated
+var OTPMismatchLabel
+var OTPMismatchLabelCreated
+var DownloadOTPLink
+var DownloadOTPLinkCreated
+var VerifyOTPButton
+var VerifyOTPButtonCreated
 Function CheckOTPPageShow
   !insertmacro MUI_HEADER_TEXT "Checking Erlang/OTP" ""
 
@@ -29,16 +34,61 @@ Function CheckOTPPageShow
     Abort
   ${EndIf}
 
-  EnumRegKey $0 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" 0
-  ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$0" ""
-  StrCpy $InstalledERTSVersion $0
-  StrCpy $OTPPath $1
+  Call VerifyOTP
 
-  ${If} $1 == ""
-    ${NSD_CreateLabel} 0 0   100% 20u "Couldn't find existing Erlang/OTP installation. Click the link below to download and install it before proceeding."
-    ${NSD_CreateLink}  0 25u 100% 20u "Download Erlang/OTP ${OTP_RELEASE}"
+  nsDialogs::Show
+FunctionEnd
+
+Function VerifyOTP
+  ${If} $NoOTPLabelCreated == "true"
+    ShowWindow $NoOTPLabel ${SW_HIDE}
+  ${EndIf}
+
+  ${If} $OTPMismatchLabelCreated == "true"
+    ShowWindow $OTPMismatchLabel ${SW_HIDE}
+  ${EndIf}
+
+  ${If} $DownloadOTPLinkCreated == "true"
+    ShowWindow $DownloadOTPLink ${SW_HIDE}
+  ${Else}
+    StrCpy $DownloadOTPLinkCreated "true"
+    ${NSD_CreateLink}  0 60u 100% 20u "Download Erlang/OTP ${OTP_RELEASE}"
     Pop $DownloadOTPLink
     ${NSD_OnClick} $DownloadOTPLink OpenOTPDownloads
+    ShowWindow $DownloadOTPLink ${SW_HIDE}
+  ${EndIf}
+
+  ${If} $VerifyOTPButtonCreated == "true"
+    ShowWindow $VerifyOTPButton ${SW_HIDE}
+  ${Else}
+    StrCpy $VerifyOTPButtonCreated "true"
+    ${NSD_CreateButton} 0 80u 25% 12u "Verify Erlang/OTP"
+    Pop $VerifyOTPButton
+    ${NSD_OnClick} $VerifyOTPButton VerifyOTP
+    ShowWindow $VerifyOTPButton ${SW_HIDE}
+  ${EndIf}
+
+  StrCpy $0 0
+  loop:
+    EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0
+    StrCmp $1 "" done
+    ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" ""
+    StrCpy $OTPPath $1
+    IntOp $0 $0 + 1
+    goto loop
+  done:
+
+  ${If} $OTPPath == ""
+    ${If} $NoOTPLabelCreated != "true"
+      StrCpy $NoOTPLabelCreated "true"
+      ${NSD_CreateLabel} 0 0   100% 20u "Couldn't find existing Erlang/OTP installation. Click the link below to download and install it before proceeding."
+      Pop $NoOTPLabel
+    ${EndIf}
+
+    ShowWindow $NoOTPLabel ${SW_SHOW}
+    ShowWindow $DownloadOTPLink ${SW_SHOW}
+    ShowWindow $VerifyOTPButton ${SW_SHOW}
+
   ${Else}
     nsExec::ExecToStack `$OTPPath\bin\erl.exe -noinput -eval "\
     io:put_chars(erlang:system_info(otp_release)),\
@@ -49,24 +99,25 @@ Function CheckOTPPageShow
     ${If} $0 == 0
       StrCpy $InstalledOTPRelease $1
       ${If} $InstalledOTPRelease == ${OTP_RELEASE}
-        ${NSD_CreateLabel} 0 0 100% 20u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath. Please proceed."
-      ${ElseIf} $2 < ${OTP_RELEASE}
-        ${NSD_CreateLabel} 0 0 100% 30u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath but this Elixir installer was precompiled for Erlang/OTP ${OTP_RELEASE}. \
-        We recommend checking if there is an Elixir version precompiled for Erlang/OTP $InstalledOTPRelease. Otherwise, proceed."
+        ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath. Please proceed."
+
       ${Else}
-        SetErrorlevel 5
-        MessageBox MB_ICONSTOP "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath but this Elixir version was precompiled for Erlang/OTP ${OTP_RELEASE}. \
-        Please upgrade your Erlang/OTP version or choose an Elixir installer matching your Erlang/OTP version"
-        Quit
+        ${If} $OTPMismatchLabelCreated != "true"
+          StrCpy $OTPMismatchLabelCreated "true"
+          ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath but this Elixir installer was precompiled for Erlang/OTP ${OTP_RELEASE}. \
+          $\r$\n$\r$\nYou can either search for another Elixir installer precompiled for Erlang/OTP $InstalledOTPRelease or download Erlang/OTP ${OTP_RELEASE} and install before proceeding."
+          Pop $OTPMismatchLabel
+        ${EndIf}
+
+        ShowWindow $OTPMismatchLabel ${SW_SHOW}
+        ShowWindow $DownloadOTPLink  ${SW_SHOW}
+        ShowWindow $VerifyOTPButton  ${SW_SHOW}
       ${EndIf}
     ${Else}
       SetErrorlevel 5
       MessageBox MB_ICONSTOP "Found existing Erlang/OTP installation at $OTPPath but checking it exited with $0: $1"
-      Quit
     ${EndIf}
   ${EndIf}
-
-  nsDialogs::Show
 FunctionEnd
 
 Function OpenOTPDownloads
@@ -97,42 +148,42 @@ Function FinishPageShow
     Abort
   ${EndIf}
 
-  ${NSD_CreateCheckbox} 0 0 195u 10u "&Add $INSTDIR\bin to %PATH%"
-  Pop $AddElixirToPathCheckbox
-  SendMessage $AddElixirToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0
+  ; we add to PATH using erlang, so there must be an OTP installed to do so.
+  ${If} "$OTPPath" != ""
+    ${NSD_CreateCheckbox} 0 0 195u 10u "&Add $INSTDIR\bin to %PATH%"
+    Pop $AddElixirToPathCheckbox
+    SendMessage $AddElixirToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0
 
-  EnumRegKey $0 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" 0
-  ReadRegStr $0 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$0" ""
-  StrCpy $OTPPath $0
-  ${If} $0 != ""
-    ${NSD_CreateCheckbox} 0 20u 195u 10u "&Add $0\bin to %PATH%"
+    ${NSD_CreateCheckbox} 0 20u 195u 10u "&Add $OTPPath\bin to %PATH%"
     Pop $AddOTPToPathCheckbox
     SendMessage $AddOTPToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0
+
+    ${NSD_CreateLabel} 0 40u 100% 20u "Note: you need to restart your shell for the environment variable changes to take effect."
   ${EndIf}
 
   nsDialogs::Show
 FunctionEnd
 
+var PathsToAdd
 Function FinishPageLeave
   ${NSD_GetState} $AddOTPToPathCheckbox $0
   ${If} $0 <> ${BST_UNCHECKED}
-    ReadRegStr $0 HKCU "Environment" "Path"
-    ${StrStr} $1 "$0" "$OTPPath\bin"
-    ${If} $1 == ""
-      WriteRegExpandStr HKCU "Environment" "Path" "$OTPPath\bin;$0"
-    ${Else}
-      MessageBox MB_OK "$OTPPath\bin already in %PATH%"
-    ${EndIf}
+    StrCpy $PathsToAdd ";$OTPPath\bin"
   ${EndIf}
 
   ${NSD_GetState} $AddElixirToPathCheckbox $0
   ${If} $0 <> ${BST_UNCHECKED}
-    ReadRegStr $0 HKCU "Environment" "Path"
-    ${StrStr} $1 "$0" "$INSTDIR\bin"
-    ${If} $1 == ""
-      WriteRegExpandStr HKCU "Environment" "Path" "$INSTDIR\bin;$0"
-    ${Else}
-      MessageBox MB_OK "$INSTDIR\bin already in %PATH%"
+    StrCpy $PathsToAdd "$PathsToAdd;$INSTDIR\bin"
+  ${EndIf}
+
+  ${If} "$PathsToAdd" != ""
+    nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" add "$PathsToAdd"`
+    Pop $0
+    Pop $1
+    ${If} $0 != 0
+      SetErrorlevel 5
+      MessageBox MB_ICONSTOP "adding paths failed with $0: $1"
+      Quit
     ${EndIf}
   ${EndIf}
 FunctionEnd
@@ -140,25 +191,27 @@ FunctionEnd
 Section "Install Elixir" SectionElixir
   SetOutPath "$INSTDIR"
   File /r "${ELIXIR_DIR}\"
+  File "update_system_path.erl"
 
   WriteUninstaller "Uninstall.exe"
 SectionEnd
 
-; Uninstall Page: Files
-
-!insertmacro MUI_UNPAGE_DIRECTORY
-!insertmacro MUI_UNPAGE_INSTFILES
-
-Section "Uninstall"
-  RMDir /r "$INSTDIR"
-SectionEnd
-
-; Uninstall Page: Finish
+; Uninstall Page: Remove from %PATH%
 
 var RemoveOTPFromPathCheckbox
 var RemoveElixirFromPathCheckbox
 Function un.FinishPageShow
-  !insertmacro MUI_HEADER_TEXT "Finish Setup" ""
+  !insertmacro MUI_HEADER_TEXT "Remove from %PATH%" ""
+
+  StrCpy $0 0
+  loop:
+    EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0
+    StrCmp $1 "" done
+    ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" ""
+    StrCpy $OTPPath $1
+    IntOp $0 $0 + 1
+    goto loop
+  done:
 
   nsDialogs::Create 1018
   Pop $Dialog
@@ -168,6 +221,7 @@ Function un.FinishPageShow
   ${EndIf}
 
   ReadRegStr $0 HKCU "Environment" "Path"
+
   ${UnStrStr} $1 "$0" "$INSTDIR\bin"
   ${If} $1 != ""
     ${NSD_CreateCheckbox} 0 0 195u 10u "&Remove $INSTDIR\bin from %PATH%"
@@ -175,9 +229,6 @@ Function un.FinishPageShow
     SendMessage $RemoveElixirFromPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0
   ${EndIf}
 
-  EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" 0
-  ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" ""
-  StrCpy $OTPPath $1
   ${UnStrStr} $1 "$0" "$OTPPath\bin"
   ${If} $1 != ""
     ${NSD_CreateCheckbox} 0 20u 195u 10u "&Remove $OTPPath\bin from %PATH%"
@@ -188,22 +239,37 @@ Function un.FinishPageShow
   nsDialogs::Show
 FunctionEnd
 
+var PathsToRemove
 Function un.FinishPageLeave
-  ReadRegStr $0 HKCU "Environment" "Path"
-
   ${NSD_GetState} $RemoveOTPFromPathCheckbox $1
   ${If} $1 <> ${BST_UNCHECKED}
-    ${UnStrRep} $0 "$0" "$OTPPath\bin;" ""
+    StrCpy $PathsToRemove ";$OTPPath\bin"
   ${EndIf}
 
   ${NSD_GetState} $RemoveElixirFromPathCheckbox $1
   ${If} $1 <> ${BST_UNCHECKED}
-    ${UnStrRep} $0 "$0" "$INSTDIR\bin;" ""
+    StrCpy $PathsToRemove "$PathsToRemove;$INSTDIR\bin"
   ${EndIf}
 
-  WriteRegExpandStr HKCU "Environment" "Path" "$0"
+  ${If} "$PathsToRemove" != ""
+    nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" remove "$PathsToRemove"`
+    Pop $0
+    Pop $1
+    ${If} $0 != 0
+      SetErrorlevel 5
+      MessageBox MB_ICONSTOP "removing paths failed with $0: $1"
+      Quit
+    ${EndIf}
+  ${EndIf}
 FunctionEnd
 
 UninstPage custom un.FinishPageShow un.FinishPageLeave
 
+!insertmacro MUI_UNPAGE_DIRECTORY
+!insertmacro MUI_UNPAGE_INSTFILES
+
+Section "Uninstall"
+  RMDir /r "$INSTDIR"
+SectionEnd
+
 !insertmacro MUI_LANGUAGE "English"

lib/elixir/scripts/windows_installer/update_system_path.erl

@@ -0,0 +1,42 @@
+#!/usr/bin/env escript
+%%! -noinput
+
+%% This file is used by the Elixir installer and uninstaller.
+main(["add", ";" ++ PathsToAdd]) ->
+  {ok, Reg} = win32reg:open([read, write]),
+  ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"),
+  {ok, SystemPath} = win32reg:value(Reg, "path"),
+
+  NewSystemPath =
+    lists:foldl(
+      fun(Elem, Acc) ->
+        Elem ++ ";" ++
+          binary_to_list(
+            iolist_to_binary(
+              string:replace(Acc, Elem ++ ";", "", all)))
+      end,
+      SystemPath,
+      string:split(PathsToAdd, ";", all)
+    ),
+
+  ok = win32reg:set_value(Reg, "Path", NewSystemPath),
+  ok;
+
+main(["remove", ";" ++ PathsToRemove]) ->
+  {ok, Reg} = win32reg:open([read, write]),
+  ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"),
+  {ok, SystemPath} = win32reg:value(Reg, "path"),
+
+  NewSystemPath =
+    lists:foldl(
+      fun(Elem, Acc) ->
+        binary_to_list(
+          iolist_to_binary(
+            string:replace(Acc, Elem ++ ";", "", all)))
+      end,
+      SystemPath,
+      string:split(PathsToRemove, ";", all)
+    ),
+
+  ok = win32reg:set_value(Reg, "Path", NewSystemPath),
+  ok.

lib/elixir/src/elixir.erl

@@ -7,10 +7,11 @@
 -export([
   string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5,
   env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3,
-  eval_quoted/4
+  eval_quoted/4, eval_external_handler/3
 ]).
 -include("elixir.hrl").
 -define(system, 'Elixir.System').
+-define(elixir_eval_env, {elixir, eval_env}).
 
 %% Top level types
 %% TODO: Remove char_list type on v2.0
@@ -357,8 +358,25 @@ eval_forms(Tree, Binding, OrigE, Opts) ->
           _ -> [Erl]
         end,
 
-      ExternalHandler = eval_external_handler(NewE),
-      {value, Value, NewBinding} = erl_eval:exprs(Exprs, ErlBinding, none, ExternalHandler),
+      ExternalHandler = eval_external_handler(),
+
+      {value, Value, NewBinding} =
+        try
+          %% ?elixir_eval_env is used by the external handler.
+          %%
+          %% The reason why we use the process dictionary to pass the environment
+          %% is because we want to avoid passing closures to erl_eval, as that
+          %% would effectively tie the eval code to the Elixir version and it is
+          %% best if it depends solely on Erlang/OTP.
+          %%
+          %% The downside is that functions that escape the eval context will no
+          %% longer have the original environment they came from.
+          erlang:put(?elixir_eval_env, NewE),
+          erl_eval:exprs(Exprs, ErlBinding, none, ExternalHandler)
+        after
+          erlang:erase(?elixir_eval_env)
+        end,
+
       PruneBefore = if Prune -> length(Binding); true -> -1 end,
 
       {DumpedBinding, DumpedVars} =
@@ -369,52 +387,60 @@ eval_forms(Tree, Binding, OrigE, Opts) ->
 
 %% TODO: Remove conditional once we require Erlang/OTP 25+.
 -if(?OTP_RELEASE >= 25).
-eval_external_handler(Env) ->
-  Fun = fun(Ann, FunOrModFun, Args) ->
-    try
-      case FunOrModFun of
-        {Mod, Fun} -> apply(Mod, Fun, Args);
-        Fun -> apply(Fun, Args)
-      end
-    catch
-      Kind:Reason:Stacktrace ->
-        %% Take everything up to the Elixir module
-        Pruned =
-          lists:takewhile(fun
-            ({elixir,_,_,_}) -> false;
-            (_) -> true
-          end, Stacktrace),
+eval_external_handler() ->
+  {value, fun ?MODULE:eval_external_handler/3}.
 
-        Caller =
-          lists:dropwhile(fun
-            ({elixir,_,_,_}) -> false;
-            (_) -> true
-          end, Stacktrace),
-
-        %% Now we prune any shared code path from erl_eval
-        {current_stacktrace, Current} =
-          erlang:process_info(self(), current_stacktrace),
-
-        %% We need to make sure that we don't generate more
-        %% frames than supported. So we do our best to drop
-        %% from the Caller, but if the caller has no frames,
-        %% we need to drop from Pruned.
-        {DroppedCaller, ToDrop} =
-          case Caller of
-            [] -> {[], true};
-            _ -> {lists:droplast(Caller), false}
-          end,
-
-        Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop),
-        File = elixir_utils:characters_to_list(?key(Env, file)),
-        Location = [{file, File}, {line, erl_anno:line(Ann)}],
-
-        %% Add file+line information at the bottom
-        Custom = lists:reverse([{elixir_eval, '__FILE__', 1, Location} | Reversed], DroppedCaller),
-        erlang:raise(Kind, Reason, Custom)
+eval_external_handler(Ann, FunOrModFun, Args) ->
+  try
+    case FunOrModFun of
+      {Mod, Fun} -> apply(Mod, Fun, Args);
+      Fun -> apply(Fun, Args)
     end
-  end,
-  {value, Fun}.
+  catch
+    Kind:Reason:Stacktrace ->
+      %% Take everything up to the Elixir module
+      Pruned =
+        lists:takewhile(fun
+          ({elixir,_,_,_}) -> false;
+          (_) -> true
+        end, Stacktrace),
+
+      Caller =
+        lists:dropwhile(fun
+          ({elixir,_,_,_}) -> false;
+          (_) -> true
+        end, Stacktrace),
+
+      %% Now we prune any shared code path from erl_eval
+      {current_stacktrace, Current} =
+        erlang:process_info(self(), current_stacktrace),
+
+      %% We need to make sure that we don't generate more
+      %% frames than supported. So we do our best to drop
+      %% from the Caller, but if the caller has no frames,
+      %% we need to drop from Pruned.
+      {DroppedCaller, ToDrop} =
+        case Caller of
+          [] -> {[], true};
+          _ -> {lists:droplast(Caller), false}
+        end,
+
+      Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop),
+
+      %% Add file+line information at the bottom
+      Bottom =
+        case erlang:get(?elixir_eval_env) of
+          #{file := File} ->
+            [{elixir_eval, '__FILE__', 1,
+             [{file, elixir_utils:characters_to_list(File)}, {line, erl_anno:line(Ann)}]}];
+
+          _ ->
+            []
+        end,
+
+      Custom = lists:reverse(Bottom ++ Reversed, DroppedCaller),
+      erlang:raise(Kind, Reason, Custom)
+  end.
 
 %% We need to check if we have dropped any frames.
 %% If we have not dropped frames, then we need to drop one
@@ -427,8 +453,8 @@ drop_common([], [{?MODULE, _, _, _} | T2], _ToDrop) -> T2;
 drop_common([], [_ | T2], true) -> T2;
 drop_common([], T2, _) -> T2.
 -else.
-eval_external_handler(_Env) ->
-  none.
+eval_external_handler() -> none.
+eval_external_handler(_Ann, _FunOrModFun, _Args) -> error(unused).
 -endif.
 
 %% Converts a quoted expression to Erlang abstract format

lib/elixir/src/elixir_compiler.erl

@@ -16,7 +16,7 @@ quoted(Forms, File, Callback) ->
 
     elixir_lexical:run(
       Env,
-      fun (LexicalEnv) -> eval_or_compile(Forms, [], LexicalEnv) end,
+      fun (LexicalEnv) -> maybe_fast_compile(Forms, [], LexicalEnv) end,
       fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end
     ),
 
@@ -32,7 +32,7 @@ file(File, Callback) ->
 %% Evaluates the given code through the Erlang compiler.
 %% It may end-up evaluating the code if it is deemed a
 %% more efficient strategy depending on the code snippet.
-eval_or_compile(Forms, Args, E) ->
+maybe_fast_compile(Forms, Args, E) ->
   case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso
         (not elixir_config:is_bootstrap()) of
     true  -> fast_compile(Forms, E);
@@ -40,8 +40,9 @@ eval_or_compile(Forms, Args, E) ->
   end,
   ok.
 
-compile(Quoted, ArgsList, E) ->
-  {Expanded, SE, EE} = elixir_expand:expand(Quoted, elixir_env:env_to_ex(E), E),
+compile(Quoted, ArgsList, #{line := Line} = E) ->
+  Block = no_tail_optimize([{line, Line}], Quoted),
+  {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E),
   elixir_env:check_unused_vars(SE, EE),
 
   {Module, Fun, Purgeable} =
@@ -55,7 +56,7 @@ spawned_compile(ExExprs, #{line := Line, file := File} = E) ->
   {ErlExprs, _} = elixir_erl_pass:translate(ExExprs, erl_anno:new(Line), S),
 
   Module = retrieve_compiler_module(),
-  Fun  = code_fun(?key(E, module)),
+  Fun = code_fun(?key(E, module)),
   Forms = code_mod(Fun, ErlExprs, Line, File, Module, Vars),
 
   {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch, no_bool_opt, no_ssa_opt]),
@@ -106,15 +107,9 @@ allows_fast_compilation(_) ->
 
 fast_compile({'__block__', _, Exprs}, E) ->
   lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs);
-fast_compile({defmodule, Meta, [Mod, [{do, TailBlock}]]}, NoLineE) ->
+fast_compile({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) ->
   E = NoLineE#{line := ?line(Meta)},
 
-  Block = {'__block__', Meta, [
-    {'=', Meta, [{result, Meta, ?MODULE}, TailBlock]},
-    {{'.', Meta, [elixir_utils, noop]}, Meta, []},
-    {result, Meta, ?MODULE}
-  ]},
-
   Expanded = case Mod of
     {'__aliases__', _, _} ->
       case elixir_aliases:expand_or_concat(Mod, E) of
@@ -129,6 +124,13 @@ fast_compile({defmodule, Meta, [Mod, [{do, TailBlock}]]}, NoLineE) ->
   ContextModules = [Expanded | ?key(E, context_modules)],
   elixir_module:compile(Expanded, Block, [], false, E#{context_modules := ContextModules}).
 
+no_tail_optimize(Meta, Block) ->
+  {'__block__', Meta, [
+    {'=', Meta, [{result, Meta, ?MODULE}, Block]},
+    {{'.', Meta, [elixir_utils, noop]}, Meta, []},
+    {result, Meta, ?MODULE}
+  ]}.
+
 %% Bootstrapper
 
 bootstrap() ->

lib/elixir/src/elixir_dispatch.erl

@@ -130,11 +130,15 @@ dispatch_require(Meta, Receiver, Name, Args, S, E, Callback) when is_atom(Receiv
 
   case elixir_rewrite:inline(Receiver, Name, Arity) of
     {AR, AN} ->
+      elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E),
       Callback(AR, AN, Args);
     false ->
       case expand_require(Meta, Receiver, {Name, Arity}, Args, S, E) of
-        {ok, Receiver, Quoted} -> expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E);
-        error -> Callback(Receiver, Name, Args)
+        {ok, Receiver, Quoted} ->
+          expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E);
+        error ->
+          elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E),
+          Callback(Receiver, Name, Args)
       end
   end;
 

lib/elixir/src/elixir_erl_compiler.erl

@@ -5,25 +5,42 @@
 spawn(Fun) ->
   CompilerInfo = get(elixir_compiler_info),
 
+  CodeDiagnostics =
+    case get(elixir_code_diagnostics) of
+      undefined -> undefined;
+      {_Tail, Log} -> {[], Log}
+    end,
+
   {_, Ref} =
     spawn_monitor(fun() ->
       put(elixir_compiler_info, CompilerInfo),
+      put(elixir_code_diagnostics, CodeDiagnostics),
 
       try Fun() of
-        Result -> exit({ok, Result})
+        Result -> exit({ok, Result, get(elixir_code_diagnostics)})
       catch
         Kind:Reason:Stack ->
-          exit({Kind, Reason, Stack})
+          exit({Kind, Reason, Stack, get(elixir_code_diagnostics)})
       end
     end),
 
   receive
-    {'DOWN', Ref, process, _, {ok, Result}} ->
+    {'DOWN', Ref, process, _, {ok, Result, Diagnostics}} ->
+      copy_diagnostics(Diagnostics),
       Result;
-    {'DOWN', Ref, process, _, {Kind, Reason, Stack}} ->
+    {'DOWN', Ref, process, _, {Kind, Reason, Stack, Diagnostics}} ->
+      copy_diagnostics(Diagnostics),
       erlang:raise(Kind, Reason, Stack)
   end.
 
+copy_diagnostics(undefined) ->
+  ok;
+copy_diagnostics({Head, _}) ->
+  case get(elixir_code_diagnostics) of
+    undefined -> ok;
+    {Tail, Log} -> put(elixir_code_diagnostics, {Head ++ Tail, Log})
+  end.
+
 forms(Forms, File, Opts) ->
   compile(Forms, File, Opts ++ compile:env_compiler_options()).
 

lib/elixir/src/elixir_erl_try.erl

@@ -55,7 +55,7 @@ normalize_rescue(Meta, Var, Pattern, Expr, ErlangAliases) ->
         dynamic_normalize(Meta, Var, ?REQUIRES_STACKTRACE);
 
       false ->
-        case lists:splitwith(fun is_normalized_with_stacktrace/1, ErlangAliases) of
+        case lists:partition(fun is_normalized_with_stacktrace/1, ErlangAliases) of
           {[], _} -> [];
           {_, []} -> {'__STACKTRACE__', Meta, nil};
           {Some, _} -> dynamic_normalize(Meta, Var, Some)

lib/elixir/src/elixir_expand.erl

@@ -856,10 +856,6 @@ expand_remote(Receiver, DotMeta, Right, Meta, Args, S, SL, #{context := Context}
 
     _ ->
       AttachedDotMeta = attach_context_module(Receiver, DotMeta, E),
-
-      is_atom(Receiver) andalso
-        elixir_env:trace({remote_function, Meta, Receiver, Right, length(Args)}, E),
-
       {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {SL, S}, E, Args),
 
       case rewrite(Context, Receiver, AttachedDotMeta, Right, Meta, EArgs, S) of

lib/elixir/src/elixir_locals.erl

@@ -118,11 +118,11 @@ format_error({function_conflict, {Receiver, {Name, Arity}}}) ->
 format_error({unused_args, {Name, Arity}}) ->
   io_lib:format("default values for the optional arguments in ~ts/~B are never used", [Name, Arity]);
 
-format_error({unused_args, {Name, Arity}, 1}) ->
-  io_lib:format("the default value for the first optional argument in ~ts/~B is never used", [Name, Arity]);
+format_error({unused_args, {Name, Arity}, Count}) when Arity - Count == 1 ->
+  io_lib:format("the default value for the last optional argument in ~ts/~B is never used", [Name, Arity]);
 
 format_error({unused_args, {Name, Arity}, Count}) ->
-  io_lib:format("the default values for the first ~B optional arguments in ~ts/~B are never used", [Count, Name, Arity]);
+  io_lib:format("the default values for the last ~B optional arguments in ~ts/~B are never used", [Arity - Count, Name, Arity]);
 
 format_error({unused_def, {Name, Arity}, defp}) ->
   io_lib:format("function ~ts/~B is unused", [Name, Arity]);

lib/elixir/src/elixir_module.erl

@@ -508,19 +508,17 @@ beam_location(ModuleAsCharlist) ->
 checker_info() ->
   case get(elixir_checker_info) of
     undefined -> undefined;
-    _ ->
-      Log =
-        case erlang:get(elixir_code_diagnostics) of
-          {_, false} -> false;
-          _ -> true
-        end,
-
-      {'Elixir.Module.ParallelChecker':get(), Log}
+    _ -> 'Elixir.Module.ParallelChecker':get()
   end.
 
 spawn_parallel_checker(undefined, _Module, _ModuleMap) ->
-  nil;
-spawn_parallel_checker({CheckerInfo, Log}, Module, ModuleMap) ->
+  ok;
+spawn_parallel_checker(CheckerInfo, Module, ModuleMap) ->
+  Log =
+    case erlang:get(elixir_code_diagnostics) of
+      {_, false} -> false;
+      _ -> true
+    end,
   'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, Log).
 
 make_module_available(Module, Binary) ->

lib/elixir/test/elixir/code_formatter/containers_test.exs

@@ -275,6 +275,8 @@ defmodule Code.Formatter.ContainersTest do
       assert_format "<< 1 :: 2 - unit(3) >>", "<<1::2-unit(3)>>"
       assert_format "<< 1 :: 2 * 3 - unit(4) >>", "<<1::2*3-unit(4)>>"
       assert_format "<< 1 :: 2 - unit(3) - 4 / 5 >>", "<<1::2-unit(3)-(4 / 5)>>"
+      assert_format "<<0 :: ( x - 1 ) * 5>>", "<<0::(x-1)*5>>"
+      assert_format "<<0 :: 2 * 3 * 4>>", "<<0::(2*3)*4>>"
     end
 
     test "in comprehensions" do

lib/elixir/test/elixir/code_fragment_test.exs

@@ -738,6 +738,14 @@ defmodule CodeFragmentTest do
                  end: {3, 5}
                }
       end
+
+      for i <- 1..11 do
+        assert CF.surround_context("Foo.Bar.Baz.foo(bar)", {1, i}) == %{
+                 context: {:alias, ~c"Foo.Bar.Baz"},
+                 begin: {1, 1},
+                 end: {1, 12}
+               }
+      end
     end
 
     test "underscored special forms" do
@@ -747,17 +755,21 @@ defmodule CodeFragmentTest do
                end: {1, 11}
              }
 
-      assert CF.surround_context("__MODULE__.Foo", {1, 12}) == %{
-               context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"},
-               begin: {1, 1},
-               end: {1, 15}
-             }
+      for i <- 1..15 do
+        assert CF.surround_context("__MODULE__.Foo", {1, i}) == %{
+                 context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"},
+                 begin: {1, 1},
+                 end: {1, 15}
+               }
+      end
 
-      assert CF.surround_context("__MODULE__.Foo.Sub", {1, 16}) == %{
-               context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"},
-               begin: {1, 1},
-               end: {1, 19}
-             }
+      for i <- 1..19 do
+        assert CF.surround_context("__MODULE__.Foo.Sub", {1, i}) == %{
+                 context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"},
+                 begin: {1, 1},
+                 end: {1, 19}
+               }
+      end
 
       assert CF.surround_context("%__MODULE__{}", {1, 5}) == %{
                context: {:struct, {:local_or_var, ~c"__MODULE__"}},
@@ -803,17 +815,21 @@ defmodule CodeFragmentTest do
     end
 
     test "attribute submodules" do
-      assert CF.surround_context("@some.Foo", {1, 8}) == %{
-               context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"},
-               begin: {1, 1},
-               end: {1, 10}
-             }
+      for i <- 1..10 do
+        assert CF.surround_context("@some.Foo", {1, i}) == %{
+                 context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"},
+                 begin: {1, 1},
+                 end: {1, 10}
+               }
+      end
 
-      assert CF.surround_context("@some.Foo.Sub", {1, 12}) == %{
-               context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"},
-               begin: {1, 1},
-               end: {1, 14}
-             }
+      for i <- 1..14 do
+        assert CF.surround_context("@some.Foo.Sub", {1, i}) == %{
+                 context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"},
+                 begin: {1, 1},
+                 end: {1, 14}
+               }
+      end
 
       assert CF.surround_context("%@some{}", {1, 5}) == %{
                context: {:struct, {:module_attribute, ~c"some"}},

lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs

@@ -226,6 +226,11 @@ defmodule Code.Normalizer.QuotedASTTest do
       assert quoted_to_string(quoted) <> "\n" == expected
     end
 
+    test "invalid block" do
+      assert quoted_to_string({:__block__, [], {:bar, [], []}}) ==
+               "{:__block__, [], {:bar, [], []}}"
+    end
+
     test "not in" do
       assert quoted_to_string(quote(do: false not in [])) == "false not in []"
     end

lib/elixir/test/elixir/code_test.exs

@@ -90,6 +90,28 @@ defmodule CodeTest do
                  Code.eval_quoted(quoted, [])
                end)
     end
+
+    test "captures unknown local calls" do
+      sample = """
+      defmodule CodeTest.UnknownLocalCall do
+        def perform do
+          foo()
+        end
+      end
+      """
+
+      assert {:rescued, [%{message: message}]} =
+               Code.with_diagnostics(fn ->
+                 try do
+                   quoted = Code.string_to_quoted!(sample, columns: true)
+                   Code.eval_quoted(quoted, [])
+                 rescue
+                   _ -> :rescued
+                 end
+               end)
+
+      assert message =~ "undefined function foo/0"
+    end
   end
 
   describe "eval_string/1,2,3" do
@@ -325,7 +347,12 @@ defmodule CodeTest do
       assert env.versioned_vars == %{}
 
       assert_receive {:trace, {:on_module, _, _}, %{module: CodeTest.TracingPruning} = trace_env}
-      assert trace_env.versioned_vars == %{{:result, Kernel} => 5, {:x, nil} => 1, {:y, nil} => 4}
+
+      assert trace_env.versioned_vars == %{
+               {:result, :elixir_compiler} => 5,
+               {:x, nil} => 1,
+               {:y, nil} => 4
+             }
     end
 
     test "with defguard" do
@@ -474,6 +501,14 @@ defmodule CodeTest do
       :code.purge(CompileCrossSample)
       :code.delete(CompileCrossSample)
     end
+
+    test "disables tail call optimization at the root" do
+      try do
+        Code.compile_string("List.flatten(123)")
+      rescue
+        _ -> assert Enum.any?(__STACKTRACE__, &match?({_, :__FILE__, 1, _}, &1))
+      end
+    end
   end
 
   test "format_string/2 returns empty iodata for empty string" do

lib/elixir/test/elixir/kernel/raise_test.exs

@@ -224,6 +224,17 @@ defmodule Kernel.RaiseTest do
       assert result == "an exception"
     end
 
+    test "named function clause (stacktrace) or runtime (no stacktrace) error" do
+      result =
+        try do
+          Access.get("foo", 0)
+        rescue
+          x in [FunctionClauseError, CaseClauseError] -> Exception.message(x)
+        end
+
+      assert result == "no function clause matching in Access.get/3"
+    end
+
     test "with higher precedence than catch" do
       result =
         try do

lib/elixir/test/elixir/kernel/tracers_test.exs

@@ -118,6 +118,7 @@ defmodule Kernel.TracersTest do
     require Integer
     true = Integer.is_odd(1)
     {1, ""} = Integer.parse("1")
+    "foo" = Atom.to_string(:foo)
     """)
 
     assert_receive {{:remote_macro, meta, Integer, :is_odd, 1}, _}
@@ -127,6 +128,10 @@ defmodule Kernel.TracersTest do
     assert_receive {{:remote_function, meta, Integer, :parse, 1}, _}
     assert meta[:line] == 3
     assert meta[:column] == 19
+
+    assert_receive {{:remote_function, meta, Atom, :to_string, 1}, _}
+    assert meta[:line] == 4
+    assert meta[:column] == 14
   end
 
   test "traces remote via captures" do

lib/elixir/test/elixir/kernel/warning_test.exs

@@ -563,17 +563,17 @@ defmodule Kernel.WarningTest do
              end
              """)
            end) =~
-             "the default values for the first 2 optional arguments in b/3 are never used\n  nofile:3"
+             "the default value for the last optional argument in b/3 is never used\n  nofile:3"
 
     assert capture_err(fn ->
              Code.eval_string(~S"""
              defmodule Sample3 do
-               def a, do: b(1)
-               defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3]
+               def a, do: b(1, 2)
+               defp b(arg1, arg2 \\ 2, arg3 \\ 3, arg4 \\ 4), do: [arg1, arg2, arg3, arg4]
              end
              """)
            end) =~
-             "the default value for the first optional argument in b/3 is never used\n  nofile:3"
+             "the default values for the last 2 optional arguments in b/4 are never used"
 
     assert capture_err(fn ->
              Code.eval_string(~S"""
@@ -587,17 +587,6 @@ defmodule Kernel.WarningTest do
     assert capture_err(fn ->
              Code.eval_string(~S"""
              defmodule Sample5 do
-               def a, do: b(1, 2, 3)
-               defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3)
-
-               defp b(arg1, arg2, arg3), do: [arg1, arg2, arg3]
-             end
-             """)
-           end) =~ "default values for the optional arguments in b/3 are never used\n  nofile:3"
-
-    assert capture_err(fn ->
-             Code.eval_string(~S"""
-             defmodule Sample6 do
                def a, do: b(1, 2)
                defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3)
 
@@ -605,9 +594,9 @@ defmodule Kernel.WarningTest do
              end
              """)
            end) =~
-             "the default values for the first 2 optional arguments in b/3 are never used\n  nofile:3"
+             "the default value for the last optional argument in b/3 is never used\n  nofile:3"
   after
-    purge([Sample1, Sample2, Sample3, Sample4, Sample5, Sample6])
+    purge([Sample1, Sample2, Sample3, Sample4, Sample5])
   end
 
   test "unused import" do

lib/elixir/test/elixir/uri_test.exs

@@ -277,6 +277,32 @@ defmodule URITest do
     test "preserves an empty query" do
       assert URI.new!("http://foo.com/?").query == ""
     end
+
+    test "without scheme, undefined port after host translates to nil" do
+      assert URI.new!("//https://www.example.com") ==
+               %URI{
+                 scheme: nil,
+                 userinfo: nil,
+                 host: "https",
+                 port: nil,
+                 path: "//www.example.com",
+                 query: nil,
+                 fragment: nil
+               }
+    end
+
+    test "with scheme, undefined port after host translates to nil" do
+      assert URI.new!("myscheme://myhost:/path/info") ==
+               %URI{
+                 scheme: "myscheme",
+                 userinfo: nil,
+                 host: "myhost",
+                 port: nil,
+                 path: "/path/info",
+                 query: nil,
+                 fragment: nil
+               }
+    end
   end
 
   test "http://http://http://@http://http://?http://#http://" do

lib/ex_unit/lib/ex_unit/callbacks.ex

@@ -549,14 +549,7 @@ defmodule ExUnit.Callbacks do
       end
 
     child_spec = Supervisor.child_spec(child_spec_or_module, opts)
-
-    case Supervisor.start_child(sup, child_spec) do
-      {:error, {:already_started, _pid}} ->
-        {:error, {:duplicate_child_name, child_spec.id}}
-
-      other ->
-        other
-    end
+    Supervisor.start_child(sup, child_spec)
   end
 
   @doc """

lib/ex_unit/lib/ex_unit/case_template.ex

@@ -130,7 +130,7 @@ defmodule ExUnit.CaseTemplate do
   The second argument passed to `use MyCase` gets forwarded to `using/2` too:
 
       defmodule SomeTestCase do
-        use MyCase, async: true, import_helpers: true, async: true
+        use MyCase, async: true, import_helpers: true
 
         test "the truth" do
           # truth/0 comes from MyApp.TestHelpers:

lib/ex_unit/lib/ex_unit/diff.ex

@@ -200,11 +200,15 @@ defmodule ExUnit.Diff do
 
       {diff, env}
     else
-      diff_value(left, right, env)
+      non_recursive_diff_value(left, right, env)
     end
   end
 
   defp diff_value(left, right, env) do
+    non_recursive_diff_value(left, right, env)
+  end
+
+  defp non_recursive_diff_value(left, right, env) do
     diff_left = escape(left) |> update_diff_meta(true)
     diff_right = escape(right) |> update_diff_meta(true)
     diff = %__MODULE__{equivalent?: false, left: diff_left, right: diff_right}

lib/ex_unit/test/ex_unit/diff_test.exs

@@ -1121,6 +1121,8 @@ defmodule ExUnit.DiffTest do
     )
   end
 
+  @compile {:no_warn_undefined, String}
+
   test "functions" do
     identity = & &1
     inspect = inspect(identity)
@@ -1130,11 +1132,15 @@ defmodule ExUnit.DiffTest do
 
     refute_diff(identity == :a, "-#{inspect}-", "+:a+")
     refute_diff({identity, identity} == :a, "-{#{inspect}, #{inspect}}", "+:a+")
-
     refute_diff({identity, :a} == {:a, identity}, "{-#{inspect}-, -:a-}", "{+:a+, +#{inspect}+}")
-
     refute_diff(%{identity => identity} == :a, "-%{#{inspect} => #{inspect}}", "+:a+")
 
+    refute_diff(
+      (&String.to_charlist/1) == (&String.unknown/1),
+      "-&String.to_charlist/1-",
+      "+&String.unknown/1"
+    )
+
     refute_diff(
       %Opaque{data: identity} == :a,
       "-#Opaque<???>-",

lib/ex_unit/test/ex_unit/formatter_test.exs

@@ -491,28 +491,25 @@ defmodule ExUnit.FormatterTest do
   test "inspect failure" do
     failure = [{:error, catch_assertion(assert :will_fail == %BadInspect{}), []}]
 
-    message = ~S'''
-      got FunctionClauseError with message:
-
-          """
-          no function clause matching in Inspect.ExUnit.FormatterTest.BadInspect.inspect/2
-          """
-
-      while inspecting:
-
-          %{__struct__: ExUnit.FormatterTest.BadInspect, key: 0}
-
-      Stacktrace:
-    '''
-
-    assert format_test_failure(test(), failure, 1, 80, &formatter/2) =~ """
+    assert format_test_failure(test(), failure, 1, 80, &formatter/2) =~ ~s'''
              1) world (Hello)
                 test/ex_unit/formatter_test.exs:1
                 Assertion with == failed
                 code:  assert :will_fail == %BadInspect{}
                 left:  :will_fail
-                right: #Inspect.Error<\n#{message}\
-           """
+                right: #Inspect.Error<
+             got FunctionClauseError with message:
+
+                 """
+                 no function clause matching in Inspect.ExUnit.FormatterTest.BadInspect.inspect/2
+                 """
+
+             while inspecting:
+
+                 #{inspect(%BadInspect{}, structs: false)}
+
+             Stacktrace:
+           '''
   end
 
   defmodule BadMessage do

lib/ex_unit/test/ex_unit/supervised_test.exs

@@ -73,19 +73,14 @@ defmodule ExUnit.SupervisedTest do
 
   test "starts a supervised process with ID checks" do
     {:ok, pid} = start_supervised({MyAgent, 0})
+    assert is_pid(pid)
 
-    assert {:error, {:duplicate_child_name, ExUnit.SupervisedTest.MyAgent}} =
-             start_supervised({MyAgent, 0})
-
-    assert {:error, {{:already_started, ^pid}, _}} = start_supervised({MyAgent, 0}, id: :another)
+    assert {:error, _} = start_supervised({MyAgent, 0})
+    assert {:error, _} = start_supervised({MyAgent, 0}, id: :another)
 
     assert_raise RuntimeError, ~r"Reason: bad child specification", fn ->
       start_supervised!(%{id: 1, start: :oops})
     end
-
-    assert_raise RuntimeError, ~r"Reason: already started", fn ->
-      start_supervised!({MyAgent, 0}, id: :another)
-    end
   end
 
   test "stops a supervised process" do

lib/iex/lib/iex.ex

@@ -582,9 +582,6 @@ defmodule IEx do
   @doc """
   Pries into the process environment.
 
-  When you start `iex`, IEx will set this function to be the
-  default `dbg/2` backend unless the `--no-pry` flag is given.
-
   This function is useful for debugging a particular chunk of code
   when executed by a particular process. The process becomes
   the evaluator of IEx commands and is temporarily changed to
@@ -600,6 +597,11 @@ defmodule IEx do
 
   See also `break!/4` for others ways to pry.
 
+  > #### `dbg/0` integration
+  >
+  > By calling `iex --dbg pry`, `iex` will set this function
+  > as the default backend for `dbg/0` calls.
+
   ## Examples
 
   Let's suppose you want to investigate what is happening
@@ -869,6 +871,7 @@ defmodule IEx do
     if Code.ensure_loaded?(:prim_tty) do
       spawn(fn ->
         {:ok, _} = Application.ensure_all_started(:iex)
+        :ok = :io.setopts(binary: true, encoding: :unicode)
         _ = for fun <- Enum.reverse(after_spawn()), do: fun.()
         IEx.Server.run([register: false] ++ opts)
       end)
@@ -928,13 +931,18 @@ defmodule IEx do
         mfa
       end
 
-    :ok = :shell.start_interactive(shell)
+    case :shell.start_interactive(shell) do
+      :ok ->
+        receive do
+          {^ref, shell} -> shell
+        after
+          15_000 ->
+            IO.puts(:stderr, "Could not start IEx CLI due to reason: :boot_timeout")
+            System.halt(1)
+        end
 
-    receive do
-      {^ref, shell} -> shell
-    after
-      15_000 ->
-        IO.puts(:stderr, "Could not start the shell after 15 seconds, aborting...")
+      {:error, reason} ->
+        IO.puts(:stderr, "Could not start IEx CLI due to reason: #{inspect(reason)}")
         System.halt(1)
     end
   end

lib/iex/lib/iex/autocomplete.ex

@@ -146,7 +146,7 @@ defmodule IEx.Autocomplete do
 
   @doc false
   def exports(mod) do
-    if Code.ensure_loaded?(mod) and function_exported?(mod, :__info__, 1) do
+    if ensure_loaded?(mod) and function_exported?(mod, :__info__, 1) do
       mod.__info__(:macros) ++ (mod.__info__(:functions) -- [__info__: 1])
     else
       mod.module_info(:exports) -- [module_info: 0, module_info: 1]
@@ -313,21 +313,23 @@ defmodule IEx.Autocomplete do
       for {alias, mod} <- aliases_from_env(shell),
           [name] = Module.split(alias),
           String.starts_with?(name, hint),
-          struct?(mod) and not function_exported?(mod, :exception, 1),
-          do: %{kind: :struct, name: name}
+          do: {mod, name}
 
     modules =
       for "Elixir." <> name = full_name <- match_modules("Elixir." <> hint, true),
           String.starts_with?(name, hint),
           mod = String.to_atom(full_name),
-          struct?(mod) and not function_exported?(mod, :exception, 1),
+          do: {mod, name}
+
+    all = aliases ++ modules
+    Code.ensure_all_loaded(Enum.map(all, &elem(&1, 0)))
+
+    refs =
+      for {mod, name} <- all,
+          function_exported?(mod, :__struct__, 1) and not function_exported?(mod, :exception, 1),
           do: %{kind: :struct, name: name}
 
-    format_expansion(aliases ++ modules, hint)
-  end
-
-  defp struct?(mod) do
-    Code.ensure_loaded?(mod) and function_exported?(mod, :__struct__, 1)
+    format_expansion(refs, hint)
   end
 
   defp expand_container_context(code, context, hint, shell) do
@@ -420,7 +422,9 @@ defmodule IEx.Autocomplete do
   defp container_context_struct(cursor, pairs, aliases, shell) do
     with {pairs, [^cursor]} <- Enum.split(pairs, -1),
          alias = value_from_alias(aliases, shell),
-         true <- Keyword.keyword?(pairs) and struct?(alias) do
+         true <-
+           Keyword.keyword?(pairs) and ensure_loaded?(alias) and
+             function_exported?(alias, :__struct__, 1) do
       {:struct, alias, pairs}
     else
       _ -> nil

lib/iex/lib/iex/broker.ex

@@ -85,9 +85,13 @@ defmodule IEx.Broker do
     yes?(IO.gets(:stdio, interrupt))
   end
 
-  defp yes?(string) do
-    is_binary(string) and String.trim(string) in ["", "y", "Y", "yes", "YES", "Yes"]
-  end
+  defp yes?(string) when is_binary(string),
+    do: String.trim(string) in ["", "y", "Y", "yes", "YES", "Yes"]
+
+  defp yes?(charlist) when is_list(charlist),
+    do: yes?(List.to_string(charlist))
+
+  defp yes?(_), do: false
 
   @doc """
   Client requests a takeover.

lib/iex/lib/iex/cli.ex

@@ -89,8 +89,7 @@ defmodule IEx.CLI do
       spawn_link(fn ->
         receive do
           {:begin, ^ref, other} ->
-            {:ok, _} = Application.ensure_all_started(:elixir)
-            System.wait_until_booted()
+            :elixir.start_cli()
             send(other, {:done, ref})
         end
       end)

lib/iex/lib/iex/introspection.ex

@@ -293,6 +293,7 @@ defmodule IEx.Introspection do
               module.module_info(:exports)
           end
           |> Enum.sort()
+          |> Enum.dedup()
 
         result =
           for {^function, arity} <- exports,

lib/logger/lib/logger.ex

@@ -831,9 +831,12 @@ defmodule Logger do
 
   defp update_translators(updater) do
     :elixir_config.serial(fn ->
+      translators = updater.(Application.fetch_env!(:logger, :translators))
+      Application.put_env(:logger, :translators, translators)
+
       with %{filters: filters} <- :logger.get_primary_config(),
            {{_, {fun, config}}, filters} <- List.keytake(filters, :logger_translator, 0) do
-        config = update_in(config.translators, updater)
+        config = %{config | translators: translators}
         :ok = :logger.set_primary_config(:filters, filters ++ [logger_translator: {fun, config}])
       end
     end)

lib/logger/test/logger/backends/handler_test.exs

@@ -58,7 +58,9 @@ defmodule Logger.Backends.HandlerTest do
   end
 
   test "add_translator/1 and remove_translator/1 for logger formats" do
+    refute {CustomTranslator, :t} in Application.fetch_env!(:logger, :translators)
     assert Logger.add_translator({CustomTranslator, :t})
+    assert {CustomTranslator, :t} in Application.fetch_env!(:logger, :translators)
 
     assert capture_log(fn ->
              :logger.info(~c"hello: ~p", [:ok])

lib/mix/lib/mix.ex

@@ -600,26 +600,29 @@ defmodule Mix do
   """
   @doc since: "1.15.0"
   def ensure_application!(app) when is_atom(app) do
-    ensure_application!(app, Mix.State.builtin_apps())
+    ensure_application!(app, Mix.State.builtin_apps(), [])
     :ok
   end
 
-  defp ensure_application!(app, builtin_apps) do
+  defp ensure_application!(app, builtin_apps, optional) do
     case builtin_apps do
       %{^app => path} ->
         Code.prepend_path(path, cache: true)
         Application.load(app)
+        optional = List.wrap(Application.spec(app, :optional_applications))
 
         Application.spec(app, :applications)
         |> List.wrap()
-        |> Enum.each(&ensure_application!(&1, builtin_apps))
+        |> Enum.each(&ensure_application!(&1, builtin_apps, optional))
 
       %{} ->
-        Mix.raise(
-          "The application \"#{app}\" could not be found. This may happen if your " <>
-            "Operating System broke Erlang into multiple packages and may be fixed " <>
-            "by installing the missing \"erlang-dev\" and \"erlang-#{app}\" packages"
-        )
+        unless app in optional do
+          Mix.raise(
+            "The application \"#{app}\" could not be found. This may happen if your " <>
+              "Operating System broke Erlang into multiple packages and may be fixed " <>
+              "by installing the missing \"erlang-dev\" and \"erlang-#{app}\" packages"
+          )
+        end
     end
   end
 
@@ -674,6 +677,9 @@ defmodule Mix do
     * `:lockfile` (since v1.14.0) - path to a lockfile to be used as a basis of
       dependency resolution.
 
+    * `:start_applications` (since v1.15.3) - if `true`, ensures that installed app
+      and its dependencies are started after install (Default: `true`)
+
   ## Examples
 
   Installing `:decimal` and `:jason`:
@@ -794,6 +800,7 @@ defmodule Mix do
     config_path = expand_path(opts[:config_path], deps, :config_path, "config/config.exs")
     system_env = Keyword.get(opts, :system_env, [])
     consolidate_protocols? = Keyword.get(opts, :consolidate_protocols, true)
+    start_applications? = Keyword.get(opts, :start_applications, true)
 
     id =
       {deps, config, system_env, consolidate_protocols?}
@@ -893,9 +900,11 @@ defmodule Mix do
             end
           end)
 
-          for %{app: app, opts: opts} <- Mix.Dep.cached(),
-              Keyword.get(opts, :runtime, true) and Keyword.get(opts, :app, true) do
-            Application.ensure_all_started(app)
+          if start_applications? do
+            for %{app: app, opts: opts} <- Mix.Dep.cached(),
+                Keyword.get(opts, :runtime, true) and Keyword.get(opts, :app, true) do
+              Application.ensure_all_started(app)
+            end
           end
 
           Mix.State.put(:installed, id)

lib/mix/lib/mix/compilers/elixir.ex

@@ -1,7 +1,7 @@
 defmodule Mix.Compilers.Elixir do
   @moduledoc false
 
-  @manifest_vsn 19
+  @manifest_vsn 20
   @checkpoint_vsn 2
 
   import Record
@@ -46,6 +46,11 @@ defmodule Mix.Compilers.Elixir do
     {all_modules, all_sources, all_local_exports, old_parents, old_cache_key, old_deps_config} =
       parse_manifest(manifest, dest)
 
+    # Prepend ourselves early because of __mix_recompile__? checks
+    # and also that, in case of nothing compiled, we already need
+    # ourselves available in the path.
+    Code.prepend_path(dest)
+
     # If modules have been added or removed from the Erlang compiler,
     # we need to recompile all references to old and new modules.
     stale =
@@ -205,10 +210,6 @@ defmodule Mix.Compilers.Elixir do
         Code.purge_compiler_modules()
       end
     else
-      # Prepend ourselves, even if we did no work, as we may have defined
-      # future compilers.
-      Code.prepend_path(dest)
-
       # We need to return ok if deps_changed? or stale_modules changed,
       # even if no code was compiled, because we need to propagate the changed
       # status to compile.protocols. This will be the case whenever:
@@ -453,6 +454,9 @@ defmodule Mix.Compilers.Elixir do
       8 -> :crypto.hash(:blake2b, contents)
       _ -> :crypto.hash(:blake2s, contents)
     end
+  rescue
+    # Blake may not be available on all OpenSSL distribution
+    _ -> :erlang.md5(contents)
   end
 
   defp set_compiler_opts(opts) do
@@ -565,72 +569,72 @@ defmodule Mix.Compilers.Elixir do
     Enum.any?(enumerable, &Map.has_key?(map, &1))
   end
 
-  defp stale_local_deps(local_deps, manifest, stale_modules, modified, old_exports) do
+  defp stale_local_deps(local_deps, manifest, stale_modules, modified, deps_exports) do
     base = Path.basename(manifest)
 
     # The stale modules so far will become both stale_modules and stale_exports,
     # as any export from a dependency needs to be recompiled.
     stale_modules = Map.from_keys(stale_modules, true)
 
-    {stale_modules, stale_exports, new_exports} =
-      for %{opts: opts} <- local_deps,
-          manifest = Path.join([opts[:build], ".mix", base]),
-          Mix.Utils.last_modified(manifest) > modified,
-          reduce: {stale_modules, stale_modules, []} do
-        {modules, exports, new_exports} ->
-          {manifest_modules, manifest_sources} = read_manifest(manifest)
+    for %{app: app, opts: opts} <- local_deps,
+        manifest = Path.join([opts[:build], ".mix", base]),
+        Mix.Utils.last_modified(manifest) > modified,
+        reduce: {stale_modules, stale_modules, deps_exports} do
+      {modules, exports, deps_exports} ->
+        {manifest_modules, manifest_sources} = read_manifest(manifest)
 
-          dep_modules =
-            for module(module: module, timestamp: timestamp) <- manifest_modules,
-                timestamp > modified,
-                do: module
+        dep_modules =
+          for module(module: module, timestamp: timestamp) <- manifest_modules,
+              timestamp > modified,
+              do: module
 
-          # If any module has a compile time dependency on a changed module
-          # within the dependency, they will be recompiled. However, export
-          # and runtime dependencies won't have recompiled so we need to
-          # propagate them to the parent app.
-          {dep_modules, _, _} =
-            fixpoint_runtime_modules(manifest_sources, Map.from_keys(dep_modules, true))
+        # If any module has a compile time dependency on a changed module
+        # within the dependency, they will be recompiled. However, export
+        # and runtime dependencies won't have recompiled so we need to
+        # propagate them to the parent app.
+        {dep_modules, _, _} =
+          fixpoint_runtime_modules(manifest_sources, Map.from_keys(dep_modules, true))
 
-          # Update exports
-          {exports, new_exports} =
-            for {module, _} <- dep_modules, reduce: {exports, new_exports} do
-              {exports, new_exports} ->
-                export = exports_md5(module, false)
+        old_exports = Map.get(deps_exports, app, %{})
 
-                # If the exports are the same, then the API did not change,
-                # so we do not mark the export as stale. Note this has to
-                # be very conservative. If the module is not loaded or if
-                # the exports were not there, we need to consider it a stale
-                # export.
-                exports =
-                  if export && old_exports[module] == export,
-                    do: exports,
-                    else: Map.put(exports, module, true)
+        # Update exports
+        {exports, new_exports} =
+          for {module, _} <- dep_modules, reduce: {exports, []} do
+            {exports, new_exports} ->
+              export = exports_md5(module, false)
 
-                # Then we store the new export if any
-                new_exports =
-                  if export,
-                    do: [{module, export} | new_exports],
-                    else: new_exports
+              # If the exports are the same, then the API did not change,
+              # so we do not mark the export as stale. Note this has to
+              # be very conservative. If the module is not loaded or if
+              # the exports were not there, we need to consider it a stale
+              # export.
+              exports =
+                if export && old_exports[module] == export,
+                  do: exports,
+                  else: Map.put(exports, module, true)
 
-                {exports, new_exports}
-            end
+              # Then we store the new export if any
+              new_exports =
+                if export,
+                  do: [{module, export} | new_exports],
+                  else: new_exports
 
-          {Map.merge(modules, dep_modules), exports, new_exports}
-      end
+              {exports, new_exports}
+          end
 
-    # Any dependency in old export but not in new export
-    # was removed so we need to mark them as stale too.
-    new_exports = Map.new(new_exports)
+        new_exports = Map.new(new_exports)
 
-    removed =
-      for {module, _} <- old_exports,
-          not is_map_key(new_exports, module),
-          do: {module, true},
-          into: %{}
+        removed =
+          for {module, _} <- old_exports,
+              not is_map_key(new_exports, module),
+              do: {module, true},
+              into: %{}
 
-    {Map.merge(stale_modules, removed), Map.merge(stale_exports, removed), new_exports}
+        modules = modules |> Map.merge(dep_modules) |> Map.merge(removed)
+        exports = Map.merge(exports, removed)
+        deps_exports = Map.put(deps_exports, app, new_exports)
+        {modules, exports, deps_exports}
+    end
   end
 
   defp fixpoint_runtime_modules(sources, modules) when modules != %{} do

lib/mix/lib/mix/dep.ex

@@ -114,12 +114,12 @@ defmodule Mix.Dep do
         write_cached_deps(top, {env, target}, load_and_cache(config, top, bottom, env, target))
 
       _ ->
-        converge(env: env, target: target)
+        converge_and_load(env: env, target: target)
     end
   end
 
   defp load_and_cache(_config, top, top, env, target) do
-    converge(env: env, target: target)
+    converge_and_load(env: env, target: target)
   end
 
   defp load_and_cache(config, _top, bottom, _env, _target) do
@@ -152,6 +152,20 @@ defmodule Mix.Dep do
     end)
   end
 
+  defp converge_and_load(opts) do
+    for %{app: app, opts: opts} = dep <- Mix.Dep.Converger.converge(opts) do
+      case Keyword.pop(opts, :app_properties) do
+        {nil, _opts} ->
+          dep
+
+        {app_properties, opts} ->
+          # We don't raise because child dependencies may be missing if manually cleaned
+          :application.load({:application, app, app_properties})
+          %{dep | opts: opts}
+      end
+    end
+  end
+
   defp read_cached_deps(project, env_target) do
     case Mix.State.read_cache({:cached_deps, project}) do
       {^env_target, deps} -> deps
@@ -164,6 +178,13 @@ defmodule Mix.Dep do
     deps
   end
 
+  @doc """
+  Although private API, this is kept to avoid breaking code on patch releases.
+  """
+  def load_on_environment(opts) do
+    Mix.Dep.Converger.converge(opts)
+  end
+
   @doc """
   Clears loaded dependencies from the cache for the current environment.
   """
@@ -174,25 +195,6 @@ defmodule Mix.Dep do
     end
   end
 
-  @doc """
-  Returns loaded dependencies recursively on the given environment.
-
-  If no environment is passed, dependencies are loaded across all
-  environments. The result is not cached.
-
-  ## Exceptions
-
-  This function raises an exception if any of the dependencies
-  provided in the project are in the wrong format.
-  """
-  def load_on_environment(opts) do
-    converge(opts)
-  end
-
-  defp converge(opts) do
-    Mix.Dep.Converger.converge(nil, nil, opts, &{&1, &2, &3}) |> elem(0)
-  end
-
   @doc """
   Filters the given dependencies by name.
 

lib/mix/lib/mix/dep/converger.ex

@@ -60,6 +60,18 @@ defmodule Mix.Dep.Converger do
     )
   end
 
+  @doc """
+  Converge without lock and accumulator updates.
+
+  Note the dependencies returned from converge are not yet loaded.
+  The relevant app keys are found under `dep.opts[:app_properties]`.
+
+  See `Mix.Dep.Loader.children/1` for options.
+  """
+  def converge(opts \\ []) do
+    converge(nil, nil, opts, &{&1, &2, &3}) |> elem(0)
+  end
+
   @doc """
   Converges all dependencies from the current project,
   including nested dependencies.
@@ -68,35 +80,15 @@ defmodule Mix.Dep.Converger do
   must return an updated dependency in case some processing
   is done.
 
+  Note the dependencies returned from converge are not yet loaded.
+  The relevant app keys are found under `dep.opts[:app_properties]`.
+
   See `Mix.Dep.Loader.children/1` for options.
   """
   def converge(acc, lock, opts, callback) do
     {deps, acc, lock} = all(acc, lock, opts, callback)
     if remote = Mix.RemoteConverger.get(), do: remote.post_converge()
-
-    deps =
-      for %{app: app, opts: opts} = dep <- topological_sort(deps) do
-        case Keyword.pop(opts, :app_properties) do
-          {nil, _opts} ->
-            dep
-
-          {app_properties, opts} ->
-            case :application.load({:application, app, app_properties}) do
-              :ok ->
-                :ok
-
-              {:error, {:already_loaded, _}} ->
-                :ok
-
-              {:error, error} ->
-                Mix.raise("Could not load application #{inspect(app)}: #{inspect(error)}")
-            end
-
-            %{dep | opts: opts}
-        end
-      end
-
-    {deps, acc, lock}
+    {topological_sort(deps), acc, lock}
   end
 
   defp all(acc, lock, opts, callback) do

lib/mix/lib/mix/release.ex

@@ -67,7 +67,7 @@ defmodule Mix.Release do
   @default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent]
   @safe_modes [:permanent, :temporary, :transient]
   @unsafe_modes [:load, :none]
-  @significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c
+  @additional_chunks ~w(Attr)c
   @copy_app_dirs ["priv"]
 
   @doc false
@@ -879,7 +879,7 @@ defmodule Mix.Release do
   @spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()}
   def strip_beam(binary, options \\ []) when is_list(options) do
     chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1)
-    all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep)
+    all_chunks = Enum.uniq(@additional_chunks ++ :beam_lib.significant_chunks() ++ chunks_to_keep)
     compress? = Keyword.get(options, :compress, false)
 
     case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do

lib/mix/lib/mix/task.ex

@@ -58,14 +58,22 @@ defmodule Mix.Task do
 
       @requirements ["app.config"]
 
-  Tasks typically depend on the `"app.config"` task, when they
-  need to access code from the current project with all apps
-  already configured, or the "app.start" task, when they also
-  need those apps to be already started:
+  A task will typically depend on one of the following tasks:
 
-      @requirements ["app.start"]
+    * "loadpaths" - this ensures dependencies are available
+      and compiled. If you are publishing a task as part of
+      a library to be used by others, and your task does not
+      need to interact with the user code in any way, this is
+      the recommended requirement
 
-  You can also run tasks directly with `run/2`.
+    * "app.config" - additionally compiles and loads the runtime
+      configuration for the current project. If you are creating
+      a task to be used within your application or as part of a
+      library, which must invoke or interact with the user code,
+      this is the minimum recommended requirement
+
+    * "app.start" - additionally starts the supervision tree of
+      the current project and its dependencies
 
   Finally, set `@recursive true` if you want the task to run
   on each umbrella child in an umbrella project.

lib/mix/lib/mix/tasks/archive.build.ex

@@ -61,7 +61,7 @@ defmodule Mix.Tasks.Archive.Build do
     project = Mix.Project.get()
 
     if project && Keyword.get(opts, :compile, true) do
-      Mix.Task.run(:compile, args)
+      Mix.Task.run(:compile, ["--no-protocol-consolidation" | args])
     end
 
     source =

lib/mix/lib/mix/tasks/compile.all.ex

@@ -43,7 +43,11 @@ defmodule Mix.Tasks.Compile.All do
       Code.delete_paths(current_paths -- loaded_paths)
     end
 
-    Code.prepend_paths(loaded_paths -- current_paths, cache: true)
+    # Add the current compilation path. compile.elixir and compile.erlang
+    # will also add this path, but only if they run, so we always add it
+    # here too. Furthermore, we don't cache it as we may still write to it.
+    compile_path = to_charlist(Mix.Project.compile_path())
+    Code.prepend_paths([compile_path | loaded_paths -- current_paths], cache: true)
 
     result =
       if "--no-compile" in args do
@@ -64,12 +68,6 @@ defmodule Mix.Tasks.Compile.All do
       Mix.AppLoader.write_cache(app_cache, Map.new(loaded_modules))
     end
 
-    # Add the current compilation path. compile.elixir and compile.erlang
-    # will also add this path, but only if they run, so we always add it
-    # here too. Furthermore, we don't cache it as we may still write to it.
-    compile_path = to_charlist(Mix.Project.compile_path())
-    _ = Code.prepend_path(compile_path)
-
     unless "--no-app-loading" in args do
       app = config[:app]
 
@@ -125,7 +123,7 @@ defmodule Mix.Tasks.Compile.All do
     Enum.reduce(Mix.ProjectStack.pop_after_compiler(compiler), result, & &1.(&2))
   end
 
-  defp project_apps(config) do
+  def project_apps(config) do
     project = Mix.Project.get!()
 
     properties =

lib/mix/lib/mix/tasks/compile.ex

@@ -148,8 +148,7 @@ defmodule Mix.Tasks.Compile do
     # If we are in an umbrella project, now load paths from all children.
     if apps_paths = Mix.Project.apps_paths(config) do
       loaded_paths =
-        apps_paths
-        |> Map.keys()
+        (Mix.Tasks.Compile.All.project_apps(config) ++ Map.keys(apps_paths))
         |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, [], fn
           {_app, path}, acc -> if path, do: [path | acc], else: acc
         end)

lib/mix/lib/mix/tasks/deps.clean.ex

@@ -41,7 +41,7 @@ defmodule Mix.Tasks.Deps.Clean do
           value = opts[switch],
           do: {key, :"#{value}"}
 
-    loaded_deps = Mix.Dep.load_on_environment(loaded_opts)
+    loaded_deps = Mix.Dep.Converger.converge(loaded_opts)
 
     apps_to_clean =
       cond do

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -57,9 +57,6 @@ defmodule Mix.Tasks.Deps.Compile do
 
     case OptionParser.parse(args, switches: @switches) do
       {opts, [], _} ->
-        # Because this command may be invoked explicitly with
-        # deps.compile, we simply try to compile any available
-        # or local dependency.
         compile(filter_available_and_local_deps(deps), opts)
 
       {opts, tail, _} ->

lib/mix/lib/mix/tasks/deps.ex

@@ -1,7 +1,7 @@
 defmodule Mix.Tasks.Deps do
   use Mix.Task
 
-  import Mix.Dep, only: [load_on_environment: 1, format_dep: 1, format_status: 1, check_lock: 1]
+  import Mix.Dep, only: [format_dep: 1, format_status: 1, check_lock: 1]
 
   @shortdoc "Lists dependencies and their status"
 
@@ -167,7 +167,7 @@ defmodule Mix.Tasks.Deps do
 
     shell = Mix.shell()
 
-    load_on_environment(loaded_opts)
+    Mix.Dep.Converger.converge(loaded_opts)
     |> Enum.sort_by(& &1.app)
     |> Enum.each(fn dep ->
       %Mix.Dep{scm: scm, manager: manager} = dep

lib/mix/lib/mix/tasks/deps.loadpaths.ex

@@ -17,7 +17,6 @@ defmodule Mix.Tasks.Deps.Loadpaths do
     * `--no-archives-check` - does not check archives
     * `--no-compile` - does not compile even if files require compilation
     * `--no-deps-check` - does not check or compile deps, only load available ones
-    * `--no-deps-loading` - does not add deps loadpaths to the code path
     * `--no-elixir-version-check` - does not check Elixir version
     * `--no-optional-deps` - does not compile or load optional deps
 

lib/mix/lib/mix/tasks/deps.tree.ex

@@ -44,7 +44,7 @@ defmodule Mix.Tasks.Deps.Tree do
           value = opts[switch],
           do: {key, :"#{value}"}
 
-    deps = Mix.Dep.load_on_environment(deps_opts)
+    deps = Mix.Dep.Converger.converge(deps_opts)
 
     root =
       case args do

lib/mix/lib/mix/tasks/deps.unlock.ex

@@ -83,7 +83,7 @@ defmodule Mix.Tasks.Deps.Unlock do
   end
 
   defp unused_apps(lock) do
-    apps = Mix.Dep.load_on_environment([]) |> Enum.map(& &1.app)
+    apps = Mix.Dep.Converger.converge([]) |> Enum.map(& &1.app)
 
     lock
     |> Map.drop(apps)

lib/mix/lib/mix/tasks/format.ex

@@ -233,6 +233,8 @@ defmodule Mix.Tasks.Format do
     {formatter_opts_and_subs, _sources} =
       eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, [dot_formatter])
 
+    formatter_opts_and_subs = load_plugins(formatter_opts_and_subs)
+
     args
     |> expand_args(cwd, dot_formatter, formatter_opts_and_subs, opts)
     |> Task.async_stream(&format_file(&1, opts), ordered: false, timeout: :infinity)
@@ -240,67 +242,11 @@ defmodule Mix.Tasks.Format do
     |> check!(opts)
   end
 
-  @doc """
-  Returns a formatter function and the formatter options to
-  be used for the given file.
-
-  The function must be called with the contents of the file
-  to be formatted. The options are returned for reflection
-  purposes.
-  """
-  @doc since: "1.13.0"
-  def formatter_for_file(file, opts \\ []) do
-    cwd = Keyword.get_lazy(opts, :root, &File.cwd!/0)
-    {dot_formatter, formatter_opts} = eval_dot_formatter(cwd, opts)
-
-    {formatter_opts_and_subs, _sources} =
-      eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, [dot_formatter])
-
-    find_formatter_and_opts_for_file(Path.expand(file, cwd), formatter_opts_and_subs)
-  end
-
-  @doc """
-  Returns formatter options to be used for the given file.
-  """
-  # TODO: Deprecate on Elixir v1.17
-  @doc deprecated: "Use formatter_for_file/2 instead"
-  def formatter_opts_for_file(file, opts \\ []) do
-    {_, formatter_opts} = formatter_for_file(file, opts)
-    formatter_opts
-  end
-
-  defp eval_dot_formatter(cwd, opts) do
-    cond do
-      dot_formatter = opts[:dot_formatter] ->
-        {dot_formatter, eval_file_with_keyword_list(dot_formatter)}
-
-      File.regular?(Path.join(cwd, ".formatter.exs")) ->
-        dot_formatter = Path.join(cwd, ".formatter.exs")
-        {".formatter.exs", eval_file_with_keyword_list(dot_formatter)}
-
-      true ->
-        {".formatter.exs", []}
-    end
-  end
-
-  # This function reads exported configuration from the imported
-  # dependencies and subdirectories and deals with caching the result
-  # of reading such configuration in a manifest file.
-  defp eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, sources) do
-    deps = Keyword.get(formatter_opts, :import_deps, [])
-    subs = Keyword.get(formatter_opts, :subdirectories, [])
+  defp load_plugins({formatter_opts, subs}) do
     plugins = Keyword.get(formatter_opts, :plugins, [])
 
-    if not is_list(deps) do
-      Mix.raise("Expected :import_deps to return a list of dependencies, got: #{inspect(deps)}")
-    end
-
-    if not is_list(subs) do
-      Mix.raise("Expected :subdirectories to return a list of directories, got: #{inspect(subs)}")
-    end
-
     if not is_list(plugins) do
-      Mix.raise("Expected :plugins to return a list of modules, got: #{inspect(plugins)}")
+      Mix.raise("Expected :plugins to return a list of directories, got: #{inspect(plugins)}")
     end
 
     if plugins != [] do
@@ -341,10 +287,69 @@ defmodule Mix.Tasks.Format do
          end}
       end)
 
-    formatter_opts =
-      formatter_opts
-      |> Keyword.put(:plugins, plugins)
-      |> Keyword.put(:sigils, sigils)
+    {Keyword.put(formatter_opts, :sigils, sigils),
+     Enum.map(subs, fn {path, opts} -> {path, load_plugins(opts)} end)}
+  end
+
+  @doc """
+  Returns a formatter function and the formatter options to
+  be used for the given file.
+
+  The function must be called with the contents of the file
+  to be formatted. The options are returned for reflection
+  purposes.
+  """
+  @doc since: "1.13.0"
+  def formatter_for_file(file, opts \\ []) do
+    cwd = Keyword.get_lazy(opts, :root, &File.cwd!/0)
+    {dot_formatter, formatter_opts} = eval_dot_formatter(cwd, opts)
+
+    {formatter_opts_and_subs, _sources} =
+      eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, [dot_formatter])
+
+    formatter_opts_and_subs = load_plugins(formatter_opts_and_subs)
+
+    find_formatter_and_opts_for_file(Path.expand(file, cwd), formatter_opts_and_subs)
+  end
+
+  @doc """
+  Returns formatter options to be used for the given file.
+  """
+  # TODO: Deprecate on Elixir v1.17
+  @doc deprecated: "Use formatter_for_file/2 instead"
+  def formatter_opts_for_file(file, opts \\ []) do
+    {_, formatter_opts} = formatter_for_file(file, opts)
+    formatter_opts
+  end
+
+  defp eval_dot_formatter(cwd, opts) do
+    cond do
+      dot_formatter = opts[:dot_formatter] ->
+        {dot_formatter, eval_file_with_keyword_list(dot_formatter)}
+
+      File.regular?(Path.join(cwd, ".formatter.exs")) ->
+        dot_formatter = Path.join(cwd, ".formatter.exs")
+        {".formatter.exs", eval_file_with_keyword_list(dot_formatter)}
+
+      true ->
+        {".formatter.exs", []}
+    end
+  end
+
+  # This function reads exported configuration from the imported
+  # dependencies and subdirectories and deals with caching the result
+  # of reading such configuration in a manifest file.
+  defp eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, sources) do
+    deps = Keyword.get(formatter_opts, :import_deps, [])
+    subs = Keyword.get(formatter_opts, :subdirectories, [])
+
+    if not is_list(deps) do
+      Mix.raise("Expected :import_deps to return a list of dependencies, got: #{inspect(deps)}")
+    end
+
+    if not is_list(subs) do
+      Mix.raise("Expected :subdirectories to return a list of directories, got: #{inspect(subs)}")
+    end
 
     if deps == [] and subs == [] do
       {{formatter_opts, []}, sources}
@@ -448,7 +453,8 @@ defmodule Mix.Tasks.Format do
       _ ->
         Mix.raise(
           "Unknown dependency #{inspect(dep)} given to :import_deps in the formatter configuration. " <>
-            "Make sure the dependency is listed in your mix.exs and you have run \"mix deps.get\""
+            "Make sure the dependency is listed in your mix.exs for environment #{inspect(Mix.env())} " <>
+            "and you have run \"mix deps.get\""
         )
     end
   end
@@ -578,8 +584,15 @@ defmodule Mix.Tasks.Format do
 
   defp recur_formatter_opts_for_file(file, {formatter_opts, subs}) do
     Enum.find_value(subs, formatter_opts, fn {sub, formatter_opts_and_subs} ->
-      if String.starts_with?(file, sub) do
-        recur_formatter_opts_for_file(file, formatter_opts_and_subs)
+      size = byte_size(sub)
+
+      case file do
+        <<prefix::binary-size(size), dir_separator, _::binary>>
+        when prefix == sub and dir_separator in [?\\, ?/] ->
+          recur_formatter_opts_for_file(file, formatter_opts_and_subs)
+
+        _ ->
+          nil
       end
     end)
   end

lib/mix/lib/mix/tasks/loadpaths.ex

@@ -20,7 +20,6 @@ defmodule Mix.Tasks.Loadpaths do
     * `--no-archives-check` - does not check archives
     * `--no-compile` - does not compile dependencies, only check and load them
     * `--no-deps-check` - does not check dependencies, only load available ones
-    * `--no-deps-loading` - does not add deps loadpaths to the code path
     * `--no-elixir-version-check` - does not check Elixir version
     * `--no-optional-deps` - does not compile or load optional deps
 

lib/mix/test/fixtures/umbrella_dep/deps/umbrella/mix.exs

@@ -4,4 +4,8 @@ defmodule Umbrella.MixProject do
   def project do
     [apps_path: "apps"]
   end
+
+  def application do
+    [extra_applications: [:runtime_tools]]
+  end
 end

lib/mix/test/mix/dep_test.exs

@@ -36,7 +36,7 @@ defmodule Mix.DepTest do
   defp assert_wrong_dependency(deps) do
     with_deps(deps, fn ->
       assert_raise Mix.Error, ~r"Dependency specified in the wrong format", fn ->
-        Mix.Dep.load_on_environment([])
+        Mix.Dep.Converger.converge([])
       end
     end)
   end
@@ -59,7 +59,7 @@ defmodule Mix.DepTest do
     in_fixture("deps_status", fn ->
       Mix.Project.push(DepsApp)
 
-      deps = Mix.Dep.load_on_environment([])
+      deps = Mix.Dep.Converger.converge([])
       assert length(deps) == 6
       assert Enum.find(deps, &match?(%Mix.Dep{app: :ok, status: {:ok, _}}, &1))
       assert Enum.find(deps, &match?(%Mix.Dep{app: :invalidvsn, status: {:invalidvsn, :ok}}, &1))
@@ -102,7 +102,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
-        deps = Mix.Dep.load_on_environment([])
+        deps = Mix.Dep.Converger.converge([])
         assert Enum.find(deps, &match?(%Mix.Dep{app: :ok, status: {:ok, _}}, &1))
       end)
     end)
@@ -115,7 +115,7 @@ defmodule Mix.DepTest do
       in_fixture("deps_status", fn ->
         send(self(), {:mix_shell_input, :yes?, false})
         msg = "Could not find an SCM for dependency :ok from Mix.DepTest.ProcessDepsApp"
-        assert_raise Mix.Error, msg, fn -> Mix.Dep.load_on_environment([]) end
+        assert_raise Mix.Error, msg, fn -> Mix.Dep.Converger.converge([]) end
       end)
     end)
   end
@@ -139,7 +139,7 @@ defmodule Mix.DepTest do
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
         assert_raise Mix.Error, ~r"Invalid requirement", fn ->
-          Mix.Dep.load_on_environment([])
+          Mix.Dep.Converger.converge([])
         end
       end)
     end)
@@ -150,7 +150,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo]
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo]
       end)
     end)
   end
@@ -172,7 +172,7 @@ defmodule Mix.DepTest do
         end
         """)
 
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:deps_repo]
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:deps_repo]
       end)
     end)
   end
@@ -185,7 +185,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo]
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo]
       end)
     end)
   end
@@ -200,7 +200,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
-        [dep1, dep2] = Mix.Dep.load_on_environment([])
+        [dep1, dep2] = Mix.Dep.Converger.converge([])
         assert dep1.manager == nil
         assert dep2.manager == :rebar3
       end)
@@ -252,7 +252,7 @@ defmodule Mix.DepTest do
         end
         """)
 
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo]
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo]
       end)
     end)
   end
@@ -362,7 +362,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_tmp("load dependency with env vars", fn ->
-        Mix.Dep.load_on_environment([])
+        Mix.Dep.Converger.converge([])
         assert {:ok, "contents dep test"} = File.read(file_path)
       end)
     end)
@@ -378,7 +378,7 @@ defmodule Mix.DepTest do
 
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
-        [git_repo, _] = Mix.Dep.load_on_environment([])
+        [git_repo, _] = Mix.Dep.Converger.converge([])
         %{app: :git_repo, status: {:overridden, _}} = git_repo
       end)
     end)
@@ -549,7 +549,7 @@ defmodule Mix.DepTest do
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
         # Both orders below are valid after topological sort
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) in [
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) in [
                  [:git_repo, :abc_repo, :deps_repo],
                  [:abc_repo, :git_repo, :deps_repo]
                ]
@@ -604,7 +604,7 @@ defmodule Mix.DepTest do
     with_deps(deps, fn ->
       in_fixture("deps_status", fn ->
         # Both orders below are valid after topological sort
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) in [
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) in [
                  [:git_repo, :abc_repo, :deps_repo],
                  [:abc_repo, :git_repo, :deps_repo]
                ]
@@ -639,13 +639,13 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          deps = Mix.Dep.load_on_environment(env: :other_env)
+          deps = Mix.Dep.Converger.converge(env: :other_env)
           assert length(deps) == 2
 
-          deps = Mix.Dep.load_on_environment([])
+          deps = Mix.Dep.Converger.converge([])
           assert length(deps) == 2
 
-          assert [dep] = Mix.Dep.load_on_environment(env: :prod)
+          assert [dep] = Mix.Dep.Converger.converge(env: :prod)
           assert dep.app == :foo
         end)
       end)
@@ -676,10 +676,10 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
         end)
       end)
@@ -694,15 +694,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :dev)
+          loaded = Mix.Dep.Converger.converge(env: :dev)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
@@ -727,15 +727,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :dev)
+          loaded = Mix.Dep.Converger.converge(env: :dev)
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
           assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
         end)
@@ -751,18 +751,18 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :dev)
+          loaded = Mix.Dep.Converger.converge(env: :dev)
           assert [] = Enum.map(loaded, & &1.app)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:git_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :prod)
+          loaded = Mix.Dep.Converger.converge(env: :prod)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
         end)
@@ -778,15 +778,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :dev)
+          loaded = Mix.Dep.Converger.converge(env: :dev)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:git_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _] = Enum.map(loaded, & &1.status)
 
@@ -811,19 +811,19 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :dev)
+          loaded = Mix.Dep.Converger.converge(env: :dev)
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
           assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :test)
+          loaded = Mix.Dep.Converger.converge(env: :test)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(env: :prod)
+          loaded = Mix.Dep.Converger.converge(env: :prod)
           assert [] = Enum.map(loaded, & &1.app)
         end)
       end)
@@ -848,7 +848,7 @@ defmodule Mix.DepTest do
             """)
 
             Mix.State.clear_cache()
-            loaded = Mix.Dep.load_on_environment([])
+            loaded = Mix.Dep.Converger.converge([])
             Enum.map(loaded, &{&1.app, &1.opts[:only]})
           end)
         end)
@@ -933,13 +933,13 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          deps = Mix.Dep.load_on_environment(target: :rpi3)
+          deps = Mix.Dep.Converger.converge(target: :rpi3)
           assert length(deps) == 2
 
-          deps = Mix.Dep.load_on_environment([])
+          deps = Mix.Dep.Converger.converge([])
           assert length(deps) == 2
 
-          assert [dep] = Mix.Dep.load_on_environment(target: :host)
+          assert [dep] = Mix.Dep.Converger.converge(target: :host)
           assert dep.app == :foo
         end)
       end)
@@ -973,15 +973,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :host)
+          loaded = Mix.Dep.Converger.converge(target: :host)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :rpi3)
+          loaded = Mix.Dep.Converger.converge(target: :rpi3)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
@@ -1006,15 +1006,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :host)
+          loaded = Mix.Dep.Converger.converge(target: :host)
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
           assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :rpi3)
+          loaded = Mix.Dep.Converger.converge(target: :rpi3)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
         end)
@@ -1030,18 +1030,18 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :host)
+          loaded = Mix.Dep.Converger.converge(target: :host)
           assert [] = Enum.map(loaded, & &1.app)
 
-          loaded = Mix.Dep.load_on_environment(target: :bbb)
+          loaded = Mix.Dep.Converger.converge(target: :bbb)
           assert [:git_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :rpi3)
+          loaded = Mix.Dep.Converger.converge(target: :rpi3)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
         end)
@@ -1057,15 +1057,15 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :host)
+          loaded = Mix.Dep.Converger.converge(target: :host)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :rpi3)
+          loaded = Mix.Dep.Converger.converge(target: :rpi3)
           assert [:git_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _] = Enum.map(loaded, & &1.status)
 
@@ -1090,19 +1090,19 @@ defmodule Mix.DepTest do
 
       with_deps(deps, fn ->
         in_fixture("deps_status", fn ->
-          loaded = Mix.Dep.load_on_environment([])
+          loaded = Mix.Dep.Converger.converge([])
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :host)
+          loaded = Mix.Dep.Converger.converge(target: :host)
           assert [:deps_repo] = Enum.map(loaded, & &1.app)
           assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :bbb)
+          loaded = Mix.Dep.Converger.converge(target: :bbb)
           assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app)
           assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status)
 
-          loaded = Mix.Dep.load_on_environment(target: :rpi3)
+          loaded = Mix.Dep.Converger.converge(target: :rpi3)
           assert [] = Enum.map(loaded, & &1.app)
         end)
       end)
@@ -1127,7 +1127,7 @@ defmodule Mix.DepTest do
             """)
 
             Mix.State.clear_cache()
-            loaded = Mix.Dep.load_on_environment([])
+            loaded = Mix.Dep.Converger.converge([])
             Enum.map(loaded, &{&1.app, &1.opts[:targets]})
           end)
         end)

lib/mix/test/mix/rebar_test.exs

@@ -149,14 +149,14 @@ defmodule Mix.RebarTest do
   describe "integration with Mix" do
     test "inherits Rebar manager" do
       Mix.Project.push(RebarAsDep)
-      deps = Mix.Dep.load_on_environment([])
+      deps = Mix.Dep.Converger.converge([])
       assert Enum.all?(deps, &(&1.manager == :rebar3))
     end
 
     test "parses Rebar dependencies from rebar.config" do
       Mix.Project.push(RebarAsDep)
 
-      deps = Mix.Dep.load_on_environment([])
+      deps = Mix.Dep.Converger.converge([])
       assert Enum.all?(deps, &(&1.manager == :rebar3))
       assert Enum.find(deps, &(&1.app == :rebar_dep))
 
@@ -175,7 +175,7 @@ defmodule Mix.RebarTest do
 
         Mix.Tasks.Deps.Get.run([])
 
-        assert Mix.Dep.load_on_environment([]) |> Enum.map(& &1.app) ==
+        assert Mix.Dep.Converger.converge([]) |> Enum.map(& &1.app) ==
                  [:git_repo, :git_rebar, :rebar_override]
       end)
     after
@@ -199,7 +199,7 @@ defmodule Mix.RebarTest do
         assert :rebar_dep.any_function() == :ok
 
         load_paths =
-          Mix.Dep.load_on_environment([])
+          Mix.Dep.Converger.converge([])
           |> Enum.map(&Mix.Dep.load_paths(&1))
           |> Enum.concat()
 

lib/mix/test/mix/tasks/archive_test.exs

@@ -32,6 +32,7 @@ defmodule Mix.Tasks.ArchiveTest do
       message = "Generated archive \"archive-0.1.0.ez\" with MIX_ENV=dev"
       assert_received {:mix_shell, :info, [^message]}
       assert File.regular?(~c"archive-0.1.0.ez")
+      assert to_charlist(Mix.Project.consolidation_path()) not in :code.get_path()
 
       assert_archive_content_default()
       refute has_in_zip_file?(~c"archive-0.1.0.ez", ~c"archive-0.1.0/priv/.dot_file")

lib/mix/test/mix/tasks/compile.elixir_test.exs

@@ -19,6 +19,14 @@ defmodule Mix.Tasks.Compile.ElixirTest do
 
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
+
+      File.write!("lib/a.ex", """
+      defmodule A, do: :ok
+
+      # Also make sure that we access the ebin directory during compilation
+      true = to_charlist(Mix.Project.compile_path()) in :code.get_path()
+      """)
+
       Mix.Tasks.Compile.Elixir.run(["--verbose"])
 
       assert File.regular?("_build/shared/lib/sample/ebin/Elixir.A.beam")
@@ -32,6 +40,14 @@ defmodule Mix.Tasks.Compile.ElixirTest do
   test "compiles a project with per environment build" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
+
+      File.write!("lib/a.ex", """
+      defmodule A, do: :ok
+
+      # Also make sure that we access the ebin directory during compilation
+      true = to_charlist(Mix.Project.compile_path()) in :code.get_path()
+      """)
+
       Mix.Tasks.Compile.Elixir.run(["--verbose"])
 
       assert File.regular?("_build/dev/lib/sample/ebin/Elixir.A.beam")
@@ -772,7 +788,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
-  test "compiles mtime changed files if content changed but not length" do
+  test "recompiles mtime changed files if content changed but not length" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == {:ok, []}
@@ -872,7 +888,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
-  test "compiles size changed files" do
+  test "recompiles size changed files" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
       past = @old_time
@@ -894,7 +910,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
-  test "compiles dependent changed modules" do
+  test "recompiles dependent changed modules" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
       File.write!("lib/a.ex", "defmodule A, do: B.module_info()")
@@ -914,7 +930,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
-  test "compiles dependent changed modules without beam files" do
+  test "recompiles dependent changed modules without beam files" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
 
@@ -942,7 +958,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     Code.put_compiler_option(:ignore_module_conflict, false)
   end
 
-  test "compiles dependent changed modules even on removal" do
+  test "recompiles dependent changed modules even on removal" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
       File.write!("lib/a.ex", "defmodule A, do: B.module_info()")
@@ -963,7 +979,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
-  test "compiles dependent changed external resources" do
+  test "recompiles dependent changed external resources" do
     in_fixture("no_mixfile", fn ->
       Mix.Project.push(MixTest.Case.Sample)
       tmp = tmp_path("c.eex")
@@ -1263,6 +1279,11 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       assert_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
       assert_received {:mix_shell, :info, ["Compiled lib/c.ex"]}
 
+      # Mix recompile should work even if the compile path
+      # was removed and the module purged
+      Code.delete_path(Mix.Project.compile_path())
+      purge([A])
+
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == {:ok, []}
       assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]}
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}

lib/mix/test/mix/tasks/format_test.exs

@@ -308,6 +308,24 @@ defmodule Mix.Tasks.FormatTest do
                '''abc
              end
              """
+
+      {formatter_function, _options} = Mix.Tasks.Format.formatter_for_file("a.ex")
+
+      assert formatter_function.("""
+             if true do
+               ~W'''
+               foo bar baz
+               '''abc
+             end
+             """) == """
+             if true do
+               ~W'''
+               foo
+               bar
+               baz
+               '''abc
+             end
+             """
     end)
   end
 
@@ -555,11 +573,18 @@ defmodule Mix.Tasks.FormatTest do
   test "reads exported configuration from subdirectories", context do
     in_tmp(context.test, fn ->
       File.write!(".formatter.exs", """
-      [subdirectories: ["lib"]]
+      [subdirectories: ["li", "lib"]]
       """)
 
+      # We also create a directory called li to ensure files
+      # from lib won't accidentally match on li.
+      File.mkdir_p!("li")
       File.mkdir_p!("lib")
 
+      File.write!("li/.formatter.exs", """
+      [inputs: "**/*", locals_without_parens: [other_fun: 2]]
+      """)
+
       File.write!("lib/.formatter.exs", """
       [inputs: "a.ex", locals_without_parens: [my_fun: 2]]
       """)

lib/mix/test/mix/umbrella_test.exs

@@ -64,6 +64,10 @@ defmodule Mix.UmbrellaTest do
         Mix.Task.run("deps.loadpaths")
         Mix.Task.run("compile", ["--verbose"])
 
+        # Extra applications are picked even for umbrellas
+        assert :code.where_is_file(~c"runtime_tools.app") != :non_existing
+        assert :code.where_is_file(~c"observer.app") == :non_existing
+
         assert_received {:mix_shell, :info, ["==> bar"]}
         assert_received {:mix_shell, :info, ["Generated bar app"]}
         assert File.regular?("_build/dev/lib/bar/ebin/Elixir.Bar.beam")
@@ -290,7 +294,7 @@ defmodule Mix.UmbrellaTest do
     in_fixture("umbrella_dep", fn ->
       Mix.Project.push(CycleDeps)
 
-      assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:foo, :bar, :umbrella]
+      assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:foo, :bar, :umbrella]
     end)
   end
 
@@ -328,7 +332,7 @@ defmodule Mix.UmbrellaTest do
         end
         """)
 
-        assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:a, :b, :bar, :foo]
+        assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:a, :b, :bar, :foo]
       end)
     end)
   end
@@ -453,7 +457,8 @@ defmodule Mix.UmbrellaTest do
         assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]}
 
         # Recompiles if export dependency is removed
-        File.rm!("../foo/lib/foo.ex")
+        File.write!("../foo/lib/foo.ex", "")
+        ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir")
         Mix.Task.clear()
 
         ExUnit.CaptureIO.capture_io(:stderr, fn ->