Release v1.10.3

This reverts commit 4477115675.

.cirrus.yml

@@ -130,11 +130,11 @@ test_windows_task:
 test_freebsd_task:
   <<: *DEFAULT_TEST_SETTINGS
 
-  name: FreeBSD 12.0
+  name: FreeBSD 12.1
   alias: FreeBSD Stable
 
   freebsd_instance:
-    image_family: freebsd-12-0
+    image_family: freebsd-12-1
     cpu: 8
     memory: 7424Mi
 

CHANGELOG.md

@@ -155,6 +155,30 @@ assert %{"status" => 200, "body" => %{"key" => "foo"}} = json_payload
 
 Now imagine that `json_payload` is a large JSON blob and the `"key"` inside the `"body"` did not have value of `"foo"`. In previous Elixir versions, if the assertion failed, Elixir would print the right side and let you up to your own devices to figure out what went wrong. In Elixir v1.10, we diff the data structure against the pattern so you can see exactly which parts of the data matched the pattern and which ones did not. Note ExUnit already performed diffing when comparing data types, this new version adds diffing when matching data against a pattern.
 
+## v1.10.3 (2020-04-25)
+
+### 1. Bug fixes
+
+#### Elixir
+
+  * [Code] Return `[{mod, bin}]` from `Code.compile_file/2`, `Code.require_file/2`, `Code.load_file/2`
+  * [Code] Make sure the formatter respects newlines before and after module attributes
+  * [Kernel.ParallelCompiler] Fix a bug where the parallel compiler would raise in long compilation cycles
+  * [Kernel.ParallelCompiler] Fix a bug where the parallel compiler would raise if some of the modules being compiled referred to a module that has been loaded directly to memory
+  * [Module] Fix accidental breaking change where bodiless clauses had their body value on `@on_definition` callbacks set to an empty list instead of `nil`
+  * [String] Undeprecate `String.normalize/2` normalize and fix infinite loop caused by certain invalid strings
+
+#### ExUnit
+
+  * [ExUnit.Assertions] Fix pattern matching diff when matching on pinned variables
+  * [ExUnit.Assertions] Fix pattern matching diff when matching variable struct names
+  * [ExUnit.Assertions] Fix pattern matching diff when matching on the binary concat operator (`<>`) and the left side is not a literal string
+  * [ExUnit.Assertions] Fix pattern matching diff when matching on pseudo-vars (`__MODULE__`, `__DIR__`, etc)
+
+#### Mix
+
+  * [mix release] Respect the `:path` option when creating a `:tar` file for releases
+
 ## v1.10.2 (2020-02-26)
 
 ### 1. Bug fixes
@@ -282,6 +306,10 @@ Now imagine that `json_payload` is a large JSON blob and the `"key"` inside the
   * [URI] Preserve slashes in URIs without authority
   * [URI] Require a nil or an absolute path on URIs with host or authority
 
+#### ExUnit
+
+  * [ExUnit.Assertions] Fix `assert_receive` and `assert match?` to behave consistently compared `receive` and `match?` when given an invalid macro
+
 #### IEx
 
   * [IEx] Exit IEx session if the group leader exits
@@ -326,6 +354,7 @@ Now imagine that `json_payload` is a large JSON blob and the `"key"` inside the
 #### Mix
 
   * [mix compile.xref] This check has been moved into the compiler and has no effect now
+  * [mix xref] `xref` now only tracks dependencies between modules and files, no longer between functions. See "Compilation tracers" to learn more about how to track this information directly
   * [mix xref deprecations] This check has been moved into the compiler and has no effect now
   * [mix xref unreachable] This check has been moved into the compiler and has no effect now
 

VERSION

@@ -1 +1 @@
-1.10.2
+1.10.3

lib/elixir/lib/code.ex

@@ -43,6 +43,15 @@ defmodule Code do
   You can configure your list of tracers via `put_compiler_option/2`. The
   following events are available to tracers:
 
+    * `:start` - (since v1.11.0) invoked whenever the compiler starts to trace
+      a new lexical context, such as a new file. Keep in mind the compiler runs
+      in parallel, so multiple files may invoke `:start` and run at the same
+      time. The value of the `lexical_tracker` of the macro environment, albeit
+      opaque, can be used to uniquely identify the environment.
+
+    * `:stop` - (since v1.11.0) invoked whenever the compiler stops tracing a
+      new lexical context, such as a new file.
+
     * `{:import, meta, module, opts}` - traced whenever `module` is imported.
       `meta` is the import AST metadata and `opts` are the import options.
 
@@ -1392,6 +1401,6 @@ defmodule Code do
   defp verify_loaded(loaded) do
     maps_binaries = Enum.map(loaded, fn {_module, map, binary} -> {map, binary} end)
     Module.ParallelChecker.verify(maps_binaries, [])
-    Enum.map(loaded, fn {module, map, _binary} -> {module, map} end)
+    Enum.map(loaded, fn {module, _map, binary} -> {module, binary} end)
   end
 end

lib/elixir/lib/code/formatter.ex

@@ -1914,8 +1914,7 @@ defmodule Code.Formatter do
     {args_docs, comments?, %{state | comments: comments}}
   end
 
-  defp each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) do
-    [arg | args] = args
+  defp each_quoted_to_algebra_with_comments([arg | args], acc, max_line, state, comments?, fun) do
     {doc_start, doc_end} = traverse_line(arg, {@max_line, @min_line})
 
     {acc, comments, comments?} =
@@ -1926,7 +1925,7 @@ defmodule Code.Formatter do
     {acc, comments, comments?} =
       extract_comments_trailing(doc_start, doc_end, acc, state.comments, comments?)
 
-    acc = [doc_triplet | acc]
+    acc = [adjust_trailing_newlines(doc_triplet, doc_end, comments) | acc]
     state = %{state | comments: comments}
     each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun)
   end
@@ -1957,6 +1956,15 @@ defmodule Code.Formatter do
     {acc, rest, comments?}
   end
 
+  # If the document is immediately followed by comment which is followed by newlines,
+  # its newlines wouldn't have considerd the comment, so we need to adjust it.
+  defp adjust_trailing_newlines({doc, next_line, newlines}, doc_end, [{line, _, _} | _])
+       when newlines > 1 and line == doc_end + 1 do
+    {doc, next_line, 1}
+  end
+
+  defp adjust_trailing_newlines(doc_triplet, _, _), do: doc_triplet
+
   defp traverse_line({expr, meta, args}, {min, max}) do
     acc =
       case Keyword.fetch(meta, :line) do

lib/elixir/lib/gen_server.ex

@@ -640,7 +640,7 @@ defmodule GenServer do
   This callback is optional.
   """
   @callback terminate(reason, state :: term) :: term
-            when reason: :normal | :shutdown | {:shutdown, term}
+            when reason: :normal | :shutdown | {:shutdown, term} | term
 
   @doc """
   Invoked to change the state of the `GenServer` when a different version of a

lib/elixir/lib/kernel.ex

@@ -4104,7 +4104,7 @@ defmodule Kernel do
       end
 
   """
-  defmacro def(call, expr \\ []) do
+  defmacro def(call, expr \\ nil) do
     define(:def, call, expr, __CALLER__)
   end
 
@@ -4134,7 +4134,7 @@ defmodule Kernel do
       ** (UndefinedFunctionError) undefined function Foo.sum/2
 
   """
-  defmacro defp(call, expr \\ []) do
+  defmacro defp(call, expr \\ nil) do
     define(:defp, call, expr, __CALLER__)
   end
 
@@ -4162,7 +4162,7 @@ defmodule Kernel do
       end
 
   """
-  defmacro defmacro(call, expr \\ []) do
+  defmacro defmacro(call, expr \\ nil) do
     define(:defmacro, call, expr, __CALLER__)
   end
 
@@ -4178,7 +4178,7 @@ defmodule Kernel do
   naming and default arguments.
 
   """
-  defmacro defmacrop(call, expr \\ []) do
+  defmacro defmacrop(call, expr \\ nil) do
     define(:defmacrop, call, expr, __CALLER__)
   end
 
@@ -4673,7 +4673,7 @@ defmodule Kernel do
             macro_definition =
               case impls do
                 [] ->
-                  define(kind, call, [], env)
+                  define(kind, call, nil, env)
 
                 [guard] ->
                   quoted =

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -239,7 +239,7 @@ defmodule Kernel.ParallelCompiler do
   defp checker_runtime_modules(modules) do
     for module <- modules,
         path = :code.which(module),
-        is_list(path) do
+        is_list(path) and path != [] do
       {module, File.read!(path)}
     end
   end
@@ -427,10 +427,11 @@ defmodule Kernel.ParallelCompiler do
 
   # The goal of this function is to find leaves in the dependency graph,
   # i.e. to find code that depends on code that we know is not being defined.
+  # Note that not all files have been compile yet, so they may not be in waiting.
   defp without_definition(waiting, files) do
     nillify_empty(
       for {pid, _, _, _} <- files,
-          {_, ^pid, ref, on, _, _} = List.keyfind(waiting, pid, 1),
+          {_, ^pid, ref, on, _, _} <- List.wrap(List.keyfind(waiting, pid, 1)),
           not Enum.any?(waiting, fn {_, _, _, _, defining, _} -> on in defining end),
           do: {ref, :not_found}
     )

lib/elixir/lib/kernel/special_forms.ex

@@ -14,7 +14,8 @@ defmodule Kernel.SpecialForms do
   forms used to define tuple and binary data structures respectively.
 
   This module also documents macros that return information about Elixir's
-  compilation environment, such as (`__ENV__/0`, `__MODULE__/0`, `__DIR__/0` and `__CALLER__/0`).
+  compilation environment, such as (`__ENV__/0`, `__MODULE__/0`, `__DIR__/0`,
+  `__STACKTRACE__/0`, and `__CALLER__/0`).
 
   Additionally, it documents two special forms, `__block__/1` and
   `__aliases__/1`, which are not intended to be called directly by the

lib/elixir/lib/port.ex

@@ -132,9 +132,9 @@ defmodule Port do
   While we encourage graceful termination by detecting if stdin/stdout has been
   closed, we do not always have control over how third-party software terminates.
   In those cases, you can wrap the application in a script that checks for stdin.
-  Here is such script in `sh`:
+  Here is such script that has been verified to work on bash shells:
 
-      #!/bin/sh
+      #!/usr/bin/env bash
 
       # Start the program in the background
       exec "$@" &

lib/elixir/lib/string.ex

@@ -455,13 +455,13 @@ defmodule String do
   For example, take the grapheme "é" which is made of the characters
   "e" and the acute accent. The following will split the string into two parts:
 
-      iex> String.split(:unicode.characters_to_nfd_binary("é"), "e")
+      iex> String.split(String.normalize("é", :nfd), "e")
       ["", "́"]
 
   However, if "é" is represented by the single character "e with acute"
   accent, then it will split the string into just one part:
 
-      iex> String.split(:unicode.characters_to_nfc_binary("é"), "e")
+      iex> String.split(String.normalize("é", :nfc), "e")
       ["é"]
 
   """
@@ -650,9 +650,9 @@ defmodule String do
 
       String.normalize(string1, :nfd) == String.normalize(string2, :nfd)
 
-  Therefore, if you plan to compare multiple strings, multiple times
-  in a row, you may normalize them upfront and compare them directly
-  to avoid multiple normalization passes.
+  If you plan to compare multiple strings, multiple times in a row, you
+  may normalize them upfront and compare them directly to avoid multiple
+  normalization passes.
 
   ## Examples
 
@@ -674,21 +674,49 @@ defmodule String do
     normalize(string1, :nfd) == normalize(string2, :nfd)
   end
 
-  @doc false
-  @deprecated "Use :unicode.characters_to_nfc_binary/1 or :unicode.characters_to_nfd_binary/1 instead"
+  @doc """
+  Converts all characters in `string` to Unicode normalization
+  form identified by `form`.
+
+  Invalid Unicode codepoints are skipped and the remaining of
+  the string is converted. If you want the algorith to stop
+  and return on invalid codepoint, use `:unicode.characters_to_nfd_binary/1`
+  and `:unicode.characters_to_nfc_binary/1` instead.
+
+  ## Forms
+
+  The supported forms are:
+
+    * `:nfd` - Normalization Form Canonical Decomposition.
+      Characters are decomposed by canonical equivalence, and
+      multiple combining characters are arranged in a specific
+      order.
+
+    * `:nfc` - Normalization Form Canonical Composition.
+      Characters are decomposed and then recomposed by canonical equivalence.
+
+  ## Examples
+
+      iex> String.normalize("yêṩ", :nfd)
+      "yêṩ"
+
+      iex> String.normalize("leña", :nfc)
+      "leña"
+
+  """
   def normalize(string, form)
 
   def normalize(string, :nfd) do
     case :unicode.characters_to_nfd_binary(string) do
       string when is_binary(string) -> string
-      {:error, bad, rest} -> bad <> normalize(rest, :nfd)
+      {:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfd)
     end
   end
 
   def normalize(string, :nfc) do
     case :unicode.characters_to_nfc_binary(string) do
       string when is_binary(string) -> string
-      {:error, bad, rest} -> bad <> normalize(rest, :nfc)
+      {:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfc)
     end
   end
 
@@ -2174,13 +2202,13 @@ defmodule String do
   For example, take the grapheme "é" which is made of the characters
   "e" and the acute accent. The following returns `true`:
 
-      iex> String.contains?(:unicode.characters_to_nfd_binary("é"), "e")
+      iex> String.contains?(String.normalize("é", :nfd), "e")
       true
 
   However, if "é" is represented by the single character "e with acute"
   accent, then it will return `false`:
 
-      iex> String.contains?(:unicode.characters_to_nfc_binary("é"), "e")
+      iex> String.contains?(String.normalize("é", :nfc), "e")
       false
 
   """

lib/elixir/pages/Compatibility and Deprecations.md

@@ -81,7 +81,6 @@ Version | Deprecated feature                                  | Replaced by (ava
 [v1.10] | Passing non-chardata to `Logger.log/2`              | Explicitly convert to string with `to_string/1` (v1.0)
 [v1.10] | `:compile_time_purge_level` in `Logger` app environment | `:compile_time_purge_matching` in `Logger` app environment (v1.7)
 [v1.10] | `Supervisor.Spec.supervise/2`                       | The new child specs outlined in `Supervisor` (v1.5)
-[v1.10] | `String.normalize/2`                                | `:unicode.characters_to_nfc_binary/1` or `:unicode.characters_to_nfd_binary/1` (Erlang/OTP 20)
 [v1.10] | `:simple_one_for_one` strategy in `Supervisor`      | `DynamicSupervisor` (v1.6)
 [v1.10] | `:restart` and `:shutdown` in `Task.Supervisor.start_link/1` | `:restart` and `:shutdown` in `Task.Supervisor.start_child/3` (v1.6)
 [v1.9]  | Enumerable keys in `Map.drop/2`, `Map.split/2`, and `Map.take/2` | Call `Enum.to_list/1` on the second argument before hand (v1.0)

lib/elixir/src/elixir_bootstrap.erl

@@ -10,11 +10,11 @@
 'MACRO-@'(Caller, Tree) ->
   unless_loaded('MACRO-@', [Caller, Tree], fun() -> nil end).
 
-'MACRO-def'(Caller, Call) -> 'MACRO-def'(Caller, Call, []).
+'MACRO-def'(Caller, Call) -> 'MACRO-def'(Caller, Call, nil).
 'MACRO-def'(Caller, Call, Expr) -> define(Caller, def, Call, Expr).
 'MACRO-defp'(Caller, Call, Expr) -> define(Caller, defp, Call, Expr).
 
-'MACRO-defmacro'(Caller, Call) -> 'MACRO-defmacro'(Caller, Call, []).
+'MACRO-defmacro'(Caller, Call) -> 'MACRO-defmacro'(Caller, Call, nil).
 'MACRO-defmacro'(Caller, Call, Expr) -> define(Caller, defmacro, Call, Expr).
 'MACRO-defmacrop'(Caller, Call, Expr) -> define(Caller, defmacrop, Call, Expr).
 

lib/elixir/src/elixir_def.erl

@@ -212,7 +212,7 @@ run_with_location_change(File, #{file := File} = E, Callback) ->
 run_with_location_change(File, E, Callback) ->
   elixir_lexical:with_file(File, E, Callback).
 
-def_to_clauses(_Kind, Meta, Args, [], [], E) ->
+def_to_clauses(_Kind, Meta, Args, [], nil, E) ->
   check_args_for_function_head(Meta, Args, E),
   [];
 def_to_clauses(_Kind, Meta, Args, Guards, [{do, Body}], _E) ->

lib/elixir/src/elixir_erl_compiler.erl

@@ -48,8 +48,10 @@ compile(Forms, File, Opts) when is_list(Forms), is_list(Opts), is_binary(File) -
   case erl_to_core(Forms, Opts) of
     {ok, CoreForms, CoreWarnings} ->
       format_warnings(Opts, CoreWarnings),
+      CompileOpts = [?NO_SPAWN_COMPILER_PROCESS, from_core, no_core_prepare,
+                     no_auto_import, return, {source, Source} | Opts],
 
-      case compile:noenv_forms(CoreForms, [?NO_SPAWN_COMPILER_PROCESS, from_core, no_auto_import, return, {source, Source} | Opts]) of
+      case compile:noenv_forms(CoreForms, CompileOpts) of
         {ok, Module, Binary, Warnings} when is_binary(Binary) ->
           format_warnings(Opts, Warnings),
           {Module, Binary};

lib/elixir/src/elixir_lexical.erl

@@ -10,6 +10,7 @@ run(#{tracers := Tracers} = E, ExecutionCallback, AfterExecutionCallback) ->
     false ->
       {ok, Pid} = ?tracker:start_link(),
       LexEnv = E#{lexical_tracker := Pid, tracers := [?MODULE | Tracers]},
+      elixir_env:trace(start, LexEnv),
 
       try ExecutionCallback(LexEnv) of
         Res ->
@@ -18,6 +19,7 @@ run(#{tracers := Tracers} = E, ExecutionCallback, AfterExecutionCallback) ->
           AfterExecutionCallback(LexEnv),
           Res
       after
+        elixir_env:trace(stop, LexEnv),
         unlink(Pid),
         ?tracker:stop(Pid)
       end;

lib/elixir/test/elixir/code_formatter/comments_test.exs

@@ -102,6 +102,108 @@ defmodule Code.Formatter.CommentsTest do
     end
   end
 
+  describe "modules attributes" do
+    test "with comments around" do
+      assert_same """
+      defmodule Sample do
+        # Comment 0
+        @moduledoc false
+        # Comment 1
+
+        # Comment 2
+        @attr1 1
+        # Comment 3
+
+        # Comment 4
+        @doc "Doc"
+        # Comment 5
+        @attr2 2
+        # Comment 6
+        def sample, do: :sample
+      end
+      """
+    end
+
+    test "with comments only after" do
+      assert_same """
+      @moduledoc false
+      # Comment 1
+
+      @attr 1
+      """
+    end
+
+    test "with too many new lines" do
+      bad = """
+      defmodule Sample do
+
+        # Comment 0
+
+
+        @moduledoc false
+
+
+        # Comment 1
+
+
+        # Comment 2
+
+
+        @attr1 1
+
+
+        # Comment 3
+
+
+        # Comment 4
+
+
+        @doc "Doc"
+
+
+        # Comment 5
+
+
+        @attr2 2
+
+
+        # Comment 6
+
+
+        def sample, do: :sample
+      end
+      """
+
+      assert_format bad, """
+      defmodule Sample do
+        # Comment 0
+
+        @moduledoc false
+
+        # Comment 1
+
+        # Comment 2
+
+        @attr1 1
+
+        # Comment 3
+
+        # Comment 4
+
+        @doc "Doc"
+
+        # Comment 5
+
+        @attr2 2
+
+        # Comment 6
+
+        def sample, do: :sample
+      end
+      """
+    end
+  end
+
   describe "interpolation" do
     test "with comment outside before, during and after" do
       assert_same ~S"""

lib/elixir/test/elixir/code_test.exs

@@ -103,12 +103,18 @@ defmodule CodeTest do
   test "compile_file/1" do
     assert Code.compile_file(fixture_path("code_sample.exs")) == []
     refute fixture_path("code_sample.exs") in Code.required_files()
+
+    assert [{CompileSample, binary}] = Code.compile_file(fixture_path("compile_sample.ex"))
+    assert is_binary(binary)
+  after
+    :code.purge(CompileSample)
+    :code.delete(CompileSample)
   end
 
   test "compile_file/1 also emits checker warnings" do
     output =
       ExUnit.CaptureIO.capture_io(:stderr, fn ->
-        Code.compile_file(PathHelpers.fixture_path("checker_warning.exs"))
+        Code.compile_file(fixture_path("checker_warning.exs"))
       end)
 
     assert output =~ "incompatible types"
@@ -122,8 +128,13 @@ defmodule CodeTest do
     Code.unrequire_files([fixture_path("code_sample.exs")])
     refute fixture_path("code_sample.exs") in Code.required_files()
     assert Code.require_file(fixture_path("code_sample.exs")) != nil
+
+    assert [{CompileSample, binary}] = Code.require_file(fixture_path("compile_sample.ex"))
+    assert is_binary(binary)
   after
-    Code.unrequire_files([fixture_path("code_sample.exs")])
+    Code.unrequire_files([fixture_path("code_sample.exs"), fixture_path("compile_sample.ex")])
+    :code.purge(CompileSample)
+    :code.delete(CompileSample)
   end
 
   describe "string_to_quoted/2" do

lib/elixir/test/elixir/kernel/tracers_test.exs

@@ -20,6 +20,15 @@ defmodule Kernel.TracersTest do
     end)
   end
 
+  test "traces start and stop" do
+    compile_string("""
+    Foo
+    """)
+
+    assert_receive {:start, %{lexical_tracker: pid}} when is_pid(pid)
+    assert_receive {:stop, %{lexical_tracker: pid}} when is_pid(pid)
+  end
+
   test "traces alias references" do
     compile_string("""
     Foo

lib/elixir/test/elixir/regex_test.exs

@@ -91,8 +91,10 @@ defmodule RegexTest do
     assert <<0xA0::utf8>> =~ ~r/[[:space:]]/u
     assert <<0xA0::utf8>> =~ ~r/\s/u
 
+    # Erlang/OTP 23 raises badarg on invalid UTF-8.
+    # Earlier versions simply would not match.
+    assert catch_error(if <<?<, 255, ?>>> =~ ~r/<.>/u, do: flunk("failed"), else: raise("failed"))
     assert <<?<, 255, ?>>> =~ ~r/<.>/
-    refute <<?<, 255, ?>>> =~ ~r/<.>/u
   end
 
   test "ungreedy" do
@@ -349,6 +351,6 @@ defmodule RegexTest do
   end
 
   defp matches_escaped?(string, match) do
-    Regex.match?(~r/#{Regex.escape(string)}/simxu, match)
+    Regex.match?(~r/#{Regex.escape(string)}/simx, match)
   end
 end

lib/elixir/test/elixir/string_test.exs

@@ -749,4 +749,79 @@ defmodule StringTest do
     assert String.myers_difference("abc", "aйbc") == [eq: "a", ins: "й", eq: "bc"]
     assert String.myers_difference("aйbc", "abc") == [eq: "a", del: "й", eq: "bc"]
   end
+
+  test "normalize/2" do
+    assert String.normalize("ŝ", :nfd) == "ŝ"
+    assert String.normalize("ḇravô", :nfd) == "ḇravô"
+    assert String.normalize("ṩierra", :nfd) == "ṩierra"
+    assert String.normalize("뢴", :nfd) == "뢴"
+    assert String.normalize("êchǭ", :nfc) == "êchǭ"
+    assert String.normalize("거̄", :nfc) == "거̄"
+    assert String.normalize("뢴", :nfc) == "뢴"
+
+    ## Error cases
+    assert String.normalize(<<15, 216>>, :nfc) == <<15, 216>>
+    assert String.normalize(<<15, 216>>, :nfd) == <<15, 216>>
+    assert String.normalize(<<216, 15>>, :nfc) == <<216, 15>>
+    assert String.normalize(<<216, 15>>, :nfd) == <<216, 15>>
+
+    ## Cases from NormalizationTest.txt
+
+    # 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F
+    # 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F
+    # HEBREW POINT QAMATS, HEBREW POINT HOLAM, HEBREW POINT HATAF SEGOL,
+    # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA,
+    # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA
+    assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟"
+
+    # 095D (exclusion list)
+    # 0922 093C
+    # DEVANAGARI LETTER RHA
+    assert String.normalize("ढ़", :nfc) == "ढ़"
+
+    # 0061 0315 0300 05AE 0340 0062
+    # 00E0 05AE 0300 0315 0062
+    # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT,
+    # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B
+    assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b"
+
+    # 0344
+    # 0308 0301
+    # COMBINING GREEK DIALYTIKA TONOS
+    assert String.normalize("\u0344", :nfc) == "\u0308\u0301"
+
+    # 115B9 0334 115AF
+    # 115B9 0334 115AF
+    # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA
+    assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯"
+    # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA,
+    # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA
+    assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟"
+
+    # 095D (exclusion list)
+    # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA,
+    # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA
+    assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟"
+
+    # 095D (exclusion list)
+    # 0922 093C
+    # DEVANAGARI LETTER RHA
+    assert String.normalize("ढ़", :nfc) == "ढ़"
+
+    # 0061 0315 0300 05AE 0340 0062
+    # 00E0 05AE 0300 0315 0062
+    # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT,
+    # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B
+    assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b"
+
+    # 0344
+    # 0308 0301
+    # COMBINING GREEK DIALYTIKA TONOS
+    assert String.normalize("\u0344", :nfc) == "\u0308\u0301"
+
+    # 115B9 0334 115AF
+    # 115B9 0334 115AF
+    # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA
+    assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯"
+  end
 end

lib/ex_unit/lib/ex_unit/diff.ex

@@ -57,7 +57,8 @@ defmodule ExUnit.Diff do
   end
 
   defp diff_quoted({name, _, context} = left, right, env)
-       when is_atom(name) and is_atom(context) do
+       when is_atom(name) and is_atom(context) and
+              name not in [:__MODULE__, :__DIR__, :__STACKTRACE__, :__ENV__, :__CALLER__] do
     diff_var(left, right, env)
   end
 
@@ -81,20 +82,16 @@ defmodule ExUnit.Diff do
     diff_tuple(Tuple.to_list(left), Tuple.to_list(right), env)
   end
 
-  defp diff_quoted({:%, _, [struct, {:%{}, _, kw}]}, %{} = right, env)
-       when is_atom(struct) and is_list(kw) do
-    diff_quoted_struct([__struct__: struct] ++ kw, struct, right, env)
+  defp diff_quoted({:%, _, [struct, {:%{}, _, kw}]}, %{} = right, env) when is_list(kw) do
+    diff_quoted_struct([__struct__: struct] ++ kw, right, env)
   end
 
-  defp diff_quoted({:%{}, _, items}, %{} = right, env) when is_list(items) do
-    if struct = items[:__struct__] do
-      diff_quoted_struct(items, struct, right, env)
-    else
-      diff_map(items, right, nil, maybe_struct(right), env)
-    end
+  defp diff_quoted({:%{}, _, kw}, %{} = right, env) when is_list(kw) do
+    diff_quoted_struct(kw, right, env)
   end
 
-  defp diff_quoted({:<>, _, _} = left, right, env) when is_binary(right) do
+  defp diff_quoted({:<>, _, [literal, _]} = left, right, env)
+       when is_binary(literal) and is_binary(right) do
     diff_string_concat(left, right, env)
   end
 
@@ -225,7 +222,7 @@ defmodule ExUnit.Diff do
   defp diff_pin({:^, _, [var]} = pin, right, %{pins: pins} = env) do
     identifier = var_context(var)
     %{^identifier => pin_value} = pins
-    {diff, post_env} = diff(pin_value, right, env)
+    {diff, post_env} = diff_value(pin_value, right, env)
 
     diff_left = update_diff_meta(pin, not diff.equivalent?)
     {%{diff | left: diff_left}, post_env}
@@ -626,8 +623,9 @@ defmodule ExUnit.Diff do
 
   # Structs
 
-  defp diff_quoted_struct(kw, struct1, right, env) do
-    left = load_struct(struct1)
+  defp diff_quoted_struct(kw, right, env) do
+    struct1 = kw[:__struct__]
+    left = load_struct(kw[:__struct__])
 
     if left && Enum.all?(kw, fn {k, _} -> Map.has_key?(left, k) end) do
       if Macro.quoted_literal?(kw) do
@@ -674,7 +672,8 @@ defmodule ExUnit.Diff do
   end
 
   defp load_struct(struct) do
-    if Code.ensure_loaded?(struct) and function_exported?(struct, :__struct__, 0) do
+    if is_atom(struct) and struct != nil and
+         Code.ensure_loaded?(struct) and function_exported?(struct, :__struct__, 0) do
       struct.__struct__
     end
   end
@@ -761,11 +760,10 @@ defmodule ExUnit.Diff do
     String.bag_distance(left, right) > 0.4
   end
 
-  defp parse_string({:<>, _, [literal, rest]}) do
+  defp parse_string({:<>, _, [literal, rest]}) when is_binary(literal) do
     {parsed, quoted, indexes, parsed_length} = parse_string(rest)
     literal_length = String.length(literal)
     length = literal_length + parsed_length
-
     {literal <> parsed, quoted, [literal_length | indexes], length}
   end
 

lib/ex_unit/test/ex_unit/diff_test.exs

@@ -141,6 +141,11 @@ defmodule ExUnit.DiffTest do
     refute_diff(^b = :a, "-^b-", "+:a+", pins)
   end
 
+  test "pseudo vars" do
+    assert_diff(__MODULE__ = ExUnit.DiffTest, [])
+    refute_diff(__MODULE__ = SomethingElse, "-__MODULE__-", "+SomethingElse+")
+  end
+
   test "integers" do
     assert_diff(123 = 123, [])
     assert_diff(-123 = -123, [])
@@ -471,6 +476,34 @@ defmodule ExUnit.DiffTest do
     refute_diff(%{a: 1} = :a, "-%{a: 1}-", "+:a+")
   end
 
+  test "maps as pinned map value" do
+    user = %{"id" => 13, "name" => "john"}
+
+    notification = %{
+      "user" => user,
+      "subtitle" => "foo"
+    }
+
+    assert_diff(
+      %{
+        "user" => ^user,
+        "subtitle" => "foo"
+      } = notification,
+      [],
+      %{{:user, nil} => user}
+    )
+
+    refute_diff(
+      %{
+        "user" => ^user,
+        "subtitle" => "bar"
+      } = notification,
+      ~s|%{"subtitle" => "-bar-", "user" => ^user}|,
+      ~s|%{"subtitle" => "+foo+", "user" => %{"id" => 13, "name" => "john"}}|,
+      %{{:user, nil} => user}
+    )
+  end
+
   test "maps outside match context" do
     assert_diff(%{a: 1} == %{a: 1}, [])
     assert_diff(%{a: 1, b: 2} == %{a: 1, b: 2}, [])
@@ -594,7 +627,7 @@ defmodule ExUnit.DiffTest do
     )
   end
 
-  test "structs with inspect difference" do
+  test "structs with same inspect but different" do
     refute_diff(
       %Opaque{data: 1} = %Opaque{data: 2},
       "%ExUnit.DiffTest.Opaque{data: -1-}",
@@ -608,7 +641,7 @@ defmodule ExUnit.DiffTest do
     )
   end
 
-  test "structs without inspect difference outside match" do
+  test "structs with same inspect but different outside match" do
     refute_diff(
       %Opaque{data: 1} == %Opaque{data: 2},
       "%ExUnit.DiffTest.Opaque{data: -1-}",
@@ -630,6 +663,66 @@ defmodule ExUnit.DiffTest do
     )
   end
 
+  test "structs with matched type" do
+    pins = %{{:type, nil} => User, {:age, nil} => 33}
+
+    # pin on __struct__
+    assert_diff(
+      %{__struct__: ^type, age: ^age, name: "john"} = %User{name: "john", age: 33},
+      [],
+      pins
+    )
+
+    refute_diff(
+      %{__struct__: ^type, age: ^age, name: "john"} = %User{name: "jane", age: 33},
+      "%{__struct__: ^type, age: ^age, name: \"j-oh-n\"}",
+      "%ExUnit.DiffTest.User{age: 33, name: \"j+a+n+e+\"}",
+      pins
+    )
+
+    refute_diff(
+      %{__struct__: ^type, age: ^age, name: "john"} = %User{name: "john", age: 35},
+      "%{__struct__: ^type, age: -^age-, name: \"john\"}",
+      "%ExUnit.DiffTest.User{age: 3+5+, name: \"john\"}",
+      pins
+    )
+
+    refute_diff(
+      %{__struct__: ^type, age: ^age, name: "john"} = ~D[2020-01-01],
+      "%{__struct__: -^type-, -age: ^age-, -name: \"john\"-}",
+      "%+Date+{calendar: Calendar.ISO, day: 1, month: 1, year: 2020}",
+      pins
+    )
+
+    # pin on %
+    assert_diff(
+      %^type{age: ^age, name: "john"} = %User{name: "john", age: 33},
+      [],
+      pins
+    )
+
+    refute_diff(
+      %^type{age: ^age, name: "john"} = %User{name: "jane", age: 33},
+      "%{__struct__: ^type, age: ^age, name: \"j-oh-n\"}",
+      "%ExUnit.DiffTest.User{age: 33, name: \"j+a+n+e+\"}",
+      pins
+    )
+
+    refute_diff(
+      %^type{age: ^age, name: "john"} = %User{name: "john", age: 35},
+      "%{__struct__: ^type, age: -^age-, name: \"john\"}",
+      "%ExUnit.DiffTest.User{age: 3+5+, name: \"john\"}",
+      pins
+    )
+
+    refute_diff(
+      %^type{age: ^age, name: "john"} = ~D[2020-01-01],
+      "%{__struct__: -^type-, -age: ^age-, -name: \"john\"-}",
+      "%+Date+{calendar: Calendar.ISO, day: 1, month: 1, year: 2020}",
+      pins
+    )
+  end
+
   test "invalid structs" do
     refute_diff(
       %{__struct__: Unknown} = %{},
@@ -721,7 +814,7 @@ defmodule ExUnit.DiffTest do
     )
   end
 
-  test "concat operator" do
+  test "concat binaries" do
     assert_diff("fox hops" <> " over the dog" = "fox hops over the dog", [])
     assert_diff("fox hops " <> "over " <> "the dog" = "fox hops over the dog", [])
 
@@ -755,6 +848,10 @@ defmodule ExUnit.DiffTest do
       ~s/"fox hops over the dog"/
     )
 
+    refute_diff("fox" <> " hops" = :a, ~s/-"fox" <> " hops"-/, "+:a+")
+  end
+
+  test "concat binaries with pin" do
     pins = %{{:x, nil} => " over the dog"}
 
     assert_diff("fox hops" <> x = "fox hops over the dog", x: " over the dog")
@@ -773,8 +870,16 @@ defmodule ExUnit.DiffTest do
       ~s/"fox hops over +t+he dog"/,
       pins
     )
+  end
 
-    refute_diff("fox" <> " hops" = :a, ~s/-"fox" <> " hops"-/, "+:a+")
+  test "concat binaries with specifiers" do
+    input = "foobar"
+
+    refute_diff(
+      <<trap::binary-size(3)>> <> "baz" = input,
+      "-<<trap::binary-size(3)>> <> \"baz\"-",
+      "+\"foobar\"+"
+    )
   end
 
   test "underscore" do

lib/iex/test/iex/helpers_test.exs

@@ -382,7 +382,7 @@ defmodule IEx.HelpersTest do
         "* def left == right\n\n  @spec term() == term() :: boolean()\n\nguard: true\n\nReturns `true` if the two terms are equal.\n\n"
 
       def_h =
-        "* defmacro def(call, expr \\\\ [])\n\nDefines a public function with the given name and body."
+        "* defmacro def(call, expr \\\\ nil)\n\nDefines a public function with the given name and body."
 
       assert capture_io(fn -> h(IEx.Helpers.pwd() / 0) end) =~ pwd_h
       assert capture_io(fn -> h(IEx.Helpers.c() / 2) end) =~ c_h

lib/mix/lib/mix/dep.ex

@@ -66,7 +66,7 @@ defmodule Mix.Dep do
           scm: Mix.SCM.t(),
           app: atom,
           requirement: String.t() | Regex.t() | nil,
-          status: atom,
+          status: {:ok, String.t() | nil} | atom | tuple,
           opts: keyword,
           top_level: boolean,
           manager: :rebar | :rebar3 | :mix | :make | nil,

lib/mix/lib/mix/dep/fetcher.ex

@@ -65,6 +65,7 @@ defmodule Mix.Dep.Fetcher do
           end
 
         if new do
+          File.touch!(Path.join(opts[:dest], ".fetch"))
           dep = put_in(dep.opts[:lock], new)
           {dep, [app | acc], Map.put(lock, app, new)}
         else
@@ -92,22 +93,18 @@ defmodule Mix.Dep.Fetcher do
     # dependency is missing, it could directly affect one of the
     # dependencies we are trying to compile, causing the whole thing
     # to fail.
-    #
-    # If there is any other dependency that is not ok, we include
-    # it for compilation too, this is our best to try to solve the
-    # maximum we can at each deps.get and deps.update.
-    deps =
+    parent_deps =
       if Enum.all?(all_deps, &available?/1) do
         Enum.uniq_by(with_depending(deps, all_deps), & &1.app)
       else
-        deps
+        []
       end
 
     # Merge the new lock on top of the old to guarantee we don't
     # leave out things that could not be fetched and save it.
     lock = Map.merge(old_lock, new_lock)
     Mix.Dep.Lock.write(lock)
-    mark_as_fetched(deps)
+    mark_as_fetched(parent_deps)
 
     # See if any of the deps diverged and abort.
     show_diverged!(Enum.filter(all_deps, &Mix.Dep.diverged?/1))

lib/mix/lib/mix/tasks/release.ex

@@ -481,7 +481,9 @@ defmodule Mix.Tasks.Release do
   will receive a `Mix.Release` struct and must return the same or
   an updated `Mix.Release` struct. It is also possible to build a tarball
   of the release by passing the `:tar` step anywhere after `:assemble`.
-  The tarball is created in `_build/MIX_ENV/RELEASE_NAME-RELEASE_VSN.tar.gz`
+  If the release `:path` is not configured, the tarball is created in
+  `_build/MIX_ENV/RELEASE_NAME-RELEASE_VSN.tar.gz` Otherwise it is
+  created inside the configured `:path`.
 
   See `Mix.Release` for more documentation on the struct and which
   fields can be modified. Note that `:steps` field itself can be
@@ -1065,8 +1067,16 @@ defmodule Mix.Tasks.Release do
   end
 
   defp make_tar(release) do
-    tar_filename = "#{release.name}-#{release.version}.tar.gz"
-    out_path = Path.join([release.path, "..", "..", tar_filename]) |> Path.expand()
+    build_path = Mix.Project.build_path()
+
+    dir_path =
+      if release.path == Path.join([build_path, "rel", Atom.to_string(release.name)]) do
+        build_path
+      else
+        release.path
+      end
+
+    out_path = Path.join(dir_path, "#{release.name}-#{release.version}.tar.gz")
     info(release, [:green, "* building ", :reset, out_path])
 
     lib_dirs =

lib/mix/test/mix/tasks/compile.elixir_test.exs

@@ -201,6 +201,32 @@ defmodule Mix.Tasks.Compile.ElixirTest do
     end)
   end
 
+  test "compiles dependent changed modules without beam files" do
+    in_fixture("no_mixfile", fn ->
+      File.write!("lib/b.ex", """
+      defmodule B do
+        def a, do: A.__info__(:module)
+      end
+      """)
+
+      Mix.Tasks.Compile.Elixir.run(["--verbose"])
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      assert_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
+
+      assert File.regular?("_build/dev/lib/sample/ebin/Elixir.A.beam")
+      assert File.regular?("_build/dev/lib/sample/ebin/Elixir.B.beam")
+
+      Code.put_compiler_option(:ignore_module_conflict, true)
+      Code.compile_file("lib/b.ex")
+      File.touch!("lib/a.ex", {{2038, 1, 1}, {0, 0, 0}})
+
+      Mix.Tasks.Compile.Elixir.run(["--verbose"])
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+    end)
+  after
+    Code.put_compiler_option(:ignore_module_conflict, false)
+  end
+
   test "compiles dependent changed modules even on removal" do
     in_fixture("no_mixfile", fn ->
       File.write!("lib/a.ex", "defmodule A, do: B.module_info()")

lib/mix/test/mix/tasks/release_test.exs

@@ -135,7 +135,7 @@ defmodule Mix.Tasks.ReleaseTest do
   end
 
   describe "tar" do
-    test "with ERTS" do
+    test "with default options" do
       in_fixture("release_test", fn ->
         config = [releases: [demo: [steps: [:assemble, :tar]]]]
 
@@ -186,15 +186,15 @@ defmodule Mix.Tasks.ReleaseTest do
       end)
     end
 
-    test "without ERTS" do
+    test "without ERTS and custom path" do
       in_fixture("release_test", fn ->
-        config = [releases: [demo: [include_erts: false, steps: [:assemble, :tar]]]]
+        config = [
+          releases: [demo: [include_erts: false, path: "tmp/rel", steps: [:assemble, :tar]]]
+        ]
 
         Mix.Project.in_project(:release_test, ".", config, fn _ ->
-          root = Path.absname("_build/#{Mix.env()}/rel/demo")
-
           Mix.Task.run("release")
-          tar_path = Path.expand(Path.join([root, "..", "..", "demo-0.1.0.tar.gz"]))
+          tar_path = Path.expand(Path.join(["tmp", "rel", "demo-0.1.0.tar.gz"]))
           message = "* building #{tar_path}"
           assert_received {:mix_shell, :info, [^message]}
           assert File.exists?(tar_path)