Change wording from "change" to "rebind" (#8475)

Prior to this change, calling a protocol function with an opaque type
would yield a warning, as Dialyzer concludes that the impl_for/1
function can't handle opaque arguments, since all clauses would
destructure their arguments in some way.

By adding a catch-all clause that does not destructure its argument,
Dialyzer no longer draws this conclusion, and the warnings go away.

As noted in the protocol.ex comment, this is technically a hack as it
relies on Dialyzer not being smart enough. However, I would not expect
it to break soon, if ever.

Signed-off-by: José Valim <jose.valim@plataformatec.com.br>

CHANGELOG.md

@@ -23,7 +23,7 @@ def format(message, opts) do
 end
 ```
 
-The `if` block above is implicitly changing the value in `message`. Now imagine we want to move the `if` block to its own function to clean up the implementation:
+The `if` block above is implicitly rebinding the outer scope variable `message` to a new value. Now imagine we want to move the `if` block to its own function to clean up the implementation:
 
 ```elixir
 def format(message, opts) do
@@ -268,6 +268,89 @@ end
 
 By restricting hierarchies in favor of named setups, it is straight-forward for the developer to glance at each describe block and know exactly the setup steps involved.
 
+## v1.3.4 (2016-10-09)
+
+### 1. Bug fixes
+
+#### Elixir
+
+  * [Kernel] Ensure the compiler does not generate unecessary variable bindings inside case statements. This improves the code emitted and make sure "unused variable warnings" are not mistakenly silenced
+  * [Kernel] Move `raise` checks to runtime to avoid crashing cover on Erlang 19.1
+  * [Protocol] Do not emit warnings when using protocols on opaque types
+
+#### ExUnit
+
+  * [ExUnit.CaptureLog] Flush Erlang's `:error_logger` before capturing to avoid mixed messages
+
+## v1.3.3 (2016-09-17)
+
+### 1. Enhancements
+
+#### Elixir
+
+  * [DateTime] Support negative integer in `DateTime.from_unix/2`
+  * [Kernel.LexicalTracker] Do not consider remote typespecs as a compile-time dependency
+  * [Kernel.ParallelCompiler] Do not emit deadlock messages when the process is waiting on itself
+  * [Kernel.Typespec] Mark struct update syntax as generated to avoid false positives from dialyzer
+
+#### ExUnit
+
+  * [ExUnit] Make ExUnit server timeout configurable
+
+#### Logger
+
+  * [Logger] Use `:ansi_color` if one is available in metadata
+
+#### Mix
+
+  * [Mix] Add support for the `:sparse` option in `Mix.SCM.Git`
+  * [Mix] Skip dependendency loading if `MIX_NO_DEPS` is set to `1`
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [System] Use `NUL` instead of `/dev/null` on Windows when building `System.build_info`
+
+#### IEx
+
+  * [IEx.Autocomplete] Resolves issue with autocompletion on structs not working
+
+#### Mix
+
+  * [Mix] Also store external resources that are not part of the current working directory in compilation manifest
+  * [Mix] Always include the compiled file source in manifests
+
+## v1.3.2 (2016-07-15)
+
+### 1. Enhancements
+
+#### Elixir
+
+  * [Kernel] Support guards in `else` clauses in `with`
+
+#### Mix
+
+  * [Mix] Add `MIX_NO_DEPS` env var for disabling dep loading. Used for third-party scripts and tools like Nix package manager
+  * [Mix] Add `mix test --listen-on-stdin` that automatically reruns tests on stdin
+  * [Mix] Disable `--warnings-as-errors` when compiling dependencies
+  * [Mix] Add `--filter` option to `mix deps.unlock` to unlock only matching dependencies
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [Enum] Return `nil` if enumerable halts in `Enum.find_index/3`
+  * [Kernel] Do not attempt to load modules that have not been required when invoking regular functions, otherwise this invalidates the `@compile {:autoload, false}` directive.
+
+#### Mix
+
+  * [Mix] Ensure missing protocol dependencies are discarded in umbrella projects with shared build
+
+#### ExUnit
+
+  * [ExUnit.Diff] Ensure no leading or trailing when diffing some maps
+
 ## v1.3.1 (2016-06-28)
 
 ### 1. Enhancements
@@ -434,3 +517,7 @@ This release deprecates many APIs that have been soft-deprecated in previous Eli
   * [String] `String.valid_character?/1` is deprecated in favor of `String.valid?/1` with pattern matching
   * [Task] `Task.find/2` is deprecated in favor of explicit message matching
   * [URI] Passing a non-map to `URI.decode_query/2` is deprecated
+
+## v1.2
+
+The CHANGELOG for v1.2 releases can be found [in the v1.2 branch](https://github.com/elixir-lang/elixir/blob/v1.2/CHANGELOG.md).

VERSION

@@ -1 +1 @@
-1.3.1
+1.3.4

lib/elixir/lib/access.ex

@@ -435,6 +435,13 @@ defmodule Access do
       ...>   prev -> {prev, String.upcase(prev)}
       ...> end)
       {"john", [%{name: "JOHN"}, %{name: "mary"}]}
+
+  `at/1` can also be used to pop elements out of a list or
+  a key inside of a list:
+
+      iex> list = [%{name: "john"}, %{name: "mary"}]
+      iex> pop_in(list, [Access.at(0)])
+      {%{name: "john"}, [%{name: "mary"}]}
       iex> pop_in(list, [Access.at(0), :name])
       {"john", [%{}, %{name: "mary"}]}
 
@@ -477,7 +484,7 @@ defmodule Access do
   defp get_and_update_at([head | rest], 0, next, updates) do
     case next.(head) do
       {get, update} -> {get, :lists.reverse([update | updates], rest)}
-      :pop -> {head, :lists.reverse([head | updates], rest)}
+      :pop -> {head, :lists.reverse(updates, rest)}
     end
   end
 

lib/elixir/lib/calendar.ex

@@ -1004,19 +1004,32 @@ defmodule DateTime do
       {:ok, %DateTime{calendar: Calendar.ISO, day: 23, hour: 22, microsecond: {211914, 3}, minute: 53,
                       month: 1, second: 43, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0,
                       year: 46302, zone_abbr: "UTC"}}
+  
+  Negative Unix times are supported, up to -#{@unix_epoch} seconds, 
+  which is equivalent to "0000-01-01T00:00:00Z" or 0 gregorian seconds.
 
+      iex> DateTime.from_unix(-12345678910)
+      {:ok, %DateTime{calendar: Calendar.ISO, day: 13, hour: 4, microsecond: {0, 0}, minute: 44, 
+                      month: 10, second: 50, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0, 
+                      year: 1578, zone_abbr: "UTC"}}
+  
+  When a Unix time before that moment is passed to `from_unix/2`, `:error` will be returned.
   """
-  @spec from_unix(non_neg_integer, :native | System.time_unit) :: {:ok, DateTime.t}
-  def from_unix(integer, unit \\ :seconds) when is_integer(integer) and integer >= 0 do
+  @spec from_unix(integer, :native | System.time_unit) :: {:ok, DateTime.t}
+  def from_unix(integer, unit \\ :seconds) when is_integer(integer) do
     total = System.convert_time_unit(integer, unit, :microseconds)
-    microsecond = rem(total, 1_000_000)
-    precision = precision_for_unit(unit)
-    {{year, month, day}, {hour, minute, second}} =
-      :calendar.gregorian_seconds_to_datetime(@unix_epoch + div(total, 1_000_000))
+    if total < -@unix_epoch * 1_000_000 do
+      :error  
+    else
+      microsecond = rem(total, 1_000_000)
+      precision = precision_for_unit(unit)
+      {{year, month, day}, {hour, minute, second}} =
+        :calendar.gregorian_seconds_to_datetime(@unix_epoch + div(total, 1_000_000))
 
-    {:ok, %DateTime{year: year, month: month, day: day,
-                    hour: hour, minute: minute, second: second, microsecond: {microsecond, precision},
-                    std_offset: 0, utc_offset: 0, zone_abbr: "UTC", time_zone: "Etc/UTC"}}
+      {:ok, %DateTime{year: year, month: month, day: day,
+                      hour: hour, minute: minute, second: second, microsecond: {microsecond, precision},
+                      std_offset: 0, utc_offset: 0, zone_abbr: "UTC", time_zone: "Etc/UTC"}}
+    end
   end
 
   def precision_for_unit(unit) do
@@ -1034,6 +1047,13 @@ defmodule DateTime do
   @doc """
   Converts the given Unix time to DateTime.
 
+  The integer can be given in different unit
+  according to `System.convert_time_unit/3` and it will
+  be converted to microseconds internally.
+
+  Unix times are always in UTC and therefore the DateTime
+  will be returned in UTC.
+
   ## Examples
 
       iex> DateTime.from_unix!(1464096368)
@@ -1046,6 +1066,15 @@ defmodule DateTime do
                 month: 5, second: 8, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0,
                 year: 2015, zone_abbr: "UTC"}
 
+  Negative Unix times are supported, up to -#{@unix_epoch} seconds, 
+  which is equivalent to "0000-01-01T00:00:00Z" or 0 gregorian seconds.
+
+      iex> DateTime.from_unix(-12345678910)
+      {:ok, %DateTime{calendar: Calendar.ISO, day: 13, hour: 4, microsecond: {0, 0}, minute: 44, 
+                      month: 10, second: 50, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0, 
+                      year: 1578, zone_abbr: "UTC"}}
+
+  When a Unix time before that moment is passed to `from_unix!/2`, an ArgumentError will be raised.
   """
   @spec from_unix!(non_neg_integer, :native | System.time_unit) :: DateTime.t
   def from_unix!(integer, unit \\ :seconds) when is_atom(unit) do
@@ -1057,7 +1086,7 @@ defmodule DateTime do
   Converts the given DateTime to Unix time.
 
   The DateTime is expected to be using the ISO calendar
-  with a year greater than or equal to 1970.
+  with a year greater than or equal to 0.
 
   It will return the integer with the given unit,
   according to `System.convert_time_unit/3`.
@@ -1073,11 +1102,19 @@ defmodule DateTime do
       iex> DateTime.to_unix(dt)
       1416517099
 
+      iex> flamel = %DateTime{calendar: Calendar.ISO, day: 22, hour: 8, microsecond: {527771, 6},
+      ...>                minute: 2, month: 3, second: 25, std_offset: 0, time_zone: "Etc/UTC",
+      ...>                utc_offset: 0, year: 1418, zone_abbr: "UTC"}
+      iex> DateTime.to_unix(flamel)
+      -17412508655
+
   """
   @spec to_unix(DateTime.t, System.time_unit) :: non_neg_integer
+  def to_unix(datetime, unit \\ :seconds)
+  
   def to_unix(%DateTime{calendar: Calendar.ISO, std_offset: std_offset, utc_offset: utc_offset,
                         hour: hour, minute: minute, second: second, microsecond: {microsecond, _},
-                        year: year, month: month, day: day}, unit \\ :seconds) when year >= 1970 do
+                        year: year, month: month, day: day}, unit) when year >= 0 do
     seconds =
       :calendar.datetime_to_gregorian_seconds({{year, month, day}, {hour, minute, second}})
       |> Kernel.-(utc_offset)

lib/elixir/lib/enum.ex

@@ -861,13 +861,13 @@ defmodule Enum do
 
   def find_index(enumerable, fun) do
     res =
-      Enumerable.reduce(enumerable, {:cont, 0}, fn(entry, acc) ->
-        if fun.(entry), do: {:halt, acc}, else: {:cont, acc + 1}
+      Enumerable.reduce(enumerable, {:cont, {:not_found, 0}}, fn(entry, {_status, index}) ->
+        if fun.(entry), do: {:halt, {:found, index}}, else: {:cont, {:not_found, index + 1}}
       end)
 
     case res do
-      {:halted, entry} -> entry
-      {:done, _} -> nil
+      {_, {:found, index}} -> index
+      {_, {:not_found, _}} -> nil
     end
   end
 

lib/elixir/lib/exception.ex

@@ -15,7 +15,11 @@ defmodule Exception do
   """
 
   @typedoc "The exception type"
-  @type t :: %{__struct__: module, __exception__: true}
+  @type t :: %{
+    required(:__struct__) => module,
+    required(:__exception__) => true,
+    atom => any
+  }
 
   @typedoc "The kind handled by formatting functions"
   @type kind :: :error | :exit | :throw | {:EXIT, pid}

lib/elixir/lib/kernel.ex

@@ -1350,31 +1350,9 @@ defmodule Kernel do
           :erlang.error unquote(alias).exception([])
         end
       _ ->
-        generated = fn fun, var ->
-          {fun, [generated: true, line: -1], [{var, [], __MODULE__}]}
+        quote do
+          :erlang.error Kernel.Utils.raise(unquote(msg))
         end
-
-        {fun, meta, [arg, [do: clauses]]} =
-          quote do
-            case unquote(msg) do
-              msg when unquote(generated.(:is_binary, :msg)) ->
-                :erlang.error RuntimeError.exception(msg)
-              atom when unquote(generated.(:is_atom, :atom)) ->
-                :erlang.error atom.exception([])
-              %{__struct__: struct, __exception__: true} = other when is_atom(struct) ->
-                :erlang.error other
-              other ->
-                message = "raise/1 expects an alias, string or exception as the first argument, got: #{inspect other}"
-                :erlang.error ArgumentError.exception(message)
-            end
-          end
-
-        clauses =
-          :lists.map(fn {:->, meta, args} ->
-            {:->, [generated: true] ++ Keyword.put(meta, :line, -1), args}
-          end, clauses)
-
-        {fun, meta, [arg, [do: clauses]]}
     end
   end
 
@@ -1862,7 +1840,7 @@ defmodule Kernel do
   In case any entry returns `nil`, its key will be removed
   and the deletion will be considered a success.
   """
-  @spec pop_in(Access.t, nonempty_list(term)) :: Access.t
+  @spec pop_in(Access.t, nonempty_list(term)) :: {term, Access.t}
   def pop_in(data, keys)
   def pop_in(nil, [h | _]), do: Access.pop(nil, h)
   def pop_in(data, keys), do: do_pop_in(data, keys)
@@ -4314,7 +4292,7 @@ defmodule Kernel do
   end
 
   # TODO: Deprecate by v1.5
-  # @doc false
+  @doc false
   defmacro to_char_list(arg) do
     quote do: Kernel.to_charlist(unquote(arg))
   end

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -107,6 +107,7 @@ defmodule Kernel.ParallelCompiler do
       :erlang.spawn_monitor fn ->
         # Set the elixir_compiler_pid used by our custom Kernel.ErrorHandler.
         :erlang.put(:elixir_compiler_pid, parent)
+        :erlang.put(:elixir_compiler_file, file)
         :erlang.process_flag(:error_handler, Kernel.ErrorHandler)
 
         exit(try do
@@ -206,8 +207,10 @@ defmodule Kernel.ParallelCompiler do
 
       {:waiting, kind, child, ref, on, defining} ->
         # Oops, we already got it, do not put it on waiting.
+        # OR
+        # We're waiting on ourselves, send :found so that we can crash with a better error
         waiting =
-          if :lists.any(&match?({^kind, ^on}, &1), result) do
+          if :lists.any(&match?({^kind, ^on}, &1), result) or on in defining do
             send child, {ref, :found}
             waiting
           else

lib/elixir/lib/kernel/parallel_require.ex

@@ -47,21 +47,22 @@ defmodule Kernel.ParallelRequire do
     wait_for_messages(files, waiting, callbacks, schedulers, result)
   end
 
-  defp spawn_requires([h | t], waiting, callbacks, schedulers, result) do
+  defp spawn_requires([file | files], waiting, callbacks, schedulers, result) do
     parent = self()
     {pid, ref} = :erlang.spawn_monitor fn ->
       :erlang.put(:elixir_compiler_pid, parent)
+      :erlang.put(:elixir_compiler_file, file)
 
       exit(try do
-        new = Code.require_file(h) || []
-        {:required, Enum.map(new, &elem(&1, 0)), h}
+        new = Code.require_file(file) || []
+        {:required, Enum.map(new, &elem(&1, 0)), file}
       catch
         kind, reason ->
           {:failure, kind, reason, System.stacktrace}
       end)
     end
 
-    spawn_requires(t, [{pid, ref} | waiting], callbacks, schedulers, result)
+    spawn_requires(files, [{pid, ref} | waiting], callbacks, schedulers, result)
   end
 
   defp wait_for_messages(files, waiting, callbacks, schedulers, result) do

lib/elixir/lib/kernel/typespec.ex

@@ -448,7 +448,7 @@ defmodule Kernel.Typespec do
   defp translate_spec(kind, meta, name, args, return, guard, caller) when is_atom(args),
     do: translate_spec(kind, meta, name, [], return, guard, caller)
   defp translate_spec(:macrocallback, meta, name, args, return, guard, caller),
-    do: translate_spec(:callback, meta, :"MACRO-#{name}", [quote(do: env :: Macro.Env.t) | args], return, guard, caller)
+    do: translate_spec(:callback, meta, :"MACRO-#{name}", macro_args(args), return, guard, caller)
   defp translate_spec(kind, meta, name, args, return, guard, caller) do
     ensure_no_defaults!(args)
 
@@ -470,6 +470,10 @@ defmodule Kernel.Typespec do
     {{kind, {name, arity}, spec}, caller.line}
   end
 
+  defp macro_args(args) do
+    [quote(do: {line :: Macro.Env.line, env :: Macro.Env.t}) | args]
+  end
+
   defp ensure_no_defaults!(args) do
     :lists.foreach fn
       {:::, _, [left, right]} ->
@@ -773,6 +777,8 @@ defmodule Kernel.Typespec do
   end
 
   defp typespec({:%, _, [name, {:%{}, meta, fields}]}, vars, caller) do
+    # We cannot set a function name to avoid tracking
+    # as a compile time dependency, because for structs it actually is one.
     module = Macro.expand(name, caller)
 
     struct =
@@ -809,6 +815,8 @@ defmodule Kernel.Typespec do
   end
 
   defp typespec({:record, meta, [atom, fields]}, vars, caller) do
+    # We cannot set a function name to avoid tracking
+    # as a compile time dependency because for records it actually is one.
     case Macro.expand({atom, [], [{atom, [], []}]}, caller) do
       keyword when is_list(keyword) ->
         types =
@@ -879,7 +887,9 @@ defmodule Kernel.Typespec do
 
   # Handle remote calls
   defp typespec({{:., meta, [remote, name]}, _, args} = orig, vars, caller) do
-    remote = Macro.expand remote, caller
+    # We set a function name to avoid tracking
+    # aliases in typespecs as compile time dependencies.
+    remote = Macro.expand remote, %{caller | function: {:typespec, 0}}
     unless is_atom(remote) do
       compile_error(caller, "invalid remote in typespec: #{Macro.to_string(orig)}")
     end

lib/elixir/lib/kernel/utils.ex

@@ -3,6 +3,9 @@ import Kernel, except: [destructure: 2, defdelegate: 2, defstruct: 2]
 defmodule Kernel.Utils do
   @moduledoc false
 
+  @doc """
+  Callback for destructure.
+  """
   def destructure(list, count) when is_list(list), do: destructure_list(list, count)
   def destructure(nil, count), do: destructure_nil(count)
 
@@ -13,6 +16,9 @@ defmodule Kernel.Utils do
   defp destructure_nil(0), do: []
   defp destructure_nil(count), do: [nil | destructure_nil(count - 1)]
 
+  @doc """
+  Callback for defdelegate.
+  """
   def defdelegate(fun, opts) do
     append_first = Keyword.get(opts, :append_first, false)
 
@@ -56,6 +62,9 @@ defmodule Kernel.Utils do
       "defdelegate/2 only accepts function parameters, got: #{Macro.to_string(code)}"
   end
 
+  @doc """
+  Callback for defstruct.
+  """
   def defstruct(module, fields) do
     case fields do
       fs when is_list(fs) ->
@@ -85,10 +94,30 @@ defmodule Kernel.Utils do
      Module.get_attribute(module, :derive)}
   end
 
+  @doc """
+  Announcing callback for defstruct.
+  """
   def announce_struct(module) do
     case :erlang.get(:elixir_compiler_pid) do
       :undefined -> :ok
       pid -> send(pid, {:struct_available, module})
     end
   end
+
+  @doc """
+  Callback for raise.
+  """
+  def raise(msg) when is_binary(msg) do
+    RuntimeError.exception(msg)
+  end
+  def raise(atom) when is_atom(atom) do
+    atom.exception([])
+  end
+  def raise(%{__struct__: struct, __exception__: true} = exception) when is_atom(struct) do
+    exception
+  end
+  def raise(other) do
+    ArgumentError.exception("raise/1 expects an alias, string or exception as " <>
+                            "the first argument, got: #{inspect other}")
+  end
 end

lib/elixir/lib/option_parser.ex

@@ -15,13 +15,24 @@ defmodule OptionParser do
   @doc """
   Parses `argv` into a keywords list.
 
-  It returns a three-element tuple as follows:
+  It returns a three-element tuple with the form `{parsed, args, invalid}`, where:
 
-     1. parsed switches,
-     2. remaining arguments,
-     3. invalid options.
+    * `parsed` is a keyword list of parsed switches with `{switch_name, value}`
+      tuples in it; `switch_name` is the atom representing the switch name while
+      `value` is the value for that switch parsed according to `opts` (see the
+      "Examples" section for more information)
+    * `args` is a list of the remaining arguments in `argv` as strings
+    * `invalid` is a list of invalid options as `{option_name, value}` where
+      `option_name` is the raw option and `value` is `nil` if the option wasn't
+      expected or the string value if the value didn't have the expected type for
+      the corresponding option
 
-  ## Examples
+  Elixir converts switches to underscored atoms, so `--source-path` becomes
+  `:source_path`. This is done to better suit Elixir conventions. However, this
+  means that switches can't contain underscores and switches that do contain
+  underscores are always returned in the list of invalid options.
+
+  Without any options, this function will try to parse all switches in the `argv`.
 
       iex> OptionParser.parse(["--debug"])
       {[debug: true], [], []}
@@ -32,65 +43,85 @@ defmodule OptionParser do
       iex> OptionParser.parse(["--source-path", "lib", "test/enum_test.exs", "--verbose"])
       {[source_path: "lib", verbose: true], ["test/enum_test.exs"], []}
 
-  By default, Elixir will try to automatically parse all switches.
   Switches followed by a value will be assigned the value, as a string.
-  Switches without an argument, like `--debug` will automatically
-  be set to `true`.
+  Switches without an argument, like `--debug` in the examples above, will
+  automatically be set to `true`.
 
-  Note: Elixir also converts the switches to underscore atoms, so
-  `--source-path` becomes `:source_path`, to better suit Elixir
-  conventions. This means that option names on the command line cannot
-  contain underscores; such options will be put in the invalid options
-  list.
+  ## Options
 
-  ## Switch Definitions
+  The following options are supported:
+
+    * `:switches` or `:strict` - see the "Switch definitions" section below
+    * `:aliases` - see the "Aliases" section below
+
+  ## Switch definitions
 
   Often it is better to explicitly list the known
-  switches and their formats. The switches can be specified via two
-  alternative options:
+  switches and their formats. The switches can be specified via one of two
+  options:
 
-    * `:switches` - defines some switches. An attempt is still made to parse
-      switches that do not appear in the list.
+    * `:switches` - defines some switches and their types. This function
+      still attempts to parse switches that are not in this list.
+    * `:strict` - defines strict switches. Any switch in `argv` that is not
+      specified in the list is returned in the invalid options list.
 
-    * `:strict` - the switches are strict. Any switch that is not specified
-      in the list is returned in the invalid options list.
+  Both these options accept a keyword list of `{name, type}` tuples where `name`
+  is an atom defining the name of the switch and `type` is an atom that
+  specifies the type for the value of this switch (see the "Types" section below
+  for the possible types and more information about type casting).
 
   Note that you should only supply the `:switches` or `:strict` option.
-  If you supply both, an error will be raised.
+  If you supply both, an `ArgumentError` exception will be raised.
 
   ### Types
 
-  Option parser switches may take 0 or 1 argument.
+  Switches parsed by `OptionParser` may take zero or one arguments.
 
-  The following switches take no argument:
+  The following switches types take no arguments:
 
-    * `:boolean` - sets the value to true when given
-    * `:count`   - counts the number of times the switch is given
+    * `:boolean` - sets the value to `true` when given (see also the
+      "Negation switches" section below)
+    * `:count` - counts the number of times the switch is given
 
-  The following switches take 1 argument:
+  The following switches take one argument:
 
-    * `:integer` - parses the upcoming value as an integer.
-    * `:float`   - parses the upcoming value as a float.
-    * `:string`  - parses the upcoming value as a string.
+    * `:integer` - parses the value as an integer
+    * `:float` - parses the value as a float
+    * `:string` - parses the value as a string
 
-  If a switch can't be parsed, it is returned in the invalid
-  options list.
+  If a switch can't be parsed according to the given type, it is returned
+  in the invalid options list.
 
   ### Modifiers
 
   Switches can be specified with modifiers, which change how
   they behave. The following modifiers are supported:
 
-    * `:keep` - keeps duplicated items instead of overriding them.
-      Works with all types except `:count`.
+    * `:keep` - keeps duplicated items instead of overriding them; works with
+      all types except `:count`. Specifying `switch_name: :keep` assumes the
+      type of `:switch_name` will be `:string`.
 
-  Note: if you want to use `:keep` with a non-string type, use a list, e.g.
-  `[foo: [:integer, :keep]]`.
+  Note that if you want to use `:keep` with a type other than `:string`, use a list
+  as the type for the switch. For example: `[foo: [:integer, :keep]]`.
 
-  ### Examples
+  ### Negation switches
 
-  Here are some examples of option parser working with different types
-  and modifiers:
+  In case a switch `SWITCH` is specified to have type `:boolean`, it may be
+  passed as `--no-SWITCH` as well which will set the option to `false`:
+
+      iex> OptionParser.parse(["--no-op", "path/to/file"], switches: [op: :boolean])
+      {[op: false], ["path/to/file"], []}
+
+  ## Aliases
+
+  A set of aliases can be specified in the `:aliases` option:
+
+      iex> OptionParser.parse(["-d"], aliases: [d: :debug])
+      {[debug: true], [], []}
+
+  ## Examples
+
+  Here are some examples of working with different types and modifiers:
 
       iex> OptionParser.parse(["--unlock", "path/to/file"], strict: [unlock: :boolean])
       {[unlock: true], ["path/to/file"], []}
@@ -121,21 +152,6 @@ defmodule OptionParser do
       iex> OptionParser.parse(["--unlock", "path/to/file", "--unlock", "path/to/another/file"], strict: [unlock: :keep])
       {[unlock: "path/to/file", unlock: "path/to/another/file"], [], []}
 
-  ### Negation switches
-
-  In case a switch is declared as boolean, it may be passed as `--no-SWITCH`
-  which will set the option to `false`:
-
-      iex> OptionParser.parse(["--no-op", "path/to/file"], switches: [op: :boolean])
-      {[op: false], ["path/to/file"], []}
-
-  ## Aliases
-
-  A set of aliases can be given as options too:
-
-      iex> OptionParser.parse(["-d"], aliases: [d: :debug])
-      {[debug: true], [], []}
-
   """
   @spec parse(argv, options) :: {parsed, argv, errors}
   def parse(argv, opts \\ []) when is_list(argv) and is_list(opts) do
@@ -146,14 +162,16 @@ defmodule OptionParser do
   The same as `parse/2` but raises an `OptionParser.ParseError`
   exception if any invalid options are given.
 
-  If there weren't any errors, returns a three-element tuple as follows:
+  If there are no errors, returns a `{parsed, rest}` tuple where:
 
-      1. parsed options,
-      2. remaining arguments,
-      3. empty list.
+    * `parsed` is the list of parsed switches (same as in `parse/2`)
+    * `rest` is the list of arguments (same as in `parse/2`)
 
   ## Examples
 
+      iex> OptionParser.parse!(["--debug", "path/to/file"], strict: [debug: :boolean])
+      {[debug: true], ["path/to/file"]}
+
       iex> OptionParser.parse!(["--limit", "xyz"], strict: [limit: :integer])
       ** (OptionParser.ParseError) 1 error found!
       --limit : Expected type integer, got "xyz"
@@ -172,7 +190,7 @@ defmodule OptionParser do
   @spec parse!(argv, options) :: {parsed, argv} | no_return
   def parse!(argv, opts \\ []) when is_list(argv) and is_list(opts) do
     case parse(argv, opts) do
-      {parsed, argv, []} -> {parsed, argv}
+      {parsed, args, []} -> {parsed, args}
       {_, _, errors} -> raise ParseError, format_errors(errors, opts)
     end
   end
@@ -201,14 +219,16 @@ defmodule OptionParser do
   The same as `parse_head/2` but raises an `OptionParser.ParseError`
   exception if any invalid options are given.
 
-  If there weren't any errors, returns a three-element tuple as follows:
+  If there are no errors, returns a `{parsed, rest}` tuple where:
 
-      1. parsed options,
-      2. remaining arguments,
-      3. empty list.
+    * `parsed` is the list of parsed switches (same as in `parse_head/2`)
+    * `rest` is the list of arguments (same as in `parse_head/2`)
 
   ## Examples
 
+      iex> OptionParser.parse_head!(["--source", "lib", "path/to/file", "--verbose"])
+      {[source: "lib"], ["path/to/file", "--verbose"]}
+
       iex> OptionParser.parse_head!(["--number", "lib", "test/enum_test.exs", "--verbose"], strict: [number: :integer])
       ** (OptionParser.ParseError) 1 error found!
       --number : Expected type integer, got "lib"
@@ -219,10 +239,10 @@ defmodule OptionParser do
       --verbose : Missing argument of type integer
       --source : Expected type integer, got "lib"
   """
-  @spec parse_head!(argv, options) :: {parsed, argv, errors} | no_return
+  @spec parse_head!(argv, options) :: {parsed, argv} | no_return
   def parse_head!(argv, opts \\ []) when is_list(argv) and is_list(opts) do
     case parse_head(argv, opts) do
-      {parsed, argv, []} -> {parsed, argv}
+      {parsed, args, []} -> {parsed, args}
       {_, _, errors} -> raise ParseError, format_errors(errors, opts)
     end
   end
@@ -250,7 +270,7 @@ defmodule OptionParser do
       {:error, ["--" | rest]} ->
         {Enum.reverse(opts), Enum.reverse(args, rest), Enum.reverse(invalid)}
 
-      {:error, [arg | rest]=remaining_args} ->
+      {:error, [arg | rest] = remaining_args} ->
         # there is no option
         if all? do
           do_parse(rest, config, opts, [arg | args], invalid, all?)
@@ -264,20 +284,20 @@ defmodule OptionParser do
   Low-level function that parses one option.
 
   It accepts the same options as `parse/2` and `parse_head/2`
-  as both functions are built on top of next. This function
+  as both functions are built on top of this function. This function
   may return:
 
     * `{:ok, key, value, rest}` - the option `key` with `value` was
       successfully parsed
 
     * `{:invalid, key, value, rest}` - the option `key` is invalid with `value`
-      (returned when the switch type does not match the one given via the
-      command line)
+      (returned when the value cannot be parsed according to the switch type)
 
     * `{:undefined, key, value, rest}` - the option `key` is undefined
       (returned in strict mode when the switch is unknown)
 
-    * `{:error, rest}` - there are no switches at the top of the given argv
+    * `{:error, rest}` - there are no switches at the head of the given `argv`
+
   """
 
   @spec next(argv, options) ::
@@ -295,15 +315,15 @@ defmodule OptionParser do
     {:error, []}
   end
 
-  defp next(["--" | _]=argv, _aliases, _switches, _strict) do
+  defp next(["--" | _] = argv, _aliases, _switches, _strict) do
     {:error, argv}
   end
 
-  defp next(["-" | _]=argv, _aliases, _switches, _strict) do
+  defp next(["-" | _] = argv, _aliases, _switches, _strict) do
     {:error, argv}
   end
 
-  defp next(["- " <> _ | _]=argv, _aliases, _switches, _strict) do
+  defp next(["- " <> _ | _] = argv, _aliases, _switches, _strict) do
     {:error, argv}
   end
 
@@ -334,8 +354,9 @@ defmodule OptionParser do
   @doc """
   Receives a key-value enumerable and converts it to argv.
 
-  Keys must be atoms. Keys with nil value are discarded,
+  Keys must be atoms. Keys with `nil` value are discarded,
   boolean values are converted to `--key` or `--no-key`
+  (if the value is `true` or `false`, respectively),
   and all other values are converted using `to_string/1`.
 
   ## Examples
@@ -364,6 +385,9 @@ defmodule OptionParser do
   @doc ~S"""
   Splits a string into argv chunks.
 
+  This function splits the given `string` into a list of strings in a similar
+  way to many shells.
+
   ## Examples
 
       iex> OptionParser.split("foo bar")
@@ -371,6 +395,7 @@ defmodule OptionParser do
 
       iex> OptionParser.split("foo \"bar baz\"")
       ["foo", "bar baz"]
+
   """
   @spec split(String.t) :: argv
   def split(string) do

lib/elixir/lib/protocol.ex

@@ -451,6 +451,16 @@ defmodule Protocol do
         end
       end, builtin)
 
+      # Define a catch-all impl_for/1 clause to pacify Dialyzer (since
+      # destructuring opaque types is illegal, Dialyzer will think none of the
+      # previous clauses matches opaque types, and without this clause, will
+      # conclude that impl_for can't handle an opaque argument). This is a hack
+      # since it relies on Dialyzer not being smart enough to conclude that all
+      # opaque types will get the any_impl_for/0 implementation.
+      Kernel.def impl_for(_) do
+        any_impl_for()
+      end
+
       @doc false
       @spec impl_for!(term) :: atom | no_return
       Kernel.def impl_for!(data) do

lib/elixir/lib/string.ex

@@ -350,6 +350,7 @@ defmodule String do
   defp parts_to_index(:infinity),                      do: 0
   defp parts_to_index(n) when is_integer(n) and n > 0, do: n
 
+  defp split_each("", _pattern, true, 1), do: []
   defp split_each(string, _pattern, _trim, 1) when is_binary(string), do: [string]
   defp split_each(string, pattern, trim, count) do
     case do_splitter(string, pattern, trim) do
@@ -402,6 +403,7 @@ defmodule String do
   end
 
   defp maybe_compile_pattern(""), do: ""
+  defp maybe_compile_pattern(pattern) when is_tuple(pattern), do: pattern
   defp maybe_compile_pattern(pattern), do: :binary.compile_pattern(pattern)
 
   @doc """

lib/elixir/lib/system.ex

@@ -112,7 +112,15 @@ defmodule System do
   # Tries to run "git rev-parse --short HEAD". In the case of success returns
   # the short revision hash. If that fails, returns an empty string.
   defmacrop get_revision do
-    :os.cmd('git rev-parse --short HEAD 2> /dev/null')
+    null =
+      case :os.type do
+        {:win32, _} -> 'NUL'
+        _ -> '/dev/null'
+      end
+
+    'git rev-parse --short HEAD 2> '
+    |> Kernel.++(null)
+    |> :os.cmd()
     |> strip
   end
 

lib/elixir/rebar.config

@@ -1,23 +1,24 @@
 {erl_opts, [
-            warn_unused_vars,
-            warn_export_all,
-            warn_shadow_vars,
-            warn_unused_import,
-            warn_unused_function,
-            warn_bif_clash,
-            warn_unused_record,
-            warn_deprecated_function,
-            warn_obsolete_guard,
-            strict_validation,
-            warn_exported_vars,
-            %% warn_export_vars,
-            %% warn_missing_spec,
-            %% warn_untyped_record,
-            %% warnings_as_errors,
-            debug_info
-           ]}.
+  warn_unused_vars,
+  warn_export_all,
+  warn_shadow_vars,
+  warn_unused_import,
+  warn_unused_function,
+  warn_bif_clash,
+  warn_unused_record,
+  warn_deprecated_function,
+  warn_obsolete_guard,
+  strict_validation,
+  warn_exported_vars,
+  %% warn_export_vars,
+  %% warn_missing_spec,
+  %% warn_untyped_record,
+  %% warnings_as_errors,
+  debug_info,
+  {platform_define, "^18.*", old_map_specs}
+]}.
 
 {yrl_opts, [
-            {report, true},
-            {verbose, false}
-           ]}.
+  {report, true},
+  {verbose, false}
+]}.

lib/elixir/src/elixir.erl

@@ -14,11 +14,16 @@
 -export_type([charlist/0, char_list/0, struct/0, as_boolean/1, keyword/0, keyword/1]).
 -type charlist() :: string().
 -type char_list() :: string().
--type struct() :: #{'__struct__' => atom()}.
 -type as_boolean(T) :: T.
 -type keyword() :: [{atom(), any()}].
 -type keyword(T) :: [{atom(), T}].
 
+-ifdef(old_map_specs).
+-type struct() :: #{'__struct__' => atom(), atom() => any()}.
+-else.
+-type struct() :: #{'__struct__' := atom(), atom() => any()}.
+-endif.
+
 %% OTP Application API
 
 -export([start/2, stop/1, config_change/3]).

lib/elixir/src/elixir_clauses.erl

@@ -198,17 +198,17 @@ normalize_vars(Key, {Ref, Counter, _Safe},
 % Generate match vars by checking if they were updated
 % or not and assigning the previous value.
 
-generate_match_vars([{Key, Value, Expr} | T], ClauseVars, Left, Right) ->
+generate_match_vars([{Key, {Value, _, _}, Expr} | T], ClauseVars, Left, Right) ->
   case maps:find(Key, ClauseVars) of
-    {ok, Value} ->
+    {ok, {Value, _, _}} ->
       generate_match_vars(T, ClauseVars, Left, Right);
-    {ok, Clause} ->
+    {ok, {Clause, _, _}} ->
       generate_match_vars(T, ClauseVars,
-        [{var, 0, element(1, Value)} | Left],
-        [{var, 0, element(1, Clause)} | Right]);
+        [{var, 0, Value} | Left],
+        [{var, 0, Clause} | Right]);
     error ->
       generate_match_vars(T, ClauseVars,
-        [{var, 0, element(1, Value)} | Left], [Expr | Right])
+        [{var, 0, Value} | Left], [Expr | Right])
   end;
 
 generate_match_vars([], _ClauseVars, Left, Right) ->

lib/elixir/src/elixir_dispatch.erl

@@ -60,7 +60,8 @@ import_function(Meta, Name, Arity, E) ->
   end.
 
 require_function(Meta, Receiver, Name, Arity, E) ->
-  case is_element({Name, Arity}, get_optional_macros(Receiver)) of
+  Required = is_element(Receiver, ?m(E, requires)),
+  case is_element({Name, Arity}, get_macros(Receiver, Required)) of
     true  -> false;
     false ->
       elixir_lexical:record_remote(Receiver, ?m(E, function), ?m(E, lexical_tracker)),
@@ -146,7 +147,7 @@ do_expand_import(Meta, {Name, Arity} = Tuple, Args, Module, E, Result) ->
       elixir_locals:record_import(Tuple, Receiver, Module, ?m(E, function)),
       {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)};
     {import, Receiver} ->
-      case expand_require([{require, false} | Meta], Receiver, Tuple, Args, E) of
+      case expand_require([{required, true} | Meta], Receiver, Tuple, Args, E) of
         {ok, _, _} = Response -> Response;
         error -> {ok, Receiver, Name, Args}
       end;
@@ -161,19 +162,15 @@ do_expand_import(Meta, {Name, Arity} = Tuple, Args, Module, E, Result) ->
 
 expand_require(Meta, Receiver, {Name, Arity} = Tuple, Args, E) ->
   check_deprecation(Meta, Receiver, Name, Arity, E),
-  Module = ?m(E, module),
+  Required = (Receiver == ?m(E, module)) orelse is_element(Receiver, ?m(E, requires)) orelse required(Meta),
 
-  case is_element(Tuple, get_optional_macros(Receiver)) of
+  case is_element(Tuple, get_macros(Receiver, Required)) of
+    true when Required ->
+      elixir_lexical:record_remote(Receiver, Name, Arity, nil, ?line(Meta), ?m(E, lexical_tracker)),
+      {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)};
     true ->
-      Requires = ?m(E, requires),
-      case (Receiver == Module) orelse is_element(Receiver, Requires) orelse skip_require(Meta) of
-        true  ->
-          elixir_lexical:record_remote(Receiver, Name, Arity, nil, ?line(Meta), ?m(E, lexical_tracker)),
-          {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)};
-        false ->
-          Info = {unrequired_module, {Receiver, Name, length(Args), Requires}},
-          elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Info)
-      end;
+      Info = {unrequired_module, {Receiver, Name, length(Args), ?m(E, requires)}},
+      elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Info);
     false ->
       error
   end.
@@ -220,8 +217,8 @@ caller(Line, E) ->
 
 %% Helpers
 
-skip_require(Meta) ->
-  lists:keyfind(require, 1, Meta) == {require, false}.
+required(Meta) ->
+  lists:keyfind(required, 1, Meta) == {required, true}.
 
 find_dispatch(Meta, Tuple, Extra, E) ->
   case is_import(Meta) of
@@ -289,9 +286,20 @@ format_error({ambiguous_call, {Mod1, Mod2, Name, Arity}}) ->
 %% INTROSPECTION
 
 %% Do not try to get macros from Erlang. Speeds up compilation a bit.
-get_optional_macros(erlang) -> [];
+get_macros(erlang, _) -> [];
 
-get_optional_macros(Receiver) ->
+get_macros(Receiver, false) ->
+  case code:is_loaded(Receiver) of
+    {file, _} ->
+      try
+        Receiver:'__info__'(macros)
+      catch
+        error:undef -> []
+      end;
+    false -> []
+  end;
+
+get_macros(Receiver, true) ->
   case code:ensure_loaded(Receiver) of
     {module, Receiver} ->
       try

lib/elixir/src/elixir_map.erl

@@ -92,9 +92,9 @@ translate_struct(Meta, Name, {'%{}', MapMeta, Args}, S) ->
 
       {TMap, TS} = translate_map(MapMeta, Assocs, Var, VS),
 
-      {{'case', Ann, TUpdate, [
+      {{'case', ?generated, TUpdate, [
         {clause, Ann, [Match], [], [TMap]},
-        {clause, Ann, [Var], [], [elixir_utils:erl_call(Ann, erlang, error, [Error])]}
+        {clause, ?generated, [Var], [], [elixir_utils:erl_call(Ann, erlang, error, [Error])]}
       ]}, TS};
     match ->
       translate_map(MapMeta, Assocs ++ [{'__struct__', Name}], nil, US);

lib/elixir/src/elixir_module.erl

@@ -378,7 +378,7 @@ load_form(Line, Data, Forms, Opts, E) ->
             ok;
           PID ->
             Ref = make_ref(),
-            PID ! {module_available, self(), Ref, ?m(E, file), Module, Binary},
+            PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary},
             receive {Ref, ack} -> ok end
         end;
       _ ->
@@ -525,7 +525,7 @@ add_beam_chunk(Bin, Id, ChunkData)
 %% the callback can't be expanded, invokes the given
 %% fun passing a possibly expanded AM:AF(Args).
 expand_callback(Line, M, F, Args, E, Fun) ->
-  Meta = [{line, Line}, {require, false}],
+  Meta = [{line, Line}, {required, true}],
 
   {EE, ET} = elixir_dispatch:dispatch_require(Meta, M, F, Args, E, fun(AM, AF, AA) ->
     Fun(AM, AF, AA),

lib/elixir/src/elixir_tokenizer.erl

@@ -135,15 +135,15 @@ tokenize([], EndLine, _Column, #elixir_tokenizer{terminators=[{Start, {StartLine
 % Base integers
 
 tokenize([$0, $x, H | T], Line, Column, Scope, Tokens) when ?is_hex(H) ->
-  {Rest, Number, Length} = tokenize_hex([H | T], []),
+  {Rest, Number, Length} = tokenize_hex(T, [H], 1),
   tokenize(Rest, Line, Column + 2 + Length, Scope, [{number, {Line, Column, Column + 2 + Length}, Number} | Tokens]);
 
 tokenize([$0, $b, H | T], Line, Column, Scope, Tokens) when ?is_bin(H) ->
-  {Rest, Number, Length} = tokenize_bin([H | T], []),
+  {Rest, Number, Length} = tokenize_bin(T, [H], 1),
   tokenize(Rest, Line, Column + 2 + Length, Scope, [{number, {Line, Column, Column + 2 + Length}, Number} | Tokens]);
 
 tokenize([$0, $o, H | T], Line, Column, Scope, Tokens) when ?is_octal(H) ->
-  {Rest, Number, Length} = tokenize_octal([H | T], []),
+  {Rest, Number, Length} = tokenize_octal(T, [H], 1),
   tokenize(Rest, Line, Column + 2 + Length, Scope, [{number, {Line, Column, Column + 2 + Length}, Number} | Tokens]);
 
 % Comments
@@ -413,8 +413,8 @@ tokenize([$. | T], Line, Column, Scope, Tokens) ->
 
 % Integers and floats
 
-tokenize([H | _] = String, Line, Column, Scope, Tokens) when ?is_digit(H) ->
-  {Rest, Number, Length} = tokenize_number(String, [], false),
+tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) ->
+  {Rest, Number, Length} = tokenize_number(T, [H], 1, false),
   tokenize(Rest, Line, Column + Length, Scope, [{number, {Line, Column, Column + Length}, Number} | Tokens]);
 
 % Identifiers (including aliases)
@@ -733,46 +733,55 @@ extract_heredoc_line(Marker, Rest, Buffer, _Counter) ->
 %% At this point, we are at least sure the first digit is a number.
 
 %% Check if we have a point followed by a number;
-tokenize_number([$., H | T], Acc, false) when ?is_digit(H) ->
-  tokenize_number(T, [H, $. | Acc], true);
+tokenize_number([$., H | T], Acc, Length, false) when ?is_digit(H) ->
+  tokenize_number(T, [H, $. | Acc], Length + 2, true);
 
 %% Check if we have an underscore followed by a number;
-tokenize_number([$_, H | T], Acc, Bool) when ?is_digit(H) ->
-  tokenize_number(T, [H | Acc], Bool);
+tokenize_number([$_, H | T], Acc, Length, Bool) when ?is_digit(H) ->
+  tokenize_number(T, [H | Acc], Length + 2, Bool);
 
 %% Check if we have e- followed by numbers (valid only for floats);
-tokenize_number([E, S, H | T], Acc, true)
+tokenize_number([E, S, H | T], Acc, Length, true)
     when (E == $E) or (E == $e), ?is_digit(H), S == $+ orelse S == $- ->
-  tokenize_number(T, [H, S, $e | Acc], true);
+  tokenize_number(T, [H, S, $e | Acc], Length + 3, true);
 
 %% Check if we have e followed by numbers (valid only for floats);
-tokenize_number([E, H | T], Acc, true)
+tokenize_number([E, H | T], Acc, Length, true)
     when (E == $E) or (E == $e), ?is_digit(H) ->
-  tokenize_number(T, [H, $e | Acc], true);
+  tokenize_number(T, [H, $e | Acc], Length + 2, true);
 
 %% Finally just numbers.
-tokenize_number([H | T], Acc, Bool) when ?is_digit(H) ->
-  tokenize_number(T, [H | Acc], Bool);
+tokenize_number([H | T], Acc, Length, Bool) when ?is_digit(H) ->
+  tokenize_number(T, [H | Acc], Length + 1, Bool);
 
 %% Cast to float...
-tokenize_number(Rest, Acc, true) ->
-  {Rest, list_to_float(lists:reverse(Acc)), length(Acc)};
+tokenize_number(Rest, Acc, Length, true) ->
+  {Rest, list_to_float(lists:reverse(Acc)), Length};
 
 %% Or integer.
-tokenize_number(Rest, Acc, false) ->
-  {Rest, list_to_integer(lists:reverse(Acc)), length(Acc)}.
+tokenize_number(Rest, Acc, Length, false) ->
+  {Rest, list_to_integer(lists:reverse(Acc)), Length}.
 
-tokenize_hex([H | T], Acc) when ?is_hex(H) -> tokenize_hex(T, [H | Acc]);
-tokenize_hex([$_, H | T], Acc) when ?is_hex(H) -> tokenize_hex(T, [H | Acc]);
-tokenize_hex(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 16), length(Acc)}.
+tokenize_hex([H | T], Acc, Length) when ?is_hex(H) ->
+  tokenize_hex(T, [H | Acc], Length + 1);
+tokenize_hex([$_, H | T], Acc, Length) when ?is_hex(H) ->
+  tokenize_hex(T, [H | Acc], Length + 2);
+tokenize_hex(Rest, Acc, Length) ->
+  {Rest, list_to_integer(lists:reverse(Acc), 16), Length}.
 
-tokenize_octal([H | T], Acc) when ?is_octal(H) -> tokenize_octal(T, [H | Acc]);
-tokenize_octal([$_, H | T], Acc) when ?is_octal(H) -> tokenize_octal(T, [H | Acc]);
-tokenize_octal(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 8), length(Acc)}.
+tokenize_octal([H | T], Acc, Length) when ?is_octal(H) ->
+  tokenize_octal(T, [H | Acc], Length + 1);
+tokenize_octal([$_, H | T], Acc, Length) when ?is_octal(H) ->
+  tokenize_octal(T, [H | Acc], Length + 2);
+tokenize_octal(Rest, Acc, Length) ->
+  {Rest, list_to_integer(lists:reverse(Acc), 8), Length}.
 
-tokenize_bin([H | T], Acc) when ?is_bin(H) -> tokenize_bin(T, [H | Acc]);
-tokenize_bin([$_, H | T], Acc) when ?is_bin(H) -> tokenize_bin(T, [H | Acc]);
-tokenize_bin(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 2), length(Acc)}.
+tokenize_bin([H | T], Acc, Length) when ?is_bin(H) ->
+  tokenize_bin(T, [H | Acc], Length + 1);
+tokenize_bin([$_, H | T], Acc, Length) when ?is_bin(H) ->
+  tokenize_bin(T, [H | Acc], Length + 2);
+tokenize_bin(Rest, Acc, Length) ->
+  {Rest, list_to_integer(lists:reverse(Acc), 2), Length}.
 
 %% Comments
 

lib/elixir/src/elixir_with.erl

@@ -107,9 +107,14 @@ build_else(Meta, ElseClauses) ->
     each_clause_to_error_match(ElseClauses)] ++ [build_raise(Meta)].
 
 each_clause_to_error_match(Clauses) ->
-  [{match, Meta, [{error, Match}], Expr} ||
+  [{match, Meta, [error_match_for_match(Match)], Expr} ||
     {'->', Meta, [[Match], Expr]} <- Clauses].
 
+error_match_for_match({'when', Meta, [Left, Right]}) ->
+  {'when', Meta, [{error, Left}, Right]};
+error_match_for_match(Match) ->
+  {error, Match}.
+
 build_raise(Meta) ->
   Other = {other, Meta, ?MODULE},
   {match, ?generated, [{error, Other}], {{'.', Meta, [erlang, error]}, Meta, [{with_clause, Other}]}}.

lib/elixir/test/elixir/calendar_test.exs

@@ -66,4 +66,25 @@ defmodule DateTimeTest do
                    utc_offset: -12600, std_offset: 3600, time_zone: "Brazil/Manaus"}
     assert to_string(dt) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus"
   end
+
+  test "from_unix/2 works with Unix times back to 0 Gregorian Seconds" do
+    assert DateTime.from_unix(-62167219200) == {:ok,
+                    %DateTime{calendar: Calendar.ISO, day: 1, hour: 0, microsecond: {0, 0},
+                              minute: 0, month: 1, second: 0, std_offset: 0, time_zone: "Etc/UTC",
+                              utc_offset: 0, year: 0, zone_abbr: "UTC"}}
+
+    assert DateTime.from_unix(-62167219201) == :error
+  end
+
+  test "to_unix/2 works with Unix times back to 0 Gregorian Seconds" do
+    gregorian_0 = %DateTime{calendar: Calendar.ISO, day: 1, hour: 0, microsecond: {0, 0},
+                            minute: 0, month: 1, second: 0, std_offset: 0, time_zone: "Etc/UTC",
+                            utc_offset: 0, year: 0, zone_abbr: "UTC"}
+    before_gregorian_0 = %DateTime{gregorian_0 | year: -1}
+
+    assert DateTime.to_unix(gregorian_0) == -62167219200
+    assert_raise FunctionClauseError, fn ->
+      DateTime.to_unix(before_gregorian_0)
+    end
+  end
 end

lib/elixir/test/elixir/enum_test.exs

@@ -195,6 +195,8 @@ defmodule EnumTest do
   test "find_index/2" do
     assert Enum.find_index([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil
     assert Enum.find_index([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) == 1
+    assert Stream.take(1..3, 3) |> Enum.find_index(fn _ -> false end) == nil
+    assert Stream.take(1..6, 6) |> Enum.find_index(fn x -> x == 5 end) == 4
   end
 
   test "find_value/2" do

lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex

@@ -0,0 +1,11 @@
+defmodule Dialyzer.Macrocallback do
+  @macrocallback required(atom) :: Macro.t
+  @macrocallback optional(atom) :: Macro.t
+  @optional_callbacks [optional: 1]
+end
+
+defmodule Dialyzer.Macrocallback.Impl do
+  @behaviour Dialyzer.Macrocallback
+  defmacro required(var), do: Macro.expand(var, __CALLER__)
+  defmacro optional(var), do: Macro.expand(var, __CALLER__)
+end

lib/elixir/test/elixir/fixtures/dialyzer/protocol_opaque.ex

@@ -0,0 +1,21 @@
+defmodule Dialyzer.ProtocolOpaque do
+  def circus() do
+    Dialyzer.ProtocolOpaque.Entity.speak(Dialyzer.ProtocolOpaque.Duck.new)
+  end
+end
+
+defprotocol Dialyzer.ProtocolOpaque.Entity do
+  def speak(entity)
+end
+
+defmodule Dialyzer.ProtocolOpaque.Duck do
+  @opaque t :: %__MODULE__{}
+  defstruct feathers: :white_and_grey
+
+  @spec new :: t
+  def new(), do: %__MODULE__{}
+
+  defimpl Dialyzer.ProtocolOpaque.Entity do
+    def speak(%Dialyzer.ProtocolOpaque.Duck{}), do: "Quack!"
+  end
+end

lib/elixir/test/elixir/fixtures/dialyzer/struct_update.ex

@@ -0,0 +1,7 @@
+defmodule Dialyzer.StructUpdate do
+  defstruct [:foo]
+
+  def update(%__MODULE__{} = struct) do
+    %__MODULE__{struct | foo: :bar}
+  end
+end

lib/elixir/test/elixir/fixtures/parallel_struct/undef.ex

@@ -0,0 +1,5 @@
+defmodule Undef do
+  def undef do
+    %__MODULE__{}
+  end
+end

lib/elixir/test/elixir/kernel/dialyzer_test.exs

@@ -16,8 +16,14 @@ defmodule Kernel.DialyzerTest do
       |> Path.join("base_plt")
       |> String.to_charlist()
 
-    # Add a few key elixir modules for types
-    files = Enum.map([Kernel, String, Keyword, Exception], &:code.which/1)
+    # Some OSs (like Windows) do not provide the HOME environment variable.
+    unless System.get_env("HOME") do
+      System.put_env("HOME", System.user_home())
+    end
+
+    # Add a few key elixir modules for types and macro functions
+    mods = [Kernel, String, Keyword, Exception, Macro, Macro.Env, :elixir_env]
+    files = Enum.map(mods, &:code.which/1)
     :dialyzer.run([analysis_type: :plt_build, output_plt: plt,
                    apps: [:erts], files: files])
 
@@ -59,12 +65,29 @@ defmodule Kernel.DialyzerTest do
     assert_dialyze_no_warnings! context
   end
 
-  @tag otp19: false
   test "no warnings on raise", context do
     copy_beam! context, Dialyzer.Raise
     assert_dialyze_no_warnings! context
   end
 
+  test "no warnings on macrocallback", context do
+    copy_beam! context, Dialyzer.Macrocallback
+    copy_beam! context, Dialyzer.Macrocallback.Impl
+    assert_dialyze_no_warnings! context
+  end
+
+  test "no warnings on struct update", context do
+    copy_beam! context, Dialyzer.StructUpdate
+    assert_dialyze_no_warnings! context
+  end
+
+  test "no warnings on protocol calls with opaque types", context do
+    copy_beam! context, Dialyzer.ProtocolOpaque
+    copy_beam! context, Dialyzer.ProtocolOpaque.Entity
+    copy_beam! context, Dialyzer.ProtocolOpaque.Duck
+    assert_dialyze_no_warnings! context
+  end
+
   defp copy_beam!(context, module) do
     name = "#{module}.beam"
     File.cp! Path.join(context[:base_dir], name),

lib/elixir/test/elixir/kernel/parallel_compiler_test.exs

@@ -28,6 +28,13 @@ defmodule Kernel.ParallelCompilerTest do
     end
   end
 
+  test "emits struct undefined error when local struct is undefined" do
+    fixtures = [fixture_path("parallel_struct/undef.ex")]
+    assert capture_io(fn ->
+      assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == {:shutdown, 1}
+    end) =~ "Undef.__struct__/1 is undefined, cannot expand struct Undef"
+  end
+
   test "does not hang on missing dependencies" do
     fixtures = [fixture_path("parallel_compiler/bat.ex")]
     assert capture_io(fn ->

lib/elixir/test/elixir/kernel/with_test.exs

@@ -66,6 +66,7 @@ defmodule Kernel.WithTest do
 
   test "else conditions" do
     assert with({:ok, res} <- 41, do: res, else: ({:error, error} -> error; res -> res + 1)) == 42
+    assert with({:ok, res} <- 41, do: res, else: (res when res == 41 -> res + 1; res -> res)) == 42
     assert with({:ok, res} <- 41, do: res, else: (_ -> :error)) == :error
   end
 

lib/elixir/test/elixir/string_test.exs

@@ -66,6 +66,9 @@ defmodule StringTest do
     assert String.split("abé", "", trim: true, parts: 2) == ["a", "bé"]
 
     assert String.split("noël", "") == ["n", "o", "ë", "l", ""]
+
+    assert String.split("x-", "-", parts: 2, trim: true) == ["x"]
+    assert String.split("x-x-", "-", parts: 3, trim: true) == ["x", "x"]
   end
 
   test "split with regex" do
@@ -79,6 +82,14 @@ defmodule StringTest do
     assert String.split("a,b", ~r{\.}) == ["a,b"]
   end
 
+  test "split with compiled pattern" do
+    pattern = :binary.compile_pattern("-")
+
+    assert String.split("x-", pattern) == ["x", ""]
+    assert String.split("x-", pattern, parts: 2, trim: true) == ["x"]
+    assert String.split("x-x-", pattern, parts: 3, trim: true) == ["x", "x"]
+  end
+
   test "splitter" do
     assert String.splitter("a,b,c", ",") |> Enum.to_list == ["a", "b", "c"]
     assert String.splitter("a,b", ".") |> Enum.to_list == ["a,b"]

lib/elixir/test/erlang/tokenizer_test.erl

@@ -25,15 +25,16 @@ op_kw_test() ->
   [{atom, {1, 1, 5}, foo}, {dual_op, {1, 5, 6}, '+'}, {atom, {1, 6, 10}, bar}] = tokenize(":foo+:bar").
 
 scientific_test() ->
-  [{number, {1, 1, 7}, 0.1}] = tokenize("1.0e-1").
+  [{number, {1, 1, 7}, 0.1}] = tokenize("1.0e-1"),
+  [{number, {1, 1, 16}, 1.2345678e-7}] = tokenize("1_234.567_8e-10").
 
 hex_bin_octal_test() ->
   [{number, {1, 1, 5}, 255}] = tokenize("0xFF"),
-  [{number, {1, 1, 5}, 255}] = tokenize("0xF_F"),
+  [{number, {1, 1, 6}, 255}] = tokenize("0xF_F"),
   [{number, {1, 1, 5}, 63}] = tokenize("0o77"),
-  [{number, {1, 1, 5}, 63}] = tokenize("0o7_7"),
+  [{number, {1, 1, 6}, 63}] = tokenize("0o7_7"),
   [{number, {1, 1, 5}, 3}] = tokenize("0b11"),
-  [{number, {1, 1, 5}, 3}] = tokenize("0b1_1").
+  [{number, {1, 1, 6}, 3}] = tokenize("0b1_1").
 
 unquoted_atom_test() ->
   [{atom, {1, 1, 3}, '+'}] = tokenize(":+"),

lib/ex_unit/lib/ex_unit.ex

@@ -181,6 +181,9 @@ defmodule ExUnit do
       and print them on test failure. Can be overridden for individual tests via
       `@tag capture_log: false`. Defaults to `false`.
 
+    * `:case_load_timeout` - the timeout to be used when loading a test case.
+      Defaults to `60_000` milliseconds.
+
     * `:colors` - a keyword list of colors to be used by some formatters.
       The only option so far is `[enabled: boolean]` which defaults to `IO.ANSI.enabled?/0`
 

lib/ex_unit/lib/ex_unit/capture_log.ex

@@ -37,7 +37,15 @@ defmodule ExUnit.CaptureLog do
   Returns the binary which is the captured output.
 
   This function mutes the `:console` backend
-  and captures any log messages sent to Logger.
+  and captures any log messages sent to Logger from the calling
+  processes. It is possible to ensure explicit log messages from other
+  processes are captured by waiting for their exit or monitor
+  signal. This does not guarantee to capture VM generated log messages
+  when a process that is spawned using a low level `Kernel` spawn function
+  (e.g. `Kernel.spawn/1`) raises an exception that isn't rescued or a throw
+  that isn't caught. A `Task`, or other OTP process, will send explicit logs
+  that are sent before its exit or monitor signal and will not cause VM
+  generated log messages.
 
   Note that when the `async` is set to `true`,
   the messages from another test might be captured.
@@ -59,6 +67,7 @@ defmodule ExUnit.CaptureLog do
     {:ok, string_io} = StringIO.open("")
 
     try do
+      _ = :gen_event.which_handlers(:error_logger)
       :ok = add_capture(string_io, opts)
       ref = ExUnit.CaptureServer.log_capture_on(self())
 

lib/ex_unit/lib/ex_unit/diff.ex

@@ -367,18 +367,29 @@ defmodule ExUnit.Diff do
       map_pair = format_key_value(key, val, keyword?)
       [[ins: ", ", ins: map_pair] | acc]
     end)
+    result =
+      if same == [] and altered == [] and missing != [] and surplus != [] do
+        [[_ | elem_diff] | rest] = result
+        [elem_diff | rest]
+      else
+        result
+      end
+
     result = Enum.reduce(surplus, result, fn({key, val}, acc) ->
       map_pair = format_key_value(key, val, keyword?)
       [[del: ", ", del: map_pair] | acc]
     end)
+
     result = Enum.reduce(altered, result, fn({key, {val1, val2}}, acc) ->
       value_diff = script_inner(val1, val2)
       [[{:eq, ", "}, {:eq, format_key(key, keyword?)}, value_diff] | acc]
     end)
+
     result = Enum.reduce(same, result, fn({key, val}, acc) ->
       map_pair = format_key_value(key, val, keyword?)
       [[eq: ", ", eq: map_pair] | acc]
     end)
+
     [[_ | elem_diff] | rest] = result
     [{:eq, "%" <> name <> "{"}, [elem_diff | rest], {:eq, "}"}]
   end

lib/ex_unit/lib/ex_unit/server.ex

@@ -1,6 +1,5 @@
 defmodule ExUnit.Server do
   @moduledoc false
-  @timeout 60_000
 
   use GenServer
 
@@ -21,11 +20,13 @@ defmodule ExUnit.Server do
   end
 
   def take_async_cases(count) do
-    GenServer.call(__MODULE__, {:take_async_cases, count}, @timeout)
+    timeout = Application.fetch_env!(:ex_unit, :case_load_timeout)
+    GenServer.call(__MODULE__, {:take_async_cases, count}, timeout)
   end
 
   def take_sync_cases() do
-    GenServer.call(__MODULE__, :take_sync_cases, @timeout)
+    timeout = Application.fetch_env!(:ex_unit, :case_load_timeout)
+    GenServer.call(__MODULE__, :take_sync_cases, timeout)
   end
 
   ## Callbacks

lib/ex_unit/mix.exs

@@ -18,6 +18,7 @@ defmodule ExUnit.Mixfile do
        assert_receive_timeout: 100,
        autorun: true,
        capture_log: false,
+       case_load_timeout: 60_000,
        colors: [],
        exclude: [],
        include: [],

lib/ex_unit/test/ex_unit/diff_test.exs

@@ -242,6 +242,8 @@ defmodule ExUnit.DiffTest do
     assert script(map2, map1) == expected
     assert script(map1, %{}) == [{:eq, "%{"}, [[del: "baz: 12"]], {:eq, "}"}]
     assert script(%{}, map1) == [{:eq, "%{"}, [[ins: "baz: 12"]], {:eq, "}"}]
+    expected = [{:eq, "%{"}, [[del: "baz: 12"], [ins: "foo: 12"]], {:eq, "}"}]
+    assert script(map1, %{foo: 12}) == expected
 
     assert script(%{}, %{}) == [eq: "%{}"]
   end

lib/iex/lib/iex/autocomplete.ex

@@ -59,9 +59,12 @@ defmodule IEx.Autocomplete do
   defp reduce(expr) do
     Enum.reduce(' ([{', expr, fn token, acc ->
       hd(:string.tokens(acc, [token]))
-    end) |> Enum.reverse |> strip_ampersand
+    end) |> Enum.reverse |> strip_ampersand |> strip_percent
   end
 
+  defp strip_percent([?% | t]), do: t
+  defp strip_percent(expr), do: expr
+
   defp strip_ampersand([?& | t]), do: t
   defp strip_ampersand(expr), do: expr
 

lib/iex/test/iex/autocomplete_test.exs

@@ -179,4 +179,13 @@ defmodule IEx.AutocompleteTest do
     :code.purge(Sample)
     :code.delete(Sample)
   end
+
+
+  defmodule MyStruct do
+    defstruct my_val: "val"
+  end
+
+  test "completion for structs" do
+    assert expand('%IEx.AutocompleteTest.MyStr') == {:yes, 'uct.', []}
+  end
 end

lib/logger/lib/logger/backends/console.ex

@@ -86,7 +86,7 @@ defmodule Logger.Backends.Console do
     colors     = configure_colors(config)
     max_buffer = Keyword.get(config, :max_buffer, 32)
     %{state | format: format, metadata: Enum.reverse(metadata),
-      level: level, colors: colors, device: device, max_buffer: max_buffer}
+              level: level, colors: colors, device: device, max_buffer: max_buffer}
   end
 
   defp configure_merge(env, options) do
@@ -149,7 +149,7 @@ defmodule Logger.Backends.Console do
     %{format: format, metadata: keys, colors: colors} = state
     format
     |> Logger.Formatter.format(level, msg, ts, take_metadata(md, keys))
-    |> color_event(level, colors)
+    |> color_event(level, colors, md)
   end
 
   defp take_metadata(metadata, keys) do
@@ -161,10 +161,11 @@ defmodule Logger.Backends.Console do
     end
   end
 
-  defp color_event(data, _level, %{enabled: false}), do: data
+  defp color_event(data, _level, %{enabled: false}, _md), do: data
 
-  defp color_event(data, level, %{enabled: true} = colors) do
-    [IO.ANSI.format_fragment(Map.fetch!(colors, level), true), data | IO.ANSI.reset]
+  defp color_event(data, level, %{enabled: true} = colors, md) do
+    color = md[:ansi_color] || Map.fetch!(colors, level)
+    [IO.ANSI.format_fragment(color, true), data | IO.ANSI.reset]
   end
 
   defp log_buffer(%{buffer_size: 0, buffer: []} = state), do: state

lib/logger/test/logger/backends/console_test.exs

@@ -109,4 +109,12 @@ defmodule Logger.Backends.ConsoleTest do
       Logger.error("hello")
     end) == IO.ANSI.cyan() <> "hello" <> IO.ANSI.reset()
   end
+
+  test "can use colors from metadata" do
+    Logger.configure_backend(:console, [format: "$message", colors: [enabled: true]])
+
+    assert capture_log(fn ->
+      Logger.log(:error, "hello", ansi_color: :yellow)
+    end) == IO.ANSI.yellow() <> "hello" <> IO.ANSI.reset()
+  end
 end

lib/mix/lib/mix/compilers/elixir.ex

@@ -246,10 +246,7 @@ defmodule Mix.Compilers.Elixir do
 
   defp get_external_resources(module, cwd) do
     for file <- Module.get_attribute(module, :external_resource),
-        File.regular?(file),
-        relative = Path.relative_to(file, cwd),
-        Path.type(relative) == :relative,
-        do: relative
+        do: Path.relative_to(file, cwd)
   end
 
   defp each_file(source) do

lib/mix/lib/mix/compilers/test.ex

@@ -264,9 +264,6 @@ defmodule Mix.Compilers.Test do
 
   defp get_external_resources(module, cwd) do
     for file <- Module.get_attribute(module, :external_resource),
-        File.regular?(file),
-        relative = Path.relative_to(file, cwd),
-        Path.type(relative) == :relative,
-        do: relative
+        do: Path.relative_to(file, cwd)
   end
 end

lib/mix/lib/mix/dep.ex

@@ -64,19 +64,22 @@ defmodule Mix.Dep do
   @doc """
   Returns loaded dependencies from the cache for the current environment.
 
-  Because the dependencies are cached during deps.check, their
+  Because the dependencies are cached during deps.loadpaths, their
   status may be outdated (for example, `:compile` did not
   yet become `:ok`). Therefore it is recommended to not rely
   on their status, also given they haven't been checked
   against the lock.
   """
   def cached do
-    if project = Mix.Project.get do
-      key = {:cached_deps, project}
-      Mix.ProjectStack.read_cache(key) ||
-        Mix.ProjectStack.write_cache(key, loaded(env: Mix.env))
-    else
-      loaded(env: Mix.env)
+    cond do
+      System.get_env("MIX_NO_DEPS") in ~w(1 true) ->
+        []
+      project = Mix.Project.get ->
+        key = {:cached_deps, project}
+        Mix.ProjectStack.read_cache(key) ||
+          Mix.ProjectStack.write_cache(key, loaded(env: Mix.env))
+      true ->
+        loaded(env: Mix.env)
     end
   end
 
@@ -225,16 +228,16 @@ defmodule Mix.Dep do
   def format_status(%Mix.Dep{app: app, status: {:divergedonly, other}} = dep) do
     recommendation =
       if Keyword.has_key?(other.opts, :only) do
-        "Ensure the parent dependency specifies a superset of the child one in"
+        "Ensure you specify at least the same environments in :only in your dep"
       else
-        "Remove the :only restriction from"
+        "Remove the :only restriction from your dep"
       end
 
-    "the dependency #{app}\n" <>
+    "the :only option for dependency #{app}\n" <>
     "#{dep_status(dep)}" <>
-    "\n  does not match the environments calculated for\n" <>
+    "\n  does not match the :only option calculated for\n" <>
     "#{dep_status(other)}" <>
-    "\n  #{recommendation} your dep"
+    "\n  #{recommendation}"
   end
 
   def format_status(%Mix.Dep{app: app, status: {:diverged, other}} = dep) do
@@ -264,7 +267,7 @@ defmodule Mix.Dep do
     do: "the dependency was built with another SCM, run \"#{mix_env_var}mix deps.compile\""
 
   defp dep_status(%Mix.Dep{app: app, requirement: req, manager: manager, opts: opts, from: from}) do
-    opts = Keyword.drop(opts, [:dest, :build, :lock, :manager])
+    opts = Keyword.drop(opts, [:dest, :build, :lock, :manager, :checkout])
     opts = opts ++ (if manager, do: [manager: manager], else: [])
     info = if req, do: {app, req, opts}, else: {app, opts}
     "\n  > In #{Path.relative_to_cwd(from)}:\n    #{inspect info}\n"

lib/mix/lib/mix/dep/converger.ex

@@ -84,7 +84,7 @@ defmodule Mix.Dep.Converger do
     if not diverged? && remote do
       # If there is a lock, it means we are doing a get/update
       # and we need to hit the remote converger which do external
-      # requests and what not. In case of deps.check, deps and so
+      # requests and what not. In case of deps.loadpaths, deps and so
       # on, there is no lock, so we won't hit this branch.
       lock = if lock_given?, do: remote.converge(deps, lock), else: lock
 

lib/mix/lib/mix/scm/git.ex

@@ -25,6 +25,11 @@ defmodule Mix.SCM.Git do
   end
 
   def accepts_options(_app, opts) do
+    opts =
+      opts
+      |> Keyword.put(:checkout, opts[:dest])
+      |> sparse_opts()
+
     cond do
       gh = opts[:github] ->
         opts
@@ -41,7 +46,9 @@ defmodule Mix.SCM.Git do
 
   def checked_out?(opts) do
     # Are we inside a Git repository?
-    File.regular?(Path.join(opts[:dest], ".git/HEAD"))
+    opts[:checkout]
+    |> Path.join(".git/HEAD")
+    |> File.regular?
   end
 
   def lock_status(opts) do
@@ -50,7 +57,7 @@ defmodule Mix.SCM.Git do
 
     cond do
       lock_rev = get_lock_rev(lock, opts) ->
-        File.cd!(opts[:dest], fn ->
+        File.cd!(opts[:checkout], fn ->
           %{origin: origin, rev: rev} = get_rev_info()
           if get_lock_repo(lock) == origin and lock_rev == rev do
             :ok
@@ -76,29 +83,75 @@ defmodule Mix.SCM.Git do
 
   def checkout(opts) do
     assert_git!()
-
-    path     = opts[:dest]
-    location = opts[:git]
-
-    _ = File.rm_rf!(path)
-    git!(~s(clone --no-checkout --progress "#{location}" "#{path}"))
-
-    File.cd! path, fn -> do_checkout(opts) end
+    path = opts[:checkout]
+    File.rm_rf!(path)
+    File.mkdir_p!(path)
+    File.cd!(path, fn ->
+      git!("init --quiet")
+      git!("--git-dir=.git remote add origin \"#{opts[:git]}\"")
+      checkout(path, opts)
+    end)
   end
 
   def update(opts) do
     assert_git!()
+    path = opts[:checkout]
+    File.cd! path, fn -> checkout(path, opts) end
+  end
 
-    File.cd! opts[:dest], fn ->
-      location = opts[:git]
-      update_origin(location)
+  defp checkout(_path, opts) do
+    # Set configuration
+    sparse_toggle(opts)
+    update_origin(opts[:git])
 
-      command = IO.iodata_to_binary(["--git-dir=.git fetch --force",
-                                     progress_switch(git_version()),
-                                     tags_switch(opts[:tag])])
+    # Fetch external data
+    command = IO.iodata_to_binary(["--git-dir=.git fetch --force --quiet",
+                                   progress_switch(git_version()),
+                                   tags_switch(opts[:tag])])
+    git!(command)
 
-      git!(command)
-      do_checkout(opts)
+    # Migrate the git repo
+    rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts)
+    git!("--git-dir=.git checkout --quiet #{rev}")
+
+    if opts[:submodules] do
+      git!("--git-dir=.git submodule update --init --recursive")
+    end
+
+    # Get the new repo lock
+    get_lock(opts)
+  end
+
+  defp sparse_opts(opts) do
+    if opts[:sparse] do
+      dest = Path.join(opts[:dest], opts[:sparse])
+      Keyword.put(opts, :dest, dest)
+    else
+      opts
+    end
+  end
+
+  defp sparse_toggle(opts) do
+    cond do
+      sparse = opts[:sparse] ->
+        sparse_check(git_version())
+        git!("--git-dir=.git config core.sparsecheckout true")
+        File.write!(".git/info/sparse-checkout", sparse)
+      File.exists?(".git/info/sparse-checkout") ->
+        File.write!(".git/info/sparse-checkout", "*")
+        git!("--git-dir=.git read-tree -mu HEAD")
+        git!("--git-dir=.git config core.sparsecheckout false")
+        File.rm(".git/info/sparse-checkout")
+      true ->
+        :ok
+    end
+  end
+
+  defp sparse_check(version) do
+    unless {1, 7, 0} <= version do
+      version = version |> Tuple.to_list |> Enum.join(".")
+      Mix.raise "Git >= 1.7.0 is required to use sparse checkout. " <>
+                "You are running version #{version}"
     end
   end
 
@@ -111,24 +164,17 @@ defmodule Mix.SCM.Git do
   ## Helpers
 
   defp validate_git_options(opts) do
-    case Keyword.take(opts, [:branch, :ref, :tag]) do
-      []  -> opts
-      [_] -> opts
-      _   ->
-        Mix.raise "You should specify only one of branch, ref or tag, and only once. " <>
-                  "Error on Git dependency: #{opts[:git]}"
-    end
+    err = "You should specify only one of branch, ref or tag, and only once. " <>
+          "Error on Git dependency: #{opts[:git]}"
+    validate_single_uniq(opts, [:branch, :ref, :tag], err)
   end
 
-  defp do_checkout(opts) do
-    rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts)
-    git!("--git-dir=.git checkout --quiet #{rev}")
-
-    if opts[:submodules] do
-      git!("--git-dir=.git submodule update --init --recursive")
+  defp validate_single_uniq(opts, take, error) do
+    case Keyword.take(opts, take) do
+      []  -> opts
+      [_] -> opts
+      _   -> Mix.raise error
     end
-
-    get_lock(opts)
   end
 
   defp get_lock(opts) do
@@ -146,7 +192,8 @@ defmodule Mix.SCM.Git do
   defp get_lock_rev(_, _), do: nil
 
   defp get_lock_opts(opts) do
-    lock_opts = Keyword.take(opts, [:branch, :ref, :tag])
+    lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse])
+
     if opts[:submodules] do
       lock_opts ++ [submodules: true]
     else

lib/mix/lib/mix/task.ex

@@ -275,7 +275,7 @@ defmodule Mix.Task do
     # 2. Otherwise we look for it in dependencies.
     # 3. Finally, we compile the current project in hope it is available.
     module =
-      get_task_or_run(proj, task, fn -> Mix.Task.run("deps.check") end) ||
+      get_task_or_run(proj, task, fn -> Mix.Task.run("deps.loadpaths") end) ||
       get_task_or_run(proj, task, fn -> Mix.Project.compile([]) end) ||
       get!(task)
 

lib/mix/lib/mix/tasks/app.start.ex

@@ -20,7 +20,7 @@ defmodule Mix.Tasks.App.Start do
       applications are started in permanent mode
 
     * `:consolidate_protocols` - when `true`, loads consolidated
-      protocols before start
+      protocols before start. The default value is `true`.
 
     * `:elixir` - matches the current elixir version against the
       given requirement

lib/mix/lib/mix/tasks/clean.ex

@@ -49,6 +49,6 @@ defmodule Mix.Tasks.Clean do
   defp loadpaths! do
     Mix.Task.run "loadpaths", ["--no-elixir-version-check", "--no-deps-check", "--no-archives-check"]
     Mix.Task.reenable "loadpaths"
-    Mix.Task.reenable "deps.check"
+    Mix.Task.reenable "deps.loadpaths"
   end
 end

lib/mix/lib/mix/tasks/compile.ex

@@ -14,7 +14,8 @@ defmodule Mix.Tasks.Compile do
       `[:yecc, :leex, :erlang, :elixir, :xref, :app]`
 
     * `:consolidate_protocols` - when `true`, runs protocol
-      consolidation via the `compile.protocols` task
+      consolidation via the `compile.protocols` task. The default
+      value is `true`.
 
     * `:build_embedded` - when `true`, activates protocol
       consolidation and does not generate symlinks in builds
@@ -95,7 +96,7 @@ defmodule Mix.Tasks.Compile do
   defp loadpaths! do
     Mix.Task.run "loadpaths", ["--no-elixir-version-check", "--no-deps-check", "--no-archives-check"]
     Mix.Task.reenable "loadpaths"
-    Mix.Task.reenable "deps.check"
+    Mix.Task.reenable "deps.loadpaths"
   end
 
   defp consolidate_protocols? do

lib/mix/lib/mix/tasks/compile.protocols.ex

@@ -9,7 +9,8 @@ defmodule Mix.Tasks.Compile.Protocols do
 
   This task is automatically invoked whenever the project
   enables `:consolidate_protocols` or `:build_embedded` in
-  its configuration.
+  its configuration. As `:consolidate_protocols` is enabled
+  in the default configuration, this is the default case.
 
   ## Consolidation
 
@@ -44,10 +45,7 @@ defmodule Mix.Tasks.Compile.Protocols do
     output   = Mix.Project.consolidation_path(config)
     manifest = Path.join(output, @manifest)
 
-    protocols_and_impls =
-      unless Mix.Project.umbrella?(config) do
-        protocols_and_impls(config)
-      end
+    protocols_and_impls = protocols_and_impls(config)
 
     cond do
       opts[:force] || Mix.Utils.stale?(Mix.Project.config_files(), [manifest]) ->
@@ -79,10 +77,15 @@ defmodule Mix.Tasks.Compile.Protocols do
                not scm.fetchable?,
                do: opts[:build])
 
-    app = Mix.Project.app_path(config)
+    app =
+      if Mix.Project.umbrella?(config) do
+        []
+      else
+        [Mix.Project.app_path(config)]
+      end
 
     protocols_and_impls =
-      for path <- [app | deps] do
+      for path <- app ++ deps do
         manifest_path = Path.join(path, ".compile.elixir")
         compile_path = Path.join(path, "ebin")
         Mix.Compilers.Elixir.protocols_and_impls(manifest_path, compile_path)
@@ -120,10 +123,24 @@ defmodule Mix.Tasks.Compile.Protocols do
   defp consolidate(protocol, paths, output, opts) do
     impls = Protocol.extract_impls(protocol, paths)
     reload(protocol)
-    {:ok, binary} = Protocol.consolidate(protocol, impls)
-    File.write!(Path.join(output, "#{protocol}.beam"), binary)
-    if opts[:verbose] do
-      Mix.shell.info "Consolidated #{inspect protocol}"
+    case Protocol.consolidate(protocol, impls) do
+      {:ok, binary} ->
+        File.write!(Path.join(output, "#{protocol}.beam"), binary)
+        if opts[:verbose] do
+          Mix.shell.info "Consolidated #{inspect protocol}"
+        end
+
+      # If we remove a dependency and we have implemented one of its
+      # protocols locally, we will mark the protocol as needing to be
+      # reconsolidated when the implementation is removed even though
+      # the protocol no longer exists. Although most times removing a
+      # dependency will trigger a full recompilation, such won't happen
+      # in umbrella apps with shared build.
+      {:error, :no_beam_info} ->
+        remove_consolidated(protocol, output)
+        if opts[:verbose] do
+          Mix.shell.info "Unavailable #{inspect protocol}"
+        end
     end
   end
 

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -133,7 +133,8 @@ defmodule Mix.Tasks.Deps.Compile do
       end
 
       try do
-        res = Mix.Task.run("compile", ["--no-deps", "--no-archives-check", "--no-elixir-version-check"])
+        res = Mix.Task.run("compile", ["--no-deps", "--no-archives-check",
+                                       "--no-elixir-version-check", "--no-warnings-as-errors"])
         :ok in List.wrap(res)
       catch
         kind, reason ->
@@ -191,7 +192,7 @@ defmodule Mix.Tasks.Deps.Compile do
       else
         "make"
       end
-    do_command(dep, config, command, true)
+    do_command(dep, config, command, true, [{"IS_DEP", "1"}])
   end
 
   defp do_compile(%Mix.Dep{opts: opts} = dep, config) do

lib/mix/lib/mix/tasks/deps.loadpaths.ex

@@ -1,16 +1,18 @@
-defmodule Mix.Tasks.Deps.Check do
+defmodule Mix.Tasks.Deps.Loadpaths do
   use Mix.Task
 
   import Mix.Dep, only: [loaded_by_name: 2, format_dep: 1, ok?: 1,
                          format_status: 1, check_lock: 1]
 
   @moduledoc """
-  Checks if all dependencies are valid,
-  loading them along the way.
+  Checks and loads all dependencies along the way.
 
   If there is an invalid dependency, its status is printed
   before aborting.
 
+  Although this task does not show up in `mix help`, it is
+  part of Mix public API and can be depended on.
+
   ## Command line options
 
     * `--no-deps-check` - do not check or compile deps, only load available ones
@@ -31,7 +33,9 @@ defmodule Mix.Tasks.Deps.Check do
         path
       end
 
-    prune_deps(load_paths, "--no-deps-check" in args)
+    unless System.get_env("MIX_NO_DEPS") in ~w(1 true) do
+      prune_deps(load_paths, "--no-deps-check" in args)
+    end
   end
 
   # If the build is per environment, we should be able to look

lib/mix/lib/mix/tasks/deps.unlock.ex

@@ -16,7 +16,7 @@ defmodule Mix.Tasks.Deps.Unlock do
 
   """
 
-  @switches [all: :boolean, unused: :boolean]
+  @switches [all: :boolean, unused: :boolean, filter: :string]
 
   @spec run(OptionParser.argv) :: :ok
   def run(args) do
@@ -29,6 +29,27 @@ defmodule Mix.Tasks.Deps.Unlock do
       opts[:unused] ->
         apps = Mix.Dep.loaded([]) |> Enum.map(& &1.app)
         Mix.Dep.Lock.read() |> Map.take(apps) |> Mix.Dep.Lock.write()
+      filter = opts[:filter] ->
+        lock = Mix.Dep.Lock.read
+        apps = Map.keys(lock)
+
+        unlock =
+          apps
+          |> Enum.filter(&String.contains?("#{&1}", filter))
+
+        if unlock == [] do
+          Mix.shell.error "warning: no dependencies were matched"
+        else
+          lock =
+            Enum.reject(lock, fn({app, _}) ->
+              app in unlock
+            end)
+          Mix.Dep.Lock.write(lock)
+          Mix.shell.info """
+          Unlocked deps:
+          * #{Enum.join(unlock, "\n* ")}
+          """
+        end
 
       apps != [] ->
         lock =

lib/mix/lib/mix/tasks/escript.ex

@@ -1,7 +1,7 @@
 defmodule Mix.Tasks.Escript do
   use Mix.Task
 
-  @shortdoc "List installed escripts."
+  @shortdoc "Lists installed escripts"
 
   @moduledoc ~S"""
   Lists all installed escripts.

lib/mix/lib/mix/tasks/help.ex

@@ -107,7 +107,7 @@ defmodule Mix.Tasks.Help do
   defp loadpaths! do
     Mix.Task.run "loadpaths", ["--no-elixir-version-check", "--no-deps-check", "--no-archives-check"]
     Mix.Task.reenable "loadpaths"
-    Mix.Task.reenable "deps.check"
+    Mix.Task.reenable "deps.loadpaths"
   end
 
   defp load_tasks() do

lib/mix/lib/mix/tasks/loadpaths.ex

@@ -31,9 +31,9 @@ defmodule Mix.Tasks.Loadpaths do
     end
 
     # --no-deps is used only internally. It has no purpose
-    # from Mix.CLI because the CLI itself already loads deps.
+    # from Mix.CLI because running a task may load deps.
     unless "--no-deps" in args do
-      Mix.Task.run "deps.check", args
+      Mix.Task.run "deps.loadpaths", args
     end
 
     if config[:app] do

lib/mix/lib/mix/tasks/test.ex

@@ -67,6 +67,9 @@ defmodule Mix.Tasks.Test do
     * `--no-elixir-version-check` - do not check the Elixir version from mix.exs
     * `--stale` - run only tests which reference modules that changed since the
       last `test --stale`. You can read more about this option in the "Stale" section below.
+    * `--listen-on-stdin` - run tests, and then listen on stdin. Receiving a newline will
+      result in the tests being run again. Very useful when combined with `--stale` and
+      external commands which produce output on stdout upon file system modification.
 
   ## Filters
 
@@ -164,7 +167,7 @@ defmodule Mix.Tasks.Test do
              exclude: :keep, seed: :integer, only: :keep, compile: :boolean,
              start: :boolean, timeout: :integer, raise: :boolean,
              deps_check: :boolean, archives_check: :boolean, elixir_version_check: :boolean,
-             stale: :boolean]
+             stale: :boolean, listen_on_stdin: :boolean]
 
   @cover [output: "cover", tool: Cover]
 
@@ -172,6 +175,15 @@ defmodule Mix.Tasks.Test do
   def run(args) do
     {opts, files} = OptionParser.parse!(args, strict: @switches)
 
+    if opts[:listen_on_stdin] do
+      System.at_exit fn _ ->
+        IO.gets(:stdio, "")
+        Mix.shell.info "Restarting..."
+        :init.restart()
+        :timer.sleep(:infinity)
+      end
+    end
+
     unless System.get_env("MIX_ENV") || Mix.env == :test do
       Mix.raise "\"mix test\" is running on environment \"#{Mix.env}\". If you are " <>
                                 "running tests along another task, please set MIX_ENV explicitly"

lib/mix/lib/mix/tasks/xref.ex

@@ -13,28 +13,28 @@ defmodule Mix.Tasks.Xref do
   ## Xref modes
 
   The `xref` task expects a mode as first argument:
-  
+
       mix xref MODE
-  
+
   All available modes are discussed below.
-  
+
   ### warnings
 
   Prints warnings for violated cross reference checks:
-  
+
       mix xref warnings
-  
+
   This is the mode used by Mix during compilation.
 
   ### unreachable
 
   Prints all unreachable "file:line: module.function/arity" entries:
-  
+
       mix xref unreachable
-  
+
   The "file:line" represents the file and line a call to an unknown
   "module.function/arity" is made.
-  
+
   ### callers CALLEE
 
   Prints all callers of the given `CALLEE`, which can be one of: `Module`,
@@ -48,7 +48,7 @@ defmodule Mix.Tasks.Xref do
 
   Prints a file dependency graph where an edge from `A` to `B` indicates
   that `A` depends on `B`.
- 
+
       mix xref graph --format dot
 
   The following options are accepted:
@@ -106,6 +106,8 @@ defmodule Mix.Tasks.Xref do
     {opts, args} =
       OptionParser.parse!(args, strict: @switches)
 
+    Mix.Task.run("loadpaths")
+
     if Keyword.get(opts, :compile, true) do
       Mix.Task.run("compile")
     end

lib/mix/test/fixtures/.gitignore

@@ -1,3 +1,4 @@
 git_repo
+git_sparse_repo
 deps_on_git_repo
 git_rebar

lib/mix/test/fixtures/rebar_dep/rebar.config

@@ -1,7 +1,3 @@
 {sub_dirs, ["apps/*"]}.
 
-{deps, [
-    {git_rebar, "0.1..*", {git, "../../test/fixtures/git_rebar", master}}
-]}.
-
 {overrides, [{add, rebar_dep, [{src_dirs, ["apps/rebar_dep/src"]}]}]}.

lib/mix/test/fixtures/rebar_dep/rebar.config.script

@@ -1 +1 @@
-CONFIG ++ [{'SCRIPT', SCRIPT}].
+CONFIG ++ [{'SCRIPT', SCRIPT}] ++ [{deps, [{git_rebar, "0.1..*", {git, filename:absname("../../test/fixtures/git_rebar"), master}}]}].

lib/mix/test/fixtures/rebar_override/rebar.config

@@ -1,7 +0,0 @@
-{deps, [
-    {git_rebar, {git, "../../test/fixtures/git_rebar"}}
-]}.
-
-{overrides, [
-  {override, git_rebar, [{deps, [{git_repo, {git, "../../test/fixtures/git_repo"}}]}]}
-]}.

lib/mix/test/fixtures/rebar_override/rebar.config.script

@@ -0,0 +1,8 @@
+[
+  {deps, [
+    {git_rebar, {git, filename:absname("../../test/fixtures/git_rebar")}}
+  ]},
+  {overrides, [
+    {override, git_rebar, [{deps, [{git_repo, {git, filename:absname("../../test/fixtures/git_repo")}}]}]}
+  ]}
+].

lib/mix/test/mix/dep_test.exs

@@ -346,12 +346,12 @@ defmodule Mix.DepTest do
         refute_received {:mix_shell, :info, ["* Getting" <> _]}
 
         assert_raise Mix.Error, "Can't continue due to errors on dependencies", fn ->
-          Mix.Tasks.Deps.Check.run([])
+          Mix.Tasks.Deps.Loadpaths.run([])
         end
 
         Mix.ProjectStack.clear_cache()
         Mix.env(:prod)
-        Mix.Tasks.Deps.Check.run([])
+        Mix.Tasks.Deps.Loadpaths.run([])
       end
     end
   end
@@ -496,7 +496,7 @@ defmodule Mix.DepTest do
         Mix.Tasks.Deps.run([])
         assert_received {:mix_shell, :info, ["* git_repo" <> _]}
         assert_received {:mix_shell, :info, [msg]}
-        assert msg =~ "Ensure the parent dependency specifies a superset of the child one"
+        assert msg =~ "Ensure you specify at least the same environments in :only in your dep"
       end
     end
   end

lib/mix/test/mix/rebar_test.exs

@@ -106,7 +106,7 @@ defmodule Mix.RebarTest do
 
     assert Enum.find(deps, fn %Mix.Dep{app: app, opts: opts} ->
       if app == :git_rebar do
-        assert Enum.find(opts, &match?({:git, "../../test/fixtures/git_rebar"}, &1))
+        assert Enum.find(opts, &match?({:git, _}, &1))
         assert Enum.find(opts, &match?({:ref, "master"}, &1))
         true
       end
@@ -155,7 +155,7 @@ defmodule Mix.RebarTest do
 
     in_tmp "get and compile dependencies for Rebar", fn ->
       Mix.Tasks.Deps.Get.run []
-      assert_received {:mix_shell, :info, ["* Getting git_rebar (../../test/fixtures/git_rebar)"]}
+      assert_received {:mix_shell, :info, ["* Getting git_rebar" <> _]}
 
       Mix.Tasks.Deps.Compile.run []
       assert_received {:mix_shell, :run, ["==> git_rebar (compile)\n"]}
@@ -185,7 +185,7 @@ defmodule Mix.RebarTest do
 
     in_tmp "get and compile dependencies for rebar3", fn ->
       Mix.Tasks.Deps.Get.run []
-      assert_received {:mix_shell, :info, ["* Getting git_rebar (../../test/fixtures/git_rebar)"]}
+      assert_received {:mix_shell, :info, ["* Getting git_rebar " <> _]}
 
       Mix.Tasks.Deps.Compile.run []
       assert_received {:mix_shell, :run, ["===> Compiling git_rebar\n"]}

lib/mix/test/mix/tasks/compile.elixir_test.exs

@@ -172,25 +172,43 @@ defmodule Mix.Tasks.Compile.ElixirTest do
 
   test "compiles dependent changed files" do
     in_fixture "no_mixfile", fn ->
+      tmp = tmp_path("c.eex")
       File.touch!("lib/a.eex")
+
       File.write!("lib/a.ex", """
       defmodule A do
-        @external_resource "lib/b.eex"
         @external_resource "lib/a.eex"
+        @external_resource #{inspect tmp}
         def a, do: :ok
       end
       """)
 
+      # Compiles with missing external resources
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :noop
       Mix.shell.flush
       purge [A, B]
 
+      # Update local existing resource
       File.touch!("lib/a.eex", {{2020, 1, 1}, {0, 0, 0}})
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
 
+      # Does not update on old existing resource
+      File.touch!("lib/a.eex", {{1970, 1, 1}, {0, 0, 0}})
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :noop
+      Mix.shell.flush
+      purge [A, B]
+
+      # Update external existing resource
+      File.touch!(tmp, {{2020, 1, 1}, {0, 0, 0}})
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
       refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
     end
+  after
+    File.rm tmp_path("c.eex")
   end
 
   test "does not recompile empty files" do
@@ -231,4 +249,28 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
     end
   end
+
+  test "does not treat remote typespecs as compile time dependencies" do
+    in_fixture "no_mixfile", fn ->
+      File.write!("lib/b.ex", """
+      defmodule B do
+        @type t :: A.t
+      end
+      """)
+
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      assert_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
+
+      Mix.shell.flush
+      purge [A, B]
+
+      future = {{2020, 1, 1}, {0, 0, 0}}
+      File.touch!("lib/a.ex", future)
+      Mix.Tasks.Compile.Elixir.run ["--verbose"]
+
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
+    end
+  end
 end

lib/mix/test/mix/tasks/deps.git_test.exs

@@ -69,6 +69,20 @@ defmodule Mix.Tasks.DepsGitTest do
     end
   end
 
+  test "gets and updates Git repos with sparse checkout" do
+    Process.put(:git_repo_opts, sparse: "sparse_dir")
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      message = "* Getting git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+      refute File.exists?("deps/git_repo/mix.exs")
+      assert File.exists?("deps/git_repo/sparse_dir/mix.exs")
+      assert File.read!("mix.lock") =~ "sparse: \"sparse_dir\""
+    end
+  end
+
   test "handles invalid .git directory" do
     Mix.Project.push GitApp
 
@@ -90,7 +104,7 @@ defmodule Mix.Tasks.DepsGitTest do
       File.rm_rf!("deps/git_repo/.git")
 
       assert_raise Mix.Error, "Can't continue due to errors on dependencies", fn ->
-        Mix.Tasks.Deps.Check.run ["git_repo"]
+        Mix.Tasks.Deps.Loadpaths.run ["git_repo"]
       end
     end
   end
@@ -126,7 +140,7 @@ defmodule Mix.Tasks.DepsGitTest do
       Code.delete_path("_build/dev/lib/git_repo/ebin")
 
       # Deps on Git repo loads it automatically on compile
-      Mix.Task.reenable "deps.check"
+      Mix.Task.reenable "deps.loadpaths"
       Mix.Tasks.Deps.Compile.run ["deps_on_git_repo"]
       assert File.exists?("_build/dev/lib/deps_on_git_repo/ebin")
     end
@@ -243,7 +257,7 @@ defmodule Mix.Tasks.DepsGitTest do
       # Update the lock and now we should get an error
       Mix.Dep.Lock.write %{git_repo: {:git, fixture_path("git_repo"), last, []}}
       assert_raise Mix.Error, fn ->
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       # Flush the errors we got, move to a clean slate
@@ -265,6 +279,61 @@ defmodule Mix.Tasks.DepsGitTest do
     purge [GitRepo, GitRepo.Mixfile]
   end
 
+  test "updates the repo when sparse is turned off" do
+    Process.put(:git_repo_opts, sparse: "sparse_dir")
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      refute File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      # Flush the errors we got, move to a clean slate
+      Mix.shell.flush
+      Mix.Task.clear
+      Process.delete(:git_repo_opts)
+      Mix.Project.pop
+      Mix.Project.push GitApp
+
+      # Calling get should update the dependency
+      Mix.Tasks.Deps.Get.run []
+      refute File.read!("mix.lock") =~ "sparse_dir"
+      assert File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      message = "* Updating git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+
+      # Check we got no error
+      refute_received {:mix_shell, :error, _}
+    end
+  end
+
+  test "updates the repo when sparse is turned on" do
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      assert File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      # Flush the errors we got, move to a clean slate
+      Mix.shell.flush
+      Mix.Task.clear
+      Process.put(:git_repo_opts, sparse: "sparse_dir")
+      Mix.Project.pop
+      Mix.Project.push GitApp
+
+      # Calling get should update the dependency
+      Mix.Tasks.Deps.Get.run []
+      assert File.read!("mix.lock") =~ "sparse_dir"
+      refute File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      message = "* Updating git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+
+      # Check we got no error
+      refute_received {:mix_shell, :error, _}
+    end
+  end
+
   test "updates the repo and the lock when the mixfile updates" do
     Mix.Project.push GitApp
     [last, first | _] = get_git_repo_revs
@@ -304,7 +373,7 @@ defmodule Mix.Tasks.DepsGitTest do
       exception = assert_raise Mix.Error, fn ->
         Mix.Tasks.Deps.Get.run []
       end
-      assert Exception.message(exception) =~ "Command \"git clone"
+      assert Exception.message(exception) =~ "Command \"git --git-dir=.git fetch"
     end
   end
 
@@ -364,7 +433,7 @@ defmodule Mix.Tasks.DepsGitTest do
     Mix.Project.push(name, file)
   end
 
-  defp get_git_repo_revs do
+  defp get_git_repo_revs() do
     File.cd! fixture_path("git_repo"), fn ->
       Regex.split ~r(\r?\n), System.cmd("git", ["log", "--format=%H"]) |> elem(0)
     end

lib/mix/test/mix/tasks/deps_test.exs

@@ -121,13 +121,13 @@ defmodule Mix.Tasks.DepsTest do
     end
   end
 
-  ## deps.check
+  ## deps.loadpaths
 
   test "checks list of dependencies and their status with success" do
     Mix.Project.push SuccessfulDepsApp
 
     in_fixture "deps_status", fn ->
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
     end
   end
 
@@ -136,7 +136,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       assert_raise Mix.Error, fn ->
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       assert_received {:mix_shell, :error, ["* ok (https://github.com/elixir-lang/ok.git)"]}
@@ -160,7 +160,7 @@ defmodule Mix.Tasks.DepsTest do
       File.rm_rf("_build")
 
       Mix.Tasks.Deps.Compile.run []
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
       assert File.exists?("_build/dev/lib/ok/ebin/ok.app")
       assert File.exists?("_build/dev/lib/ok/priv/sample")
 
@@ -174,7 +174,7 @@ defmodule Mix.Tasks.DepsTest do
       Mix.Project.pop
       Mix.Project.push SuccessfulDepsApp
 
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
       refute to_charlist(Path.expand("_build/dev/lib/ok/ebin/")) in :code.get_path
       assert File.exists?("_build/dev/lib/ok/ebin/ok.app")
       assert File.exists?("_build/dev/lib/sample/ebin/sample.app")
@@ -185,7 +185,7 @@ defmodule Mix.Tasks.DepsTest do
       Mix.Project.pop
       Mix.Project.push SuccessfulDepsApp
 
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
       refute File.exists?("_build/dev/lib/ok/ebin/ok.app")
       assert File.exists?("_build/dev/lib/sample/ebin/sample.app")
     end
@@ -238,6 +238,21 @@ defmodule Mix.Tasks.DepsTest do
     end
   end
 
+  test "unlocks filtered deps", context do
+    Mix.Project.push DepsApp
+    in_tmp context.test, fn ->
+      Mix.Dep.Lock.write %{git_repo: "abcdef", another: "hash", another_one: "hash"}
+      Mix.Tasks.Deps.Unlock.run ["--filter", "another"]
+      assert Mix.Dep.Lock.read == %{git_repo: "abcdef"}
+      output = """
+      Unlocked deps:
+      * another
+      * another_one
+      """
+      assert_received {:mix_shell, :info, [^output]}
+    end
+  end
+
   ## Deps environment
 
   defmodule DepsEnvApp do
@@ -364,7 +379,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       assert_raise Mix.Error, fn ->
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
       assert_received {:mix_shell, :error, ["  the dependency git_repo in mix.exs is overriding a child dependency" <> _]}
 
@@ -385,7 +400,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       assert_raise Mix.Error, fn ->
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       assert_received {:mix_shell, :error, ["  different specs were given for the git_repo app:" <> _ = msg]}
@@ -416,7 +431,7 @@ defmodule Mix.Tasks.DepsTest do
 
       assert_raise Mix.Error, fn ->
         Mix.Tasks.Deps.Get.run []
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       assert_received {:mix_shell, :error, ["  the dependency git_repo 0.1.0" <> _ = msg]}
@@ -447,7 +462,7 @@ defmodule Mix.Tasks.DepsTest do
 
       assert_raise Mix.Error, fn ->
         Mix.Tasks.Deps.Get.run []
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       assert_received {:mix_shell, :error, ["  the dependency git_repo in mix.exs is overriding" <> _]}
@@ -503,7 +518,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       assert_raise Mix.Error, fn ->
-        Mix.Tasks.Deps.Check.run []
+        Mix.Tasks.Deps.Loadpaths.run []
       end
 
       receive do
@@ -523,7 +538,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       Mix.Tasks.Deps.Compile.run []
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
 
       File.mkdir_p!("_build/dev/lib/ok/ebin")
       manifest_data = :erlang.term_to_binary({:v1, "the_future", :scm})
@@ -535,8 +550,8 @@ defmodule Mix.Tasks.DepsTest do
       Mix.Tasks.Deps.run []
       assert_received {:mix_shell, :info, [^msg]}
 
-      # deps.check will automatically recompile it
-      Mix.Tasks.Deps.Check.run []
+      # deps.loadpaths will automatically recompile it
+      Mix.Tasks.Deps.Loadpaths.run []
 
       Mix.Tasks.Deps.run []
       refute_received {:mix_shell, :info, [^msg]}
@@ -548,7 +563,7 @@ defmodule Mix.Tasks.DepsTest do
 
     in_fixture "deps_status", fn ->
       Mix.Tasks.Deps.Compile.run []
-      Mix.Tasks.Deps.Check.run []
+      Mix.Tasks.Deps.Loadpaths.run []
 
       File.mkdir_p!("_build/dev/lib/ok/ebin")
       manifest_data = :erlang.term_to_binary({:v1, System.version, :scm})
@@ -560,8 +575,8 @@ defmodule Mix.Tasks.DepsTest do
       Mix.Tasks.Deps.run []
       assert_received {:mix_shell, :info, [^msg]}
 
-      # deps.check will automatically recompile it
-      Mix.Tasks.Deps.Check.run []
+      # deps.loadpaths will automatically recompile it
+      Mix.Tasks.Deps.Loadpaths.run []
 
       Mix.Tasks.Deps.run []
       refute_received {:mix_shell, :info, [^msg]}

lib/mix/test/mix/tasks/test_test.exs

@@ -104,6 +104,84 @@ defmodule Mix.Tasks.TestTest do
     end
   end
 
+  test "--listen-on-stdin: runs tests after input" do
+    in_fixture "test_stale", fn ->
+      port = mix_port(~w[test --stale --listen-on-stdin])
+
+      assert receive_until_match(port, "seed", []) =~ "2 tests"
+
+      :erlang.port_command(port, "\n")
+
+      assert receive_until_match(port, "No stale tests.", []) =~ "Restarting..."
+    end
+  end
+
+  test "--listen-on-stdin: does not exit on compilation failure" do
+    in_fixture "test_stale", fn ->
+      File.write!("lib/b.ex", """
+      defmodule B do
+        def f, do: error_not_a_var
+      end
+      """)
+
+      port = mix_port(~w[test --listen-on-stdin])
+
+      assert receive_until_match(port, "error", []) =~ "lib/b.ex"
+
+      File.write!("lib/b.ex", """
+      defmodule B do
+        def f, do: A.f
+      end
+      """)
+
+      Port.command(port, "\n")
+
+      assert receive_until_match(port, "seed", []) =~ "2 tests"
+
+      File.write!("test/b_test_stale.exs", """
+      defmodule BTest do
+        use ExUnit.Case
+
+        test "f" do
+          assert B.f() == error_not_a_var
+        end
+      end
+      """)
+
+      Port.command(port, "\n")
+
+      assert receive_until_match(port, "undefined function error_not_a_var", []) =~ "test/b_test_stale.exs"
+
+      File.write!("test/b_test_stale.exs", """
+      defmodule BTest do
+        use ExUnit.Case
+
+        test "f" do
+          assert B.f() == :ok
+        end
+      end
+      """)
+
+      Port.command(port, "\n")
+
+      assert receive_until_match(port, "seed", []) =~ "2 tests"
+    end
+  end
+
+  defp receive_until_match(port, expected, acc) do
+    receive do
+      {^port, {:data, charlist}} ->
+        string = to_string(charlist)
+        acc = [acc | string]
+
+        if string =~ expected do
+          IO.iodata_to_binary(acc)
+        else
+          receive_until_match(port, expected, acc)
+        end
+    end
+  end
+
   defp set_all_mtimes(time \\ {{2010, 1, 1}, {0, 0, 0}}) do
     Enum.each(Path.wildcard("**", match_dot: true), &File.touch!(&1, time))
   end

lib/mix/test/mix/tasks/xref_test.exs

@@ -564,6 +564,32 @@ defmodule Mix.Tasks.XrefTest do
     end
   end
 
+  test "graph: with dynamic module" do
+    in_fixture "no_mixfile", fn ->
+      File.write! "lib/a.ex", """
+      B.define()
+      """
+
+      File.write! "lib/b.ex", """
+      defmodule B do
+        def define do
+          defmodule A do
+          end
+        end
+      end
+      """
+
+      assert Mix.Task.run("xref", ["graph"]) == :ok
+
+      assert """
+      Compiling 2 files (.ex)
+      Generated sample app
+      lib/a.ex
+      lib/b.ex
+      """ = receive_until_no_messages([])
+    end
+  end
+
   defp assert_graph(opts \\ [], dot \\ false, expected) do
     in_fixture "no_mixfile", fn ->
       File.write! "lib/a.ex", """

lib/mix/test/mix/umbrella_test.exs

@@ -14,7 +14,7 @@ defmodule Mix.UmbrellaTest do
 
         # Ensure we can compile and run checks
         Mix.Task.run "deps.compile"
-        Mix.Task.run "deps.check"
+        Mix.Task.run "deps.loadpaths"
         Mix.Task.run "compile", ["--verbose"]
 
         assert_received {:mix_shell, :info, ["==> bar"]}
@@ -364,6 +364,53 @@ defmodule Mix.UmbrellaTest do
     end)
   end
 
+  test "reconsolidates using umbrella parent information on shared _build" do
+    in_fixture("umbrella_dep/deps/umbrella", fn ->
+      File.write!("apps/bar/lib/bar.ex", """
+      defprotocol Bar do
+        def bar(arg)
+      end
+      defimpl Bar, for: List do
+        def bar(list), do: list
+      end
+      """)
+
+      Mix.Project.in_project(:foo, "apps/foo", [build_path: "../../_build"], fn _ ->
+        Mix.Task.run("compile.protocols")
+        refute Code.ensure_loaded?(Bar)
+      end)
+
+      Mix.Project.in_project(:umbrella, ".", fn _ ->
+        Mix.Task.run("compile.protocols")
+        Mix.Task.run("app.start")
+        assert Protocol.consolidated?(Bar)
+      end)
+    end)
+  end
+
+  test "reconsolidates using umbrella child information on shared _build" do
+    in_fixture("umbrella_dep/deps/umbrella", fn ->
+      File.write!("apps/bar/lib/bar.ex", """
+      defprotocol Bar do
+        def foo(arg)
+      end
+      defimpl Bar, for: List do
+        def foo(list), do: list
+      end
+      """)
+
+      Mix.Project.in_project(:umbrella, ".", fn _ ->
+        Mix.Task.run("compile.protocols")
+      end)
+
+      # Emulate the dependency being removed
+      Mix.Project.in_project(:foo, "apps/foo", [build_path: "../../_build", deps: []], fn _ ->
+        File.rm_rf "../../_build/dev/lib/bar"
+        Mix.Task.run("compile.protocols")
+      end)
+    end)
+  end
+
   defmodule Selective do
     def project do
       [apps_path: "apps",

lib/mix/test/test_helper.exs

@@ -136,6 +136,15 @@ defmodule MixTest.Case do
                env: envs) |> elem(0)
   end
 
+  def mix_port(args, envs \\ []) when is_list(args) do
+    :erlang.open_port({:spawn_executable, elixir_executable()}, [
+      {:args, ["-r", mix_executable(), "--" | args]},
+      {:env, envs},
+      :use_stdio,
+      :stderr_to_stdout
+    ])
+  end
+
   defp mix_executable do
     Path.expand("../../../bin/mix", __DIR__)
   end
@@ -221,6 +230,30 @@ unless File.dir?(target) do
   end
   """
 
+  ## Sparse
+  subdir = Path.join(target, "sparse_dir")
+  File.mkdir_p!(Path.join(subdir, "lib"))
+
+  File.write! Path.join(subdir, "mix.exs"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo.Mixfile do
+    use Mix.Project
+
+    def project do
+      [app: :git_sparse_repo, version: "0.1.0"]
+    end
+  end
+  """
+
+  File.write! Path.join(subdir, "lib/git_sparse_repo.ex"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo do
+    def hello do
+      "World"
+    end
+  end
+  """
+
   File.cd! target, fn ->
     System.cmd("git", ~w[add .])
     System.cmd("git", ~w[commit -m "lib"])

src/elixir.app.src

@@ -1,6 +1,6 @@
 {application, elixir,
 [{description, "elixir"},
- {vsn, "1.3.1"},
+ {vsn, "1.3.4"},
  {modules, [
 	elixir
   ]},