Release v1.3.3

Solves https://github.com/phoenixframework/phoenix_live_reload/issues/37

CHANGELOG.md

@@ -268,6 +268,45 @@ end
 
 By restricting hierarchies in favor of named setups, it is straight-forward for the developer to glance at each describe block and know exactly the setup steps involved.
 
+## v1.3.3 (2016-09-17)
+
+### 1. Enhancements
+
+#### Elixir
+
+  * [DateTime] Support negative integer in `DateTime.from_unix/2`
+  * [Kernel.LexicalTracker] Do not consider remote typespecs as a compile-time dependency
+  * [Kernel.ParallelCompiler] Do not emit deadlock messages when the process is waiting on itself
+  * [Kernel.Typespec] Mark struct update syntax as generated to avoid false positives from dialyzer
+
+#### ExUnit
+
+  * [ExUnit] Make ExUnit server timeout configurable
+
+#### Logger
+
+  * [Logger] Use `:ansi_color` if one is available in metadata
+
+#### Mix
+
+  * [Mix] Add support for the `:sparse` option in `Mix.SCM.Git`
+  * [Mix] Skip dependendency loading if `MIX_NO_DEPS` is set to `1`
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [System] Use `NUL` instead of `/dev/null` on Windows when building `System.build_info`
+
+#### IEx
+
+  * [IEx.Autocomplete] Resolves issue with autocompletion on structs not working
+
+#### Mix
+
+  * [Mix] Also store external resources that are not part of the current working directory in compilation manifest
+  * [Mix] Always include the compiled file source in manifests
+
 ## v1.3.2 (2016-07-15)
 
 ### 1. Enhancements

VERSION

@@ -1 +1 @@
-1.3.2
+1.3.3

lib/elixir/lib/calendar.ex

@@ -1004,19 +1004,32 @@ defmodule DateTime do
       {:ok, %DateTime{calendar: Calendar.ISO, day: 23, hour: 22, microsecond: {211914, 3}, minute: 53,
                       month: 1, second: 43, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0,
                       year: 46302, zone_abbr: "UTC"}}
+  
+  Negative Unix times are supported, up to -#{@unix_epoch} seconds, 
+  which is equivalent to "0000-01-01T00:00:00Z" or 0 gregorian seconds.
 
+      iex> DateTime.from_unix(-12345678910)
+      {:ok, %DateTime{calendar: Calendar.ISO, day: 13, hour: 4, microsecond: {0, 0}, minute: 44, 
+                      month: 10, second: 50, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0, 
+                      year: 1578, zone_abbr: "UTC"}}
+  
+  When a Unix time before that moment is passed to `from_unix/2`, `:error` will be returned.
   """
-  @spec from_unix(non_neg_integer, :native | System.time_unit) :: {:ok, DateTime.t}
-  def from_unix(integer, unit \\ :seconds) when is_integer(integer) and integer >= 0 do
+  @spec from_unix(integer, :native | System.time_unit) :: {:ok, DateTime.t}
+  def from_unix(integer, unit \\ :seconds) when is_integer(integer) do
     total = System.convert_time_unit(integer, unit, :microseconds)
-    microsecond = rem(total, 1_000_000)
-    precision = precision_for_unit(unit)
-    {{year, month, day}, {hour, minute, second}} =
-      :calendar.gregorian_seconds_to_datetime(@unix_epoch + div(total, 1_000_000))
+    if total < -@unix_epoch * 1_000_000 do
+      :error  
+    else
+      microsecond = rem(total, 1_000_000)
+      precision = precision_for_unit(unit)
+      {{year, month, day}, {hour, minute, second}} =
+        :calendar.gregorian_seconds_to_datetime(@unix_epoch + div(total, 1_000_000))
 
-    {:ok, %DateTime{year: year, month: month, day: day,
-                    hour: hour, minute: minute, second: second, microsecond: {microsecond, precision},
-                    std_offset: 0, utc_offset: 0, zone_abbr: "UTC", time_zone: "Etc/UTC"}}
+      {:ok, %DateTime{year: year, month: month, day: day,
+                      hour: hour, minute: minute, second: second, microsecond: {microsecond, precision},
+                      std_offset: 0, utc_offset: 0, zone_abbr: "UTC", time_zone: "Etc/UTC"}}
+    end
   end
 
   def precision_for_unit(unit) do
@@ -1034,6 +1047,13 @@ defmodule DateTime do
   @doc """
   Converts the given Unix time to DateTime.
 
+  The integer can be given in different unit
+  according to `System.convert_time_unit/3` and it will
+  be converted to microseconds internally.
+
+  Unix times are always in UTC and therefore the DateTime
+  will be returned in UTC.
+
   ## Examples
 
       iex> DateTime.from_unix!(1464096368)
@@ -1046,6 +1066,15 @@ defmodule DateTime do
                 month: 5, second: 8, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0,
                 year: 2015, zone_abbr: "UTC"}
 
+  Negative Unix times are supported, up to -#{@unix_epoch} seconds, 
+  which is equivalent to "0000-01-01T00:00:00Z" or 0 gregorian seconds.
+
+      iex> DateTime.from_unix(-12345678910)
+      {:ok, %DateTime{calendar: Calendar.ISO, day: 13, hour: 4, microsecond: {0, 0}, minute: 44, 
+                      month: 10, second: 50, std_offset: 0, time_zone: "Etc/UTC", utc_offset: 0, 
+                      year: 1578, zone_abbr: "UTC"}}
+
+  When a Unix time before that moment is passed to `from_unix!/2`, an ArgumentError will be raised.
   """
   @spec from_unix!(non_neg_integer, :native | System.time_unit) :: DateTime.t
   def from_unix!(integer, unit \\ :seconds) when is_atom(unit) do
@@ -1057,7 +1086,7 @@ defmodule DateTime do
   Converts the given DateTime to Unix time.
 
   The DateTime is expected to be using the ISO calendar
-  with a year greater than or equal to 1970.
+  with a year greater than or equal to 0.
 
   It will return the integer with the given unit,
   according to `System.convert_time_unit/3`.
@@ -1073,11 +1102,19 @@ defmodule DateTime do
       iex> DateTime.to_unix(dt)
       1416517099
 
+      iex> flamel = %DateTime{calendar: Calendar.ISO, day: 22, hour: 8, microsecond: {527771, 6},
+      ...>                minute: 2, month: 3, second: 25, std_offset: 0, time_zone: "Etc/UTC",
+      ...>                utc_offset: 0, year: 1418, zone_abbr: "UTC"}
+      iex> DateTime.to_unix(flamel)
+      -17412508655
+
   """
   @spec to_unix(DateTime.t, System.time_unit) :: non_neg_integer
+  def to_unix(datetime, unit \\ :seconds)
+  
   def to_unix(%DateTime{calendar: Calendar.ISO, std_offset: std_offset, utc_offset: utc_offset,
                         hour: hour, minute: minute, second: second, microsecond: {microsecond, _},
-                        year: year, month: month, day: day}, unit \\ :seconds) when year >= 1970 do
+                        year: year, month: month, day: day}, unit) when year >= 0 do
     seconds =
       :calendar.datetime_to_gregorian_seconds({{year, month, day}, {hour, minute, second}})
       |> Kernel.-(utc_offset)

lib/elixir/lib/enum.ex

@@ -861,7 +861,7 @@ defmodule Enum do
 
   def find_index(enumerable, fun) do
     res =
-      Enumerable.reduce(enumerable, {:cont, {:not_found, 0}}, fn(entry, {status, index}) ->
+      Enumerable.reduce(enumerable, {:cont, {:not_found, 0}}, fn(entry, {_status, index}) ->
         if fun.(entry), do: {:halt, {:found, index}}, else: {:cont, {:not_found, index + 1}}
       end)
 

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -107,6 +107,7 @@ defmodule Kernel.ParallelCompiler do
       :erlang.spawn_monitor fn ->
         # Set the elixir_compiler_pid used by our custom Kernel.ErrorHandler.
         :erlang.put(:elixir_compiler_pid, parent)
+        :erlang.put(:elixir_compiler_file, file)
         :erlang.process_flag(:error_handler, Kernel.ErrorHandler)
 
         exit(try do
@@ -206,8 +207,10 @@ defmodule Kernel.ParallelCompiler do
 
       {:waiting, kind, child, ref, on, defining} ->
         # Oops, we already got it, do not put it on waiting.
+        # OR
+        # We're waiting on ourselves, send :found so that we can crash with a better error
         waiting =
-          if :lists.any(&match?({^kind, ^on}, &1), result) do
+          if :lists.any(&match?({^kind, ^on}, &1), result) or on in defining do
             send child, {ref, :found}
             waiting
           else

lib/elixir/lib/kernel/parallel_require.ex

@@ -47,21 +47,22 @@ defmodule Kernel.ParallelRequire do
     wait_for_messages(files, waiting, callbacks, schedulers, result)
   end
 
-  defp spawn_requires([h | t], waiting, callbacks, schedulers, result) do
+  defp spawn_requires([file | files], waiting, callbacks, schedulers, result) do
     parent = self()
     {pid, ref} = :erlang.spawn_monitor fn ->
       :erlang.put(:elixir_compiler_pid, parent)
+      :erlang.put(:elixir_compiler_file, file)
 
       exit(try do
-        new = Code.require_file(h) || []
-        {:required, Enum.map(new, &elem(&1, 0)), h}
+        new = Code.require_file(file) || []
+        {:required, Enum.map(new, &elem(&1, 0)), file}
       catch
         kind, reason ->
           {:failure, kind, reason, System.stacktrace}
       end)
     end
 
-    spawn_requires(t, [{pid, ref} | waiting], callbacks, schedulers, result)
+    spawn_requires(files, [{pid, ref} | waiting], callbacks, schedulers, result)
   end
 
   defp wait_for_messages(files, waiting, callbacks, schedulers, result) do

lib/elixir/lib/kernel/typespec.ex

@@ -777,6 +777,8 @@ defmodule Kernel.Typespec do
   end
 
   defp typespec({:%, _, [name, {:%{}, meta, fields}]}, vars, caller) do
+    # We cannot set a function name to avoid tracking
+    # as a compile time dependency, because for structs it actually is one.
     module = Macro.expand(name, caller)
 
     struct =
@@ -813,6 +815,8 @@ defmodule Kernel.Typespec do
   end
 
   defp typespec({:record, meta, [atom, fields]}, vars, caller) do
+    # We cannot set a function name to avoid tracking
+    # as a compile time dependency because for records it actually is one.
     case Macro.expand({atom, [], [{atom, [], []}]}, caller) do
       keyword when is_list(keyword) ->
         types =
@@ -883,7 +887,9 @@ defmodule Kernel.Typespec do
 
   # Handle remote calls
   defp typespec({{:., meta, [remote, name]}, _, args} = orig, vars, caller) do
-    remote = Macro.expand remote, caller
+    # We set a function name to avoid tracking
+    # aliases in typespecs as compile time dependencies.
+    remote = Macro.expand remote, %{caller | function: {:typespec, 0}}
     unless is_atom(remote) do
       compile_error(caller, "invalid remote in typespec: #{Macro.to_string(orig)}")
     end

lib/elixir/lib/string.ex

@@ -350,6 +350,7 @@ defmodule String do
   defp parts_to_index(:infinity),                      do: 0
   defp parts_to_index(n) when is_integer(n) and n > 0, do: n
 
+  defp split_each("", _pattern, true, 1), do: []
   defp split_each(string, _pattern, _trim, 1) when is_binary(string), do: [string]
   defp split_each(string, pattern, trim, count) do
     case do_splitter(string, pattern, trim) do
@@ -402,6 +403,7 @@ defmodule String do
   end
 
   defp maybe_compile_pattern(""), do: ""
+  defp maybe_compile_pattern(pattern) when is_tuple(pattern), do: pattern
   defp maybe_compile_pattern(pattern), do: :binary.compile_pattern(pattern)
 
   @doc """

lib/elixir/lib/system.ex

@@ -112,7 +112,15 @@ defmodule System do
   # Tries to run "git rev-parse --short HEAD". In the case of success returns
   # the short revision hash. If that fails, returns an empty string.
   defmacrop get_revision do
-    :os.cmd('git rev-parse --short HEAD 2> /dev/null')
+    null =
+      case :os.type do
+        {:win32, _} -> 'NUL'
+        _ -> '/dev/null'
+      end
+
+    'git rev-parse --short HEAD 2> '
+    |> Kernel.++(null)
+    |> :os.cmd()
     |> strip
   end
 

lib/elixir/src/elixir_map.erl

@@ -92,9 +92,9 @@ translate_struct(Meta, Name, {'%{}', MapMeta, Args}, S) ->
 
       {TMap, TS} = translate_map(MapMeta, Assocs, Var, VS),
 
-      {{'case', Ann, TUpdate, [
+      {{'case', ?generated, TUpdate, [
         {clause, Ann, [Match], [], [TMap]},
-        {clause, Ann, [Var], [], [elixir_utils:erl_call(Ann, erlang, error, [Error])]}
+        {clause, ?generated, [Var], [], [elixir_utils:erl_call(Ann, erlang, error, [Error])]}
       ]}, TS};
     match ->
       translate_map(MapMeta, Assocs ++ [{'__struct__', Name}], nil, US);

lib/elixir/src/elixir_module.erl

@@ -378,7 +378,7 @@ load_form(Line, Data, Forms, Opts, E) ->
             ok;
           PID ->
             Ref = make_ref(),
-            PID ! {module_available, self(), Ref, ?m(E, file), Module, Binary},
+            PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary},
             receive {Ref, ack} -> ok end
         end;
       _ ->

lib/elixir/test/elixir/calendar_test.exs

@@ -66,4 +66,25 @@ defmodule DateTimeTest do
                    utc_offset: -12600, std_offset: 3600, time_zone: "Brazil/Manaus"}
     assert to_string(dt) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus"
   end
+
+  test "from_unix/2 works with Unix times back to 0 Gregorian Seconds" do
+    assert DateTime.from_unix(-62167219200) == {:ok,
+                    %DateTime{calendar: Calendar.ISO, day: 1, hour: 0, microsecond: {0, 0},
+                              minute: 0, month: 1, second: 0, std_offset: 0, time_zone: "Etc/UTC",
+                              utc_offset: 0, year: 0, zone_abbr: "UTC"}}
+
+    assert DateTime.from_unix(-62167219201) == :error
+  end
+
+  test "to_unix/2 works with Unix times back to 0 Gregorian Seconds" do
+    gregorian_0 = %DateTime{calendar: Calendar.ISO, day: 1, hour: 0, microsecond: {0, 0},
+                            minute: 0, month: 1, second: 0, std_offset: 0, time_zone: "Etc/UTC",
+                            utc_offset: 0, year: 0, zone_abbr: "UTC"}
+    before_gregorian_0 = %DateTime{gregorian_0 | year: -1}
+
+    assert DateTime.to_unix(gregorian_0) == -62167219200
+    assert_raise FunctionClauseError, fn ->
+      DateTime.to_unix(before_gregorian_0)
+    end
+  end
 end

lib/elixir/test/elixir/fixtures/dialyzer/struct_update.ex

@@ -0,0 +1,7 @@
+defmodule Dialyzer.StructUpdate do
+  defstruct [:foo]
+
+  def update(%__MODULE__{} = struct) do
+    %__MODULE__{struct | foo: :bar}
+  end
+end

lib/elixir/test/elixir/fixtures/parallel_struct/undef.ex

@@ -0,0 +1,5 @@
+defmodule Undef do
+  def undef do
+    %__MODULE__{}
+  end
+end

lib/elixir/test/elixir/kernel/dialyzer_test.exs

@@ -76,6 +76,11 @@ defmodule Kernel.DialyzerTest do
     assert_dialyze_no_warnings! context
   end
 
+  test "no warnings on struct update", context do
+    copy_beam! context, Dialyzer.StructUpdate
+    assert_dialyze_no_warnings! context
+  end
+
   defp copy_beam!(context, module) do
     name = "#{module}.beam"
     File.cp! Path.join(context[:base_dir], name),

lib/elixir/test/elixir/kernel/parallel_compiler_test.exs

@@ -28,6 +28,13 @@ defmodule Kernel.ParallelCompilerTest do
     end
   end
 
+  test "emits struct undefined error when local struct is undefined" do
+    fixtures = [fixture_path("parallel_struct/undef.ex")]
+    assert capture_io(fn ->
+      assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == {:shutdown, 1}
+    end) =~ "Undef.__struct__/1 is undefined, cannot expand struct Undef"
+  end
+
   test "does not hang on missing dependencies" do
     fixtures = [fixture_path("parallel_compiler/bat.ex")]
     assert capture_io(fn ->

lib/elixir/test/elixir/string_test.exs

@@ -66,6 +66,9 @@ defmodule StringTest do
     assert String.split("abé", "", trim: true, parts: 2) == ["a", "bé"]
 
     assert String.split("noël", "") == ["n", "o", "ë", "l", ""]
+
+    assert String.split("x-", "-", parts: 2, trim: true) == ["x"]
+    assert String.split("x-x-", "-", parts: 3, trim: true) == ["x", "x"]
   end
 
   test "split with regex" do
@@ -79,6 +82,14 @@ defmodule StringTest do
     assert String.split("a,b", ~r{\.}) == ["a,b"]
   end
 
+  test "split with compiled pattern" do
+    pattern = :binary.compile_pattern("-")
+
+    assert String.split("x-", pattern) == ["x", ""]
+    assert String.split("x-", pattern, parts: 2, trim: true) == ["x"]
+    assert String.split("x-x-", pattern, parts: 3, trim: true) == ["x", "x"]
+  end
+
   test "splitter" do
     assert String.splitter("a,b,c", ",") |> Enum.to_list == ["a", "b", "c"]
     assert String.splitter("a,b", ".") |> Enum.to_list == ["a,b"]

lib/ex_unit/lib/ex_unit.ex

@@ -181,6 +181,9 @@ defmodule ExUnit do
       and print them on test failure. Can be overridden for individual tests via
       `@tag capture_log: false`. Defaults to `false`.
 
+    * `:case_load_timeout` - the timeout to be used when loading a test case.
+      Defaults to `60_000` milliseconds.
+
     * `:colors` - a keyword list of colors to be used by some formatters.
       The only option so far is `[enabled: boolean]` which defaults to `IO.ANSI.enabled?/0`
 

lib/ex_unit/lib/ex_unit/server.ex

@@ -1,6 +1,5 @@
 defmodule ExUnit.Server do
   @moduledoc false
-  @timeout 60_000
 
   use GenServer
 
@@ -21,11 +20,13 @@ defmodule ExUnit.Server do
   end
 
   def take_async_cases(count) do
-    GenServer.call(__MODULE__, {:take_async_cases, count}, @timeout)
+    timeout = Application.fetch_env!(:ex_unit, :case_load_timeout)
+    GenServer.call(__MODULE__, {:take_async_cases, count}, timeout)
   end
 
   def take_sync_cases() do
-    GenServer.call(__MODULE__, :take_sync_cases, @timeout)
+    timeout = Application.fetch_env!(:ex_unit, :case_load_timeout)
+    GenServer.call(__MODULE__, :take_sync_cases, timeout)
   end
 
   ## Callbacks

lib/ex_unit/mix.exs

@@ -18,6 +18,7 @@ defmodule ExUnit.Mixfile do
        assert_receive_timeout: 100,
        autorun: true,
        capture_log: false,
+       case_load_timeout: 60_000,
        colors: [],
        exclude: [],
        include: [],

lib/iex/lib/iex/autocomplete.ex

@@ -59,9 +59,12 @@ defmodule IEx.Autocomplete do
   defp reduce(expr) do
     Enum.reduce(' ([{', expr, fn token, acc ->
       hd(:string.tokens(acc, [token]))
-    end) |> Enum.reverse |> strip_ampersand
+    end) |> Enum.reverse |> strip_ampersand |> strip_percent
   end
 
+  defp strip_percent([?% | t]), do: t
+  defp strip_percent(expr), do: expr
+
   defp strip_ampersand([?& | t]), do: t
   defp strip_ampersand(expr), do: expr
 

lib/iex/test/iex/autocomplete_test.exs

@@ -179,4 +179,13 @@ defmodule IEx.AutocompleteTest do
     :code.purge(Sample)
     :code.delete(Sample)
   end
+
+
+  defmodule MyStruct do
+    defstruct my_val: "val"
+  end
+
+  test "completion for structs" do
+    assert expand('%IEx.AutocompleteTest.MyStr') == {:yes, 'uct.', []}
+  end
 end

lib/logger/lib/logger/backends/console.ex

@@ -86,7 +86,7 @@ defmodule Logger.Backends.Console do
     colors     = configure_colors(config)
     max_buffer = Keyword.get(config, :max_buffer, 32)
     %{state | format: format, metadata: Enum.reverse(metadata),
-      level: level, colors: colors, device: device, max_buffer: max_buffer}
+              level: level, colors: colors, device: device, max_buffer: max_buffer}
   end
 
   defp configure_merge(env, options) do
@@ -149,7 +149,7 @@ defmodule Logger.Backends.Console do
     %{format: format, metadata: keys, colors: colors} = state
     format
     |> Logger.Formatter.format(level, msg, ts, take_metadata(md, keys))
-    |> color_event(level, colors)
+    |> color_event(level, colors, md)
   end
 
   defp take_metadata(metadata, keys) do
@@ -161,10 +161,11 @@ defmodule Logger.Backends.Console do
     end
   end
 
-  defp color_event(data, _level, %{enabled: false}), do: data
+  defp color_event(data, _level, %{enabled: false}, _md), do: data
 
-  defp color_event(data, level, %{enabled: true} = colors) do
-    [IO.ANSI.format_fragment(Map.fetch!(colors, level), true), data | IO.ANSI.reset]
+  defp color_event(data, level, %{enabled: true} = colors, md) do
+    color = md[:ansi_color] || Map.fetch!(colors, level)
+    [IO.ANSI.format_fragment(color, true), data | IO.ANSI.reset]
   end
 
   defp log_buffer(%{buffer_size: 0, buffer: []} = state), do: state

lib/logger/test/logger/backends/console_test.exs

@@ -109,4 +109,12 @@ defmodule Logger.Backends.ConsoleTest do
       Logger.error("hello")
     end) == IO.ANSI.cyan() <> "hello" <> IO.ANSI.reset()
   end
+
+  test "can use colors from metadata" do
+    Logger.configure_backend(:console, [format: "$message", colors: [enabled: true]])
+
+    assert capture_log(fn ->
+      Logger.log(:error, "hello", ansi_color: :yellow)
+    end) == IO.ANSI.yellow() <> "hello" <> IO.ANSI.reset()
+  end
 end

lib/mix/lib/mix/compilers/elixir.ex

@@ -246,10 +246,7 @@ defmodule Mix.Compilers.Elixir do
 
   defp get_external_resources(module, cwd) do
     for file <- Module.get_attribute(module, :external_resource),
-        File.regular?(file),
-        relative = Path.relative_to(file, cwd),
-        Path.type(relative) == :relative,
-        do: relative
+        do: Path.relative_to(file, cwd)
   end
 
   defp each_file(source) do

lib/mix/lib/mix/compilers/test.ex

@@ -264,9 +264,6 @@ defmodule Mix.Compilers.Test do
 
   defp get_external_resources(module, cwd) do
     for file <- Module.get_attribute(module, :external_resource),
-        File.regular?(file),
-        relative = Path.relative_to(file, cwd),
-        Path.type(relative) == :relative,
-        do: relative
+        do: Path.relative_to(file, cwd)
   end
 end

lib/mix/lib/mix/dep.ex

@@ -71,12 +71,15 @@ defmodule Mix.Dep do
   against the lock.
   """
   def cached do
-    if project = Mix.Project.get do
-      key = {:cached_deps, project}
-      Mix.ProjectStack.read_cache(key) ||
-        Mix.ProjectStack.write_cache(key, loaded(env: Mix.env))
-    else
-      loaded(env: Mix.env)
+    cond do
+      System.get_env("MIX_NO_DEPS") in ~w(1 true) ->
+        []
+      project = Mix.Project.get ->
+        key = {:cached_deps, project}
+        Mix.ProjectStack.read_cache(key) ||
+          Mix.ProjectStack.write_cache(key, loaded(env: Mix.env))
+      true ->
+        loaded(env: Mix.env)
     end
   end
 
@@ -264,7 +267,7 @@ defmodule Mix.Dep do
     do: "the dependency was built with another SCM, run \"#{mix_env_var}mix deps.compile\""
 
   defp dep_status(%Mix.Dep{app: app, requirement: req, manager: manager, opts: opts, from: from}) do
-    opts = Keyword.drop(opts, [:dest, :build, :lock, :manager])
+    opts = Keyword.drop(opts, [:dest, :build, :lock, :manager, :checkout])
     opts = opts ++ (if manager, do: [manager: manager], else: [])
     info = if req, do: {app, req, opts}, else: {app, opts}
     "\n  > In #{Path.relative_to_cwd(from)}:\n    #{inspect info}\n"

lib/mix/lib/mix/scm/git.ex

@@ -25,6 +25,11 @@ defmodule Mix.SCM.Git do
   end
 
   def accepts_options(_app, opts) do
+    opts =
+      opts
+      |> Keyword.put(:checkout, opts[:dest])
+      |> sparse_opts()
+
     cond do
       gh = opts[:github] ->
         opts
@@ -41,7 +46,9 @@ defmodule Mix.SCM.Git do
 
   def checked_out?(opts) do
     # Are we inside a Git repository?
-    File.regular?(Path.join(opts[:dest], ".git/HEAD"))
+    opts[:checkout]
+    |> Path.join(".git/HEAD")
+    |> File.regular?
   end
 
   def lock_status(opts) do
@@ -50,7 +57,7 @@ defmodule Mix.SCM.Git do
 
     cond do
       lock_rev = get_lock_rev(lock, opts) ->
-        File.cd!(opts[:dest], fn ->
+        File.cd!(opts[:checkout], fn ->
           %{origin: origin, rev: rev} = get_rev_info()
           if get_lock_repo(lock) == origin and lock_rev == rev do
             :ok
@@ -76,29 +83,75 @@ defmodule Mix.SCM.Git do
 
   def checkout(opts) do
     assert_git!()
-
-    path     = opts[:dest]
-    location = opts[:git]
-
-    _ = File.rm_rf!(path)
-    git!(~s(clone --no-checkout --progress "#{location}" "#{path}"))
-
-    File.cd! path, fn -> do_checkout(opts) end
+    path = opts[:checkout]
+    File.rm_rf!(path)
+    File.mkdir_p!(path)
+    File.cd!(path, fn ->
+      git!("init --quiet")
+      git!("--git-dir=.git remote add origin \"#{opts[:git]}\"")
+      checkout(path, opts)
+    end)
   end
 
   def update(opts) do
     assert_git!()
+    path = opts[:checkout]
+    File.cd! path, fn -> checkout(path, opts) end
+  end
 
-    File.cd! opts[:dest], fn ->
-      location = opts[:git]
-      update_origin(location)
+  defp checkout(_path, opts) do
+    # Set configuration
+    sparse_toggle(opts)
+    update_origin(opts[:git])
 
-      command = IO.iodata_to_binary(["--git-dir=.git fetch --force",
-                                     progress_switch(git_version()),
-                                     tags_switch(opts[:tag])])
+    # Fetch external data
+    command = IO.iodata_to_binary(["--git-dir=.git fetch --force --quiet",
+                                   progress_switch(git_version()),
+                                   tags_switch(opts[:tag])])
+    git!(command)
 
-      git!(command)
-      do_checkout(opts)
+    # Migrate the git repo
+    rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts)
+    git!("--git-dir=.git checkout --quiet #{rev}")
+
+    if opts[:submodules] do
+      git!("--git-dir=.git submodule update --init --recursive")
+    end
+
+    # Get the new repo lock
+    get_lock(opts)
+  end
+
+  defp sparse_opts(opts) do
+    if opts[:sparse] do
+      dest = Path.join(opts[:dest], opts[:sparse])
+      Keyword.put(opts, :dest, dest)
+    else
+      opts
+    end
+  end
+
+  defp sparse_toggle(opts) do
+    cond do
+      sparse = opts[:sparse] ->
+        sparse_check(git_version())
+        git!("--git-dir=.git config core.sparsecheckout true")
+        File.write!(".git/info/sparse-checkout", sparse)
+      File.exists?(".git/info/sparse-checkout") ->
+        File.write!(".git/info/sparse-checkout", "*")
+        git!("--git-dir=.git read-tree -mu HEAD")
+        git!("--git-dir=.git config core.sparsecheckout false")
+        File.rm(".git/info/sparse-checkout")
+      true ->
+        :ok
+    end
+  end
+
+  defp sparse_check(version) do
+    unless {1, 7, 0} <= version do
+      version = version |> Tuple.to_list |> Enum.join(".")
+      Mix.raise "Git >= 1.7.0 is required to use sparse checkout. " <>
+                "You are running version #{version}"
     end
   end
 
@@ -111,24 +164,17 @@ defmodule Mix.SCM.Git do
   ## Helpers
 
   defp validate_git_options(opts) do
-    case Keyword.take(opts, [:branch, :ref, :tag]) do
-      []  -> opts
-      [_] -> opts
-      _   ->
-        Mix.raise "You should specify only one of branch, ref or tag, and only once. " <>
-                  "Error on Git dependency: #{opts[:git]}"
-    end
+    err = "You should specify only one of branch, ref or tag, and only once. " <>
+          "Error on Git dependency: #{opts[:git]}"
+    validate_single_uniq(opts, [:branch, :ref, :tag], err)
   end
 
-  defp do_checkout(opts) do
-    rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts)
-    git!("--git-dir=.git checkout --quiet #{rev}")
-
-    if opts[:submodules] do
-      git!("--git-dir=.git submodule update --init --recursive")
+  defp validate_single_uniq(opts, take, error) do
+    case Keyword.take(opts, take) do
+      []  -> opts
+      [_] -> opts
+      _   -> Mix.raise error
     end
-
-    get_lock(opts)
   end
 
   defp get_lock(opts) do
@@ -146,7 +192,8 @@ defmodule Mix.SCM.Git do
   defp get_lock_rev(_, _), do: nil
 
   defp get_lock_opts(opts) do
-    lock_opts = Keyword.take(opts, [:branch, :ref, :tag])
+    lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse])
+
     if opts[:submodules] do
       lock_opts ++ [submodules: true]
     else

lib/mix/lib/mix/tasks/app.start.ex

@@ -20,7 +20,7 @@ defmodule Mix.Tasks.App.Start do
       applications are started in permanent mode
 
     * `:consolidate_protocols` - when `true`, loads consolidated
-      protocols before start
+      protocols before start. The default value is `true`.
 
     * `:elixir` - matches the current elixir version against the
       given requirement

lib/mix/lib/mix/tasks/compile.ex

@@ -14,7 +14,8 @@ defmodule Mix.Tasks.Compile do
       `[:yecc, :leex, :erlang, :elixir, :xref, :app]`
 
     * `:consolidate_protocols` - when `true`, runs protocol
-      consolidation via the `compile.protocols` task
+      consolidation via the `compile.protocols` task. The default
+      value is `true`.
 
     * `:build_embedded` - when `true`, activates protocol
       consolidation and does not generate symlinks in builds

lib/mix/lib/mix/tasks/compile.protocols.ex

@@ -9,7 +9,8 @@ defmodule Mix.Tasks.Compile.Protocols do
 
   This task is automatically invoked whenever the project
   enables `:consolidate_protocols` or `:build_embedded` in
-  its configuration.
+  its configuration. As `:consolidate_protocols` is enabled
+  in the default configuration, this is the default case.
 
   ## Consolidation
 

lib/mix/lib/mix/tasks/deps.loadpaths.ex

@@ -33,7 +33,9 @@ defmodule Mix.Tasks.Deps.Loadpaths do
         path
       end
 
-    prune_deps(load_paths, "--no-deps-check" in args)
+    unless System.get_env("MIX_NO_DEPS") in ~w(1 true) do
+      prune_deps(load_paths, "--no-deps-check" in args)
+    end
   end
 
   # If the build is per environment, we should be able to look

lib/mix/lib/mix/tasks/loadpaths.ex

@@ -32,7 +32,7 @@ defmodule Mix.Tasks.Loadpaths do
 
     # --no-deps is used only internally. It has no purpose
     # from Mix.CLI because running a task may load deps.
-    unless "--no-deps" in args or System.get_env("MIX_NO_DEPS") in ~w(1 true) do
+    unless "--no-deps" in args do
       Mix.Task.run "deps.loadpaths", args
     end
 

lib/mix/lib/mix/tasks/xref.ex

@@ -13,28 +13,28 @@ defmodule Mix.Tasks.Xref do
   ## Xref modes
 
   The `xref` task expects a mode as first argument:
-  
+
       mix xref MODE
-  
+
   All available modes are discussed below.
-  
+
   ### warnings
 
   Prints warnings for violated cross reference checks:
-  
+
       mix xref warnings
-  
+
   This is the mode used by Mix during compilation.
 
   ### unreachable
 
   Prints all unreachable "file:line: module.function/arity" entries:
-  
+
       mix xref unreachable
-  
+
   The "file:line" represents the file and line a call to an unknown
   "module.function/arity" is made.
-  
+
   ### callers CALLEE
 
   Prints all callers of the given `CALLEE`, which can be one of: `Module`,
@@ -48,7 +48,7 @@ defmodule Mix.Tasks.Xref do
 
   Prints a file dependency graph where an edge from `A` to `B` indicates
   that `A` depends on `B`.
- 
+
       mix xref graph --format dot
 
   The following options are accepted:
@@ -106,6 +106,8 @@ defmodule Mix.Tasks.Xref do
     {opts, args} =
       OptionParser.parse!(args, strict: @switches)
 
+    Mix.Task.run("loadpaths")
+
     if Keyword.get(opts, :compile, true) do
       Mix.Task.run("compile")
     end

lib/mix/test/fixtures/.gitignore

@@ -1,3 +1,4 @@
 git_repo
+git_sparse_repo
 deps_on_git_repo
 git_rebar

lib/mix/test/fixtures/rebar_dep/rebar.config

@@ -1,7 +1,3 @@
 {sub_dirs, ["apps/*"]}.
 
-{deps, [
-    {git_rebar, "0.1..*", {git, "../../test/fixtures/git_rebar", master}}
-]}.
-
 {overrides, [{add, rebar_dep, [{src_dirs, ["apps/rebar_dep/src"]}]}]}.

lib/mix/test/fixtures/rebar_dep/rebar.config.script

@@ -1 +1 @@
-CONFIG ++ [{'SCRIPT', SCRIPT}].
+CONFIG ++ [{'SCRIPT', SCRIPT}] ++ [{deps, [{git_rebar, "0.1..*", {git, filename:absname("../../test/fixtures/git_rebar"), master}}]}].

lib/mix/test/fixtures/rebar_override/rebar.config

@@ -1,7 +0,0 @@
-{deps, [
-    {git_rebar, {git, "../../test/fixtures/git_rebar"}}
-]}.
-
-{overrides, [
-  {override, git_rebar, [{deps, [{git_repo, {git, "../../test/fixtures/git_repo"}}]}]}
-]}.

lib/mix/test/fixtures/rebar_override/rebar.config.script

@@ -0,0 +1,8 @@
+[
+  {deps, [
+    {git_rebar, {git, filename:absname("../../test/fixtures/git_rebar")}}
+  ]},
+  {overrides, [
+    {override, git_rebar, [{deps, [{git_repo, {git, filename:absname("../../test/fixtures/git_repo")}}]}]}
+  ]}
+].

lib/mix/test/mix/rebar_test.exs

@@ -106,7 +106,7 @@ defmodule Mix.RebarTest do
 
     assert Enum.find(deps, fn %Mix.Dep{app: app, opts: opts} ->
       if app == :git_rebar do
-        assert Enum.find(opts, &match?({:git, "../../test/fixtures/git_rebar"}, &1))
+        assert Enum.find(opts, &match?({:git, _}, &1))
         assert Enum.find(opts, &match?({:ref, "master"}, &1))
         true
       end
@@ -155,7 +155,7 @@ defmodule Mix.RebarTest do
 
     in_tmp "get and compile dependencies for Rebar", fn ->
       Mix.Tasks.Deps.Get.run []
-      assert_received {:mix_shell, :info, ["* Getting git_rebar (../../test/fixtures/git_rebar)"]}
+      assert_received {:mix_shell, :info, ["* Getting git_rebar" <> _]}
 
       Mix.Tasks.Deps.Compile.run []
       assert_received {:mix_shell, :run, ["==> git_rebar (compile)\n"]}
@@ -185,7 +185,7 @@ defmodule Mix.RebarTest do
 
     in_tmp "get and compile dependencies for rebar3", fn ->
       Mix.Tasks.Deps.Get.run []
-      assert_received {:mix_shell, :info, ["* Getting git_rebar (../../test/fixtures/git_rebar)"]}
+      assert_received {:mix_shell, :info, ["* Getting git_rebar " <> _]}
 
       Mix.Tasks.Deps.Compile.run []
       assert_received {:mix_shell, :run, ["===> Compiling git_rebar\n"]}

lib/mix/test/mix/tasks/compile.elixir_test.exs

@@ -172,25 +172,43 @@ defmodule Mix.Tasks.Compile.ElixirTest do
 
   test "compiles dependent changed files" do
     in_fixture "no_mixfile", fn ->
+      tmp = tmp_path("c.eex")
       File.touch!("lib/a.eex")
+
       File.write!("lib/a.ex", """
       defmodule A do
-        @external_resource "lib/b.eex"
         @external_resource "lib/a.eex"
+        @external_resource #{inspect tmp}
         def a, do: :ok
       end
       """)
 
+      # Compiles with missing external resources
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :noop
       Mix.shell.flush
       purge [A, B]
 
+      # Update local existing resource
       File.touch!("lib/a.eex", {{2020, 1, 1}, {0, 0, 0}})
       assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
 
+      # Does not update on old existing resource
+      File.touch!("lib/a.eex", {{1970, 1, 1}, {0, 0, 0}})
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :noop
+      Mix.shell.flush
+      purge [A, B]
+
+      # Update external existing resource
+      File.touch!(tmp, {{2020, 1, 1}, {0, 0, 0}})
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
       refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
     end
+  after
+    File.rm tmp_path("c.eex")
   end
 
   test "does not recompile empty files" do
@@ -231,4 +249,28 @@ defmodule Mix.Tasks.Compile.ElixirTest do
       assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
     end
   end
+
+  test "does not treat remote typespecs as compile time dependencies" do
+    in_fixture "no_mixfile", fn ->
+      File.write!("lib/b.ex", """
+      defmodule B do
+        @type t :: A.t
+      end
+      """)
+
+      assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == :ok
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      assert_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
+
+      Mix.shell.flush
+      purge [A, B]
+
+      future = {{2020, 1, 1}, {0, 0, 0}}
+      File.touch!("lib/a.ex", future)
+      Mix.Tasks.Compile.Elixir.run ["--verbose"]
+
+      assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]}
+      refute_received {:mix_shell, :info, ["Compiled lib/b.ex"]}
+    end
+  end
 end

lib/mix/test/mix/tasks/deps.git_test.exs

@@ -69,6 +69,20 @@ defmodule Mix.Tasks.DepsGitTest do
     end
   end
 
+  test "gets and updates Git repos with sparse checkout" do
+    Process.put(:git_repo_opts, sparse: "sparse_dir")
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      message = "* Getting git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+      refute File.exists?("deps/git_repo/mix.exs")
+      assert File.exists?("deps/git_repo/sparse_dir/mix.exs")
+      assert File.read!("mix.lock") =~ "sparse: \"sparse_dir\""
+    end
+  end
+
   test "handles invalid .git directory" do
     Mix.Project.push GitApp
 
@@ -265,6 +279,61 @@ defmodule Mix.Tasks.DepsGitTest do
     purge [GitRepo, GitRepo.Mixfile]
   end
 
+  test "updates the repo when sparse is turned off" do
+    Process.put(:git_repo_opts, sparse: "sparse_dir")
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      refute File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      # Flush the errors we got, move to a clean slate
+      Mix.shell.flush
+      Mix.Task.clear
+      Process.delete(:git_repo_opts)
+      Mix.Project.pop
+      Mix.Project.push GitApp
+
+      # Calling get should update the dependency
+      Mix.Tasks.Deps.Get.run []
+      refute File.read!("mix.lock") =~ "sparse_dir"
+      assert File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      message = "* Updating git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+
+      # Check we got no error
+      refute_received {:mix_shell, :error, _}
+    end
+  end
+
+  test "updates the repo when sparse is turned on" do
+    Mix.Project.push GitApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      assert File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      # Flush the errors we got, move to a clean slate
+      Mix.shell.flush
+      Mix.Task.clear
+      Process.put(:git_repo_opts, sparse: "sparse_dir")
+      Mix.Project.pop
+      Mix.Project.push GitApp
+
+      # Calling get should update the dependency
+      Mix.Tasks.Deps.Get.run []
+      assert File.read!("mix.lock") =~ "sparse_dir"
+      refute File.exists?("deps/git_repo/lib/git_repo.ex")
+
+      message = "* Updating git_repo (#{fixture_path("git_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+
+      # Check we got no error
+      refute_received {:mix_shell, :error, _}
+    end
+  end
+
   test "updates the repo and the lock when the mixfile updates" do
     Mix.Project.push GitApp
     [last, first | _] = get_git_repo_revs
@@ -304,7 +373,7 @@ defmodule Mix.Tasks.DepsGitTest do
       exception = assert_raise Mix.Error, fn ->
         Mix.Tasks.Deps.Get.run []
       end
-      assert Exception.message(exception) =~ "Command \"git clone"
+      assert Exception.message(exception) =~ "Command \"git --git-dir=.git fetch"
     end
   end
 
@@ -364,7 +433,7 @@ defmodule Mix.Tasks.DepsGitTest do
     Mix.Project.push(name, file)
   end
 
-  defp get_git_repo_revs do
+  defp get_git_repo_revs() do
     File.cd! fixture_path("git_repo"), fn ->
       Regex.split ~r(\r?\n), System.cmd("git", ["log", "--format=%H"]) |> elem(0)
     end

lib/mix/test/mix/tasks/xref_test.exs

@@ -564,6 +564,32 @@ defmodule Mix.Tasks.XrefTest do
     end
   end
 
+  test "graph: with dynamic module" do
+    in_fixture "no_mixfile", fn ->
+      File.write! "lib/a.ex", """
+      B.define()
+      """
+
+      File.write! "lib/b.ex", """
+      defmodule B do
+        def define do
+          defmodule A do
+          end
+        end
+      end
+      """
+
+      assert Mix.Task.run("xref", ["graph"]) == :ok
+
+      assert """
+      Compiling 2 files (.ex)
+      Generated sample app
+      lib/a.ex
+      lib/b.ex
+      """ = receive_until_no_messages([])
+    end
+  end
+
   defp assert_graph(opts \\ [], dot \\ false, expected) do
     in_fixture "no_mixfile", fn ->
       File.write! "lib/a.ex", """

lib/mix/test/test_helper.exs

@@ -230,6 +230,30 @@ unless File.dir?(target) do
   end
   """
 
+  ## Sparse
+  subdir = Path.join(target, "sparse_dir")
+  File.mkdir_p!(Path.join(subdir, "lib"))
+
+  File.write! Path.join(subdir, "mix.exs"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo.Mixfile do
+    use Mix.Project
+
+    def project do
+      [app: :git_sparse_repo, version: "0.1.0"]
+    end
+  end
+  """
+
+  File.write! Path.join(subdir, "lib/git_sparse_repo.ex"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo do
+    def hello do
+      "World"
+    end
+  end
+  """
+
   File.cd! target, fn ->
     System.cmd("git", ~w[add .])
     System.cmd("git", ~w[commit -m "lib"])

src/elixir.app.src

@@ -1,6 +1,6 @@
 {application, elixir,
 [{description, "elixir"},
- {vsn, "1.3.2"},
+ {vsn, "1.3.3"},
  {modules, [
 	elixir
   ]},