Fix Path.absname/1 to correctly handle UNC paths on Windows (#9689)

Now excludes files that does not exist on the file system. This could
happen e.g. if the release is built with `include_erts: false`.

Otherwise building the tar would fail with `:enoent`.

CHANGELOG.md

@@ -58,6 +58,91 @@ Another important change related to configuration is that `mix new` will no long
 
 There are many other enhancements. The Elixir CLI got a handful of new options in order to best support releases. `Logger` now computes its sync/async/discard thresholds in a decentralized fashion, reducing contention. `EEx` templates support more complex expressions than before. Finally, there is a new `~U` sigil for working with UTC DateTimes as well as new functions in the `File`, `Registry`, and `System` modules.
 
+## v1.9.4 (2019-11-05)
+
+### 1. Bug fixes
+
+#### Mix
+
+  * [mix local.hex] Remove invalid deprecation warning on `mix local.hex` command
+
+## v1.9.3 (2019-11-05)
+
+Note this release deprecates the use of URLs on `mix archive.install`, `mix escript.install`, and `mix local.rebar`. Support for passing URLs to said commands will be fully removed on Elixir v1.10, as they are unsafe. Thanks to Bram Verburg for the report and for providing a fix.
+
+The alternative is straight-forward: you can simply download the artifact via the command line and then invoke the command with a file system path. For example, instead of:
+
+    $ mix archive.install https://example.org/installer.ez
+
+You can execute on Unix (Linux, MacOS X):
+
+    $ wget https://example.org/installer.ez
+    $ mix archive.install installer.ez
+
+or
+
+    $ curl -o installer.ez https://example.org/installer.ez
+    $ mix archive.install installer.ez
+
+On Windows (Win7 or later):
+
+    > powershell -Command "Invoke-WebRequest https://example.org/installer.ez -OutFile installer.ez"
+    > mix archive.install installer.ez
+
+or
+
+    > powershell -Command "(New-Object Net.WebClient).DownloadFile('https://example.org/installer.ez', 'installer.ez')"
+    > mix archive.install installer.ez
+
+Note that, if you are a library author, consider providing installable escripts and archives through Hex, such as Phoenix:
+
+    $ mix archive.install hex phx_new
+
+Installations through Hex are always safe and they come with version management and all other benefits from Hex too.
+
+### 1. Enhancements
+
+#### Mix
+
+  * [mix release] Add :tar option for releases to create a tarball
+
+### 2. Bug fixes
+
+#### Mix
+
+  * [mix release] Use `default_release` option when name is not given
+  * [mix release] Make release's boot script contents deterministic
+
+### 3. Deprecations
+
+#### Mix
+
+  * [mix archive.install] Warn when installing from URI
+  * [mix escript.install] Warn when installing from URI
+  * [mix local.rebar] Warn when installing from URI
+
+## v1.9.2 (2019-10-12)
+
+### 1. Enhancements
+
+#### Mix
+
+  * [mix release] Allow `{:from_app, app_name}` as a version for releases
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [Kernel] Ensure compilation works for a variable named `super`
+  * [Kernel] Ensure capture operator of a local function expands correctly inside a macro
+  * [Regex] Ensure dynamic recompilation of regexes considers options. This fixes an issue where parsing the protocol in `URI.parse/1` seemingly looked case sensitive when running Elixir precompiled on another machine
+
+#### Mix
+
+  * [mix release] Use `Base.encode32` when generating cookie to avoid unsafe chars
+  * [mix release] Fix `install` command on Windows
+  * [mix release] Quote executable path on Windows to ensure it works on directories with spaces
+
 ## v1.9.1 (2019-07-18)
 
 ### 1. Enhancements

VERSION

@@ -1 +1 @@
-1.9.1
+1.9.4

bin/elixir.bat

@@ -156,9 +156,9 @@ if not !runMode! == "iex" (
   set beforeExtra=-noshell -s elixir start_cli !beforeExtra!
 )
 if defined useWerl (
-  start !ERTS_BIN!werl.exe !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
+  start "" "!ERTS_BIN!werl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
 ) else (
-  !ERTS_BIN!erl.exe !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
+  "!ERTS_BIN!erl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
 )
 :end
 endlocal

lib/elixir/lib/config.ex

@@ -55,7 +55,7 @@ defmodule Config do
 
   It has to be replaced by:
 
-      for config <- "apps/*/config/config.exs" |> Path.expand() |> Path.wildcard() do
+      for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do
         import_config config
       end
 

lib/elixir/lib/exception.ex

@@ -208,7 +208,7 @@ defmodule Exception do
          {_, kind, _, clauses} <- List.keyfind(defs, {function, arity}, 0) do
       clauses =
         for {meta, ex_args, guards, _block} <- clauses do
-          scope = :elixir_erl.scope(meta)
+          scope = :elixir_erl.scope(meta, true)
 
           {erl_args, scope} =
             :elixir_erl_clauses.match(&:elixir_erl_pass.translate_args/2, ex_args, scope)

lib/elixir/lib/kernel.ex

@@ -4433,14 +4433,18 @@ defmodule Kernel do
         defoverridable message: 1
 
         @impl true
-        def exception(msg) when is_binary(msg) do
+        def exception(msg) when Kernel.is_binary(msg) do
           exception(message: msg)
         end
       end
 
       # TODO: Change the implementation on v2.0 to simply call Kernel.struct!/2
+      # Calls to Kernel functions must be fully-qualified to ensure
+      # reproducible builds; otherwise, this macro will generate ASTs
+      # with different metadata (:import, :context) depending on if
+      # it is the bootstrapped version or not.
       @impl true
-      def exception(args) when is_list(args) do
+      def exception(args) when Kernel.is_list(args) do
         struct = __struct__()
         {valid, invalid} = Enum.split_with(args, fn {k, _} -> Map.has_key?(struct, k) end)
 
@@ -4451,9 +4455,9 @@ defmodule Kernel do
           _ ->
             IO.warn(
               "the following fields are unknown when raising " <>
-                "#{inspect(__MODULE__)}: #{inspect(invalid)}. " <>
+                "#{Kernel.inspect(__MODULE__)}: #{Kernel.inspect(invalid)}. " <>
                 "Please make sure to only give known fields when raising " <>
-                "or redefine #{inspect(__MODULE__)}.exception/1 to " <>
+                "or redefine #{Kernel.inspect(__MODULE__)}.exception/1 to " <>
                 "discard unknown fields. Future Elixir versions will raise on " <>
                 "unknown fields given to raise/2"
             )
@@ -4880,16 +4884,24 @@ defmodule Kernel do
   @doc ~S"""
   Handles the sigil `~S` for strings.
 
-  It simply returns a string without escaping characters and without
-  interpolations.
+  It returns a string without interpolations and without escape
+  characters, except for the escaping of the closing sigil character
+  itself.
 
   ## Examples
 
       iex> ~S(foo)
       "foo"
-
       iex> ~S(f#{o}o)
       "f\#{o}o"
+      iex> ~S(\o/)
+      "\\o/"
+
+  However, if you want to re-use the sigil character itself on
+  the string, you need to escape it:
+
+      iex> ~S((\))
+      "()"
 
   """
   defmacro sigil_S(term, modifiers)
@@ -4926,8 +4938,9 @@ defmodule Kernel do
   @doc ~S"""
   Handles the sigil `~C` for charlists.
 
-  It simply returns a charlist without escaping characters and without
-  interpolations.
+  It returns a charlist without interpolations and without escape
+  characters, except for the escaping of the closing sigil character
+  itself.
 
   ## Examples
 
@@ -5007,8 +5020,10 @@ defmodule Kernel do
   @doc ~S"""
   Handles the sigil `~R` for regular expressions.
 
-  It returns a regular expression pattern without escaping
-  nor interpreting interpolations.
+  It returns a regular expression pattern without interpolations and
+  without escape characters. Note it still supports escape of Regex
+  tokens (such as escaping `+` or `?`) and it also requires you to
+  escape the closing sigil character itself if it appears on the Regex.
 
   More information on regexes can be found in the `Regex` module.
 
@@ -5171,8 +5186,9 @@ defmodule Kernel do
   @doc ~S"""
   Handles the sigil `~W` for list of words.
 
-  It returns a list of "words" split by whitespace without escaping nor
-  interpreting interpolations.
+  It returns a list of "words" split by whitespace without interpolations
+  and without escape characters, except for the escaping of the closing
+  sigil character itself.
 
   ## Modifiers
 

lib/elixir/lib/path.ex

@@ -95,6 +95,8 @@ defmodule Path do
     absname(absname_join(name), cwd)
   end
 
+  @slash [?/, ?\\]
+
   # Joins a list
   defp absname_join([name1, name2 | rest]), do: absname_join([absname_join(name1, name2) | rest])
 
@@ -110,6 +112,11 @@ defmodule Path do
     do_absname_join(rest, relativename, [?:, uc_letter + ?a - ?A], :win32)
   end
 
+  defp do_absname_join(<<c1, c2, rest::binary>>, relativename, [], :win32)
+       when c1 in @slash and c2 in @slash do
+    do_absname_join(rest, relativename, '//', :win32)
+  end
+
   defp do_absname_join(<<?\\, rest::binary>>, relativename, result, :win32),
     do: do_absname_join(<<?/, rest::binary>>, relativename, result, :win32)
 
@@ -254,8 +261,6 @@ defmodule Path do
   defp unix_pathtype([list | rest]) when is_list(list), do: unix_pathtype(list ++ rest)
   defp unix_pathtype(relative), do: {:relative, relative}
 
-  @slash [?/, ?\\]
-
   defp win32_pathtype([list | rest]) when is_list(list), do: win32_pathtype(list ++ rest)
 
   defp win32_pathtype([char, list | rest]) when is_list(list),

lib/elixir/lib/regex.ex

@@ -461,13 +461,13 @@ defmodule Regex do
   end
 
   defp safe_run(
-         %Regex{re_pattern: compiled, source: source, re_version: version},
+         %Regex{re_pattern: compiled, source: source, re_version: version, opts: compile_opts},
          string,
          options
        ) do
     case version() do
       ^version -> :re.run(string, compiled, options)
-      _ -> :re.run(string, source, options)
+      _ -> :re.run(string, source, translate_options(compile_opts, options))
     end
   end
 

lib/elixir/src/elixir.hrl

@@ -13,6 +13,7 @@
   backup_vars=nil,         %% a copy of vars to be used on ^var
   extra_guards=[],         %% extra guards from args expansion
   counter=#{},             %% a map counting the variables defined
+  expand_captures=false,   %% a boolean to control if captures should be expanded
   stacktrace=false         %% holds information about the stacktrace variable
 }).
 

lib/elixir/src/elixir_erl.erl

@@ -1,7 +1,7 @@
 %% Compiler backend to Erlang.
 -module(elixir_erl).
 -export([elixir_to_erl/1, definition_to_anonymous/4, compile/1, consolidate/3,
-         get_ann/1, debug_info/4, scope/1, format_error/1]).
+         get_ann/1, debug_info/4, scope/2, format_error/1]).
 -include("elixir.hrl").
 -define(typespecs, 'Elixir.Kernel.Typespec').
 
@@ -48,7 +48,7 @@ get_ann([], Gen, Line) -> erl_anno:set_generated(Gen, Line).
 %% Converts an Elixir definition to an anonymous function.
 
 definition_to_anonymous(Module, Kind, Meta, Clauses) ->
-  ErlClauses = [translate_clause(Kind, Clause) || Clause <- Clauses],
+  ErlClauses = [translate_clause(Kind, Clause, true) || Clause <- Clauses],
   Fun = {'fun', ?ann(Meta), {clauses, ErlClauses}},
   LocalHandler = fun(LocalName, LocalArgs) -> invoke_local(Module, LocalName, LocalArgs) end,
   {value, Result, _Binding} = erl_eval:expr(Fun, [], {value, LocalHandler}),
@@ -117,8 +117,8 @@ elixir_to_erl_cons(T) -> elixir_to_erl(T).
 
 %% Returns a scope for translation.
 
-scope(_Meta) ->
-  #elixir_erl{}.
+scope(_Meta, ExpandCaptures) ->
+  #elixir_erl{expand_captures=ExpandCaptures}.
 
 %% Static compilation hook, used in protocol consolidation
 
@@ -220,15 +220,15 @@ add_definition(Meta, Body, {Head, Tail}) ->
   end.
 
 translate_definition(Kind, Meta, {Name, Arity}, Clauses) ->
-  ErlClauses = [translate_clause(Kind, Clause) || Clause <- Clauses],
+  ErlClauses = [translate_clause(Kind, Clause, false) || Clause <- Clauses],
 
   case is_macro(Kind) of
     true -> {function, ?ann(Meta), elixir_utils:macro_name(Name), Arity + 1, ErlClauses};
     false -> {function, ?ann(Meta), Name, Arity, ErlClauses}
   end.
 
-translate_clause(Kind, {Meta, Args, Guards, Body}) ->
-  S = scope(Meta),
+translate_clause(Kind, {Meta, Args, Guards, Body}, ExpandCaptures) ->
+  S = scope(Meta, ExpandCaptures),
 
   {TClause, TS} = elixir_erl_clauses:clause(Meta,
                     fun elixir_erl_pass:translate_args/2, Args, Body, Guards, S),

lib/elixir/src/elixir_erl_pass.erl

@@ -69,7 +69,7 @@ translate({'__CALLER__', Meta, Atom}, S) when is_atom(Atom) ->
 translate({'__STACKTRACE__', Meta, Atom}, S = #elixir_erl{stacktrace={Var, _}}) when is_atom(Atom) ->
   {{var, ?ann(Meta), Var}, S#elixir_erl{stacktrace={Var, true}}};
 
-translate({'super', Meta, Args}, S) ->
+translate({'super', Meta, Args}, S) when is_list(Args) ->
   %% In the expanded AST, super is used to invoke a function
   %% in the current module originated from a default clause
   %% or a super call.
@@ -96,7 +96,15 @@ translate({'&', Meta, [{'/', _, [{{'.', _, [Remote, Fun]}, _, []}, Arity]}]}, S)
   {{'fun', Ann, {function, TRemote, TFun, TArity}}, SR};
 translate({'&', Meta, [{'/', _, [{Fun, _, Atom}, Arity]}]}, S)
     when is_atom(Fun), is_atom(Atom), is_integer(Arity) ->
-  {{'fun', ?ann(Meta), {function, Fun, Arity}}, S};
+  case S of
+    #elixir_erl{expand_captures=true} ->
+      Vars = [{list_to_atom("arg" ++ integer_to_list(Counter)), [], ?MODULE}
+              || Counter <- tl(lists:seq(0, Arity))],
+      translate({'fn', Meta, [{'->', Meta, [Vars, {Fun, Meta, Vars}]}]}, S);
+
+    #elixir_erl{expand_captures=false} ->
+      {{'fun', ?ann(Meta), {function, Fun, Arity}}, S}
+  end;
 
 translate({fn, Meta, Clauses}, S) ->
   Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) ->

lib/elixir/test/elixir/kernel/macros_test.exs

@@ -34,6 +34,10 @@ defmodule Kernel.MacrosTest do
     quote(do: 1 + unquote(value))
   end
 
+  defmacro my_macro_with_capture(value) do
+    Enum.map(value, &by_two/1)
+  end
+
   test "require" do
     assert Kernel.MacrosTest.Nested.value() == 1
   end
@@ -42,7 +46,7 @@ defmodule Kernel.MacrosTest do
     assert Nested.value() == 1
   end
 
-  test "local but private macro" do
+  test "local with private macro" do
     assert my_private_macro() == 4
   end
 
@@ -54,6 +58,10 @@ defmodule Kernel.MacrosTest do
     assert my_macro_with_local(4) == 17
   end
 
+  test "local with capture" do
+    assert my_macro_with_capture([1, 2, 3]) == [2, 4, 6]
+  end
+
   test "macros cannot be called dynamically" do
     x = Nested
     assert_raise UndefinedFunctionError, fn -> x.value end

lib/elixir/test/elixir/kernel/overridable_test.exs

@@ -254,6 +254,11 @@ defmodule Kernel.OverridableTest do
     assert Overridable.capture_super_macro(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]
   end
 
+  test "super as a variable" do
+    super = :ok
+    assert super == :ok
+  end
+
   test "overridable with many clauses" do
     assert Overridable.many_clauses(0) == 11
     assert Overridable.many_clauses(1) == 13

lib/elixir/test/elixir/path_test.exs

@@ -39,6 +39,13 @@ defmodule PathTest do
   describe "Windows" do
     @describetag :windows
 
+    test "absname/1" do
+      assert Path.absname("//host/path") == "//host/path"
+      assert Path.absname("\\\\host\\path") == "//host/path"
+      assert Path.absname("\\/host\\path") == "//host/path"
+      assert Path.absname("/\\host\\path") == "//host/path"
+    end
+
     test "relative/1" do
       assert Path.relative("C:/usr/local/bin") == "usr/local/bin"
       assert Path.relative("C:\\usr\\local\\bin") == "usr\\local\\bin"
@@ -67,6 +74,11 @@ defmodule PathTest do
       assert Path.type("/usr/local/bin") == :volumerelative
       assert Path.type('usr/local/bin') == :relative
       assert Path.type("../usr/local/bin") == :relative
+
+      assert Path.type("//host/path") == :absolute
+      assert Path.type("\\\\host\\path") == :absolute
+      assert Path.type("/\\host\\path") == :absolute
+      assert Path.type("\\/host\\path") == :absolute
     end
 
     test "split/1" do

lib/elixir/test/elixir/regex_test.exs

@@ -194,6 +194,10 @@ defmodule RegexTest do
     assert Regex.run(@re_19_3_little, "abcd abce", capture: :all_names) == ["d"]
   end
 
+  test "run/3 with regexes with options compiled in different systems" do
+    assert Regex.run(%{~r/foo/i | re_version: "bad version"}, "FOO") == ["FOO"]
+  end
+
   test "scan/2" do
     assert Regex.scan(~r"c(d|e)", "abcd abce") == [["cd", "d"], ["ce", "e"]]
     assert Regex.scan(~r"c(?:d|e)", "abcd abce") == [["cd"], ["ce"]]
@@ -213,6 +217,10 @@ defmodule RegexTest do
     assert Regex.scan(@re_19_3_little, "abcd abce", capture: :all_names) == [["d"], ["e"]]
   end
 
+  test "scan/2 with regexes with options compiled in different systems" do
+    assert Regex.scan(%{~r/foo/i | re_version: "bad version"}, "FOO") == [["FOO"]]
+  end
+
   test "split/2,3" do
     assert Regex.split(~r",", "") == [""]
     assert Regex.split(~r",", "", trim: true) == []

lib/mix/lib/mix/compilers/elixir.ex

@@ -520,10 +520,17 @@ defmodule Mix.Compilers.Elixir do
         split_manifest(data, compile_path)
 
       [v | data] when is_integer(v) ->
-        for module <- data, is_record(module, :module) do
-          File.rm(Path.join(compile_path, module(module, :beam)))
-          :code.purge(module(module, :module))
-          :code.delete(module(module, :module))
+        try do
+          for module <- data, is_record(module, :module) do
+            File.rm(Path.join(compile_path, module(module, :beam)))
+            :code.purge(module(module, :module))
+            :code.delete(module(module, :module))
+          end
+        rescue
+          _ ->
+            Mix.raise(
+              "Cannot clean-up stale manifest, please run \"mix clean --deps\" manually before proceeding"
+            )
         end
 
         {[], []}

lib/mix/lib/mix/local/installer.ex

@@ -107,7 +107,7 @@ defmodule Mix.Local.Installer do
           module.install(basename, binary, previous_files)
 
         :badpath ->
-          Mix.raise("Expected #{inspect(src)} to be a URL or a local file path")
+          Mix.raise("Expected #{inspect(src)} to be a local file path")
 
         {:local, message} ->
           Mix.raise(message)
@@ -164,7 +164,7 @@ defmodule Mix.Local.Installer do
     cond do
       local_path?(url_or_path) -> {:local, url_or_path}
       file_url?(url_or_path) -> {:url, url_or_path}
-      true -> {:error, "Expected #{inspect(url_or_path)} to be a URL or a local file path"}
+      true -> {:error, "Expected #{inspect(url_or_path)} to be a local file path"}
     end
   end
 

lib/mix/lib/mix/release.ex

@@ -7,7 +7,8 @@ defmodule Mix.Release do
   The Mix.Release struct has the following read-only fields:
 
     * `:name` - the name of the release as an atom
-    * `:version` - the version of the release as a string
+    * `:version` - the version of the release as a string or
+       `{:from_app, app_name}
     * `:path` - the path to the release root
     * `:version_path` - the path to the release version inside the release
     * `:applications` - a map of application with their definitions
@@ -24,7 +25,8 @@ defmodule Mix.Release do
       and `term` is the value given to it on `c:Config.Provider.init/1`
     * `:options` - a keyword list with all other user supplied release options
     * `:steps` - a list of functions that receive the release and returns a release.
-      Must also contain the atom `:assemble` which is the internal assembling step
+      Must also contain the atom `:assemble` which is the internal assembling step.
+      May also contain the atom `:tar` to create a tarball of the release.
 
   """
   defstruct [
@@ -47,7 +49,7 @@ defmodule Mix.Release do
           name: atom(),
           version: String.t(),
           path: String.t(),
-          version_path: String.t(),
+          version_path: String.t() | {:from_app, application()},
           applications: %{application() => keyword()},
           boot_scripts: %{atom() => [{application(), mode()}]},
           erts_version: charlist(),
@@ -112,6 +114,24 @@ defmodule Mix.Release do
           )
       end)
 
+    version =
+      case version do
+        {:from_app, app} ->
+          Application.load(app)
+          version = Application.spec(app, :vsn)
+
+          if !version do
+            Mix.raise(
+              "Could not find version for #{inspect(app)}, please make sure the application exists"
+            )
+          end
+
+          to_string(version)
+
+        _ ->
+          version
+      end
+
     {config_providers, opts} = Keyword.pop(opts, :config_providers, [])
     {steps, opts} = Keyword.pop(opts, :steps, [:assemble])
     validate_steps!(steps)
@@ -158,10 +178,16 @@ defmodule Mix.Release do
         {name, opts}
 
       [_ | _] ->
-        Mix.raise(
-          "\"mix release\" was invoked without a name but there are multiple releases. " <>
-            "Please call \"mix release NAME\" or set :default_release in your project configuration"
-        )
+        case Keyword.get(config, :default_release) do
+          nil ->
+            Mix.raise(
+              "\"mix release\" was invoked without a name but there are multiple releases. " <>
+                "Please call \"mix release NAME\" or set :default_release in your project configuration"
+            )
+
+          name ->
+            lookup_release(name, config)
+        end
     end
   end
 
@@ -292,10 +318,12 @@ defmodule Mix.Release do
 
   defp build_start_boot(all_apps, specified_apps) do
     specified_apps ++
-      for(
-        {app, props} <- all_apps,
-        not List.keymember?(specified_apps, app, 0),
-        do: {app, default_mode(props)}
+      Enum.sort(
+        for(
+          {app, props} <- all_apps,
+          not List.keymember?(specified_apps, app, 0),
+          do: {app, default_mode(props)}
+        )
       )
   end
 
@@ -310,12 +338,14 @@ defmodule Mix.Release do
   end
 
   defp validate_steps!(steps) do
-    if not is_list(steps) or Enum.any?(steps, &(&1 != :assemble and not is_function(&1, 1))) do
+    valid_atoms = [:assemble, :tar]
+
+    if not is_list(steps) or Enum.any?(steps, &(&1 not in valid_atoms and not is_function(&1, 1))) do
       Mix.raise("""
         The :steps option must be a list of:
 
         * anonymous function that receives one argument
-        * the atom :assemble
+        * the atom :assemble or :tar
 
       Got: #{inspect(steps)}
       """)
@@ -325,6 +355,14 @@ defmodule Mix.Release do
       Mix.raise("The :steps option must contain the atom :assemble once, got: #{inspect(steps)}")
     end
 
+    if :assemble in Enum.drop_while(steps, &(&1 != :tar)) do
+      Mix.raise("The :tar step must come after :assemble")
+    end
+
+    if Enum.count(steps, &(&1 == :tar)) > 1 do
+      Mix.raise("The :steps option can only contain the atom :tar once")
+    end
+
     :ok
   end
 
@@ -409,7 +447,7 @@ defmodule Mix.Release do
     end
   end
 
-  defp random_cookie, do: Base.url_encode64(:crypto.strong_rand_bytes(40))
+  defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32))
 
   @doc """
   Makes the start_erl.data file with the

lib/mix/lib/mix/tasks/archive.install.ex

@@ -12,7 +12,7 @@ defmodule Mix.Tasks.Archive.Install do
 
       mix do archive.build, archive.install
 
-  If an argument is provided, it should be a local path or a URL to a
+  If an argument is provided, it should be a local path to a
   prebuilt archive, a Git repository, a GitHub repository, or a Hex
   package.
 
@@ -35,13 +35,12 @@ defmodule Mix.Tasks.Archive.Install do
       mix some_task
 
   Note that installing via Git, GitHub, or Hex fetches the source
-  of the archive and builds it, while using a URL or a local path
-  fetches a pre-built archive.
+  of the archive and builds it, while using local path uses a pre-built archive.
 
   ## Command line options
 
     * `--sha512` - checks the archive matches the given SHA-512 checksum. Only
-      applies to installations via URL or local path
+      applies to installations via a local path
 
     * `--force` - forces installation without a shell prompt; primarily
       intended for automation in build systems like Make
@@ -55,9 +54,6 @@ defmodule Mix.Tasks.Archive.Install do
     * `--organization` - specifies an organization to use if fetching the package
       from a private Hex repository
 
-    * `--timeout` - sets a request timeout in milliseconds for fetching
-      archives from URLs. Default is 60 seconds
-
   """
 
   @behaviour Mix.Local.Installer
@@ -77,17 +73,28 @@ defmodule Mix.Tasks.Archive.Install do
   end
 
   @impl true
-  def check_install_spec({local_or_url, path_or_url} = _install_spec, _opts)
-      when local_or_url in [:local, :url] do
-    if Path.extname(path_or_url) == ".ez" do
-      :ok
-    else
-      {:error, "Expected a local file path or a file URL ending in \".ez\"."}
+  def check_install_spec({:local, path} = _install_spec, _opts) do
+    check_extname(path)
+  end
+
+  def check_install_spec({:url, url} = _install_spec, opts) do
+    unless opts[:sha512] do
+      Mix.Utils.warn_install_over_http_deprecated("archive.install", url)
     end
+
+    check_extname(url)
   end
 
   def check_install_spec(_, _), do: :ok
 
+  defp check_extname(path_or_url) do
+    if Path.extname(path_or_url) == ".ez" do
+      :ok
+    else
+      {:error, "Expected a local file path ending in \".ez\"."}
+    end
+  end
+
   @impl true
   def find_previous_versions(src) do
     app =

lib/mix/lib/mix/tasks/escript.install.ex

@@ -12,12 +12,11 @@ defmodule Mix.Tasks.Escript.Install do
 
       mix do escript.build, escript.install
 
-  If an argument is provided, it should be a local path or a URL to a prebuilt escript,
+  If an argument is provided, it should be a local path to a prebuilt escript,
   a Git repository, a GitHub repository, or a Hex package.
 
       mix escript.install escript
       mix escript.install path/to/escript
-      mix escript.install https://example.com/my_escript
       mix escript.install git https://path/to/git/repo
       mix escript.install git https://path/to/git/repo branch git_branch
       mix escript.install git https://path/to/git/repo tag git_tag
@@ -40,7 +39,7 @@ defmodule Mix.Tasks.Escript.Install do
   ## Command line options
 
     * `--sha512` - checks the escript matches the given SHA-512 checksum. Only
-      applies to installations via URL or local path
+      applies to installations via a local path
 
     * `--force` - forces installation without a shell prompt; primarily
       intended for automation in build systems like Make
@@ -54,9 +53,6 @@ defmodule Mix.Tasks.Escript.Install do
     * `--organization` - specifies an organization to use if fetching the package
       from a private Hex repository
 
-    * `--timeout` - sets a request timeout in milliseconds for fetching
-      archives from URLs. Default is 60 seconds
-
   """
 
   @behaviour Mix.Local.Installer
@@ -79,6 +75,14 @@ defmodule Mix.Tasks.Escript.Install do
   end
 
   @impl true
+  def check_install_spec({:url, url} = _install_spec, opts) do
+    unless opts[:sha512] do
+      Mix.Utils.warn_install_over_http_deprecated("escript.install", url)
+    end
+
+    :ok
+  end
+
   def check_install_spec(_, _), do: :ok
 
   @impl true

lib/mix/lib/mix/tasks/local.rebar.ex

@@ -25,9 +25,9 @@ defmodule Mix.Tasks.Local.Rebar do
 
   ## Command line options
 
-    * `rebar PATH` - specifies a path or URL for `rebar`
+    * `rebar PATH` - specifies a path for `rebar`
 
-    * `rebar3 PATH` - specifies a path or URL for `rebar3`
+    * `rebar3 PATH` - specifies a path for `rebar3`
 
     * `--sha512` - checks the archive matches the given SHA-512 checksum
 
@@ -48,9 +48,11 @@ defmodule Mix.Tasks.Local.Rebar do
 
     case argv do
       ["rebar", path | _] ->
+        if file_url?(path), do: warn_install_over_http_deprecated(:rebar, path)
         install_from_path(:rebar, path, opts)
 
       ["rebar3", path | _] ->
+        if file_url?(path), do: warn_install_over_http_deprecated(:rebar3, path)
         install_from_path(:rebar3, path, opts)
 
       [] ->
@@ -77,7 +79,7 @@ defmodule Mix.Tasks.Local.Rebar do
           Mix.shell().info([:green, "* creating ", :reset, Path.relative_to_cwd(local)])
 
         :badpath ->
-          Mix.raise("Expected #{inspect(path)} to be a URL or a local file path")
+          Mix.raise("Expected #{inspect(path)} to be a local file path")
 
         {:local, message} ->
           Mix.raise(message)
@@ -100,6 +102,10 @@ defmodule Mix.Tasks.Local.Rebar do
     true
   end
 
+  defp file_url?(url_or_path) do
+    URI.parse(url_or_path).scheme in ["http", "https"]
+  end
+
   defp install_from_s3(manager, list_url, escript_url, opts) do
     hex_mirror = Mix.Hex.mirror()
     list_url = hex_mirror <> list_url
@@ -114,4 +120,37 @@ defmodule Mix.Tasks.Local.Rebar do
 
     install_from_path(manager, url, Keyword.put(opts, :sha512, sha512))
   end
+
+  defp warn_install_over_http_deprecated(manager, url) do
+    shell = Mix.shell()
+    basename = Path.basename(url)
+
+    shell.error("""
+    Warning: the use of HTTP/HTTPS URLs with `mix local.rebar` is deprecated")
+
+    Run `mix help local.rebar` for details on installing #{manager} from Hex's CDN.
+    Alternatively you can fetch the file using an external HTTP client and then
+    install it locally:
+
+    Unix (Linux, MacOS X):
+
+        $ wget #{url}
+        $ mix local.rebar #{manager} #{basename}
+
+    or
+
+        $ curl -o #{basename} #{url}
+        $ mix local.rebar #{manager} #{basename}
+
+    Windows (Win7 or later):
+
+        > powershell -Command "Invoke-WebRequest #{url} -OutFile #{basename}"
+        > mix local.rebar #{manager} #{basename}
+
+    or
+
+        > powershell -Command "(New-Object Net.WebClient).DownloadFile('#{url}', '#{basename}')"
+        > mix local.rebar #{manager} #{basename}
+    """)
+  end
 end

lib/mix/lib/mix/tasks/release.ex

@@ -386,8 +386,10 @@ defmodule Mix.Tasks.Release do
     * `:path` - the path the release should be installed to.
       Defaults to `"_build/MIX_ENV/rel/RELEASE_NAME"`.
 
-    * `:version` - the release version as a string. Defaults to the current
-      application version.
+    * `:version` - the release version as a string or `{:from_app, app_name}`.
+      Defaults to the current application version. The `{:from_app, app_name}` format
+      can be used to easily reference the application version from another application.
+      This is particularly useful in umbrella applications.
 
     * `:quiet` - a boolean that controls if releases should write to the standard
       output its steps. Defaults to `false`.
@@ -489,7 +491,9 @@ defmodule Mix.Tasks.Release do
   can pass anonymous functions before and after the `:assemble` to
   customize your release assembling pipeline. Those anonymous functions
   will receive a `Mix.Release` struct and must return the same or
-  an updated `Mix.Release` struct.
+  an updated `Mix.Release` struct. It is also possible to build a tarball
+  of the release by passing the `:tar` step anywhere after `:assemble`.
+  The tarball is created in `_build/MIX_ENV/rel/RELEASE_NAME-RELEASE_VSN.tar.gz`
 
   See `Mix.Release` for more documentation on the struct and which
   fields can be modified. Note that `:steps` field itself can be
@@ -951,6 +955,10 @@ defmodule Mix.Tasks.Release do
     end
   end
 
+  defp run_steps(%{steps: [:tar | steps]} = release) do
+    %{release | steps: steps} |> make_tar() |> run_steps()
+  end
+
   defp run_steps(%{steps: [:assemble | steps]} = release) do
     %{release | steps: steps} |> assemble() |> run_steps()
   end
@@ -1004,6 +1012,36 @@ defmodule Mix.Tasks.Release do
     release
   end
 
+  defp make_tar(release) do
+    tar_filename = "#{release.name}-#{release.version}.tar.gz"
+    out_path = Path.join([release.path, "..", "..", tar_filename]) |> Path.expand()
+    info(release, [:green, "* building ", :reset, out_path])
+
+    lib_dirs =
+      Enum.reduce(release.applications, [], fn {name, app_config}, acc ->
+        vsn = Keyword.fetch!(app_config, :vsn)
+        [Path.join("lib", "#{name}-#{vsn}") | acc]
+      end)
+
+    release_files =
+      for basename <- File.ls!(Path.join(release.path, "releases")),
+          not File.dir?(Path.join([release.path, "releases", basename])),
+          do: Path.join("releases", basename)
+
+    dirs =
+      ["bin", Path.join("releases", release.version), "erts-#{release.erts_version}"] ++
+        lib_dirs ++ release_files
+
+    files =
+      dirs
+      |> Enum.filter(&File.exists?(Path.join(release.path, &1)))
+      |> Enum.map(&{String.to_charlist(&1), String.to_charlist(Path.join(release.path, &1))})
+
+    File.rm(out_path)
+    :ok = :erl_tar.create(String.to_charlist(out_path), files, [:dereference, :compressed])
+    release
+  end
+
   # build_rel
 
   defp build_rel(release, config) do

lib/mix/lib/mix/tasks/release.init.ex

@@ -285,7 +285,7 @@ defmodule Mix.Tasks.Release.Init do
       findstr "RUNTIME_CONFIG=true" "!RELEASE_SYS_CONFIG!.config" >nul 2>&1 && (
         for /f "skip=1" %%X in ('wmic os get localdatetime') do if not defined TIMESTAMP set TIMESTAMP=%%X
         set RELEASE_SYS_CONFIG=!RELEASE_TMP!\!RELEASE_NAME!-!RELEASE_VSN!-!TIMESTAMP:~0,11!-!RANDOM!.runtime
-        mkdir "!RELEASE_TMP!" >nul
+        mkdir "!RELEASE_TMP!" >nul 2>&1
         copy /y "!REL_VSN_DIR!\sys.config" "!RELEASE_SYS_CONFIG!.config" >nul || (
           echo Cannot start release because it could not write to "!RELEASE_SYS_CONFIG!.config"
           goto end
@@ -378,7 +378,8 @@ defmodule Mix.Tasks.Release.Init do
     )
 
     !ERLSRV! add !RELEASE_NAME!_!RELEASE_NAME! ^
-      -name "!RELEASE_NODE!" ^
+      -!RELEASE_DISTRIBUTION! "!RELEASE_NODE!" ^
+      -env RELEASE_ROOT=!RELEASE_ROOT! -env RELEASE_NAME=!RELEASE_NAME! -env RELEASE_VSN=!RELEASE_VSN! -env RELEASE_COOKIE=!RELEASE_COOKIE! -env RELEASE_NODE=!RELEASE_NODE! -env RELEASE_VM_ARGS=!RELEASE_VM_ARGS! -env RELEASE_TMP=!RELEASE_TMP! -env RELEASE_SYS_CONFIG=!RELEASE_SYS_CONFIG! ^
       -args "-setcookie !RELEASE_COOKIE! -config !RELEASE_SYS_CONFIG! -mode !RELEASE_MODE! -boot !REL_VSN_DIR!\start -boot_var RELEASE_LIB !RELEASE_ROOT!\lib -args_file !REL_VSN_DIR!\vm.args"
 
     if %ERRORLEVEL% EQU 0 (

lib/mix/lib/mix/utils.ex

@@ -676,4 +676,38 @@ defmodule Mix.Utils do
 
     [proxy_auth: {user, pass}]
   end
+
+  def warn_install_over_http_deprecated(task_name, url) do
+    basename = Path.basename(url)
+
+    shell = Mix.shell()
+
+    shell.error("""
+    Warning: the use of HTTP/HTTPS URLs with `mix #{task_name}` is deprecated
+
+    Run `mix help #{task_name}` for details on the available alternatives, using
+    hex, git or github. Alternatively you can fetch the file using an external HTTP
+    client and then install it locally:
+
+    Unix (Linux, MacOS X):
+
+        $ wget #{url}
+        $ mix #{task_name} #{basename}
+
+    or
+
+        $ curl -o #{basename} #{url}
+        $ mix #{task_name} #{basename}
+
+    Windows (Win7 or later):
+
+        > powershell -Command "Invoke-WebRequest #{url} -OutFile #{basename}"
+        > mix #{task_name} #{basename}
+
+    or
+
+        > powershell -Command "(New-Object Net.WebClient).DownloadFile('#{url}', '#{basename}')"
+        > mix #{task_name} #{basename}
+    """)
+  end
 end

lib/mix/test/mix/release_test.exs

@@ -50,6 +50,21 @@ defmodule Mix.ReleaseTest do
       assert release.options[:quiet]
     end
 
+    test "allows specifying the version from an application" do
+      overrides = [version: {:from_app, :elixir}]
+      release = from_config!(nil, config(), overrides)
+
+      assert release.version == to_string(Application.spec(:elixir, :vsn))
+    end
+
+    test "raises when :from_app is used with an app that doesn't exist" do
+      overrides = [version: {:from_app, :not_valid}]
+
+      assert_raise Mix.Error,
+                   ~r"Could not find version for :not_valid, please make sure the application exists",
+                   fn -> from_config!(nil, config(), overrides) end
+    end
+
     test "includes applications" do
       release = from_config!(nil, config(), [])
       assert release.applications.mix[:path] == to_charlist(Application.app_dir(:mix))
@@ -80,7 +95,7 @@ defmodule Mix.ReleaseTest do
     test "uses chosen release via the default_release" do
       release =
         from_config!(
-          :bar,
+          nil,
           config(
             default_release: :bar,
             releases: [foo: [version: "0.2.0"], bar: [version: "0.3.0"]]
@@ -153,6 +168,14 @@ defmodule Mix.ReleaseTest do
                    ~r"The :steps option must contain the atom :assemble once",
                    fn -> release(steps: [:assemble, :assemble]) end
 
+      assert_raise Mix.Error,
+                   ~r"The :tar step must come after :assemble",
+                   fn -> release(steps: [:tar, :assemble]) end
+
+      assert_raise Mix.Error,
+                   ~r"The :steps option can only contain the atom :tar once",
+                   fn -> release(steps: [:assemble, :tar, :tar]) end
+
       assert_raise Mix.Error,
                    ~r"The :steps option must be",
                    fn -> release(steps: [:foo]) end

lib/mix/test/mix/tasks/archive_test.exs

@@ -82,6 +82,8 @@ defmodule Mix.Tasks.ArchiveTest do
       # Try to override it with URL
       send(self(), {:mix_shell_input, :yes?, false})
       Mix.Tasks.Archive.Install.run(["https://example.com/archive-0.1.0?hello.ez"])
+
+      assert_received {:mix_shell, :error, ["Warning: the use of HTTP/HTTPS URLs" <> _]}
       assert_received {:mix_shell, :yes?, ["Found existing entry: " <> _]}
 
       # Loading the archive should emit warning again
@@ -116,7 +118,7 @@ defmodule Mix.Tasks.ArchiveTest do
   end
 
   test "archive install missing file" do
-    message = ~r[Expected "./unlikely-to-exist-0.1.0.ez" to be a URL or a local file path]
+    message = ~r[Expected "./unlikely-to-exist-0.1.0.ez" to be a local file path]
 
     assert_raise Mix.Error, message, fn ->
       Mix.Tasks.Archive.Install.run(["./unlikely-to-exist-0.1.0.ez"])

lib/mix/test/mix/tasks/escript_test.exs

@@ -242,6 +242,13 @@ defmodule Mix.Tasks.EscriptTest do
       assert_received {:mix_shell, :info, ["* escript_test"]}
       refute_received {:mix_shell, :info, ["* escript_test.bat"]}
 
+      # Try to override it with URL
+      send(self(), {:mix_shell_input, :yes?, false})
+      Mix.Tasks.Escript.Install.run(["https://example.com/escript_test"])
+
+      assert_received {:mix_shell, :error, ["Warning: the use of HTTP/HTTPS URLs" <> _]}
+      assert_received {:mix_shell, :yes?, ["Found existing entry: " <> _]}
+
       # check uninstall confirmation
       send(self(), {:mix_shell_input, :yes?, false})
       Mix.Tasks.Escript.Uninstall.run(["escript_test"])

lib/mix/test/mix/tasks/release_test.exs

@@ -36,6 +36,74 @@ defmodule Mix.Tasks.ReleaseTest do
       end)
     end
 
+    test "tar" do
+      in_fixture("release_test", fn ->
+        config = [releases: [demo: [steps: [:assemble, :tar]]]]
+
+        Mix.Project.in_project(:release_test, ".", config, fn _ ->
+          root = Path.absname("_build/#{Mix.env()}/rel/demo")
+
+          ignored_app_path = Path.join([root, "lib", "ignored_app-0.1.0", "ebin"])
+          File.mkdir_p!(ignored_app_path)
+          File.touch(Path.join(ignored_app_path, "ignored_app.app"))
+
+          ignored_release_path = Path.join([root, "releases", "ignored_dir"])
+          File.mkdir_p!(ignored_release_path)
+          File.touch(Path.join(ignored_release_path, "ignored"))
+
+          Mix.Task.run("release")
+          tar_path = Path.expand(Path.join([root, "..", "..", "demo-0.1.0.tar.gz"]))
+          message = "* building #{tar_path}"
+          assert_received {:mix_shell, :info, [^message]}
+          assert File.exists?(tar_path)
+
+          {:ok, files} = String.to_charlist(tar_path) |> :erl_tar.table([:compressed])
+
+          files = Enum.map(files, &to_string/1)
+          files_with_versions = File.ls!(Path.join(root, "lib"))
+
+          assert "bin/demo" in files
+          assert "releases/0.1.0/sys.config" in files
+          assert "releases/0.1.0/vm.args" in files
+          assert "releases/COOKIE" in files
+          assert "releases/start_erl.data" in files
+          assert Enum.any?(files, &(&1 =~ "erts"))
+          assert Enum.any?(files, &(&1 =~ "stdlib"))
+
+          for dir <- files_with_versions -- ["ignored_app-0.1.0"] do
+            [name | _] = String.split(dir, "-")
+            assert "lib/#{dir}/ebin/#{name}.app" in files
+          end
+
+          refute "lib/ignored_app-0.1.0/ebin/ignored_app.app" in files
+          refute "releases/ignored_dir/ignored" in files
+        end)
+      end)
+    end
+
+    test "tar without ERTS" do
+      in_fixture("release_test", fn ->
+        config = [releases: [demo: [include_erts: false, steps: [:assemble, :tar]]]]
+
+        Mix.Project.in_project(:release_test, ".", config, fn _ ->
+          root = Path.absname("_build/#{Mix.env()}/rel/demo")
+
+          Mix.Task.run("release")
+          tar_path = Path.expand(Path.join([root, "..", "..", "demo-0.1.0.tar.gz"]))
+          message = "* building #{tar_path}"
+          assert_received {:mix_shell, :info, [^message]}
+          assert File.exists?(tar_path)
+
+          {:ok, files} = String.to_charlist(tar_path) |> :erl_tar.table([:compressed])
+          files = Enum.map(files, &to_string/1)
+
+          assert "bin/demo" in files
+          refute Enum.any?(files, &(&1 =~ "erts"))
+          refute Enum.any?(files, &(&1 =~ "stdlib"))
+        end)
+      end)
+    end
+
     test "steps" do
       in_fixture("release_test", fn ->
         last_step = fn release ->
@@ -281,10 +349,11 @@ defmodule Mix.Tasks.ReleaseTest do
         open_port(script, ['start'])
         wait_until_decoded(Path.join(root, "RELEASE_BOOTED"))
         assert System.cmd(script, ["rpc", "ReleaseTest.hello_world"]) == {"hello world\n", 0}
-        assert System.cmd(script, ["stop"]) == {"", 0}
 
         assert {pid, 0} = System.cmd(script, ["pid"])
         assert pid != "\n"
+
+        assert System.cmd(script, ["stop"]) == {"", 0}
       end)
     end)
   end