Release v1.9.2

Closes #9343.

CHANGELOG.md

@@ -58,6 +58,28 @@ Another important change related to configuration is that `mix new` will no long
 
 There are many other enhancements. The Elixir CLI got a handful of new options in order to best support releases. `Logger` now computes its sync/async/discard thresholds in a decentralized fashion, reducing contention. `EEx` templates support more complex expressions than before. Finally, there is a new `~U` sigil for working with UTC DateTimes as well as new functions in the `File`, `Registry`, and `System` modules.
 
+## v1.9.2 (2019-10-12)
+
+### 1. Enhancements
+
+#### Mix
+
+  * [mix release] Allow `{:from_app, app_name}` as a version for releases
+
+### 2. Bug fixes
+
+#### Elixir
+
+  * [Kernel] Ensure compilation works for a variable named `super`
+  * [Kernel] Ensure capture operator of a local function expands correctly inside a macro
+  * [Regex] Ensure dynamic recompilation of regexes considers options. This fixes an issue where parsing the protocol in `URI.parse/1` seemingly looked case sensitive when running Elixir precompiled on another machine
+
+#### Mix
+
+  * [mix release] Use `Base.encode32` when generating cookie to avoid unsafe chars
+  * [mix release] Fix `install` command on Windows
+  * [mix release] Quote executable path on Windows to ensure it works on directories with spaces
+
 ## v1.9.1 (2019-07-18)
 
 ### 1. Enhancements

VERSION

@@ -1 +1 @@
-1.9.1
+1.9.2

bin/elixir.bat

@@ -156,9 +156,9 @@ if not !runMode! == "iex" (
   set beforeExtra=-noshell -s elixir start_cli !beforeExtra!
 )
 if defined useWerl (
-  start !ERTS_BIN!werl.exe !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
+  start "" "!ERTS_BIN!werl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
 ) else (
-  !ERTS_BIN!erl.exe !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
+  "!ERTS_BIN!erl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir!
 )
 :end
 endlocal

lib/elixir/lib/config.ex

@@ -55,7 +55,7 @@ defmodule Config do
 
   It has to be replaced by:
 
-      for config <- "apps/*/config/config.exs" |> Path.expand() |> Path.wildcard() do
+      for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do
         import_config config
       end
 

lib/elixir/lib/exception.ex

@@ -208,7 +208,7 @@ defmodule Exception do
          {_, kind, _, clauses} <- List.keyfind(defs, {function, arity}, 0) do
       clauses =
         for {meta, ex_args, guards, _block} <- clauses do
-          scope = :elixir_erl.scope(meta)
+          scope = :elixir_erl.scope(meta, true)
 
           {erl_args, scope} =
             :elixir_erl_clauses.match(&:elixir_erl_pass.translate_args/2, ex_args, scope)

lib/elixir/lib/kernel.ex

@@ -4433,14 +4433,18 @@ defmodule Kernel do
         defoverridable message: 1
 
         @impl true
-        def exception(msg) when is_binary(msg) do
+        def exception(msg) when Kernel.is_binary(msg) do
           exception(message: msg)
         end
       end
 
       # TODO: Change the implementation on v2.0 to simply call Kernel.struct!/2
+      # Calls to Kernel functions must be fully-qualified to ensure
+      # reproducible builds; otherwise, this macro will generate ASTs
+      # with different metadata (:import, :context) depending on if
+      # it is the bootstrapped version or not.
       @impl true
-      def exception(args) when is_list(args) do
+      def exception(args) when Kernel.is_list(args) do
         struct = __struct__()
         {valid, invalid} = Enum.split_with(args, fn {k, _} -> Map.has_key?(struct, k) end)
 
@@ -4451,9 +4455,9 @@ defmodule Kernel do
           _ ->
             IO.warn(
               "the following fields are unknown when raising " <>
-                "#{inspect(__MODULE__)}: #{inspect(invalid)}. " <>
+                "#{Kernel.inspect(__MODULE__)}: #{Kernel.inspect(invalid)}. " <>
                 "Please make sure to only give known fields when raising " <>
-                "or redefine #{inspect(__MODULE__)}.exception/1 to " <>
+                "or redefine #{Kernel.inspect(__MODULE__)}.exception/1 to " <>
                 "discard unknown fields. Future Elixir versions will raise on " <>
                 "unknown fields given to raise/2"
             )

lib/elixir/lib/regex.ex

@@ -461,13 +461,13 @@ defmodule Regex do
   end
 
   defp safe_run(
-         %Regex{re_pattern: compiled, source: source, re_version: version},
+         %Regex{re_pattern: compiled, source: source, re_version: version, opts: compile_opts},
          string,
          options
        ) do
     case version() do
       ^version -> :re.run(string, compiled, options)
-      _ -> :re.run(string, source, options)
+      _ -> :re.run(string, source, translate_options(compile_opts, options))
     end
   end
 

lib/elixir/src/elixir.hrl

@@ -13,6 +13,7 @@
   backup_vars=nil,         %% a copy of vars to be used on ^var
   extra_guards=[],         %% extra guards from args expansion
   counter=#{},             %% a map counting the variables defined
+  expand_captures=false,   %% a boolean to control if captures should be expanded
   stacktrace=false         %% holds information about the stacktrace variable
 }).
 

lib/elixir/src/elixir_erl.erl

@@ -1,7 +1,7 @@
 %% Compiler backend to Erlang.
 -module(elixir_erl).
 -export([elixir_to_erl/1, definition_to_anonymous/4, compile/1, consolidate/3,
-         get_ann/1, debug_info/4, scope/1, format_error/1]).
+         get_ann/1, debug_info/4, scope/2, format_error/1]).
 -include("elixir.hrl").
 -define(typespecs, 'Elixir.Kernel.Typespec').
 
@@ -48,7 +48,7 @@ get_ann([], Gen, Line) -> erl_anno:set_generated(Gen, Line).
 %% Converts an Elixir definition to an anonymous function.
 
 definition_to_anonymous(Module, Kind, Meta, Clauses) ->
-  ErlClauses = [translate_clause(Kind, Clause) || Clause <- Clauses],
+  ErlClauses = [translate_clause(Kind, Clause, true) || Clause <- Clauses],
   Fun = {'fun', ?ann(Meta), {clauses, ErlClauses}},
   LocalHandler = fun(LocalName, LocalArgs) -> invoke_local(Module, LocalName, LocalArgs) end,
   {value, Result, _Binding} = erl_eval:expr(Fun, [], {value, LocalHandler}),
@@ -117,8 +117,8 @@ elixir_to_erl_cons(T) -> elixir_to_erl(T).
 
 %% Returns a scope for translation.
 
-scope(_Meta) ->
-  #elixir_erl{}.
+scope(_Meta, ExpandCaptures) ->
+  #elixir_erl{expand_captures=ExpandCaptures}.
 
 %% Static compilation hook, used in protocol consolidation
 
@@ -220,15 +220,15 @@ add_definition(Meta, Body, {Head, Tail}) ->
   end.
 
 translate_definition(Kind, Meta, {Name, Arity}, Clauses) ->
-  ErlClauses = [translate_clause(Kind, Clause) || Clause <- Clauses],
+  ErlClauses = [translate_clause(Kind, Clause, false) || Clause <- Clauses],
 
   case is_macro(Kind) of
     true -> {function, ?ann(Meta), elixir_utils:macro_name(Name), Arity + 1, ErlClauses};
     false -> {function, ?ann(Meta), Name, Arity, ErlClauses}
   end.
 
-translate_clause(Kind, {Meta, Args, Guards, Body}) ->
-  S = scope(Meta),
+translate_clause(Kind, {Meta, Args, Guards, Body}, ExpandCaptures) ->
+  S = scope(Meta, ExpandCaptures),
 
   {TClause, TS} = elixir_erl_clauses:clause(Meta,
                     fun elixir_erl_pass:translate_args/2, Args, Body, Guards, S),

lib/elixir/src/elixir_erl_pass.erl

@@ -69,7 +69,7 @@ translate({'__CALLER__', Meta, Atom}, S) when is_atom(Atom) ->
 translate({'__STACKTRACE__', Meta, Atom}, S = #elixir_erl{stacktrace={Var, _}}) when is_atom(Atom) ->
   {{var, ?ann(Meta), Var}, S#elixir_erl{stacktrace={Var, true}}};
 
-translate({'super', Meta, Args}, S) ->
+translate({'super', Meta, Args}, S) when is_list(Args) ->
   %% In the expanded AST, super is used to invoke a function
   %% in the current module originated from a default clause
   %% or a super call.
@@ -96,7 +96,15 @@ translate({'&', Meta, [{'/', _, [{{'.', _, [Remote, Fun]}, _, []}, Arity]}]}, S)
   {{'fun', Ann, {function, TRemote, TFun, TArity}}, SR};
 translate({'&', Meta, [{'/', _, [{Fun, _, Atom}, Arity]}]}, S)
     when is_atom(Fun), is_atom(Atom), is_integer(Arity) ->
-  {{'fun', ?ann(Meta), {function, Fun, Arity}}, S};
+  case S of
+    #elixir_erl{expand_captures=true} ->
+      Vars = [{list_to_atom("arg" ++ integer_to_list(Counter)), [], ?MODULE}
+              || Counter <- tl(lists:seq(0, Arity))],
+      translate({'fn', Meta, [{'->', Meta, [Vars, {Fun, Meta, Vars}]}]}, S);
+
+    #elixir_erl{expand_captures=false} ->
+      {{'fun', ?ann(Meta), {function, Fun, Arity}}, S}
+  end;
 
 translate({fn, Meta, Clauses}, S) ->
   Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) ->

lib/elixir/test/elixir/kernel/macros_test.exs

@@ -34,6 +34,10 @@ defmodule Kernel.MacrosTest do
     quote(do: 1 + unquote(value))
   end
 
+  defmacro my_macro_with_capture(value) do
+    Enum.map(value, &by_two/1)
+  end
+
   test "require" do
     assert Kernel.MacrosTest.Nested.value() == 1
   end
@@ -42,7 +46,7 @@ defmodule Kernel.MacrosTest do
     assert Nested.value() == 1
   end
 
-  test "local but private macro" do
+  test "local with private macro" do
     assert my_private_macro() == 4
   end
 
@@ -54,6 +58,10 @@ defmodule Kernel.MacrosTest do
     assert my_macro_with_local(4) == 17
   end
 
+  test "local with capture" do
+    assert my_macro_with_capture([1, 2, 3]) == [2, 4, 6]
+  end
+
   test "macros cannot be called dynamically" do
     x = Nested
     assert_raise UndefinedFunctionError, fn -> x.value end

lib/elixir/test/elixir/kernel/overridable_test.exs

@@ -254,6 +254,11 @@ defmodule Kernel.OverridableTest do
     assert Overridable.capture_super_macro(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]
   end
 
+  test "super as a variable" do
+    super = :ok
+    assert super == :ok
+  end
+
   test "overridable with many clauses" do
     assert Overridable.many_clauses(0) == 11
     assert Overridable.many_clauses(1) == 13

lib/elixir/test/elixir/regex_test.exs

@@ -194,6 +194,10 @@ defmodule RegexTest do
     assert Regex.run(@re_19_3_little, "abcd abce", capture: :all_names) == ["d"]
   end
 
+  test "run/3 with regexes with options compiled in different systems" do
+    assert Regex.run(%{~r/foo/i | re_version: "bad version"}, "FOO") == ["FOO"]
+  end
+
   test "scan/2" do
     assert Regex.scan(~r"c(d|e)", "abcd abce") == [["cd", "d"], ["ce", "e"]]
     assert Regex.scan(~r"c(?:d|e)", "abcd abce") == [["cd"], ["ce"]]
@@ -213,6 +217,10 @@ defmodule RegexTest do
     assert Regex.scan(@re_19_3_little, "abcd abce", capture: :all_names) == [["d"], ["e"]]
   end
 
+  test "scan/2 with regexes with options compiled in different systems" do
+    assert Regex.scan(%{~r/foo/i | re_version: "bad version"}, "FOO") == [["FOO"]]
+  end
+
   test "split/2,3" do
     assert Regex.split(~r",", "") == [""]
     assert Regex.split(~r",", "", trim: true) == []

lib/mix/lib/mix/compilers/elixir.ex

@@ -520,10 +520,17 @@ defmodule Mix.Compilers.Elixir do
         split_manifest(data, compile_path)
 
       [v | data] when is_integer(v) ->
-        for module <- data, is_record(module, :module) do
-          File.rm(Path.join(compile_path, module(module, :beam)))
-          :code.purge(module(module, :module))
-          :code.delete(module(module, :module))
+        try do
+          for module <- data, is_record(module, :module) do
+            File.rm(Path.join(compile_path, module(module, :beam)))
+            :code.purge(module(module, :module))
+            :code.delete(module(module, :module))
+          end
+        rescue
+          _ ->
+            Mix.raise(
+              "Cannot clean-up stale manifest, please run \"mix clean --deps\" manually before proceeding"
+            )
         end
 
         {[], []}

lib/mix/lib/mix/release.ex

@@ -7,7 +7,8 @@ defmodule Mix.Release do
   The Mix.Release struct has the following read-only fields:
 
     * `:name` - the name of the release as an atom
-    * `:version` - the version of the release as a string
+    * `:version` - the version of the release as a string or
+       `{:from_app, app_name}
     * `:path` - the path to the release root
     * `:version_path` - the path to the release version inside the release
     * `:applications` - a map of application with their definitions
@@ -47,7 +48,7 @@ defmodule Mix.Release do
           name: atom(),
           version: String.t(),
           path: String.t(),
-          version_path: String.t(),
+          version_path: String.t() | {:from_app, application()},
           applications: %{application() => keyword()},
           boot_scripts: %{atom() => [{application(), mode()}]},
           erts_version: charlist(),
@@ -112,6 +113,24 @@ defmodule Mix.Release do
           )
       end)
 
+    version =
+      case version do
+        {:from_app, app} ->
+          Application.load(app)
+          version = Application.spec(app, :vsn)
+
+          if !version do
+            Mix.raise(
+              "Could not find version for #{inspect(app)}, please make sure the application exists"
+            )
+          end
+
+          to_string(version)
+
+        _ ->
+          version
+      end
+
     {config_providers, opts} = Keyword.pop(opts, :config_providers, [])
     {steps, opts} = Keyword.pop(opts, :steps, [:assemble])
     validate_steps!(steps)
@@ -409,7 +428,7 @@ defmodule Mix.Release do
     end
   end
 
-  defp random_cookie, do: Base.url_encode64(:crypto.strong_rand_bytes(40))
+  defp random_cookie, do: Base.encode32(:crypto.strong_rand_bytes(32))
 
   @doc """
   Makes the start_erl.data file with the

lib/mix/lib/mix/tasks/release.ex

@@ -386,8 +386,10 @@ defmodule Mix.Tasks.Release do
     * `:path` - the path the release should be installed to.
       Defaults to `"_build/MIX_ENV/rel/RELEASE_NAME"`.
 
-    * `:version` - the release version as a string. Defaults to the current
-      application version.
+    * `:version` - the release version as a string or `{:from_app, app_name}`.
+      Defaults to the current application version. The `{:from_app, app_name}` format
+      can be used to easily reference the application version from another application.
+      This is particularly useful in umbrella applications.
 
     * `:quiet` - a boolean that controls if releases should write to the standard
       output its steps. Defaults to `false`.

lib/mix/lib/mix/tasks/release.init.ex

@@ -285,7 +285,7 @@ defmodule Mix.Tasks.Release.Init do
       findstr "RUNTIME_CONFIG=true" "!RELEASE_SYS_CONFIG!.config" >nul 2>&1 && (
         for /f "skip=1" %%X in ('wmic os get localdatetime') do if not defined TIMESTAMP set TIMESTAMP=%%X
         set RELEASE_SYS_CONFIG=!RELEASE_TMP!\!RELEASE_NAME!-!RELEASE_VSN!-!TIMESTAMP:~0,11!-!RANDOM!.runtime
-        mkdir "!RELEASE_TMP!" >nul
+        mkdir "!RELEASE_TMP!" >nul 2>&1
         copy /y "!REL_VSN_DIR!\sys.config" "!RELEASE_SYS_CONFIG!.config" >nul || (
           echo Cannot start release because it could not write to "!RELEASE_SYS_CONFIG!.config"
           goto end
@@ -378,7 +378,8 @@ defmodule Mix.Tasks.Release.Init do
     )
 
     !ERLSRV! add !RELEASE_NAME!_!RELEASE_NAME! ^
-      -name "!RELEASE_NODE!" ^
+      -!RELEASE_DISTRIBUTION! "!RELEASE_NODE!" ^
+      -env RELEASE_ROOT=!RELEASE_ROOT! -env RELEASE_NAME=!RELEASE_NAME! -env RELEASE_VSN=!RELEASE_VSN! -env RELEASE_COOKIE=!RELEASE_COOKIE! -env RELEASE_NODE=!RELEASE_NODE! -env RELEASE_VM_ARGS=!RELEASE_VM_ARGS! -env RELEASE_TMP=!RELEASE_TMP! -env RELEASE_SYS_CONFIG=!RELEASE_SYS_CONFIG! ^
       -args "-setcookie !RELEASE_COOKIE! -config !RELEASE_SYS_CONFIG! -mode !RELEASE_MODE! -boot !REL_VSN_DIR!\start -boot_var RELEASE_LIB !RELEASE_ROOT!\lib -args_file !REL_VSN_DIR!\vm.args"
 
     if %ERRORLEVEL% EQU 0 (

lib/mix/test/mix/release_test.exs

@@ -50,6 +50,21 @@ defmodule Mix.ReleaseTest do
       assert release.options[:quiet]
     end
 
+    test "allows specifying the version from an application" do
+      overrides = [version: {:from_app, :elixir}]
+      release = from_config!(nil, config(), overrides)
+
+      assert release.version == to_string(Application.spec(:elixir, :vsn))
+    end
+
+    test "raises when :from_app is used with an app that doesn't exist" do
+      overrides = [version: {:from_app, :not_valid}]
+
+      assert_raise Mix.Error,
+                   ~r"Could not find version for :not_valid, please make sure the application exists",
+                   fn -> from_config!(nil, config(), overrides) end
+    end
+
     test "includes applications" do
       release = from_config!(nil, config(), [])
       assert release.applications.mix[:path] == to_charlist(Application.app_dir(:mix))

lib/mix/test/mix/tasks/release_test.exs

@@ -281,10 +281,11 @@ defmodule Mix.Tasks.ReleaseTest do
         open_port(script, ['start'])
         wait_until_decoded(Path.join(root, "RELEASE_BOOTED"))
         assert System.cmd(script, ["rpc", "ReleaseTest.hello_world"]) == {"hello world\n", 0}
-        assert System.cmd(script, ["stop"]) == {"", 0}
 
         assert {pid, 0} = System.cmd(script, ["pid"])
         assert pid != "\n"
+
+        assert System.cmd(script, ["stop"]) == {"", 0}
       end)
     end)
   end