Release v1.4.2

Closes #5763.
Closes #5772.

Signed-off-by: José Valim <jose.valim@plataformatec.com.br>

CHANGELOG.md

@@ -109,6 +109,31 @@ You can now also install archives from Hex in this way. Since they are fetched a
 
 It is also possible to install escripts and archives by providing a Git/GitHub repo. See `mix help escript.install` and `mix help archive.install` for more details.
 
+## v1.4.2 (2017-02-16)
+
+### 1. Bug fixes
+
+#### EEx
+
+  * [EEx] Support middle expressions on trim mode
+
+#### Elixir
+
+  * [Calendar] Correct typo on Calendar types
+  * [Kernel] Ensure redefined functions point to the proper source
+  * [OptionParser] Add `:allow_inexistent_atoms` to support unsafe behaviour prior to v1.4
+  * [Stream] Allow consuming multiple items from suspended enumerable in `Stream.transform/3`
+  * [String] Incorporate new grapheme rules in Unicode 9
+
+#### IEx
+
+  * [IEx.Autocomplete] Do not crash on aliases which are not known at compile time
+
+#### Mix
+
+  * [Mix.Umbrella] Ensure umbrella projects can depend on other umbrella projects
+  * [Mix.Archive] Ensure previous archives with `.ez` extension are deleted
+
 ## v1.4.1 (2017-01-26)
 
 ### 1. Bug fixes

RELEASE.md

@@ -22,7 +22,7 @@ This document simply outlines the release process:
 
 9. Publish new zips with `make zips`, upload `Precompiled.zip` and `Docs.zip` to GitHub Releases
 
-10. Add the release to `elixir.csv` file in `elixir-lang/elixir-lang.github.com`
+10. Add the release to `elixir.csv` and `_data/elixir-versions.yml` files in `elixir-lang/elixir-lang.github.com`
 
 11. After a new `vMAJOR.MINOR`, move back to master, bump versions, start new CHANGELOG, add `-dev` back and commit "Start vMAJOR.MINOR+1"
 

VERSION

@@ -1 +1 @@
-1.4.1
+1.4.2

lib/eex/lib/eex/compiler.ex

@@ -90,7 +90,9 @@ defmodule EEx.Compiler do
       {contents, start, list}
     end
   end
-
+  defp look_ahead_text([{:middle_expr, line, _, chars} | t], _start, contents) do
+    {contents ++ chars, line, t}
+  end
   defp look_ahead_text(t, start, contents) do
     {contents, start, t}
   end

lib/eex/lib/eex/tokenizer.ex

@@ -181,7 +181,6 @@ defmodule EEx.Tokenizer do
   # If trim mode is enabled and the token is on a line with
   # only itself and whitespace, trim the whitespace around it,
   # including the line break following it if there is one.
-
   defp trim_if_needed(rest, line, opts, buffer, acc) do
     original = {rest, line, buffer}
     if opts[:trim] do

lib/eex/test/eex_test.exs

@@ -331,6 +331,19 @@ foo
     assert_eval expected, string, [], trim: true
   end
 
+  test "trim mode with middle expression" do
+    string = """
+    <%= cond do %>
+    <% false -> %>
+      this
+    <% true -> %>
+      that
+    <% end %>
+    """
+    expected = "  that\n"
+    assert_eval expected, string, [], trim: true
+  end
+
   test "evaluates the source from a given file" do
     filename = Path.join(__DIR__, "fixtures/eex_template.eex")
     result = EEx.eval_file(filename)

lib/elixir/lib/calendar.ex

@@ -57,13 +57,13 @@ defmodule Calendar do
   @type time :: %{optional(any) => any, hour: hour, minute: minute, second: second, microsecond: microsecond}
 
   @typedoc "Any map/struct that contains the naive_datetime fields"
-  @type naive_date_time :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day,
-                             hour: hour, minute: minute, second: second, microsecond: microsecond}
+  @type naive_datetime :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day,
+                            hour: hour, minute: minute, second: second, microsecond: microsecond}
 
   @typedoc "Any map/struct that contains the datetime fields"
-  @type date_time :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day,
-                       hour: hour, minute: minute, second: second, microsecond: microsecond,
-                       time_zone: time_zone, zone_abbr: zone_abbr, utc_offset: utc_offset, std_offset: std_offset}
+  @type datetime :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day,
+                      hour: hour, minute: minute, second: second, microsecond: microsecond,
+                      time_zone: time_zone, zone_abbr: zone_abbr, utc_offset: utc_offset, std_offset: std_offset}
 
   @doc """
   Returns how many days there are in the given year-month.

lib/elixir/lib/option_parser.ex

@@ -30,35 +30,33 @@ defmodule OptionParser do
   Elixir converts switches to underscored atoms, so `--source-path` becomes
   `:source_path`. This is done to better suit Elixir conventions. However, this
   means that switches can't contain underscores and switches that do contain
-  underscores are always returned in the list of invalid options.
+  underscores are always returned in the list of invalid switches.
 
-  Without any options, this function will try to parse all switches in the `argv`.
+  When parsing, it is common to list switches and their expected types:
 
-      iex> OptionParser.parse(["--debug"])
+      iex> OptionParser.parse(["--debug"], switches: [debug: :boolean])
       {[debug: true], [], []}
 
-      iex> OptionParser.parse(["--source", "lib"])
+      iex> OptionParser.parse(["--source", "lib"], switches: [source: :string])
       {[source: "lib"], [], []}
 
-      iex> OptionParser.parse(["--source-path", "lib", "test/enum_test.exs", "--verbose"])
+      iex> OptionParser.parse(["--source-path", "lib", "test/enum_test.exs", "--verbose"],
+      ...>                    switches: [source_path: :string, verbose: :boolean])
       {[source_path: "lib", verbose: true], ["test/enum_test.exs"], []}
 
-  Switches followed by a value will be assigned the value, as a string.
-  Switches without an argument, like `--debug` in the examples above, will
-  automatically be set to `true`.
+  We will explore the valid switches and operation modes of option parser below.
 
   ## Options
 
   The following options are supported:
 
     * `:switches` or `:strict` - see the "Switch definitions" section below
+    * `:allow_nonexistent_atoms` - see the "Parsing dynamic switches" section below
     * `:aliases` - see the "Aliases" section below
 
   ## Switch definitions
 
-  Often it is better to explicitly list the known
-  switches and their formats. The switches can be specified via one of two
-  options:
+  Switches can be specified via one of two options:
 
     * `:switches` - defines some switches and their types. This function
       still attempts to parse switches that are not in this list.
@@ -89,20 +87,20 @@ defmodule OptionParser do
     * `:float` - parses the value as a float
     * `:string` - parses the value as a string
 
-  If a switch can't be parsed according to the given type, it is returned
-  in the invalid options list.
+  If a switch can't be parsed according to the given type, it is
+  returned in the invalid options list.
 
   ### Modifiers
 
   Switches can be specified with modifiers, which change how
   they behave. The following modifiers are supported:
 
-    * `:keep` - keeps duplicated items instead of overriding them; works with
-      all types except `:count`. Specifying `switch_name: :keep` assumes the
-      type of `:switch_name` will be `:string`.
+    * `:keep` - keeps duplicated items instead of overriding them;
+      works with all types except `:count`. Specifying `switch_name: :keep`
+      assumes the type of `:switch_name` will be `:string`.
 
-  Note that if you want to use `:keep` with a type other than `:string`, use a list
-  as the type for the switch. For example: `[foo: [:integer, :keep]]`.
+  To use `:keep` with a type other than `:string`, use a list as the type
+  for the switch. For example: `[foo: [:integer, :keep]]`.
 
   ### Negation switches
 
@@ -112,6 +110,46 @@ defmodule OptionParser do
       iex> OptionParser.parse(["--no-op", "path/to/file"], switches: [op: :boolean])
       {[op: false], ["path/to/file"], []}
 
+  ### Parsing dynamic switches
+
+  `OptionParser` also includes a dynamic mode where it will attempt to parse
+  switches dynamically. Such can be done by not specifying the `:switches` or
+  `:strict` option.
+
+      iex> OptionParser.parse(["--debug"])
+      {[debug: true], [], []}
+
+
+  Switches followed by a value will be assigned the value, as a string. Switches
+  without an argument, like `--debug` in the examples above, will automatically be
+  set to `true`.
+
+  Since Elixir converts switches to atoms, the dynamic mode will only parse
+  switches that translates to atoms used by the runtime. Therefore, the code below
+  likely won't parse the given option since the `:option_parser_example` atom is
+  never used anywhere:
+
+      OptionParser.parse(["--option-parser-example"])
+      # Does nothing more...
+
+  However, the code below does since the `:option_parser_example` atom is used
+  at some point later (or earlier) on:
+
+      {opts, _, _} = OptionParser.parse(["--option-parser-example"])
+      opts[:option_parser_example]
+
+  In other words, when using dynamic mode, Elixir will do the correct thing and
+  only parse options that are used by the runtime, ignoring all others. If you
+  would like to parse all switches, regardless if they exist or not, you can
+  force creation of atoms by passing `allow_nonexistent_atoms: true` as option.
+  Such option is useful when you are building command-line applications that
+  receive dynamically-named arguments but must be used with care on long-running
+  systems.
+
+  Switches followed by a value will be assigned the value, as a string.
+  Switches without an argument, like `--debug` in the examples above, will
+  automatically be set to `true`.
+
   ## Aliases
 
   A set of aliases can be specified in the `:aliases` option:
@@ -203,10 +241,12 @@ defmodule OptionParser do
 
   ## Example
 
-      iex> OptionParser.parse_head(["--source", "lib", "test/enum_test.exs", "--verbose"])
+      iex> OptionParser.parse_head(["--source", "lib", "test/enum_test.exs", "--verbose"],
+      ...>                         switches: [source: :string, verbose: :boolean])
       {[source: "lib"], ["test/enum_test.exs", "--verbose"], []}
 
-      iex> OptionParser.parse_head(["--verbose", "--source", "lib", "test/enum_test.exs", "--unlock"])
+      iex> OptionParser.parse_head(["--verbose", "--source", "lib", "test/enum_test.exs", "--unlock"],
+      ...>                         switches: [source: :string, verbose: :boolean, unlock: :boolean])
       {[verbose: true, source: "lib"], ["test/enum_test.exs", "--unlock"], []}
 
   """
@@ -226,10 +266,12 @@ defmodule OptionParser do
 
   ## Examples
 
-      iex> OptionParser.parse_head!(["--source", "lib", "path/to/file", "--verbose"])
+      iex> OptionParser.parse_head!(["--source", "lib", "path/to/file", "--verbose"],
+      ...>                         switches: [source: :string, verbose: :boolean])
       {[source: "lib"], ["path/to/file", "--verbose"]}
 
-      iex> OptionParser.parse_head!(["--number", "lib", "test/enum_test.exs", "--verbose"], strict: [number: :integer])
+      iex> OptionParser.parse_head!(["--number", "lib", "test/enum_test.exs", "--verbose"],
+      ...>                          strict: [number: :integer])
       ** (OptionParser.ParseError) 1 error found!
       --number : Expected type integer, got "lib"
 
@@ -251,8 +293,8 @@ defmodule OptionParser do
     {Enum.reverse(opts), Enum.reverse(args), Enum.reverse(invalid)}
   end
 
-  defp do_parse(argv, {aliases, switches, strict?} = config, opts, args, invalid, all?) do
-    case next(argv, aliases, switches, strict?) do
+  defp do_parse(argv, {aliases, switches, strict?, allow_nonexistent_atoms?} = config, opts, args, invalid, all?) do
+    case next(argv, aliases, switches, strict?, allow_nonexistent_atoms?) do
       {:ok, option, value, rest} ->
         # the option exists and it was successfully parsed
         kinds = List.wrap Keyword.get(switches, option)
@@ -299,7 +341,6 @@ defmodule OptionParser do
     * `{:error, rest}` - there are no switches at the head of the given `argv`
 
   """
-
   @spec next(argv, options) ::
         {:ok, key :: atom, value :: term, argv} |
         {:invalid, String.t, String.t | nil, argv} |
@@ -307,35 +348,35 @@ defmodule OptionParser do
         {:error, argv}
 
   def next(argv, opts \\ []) when is_list(argv) and is_list(opts) do
-    {aliases, switches, strict?} = compile_config(opts)
-    next(argv, aliases, switches, strict?)
+    {aliases, switches, strict?, allow_nonexistent_atoms?} = compile_config(opts)
+    next(argv, aliases, switches, strict?, allow_nonexistent_atoms?)
   end
 
-  defp next([], _aliases, _switches, _strict?) do
+  defp next([], _aliases, _switches, _strict?, _allow_nonexistent_atoms?) do
     {:error, []}
   end
 
-  defp next(["--" | _] = argv, _aliases, _switches, _strict?) do
+  defp next(["--" | _] = argv, _aliases, _switches, _strict?, _allow_nonexistent_atoms?) do
     {:error, argv}
   end
 
-  defp next(["-" | _] = argv, _aliases, _switches, _strict?) do
+  defp next(["-" | _] = argv, _aliases, _switches, _strict?, _allow_nonexistent_atoms?) do
     {:error, argv}
   end
 
-  defp next(["- " <> _ | _] = argv, _aliases, _switches, _strict?) do
+  defp next(["- " <> _ | _] = argv, _aliases, _switches, _strict?, _allow_nonexistent_atoms?) do
     {:error, argv}
   end
 
   # Handles --foo or --foo=bar
-  defp next(["--" <> option | rest], _aliases, switches, strict?) do
+  defp next(["--" <> option | rest], _aliases, switches, strict?, allow_nonexistent_atoms?) do
     {option, value} = split_option(option)
-    tagged = tag_option(option, switches)
-    do_next(tagged, value, "--" <> option, rest, switches, strict?)
+    tagged = tag_option(option, switches, allow_nonexistent_atoms?)
+    do_next(tagged, value, "--" <> option, rest, switches, strict?, allow_nonexistent_atoms?)
   end
 
   # Handles -a, -abc, -abc=something
-  defp next(["-" <> option | rest] = argv, aliases, switches, strict?) do
+  defp next(["-" <> option | rest] = argv, aliases, switches, strict?, allow_nonexistent_atoms?) do
     {option, value} = split_option(option)
     original = "-" <> option
 
@@ -345,26 +386,26 @@ defmodule OptionParser do
       String.contains?(option, ["-", "_"]) ->
         {:undefined, original, value, rest}
       String.length(option) > 1 ->
-        key = get_option_key(option)
+        key = get_option_key(option, allow_nonexistent_atoms?)
         option_key = aliases[key]
         if key && option_key do
           IO.warn "multi-letter aliases are deprecated, got: #{inspect(key)}"
-          do_next({:default, option_key}, value, original, rest, switches, strict?)
+          do_next({:default, option_key}, value, original, rest, switches, strict?, allow_nonexistent_atoms?)
         else
-          next(expand_multiletter_alias(option, value) ++ rest, aliases, switches, strict?)
+          next(expand_multiletter_alias(option, value) ++ rest, aliases, switches, strict?, allow_nonexistent_atoms?)
         end
       true ->
         # We have a regular one-letter alias here
-        tagged = tag_oneletter_alias(option, aliases)
-        do_next(tagged, value, original, rest, switches, strict?)
+        tagged = tag_oneletter_alias(option, aliases, allow_nonexistent_atoms?)
+        do_next(tagged, value, original, rest, switches, strict?, allow_nonexistent_atoms?)
     end
   end
 
-  defp next(argv, _aliases, _switches, _strict?) do
+  defp next(argv, _aliases, _switches, _strict?, _allow_nonexistent_atoms?) do
     {:error, argv}
   end
 
-  defp do_next(tagged, value, original, rest, switches, strict?) do
+  defp do_next(tagged, value, original, rest, switches, strict?, allow_nonexistent_atoms?) do
     if strict? and not option_defined?(tagged, switches) do
       {:undefined, original, value, rest}
     else
@@ -489,6 +530,7 @@ defmodule OptionParser do
 
   defp compile_config(opts) do
     aliases = opts[:aliases] || []
+    allow_nonexistent_atoms? = opts[:allow_nonexistent_atoms] || false
 
     {switches, strict?} = cond do
       opts[:switches] && opts[:strict] ->
@@ -501,7 +543,7 @@ defmodule OptionParser do
         {[], false}
     end
 
-    {aliases, switches, strict?}
+    {aliases, switches, strict?, allow_nonexistent_atoms?}
   end
 
   defp validate_option(value, kinds) do
@@ -552,27 +594,27 @@ defmodule OptionParser do
     end
   end
 
-  defp tag_option("no-" <> option = original, switches) do
+  defp tag_option("no-" <> option = original, switches, allow_nonexistent_atoms?) do
     cond do
-      (negated = get_option_key(option)) && :boolean in List.wrap(switches[negated]) ->
+      (negated = get_option_key(option, allow_nonexistent_atoms?)) && :boolean in List.wrap(switches[negated]) ->
         {:negated, negated}
-      option_key = get_option_key(original) ->
+      option_key = get_option_key(original, allow_nonexistent_atoms?) ->
         {:default, option_key}
       true ->
         :unknown
     end
   end
 
-  defp tag_option(option, _switches) do
-    if option_key = get_option_key(option) do
+  defp tag_option(option, _switches, allow_nonexistent_atoms?) do
+    if option_key = get_option_key(option, allow_nonexistent_atoms?) do
       {:default, option_key}
     else
       :unknown
     end
   end
 
-  defp tag_oneletter_alias(alias, aliases) when is_binary(alias) do
-    if option_key = aliases[to_existing_key(alias)] do
+  defp tag_oneletter_alias(alias, aliases, allow_nonexistent_atoms?) when is_binary(alias) do
+    if option_key = aliases[to_existing_key(alias, allow_nonexistent_atoms?)] do
       {:default, option_key}
     else
       :unknown
@@ -662,13 +704,15 @@ defmodule OptionParser do
   defp to_underscore(<<>>, acc),
     do: acc
 
-  def get_option_key(option) do
+  def get_option_key(option, allow_nonexistent_atoms?) do
     if string = to_underscore(option) do
-      to_existing_key(string)
+      to_existing_key(string, allow_nonexistent_atoms?)
     end
   end
 
-  defp to_existing_key(option) do
+  defp to_existing_key(option, true),
+    do: String.to_atom(option)
+  defp to_existing_key(option, false) do
     try do
       String.to_existing_atom(option)
     rescue
@@ -702,7 +746,8 @@ defmodule OptionParser do
   end
 
   defp get_type(option, opts, types) do
-    key = option |> String.trim_leading("-") |> get_option_key()
+    allow_nonexistent_atoms? = opts[:allow_nonexistent_atoms] || false
+    key = option |> String.trim_leading("-") |> get_option_key(allow_nonexistent_atoms?)
 
     if option_key = opts[:aliases][key] do
       types[option_key]

lib/elixir/lib/stream.ex

@@ -789,16 +789,18 @@ defmodule Stream do
         do_after(after_fun, user_acc)
         :erlang.raise(kind, reason, stacktrace)
     else
-      {:suspended, [val], next} ->
-        do_transform_user(val, user_acc, user, fun, :cont, next, inner_acc, inner, after_fun)
-      {_, [val]} ->
-        do_transform_user(val, user_acc, user, fun, :halt, next, inner_acc, inner, after_fun)
-      {_, []} ->
-        do_transform(user_acc, user, fun, :halt, next, inner_acc, inner, after_fun)
+      {:suspended, vals, next} ->
+        do_transform_user(:lists.reverse(vals), user_acc, user, fun, :cont, next, inner_acc, inner, after_fun)
+      {_, vals} ->
+        do_transform_user(:lists.reverse(vals), user_acc, user, fun, :halt, next, inner_acc, inner, after_fun)
     end
   end
 
-  defp do_transform_user(val, user_acc, user, fun, next_op, next, inner_acc, inner, after_fun) do
+  defp do_transform_user([], user_acc, user, fun, next_op, next, inner_acc, inner, after_fun) do
+    do_transform(user_acc, user, fun, next_op, next, inner_acc, inner, after_fun)
+  end
+
+  defp do_transform_user([val | vals], user_acc, user, fun, next_op, next, inner_acc, inner, after_fun) do
     user.(val, user_acc)
   catch
     kind, reason ->
@@ -808,20 +810,20 @@ defmodule Stream do
       :erlang.raise(kind, reason, stacktrace)
   else
     {[], user_acc} ->
-      do_transform(user_acc, user, fun, next_op, next, inner_acc, inner, after_fun)
+      do_transform_user(vals, user_acc, user, fun, next_op, next, inner_acc, inner, after_fun)
     {list, user_acc} when is_list(list) ->
-      do_list_transform(user_acc, user, fun, next_op, next, inner_acc, inner,
+      do_list_transform(vals, user_acc, user, fun, next_op, next, inner_acc, inner,
                         &Enumerable.List.reduce(list, &1, fun), after_fun)
     {:halt, user_acc} ->
       next.({:halt, []})
       do_after(after_fun, user_acc)
       {:halted, elem(inner_acc, 1)}
     {other, user_acc} ->
-      do_enum_transform(user_acc, user, fun, next_op, next, inner_acc, inner,
+      do_enum_transform(vals, user_acc, user, fun, next_op, next, inner_acc, inner,
                         &Enumerable.reduce(other, &1, inner), after_fun)
   end
 
-  defp do_list_transform(user_acc, user, fun, next_op, next, inner_acc, inner, reduce, after_fun) do
+  defp do_list_transform(vals, user_acc, user, fun, next_op, next, inner_acc, inner, reduce, after_fun) do
     try do
       reduce.(inner_acc)
     catch
@@ -832,17 +834,17 @@ defmodule Stream do
         :erlang.raise(kind, reason, stacktrace)
     else
       {:done, acc} ->
-        do_transform(user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
+        do_transform_user(vals, user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
       {:halted, acc} ->
         next.({:halt, []})
         do_after(after_fun, user_acc)
         {:halted, acc}
       {:suspended, acc, c} ->
-        {:suspended, acc, &do_list_transform(user_acc, user, fun, next_op, next, &1, inner, c, after_fun)}
+        {:suspended, acc, &do_list_transform(vals, user_acc, user, fun, next_op, next, &1, inner, c, after_fun)}
     end
   end
 
-  defp do_enum_transform(user_acc, user, fun, next_op, next, {op, inner_acc}, inner, reduce, after_fun) do
+  defp do_enum_transform(vals, user_acc, user, fun, next_op, next, {op, inner_acc}, inner, reduce, after_fun) do
     try do
       reduce.({op, [:outer | inner_acc]})
     catch
@@ -855,15 +857,15 @@ defmodule Stream do
       # Only take into account outer halts when the op is not halt itself.
       # Otherwise, we were the ones wishing to halt, so we should just stop.
       {:halted, [:outer | acc]} when op != :halt ->
-        do_transform(user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
+        do_transform_user(vals, user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
       {:halted, [_ | acc]} ->
         next.({:halt, []})
         do_after(after_fun, user_acc)
         {:halted, acc}
       {:done, [_ | acc]} ->
-        do_transform(user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
+        do_transform_user(vals, user_acc, user, fun, next_op, next, {:cont, acc}, inner, after_fun)
       {:suspended, [_ | acc], c} ->
-        {:suspended, acc, &do_enum_transform(user_acc, user, fun, next_op, next, &1, inner, c, after_fun)}
+        {:suspended, acc, &do_enum_transform(vals, user_acc, user, fun, next_op, next, &1, inner, c, after_fun)}
     end
   end
 

lib/elixir/src/elixir_def.erl

@@ -303,12 +303,12 @@ store_each(Check, Kind, File, Location, Module, Defaults, {function, Ann, Name,
 
   {FinalAnn, FinalLocation, FinalDefaults} = case ets:take(Defs, {def, Tuple}) of
     [{_, StoredKind, StoredAnn, StoredFile, StoredCheck,
-        StoredLocation, {StoredDefaults, LastHasBody, LastDefaults}}] ->
+        _StoredLocation, {StoredDefaults, LastHasBody, LastDefaults}}] ->
       check_valid_kind(Ann, File, Name, Arity, Kind, StoredKind),
       (Check and StoredCheck) andalso
         check_valid_clause(Ann, File, Name, Arity, Kind, Data, StoredAnn, StoredFile),
       check_valid_defaults(Ann, File, Name, Arity, Kind, Defaults, StoredDefaults, LastDefaults, LastHasBody),
-      {StoredAnn, StoredLocation, {max(Defaults, StoredDefaults), HasBody, Defaults}};
+      {StoredAnn, Location, {max(Defaults, StoredDefaults), HasBody, Defaults}};
     [] ->
       {Ann, Location, {Defaults, HasBody, Defaults}}
   end,

lib/elixir/test/elixir/kernel/warning_test.exs

@@ -28,6 +28,31 @@ defmodule Kernel.WarningTest do
     purge Sample
   end
 
+  test "unused variable in redefined function in different file" do
+    output = capture_err(fn ->
+      Code.eval_string """
+      defmodule Sample do
+        defmacro __using__(_) do
+          quote location: :keep do
+            def function(arg)
+          end
+        end
+      end
+      """
+      Code.eval_string("""
+      defmodule RedefineSample do
+        use Sample
+        def function(var123), do: nil
+      end
+      """, [], file: "redefine_sample.ex")
+    end)
+    assert output =~ "redefine_sample.ex:3"
+    assert output =~ "variable \"var123\" is unused"
+  after
+    purge Sample
+    purge RedefineSample
+  end
+
   test "useless literal" do
     message = "code block contains unused literal \"oops\""
 

lib/elixir/test/elixir/option_parser_test.exs

@@ -256,6 +256,11 @@ defmodule OptionParserTest do
            == {[foo: true, boo: "-"], ["-"], []}
   end
 
+  test "allow nonexistent atoms" do
+    assert OptionParser.parse(["--option-key-creates-atom"], allow_nonexistent_atoms: true) ==
+           {[{String.to_atom("option_key_creates_atom"), true}], [], []}
+  end
+
   test "correctly handles negative integers" do
     assert OptionParser.parse(["arg1", "-43"])
       == {[], ["arg1", "-43"], []}

lib/elixir/test/elixir/stream_test.exs

@@ -481,6 +481,14 @@ defmodule StreamTest do
     assert Process.get(:stream_transform)
   end
 
+  test "transform/3 (via flat_map) handles multiple returns from suspension" do
+    assert [false]
+           |> Stream.take(1)
+           |> Stream.concat([true])
+           |> Stream.flat_map(&[&1])
+           |> Enum.to_list() == [false, true]
+  end
+
   test "iterate/2" do
     stream = Stream.iterate(0, &(&1+2))
     assert Enum.take(stream, 5) == [0, 2, 4, 6, 8]

lib/elixir/test/elixir/string_test.exs

@@ -449,7 +449,8 @@ defmodule StringTest do
     # CLRF
     assert String.graphemes("\r\n\f") == ["\r\n", "\f"]
     # Regional indicator
-    assert String.graphemes("\u{1F1E6}\u{1F1E7}\u{1F1E8}") == ["\u{1F1E6}\u{1F1E7}\u{1F1E8}"]
+    assert String.graphemes("\u{1F1E6}\u{1F1E7}") == ["\u{1F1E6}\u{1F1E7}"]
+    assert String.graphemes("\u{1F1E6}\u{1F1E7}\u{1F1E8}") == ["\u{1F1E6}\u{1F1E7}", "\u{1F1E8}"]
     # Hangul
     assert String.graphemes("\u1100\u115D\uB4A4") == ["ᄀᅝ뒤"]
     # Special Marking with Extended

lib/elixir/unicode/unicode.ex

@@ -52,20 +52,6 @@ defmodule String.Unicode do
     end
   end
 
-  # Handle Hangul L
-  for codepoint <- cluster["L"] do
-    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
-      next_hangul_l_size(rest, unquote(byte_size(codepoint)))
-    end
-  end
-
-  # Handle Hangul T
-  for codepoint <- cluster["T"] do
-    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
-      next_hangul_t_size(rest, unquote(byte_size(codepoint)))
-    end
-  end
-
   # Handle Regional
   for codepoint <- cluster["Regional_Indicator"] do
     def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
@@ -73,14 +59,48 @@ defmodule String.Unicode do
     end
   end
 
-  # Handle extended entries
+  # Handle Hangul L
+  for codepoint <- cluster["L"] do
+    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
+      next_hangul_l_size(rest, unquote(byte_size(codepoint)))
+    end
+  end
 
+  # Handle Hangul V
+  for codepoint <- cluster["LV"] ++ cluster["V"]  do
+    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
+      next_hangul_v_size(rest, unquote(byte_size(codepoint)))
+    end
+  end
+
+  # Handle Hangul T
+  for codepoint <- cluster["LVT"] ++ cluster["T"] do
+    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
+      next_hangul_t_size(rest, unquote(byte_size(codepoint)))
+    end
+  end
+
+  # Handle E_Base
+  for codepoint <- cluster["E_Base"] ++ cluster["E_Base_GAZ"] do
+    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
+      next_extend_size(rest, unquote(byte_size(codepoint)), :e_base)
+    end
+  end
+
+  # Handle ZWJ
+  for codepoint <- cluster["ZWJ"] do
+    def next_grapheme_size(<<unquote(codepoint), rest::binary>>) do
+      next_extend_size(rest, unquote(byte_size(codepoint)), :zwj)
+    end
+  end
+
+  # Handle extended entries
   def next_grapheme_size(<<cp::utf8, rest::binary>>) do
     case cp do
-      x when x <= 0x007F -> next_extend_size(rest, 1)
-      x when x <= 0x07FF -> next_extend_size(rest, 2)
-      x when x <= 0xFFFF -> next_extend_size(rest, 3)
-      _                  -> next_extend_size(rest, 4)
+      x when x <= 0x007F -> next_extend_size(rest, 1, :other)
+      x when x <= 0x07FF -> next_extend_size(rest, 2, :other)
+      x when x <= 0xFFFF -> next_extend_size(rest, 3, :other)
+      _                  -> next_extend_size(rest, 4, :other)
     end
   end
 
@@ -92,82 +112,139 @@ defmodule String.Unicode do
     nil
   end
 
-  # Handle Hangul L
-  for codepoint <- cluster["L"] do
-    defp next_hangul_l_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_hangul_l_size(rest, size + unquote(byte_size(codepoint)))
-    end
-  end
-
-  for codepoint <- cluster["LV"] do
-    defp next_hangul_l_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_hangul_v_size(rest, size + unquote(byte_size(codepoint)))
-    end
-  end
-
-  for codepoint <- cluster["LVT"] do
-    defp next_hangul_l_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_hangul_t_size(rest, size + unquote(byte_size(codepoint)))
-    end
-  end
-
+  # Handle hanguls
   defp next_hangul_l_size(rest, size) do
-    next_hangul_v_size(rest, size)
-  end
-
-  # Handle Hangul V
-  for codepoint <- cluster["V"] do
-    defp next_hangul_v_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_hangul_v_size(rest, size + unquote(byte_size(codepoint)))
+    case next_hangul(rest, size) do
+      {:l, rest, size} -> next_hangul_l_size(rest, size)
+      {:v, rest, size} -> next_hangul_v_size(rest, size)
+      {:lv, rest, size} -> next_hangul_v_size(rest, size)
+      {:lvt, rest, size} -> next_hangul_t_size(rest, size)
+      _ -> next_extend_size(rest, size, :other)
     end
   end
 
   defp next_hangul_v_size(rest, size) do
-    next_hangul_t_size(rest, size)
-  end
-
-  # Handle Hangul T
-  for codepoint <- cluster["T"] do
-    defp next_hangul_t_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_hangul_t_size(rest, size + unquote(byte_size(codepoint)))
+    case next_hangul(rest, size) do
+      {:v, rest, size} -> next_hangul_v_size(rest, size)
+      {:t, rest, size} -> next_hangul_t_size(rest, size)
+      _ -> next_extend_size(rest, size, :other)
     end
   end
 
   defp next_hangul_t_size(rest, size) do
-    next_extend_size(rest, size)
+    case next_hangul(rest, size) do
+      {:t, rest, size} -> next_hangul_t_size(rest, size)
+      _ -> next_extend_size(rest, size, :other)
+    end
+  end
+
+  for codepoint <- cluster["L"] do
+    defp next_hangul(<<unquote(codepoint), rest::binary>>, size) do
+      {:l, rest, size + unquote(byte_size(codepoint))}
+    end
+  end
+
+  for codepoint <- cluster["V"] do
+    defp next_hangul(<<unquote(codepoint), rest::binary>>, size) do
+      {:v, rest, size + unquote(byte_size(codepoint))}
+    end
+  end
+
+  for codepoint <- cluster["T"] do
+    defp next_hangul(<<unquote(codepoint), rest::binary>>, size) do
+      {:t, rest, size + unquote(byte_size(codepoint))}
+    end
+  end
+
+  for codepoint <- cluster["LV"] do
+    defp next_hangul(<<unquote(codepoint), rest::binary>>, size) do
+      {:lv, rest, size + unquote(byte_size(codepoint))}
+    end
+  end
+
+  for codepoint <- cluster["LVT"] do
+    defp next_hangul(<<unquote(codepoint), rest::binary>>, size) do
+      {:lvt, rest, size + unquote(byte_size(codepoint))}
+    end
+  end
+
+  defp next_hangul(_, _) do
+    false
   end
 
   # Handle regional
   for codepoint <- cluster["Regional_Indicator"] do
     defp next_regional_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_regional_size(rest, size + unquote(byte_size(codepoint)))
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :other)
     end
   end
-
   defp next_regional_size(rest, size) do
-    next_extend_size(rest, size)
+    next_extend_size(rest, size, :other)
   end
 
-  # Handle Extend+SpacingMark
-  for codepoint <- cluster["Extend"] ++ cluster["SpacingMark"]  do
-    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size) do
-      next_extend_size(rest, size + unquote(byte_size(codepoint)))
+  # Handle Extend+SpacingMark+ZWJ
+  for codepoint <- cluster["Extend"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, marker) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), keep_ebase(marker))
     end
   end
 
-  defp next_extend_size(rest, size) do
+  for codepoint <- cluster["SpacingMark"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, _marker) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :other)
+    end
+  end
+
+  for codepoint <- cluster["ZWJ"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, _marker) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :zwj)
+    end
+  end
+
+  for codepoint <- cluster["E_Modifier"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, :e_base) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :other)
+    end
+  end
+
+  for codepoint <- cluster["Glue_After_Zwj"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, :zwj) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :other)
+    end
+  end
+
+  for codepoint <- cluster["E_Base_GAZ"] do
+    defp next_extend_size(<<unquote(codepoint), rest::binary>>, size, :zwj) do
+      next_extend_size(rest, size + unquote(byte_size(codepoint)), :e_base)
+    end
+  end
+
+  defp next_extend_size(rest, size, _) do
     {size, rest}
   end
 
+  defp keep_ebase(:e_base), do: :e_base
+  defp keep_ebase(_), do: :other
+
   # Handle Prepend
   for codepoint <- cluster["Prepend"] do
     defp next_prepend_size(<<unquote(codepoint), rest::binary>>, size) do
       next_prepend_size(rest, size + unquote(byte_size(codepoint)))
     end
   end
-  
+
+  # However, if we see a control character, we have to break it
+  for codepoint <- cluster["CR"] ++ cluster["LF"] ++ cluster["Control"] do
+    defp next_prepend_size(<<unquote(codepoint), _::binary>> = rest, size) do
+      {size, rest}
+    end
+  end
+
   defp next_prepend_size(rest, size) do
-    {size, rest}
+    case next_grapheme_size(rest) do
+      {more, rest} ->  {more + size, rest}
+      nil -> {size, rest}
+    end
   end
 
   # Graphemes

lib/iex/lib/iex/autocomplete.ex

@@ -115,9 +115,10 @@ defmodule IEx.Autocomplete do
 
   # Elixir.fun
   defp expand_call({:__aliases__, _, list}, hint, server) do
-    expand_alias(list, server)
-    |> normalize_module
-    |> expand_require(hint)
+    case expand_alias(list, server) do
+      {:ok, alias} -> expand_require(alias, hint)
+      :error -> no()
+    end
   end
 
   # variable.fun_or_key
@@ -171,9 +172,10 @@ defmodule IEx.Autocomplete do
   end
 
   defp expand_elixir_modules(list, hint, server) do
-    expand_alias(list, server)
-    |> normalize_module
-    |> expand_elixir_modules_from_aliases(hint, [])
+    case expand_alias(list, server) do
+      {:ok, alias} -> expand_elixir_modules_from_aliases(alias, hint, [])
+      :error -> no()
+    end
   end
 
   defp expand_elixir_modules_from_aliases(mod, hint, aliases) do
@@ -183,19 +185,16 @@ defmodule IEx.Autocomplete do
     |> format_expansion(hint)
   end
 
-  defp expand_alias([name | rest] = list, server) do
-    module = Module.concat(Elixir, name)
-    Enum.find_value aliases_from_env(server), list, fn {alias, mod} ->
-      if alias === module do
-        case Atom.to_string(mod) do
-          "Elixir." <> mod ->
-            Module.concat [mod | rest]
-          _ ->
-            mod
-        end
-      end
+  defp expand_alias([name | rest], server) when is_atom(name) do
+    case Keyword.fetch(aliases_from_env(server), Module.concat(Elixir, name)) do
+      {:ok, name} when rest == [] -> {:ok, name}
+      {:ok, name} -> {:ok, Module.concat([name | rest])}
+      :error -> {:ok, Module.concat([name | rest])}
     end
   end
+  defp expand_alias([_ | _], _) do
+    :error
+  end
 
   defp match_aliases(hint, server) do
     for {alias, _mod} <- aliases_from_env(server),
@@ -220,14 +219,6 @@ defmodule IEx.Autocomplete do
 
   ## Helpers
 
-  defp normalize_module(mod) do
-    if is_list(mod) do
-      Module.concat(mod)
-    else
-      mod
-    end
-  end
-
   defp match_modules(hint, root) do
     get_modules(root)
     |> :lists.usort()

lib/iex/test/iex/autocomplete_test.exs

@@ -90,6 +90,7 @@ defmodule IEx.AutocompleteTest do
     assert expand('.')   == {:no, '', []}
     assert expand('Xyz') == {:no, '', []}
     assert expand('x.Foo') == {:no, '', []}
+    assert expand('x.Foo.get_by') == {:no, '', []}
   end
 
   test "Elixir root submodule completion" do

lib/mix/lib/mix/project.ex

@@ -233,9 +233,10 @@ defmodule Mix.Project do
   @spec apps_paths() :: %{atom => Path.t} | nil
   def apps_paths(config \\ config()) do
     if apps_path = config[:apps_path] do
-      Mix.ProjectStack.read_cache(:apps_path) ||
-        Mix.ProjectStack.write_cache(:apps_path,
-          config[:apps] |> umbrella_apps(apps_path) |> to_apps_path(apps_path))
+      key = {:apps_paths, Mix.Project.get!}
+      Mix.ProjectStack.read_cache(key) ||
+        Mix.ProjectStack.write_cache(key,
+          config[:apps] |> umbrella_apps(apps_path) |> to_apps_paths(apps_path))
     end
   end
 
@@ -249,7 +250,7 @@ defmodule Mix.Project do
     apps
   end
 
-  defp to_apps_path(apps, apps_path) do
+  defp to_apps_paths(apps, apps_path) do
     for app <- apps,
         path = path_with_mix_exs_otherwise_warn(app, apps_path),
         do: {app, path},

lib/mix/lib/mix/project_stack.ex

@@ -18,7 +18,7 @@ defmodule Mix.ProjectStack do
     get_and_update fn %{stack: stack} = state ->
       # Consider the first children to always have io_done
       # because we don't need to print anything unless another
-      # project talks ahold of the shell.
+      # project takes ahold of the shell.
       io_done? = stack == []
 
       config  = Keyword.merge(config, state.post_config)

lib/mix/lib/mix/tasks/archive.install.ex

@@ -106,7 +106,7 @@ defmodule Mix.Tasks.Archive.Install do
 
   defp archives(name) do
     Mix.Local.path_for(:archive)
-    |> Path.join(name)
+    |> Path.join(name <> "{,*.ez}")
     |> Path.wildcard
   end
 

src/elixir.app.src

@@ -1,6 +1,6 @@
 {application, elixir,
 [{description, "elixir"},
- {vsn, "1.4.1"},
+ {vsn, "1.4.2"},
  {modules, [
 	elixir
   ]},