From b962ae3e5fda2fd5b1ca28a8aa307657d33e9101 Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Mon, 23 Feb 2015 12:00:25 +0100 Subject: [PATCH] Replace `String.levenshtein_distance/2` with `String.jaro_distance/2` --- lib/elixir/lib/string.ex | 107 ++++++++++++++++++------- lib/elixir/test/elixir/string_test.exs | 59 ++++++-------- lib/mix/lib/mix/exceptions.ex | 2 +- 3 files changed, 102 insertions(+), 66 deletions(-) diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index df7b3b9114..a32f4bdd5f 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -1441,50 +1441,95 @@ defmodule String do end @doc """ - Returns an integer representing Levenshtein distance between - `source` and `target`. + Returns a float value between 0 (equates to no similarity) and 1 (is an exact match) + representing [Jaro](http://en.wikipedia.org/wiki/Jaro–Winkler_distance) + distance between `str1` and `str2`. - Levenshtein distance between two words is the minimum number of - single-character edits (i.e. insertions, deletions or substitutions) - required to change one word into the other. - - The algorithm used is based on the [Wagner-Fisher iterative - implementation](http://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows). + The Jaro distance metric is designed and best suited for short strings such as person names. ## Examples - iex> String.levenshtein_distance("kitten", "sitting") - 3 + iex> String.jaro_distance("dwayne", "duane") + 0.8222222222222223 + iex> String.jaro_distance("even", "odd") + 0.0 + """ - @spec levenshtein_distance(t, t) :: non_neg_integer - def levenshtein_distance(source, target) - def levenshtein_distance(source, source), do: 0 + @spec jaro_distance(t, t) :: 0..1 - def levenshtein_distance(source, <<>>), do: length(source) + def jaro_distance(str, str), do: 1.0 + def jaro_distance(_str, ""), do: 0.0 + def jaro_distance("", _str), do: 0.0 - def levenshtein_distance(<<>>, target), do: length(target) + def jaro_distance(str1, str2) do + {chars1, len1} = decompose(str1) + {chars2, len2} = decompose(str2) - def levenshtein_distance(source, target) do - source = graphemes(source) - target = graphemes(target) - distlist = 0..Kernel.length(target) |> Enum.to_list - do_distance(source, target, distlist, 1) + case match(chars1, len1, chars2, len2) do + {0, _trans} -> 0.0 + {comm, trans} -> + ((comm / len1) + + (comm / len2) + + ((comm - trans) / comm)) / 3 + end end - defp do_distance([], _, distlist, _), do: List.last(distlist) - - defp do_distance([src_hd | src_tl], target, distlist, step) do - distlist = distlist(target, distlist, src_hd, [step], step) - do_distance(src_tl, target, distlist, step + 1) + @compile {:inline, decompose: 1} + defp decompose(str) do + chars = graphemes(str) + {chars, Kernel.length(chars)} end - defp distlist([], _, _, new_distlist, _), do: Enum.reverse(new_distlist) + defp match(chars1, len1, chars2, len2) do + if len1 < len2 do + match(chars1, chars2, div(len2, 2) - 1) + else + match(chars2, chars1, div(len1, 2) - 1) + end + end - defp distlist([target_hd | target_tl], [distlist_hd | distlist_tl], - grapheme, new_distlist, last_dist) do - diff = if target_hd != grapheme, do: 1, else: 0 - min = min(min(last_dist + 1, hd(distlist_tl) + 1), distlist_hd + diff) - distlist(target_tl, distlist_tl, grapheme, [min | new_distlist], min) + defp match(chars1, chars2, lim) do + match(chars1, chars2, {0, lim}, {0, 0, -1}, 0) + end + + defp match([char | rest], chars, range, state, idx) do + {chars, state} = submatch(char, chars, range, state, idx) + + case range do + {lim, lim} -> match(rest, tl(chars), range, state, idx + 1) + {pre, lim} -> match(rest, chars, {pre + 1, lim}, state, idx + 1) + end + end + + defp match([], _, _, {comm, trans, _}, _), do: {comm, trans} + + defp submatch(char, chars, {pre, _} = range, state, idx) do + case detect(char, chars, range) do + nil -> {chars, state} + {subidx, chars} -> + {chars, proceed(state, idx - pre + subidx)} + end + end + + defp detect(char, chars, {pre, lim}) do + detect(char, chars, pre + 1 + lim, 0, []) + end + + defp detect(_char, _chars, 0, _idx, _acc), do: nil + defp detect(_char, [], _lim, _idx, _acc), do: nil + + defp detect(char, [char | rest], _lim, idx, acc), + do: {idx, Enum.reverse(acc, [nil | rest])} + + defp detect(char, [other | rest], lim, idx, acc), + do: detect(char, rest, lim - 1, idx + 1, [other | acc]) + + defp proceed({comm, trans, former}, current) do + if current < former do + {comm + 1, trans + 1, current} + else + {comm + 1, trans, current} + end end end diff --git a/lib/elixir/test/elixir/string_test.exs b/lib/elixir/test/elixir/string_test.exs index 58811d4795..cd28b107b2 100644 --- a/lib/elixir/test/elixir/string_test.exs +++ b/lib/elixir/test/elixir/string_test.exs @@ -517,39 +517,30 @@ defmodule StringTest do assert_raise ArgumentError, fn -> String.to_float(three.()) end end - test :levenshtein_distance do - # With empty strings - assert String.levenshtein_distance( "", "") == 0 - assert String.levenshtein_distance( "a", "") == 1 - assert String.levenshtein_distance( "", "a") == 1 - assert String.levenshtein_distance("abc", "") == 3 - assert String.levenshtein_distance( "", "abc") == 3 - # With equal strings - assert String.levenshtein_distance( "a", "a") == 0 - assert String.levenshtein_distance("abc", "abc") == 0 - assert String.levenshtein_distance( "", "a") == 1 - # Only with inserts - assert String.levenshtein_distance( "a", "ab") == 1 - assert String.levenshtein_distance( "b", "ab") == 1 - assert String.levenshtein_distance( "ac", "abc") == 1 - assert String.levenshtein_distance("abcdefg", "xabxcdxxefxgx") == 6 - # Only with deletions - assert String.levenshtein_distance( "a", "") == 1 - assert String.levenshtein_distance( "ab", "a") == 1 - assert String.levenshtein_distance( "ab", "b") == 1 - assert String.levenshtein_distance("abc", "ac") == 1 - assert String.levenshtein_distance("xabxcdxxefxgx", "abcdefg") == 6 - # Only with substitutions - assert String.levenshtein_distance( "a", "b") == 1 - assert String.levenshtein_distance( "ab", "ac") == 1 - assert String.levenshtein_distance( "ac", "bc") == 1 - assert String.levenshtein_distance("abc", "axc") == 1 - assert String.levenshtein_distance("xabxcdxxefxgx", "1ab2cd34ef5g6") == 6 - # With different operations - assert String.levenshtein_distance("example", "samples") == 3 - assert String.levenshtein_distance("sturgeon", "urgently") == 6 - assert String.levenshtein_distance("levenshtein", "frankenstein") == 6 - assert String.levenshtein_distance("distance", "difference") == 5 - assert String.levenshtein_distance("erlang was neat", "elixir is great") == 9 + test :jaro_distance do + assert String.jaro_distance("same", "same") == 1.0 + assert String.jaro_distance("any", "") == 0.0 + assert String.jaro_distance("", "any") == 0.0 + assert String.jaro_distance("martha", "marhta") == 0.9444444444444445 + assert String.jaro_distance("martha", "marhha") == 0.888888888888889 + assert String.jaro_distance("marhha", "martha") == 0.888888888888889 + assert String.jaro_distance("dwayne", "duane") == 0.8222222222222223 + assert String.jaro_distance("dixon", "dicksonx") == 0.7666666666666666 + assert String.jaro_distance("xdicksonx", "dixon") == 0.7851851851851852 + assert String.jaro_distance("shackleford", "shackelford") == 0.9696969696969697 + assert String.jaro_distance("dunningham", "cunnigham") == 0.8962962962962964 + assert String.jaro_distance("nichleson", "nichulson") == 0.9259259259259259 + assert String.jaro_distance("jones", "johnson") == 0.7904761904761904 + assert String.jaro_distance("massey", "massie") == 0.888888888888889 + assert String.jaro_distance("abroms", "abrams") == 0.888888888888889 + assert String.jaro_distance("hardin", "martinez") == 0.7222222222222222 + assert String.jaro_distance("itman", "smith") == 0.4666666666666666 + assert String.jaro_distance("jeraldine", "geraldine") == 0.9259259259259259 + assert String.jaro_distance("michelle", "michael") == 0.8690476190476191 + assert String.jaro_distance("julies", "julius") == 0.888888888888889 + assert String.jaro_distance("tanya", "tonya") == 0.8666666666666667 + assert String.jaro_distance("sean", "susan") == 0.7833333333333333 + assert String.jaro_distance("jon", "john") == 0.9166666666666666 + assert String.jaro_distance("jon", "jan") == 0.7777777777777777 end end diff --git a/lib/mix/lib/mix/exceptions.ex b/lib/mix/lib/mix/exceptions.ex index 6d7b10ee40..9b4c9f1be0 100644 --- a/lib/mix/lib/mix/exceptions.ex +++ b/lib/mix/lib/mix/exceptions.ex @@ -22,7 +22,7 @@ defmodule Mix.NoTaskError do end defp similar?(source, target) do - String.levenshtein_distance(source, target) < 2 + String.jaro_distance(source, target) > 0.77 end end