diff --git a/lib/ex_unit/lib/ex_unit.ex b/lib/ex_unit/lib/ex_unit.ex
index d3e1d076b1..261c35aa3c 100644
--- a/lib/ex_unit/lib/ex_unit.ex
+++ b/lib/ex_unit/lib/ex_unit.ex
@@ -254,9 +254,6 @@ defmodule ExUnit do
       is reached. All tests within a module that fail when using the `setup_all/1,2` callbacks
       are counted as failures. Defaults to `:infinity`;
 
-    * `:module_load_timeout` - the timeout to be used when loading a test module in milliseconds,
-      defaults to `60_000`;
-
     * `:only_test_ids` - a list of `{module_name, test_name}` tuples that limits
       what tests get run;
 
diff --git a/lib/ex_unit/lib/ex_unit/server.ex b/lib/ex_unit/lib/ex_unit/server.ex
index 6eda96aeea..6b70d0a21a 100644
--- a/lib/ex_unit/lib/ex_unit/server.ex
+++ b/lib/ex_unit/lib/ex_unit/server.ex
@@ -1,6 +1,7 @@
 defmodule ExUnit.Server do
   @moduledoc false
   @name __MODULE__
+  @timeout :infinity
 
   use GenServer
 
@@ -8,26 +9,29 @@ defmodule ExUnit.Server do
     GenServer.start_link(__MODULE__, :ok, name: @name)
   end
 
-  def add_async_module(name) do
-    GenServer.cast(@name, {:add_async_module, name})
-  end
+  def add_async_module(name), do: add(name, :async)
+  def add_sync_module(name), do: add(name, :sync)
 
-  def add_sync_module(name) do
-    GenServer.cast(@name, {:add_sync_module, name})
+  defp add(name, type) do
+    case GenServer.call(@name, {:add, name, type}, @timeout) do
+      :ok ->
+        :ok
+
+      :already_running ->
+        raise "cannot add #{type} case named #{inspect(name)} to test suite after the suite starts running"
+    end
   end
 
   def modules_loaded do
-    GenServer.call(@name, :modules_loaded)
+    GenServer.call(@name, :modules_loaded, @timeout)
   end
 
   def take_async_modules(count) do
-    timeout = Application.fetch_env!(:ex_unit, :module_load_timeout)
-    GenServer.call(@name, {:take_async_modules, count}, timeout)
+    GenServer.call(@name, {:take_async_modules, count}, @timeout)
   end
 
   def take_sync_modules() do
-    timeout = Application.fetch_env!(:ex_unit, :module_load_timeout)
-    GenServer.call(@name, :take_sync_modules, timeout)
+    GenServer.call(@name, :take_sync_modules, @timeout)
   end
 
   ## Callbacks
@@ -59,16 +63,21 @@ defmodule ExUnit.Server do
     {:reply, diff, take_modules(%{state | loaded: :done})}
   end
 
-  def handle_cast({:add_async_module, name}, %{async_modules: modules, loaded: loaded} = state)
+  def handle_call({:add, name, :async}, _from, %{loaded: loaded} = state)
       when is_integer(loaded) do
-    {:noreply, take_modules(%{state | async_modules: [name | modules]})}
+    state = update_in(state.async_modules, &[name | &1])
+    {:reply, :ok, take_modules(state)}
   end
 
-  def handle_cast({:add_sync_module, name}, %{sync_modules: modules, loaded: loaded} = state)
+  def handle_call({:add, name, :sync}, _from, %{loaded: loaded} = state)
       when is_integer(loaded) do
-    {:noreply, %{state | sync_modules: [name | modules]}}
+    state = update_in(state.sync_modules, &[name | &1])
+    {:reply, :ok, state}
   end
 
+  def handle_call({:add, _name, _type}, _from, state),
+    do: {:reply, :already_running, state}
+
   defp take_modules(%{waiting: nil} = state) do
     state
   end
diff --git a/lib/ex_unit/mix.exs b/lib/ex_unit/mix.exs
index cf7736838d..a9199b3bac 100644
--- a/lib/ex_unit/mix.exs
+++ b/lib/ex_unit/mix.exs
@@ -21,7 +21,6 @@ defmodule ExUnit.MixProject do
         assert_receive_timeout: 100,
         autorun: true,
         capture_log: false,
-        module_load_timeout: 60000,
         colors: [],
         exclude: [],
         formatters: [ExUnit.CLIFormatter],