Run tests in async mode where applicable (#2542)

* Set pg pool size for MIX_ENV=test

* Include slow tests in CI run

* Exclude slow tests by default

* Mark tests slow/async where applicable

* Restructure captcha mocks

* Revert async where env is relied upon

* Add --max-failures=1 to CI run

* Set warnings as errors

* Disable async where various mocks are used

* Revert "Disable async where various mocks are used"

This reverts commit 2446b72a29.

* Disable async for test using vcr

.github/workflows/elixir.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Check Credo Warnings
         run: mix credo diff --from-git-merge-base origin/master
       - name: Run tests
-        run: mix test
+        run: mix test --include slow --max-failures 1 --warnings-as-errors
       - name: Check Dialyzer
         run: mix dialyzer
 env:

config/.env.test

@@ -1,4 +1,4 @@
-DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_test
+DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/plausible_test?pool_size=40
 CLICKHOUSE_DATABASE_URL=http://127.0.0.1:8123/plausible_test
 SECRET_KEY_BASE=/njrhntbycvastyvtk1zycwfm981vpo/0xrvwjjvemdakc/vsvbrevlwsc6u8rcg
 BASE_URL=http://localhost:8000

test/plausible/google/api_test.exs

@@ -1,5 +1,5 @@
 defmodule Plausible.Google.ApiTest do
-  use Plausible.DataCase, async: true
+  use Plausible.DataCase, async: false
   use ExVCR.Mock, adapter: ExVCR.Adapter.Finch
   alias Plausible.Google.Api
 
@@ -25,6 +25,7 @@ defmodule Plausible.Google.ApiTest do
                     |> Enum.map(&File.read!/1)
                     |> Enum.map(&Jason.decode!/1)
 
+  @tag :slow
   test "import_analytics/4 refreshes OAuth token when needed", %{site: site} do
     past = DateTime.add(DateTime.utc_now(), -3600, :second)
     auth = {"redacted_access_token", "redacted_refresh_token", DateTime.to_iso8601(past)}

test/plausible/google/buffer_test.exs

@@ -39,6 +39,7 @@ defmodule Plausible.Google.BufferTest do
     assert imported_count(site, "imported_visitors") == 10, "expected to have flushed"
   end
 
+  @tag :slow
   test "insert_many/3 uses separate buffers for each table", %{site: site} do
     {:ok, pid} = Buffer.start_link()
 
@@ -72,6 +73,7 @@ defmodule Plausible.Google.BufferTest do
     assert imported_count(site, "imported_visitors") == 50, "expected to have flushed"
   end
 
+  @tag :slow
   test "flush/2 flushes all buffers", %{site: site} do
     {:ok, pid} = Buffer.start_link()
 

test/plausible/google/vcr_test.exs

@@ -7,6 +7,7 @@ defmodule Plausible.Google.Api.VCRTest do
   # We need real HTTP Client for VCR tests
   setup_patch_env(:http_impl, Plausible.HTTPClient)
 
+  @tag :slow
   test "imports page views from Google Analytics", %{site: site} do
     use_cassette "google_analytics_import#1", match_requests_on: [:request_body] do
       view_id = "54297898"

test/plausible_web/controllers/api/external_controller_test.exs

@@ -97,11 +97,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
         screen_width: 1440
       }
 
+      t1 = System.convert_time_unit(System.monotonic_time(), :native, :millisecond)
+
       conn
       |> put_req_header("user-agent", @user_agent)
       |> post("/api/event", params)
 
-      :timer.sleep(1500)
+      t2 = System.convert_time_unit(System.monotonic_time(), :native, :millisecond)
+
+      # timestamps are in second precision, so we need to wait for it to flip
+      Process.sleep(1000 - (t2 - t1))
 
       conn
       |> put_req_header("user-agent", @user_agent)

test/plausible_web/controllers/api/external_sites_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: false
   use Plausible.Repo
 
   setup %{conn: conn} do
@@ -62,7 +62,7 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     end
 
     test "does not allow creating more sites than the limit", %{conn: conn, user: user} do
-      Application.put_env(:plausible, :site_limit, 3)
+      patch_env(:site_limit, 3)
       insert(:site, members: [user])
       insert(:site, members: [user])
       insert(:site, members: [user])

test/plausible_web/controllers/api/internal_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.Api.InternalControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
   use Plausible.Repo
 
   describe "GET /api/:domain/status" do

test/plausible_web/controllers/api/paddle_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.Api.PaddleControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
   use Plausible.Repo
 
   @body %{

test/plausible_web/controllers/auth_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.AuthControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
   use Bamboo.Test
   use Plausible.Repo
 
@@ -15,9 +15,12 @@ defmodule PlausibleWeb.AuthControllerTest do
   end
 
   describe "POST /register" do
-    test "registering sends an activation link", %{conn: conn} do
+    setup do
       mock_captcha_success()
+      :ok
+    end
 
+    test "registering sends an activation link", %{conn: conn} do
       post(conn, "/register",
         user: %{
           name: "Jane Doe",
@@ -33,8 +36,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "user is redirected to activate page after registration", %{conn: conn} do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register",
           user: %{
@@ -49,8 +50,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "creates user record", %{conn: conn} do
-      mock_captcha_success()
-
       post(conn, "/register",
         user: %{
           name: "Jane Doe",
@@ -65,8 +64,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "logs the user in", %{conn: conn} do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register",
           user: %{
@@ -81,8 +78,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "user is redirected to activation after registration", %{conn: conn} do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register",
           user: %{
@@ -95,22 +90,6 @@ defmodule PlausibleWeb.AuthControllerTest do
 
       assert redirected_to(conn) == "/activate"
     end
-
-    test "renders captcha errors in case of captcha input verification failure", %{conn: conn} do
-      mock_captcha_failure()
-
-      conn =
-        post(conn, "/register",
-          user: %{
-            name: "Jane Doe",
-            email: "user@example.com",
-            password: "very-secret",
-            password_confirmation: "very-secret"
-          }
-        )
-
-      assert html_response(conn, 200) =~ "Please complete the captcha"
-    end
   end
 
   describe "GET /register/invitations/:invitation_id" do
@@ -134,6 +113,7 @@ defmodule PlausibleWeb.AuthControllerTest do
 
   describe "POST /register/invitation/:invitation_id" do
     setup do
+      mock_captcha_success()
       inviter = insert(:user)
       site = insert(:site, members: [inviter])
 
@@ -149,8 +129,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "registering sends an activation link", %{conn: conn, invitation: invitation} do
-      mock_captcha_success()
-
       post(conn, "/register/invitation/#{invitation.invitation_id}",
         user: %{
           name: "Jane Doe",
@@ -169,8 +147,6 @@ defmodule PlausibleWeb.AuthControllerTest do
       conn: conn,
       invitation: invitation
     } do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register/invitation/#{invitation.invitation_id}",
           user: %{
@@ -185,8 +161,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "creates user record", %{conn: conn, invitation: invitation} do
-      mock_captcha_success()
-
       post(conn, "/register/invitation/#{invitation.invitation_id}",
         user: %{
           name: "Jane Doe",
@@ -204,8 +178,6 @@ defmodule PlausibleWeb.AuthControllerTest do
       conn: conn,
       invitation: invitation
     } do
-      mock_captcha_success()
-
       post(conn, "/register/invitation/#{invitation.invitation_id}",
         user: %{
           name: "Jane Doe",
@@ -220,8 +192,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "logs the user in", %{conn: conn, invitation: invitation} do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register/invitation/#{invitation.invitation_id}",
           user: %{
@@ -236,8 +206,6 @@ defmodule PlausibleWeb.AuthControllerTest do
     end
 
     test "user is redirected to activation after registration", %{conn: conn} do
-      mock_captcha_success()
-
       conn =
         post(conn, "/register",
           user: %{
@@ -250,13 +218,29 @@ defmodule PlausibleWeb.AuthControllerTest do
 
       assert redirected_to(conn) == "/activate"
     end
+  end
+
+  describe "captcha failure" do
+    setup do
+      mock_captcha_failure()
+      inviter = insert(:user)
+      site = insert(:site, members: [inviter])
+
+      invitation =
+        insert(:invitation,
+          site_id: site.id,
+          inviter: inviter,
+          email: "user@email.co",
+          role: :admin
+        )
+
+      {:ok, %{site: site, invitation: invitation}}
+    end
 
     test "renders captcha errors in case of captcha input verification failure", %{
       conn: conn,
       invitation: invitation
     } do
-      mock_captcha_failure()
-
       conn =
         post(conn, "/register/invitation/#{invitation.invitation_id}",
           user: %{

test/plausible_web/controllers/billing_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.BillingControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
 
   describe "GET /upgrade" do
     setup [:create_user, :log_in]

test/plausible_web/controllers/unsubscribe_controller_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.UnsubscribeControllerTest do
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
   use Plausible.Repo
 
   describe "GET /sites/:website/weekly-report/unsubscribe" do

test/plausible_web/plugs/auth_plug_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.AuthPlugTest do
-  use Plausible.DataCase
+  use Plausible.DataCase, async: true
   use Plug.Test
   alias PlausibleWeb.AuthPlug
 

test/plausible_web/plugs/favicon_test.exs

@@ -1,5 +1,5 @@
 defmodule PlausibleWeb.FaviconTest do
-  use Plausible.DataCase
+  use Plausible.DataCase, async: true
   use Plug.Test
   alias PlausibleWeb.Favicon
 

test/plausible_web/plugs/firewall_test.exs

@@ -7,10 +7,7 @@ defmodule PlausibleWeb.FirewallTest do
   @blocked_ip "127.0.0.2"
   @opts [blocklist: [@blocked_ip]]
 
-  setup do
-    Application.put_env(:plausible, PlausibleWeb.Firewall, blocklist: [@blocked_ip])
-    :ok
-  end
+  setup_patch_env(PlausibleWeb.Firewall, blocklist: [@blocked_ip])
 
   test "ignores request if IP is allowed" do
     conn =

test/plausible_web/plugs/tracker_test.exs

@@ -13,7 +13,7 @@ defmodule PlausibleWeb.TrackerTest do
   If you're making changes in the tracker template files (`src/plausible.js`, `compile.js`),
   do regenerate the files before running tests, so they're up to date.
   """
-  use PlausibleWeb.ConnCase
+  use PlausibleWeb.ConnCase, async: true
   use Plug.Test
 
   test "returns legacy script p.js" do

test/test_helper.exs

@@ -1,6 +1,6 @@
 {:ok, _} = Application.ensure_all_started(:ex_machina)
 Plausible.Test.ClickhouseSetup.run()
 Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
-ExUnit.start()
+ExUnit.start(exclude: :slow)
 Application.ensure_all_started(:double)
 Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)