Adds same glob patterns as exclusions for pageview goals (#750)

* First Pass - 90% wrong

* Second pass - basically done

* Swaps N+1 for multiMatchAllIndices & friends

* New version of query with just one regex computation

* Slight formatting, basic test

* Updates tests to match master new events

* Changelog

* Adds more sophisticated glob tests

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 - Ability to view and filter by entry and exit pages, in addition to regular page hits plausible/analytics#712
 - 30 day and 6 month keybindings (`T` and `S`, respectively) plausible/analytics#709
 - Site switching keybinds (1-9 for respective sites) plausible/analytics#735
+- Glob (wildcard) based pageview goals plausible/analytics#750
 
 ### Fixed
 - Capitalized date/time selection keybinds not working plausible/analytics#709

lib/plausible/goal/schema.ex

@@ -18,17 +18,22 @@ defmodule Plausible.Goal do
   end
 
   defp validate_event_name_and_page_path(changeset) do
-    if present?(changeset, :event_name) || present?(changeset, :page_path) do
+    if validate_page_path(changeset) || validate_event_name(changeset) do
       changeset
     else
       changeset
-      |> add_error(:event_name, "this field is required")
-      |> add_error(:page_path, "this field is required")
+      |> add_error(:event_name, "this field is required and cannot be blank")
+      |> add_error(:page_path, "this field is required and must start with a /")
     end
   end
 
-  defp present?(changeset, field) do
-    value = get_field(changeset, field)
-    value && value != ""
+  defp validate_page_path(changeset) do
+    value = get_field(changeset, :page_path)
+    value && String.match?(value, ~r/^\/.*/)
+  end
+
+  defp validate_event_name(changeset) do
+    value = get_field(changeset, :event_name)
+    value && String.match?(value, ~r/^.+/)
   end
 end

lib/plausible/stats/clickhouse.ex

@@ -925,24 +925,37 @@ defmodule Plausible.Stats.Clickhouse do
   end
 
   defp fetch_pageview_goals(goals, site, query) do
-    pages =
+    goals =
       Enum.map(goals, fn goal -> goal.page_path end)
       |> Enum.filter(& &1)
 
-    if Enum.count(pages) > 0 do
-      q =
-        from(
-          e in base_query_w_sessions(site, query),
-          where: fragment("? IN tuple(?)", e.pathname, ^pages),
-          group_by: e.pathname,
-          select: %{
-            name: fragment("concat('Visit ', ?) as name", e.pathname),
-            count: fragment("uniq(user_id) as count"),
-            total_count: fragment("count(*) as total_count")
-          }
-        )
+    if Enum.count(goals) > 0 do
+      regex_goals =
+        Enum.map(goals, fn g ->
+          "^#{g}\/?$"
+          |> String.replace(~r/\*\*/, ".*")
+          |> String.replace(~r/(?<!\.)\*/, "[^/]*")
+        end)
 
-      ClickhouseRepo.all(q)
+      from(
+        e in base_query_w_sessions(site, query),
+        where:
+          fragment(
+            "notEmpty(multiMatchAllIndices(?, array(?)) as indices)",
+            e.pathname,
+            ^regex_goals
+          ),
+        select: %{
+          index: fragment("arrayJoin(indices) as index"),
+          count: fragment("uniq(user_id) as count"),
+          total_count: fragment("count(*) as total_count")
+        },
+        group_by: fragment("index")
+      )
+      |> ClickhouseRepo.all()
+      |> Enum.map(fn x ->
+        Map.put(x, :name, "Visit #{Enum.at(goals, x[:index] - 1)}") |> Map.delete(:index)
+      end)
     else
       []
     end

test/plausible_web/controllers/api/stats_controller/conversions_test.exs

@@ -79,4 +79,86 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
              ]
     end
   end
+
+  describe "GET /api/stats/:domain/conversions - with glob goals" do
+    setup [:create_user, :log_in, :create_site]
+
+    test "returns correct and sorted glob goal counts", %{conn: conn, site: site} do
+      insert(:goal, %{domain: site.domain, page_path: "/register"})
+      insert(:goal, %{domain: site.domain, page_path: "/reg*"})
+      insert(:goal, %{domain: site.domain, page_path: "/*/register"})
+      insert(:goal, %{domain: site.domain, page_path: "/billing**/success"})
+      insert(:goal, %{domain: site.domain, page_path: "/billing*/success"})
+      insert(:goal, %{domain: site.domain, page_path: "/signup"})
+      insert(:goal, %{domain: site.domain, page_path: "/signup/*"})
+      insert(:goal, %{domain: site.domain, page_path: "/signup/**"})
+      insert(:goal, %{domain: site.domain, page_path: "/*"})
+      insert(:goal, %{domain: site.domain, page_path: "/**"})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/conversions?period=day&date=2019-07-01"
+        )
+
+      assert json_response(conn, 200) == [
+               %{
+                 "conversion_rate" => 100.0,
+                 "count" => 8,
+                 "name" => "Visit /**",
+                 "total_count" => 8,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 37.5,
+                 "count" => 3,
+                 "name" => "Visit /*",
+                 "total_count" => 3,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 37.5,
+                 "count" => 3,
+                 "name" => "Visit /signup/**",
+                 "total_count" => 3,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 25.0,
+                 "count" => 2,
+                 "name" => "Visit /billing**/success",
+                 "total_count" => 2,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 25.0,
+                 "count" => 2,
+                 "name" => "Visit /reg*",
+                 "total_count" => 2,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 12.5,
+                 "count" => 1,
+                 "name" => "Visit /billing*/success",
+                 "total_count" => 1,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 12.5,
+                 "count" => 1,
+                 "name" => "Visit /register",
+                 "total_count" => 1,
+                 "prop_names" => nil
+               },
+               %{
+                 "conversion_rate" => 12.5,
+                 "count" => 1,
+                 "name" => "Visit /signup/*",
+                 "total_count" => 1,
+                 "prop_names" => nil
+               }
+             ]
+    end
+  end
 end

test/support/clickhouse_setup.ex

@@ -162,6 +162,54 @@ defmodule Plausible.Test.ClickhouseSetup do
         referrer: "t.co/b-link",
         referrer_source: "Twitter",
         timestamp: Timex.now() |> Timex.shift(days: -5)
+      },
+      %{
+        name: "pageview",
+        pathname: "/register",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/signup/new",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/billing/success",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/reg",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/signup/new/2",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/signup/new/3",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
+      },
+      %{
+        name: "pageview",
+        pathname: "/billing/upgrade/success",
+        domain: "test-site.com",
+        timestamp: ~N[2019-07-01 23:00:00]
       }
     ])