From d687663b80c16285485ad8849949de773adf1e89 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 31 Oct 2019 11:49:46 +0800 Subject: [PATCH] Use new /event endpoint for pageviews --- assets/js/p.js | 3 +- assets/js/plausible.js | 3 +- lib/plausible/event/schema.ex | 4 +- .../controllers/api/external_controller.ex | 23 +++-- lib/plausible_web/router.ex | 2 +- .../api/external_controller_test.exs | 84 +++++++++++-------- 6 files changed, 68 insertions(+), 51 deletions(-) diff --git a/assets/js/p.js b/assets/js/p.js index e6ff37f5f..f52983924 100644 --- a/assets/js/p.js +++ b/assets/js/p.js @@ -83,10 +83,11 @@ if (window.document.visibilityState === 'prerender') return ignore('document is prerendering'); var payload = getUserData() + payload.name = 'pageview' payload.url = getUrl() var request = new XMLHttpRequest(); - request.open('POST', plausibleHost + '/api/page', true); + request.open('POST', plausibleHost + '/api/event', true); request.setRequestHeader('Content-Type', 'text/plain'); request.send(JSON.stringify(payload)); diff --git a/assets/js/plausible.js b/assets/js/plausible.js index e024887d6..20adb60e9 100644 --- a/assets/js/plausible.js +++ b/assets/js/plausible.js @@ -83,10 +83,11 @@ if (window.document.visibilityState === 'prerender') return ignore('document is prerendering'); var payload = getUserData() + payload.name = 'pageview' payload.url = getUrl() var request = new XMLHttpRequest(); - request.open('POST', plausibleHost + '/api/page', true); + request.open('POST', plausibleHost + '/api/event', true); request.setRequestHeader('Content-Type', 'text/plain'); request.send(JSON.stringify(payload)); diff --git a/lib/plausible/event/schema.ex b/lib/plausible/event/schema.ex index 2e0488315..0fbfaf6b9 100644 --- a/lib/plausible/event/schema.ex +++ b/lib/plausible/event/schema.ex @@ -21,7 +21,7 @@ defmodule Plausible.Event do def changeset(pageview, attrs) do pageview - |> cast(attrs, [:hostname, :pathname, :referrer, :new_visitor, :user_id, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) - |> validate_required([:hostname, :pathname, :new_visitor, :user_id]) + |> cast(attrs, [:name, :hostname, :pathname, :referrer, :new_visitor, :user_id, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) + |> validate_required([:name, :hostname, :pathname, :new_visitor, :user_id]) end end diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index 5718fc482..c4cf51ec7 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -2,20 +2,16 @@ defmodule PlausibleWeb.Api.ExternalController do use PlausibleWeb, :controller require Logger - @blacklist_user_ids [ - "e8150466-7ddb-4771-bcf5-7c58f232e8a6" - ] - - def page(conn, _params) do + def event(conn, _params) do params = parse_body(conn) - case create_pageview(conn, params) do - {:ok, _pageview} -> + case create_event(conn, params) do + {:ok, _event} -> conn |> send_resp(202, "") {:error, changeset} -> request = Sentry.Plug.build_request_interface_data(conn, []) - Sentry.capture_message("Error processing pageview", extra: %{errors: inspect(changeset.errors), params: params, request: request}) - Logger.error("Error processing pageview: #{inspect(changeset)}") + Sentry.capture_message("Error processing event", extra: %{errors: inspect(changeset.errors), params: params, request: request}) + Logger.error("Error processing event: #{inspect(changeset)}") conn |> send_resp(400, "") end end @@ -26,11 +22,11 @@ defmodule PlausibleWeb.Api.ExternalController do send_resp(conn, 200, "") end - defp create_pageview(conn, params) do + defp create_event(conn, params) do uri = URI.parse(params["url"]) country_code = Plug.Conn.get_req_header(conn, "cf-ipcountry") |> List.first user_agent = Plug.Conn.get_req_header(conn, "user-agent") |> List.first - if UAInspector.bot?(user_agent) || params["uid"] in @blacklist_user_ids do + if UAInspector.bot?(user_agent) do {:ok, nil} else ua = if user_agent do @@ -42,7 +38,8 @@ defmodule PlausibleWeb.Api.ExternalController do RefInspector.parse(ref) end - pageview_attrs = %{ + event_attrs = %{ + name: params["name"], hostname: strip_www(uri.host), pathname: uri.path, new_visitor: params["new_visitor"], @@ -55,7 +52,7 @@ defmodule PlausibleWeb.Api.ExternalController do screen_size: calculate_screen_size(params["screen_width"]) } - Plausible.Event.changeset(%Plausible.Event{name: "pageview"}, pageview_attrs) + Plausible.Event.changeset(%Plausible.Event{}, event_attrs) |> Plausible.Repo.insert end end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 40ed821f4..9109b2231 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -29,7 +29,7 @@ defmodule PlausibleWeb.Router do scope "/api", PlausibleWeb do pipe_through :api - post "/page", Api.ExternalController, :page + post "/event", Api.ExternalController, :event get "/error", Api.ExternalController, :error post "/paddle/webhook", Api.PaddleController, :webhook diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index db9604c69..f8b10101c 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -5,9 +5,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" @country_code "EE" - describe "POST /api/page" do - test "records the pageview", %{conn: conn} do + describe "POST /api/event" do + test "records the event", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "http://m.facebook.com/", new_visitor: true, @@ -19,7 +20,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) |> put_req_header("cf-ipcountry", @country_code) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -32,6 +33,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "www. is stripped from hostname", %{conn: conn} do params = %{ + name: "pageview", url: "http://www.example.com/", uid: UUID.uuid4(), new_visitor: true @@ -39,7 +41,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -48,6 +50,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "bots and crawlers are ignored", %{conn: conn} do params = %{ + name: "pageview", url: "http://www.example.com/", new_visitor: true } @@ -55,7 +58,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", "generic crawler") - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageviews = Repo.all(Plausible.Event) @@ -64,6 +67,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "parses user_agent", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", new_visitor: false, uid: UUID.uuid4() @@ -72,7 +76,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -83,6 +87,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "parses referrer", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "https://facebook.com", new_visitor: false, @@ -92,7 +97,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -102,6 +107,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "ignores when referrer is internal", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "https://gigride.live", new_visitor: false, @@ -111,7 +117,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -121,6 +127,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "ignores localhost referrer", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "http://localhost:4000/", new_visitor: true, @@ -130,7 +137,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -140,6 +147,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "parses subdomain referrer", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "https://blog.gigride.live", new_visitor: false, @@ -149,7 +157,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -159,6 +167,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "referrer is cleaned", %{conn: conn} do params = %{ + name: "pageview", url: "http://www.example.com/", referrer: "https://www.indiehackers.com/page?query=param#hash", uid: UUID.uuid4(), @@ -167,7 +176,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -176,6 +185,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "?ref= query param controls the referrer source", %{conn: conn} do params = %{ + name: "pageview", url: "http://www.example.com/?wat=wet&ref=traffic-source", referrer: "https://www.indiehackers.com/page", uid: UUID.uuid4(), @@ -184,7 +194,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -193,6 +203,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "?utm_source= query param controls the referrer source", %{conn: conn} do params = %{ + name: "pageview", url: "http://www.example.com/?wat=wet&utm_source=traffic-source", referrer: "https://www.indiehackers.com/page", uid: UUID.uuid4(), @@ -201,31 +212,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) assert pageview.referrer_source == "traffic-source" end - test "ignores pageviews from a user blacklist", %{conn: conn} do - params = %{ - url: "http://gigride.live/", - referrer: "https://blog.gigride.live", - new_visitor: false, - uid: "e8150466-7ddb-4771-bcf5-7c58f232e8a6" - } - - conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) - - assert Repo.aggregate(Plausible.Event, :count, :id) == 0 - end - test "if it's an :unknown referrer, just the domain is used", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "https://www.indiehackers.com/landing-page-feedback", new_visitor: false, @@ -235,7 +231,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -245,6 +241,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "if the referrer is not http or https, it is ignored", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", referrer: "android-app://com.google.android.gm", new_visitor: false, @@ -254,7 +251,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -266,6 +263,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "screen size is calculated from screen_width", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", new_visitor: true, screen_width: 480, @@ -275,7 +273,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) @@ -285,6 +283,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "screen size is nil if screen_width is missing", %{conn: conn} do params = %{ + name: "pageview", url: "http://gigride.live/", new_visitor: true, uid: UUID.uuid4() @@ -293,11 +292,30 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn = conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> post("/api/page", Jason.encode!(params)) + |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) assert response(conn, 202) == "" assert pageview.screen_size == nil end + + test "can trigger a custom event", %{conn: conn} do + params = %{ + name: "custom event", + url: "http://gigride.live/", + new_visitor: false, + uid: UUID.uuid4() + } + + conn = conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) + + event = Repo.one(Plausible.Event) + + assert response(conn, 202) == "" + assert event.name == "custom event" + end end