From 81c12884cd36be524a218207399ef4d630688b8d Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 29 Dec 2020 15:17:27 +0200 Subject: [PATCH] Add elixir action (#526) * Add elixir action * Format the codebase * Add postgresql * Postgres config * Run postgres on localhost * Add clickhouse to CI --- .github/workflows/elixir.yml | 61 +++++++++++++ lib/plausible/auth/auth.ex | 50 +++++++---- lib/plausible/auth/user.ex | 1 + lib/plausible/session/write_buffer.ex | 6 +- lib/plausible/site/schema.ex | 4 +- lib/plausible/stats/clickhouse.ex | 15 ++-- lib/plausible/stats/query.ex | 25 ++++-- .../controllers/api/external_controller.ex | 15 ++-- .../controllers/api/stats_controller.ex | 1 - .../controllers/auth_controller.ex | 35 ++++++-- .../controllers/site_controller.ex | 86 ++++++++++++------- lib/plausible_web/email.ex | 8 +- lib/plausible_web/plugs/require_logged_out.ex | 5 +- lib/plausible_web/plugs/tracker.ex | 7 +- lib/plausible_web/router.ex | 1 + lib/plausible_web/views/layout_view.ex | 3 +- lib/workers/clean_email_verification_codes.ex | 2 +- lib/workers/spike_notifier.ex | 33 ++++--- ...1210085345_add_email_verified_to_users.exs | 2 +- ...20201214072008_add_theme_pref_to_users.exs | 2 +- .../api/external_controller_test.exs | 17 ++-- .../api/stats_controller/browsers_test.exs | 9 +- .../operating_systems_test.exs | 9 +- .../controllers/auth_controller_test.exs | 43 +++++++--- .../controllers/site_controller_test.exs | 20 ++++- test/support/clickhouse_setup.ex | 2 +- .../clean_email_verification_codes_test.exs | 10 ++- test/workers/spike_notifier_test.exs | 32 +++++-- 28 files changed, 370 insertions(+), 134 deletions(-) create mode 100644 .github/workflows/elixir.yml diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml new file mode 100644 index 000000000..3e97d60eb --- /dev/null +++ b/.github/workflows/elixir.yml @@ -0,0 +1,61 @@ +name: Elixir CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + name: Build and test + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:12 + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + clickhouse: + image: yandex/clickhouse-server:20 + ports: + - 8123:8123 + env: + options: >- + --health-cmd nc -zw3 localhost 8124 + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - name: Read .tool-versions + uses: marocchino/tool-versions-action@v1 + id: versions + - name: Set up Elixir + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{steps.versions.outputs.elixir}} + otp-version: ${{ steps.versions.outputs.erlang}} + - name: Restore dependencies cache + uses: actions/cache@v2 + with: + path: deps + key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-mix- + - name: Install dependencies + run: mix deps.get + - name: Check Formatting + run: mix format --check-formatted + - name: Run tests + run: mix test +env: + MIX_ENV: test diff --git a/lib/plausible/auth/auth.ex b/lib/plausible/auth/auth.ex index c08d3e6e5..6475cbda4 100644 --- a/lib/plausible/auth/auth.ex +++ b/lib/plausible/auth/auth.ex @@ -4,11 +4,18 @@ defmodule Plausible.Auth do alias Plausible.Stats.Clickhouse, as: Stats def issue_email_verification(user) do - Repo.update_all(from(c in "email_verification_codes", where: c.user_id == ^user.id), [set: [user_id: nil]]) + Repo.update_all(from(c in "email_verification_codes", where: c.user_id == ^user.id), + set: [user_id: nil] + ) - code = Repo.one(from(c in "email_verification_codes", where: is_nil(c.user_id), select: c.code, limit: 1)) + code = + Repo.one( + from(c in "email_verification_codes", where: is_nil(c.user_id), select: c.code, limit: 1) + ) - Repo.update_all(from(c in "email_verification_codes", where: c.code == ^code), [set: [user_id: user.id, issued_at: Timex.now()]]) + Repo.update_all(from(c in "email_verification_codes", where: c.code == ^code), + set: [user_id: user.id, issued_at: Timex.now()] + ) code end @@ -18,21 +25,34 @@ defmodule Plausible.Auth do end def verify_email(user, code) do - found_code = Repo.one( - from c in "email_verification_codes", - where: c.user_id == ^user.id, - where: c.code == ^code, - select: %{code: c.code, issued: c.issued_at} - ) + found_code = + Repo.one( + from c in "email_verification_codes", + where: c.user_id == ^user.id, + where: c.code == ^code, + select: %{code: c.code, issued: c.issued_at} + ) cond do - is_nil(found_code) -> {:error, :incorrect} - is_expired?(found_code[:issued]) -> {:error, :expired} + is_nil(found_code) -> + {:error, :incorrect} + + is_expired?(found_code[:issued]) -> + {:error, :expired} + true -> - {:ok, _} = Ecto.Multi.new - |> Ecto.Multi.update(:user, Plausible.Auth.User.changeset(user, %{email_verified: true})) - |> Ecto.Multi.update_all(:codes, from(c in "email_verification_codes", where: c.user_id == ^user.id), [set: [user_id: nil]]) - |> Repo.transaction + {:ok, _} = + Ecto.Multi.new() + |> Ecto.Multi.update( + :user, + Plausible.Auth.User.changeset(user, %{email_verified: true}) + ) + |> Ecto.Multi.update_all( + :codes, + from(c in "email_verification_codes", where: c.user_id == ^user.id), + set: [user_id: nil] + ) + |> Repo.transaction() :ok end diff --git a/lib/plausible/auth/user.ex b/lib/plausible/auth/user.ex index eef1714b7..67848868d 100644 --- a/lib/plausible/auth/user.ex +++ b/lib/plausible/auth/user.ex @@ -60,5 +60,6 @@ defmodule Plausible.Auth.User do hash = Plausible.Auth.Password.hash(changes[:password]) change(changeset, password_hash: hash) end + def hash_password(changeset), do: changeset end diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.ex index 7d3713740..3a85a8e74 100644 --- a/lib/plausible/session/write_buffer.ex +++ b/lib/plausible/session/write_buffer.ex @@ -51,9 +51,11 @@ defmodule Plausible.Session.WriteBuffer do sessions -> Logger.info("Flushing #{length(sessions)} sessions") - sessions = sessions + + sessions = + sessions |> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__))) - |> Enum.reverse + |> Enum.reverse() Plausible.ClickhouseRepo.insert_all(Plausible.ClickhouseSession, sessions) end diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 985296dce..edb99938c 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -23,7 +23,9 @@ defmodule Plausible.Site do site |> cast(attrs, [:domain, :timezone]) |> validate_required([:domain, :timezone]) - |> validate_format(:domain, ~r/^[a-zA-z0-9\-\.\/\:]*$/, message: "only letters, numbers, slashes and period allowed") + |> validate_format(:domain, ~r/^[a-zA-z0-9\-\.\/\:]*$/, + message: "only letters, numbers, slashes and period allowed" + ) |> unique_constraint(:domain) |> clean_domain end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 5570cd95f..d1acdcf7a 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -167,6 +167,7 @@ defmodule Plausible.Stats.Clickhouse do def unique_visitors(site, query) do query = if query.period == "realtime", do: %Query{query | period: "30m"}, else: query + ClickhouseRepo.one( from e in base_query_w_sessions(site, query), select: fragment("uniq(user_id)") @@ -764,6 +765,7 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.filter(fn row -> row[:count] > 0 end) |> Enum.map(fn row -> uri = URI.parse(row[:name]) + if uri.host && uri.scheme do Map.put(row, :is_url, true) else @@ -773,15 +775,16 @@ defmodule Plausible.Stats.Clickhouse do end def last_24h_visitors([]), do: %{} + def last_24h_visitors(sites) do domains = Enum.map(sites, & &1.domain) ClickhouseRepo.all( from e in "events", - group_by: e.domain, - where: fragment("? IN tuple(?)", e.domain, ^domains), - where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"), - select: {e.domain, fragment("uniq(user_id)")} + group_by: e.domain, + where: fragment("? IN tuple(?)", e.domain, ^domains), + where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"), + select: {e.domain, fragment("uniq(user_id)")} ) |> Enum.into(%{}) end @@ -968,8 +971,8 @@ defmodule Plausible.Stats.Clickhouse do q = if query.filters["source"] || query.filters['referrer'] || query.filters["utm_medium"] || query.filters["utm_source"] || query.filters["utm_campaign"] || query.filters["screen"] || - query.filters["browser"] || query.filters["browser_version"] || query.filters["os"] || - query.filters["os_version"] || query.filters["country"] do + query.filters["browser"] || query.filters["browser_version"] || query.filters["os"] || + query.filters["os_version"] || query.filters["country"] do from( e in q, join: sq in subquery(sessions_q), diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 35eddd8c2..8f2e9db61 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -7,13 +7,24 @@ defmodule Plausible.Stats.Query do end def shift_back(%__MODULE__{period: "month"} = query, site) do - {new_first, new_last} = if Timex.compare(Timex.now(site.timezone), query.date_range.first, :month) == 0 do # Querying current month to date - diff = Timex.diff(Timex.beginning_of_month(Timex.now(site.timezone)), Timex.now(site.timezone), :days) - 1 - {query.date_range.first |> Timex.shift(days: diff), Timex.now(site.timezone) |> Timex.to_date |> Timex.shift(days: diff)} - else - diff = Timex.diff(query.date_range.first, query.date_range.last, :days) - 1 - {query.date_range.first |> Timex.shift(days: diff), query.date_range.last |> Timex.shift(days: diff)} - end + # Querying current month to date + {new_first, new_last} = + if Timex.compare(Timex.now(site.timezone), query.date_range.first, :month) == 0 do + diff = + Timex.diff( + Timex.beginning_of_month(Timex.now(site.timezone)), + Timex.now(site.timezone), + :days + ) - 1 + + {query.date_range.first |> Timex.shift(days: diff), + Timex.now(site.timezone) |> Timex.to_date() |> Timex.shift(days: diff)} + else + diff = Timex.diff(query.date_range.first, query.date_range.last, :days) - 1 + + {query.date_range.first |> Timex.shift(days: diff), + query.date_range.last |> Timex.shift(days: diff)} + end Map.put(query, :date_range, Date.range(new_first, new_last)) end diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index ea37b9010..6f910dc9f 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -119,9 +119,11 @@ defmodule PlausibleWeb.Api.ExternalController do end end - defp is_bot?(%UAInspector.Result.Bot{}), do: true - defp is_bot?(%UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}}), do: true + + defp is_bot?(%UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}}), + do: true + defp is_bot?(_), do: false defp parse_meta(params) do @@ -137,8 +139,9 @@ defmodule PlausibleWeb.Api.ExternalController do defp get_pathname(nil, _), do: "/" defp get_pathname(uri, hash_mode) do - pathname = (uri.path || "/") - |> URI.decode + pathname = + (uri.path || "/") + |> URI.decode() if hash_mode && uri.fragment do pathname <> "#" <> URI.decode(uri.fragment) @@ -158,9 +161,8 @@ defmodule PlausibleWeb.Api.ExternalController do end end - - defp parse_referrer(_, nil), do: nil + defp parse_referrer(uri, referrer_str) do referrer_uri = URI.parse(referrer_str) @@ -222,6 +224,7 @@ defmodule PlausibleWeb.Api.ExternalController do end defp major_minor(:unknown), do: "" + defp major_minor(version) do version |> String.split(".") diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 97f724571..643858c62 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -84,7 +84,6 @@ defmodule PlausibleWeb.Api.StatsController do prev_bounce_rate = Stats.bounce_rate(site, prev_query) change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate - visit_duration = if !query.filters["page"] do duration = Stats.visit_duration(site, query) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 5cb5bb0fb..2194049e5 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -8,7 +8,14 @@ defmodule PlausibleWeb.AuthController do when action in [:register_form, :register, :login_form, :login] plug PlausibleWeb.RequireAccountPlug - when action in [:user_settings, :save_settings, :delete_me, :password_form, :set_password, :activate_form] + when action in [ + :user_settings, + :save_settings, + :delete_me, + :password_form, + :set_password, + :activate_form + ] def register_form(conn, _params) do if Keyword.fetch!(Application.get_env(:plausible, :selfhost), :disable_registration) do @@ -37,7 +44,10 @@ defmodule PlausibleWeb.AuthController do conn |> put_session(:current_user_id, user.id) - |> put_resp_cookie("logged_in", "true", [http_only: false, max_age: 60 * 60 * 24 * 365 * 5000]) + |> put_resp_cookie("logged_in", "true", + http_only: false, + max_age: 60 * 60 * 24 * 365 * 5000 + ) |> redirect(to: "/activate") {:error, changeset} -> @@ -58,12 +68,16 @@ defmodule PlausibleWeb.AuthController do def activate_form(conn, _params) do user = conn.assigns[:current_user] - has_code = Repo.exists?( - from c in "email_verification_codes", - where: c.user_id == ^user.id - ) + has_code = + Repo.exists?( + from c in "email_verification_codes", + where: c.user_id == ^user.id + ) - render(conn, "activate.html", has_pin: has_code, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "activate.html", + has_pin: has_code, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def activate(conn, %{"code" => code}) do @@ -73,12 +87,14 @@ defmodule PlausibleWeb.AuthController do case Auth.verify_email(user, code) do :ok -> redirect(conn, to: "/sites/new") + {:error, :incorrect} -> render(conn, "activate.html", error: "Incorrect activation code", has_pin: true, layout: {PlausibleWeb.LayoutView, "focus.html"} ) + {:error, :expired} -> render(conn, "activate.html", error: "Code is expired, please request another one", @@ -220,7 +236,10 @@ defmodule PlausibleWeb.AuthController do conn |> put_session(:current_user_id, user.id) - |> put_resp_cookie("logged_in", "true", [http_only: false, max_age: 60 * 60 * 24 * 365 * 5000]) + |> put_resp_cookie("logged_in", "true", + http_only: false, + max_age: 60 * 60 * 24 * 365 * 5000 + ) |> put_session(:login_dest, nil) |> redirect(to: login_dest) else diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 2676340fe..01f38bdd9 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -7,12 +7,15 @@ defmodule PlausibleWeb.SiteController do def index(conn, _params) do user = conn.assigns[:current_user] - sites = Repo.all( - from s in Plausible.Site, - join: sm in Plausible.Site.Membership, on: sm.site_id == s.id, - where: sm.user_id == ^user.id, - order_by: s.domain - ) + + sites = + Repo.all( + from s in Plausible.Site, + join: sm in Plausible.Site.Membership, + on: sm.site_id == s.id, + where: sm.user_id == ^user.id, + order_by: s.domain + ) visitors = Plausible.Stats.Clickhouse.last_24h_visitors(sites) render(conn, "index.html", sites: sites, visitors: visitors) @@ -22,12 +25,17 @@ defmodule PlausibleWeb.SiteController do current_user = conn.assigns[:current_user] changeset = Plausible.Site.changeset(%Plausible.Site{}) - is_first_site = !Repo.exists?( - from sm in Plausible.Site.Membership, - where: sm.user_id == ^current_user.id - ) + is_first_site = + !Repo.exists?( + from sm in Plausible.Site.Membership, + where: sm.user_id == ^current_user.id + ) - render(conn, "new.html", changeset: changeset, is_first_site: is_first_site, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "new.html", + changeset: changeset, + is_first_site: is_first_site, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def create_site(conn, %{"site" => site_params}) do @@ -37,11 +45,13 @@ defmodule PlausibleWeb.SiteController do {:ok, %{site: site}} -> Plausible.Slack.notify("#{user.name} created #{site.domain} [email=#{user.email}]") - is_first_site = !Repo.exists?( - from sm in Plausible.Site.Membership, - where: sm.user_id == ^user.id - and sm.site_id != ^site.id - ) + is_first_site = + !Repo.exists?( + from sm in Plausible.Site.Membership, + where: + sm.user_id == ^user.id and + sm.site_id != ^site.id + ) if is_first_site do PlausibleWeb.Email.welcome_email(user) @@ -53,10 +63,11 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: "/#{URI.encode_www_form(site.domain)}/snippet") {:error, :site, changeset, _} -> - is_first_site = !Repo.exists?( - from sm in Plausible.Site.Membership, - where: sm.user_id == ^user.id - ) + is_first_site = + !Repo.exists?( + from sm in Plausible.Site.Membership, + where: sm.user_id == ^user.id + ) render(conn, "new.html", changeset: changeset, @@ -68,19 +79,26 @@ defmodule PlausibleWeb.SiteController do def add_snippet(conn, %{"website" => website}) do user = conn.assigns[:current_user] + site = Sites.get_for_user!(conn.assigns[:current_user].id, website) |> Repo.preload(:custom_domain) - is_first_site = !Repo.exists?( - from sm in Plausible.Site.Membership, - where: sm.user_id == ^user.id - and sm.site_id != ^site.id - ) + is_first_site = + !Repo.exists?( + from sm in Plausible.Site.Membership, + where: + sm.user_id == ^user.id and + sm.site_id != ^site.id + ) conn |> assign(:skip_plausible_tracking, true) - |> render("snippet.html", site: site, is_first_site: is_first_site, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("snippet.html", + site: site, + is_first_site: is_first_site, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def new_goal(conn, %{"website" => website}) do @@ -129,8 +147,9 @@ defmodule PlausibleWeb.SiteController do end def settings_general(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:custom_domain) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:custom_domain) conn |> assign(:skip_plausible_tracking, true) @@ -168,7 +187,8 @@ defmodule PlausibleWeb.SiteController do end def settings_search_console(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) |> Repo.preload(:google_auth) search_console_domains = @@ -200,12 +220,16 @@ defmodule PlausibleWeb.SiteController do end def settings_custom_domain(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) |> Repo.preload(:custom_domain) conn |> assign(:skip_plausible_tracking, true) - |> render("settings_custom_domain.html", site: site, layout: {PlausibleWeb.LayoutView, "site_settings.html"}) + |> render("settings_custom_domain.html", + site: site, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) end def settings_danger_zone(conn, %{"website" => website}) do diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex index f89c6b3f9..4557d3ae7 100644 --- a/lib/plausible_web/email.ex +++ b/lib/plausible_web/email.ex @@ -13,6 +13,7 @@ defmodule PlausibleWeb.Email do |> subject("#{code} is your Plausible email verification code") |> render("activation_email.html", user: user, code: code) end + def welcome_email(user) do base_email() |> to(user) @@ -98,7 +99,12 @@ defmodule PlausibleWeb.Email do |> to(email) |> tag("spike-notification") |> subject("Traffic spike on #{site.domain}") - |> render("spike_notification.html", %{site: site, current_visitors: current_visitors, sources: sources, link: dashboard_link}) + |> render("spike_notification.html", %{ + site: site, + current_visitors: current_visitors, + sources: sources, + link: dashboard_link + }) end def cancellation_email(user) do diff --git a/lib/plausible_web/plugs/require_logged_out.ex b/lib/plausible_web/plugs/require_logged_out.ex index ee6c17454..ac35a4e7f 100644 --- a/lib/plausible_web/plugs/require_logged_out.ex +++ b/lib/plausible_web/plugs/require_logged_out.ex @@ -9,7 +9,10 @@ defmodule PlausibleWeb.RequireLoggedOutPlug do cond do conn.assigns[:current_user] -> conn - |> put_resp_cookie("logged_in", "true", [http_only: false, max_age: 60 * 60 * 24 * 365 * 5000]) + |> put_resp_cookie("logged_in", "true", + http_only: false, + max_age: 60 * 60 * 24 * 365 * 5000 + ) |> Phoenix.Controller.redirect(to: "/sites") |> halt diff --git a/lib/plausible_web/plugs/tracker.ex b/lib/plausible_web/plugs/tracker.ex index 553225ab7..b55a99d38 100644 --- a/lib/plausible_web/plugs/tracker.ex +++ b/lib/plausible_web/plugs/tracker.ex @@ -20,8 +20,7 @@ defmodule PlausibleWeb.Tracker do def init(_) do templates = Enum.reduce(@templates, %{}, fn template_filename, rendered_templates -> - rendered = - EEx.compile_file("priv/tracker/js/" <> template_filename) + rendered = EEx.compile_file("priv/tracker/js/" <> template_filename) aliases = Map.get(@aliases, template_filename, []) @@ -36,7 +35,9 @@ defmodule PlausibleWeb.Tracker do def call(conn, templates: templates) do case templates[conn.request_path] do - nil -> conn + nil -> + conn + found -> {js, _} = Code.eval_quoted(found, base_url: PlausibleWeb.Endpoint.url()) send_js(conn, js) diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 2e0014046..57ef97cf7 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -135,6 +135,7 @@ defmodule PlausibleWeb.Router do post "/sites/:website/spike-notification/enable", SiteController, :enable_spike_notification post "/sites/:website/spike-notification/disable", SiteController, :disable_spike_notification put "/sites/:website/spike-notification", SiteController, :update_spike_notification + post "/sites/:website/spike-notification/recipients", SiteController, :add_spike_notification_recipient diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index d8a776bba..68053c0c6 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -29,11 +29,10 @@ defmodule PlausibleWeb.LayoutView do [key: "Search Console", value: "search-console"], [key: "Email reports", value: "email-reports"], [key: "Custom domain", value: "custom-domain"], - [key: "Danger zone", value: "danger-zone"], + [key: "Danger zone", value: "danger-zone"] ] end - def trial_notificaton(user) do case Plausible.Billing.trial_days_left(user) do days when days > 1 -> diff --git a/lib/workers/clean_email_verification_codes.ex b/lib/workers/clean_email_verification_codes.ex index 72069d362..5b7af9272 100644 --- a/lib/workers/clean_email_verification_codes.ex +++ b/lib/workers/clean_email_verification_codes.ex @@ -9,7 +9,7 @@ defmodule Plausible.Workers.CleanEmailVerificationCodes do where: not is_nil(c.user_id), where: c.issued_at < fragment("now() - INTERVAL '4 hours'") ), - [set: [user_id: nil]] + set: [user_id: nil] ) end end diff --git a/lib/workers/spike_notifier.ex b/lib/workers/spike_notifier.ex index add1af17f..5d2a0fe6d 100644 --- a/lib/workers/spike_notifier.ex +++ b/lib/workers/spike_notifier.ex @@ -7,12 +7,13 @@ defmodule Plausible.Workers.SpikeNotifier do @impl Oban.Worker def perform(_args, _job, clickhouse \\ Plausible.Stats.Clickhouse) do - notifications = Repo.all( - from sn in SpikeNotification, - where: is_nil(sn.last_sent), - or_where: sn.last_sent < fragment("now() - INTERVAL ?", @at_most_every), - preload: :site - ) + notifications = + Repo.all( + from sn in SpikeNotification, + where: is_nil(sn.last_sent), + or_where: sn.last_sent < fragment("now() - INTERVAL ?", @at_most_every), + preload: :site + ) for notification <- notifications do query = Query.from(notification.site.timezone, %{"period" => "realtime"}) @@ -27,20 +28,30 @@ defmodule Plausible.Workers.SpikeNotifier do for recipient <- notification.recipients do send_notification(recipient, notification.site, current_visitors, sources) end + notification |> SpikeNotification.was_sent() - |> Repo.update + |> Repo.update() end end defp send_notification(recipient, site, current_visitors, sources) do site = Repo.preload(site, :members) - dashboard_link = if Enum.member?(site.members, recipient) do - PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain) - end + dashboard_link = + if Enum.member?(site.members, recipient) do + PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain) + end + + template = + PlausibleWeb.Email.spike_notification( + recipient, + site, + current_visitors, + sources, + dashboard_link + ) - template = PlausibleWeb.Email.spike_notification(recipient, site, current_visitors, sources, dashboard_link) try do Plausible.Mailer.send_email(template) rescue diff --git a/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs b/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs index 822409c1e..d170f1952 100644 --- a/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs +++ b/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs @@ -9,6 +9,6 @@ defmodule Plausible.Repo.Migrations.AddEmailVerifiedToUsers do flush() - Repo.update_all("users", [set: [email_verified: true]]) + Repo.update_all("users", set: [email_verified: true]) end end diff --git a/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs b/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs index 48e4eb135..3bd588008 100644 --- a/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs +++ b/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs @@ -3,7 +3,7 @@ defmodule Plausible.Repo.Migrations.AddThemePrefToUsers do def change do alter table(:users) do - add_if_not_exists :theme, :string, default: "system" + add_if_not_exists(:theme, :string, default: "system") end end end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index ebc3c8330..25a8a58d3 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -110,7 +110,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/85.0.4183.83 Safari/537.36") + |> put_req_header( + "user-agent", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/85.0.4183.83 Safari/537.36" + ) |> post("/api/event", Jason.encode!(params)) assert get_event("headless-chrome-test.com") == nil @@ -396,10 +399,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do name: "Signup", url: "http://gigride.live/", domain: "custom-prop-test.com", - props: Jason.encode!(%{ - bool_test: true, - number_test: 12 - }) + props: + Jason.encode!(%{ + bool_test: true, + number_test: 12 + }) } conn @@ -508,7 +512,8 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "decodes URL pathname, fragment and search", %{conn: conn} do params = %{ n: "pageview", - u: "https://test.com/%EF%BA%9D%EF%BB%AD%EF%BA%8E%EF%BA%8B%EF%BA%AF-%EF%BB%AE%EF%BB%A4%EF%BA%B3%EF%BA%8E%EF%BA%92%EF%BB%97%EF%BA%8E%EF%BA%97?utm_source=%25balle%25", + u: + "https://test.com/%EF%BA%9D%EF%BB%AD%EF%BA%8E%EF%BA%8B%EF%BA%AF-%EF%BB%AE%EF%BB%A4%EF%BA%B3%EF%BA%8E%EF%BA%92%EF%BB%97%EF%BA%8E%EF%BA%97?utm_source=%25balle%25", d: "url-decode-test.com", h: 1 } diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index e00252d4a..9b9ced495 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -20,7 +20,14 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "returns top browser versions by unique visitors", %{conn: conn, site: site} do filters = Jason.encode!(%{browser: "Chrome"}) - conn = get(conn, "/api/stats/#{site.domain}/browser-versions?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/browser-versions?period=day&date=2019-01-01&filters=#{ + filters + }" + ) assert json_response(conn, 200) == [ %{"name" => "78.0", "count" => 1, "percentage" => 100} diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index 71658d93f..5329e4afa 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -20,7 +20,14 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "returns top OS versions by unique visitors", %{conn: conn, site: site} do filters = Jason.encode!(%{os: "Mac"}) - conn = get(conn, "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2019-01-01&filters=#{ + filters + }" + ) assert json_response(conn, 200) == [ %{"name" => "10.15", "count" => 1, "percentage" => 100} diff --git a/test/plausible_web/controllers/auth_controller_test.exs b/test/plausible_web/controllers/auth_controller_test.exs index 81391b256..6a927f4fe 100644 --- a/test/plausible_web/controllers/auth_controller_test.exs +++ b/test/plausible_web/controllers/auth_controller_test.exs @@ -49,7 +49,7 @@ defmodule PlausibleWeb.AuthControllerTest do name: "Jane Doe", email: "user@example.com", password: "very-secret", - password_confirmation: "very-secret", + password_confirmation: "very-secret" } ) @@ -63,7 +63,7 @@ defmodule PlausibleWeb.AuthControllerTest do name: "Jane Doe", email: "user@example.com", password: "very-secret", - password_confirmation: "very-secret", + password_confirmation: "very-secret" } ) @@ -80,9 +80,11 @@ defmodule PlausibleWeb.AuthControllerTest do assert html_response(conn, 200) =~ "Request activation code" end - test "if user does have a code: prompts user to enter the activation code from their email", %{conn: conn} do - conn = post(conn, "/activate/request-code") - |> get("/activate") + test "if user does have a code: prompts user to enter the activation code from their email", + %{conn: conn} do + conn = + post(conn, "/activate/request-code") + |> get("/activate") assert html_response(conn, 200) =~ "Please enter the 4-digit code we sent to" end @@ -94,7 +96,12 @@ defmodule PlausibleWeb.AuthControllerTest do test "associates an activation pin with the user account", %{conn: conn, user: user} do post(conn, "/activate/request-code") - code = Repo.one(from c in "email_verification_codes", where: c.user_id == ^user.id, select: %{user_id: c.user_id, issued_at: c.issued_at}) + code = + Repo.one( + from c in "email_verification_codes", + where: c.user_id == ^user.id, + select: %{user_id: c.user_id, issued_at: c.issued_at} + ) assert code[:user_id] == user.id assert Timex.after?(code[:issued_at], Timex.now() |> Timex.shift(seconds: -10)) @@ -119,11 +126,13 @@ defmodule PlausibleWeb.AuthControllerTest do end test "with expired pin - reloads the form with error", %{conn: conn, user: user} do - Repo.insert_all("email_verification_codes", [%{ - code: 1234, - user_id: user.id, - issued_at: Timex.shift(Timex.now(), days: -1) - }]) + Repo.insert_all("email_verification_codes", [ + %{ + code: 1234, + user_id: user.id, + issued_at: Timex.shift(Timex.now(), days: -1) + } + ]) conn = post(conn, "/activate", %{code: "1234"}) @@ -134,7 +143,11 @@ defmodule PlausibleWeb.AuthControllerTest do Repo.update!(Plausible.Auth.User.changeset(user, %{email_verified: false})) post(conn, "/activate/request-code") - code = Repo.one(from c in "email_verification_codes", where: c.user_id == ^user.id, select: c.code) |> Integer.to_string + code = + Repo.one( + from c in "email_verification_codes", where: c.user_id == ^user.id, select: c.code + ) + |> Integer.to_string() conn = post(conn, "/activate", %{code: code}) user = Repo.get_by(Plausible.Auth.User, id: user.id) @@ -147,7 +160,11 @@ defmodule PlausibleWeb.AuthControllerTest do Repo.update!(Plausible.Auth.User.changeset(user, %{email_verified: false})) post(conn, "/activate/request-code") - code = Repo.one(from c in "email_verification_codes", where: c.user_id == ^user.id, select: c.code) |> Integer.to_string + code = + Repo.one( + from c in "email_verification_codes", where: c.user_id == ^user.id, select: c.code + ) + |> Integer.to_string() post(conn, "/activate", %{code: code}) diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index 8b1e7c1f4..751dcfad5 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -57,7 +57,10 @@ defmodule PlausibleWeb.SiteControllerTest do assert_email_delivered_with(subject: "Welcome to Plausible") end - test "does not send welcome email if user already has a previous site", %{conn: conn, user: user} do + test "does not send welcome email if user already has a previous site", %{ + conn: conn, + user: user + } do insert(:site, members: [user]) post(conn, "/sites", %{ @@ -395,7 +398,11 @@ defmodule PlausibleWeb.SiteControllerTest do describe "POST /sites/:website/spike-notification/enable" do setup [:create_user, :log_in, :create_site] - test "creates a spike notification record with the user email", %{conn: conn, site: site, user: user} do + test "creates a spike notification record with the user email", %{ + conn: conn, + site: site, + user: user + } do post(conn, "/sites/#{site.domain}/spike-notification/enable") notification = Repo.get_by(Plausible.Site.SpikeNotification, site_id: site.id) @@ -420,7 +427,10 @@ defmodule PlausibleWeb.SiteControllerTest do test "updates spike notification threshold", %{conn: conn, site: site} do insert(:spike_notification, site: site, threshold: 10) - put(conn, "/sites/#{site.domain}/spike-notification", %{"spike_notification" => %{"threshold" => "15"}}) + + put(conn, "/sites/#{site.domain}/spike-notification", %{ + "spike_notification" => %{"threshold" => "15"} + }) notification = Repo.get_by(Plausible.Site.SpikeNotification, site_id: site.id) assert notification.threshold == 15 @@ -433,7 +443,9 @@ defmodule PlausibleWeb.SiteControllerTest do test "adds a recipient to the spike notification", %{conn: conn, site: site} do insert(:spike_notification, site: site) - post(conn, "/sites/#{site.domain}/spike-notification/recipients", recipient: "user@email.com") + post(conn, "/sites/#{site.domain}/spike-notification/recipients", + recipient: "user@email.com" + ) report = Repo.get_by(Plausible.Site.SpikeNotification, site_id: site.id) assert report.recipients == ["user@email.com"] diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex index 73ba2e514..89f085c0a 100644 --- a/test/support/clickhouse_setup.ex +++ b/test/support/clickhouse_setup.ex @@ -17,7 +17,7 @@ defmodule Plausible.Test.ClickhouseSetup do referrer_source: "10words", referrer: "10words.com/page1", timestamp: ~N[2019-01-01 00:00:00], - session_id: @conversion_1_session_id, + session_id: @conversion_1_session_id }, %{ name: "pageview", diff --git a/test/workers/clean_email_verification_codes_test.exs b/test/workers/clean_email_verification_codes_test.exs index 7062d637e..ae778065d 100644 --- a/test/workers/clean_email_verification_codes_test.exs +++ b/test/workers/clean_email_verification_codes_test.exs @@ -3,8 +3,14 @@ defmodule Plausible.Workers.CleanEmailVerificationCodesTest do alias Plausible.Workers.CleanEmailVerificationCodes defp issue_code(user, issued_at) do - code = Repo.one(from(c in "email_verification_codes", where: is_nil(c.user_id), select: c.code, limit: 1)) - Repo.update_all(from(c in "email_verification_codes", where: c.code == ^code), [set: [user_id: user.id, issued_at: issued_at]]) + code = + Repo.one( + from(c in "email_verification_codes", where: is_nil(c.user_id), select: c.code, limit: 1) + ) + + Repo.update_all(from(c in "email_verification_codes", where: c.code == ^code), + set: [user_id: user.id, issued_at: issued_at] + ) end test "cleans codes that are more than 4 hours old" do diff --git a/test/workers/spike_notifier_test.exs b/test/workers/spike_notifier_test.exs index 22a2ce158..d0fd500a2 100644 --- a/test/workers/spike_notifier_test.exs +++ b/test/workers/spike_notifier_test.exs @@ -6,10 +6,17 @@ defmodule Plausible.Workers.SpikeNotifierTest do test "does not notify anyone if current visitors does not exceed notification threshold" do site = insert(:site) - insert(:spike_notification, site: site, threshold: 10, recipients: ["jerod@example.com", "uku@example.com"]) - clickhouse_stub = stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 5 end) - |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) + insert(:spike_notification, + site: site, + threshold: 10, + recipients: ["jerod@example.com", "uku@example.com"] + ) + + clickhouse_stub = + stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 5 end) + |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) + SpikeNotifier.perform(nil, nil, clickhouse_stub) assert_no_emails_delivered() @@ -17,10 +24,17 @@ defmodule Plausible.Workers.SpikeNotifierTest do test "notifies all recipients when traffic is higher than configured threshold" do site = insert(:site) - insert(:spike_notification, site: site, threshold: 10, recipients: ["jerod@example.com", "uku@example.com"]) - clickhouse_stub = stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end) - |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) + insert(:spike_notification, + site: site, + threshold: 10, + recipients: ["jerod@example.com", "uku@example.com"] + ) + + clickhouse_stub = + stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end) + |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) + SpikeNotifier.perform(nil, nil, clickhouse_stub) assert_email_delivered_with( @@ -37,8 +51,10 @@ defmodule Plausible.Workers.SpikeNotifierTest do test "does not notify anyone if a notification already went out in the last 12 hours" do site = insert(:site) insert(:spike_notification, site: site, threshold: 10, recipients: ["uku@example.com"]) - clickhouse_stub = stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end) - |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) + + clickhouse_stub = + stub(Plausible.Stats.Clickhouse, :current_visitors, fn _site, _query -> 10 end) + |> stub(:top_sources, fn _site, _query, _limit, _page, _show_noref -> [] end) SpikeNotifier.perform(nil, nil, clickhouse_stub)