diff --git a/CHANGELOG.md b/CHANGELOG.md index ba2a7cd1dc..733a4b394a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ### Added +- Hostname Allow List in Site Settings - Pages Block List in Site Settings - Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index ee917857cf..2d219ce54b 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -92,6 +92,21 @@ defmodule Plausible.Application do interval: :timer.seconds(35), warmer_fn: :refresh_updated_recently ]}, + {Plausible.Shield.HostnameRuleCache, ttl_check_interval: false, ets_options: [:bag]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.HostnameRuleCache.All, + cache_impl: Plausible.Shield.HostnameRuleCache, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), + warmer_fn: :refresh_all + ]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.HostnameRuleCache.RecentlyUpdated, + cache_impl: Plausible.Shield.HostnameRuleCache, + interval: :timer.seconds(25), + warmer_fn: :refresh_updated_recently + ]}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, PlausibleWeb.Endpoint, {Oban, Application.get_env(:plausible, Oban)}, diff --git a/lib/plausible/ecto/types/compiled_regex.ex b/lib/plausible/ecto/types/compiled_regex.ex new file mode 100644 index 0000000000..3728a11942 --- /dev/null +++ b/lib/plausible/ecto/types/compiled_regex.ex @@ -0,0 +1,14 @@ +defmodule Plausible.Ecto.Types.CompiledRegex do + @moduledoc """ + Ensures that the regex is compiled on load + """ + use Ecto.Type + + def type, do: :string + + def cast(val) when is_binary(val), do: {:ok, val} + def cast(_), do: :error + + def load(val), do: {:ok, Regex.compile!(val)} + def dump(val), do: {:ok, val} +end diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 567f66bdb0..8baf66a923 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -30,6 +30,7 @@ defmodule Plausible.Ingestion.Event do | :site_ip_blocklist | :site_country_blocklist | :site_page_blocklist + | :site_hostname_allowlist @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -104,6 +105,7 @@ defmodule Plausible.Ingestion.Event do defp pipeline() do [ drop_datacenter_ip: &drop_datacenter_ip/1, + drop_shield_rule_hostname: &drop_shield_rule_hostname/1, drop_shield_rule_page: &drop_shield_rule_page/1, drop_shield_rule_ip: &drop_shield_rule_ip/1, put_geolocation: &put_geolocation/1, @@ -183,6 +185,14 @@ defmodule Plausible.Ingestion.Event do end end + defp drop_shield_rule_hostname(%__MODULE__{} = event) do + if Plausible.Shields.hostname_allowed?(event.domain, event.request.hostname) do + event + else + drop(event, :site_hostname_allowlist) + end + end + defp drop_shield_rule_page(%__MODULE__{} = event) do if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do drop(event, :site_page_blocklist) diff --git a/lib/plausible/shield/hostname_rule.ex b/lib/plausible/shield/hostname_rule.ex new file mode 100644 index 0000000000..27d212ff5c --- /dev/null +++ b/lib/plausible/shield/hostname_rule.ex @@ -0,0 +1,67 @@ +defmodule Plausible.Shield.HostnameRule do + @moduledoc """ + Schema for Hostnames allow list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_hostname" do + belongs_to :site, Plausible.Site + field :hostname, :string + field :hostname_pattern, Plausible.Ecto.Types.CompiledRegex + field :action, Ecto.Enum, values: [:deny, :allow], default: :allow + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule \\ %__MODULE__{}, attrs) do + rule + |> cast(attrs, [:site_id, :hostname]) + |> validate_required([:site_id, :hostname]) + |> validate_length(:hostname, max: 250) + |> store_regex() + |> unique_constraint(:hostname_pattern, + name: :shield_rules_hostname_site_id_hostname_pattern_index, + error_key: :hostname, + message: "rule already exists" + ) + end + + defp store_regex(changeset) do + case fetch_change(changeset, :hostname) do + {:ok, hostname} -> + hostname + |> build_regex() + |> verify_and_put_regex(changeset) + + :error -> + changeset + end + end + + defp build_regex(hostname) do + regex = + hostname + |> Regex.escape() + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + "^#{regex}$" + end + + defp verify_and_put_regex(regex, changeset) do + case Regex.compile(regex) do + {:ok, _} -> + put_change(changeset, :hostname_pattern, regex) + + {:error, _} -> + add_error(changeset, :hostname, "could not compile regular expression") + end + end +end diff --git a/lib/plausible/shield/hostname_rule_cache.ex b/lib/plausible/shield/hostname_rule_cache.ex new file mode 100644 index 0000000000..14f2d3aa19 --- /dev/null +++ b/lib/plausible/shield/hostname_rule_cache.ex @@ -0,0 +1,70 @@ +defmodule Plausible.Shield.HostnameRuleCache do + @moduledoc """ + Allows retrieving Hostname Rules by domain + """ + alias Plausible.Shield.HostnameRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :hostname_allowlist_by_domain + + @cached_schema_fields ~w( + id + hostname_pattern + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_hostname_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(HostnameRule, :count) + end + + @impl true + def base_db_query() do + from rule in HostnameRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source(domain) do + query = + base_db_query() + |> where([..., site], site.domain == ^domain) + + case Plausible.Repo.all(query) do + [_ | _] = results -> + Enum.map(results, fn {_, _, rule} -> + %HostnameRule{rule | from_cache?: false} + end) + + _ -> + nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {domain, object}, + {domain_changed_from, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shield/page_rule.ex b/lib/plausible/shield/page_rule.ex index e874e185c4..0a238b75b6 100644 --- a/lib/plausible/shield/page_rule.ex +++ b/lib/plausible/shield/page_rule.ex @@ -1,18 +1,3 @@ -defmodule CompiledRegex do - @moduledoc """ - Ensures that the regex is compiled on load - """ - use Ecto.Type - - def type, do: :string - - def cast(val) when is_binary(val), do: {:ok, val} - def cast(_), do: :error - - def load(val), do: {:ok, Regex.compile!(val)} - def dump(val), do: {:ok, val} -end - defmodule Plausible.Shield.PageRule do @moduledoc """ Schema for Pages block list @@ -26,7 +11,7 @@ defmodule Plausible.Shield.PageRule do schema "shield_rules_page" do belongs_to :site, Plausible.Site field :page_path, :string - field :page_path_pattern, CompiledRegex + field :page_path_pattern, Plausible.Ecto.Types.CompiledRegex field :action, Ecto.Enum, values: [:deny, :allow], default: :deny field :added_by, :string diff --git a/lib/plausible/shields.ex b/lib/plausible/shields.ex index 942b1fba09..0bf16f79f9 100644 --- a/lib/plausible/shields.ex +++ b/lib/plausible/shields.ex @@ -16,11 +16,51 @@ defmodule Plausible.Shields do @maximum_page_rules 30 def maximum_page_rules(), do: @maximum_page_rules + @maximum_hostname_rules 10 + def maximum_hostname_rules(), do: @maximum_hostname_rules + @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] def list_ip_rules(site_or_id) do list(Shield.IPRule, site_or_id) end + @spec hostname_allowed?(Site.t() | String.t(), String.t()) :: boolean() + def hostname_allowed?(%Site{domain: domain}, hostname) do + hostname_allowed?(domain, hostname) + end + + def hostname_allowed?(domain, hostname) when is_binary(domain) and is_binary(hostname) do + hostname_rules = Shield.HostnameRuleCache.get(domain) + + if hostname_rules do + hostname_rules + |> List.wrap() + |> Enum.find_value(false, fn rule -> + rule.action == :allow and + Regex.match?(rule.hostname_pattern, hostname) + end) + else + true + end + end + + @spec allowed_hostname_patterns(Site.t() | String.t()) :: list(String.t()) | :all + def allowed_hostname_patterns(%Site{domain: domain}) do + allowed_hostname_patterns(domain) + end + + def allowed_hostname_patterns(domain) when is_binary(domain) do + hostname_rules = Shield.HostnameRuleCache.get(domain) + + if hostname_rules do + hostname_rules + |> List.wrap() + |> Enum.map(&Regex.source(&1.hostname_pattern)) + else + :all + end + end + @spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean() def ip_blocked?(%Site{domain: domain}, address) do ip_blocked?(domain, address) @@ -133,6 +173,28 @@ defmodule Plausible.Shields do count(Shield.PageRule, site_or_id) end + @spec list_hostname_rules(Site.t() | non_neg_integer()) :: [Shield.HostnameRule.t()] + def list_hostname_rules(site_or_id) do + list(Shield.HostnameRule, site_or_id) + end + + @spec add_hostname_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.HostnameRule.t()} | {:error, Ecto.Changeset.t()} + def add_hostname_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:hostname, @maximum_hostname_rules}) + add(Shield.HostnameRule, site_or_id, params, opts) + end + + @spec remove_hostname_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_hostname_rule(site_or_id, rule_id) do + remove(Shield.HostnameRule, site_or_id, rule_id) + end + + @spec count_hostname_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_hostname_rules(site_or_id) do + count(Shield.HostnameRule, site_or_id) + end + defp list(schema, %Site{id: id}) do list(schema, id) end diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index fb95ac13ac..ec357b9943 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -3,6 +3,7 @@ defmodule Plausible.Stats.FilterSuggestions do use Plausible.ClickhouseRepo use Plausible.Stats.Fragments import Plausible.Stats.Base + import Ecto.Query alias Plausible.Stats.Query def filter_suggestions(site, query, "country", filter_search) do @@ -232,10 +233,21 @@ defmodule Plausible.Stats.FilterSuggestions do ) :hostname -> - from(e in q, - select: e.hostname, - where: fragment("? ilike ?", e.hostname, ^filter_query) - ) + q = + from(e in q, + select: e.hostname, + where: fragment("? ilike ?", e.hostname, ^filter_query) + ) + + case Plausible.Shields.allowed_hostname_patterns(site.domain) do + :all -> + q + + limited_to when is_list(limited_to) -> + from(e in q, + where: fragment("multiMatchAny(?, ?)", e.hostname, ^limited_to) + ) + end :entry_page -> from(e in q, diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 1463565d7a..06350bdeee 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -259,7 +259,7 @@ defmodule PlausibleWeb.SiteController do end def settings_shields(conn, %{"shield" => shield}) - when shield in ["ip_addresses", "countries", "pages"] do + when shield in ["ip_addresses", "countries", "pages", "hostnames"] do site = conn.assigns.site conn diff --git a/lib/plausible_web/live/shields/hostname_rules.ex b/lib/plausible_web/live/shields/hostname_rules.ex new file mode 100644 index 0000000000..a78027f30e --- /dev/null +++ b/lib/plausible_web/live/shields/hostname_rules.ex @@ -0,0 +1,312 @@ +defmodule PlausibleWeb.Live.Shields.HostnameRules do + @moduledoc """ + LiveView allowing hostname Rules management + """ + + use Phoenix.LiveComponent, global_prefixes: ~w(x-) + use Phoenix.HTML + + alias PlausibleWeb.Live.Components.Modal + alias Plausible.Shields + alias Plausible.Shield + + import PlausibleWeb.ErrorHelpers + + def update(assigns, socket) do + socket = + socket + |> assign( + hostname_rules_count: assigns.hostname_rules_count, + site: assigns.site, + current_user: assigns.current_user, + form: new_form() + ) + |> assign_new(:hostname_rules, fn %{site: site} -> + Shields.list_hostname_rules(site) + end) + |> assign_new(:redundant_rules, fn %{hostname_rules: hostname_rules} -> + detect_redundancy(hostname_rules) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+
+
+

+ Hostnames Allow List +

+

+ Accept incoming traffic only from familiar hostnames. +

+ + +
+
+
+ + + Add Hostname + +
+ = Shields.maximum_hostname_rules()} + class="mt-4" + title="Maximum number of hostnames reached" + > +

+ You've reached the maximum number of hostnames you can block (<%= Shields.maximum_hostname_rules() %>). Please remove one before adding another. +

+ +
+ + <.live_component module={Modal} id="hostname-rule-form-modal"> + <.form + :let={f} + for={@form} + phx-submit="save-hostname-rule" + phx-target={@myself} + class="max-w-md w-full mx-auto bg-white dark:bg-gray-800" + > +

Add Hostname to Allow List

+ + <.live_component + submit_name="hostname_rule[hostname]" + submit_value={f[:hostname].value} + display_value={f[:hostname].value || ""} + module={PlausibleWeb.Live.Components.ComboBox} + suggest_fun={fn input, options -> suggest_hostnames(input, options, @site) end} + id={f[:hostname].id} + creatable + /> + + <%= error_tag(f, :hostname) %> + +

+ You can use a wildcard (*) to match multiple hostnames. For example, + *.<%= @site.domain %> + will match all subdomains.

+ + <%= if @hostname_rules_count >= 1 do %> + Once added, we will start accepting traffic from this hostname within a few minutes. + <% else %> + NB: Once added, we will start rejecting traffic from non-matching hostnames within a few minutes. + <% end %> +

+
+ + Add Hostname → + +
+ + + +

+ No Hostname Rules configured for this Site.

+ + Traffic from all hostnames is currently accepted. + +

+
+ + + + + + + + + + <%= for rule <- @hostname_rules do %> + + + + + + + <% end %> + +
+ hostname + + Status + + Remove +
+
+ + <%= rule.hostname %> + +
+ 		+
+ + Blocked + + + Allowed + + + + + +
+ 		+ +
+
+
+
+ """ + end + + def handle_event("save-hostname-rule", %{"hostname_rule" => params}, socket) do + user = socket.assigns.current_user + + case Shields.add_hostname_rule( + socket.assigns.site.id, + params, + added_by: user + ) do + {:ok, rule} -> + hostname_rules = [rule | socket.assigns.hostname_rules] + + socket = + socket + |> Modal.close("hostname-rule-form-modal") + |> assign( + form: new_form(), + hostname_rules: hostname_rules, + hostname_rules_count: socket.assigns.hostname_rules_count + 1, + redundant_rules: detect_redundancy(hostname_rules) + ) + + # Make sure to clear the combobox input after adding a hostname rule, on subsequent modal reopening + send_update(PlausibleWeb.Live.Components.ComboBox, + id: "hostname_rule_hostname_code", + display_value: "" + ) + + send_flash( + :success, + "Hostname rule added successfully. Traffic will be limited within a few minutes." + ) + + {:noreply, socket} + + {:error, %Ecto.Changeset{} = changeset} -> + {:noreply, assign(socket, form: to_form(changeset))} + end + end + + def handle_event("remove-hostname-rule", %{"rule-id" => rule_id}, socket) do + Shields.remove_hostname_rule(socket.assigns.site.id, rule_id) + + send_flash( + :success, + "Hostname rule removed successfully. Traffic will be re-adjusted within a few minutes." + ) + + hostname_rules = Enum.reject(socket.assigns.hostname_rules, &(&1.id == rule_id)) + + {:noreply, + socket + |> assign( + hostname_rules_count: socket.assigns.hostname_rules_count - 1, + hostname_rules: hostname_rules, + redundant_rules: detect_redundancy(hostname_rules) + )} + end + + def send_flash(kind, message) do + send(self(), {:flash, kind, message}) + end + + defp new_form() do + %Shield.HostnameRule{} + |> Shield.HostnameRule.changeset(%{}) + |> to_form() + end + + defp format_added_at(dt, tz) do + dt + |> Plausible.Timezones.to_datetime_in_timezone(tz) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") + end + + def suggest_hostnames(input, _options, site) do + query = Plausible.Stats.Query.from(site, %{}) + + site + |> Plausible.Stats.filter_suggestions(query, "hostname", input) + |> Enum.map(fn %{label: label, value: value} -> {label, value} end) + end + + defp detect_redundancy(hostname_rules) do + hostname_rules + |> Enum.reduce(%{}, fn rule, acc -> + {[^rule], remaining_rules} = + Enum.split_with( + hostname_rules, + fn r -> r == rule end + ) + + conflicting = + remaining_rules + |> Enum.filter(fn candidate -> + rule + |> Map.fetch!(:hostname_pattern) + |> maybe_compile() + |> Regex.match?(candidate.hostname) + end) + |> Enum.map(& &1.id) + + Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc -> + Map.update(acc, conflicting_rule_id, [rule.hostname], fn existing -> + [rule.hostname | existing] + end) + end) + end) + end + + defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern) + defp maybe_compile(%Regex{} = pattern), do: pattern +end diff --git a/lib/plausible_web/live/shields/hostnames.ex b/lib/plausible_web/live/shields/hostnames.ex new file mode 100644 index 0000000000..3bc7b424ca --- /dev/null +++ b/lib/plausible_web/live/shields/hostnames.ex @@ -0,0 +1,53 @@ +defmodule PlausibleWeb.Live.Shields.Hostnames do + @moduledoc """ + LiveView for Hostnames Shield + """ + use PlausibleWeb, :live_view + use Phoenix.HTML + + alias Plausible.Shields + alias Plausible.Sites + + def mount( + _params, + %{ + "domain" => domain, + "current_user_id" => user_id + }, + socket + ) do + socket = + socket + |> assign_new(:site, fn -> + Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin]) + end) + |> assign_new(:hostname_rules_count, fn %{site: site} -> + Shields.count_hostname_rules(site) + end) + |> assign_new(:current_user, fn -> + Plausible.Repo.get(Plausible.Auth.User, user_id) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+ <.flash_messages flash={@flash} /> + <.live_component + module={PlausibleWeb.Live.Shields.HostnameRules} + current_user={@current_user} + hostname_rules_count={@hostname_rules_count} + site={@site} + id="hostname-rules-#{@current_user.id}" + /> +
+ """ + end + + def handle_info({:flash, kind, message}, socket) do + socket = put_live_flash(socket, kind, message) + {:noreply, socket} + end +end diff --git a/lib/plausible_web/templates/site/settings_shields.html.heex b/lib/plausible_web/templates/site/settings_shields.html.heex index 489d49daf7..a96814822b 100644 --- a/lib/plausible_web/templates/site/settings_shields.html.heex +++ b/lib/plausible_web/templates/site/settings_shields.html.heex @@ -21,4 +21,11 @@ "domain" => @site.domain } ) %> + <% "hostnames" -> %> + <%= live_render(@conn, PlausibleWeb.Live.Shields.Hostnames, + session: %{ + "site_id" => @site.id, + "domain" => @site.domain + } + ) %> <% end %> diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index dad3cb9ae9..b291234850 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -69,8 +69,9 @@ defmodule PlausibleWeb.LayoutView do [ %{key: "IP Addresses", value: "shields/ip_addresses"}, %{key: "Countries", value: "shields/countries"}, - if FunWithFlags.enabled?(:shield_pages, for: conn.assigns[:site]) do - %{key: "Pages", value: "shields/pages"} + %{key: "Pages", value: "shields/pages"}, + if FunWithFlags.enabled?(:shield_hostnames, for: conn.assigns[:site]) do + %{key: "Hostnames", value: "shields/hostnames"} end ] |> Enum.reject(&is_nil/1) diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index e9828b97e5..08f7214590 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -11,8 +11,8 @@ # and so on) as they will fail if something goes wrong. FunWithFlags.enable(:imports_exports) -FunWithFlags.enable(:shield_pages) FunWithFlags.enable(:hostname_filter) +FunWithFlags.enable(:shield_hostnames) user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 8714e0848b..9176089203 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -177,6 +177,43 @@ defmodule Plausible.Ingestion.EventTest do assert dropped.drop_reason == :site_page_blocklist end + test "event pipeline drops a request when hostname allowlist is defined and hostname is not on the list" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + domain: site.domain + } + + conn = build_conn(:post, "/api/events", payload) + + {:ok, _} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "subdomain.dummy.site"}) + + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :site_hostname_allowlist + end + + test "event pipeline passes a request when hostname allowlist is defined and hostname is on the list" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://subdomain.dummy.site", + domain: site.domain + } + + conn = build_conn(:post, "/api/events", payload) + + {:ok, _} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "subdomain.dummy.site"}) + + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request) + end + test "event pipeline drops events for site with accept_trafic_until in the past" do yesterday = Date.add(Date.utc_today(), -1) diff --git a/test/plausible/shield/hostname_rule_cache_test.exs b/test/plausible/shield/hostname_rule_cache_test.exs new file mode 100644 index 0000000000..0cc66995be --- /dev/null +++ b/test/plausible/shield/hostname_rule_cache_test.exs @@ -0,0 +1,128 @@ +defmodule Plausible.Shield.HostnameRuleCacheTest do + use Plausible.DataCase, async: true + + alias Plausible.Shield.HostnameRule + alias Plausible.Shield.HostnameRuleCache + alias Plausible.Shields + + describe "public cache interface" do + test "cache caches hostname rules", %{test: test} do + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = + Supervisor.start_link( + [ + {HostnameRuleCache, + [cache_name: test, child_id: :hostname_rules_cache_id, ets_options: [:bag]]} + ], + strategy: :one_for_one, + name: :"cache_supervisor_#{test}" + ) + + site1 = insert(:site) + site2 = insert(:site) + + {:ok, %{id: rid1}} = Shields.add_hostname_rule(site1, %{"hostname" => "1.example.com"}) + {:ok, %{id: rid2}} = Shields.add_hostname_rule(site1, %{"hostname" => "2.example.com"}) + {:ok, %{id: rid3}} = Shields.add_hostname_rule(site2, %{"hostname" => "3.example.com"}) + + :ok = HostnameRuleCache.refresh_all(cache_name: test) + + :ok = Shields.remove_hostname_rule(site1, rid1) + + # cache is stale + assert HostnameRuleCache.size(test) == 3 + + # the rule order should be deterministic, but with 1s timestamp sorting precision + # race conditions may happen during tests + assert rules = HostnameRuleCache.get(site1.domain, cache_opts) + rule_ids = Enum.map(rules, & &1.id) + assert rid1 in rule_ids + assert rid2 in rule_ids + assert length(rules) == 2 + + assert %HostnameRule{from_cache?: true, id: ^rid3} = + HostnameRuleCache.get(site2.domain, cache_opts) + + refute HostnameRuleCache.get("rogue.example.com", cache_opts) + end + + test "hostname path patterns are already compiled when fetched from cache", %{test: test} do + site = insert(:site) + + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = Shields.add_hostname_rule(site, %{"hostname" => "*example.com"}) + :ok = HostnameRuleCache.refresh_all(cache_name: test) + assert regex = HostnameRuleCache.get(site.domain, cache_opts).hostname_pattern + assert regex == ~r/^.*example\.com$/ + end + + test "cache allows lookups for hostname paths on sites with changed domain", %{test: test} do + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + site = insert(:site, domain: "new.example.com", domain_changed_from: "old.example.com") + + {:ok, _} = Shields.add_hostname_rule(site, %{"hostname" => "#{test}"}) + :ok = HostnameRuleCache.refresh_all(cache_name: test) + + assert HostnameRuleCache.get("old.example.com", cache_opts) + assert HostnameRuleCache.get("new.example.com", cache_opts) + refute HostnameRuleCache.get("rogue.example.com", cache_opts) + end + + test "refreshes only recently added hostnames rules", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain = "site1.example.com" + site = insert(:site, domain: domain) + + cache_opts = [cache_name: test, force?: true] + + yesterday = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-1 * 60 * 60 * 24) + |> NaiveDateTime.truncate(:second) + + {:ok, r1} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "test1.example.com"}) + Ecto.Changeset.change(r1, inserted_at: yesterday, updated_at: yesterday) |> Repo.update!() + {:ok, _} = Plausible.Shields.add_hostname_rule(site, %{"hostname" => "test2.example.com"}) + + assert HostnameRuleCache.get(domain, cache_opts) == nil + + assert :ok = HostnameRuleCache.refresh_updated_recently(cache_opts) + + assert %{hostname_pattern: ~r/^test2\.example\.com$/} = + HostnameRuleCache.get(domain, cache_opts) + + assert :ok = HostnameRuleCache.refresh_all(cache_opts) + + assert [_, _] = HostnameRuleCache.get(domain, cache_opts) + end + end + + test "get_from_source", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain = "site1.example.com" + site = insert(:site, domain: domain) + + cache_opts = [cache_name: test, force?: true] + + {:ok, _} = Shields.add_hostname_rule(site, %{"hostname" => "#{test}"}) + {:ok, _} = Shields.add_hostname_rule(site, %{"hostname" => "#{test}*"}) + + :ok = HostnameRuleCache.refresh_all(cache_opts) + + assert length(HostnameRuleCache.get(domain, cache_opts)) == + length(HostnameRuleCache.get_from_source(domain)) + end + + defp start_test_cache(cache_name) do + %{start: {m, f, a}} = + HostnameRuleCache.child_spec(cache_name: cache_name, ets_options: [:bag]) + + apply(m, f, a) + end +end diff --git a/lib/plausible/shields/country_test.exs b/test/plausible/shields/country_test.exs similarity index 100% rename from lib/plausible/shields/country_test.exs rename to test/plausible/shields/country_test.exs diff --git a/test/plausible/shields/hostname_test.exs b/test/plausible/shields/hostname_test.exs new file mode 100644 index 0000000000..75c7ca9951 --- /dev/null +++ b/test/plausible/shields/hostname_test.exs @@ -0,0 +1,188 @@ +defmodule Plausible.Shields.HostnameTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_hostname_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_hostname_rule(site, %{}) + assert changeset.errors == [hostname: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "lengthy", %{site: site} do + long = "/" <> :binary.copy("a", 251) + assert {:error, changeset} = add_hostname_rule(site, %{"hostname" => long}) + assert [hostname: {"should be at most %{count} character(s)", _}] = changeset.errors + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_hostname_rule(site, %{"hostname" => "/test"}) + assert {:error, changeset} = add_hostname_rule(site, %{"hostname" => "/test"}) + refute changeset.valid? + + assert changeset.errors == [ + hostname: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_hostname_site_id_hostname_pattern_index"} + ]} + ] + end + + test "equivalent rules are counted as dupes", %{site: site} do + assert {:ok, _} = add_hostname_rule(site, %{"hostname" => "test*"}) + assert {:error, changeset} = add_hostname_rule(site, %{"hostname" => "test**"}) + + assert changeset.errors == [ + hostname: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_hostname_site_id_hostname_pattern_index"} + ]} + ] + end + + test "regex storage: wildcard", %{site: site} do + assert {:ok, rule} = add_hostname_rule(site, %{"hostname" => "*test"}) + assert rule.hostname_pattern == "^.*test$" + end + + test "regex storage: no wildcard", %{site: site} do + assert {:ok, rule} = add_hostname_rule(site, %{"hostname" => "test"}) + assert rule.hostname_pattern == "^test$" + end + + test "regex storage: escaping", %{site: site} do + assert {:ok, rule} = add_hostname_rule(site, %{"hostname" => "test.*.**.|+[0-9]"}) + assert rule.hostname_pattern == "^test\\..*\\..*\\.\\|\\+\\[0\\-9\\]$" + end + + test "over limit", %{site: site} do + for i <- 1..maximum_hostname_rules() do + assert {:ok, _} = + add_hostname_rule(site, %{"hostname" => "test-#{i}"}) + end + + assert count_hostname_rules(site) == maximum_hostname_rules() + + assert {:error, changeset} = + add_hostname_rule(site, %{"hostname" => "test.limit"}) + + refute changeset.valid? + assert changeset.errors == [hostname: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_hostname_rule(site, %{"hostname" => "test.example.com"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "hostname pattern matching" do + test "no wildcard", %{site: site} do + assert {:ok, _} = add_hostname_rule(site, %{"hostname" => "test.example.com"}) + assert hostname_allowed?(site, "test.example.com") + refute hostname_allowed?(site, "subdmoain.example.com") + refute hostname_allowed?(site, "test") + end + + test "wildcard - subdomains", %{site: site} do + assert {:ok, _} = add_hostname_rule(site, %{"hostname" => "*.example.com"}) + refute hostname_allowed?(site, "example.com") + refute hostname_allowed?(site, "example.com.pl") + assert hostname_allowed?(site, "subdomain.example.com") + end + + test "wildcard - any prefix", %{site: site} do + assert {:ok, _} = add_hostname_rule(site, %{"hostname" => "*example.com"}) + assert hostname_allowed?(site, "example.com") + refute hostname_allowed?(site, "example.com.pl") + assert hostname_allowed?(site, "subdomain.example.com") + end + end + + describe "remove_hostname_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_hostname_rule(site, %{"hostname" => "test"}) + assert remove_hostname_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.HostnameRule, rule.id) + assert remove_hostname_rule(site, rule.id) == :ok + end + end + + describe "list_hostname_rules/1" do + test "empty", %{site: site} do + assert(list_hostname_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, %{id: id1}} = add_hostname_rule(site, %{"hostname" => "test1.example.com"}) + :timer.sleep(1000) + {:ok, %{id: id2}} = add_hostname_rule(site, %{"hostname" => "test2.example.com"}) + assert [%{id: ^id2}, %{id: ^id1}] = list_hostname_rules(site) + end + end + + describe "count_hostname_rules/1" do + test "counts", %{site: site} do + assert count_hostname_rules(site) == 0 + {:ok, _} = add_hostname_rule(site, %{"hostname" => "test1"}) + assert count_hostname_rules(site) == 1 + {:ok, _} = add_hostname_rule(site, %{"hostname" => "test2"}) + assert count_hostname_rules(site) == 2 + end + end + + describe "allowed_hostname_patterns/1" do + test "returns a list of regular expressions when rules are defined", %{site: site} do + {:ok, _} = add_hostname_rule(site, %{"hostname" => "example.com"}) + {:ok, _} = add_hostname_rule(site, %{"hostname" => "another.example.com"}) + {:ok, _} = add_hostname_rule(site, %{"hostname" => "app.*"}) + + allowed = allowed_hostname_patterns(site.domain) + + assert length(allowed) == 3 + assert "^example\\.com$" in allowed + assert "^another\\.example\\.com$" in allowed + assert "^app\\..*$" in allowed + end + end + + describe "Hostname Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_hostname_rules(site.id) == 0 + assert list_hostname_rules(site.id) == [] + + assert {:ok, rule} = + add_hostname_rule(site.id, %{ + "hostname" => "blog.example.com" + }) + + add_hostname_rule(site2, %{"hostname" => "portral.example.com"}) + + assert count_hostname_rules(site) == 1 + assert [%{id: rule_id}] = list_hostname_rules(site) + assert rule.id == rule_id + assert rule.hostname == "blog.example.com" + assert rule.action == :allow + refute rule.from_cache? + assert hostname_allowed?(site, "blog.example.com") + refute hostname_allowed?(site, "portal.example.com") + end + end +end diff --git a/lib/plausible/shields/ip_test.exs b/test/plausible/shields/ip_test.exs similarity index 100% rename from lib/plausible/shields/ip_test.exs rename to test/plausible/shields/ip_test.exs diff --git a/lib/plausible/shields/page_test.exs b/test/plausible/shields/page_test.exs similarity index 96% rename from lib/plausible/shields/page_test.exs rename to test/plausible/shields/page_test.exs index 6134e9903e..121672568f 100644 --- a/lib/plausible/shields/page_test.exs +++ b/test/plausible/shields/page_test.exs @@ -130,10 +130,10 @@ defmodule Plausible.Shields.PageTest do @tag :slow test "many", %{site: site} do - {:ok, r1} = add_page_rule(site, %{"page_path" => "/test"}) + {:ok, %{id: id1}} = add_page_rule(site, %{"page_path" => "/test1"}) :timer.sleep(1000) - {:ok, r2} = add_page_rule(site, %{"page_path" => "/test"}) - assert [^r2, ^r1] = list_page_rules(site) + {:ok, %{id: id2}} = add_page_rule(site, %{"page_path" => "/test2"}) + assert [%{id: ^id2}, %{id: ^id1}] = list_page_rules(site) end end diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs index 07b931901c..8017a534b0 100644 --- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs @@ -279,6 +279,64 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do assert %{"label" => "host-bob.example.com", "value" => "host-bob.example.com"} in suggestions end + test "returns suggestions for hostnames limited by shields", %{conn: conn1, user: user} do + {:ok, [site: site]} = create_new_site(%{user: user}) + Plausible.Shields.add_hostname_rule(site, %{"hostname" => "*.example.com"}) + Plausible.Shields.add_hostname_rule(site, %{"hostname" => "erin.rogue.com"}) + + populate_stats(site, [ + build(:pageview, + pathname: "/", + hostname: "host-alice.example.com" + ), + build(:pageview, + pathname: "/some-other-page", + hostname: "host-bob.example.com", + user_id: 123 + ), + build(:pageview, pathname: "/exit", hostname: "host-carol.example.com", user_id: 123), + build(:pageview, + pathname: "/", + hostname: "host-dave.rogue.com" + ), + build(:pageview, + pathname: "/", + hostname: "erin.rogue.com" + ) + ]) + + conn = + get( + conn1, + "/api/stats/#{site.domain}/suggestions/hostname?q=host" + ) + + assert json_response(conn, 200) == + [ + %{"label" => "host-alice.example.com", "value" => "host-alice.example.com"}, + %{"label" => "host-carol.example.com", "value" => "host-carol.example.com"}, + %{"label" => "host-bob.example.com", "value" => "host-bob.example.com"} + ] + + conn = + get( + conn1, + "/api/stats/#{site.domain}/suggestions/hostname?q=dave" + ) + + assert json_response(conn, 200) == [] + + conn = + get( + conn1, + "/api/stats/#{site.domain}/suggestions/hostname?q=rogue" + ) + + assert json_response(conn, 200) == [ + %{"label" => "erin.rogue.com", "value" => "erin.rogue.com"} + ] + end + test "returns suggestions for referrers", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, diff --git a/test/plausible_web/live/shields/hostnames_test.exs b/test/plausible_web/live/shields/hostnames_test.exs new file mode 100644 index 0000000000..cc30500189 --- /dev/null +++ b/test/plausible_web/live/shields/hostnames_test.exs @@ -0,0 +1,199 @@ +defmodule PlausibleWeb.Live.Shields.HostnamesTest do + use PlausibleWeb.ConnCase, async: true + + import Phoenix.LiveViewTest + import Plausible.Test.Support.HTML + + alias Plausible.Shields + + setup [:create_user, :create_site, :log_in] + + describe "Hostname Rules - static" do + test "renders hostname rules hostname with empty list", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/hostnames") + resp = html_response(conn, 200) + + assert resp =~ "No Hostname Rules configured for this Site" + assert resp =~ "Hostnames Allow List" + assert resp =~ "Traffic from all hostnames is currently accepted." + end + + test "lists hostname rules with remove actions", %{conn: conn, site: site} do + {:ok, r1} = + Shields.add_hostname_rule(site, %{"hostname" => "example.com"}) + + {:ok, r2} = + Shields.add_hostname_rule(site, %{"hostname" => "example.org"}) + + conn = get(conn, "/#{site.domain}/settings/shields/hostnames") + resp = html_response(conn, 200) + + assert resp =~ "example.com" + assert resp =~ "example.org" + + assert remove_button_1 = find(resp, "#remove-hostname-rule-#{r1.id}") + assert remove_button_2 = find(resp, "#remove-hostname-rule-#{r2.id}") + + assert text_of_attr(remove_button_1, "phx-click" == "remove-hostname-rule") + assert text_of_attr(remove_button_1, "phx-value-rule-id" == r1.id) + assert text_of_attr(remove_button_2, "phx-click" == "remove-hostname-rule") + assert text_of_attr(remove_button_2, "phx-value-rule-id" == r2.id) + end + + test "add rule button is rendered", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/hostnames") + resp = html_response(conn, 200) + + assert element_exists?(resp, ~s/button#add-hostname-rule[x-data]/) + attr = text_of_attr(resp, ~s/button#add-hostname-rule/, "x-on:click") + + assert attr =~ "open-modal" + assert attr =~ "hostname-rule-form-modal" + end + + test "add rule button is not rendered when maximum reached", %{conn: conn, site: site} do + for i <- 1..Shields.maximum_hostname_rules() do + assert {:ok, _} = + Shields.add_hostname_rule(site, %{"hostname" => "#{i}.example.com"}) + end + + conn = get(conn, "/#{site.domain}/settings/shields/hostnames") + resp = html_response(conn, 200) + + refute element_exists?(resp, ~s/button#add-hostname-rule[x-data]/) + assert resp =~ "Maximum number of hostnames reached" + assert resp =~ "You've reached the maximum number of hostnames you can block (10)" + end + end + + describe "Hostname Rules - LiveView" do + test "modal contains form", %{site: site, conn: conn} do + lv = get_liveview(conn, site) + html = render(lv) + + assert element_exists?( + html, + ~s/form[phx-submit="save-hostname-rule"] input[name="hostname_rule\[hostname\]"]/ + ) + + assert submit_button(html, ~s/form[phx-submit="save-hostname-rule"]/) + end + + test "if no rules are added yet, form displays hint", %{site: site, conn: conn} do + lv = get_liveview(conn, site) + html = render(lv) + + assert text(html) =~ + "NB: Once added, we will start rejecting traffic from non-matching hostnames within a few minutes. +" + + refute text(html) =~ "we will start accepting" + end + + test "if rules are added, form changes the hint", %{site: site, conn: conn} do + {:ok, _} = + Shields.add_hostname_rule(site, %{"hostname" => "*.example.com"}) + + lv = get_liveview(conn, site) + html = render(lv) + + refute text(html) =~ "we will start rejecting traffic" + + assert text(html) =~ + "Once added, we will start accepting traffic from this hostname within a few minutes." + end + + test "submitting a valid Hostname saves it", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "hostname_rule[hostname]" => "*.example.com" + }) + + html = render(lv) + + assert html =~ "*.example.com" + + assert [%{hostname: "*.example.com"}] = Shields.list_hostname_rules(site) + end + + test "submitting invalid Hostname renders error", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "hostname_rule[hostname]" => :binary.copy("a", 256) + }) + + html = render(lv) + assert html =~ "should be at most 250 character(s)" + end + + test "clicking Remove deletes the rule", %{conn: conn, site: site} do + {:ok, _} = + Shields.add_hostname_rule(site, %{"hostname" => "*.example.com"}) + + lv = get_liveview(conn, site) + + html = render(lv) + assert html =~ "*.example.com" + + lv |> element(~s/button[phx-click="remove-hostname-rule"]/) |> render_click() + + html = render(lv) + refute html =~ "*.example.com" + + assert Shields.count_hostname_rules(site) == 0 + end + + test "conclicting rules are annotated with a warning", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "hostname_rule[hostname]" => "*example.com" + }) + + html = render(lv) + refute html =~ "This rule might be redundant" + + lv + |> element("form") + |> render_submit(%{ + "hostname_rule[hostname]" => "subdomain.example.com" + }) + + html = render(lv) + + assert html =~ "*example.com" + assert html =~ "subdomain.example.com" + + assert html =~ + "This rule might be redundant because the following rules may match first:\n\n*example.com" + + broader_rule_id = + site + |> Shields.list_hostname_rules() + |> Enum.find(&(&1.hostname == "*example.com")) + |> Map.fetch!(:id) + + lv |> element(~s/button#remove-hostname-rule-#{broader_rule_id}/) |> render_click() + html = render(lv) + + assert html =~ "subdomain.example.com" + refute html =~ "*example.com" + refute html =~ "This rule might be redundant" + end + + defp get_liveview(conn, site) do + conn = assign(conn, :live_module, PlausibleWeb.Live.Shields) + {:ok, lv, _html} = live(conn, "/#{site.domain}/settings/shields/hostnames") + + lv + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index fb033e53f1..196f780659 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,8 +1,11 @@ +if not Enum.empty?(Path.wildcard("lib/**/*_test.exs")) do + raise "Oops, test(s) found in `lib/` directory. Move them to `test/`." +end + {:ok, _} = Application.ensure_all_started(:ex_machina) Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) Application.ensure_all_started(:double) FunWithFlags.enable(:imports_exports) -FunWithFlags.enable(:shield_pages) # Temporary flag to test `experimental_reduced_joins` flag on all tests. if System.get_env("TEST_EXPERIMENTAL_REDUCED_JOINS") == "1" do