plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-28 21:47:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Implement pagination and include.total_rows

Browse Source 
Offset-based pagination is used to make sure Looker integration
is able to work as efficiently as possible. To know how many
requests users need to do `include.total_rows` option was added.
This commit is contained in:
Karl-Aksel Puulmann 2024-09-11 16:24:26 +03:00
parent c673b5f2fe
commit 38b6be8f02
9 changed files with 254 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/plausible/stats/filters/query_parser.ex
View File

@ -5,7 +5,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
@default_include %{
imports: false,
time_labels: false
time_labels: false,
total_rows: false
}
@default_pagination %{
limit: 10_000,
offset: 0
}
def parse(site, schema_type, params, now \\ nil) when is_map(params) do
@ -22,6 +28,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
{:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),
preloaded_goals <- preload_goals_if_needed(site, filters, dimensions),
query = %{
metrics: metrics,
@ -31,7 +38,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
order_by: order_by,
timezone: timezone,
preloaded_goals: preloaded_goals,
include: include
include: include,
pagination: pagination
},
:ok <- validate_order_by(query),
:ok <- validate_custom_props_access(site, query),
@ -310,18 +318,15 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}
defp parse_include(include) when is_map(include) do
with {:ok, parsed_include_list} <- parse_list(include, &parse_include_value/1) do
include = Map.merge(@default_include, Enum.into(parsed_include_list, %{}))
{:ok, include}
end
{:ok, Map.merge(@default_include, atomize_keys(include))}
end
defp parse_include_value({"imports", value}) when is_boolean(value),
do: {:ok, {:imports, value}}
defp parse_pagination(pagination) when is_map(pagination) do
{:ok, Map.merge(@default_pagination, atomize_keys(pagination))}
end
defp parse_include_value({"time_labels", value}) when is_boolean(value),
do: {:ok, {:time_labels, value}}
defp atomize_keys(map),
do: Map.new(map, fn {key, value} -> {String.to_existing_atom(key), value} end)
defp parse_filter_key_string(filter_key, error_message \\ "") do
case filter_key do

6
lib/plausible/stats/query.ex
View File

@ -20,9 +20,11 @@ defmodule Plausible.Stats.Query do
preloaded_goals: [],
include: %{
imports: false,
time_labels: false
time_labels: false,
total_rows: false
},
debug_metadata: %{}
debug_metadata: %{},
pagination: nil
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Imported, Legacy}

13
lib/plausible/stats/query_result.ex
View File

@ -26,7 +26,7 @@ defmodule Plausible.Stats.QueryResult do
struct!(
__MODULE__,
results: results_list,
meta: meta(query),
meta: meta(query, results),
query:
Jason.OrderedObject.new(
site_id: site.domain,
@ -38,7 +38,8 @@ defmodule Plausible.Stats.QueryResult do
filters: query.filters,
dimensions: query.dimensions,
order_by: query.order_by |> Enum.map(&Tuple.to_list/1),
include: query.include |> Map.filter(fn {_key, val} -> val end)
include: query.include |> Map.filter(fn {_key, val} -> val end),
pagination: query.pagination
)
)
end
@ -70,7 +71,7 @@ defmodule Plausible.Stats.QueryResult do
@imports_unsupported_interval_warning "Imported stats are not included because the time dimension (i.e. the interval) is too short."
defp meta(query) do
defp meta(query, results) do
%{
imports_included: if(query.include.imports, do: query.include_imported, else: nil),
imports_skip_reason:
@ -82,12 +83,16 @@ defmodule Plausible.Stats.QueryResult do
_ -> nil
end,
time_labels:
if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil)
if(query.include.time_labels, do: Plausible.Stats.Time.time_labels(query), else: nil),
total_rows: if(query.include.total_rows, do: total_rows(results), else: nil)
}
|> Enum.reject(fn {_, value} -> is_nil(value) end)
|> Enum.into(%{})
end
defp total_rows([]), do: 0
defp total_rows([first_row | _rest]), do: first_row.total_rows
defp to_iso8601(datetime, timezone) do
datetime
|> DateTime.shift_zone!(timezone)

18
lib/plausible/stats/sql/query_builder.ex
View File

@ -23,6 +23,8 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
{event_q, event_query},
{sessions_q, sessions_query}
)
|> paginate(query.pagination)
|> select_total_rows(query.include.total_rows)
end
defp build_events_query(_site, %Query{metrics: []}), do: nil
@ -184,6 +186,22 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> build_order_by(events_query)
end
# NOTE: Old queries do their own pagination
defp paginate(q, nil = _pagination), do: q
defp paginate(q, pagination) do
q
|> limit(^pagination.limit)
|> offset(^pagination.offset)
end
defp select_total_rows(q, false = _include_total_rows), do: q
defp select_total_rows(q, true = _include_total_rows) do
q
|> select_merge([], %{total_rows: fragment("count() over ()")})
end
def build_group_by_join(%Query{dimensions: []}), do: true
def build_group_by_join(query) do

2
lib/plausible/stats/timeseries.ex
View File

@ -32,7 +32,7 @@ defmodule Plausible.Stats.Timeseries do
dimensions: [time_dimension(query)],
order_by: [{time_dimension(query), :asc}],
v2: true,
include: %{time_labels: true, imports: query.include.imports}
include: %{time_labels: true, imports: query.include.imports, total_rows: false}
)
|> QueryOptimizer.optimize()

22
priv/json-schemas/query-api-schema.json
View File

@ -48,6 +48,11 @@
},
"imports": {
"type": "boolean"
},
"total_rows": {
"type": "boolean",
"default": false,
"description": "If set, returns the total number of result rows rows before pagination under `meta.total_rows`"
}
}
},
@ -59,6 +64,23 @@
"US/Eastern",
"UTC"
]
},
"pagination": {
"type": "object",
"properties": {
"limit": {
"type": "integer",
"minimum": 1,
"maximum": 10000,
"description": "Number of rows to limit result to."
},
"offset": {
"type": "integer",
"minimum": 0,
"default": 0,
"description": "Pagination offset."
}
}
}
},
"required": ["site_id", "metrics", "date_range"],

114
test/plausible/stats/query_parser_test.exs
View File

@ -68,7 +68,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: Map.get(date_params, "timezone", site.timezone),
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
}
@ -90,7 +91,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -131,7 +133,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
},
:internal
@ -197,7 +200,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
},
:internal
@ -322,7 +326,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -347,7 +352,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -373,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -439,7 +446,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
@ -458,7 +466,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -518,7 +527,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -565,7 +575,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -591,7 +602,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["time"],
"include" => %{"imports" => true, "time_labels" => true}
"include" => %{"imports" => true, "time_labels" => true, "total_rows" => true}
}
|> check_success(site, %{
metrics: [:visitors],
@ -600,7 +611,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["time"],
order_by: nil,
timezone: site.timezone,
include: %{imports: true, time_labels: true},
include: %{imports: true, time_labels: true, total_rows: true},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -626,6 +638,49 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
end
describe "pagination validation" do
test "setting pagination values", %{site: site} do
%{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["time"],
"pagination" => %{"limit" => 100, "offset" => 200}
}
|> check_success(site, %{
metrics: [:visitors],
utc_time_range: @date_range_day,
filters: [],
dimensions: ["time"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 100, offset: 200},
preloaded_goals: []
})
end
test "out of range limit value", %{site: site} do
%{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"pagination" => %{"limit" => 100_000}
}
|> check_error(site, "#/pagination/limit: Expected the value to be <= 10000")
end
test "out of range offset value", %{site: site} do
%{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"pagination" => %{"offset" => -5}
}
|> check_error(site, "#/pagination/offset: Expected the value to be >= 0")
end
end
describe "event:goal filter validation" do
test "valid filters", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
@ -652,7 +707,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: ^expected_timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: [
%Plausible.Goal{page_path: "/thank-you"},
%Plausible.Goal{event_name: "Signup"}
@ -932,7 +988,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -953,7 +1010,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -973,7 +1031,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:props:foobar"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1034,7 +1093,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}],
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1054,7 +1114,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:name"],
order_by: [{"event:name", :desc}],
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1121,7 +1182,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: unquote(timezone),
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
}
)
@ -1250,7 +1312,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1333,7 +1396,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:device"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1365,7 +1429,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:page"],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end
@ -1384,7 +1449,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
include: %{imports: false, time_labels: false},
include: %{imports: false, time_labels: false, total_rows: false},
pagination: %{limit: 10_000, offset: 0},
preloaded_goals: []
})
end

4
test/plausible/stats/query_result_test.exs
View File

@ -64,6 +64,10 @@ defmodule Plausible.Stats.QueryResultTest do
],
"include": {
"imports": true
},
"pagination": {
"offset": 0,
"limit": 10000
}
}
}\

99
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

@ -3225,15 +3225,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
)
])
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"property" => "event:page",
"metrics" => "bounce_rate"
})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
@ -3274,4 +3265,94 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert results2 == [%{"metrics" => [3], "dimensions" => []}]
end
end
describe "pagination" do
setup %{site: site} = context do
populate_stats(site, [
build(:pageview, pathname: "/1"),
build(:pageview, pathname: "/2"),
build(:pageview, pathname: "/3"),
build(:pageview, pathname: "/4"),
build(:pageview, pathname: "/5"),
build(:pageview, pathname: "/6"),
build(:pageview, pathname: "/7"),
build(:pageview, pathname: "/8")
])
context
end
test "pagination above total count - all results are returned", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["pageviews"],
"date_range" => "all",
"dimensions" => ["event:page"],
"order_by" => [["event:page", "asc"]],
"include" => %{"total_rows" => true},
"pagination" => %{"limit" => 10}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["/1"], "metrics" => [1]},
%{"dimensions" => ["/2"], "metrics" => [1]},
%{"dimensions" => ["/3"], "metrics" => [1]},
%{"dimensions" => ["/4"], "metrics" => [1]},
%{"dimensions" => ["/5"], "metrics" => [1]},
%{"dimensions" => ["/6"], "metrics" => [1]},
%{"dimensions" => ["/7"], "metrics" => [1]},
%{"dimensions" => ["/8"], "metrics" => [1]}
]
assert json_response(conn, 200)["meta"]["total_rows"] == 8
end
test "pagination with offset", %{conn: conn, site: site} do
query = %{
"site_id" => site.domain,
"metrics" => ["pageviews"],
"date_range" => "all",
"dimensions" => ["event:page"],
"order_by" => [["event:page", "asc"]],
"include" => %{"total_rows" => true}
}
conn1 = post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3}))
assert json_response(conn1, 200)["results"] == [
%{"dimensions" => ["/1"], "metrics" => [1]},
%{"dimensions" => ["/2"], "metrics" => [1]},
%{"dimensions" => ["/3"], "metrics" => [1]}
]
assert json_response(conn1, 200)["meta"]["total_rows"] == 8
conn2 =
post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 3}))
assert json_response(conn2, 200)["results"] == [
%{"dimensions" => ["/4"], "metrics" => [1]},
%{"dimensions" => ["/5"], "metrics" => [1]},
%{"dimensions" => ["/6"], "metrics" => [1]}
]
assert json_response(conn2, 200)["meta"]["total_rows"] == 8
conn3 =
post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 6}))
assert json_response(conn3, 200)["results"] == [
%{"dimensions" => ["/7"], "metrics" => [1]},
%{"dimensions" => ["/8"], "metrics" => [1]}
]
assert json_response(conn3, 200)["meta"]["total_rows"] == 8
conn4 =
post(conn, "/api/v2/query", Map.put(query, "pagination", %{"limit" => 3, "offset" => 9}))
assert json_response(conn4, 200)["results"] == []
end
end
end