plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-26 11:44:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Increase Google Analytics timeout and lower page size (#2326)

Browse Source 
The Google Analytics report request may take some time, especially with big
imports with thousands of pages. To mitigate the issue this commit increases
the timeout to 60s and lowers the page size to 7,500 records per request.
This commit is contained in:
Vinicius Brasil 2022-10-13 09:10:00 -03:00 committed by GitHub
parent 76806a8924
commit fb05e447f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
fixture/vcr_cassettes/google_analytics_import#1.json
View File

@ -1,7 +1,7 @@
[
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -32,7 +32,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -63,7 +63,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -94,7 +94,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -125,7 +125,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -156,7 +156,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -187,7 +187,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -218,7 +218,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -249,7 +249,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -280,7 +280,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -311,7 +311,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@ -342,7 +342,7 @@
},
{
"request": {
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},

2
lib/plausible/google/api.ex
View File

@ -114,7 +114,7 @@ defmodule Plausible.Google.Api do
expires_at :: String.t()
}
@per_page 10_000
@per_page 7_500
@max_attempts 5
@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
:ok | {:error, term()}

2
lib/plausible/google/http.ex
View File

@ -34,7 +34,7 @@ defmodule Plausible.Google.HTTP do
[{"Authorization", "Bearer #{report_request.access_token}"}],
params
)
|> http_client.request(Plausible.Finch, receive_timeout: 30_000)
|> http_client.request(Plausible.Finch, receive_timeout: 60_000)
with {:ok, %{status: 200, body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),

10
test/plausible/google/api_test.exs
View File

@ -77,11 +77,11 @@ defmodule Plausible.Google.ApiTest do
buffer: buffer
)
assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
end
test "does not fail when report does not have rows key", %{site: site, buffer: buffer} do