2019-09-02 14:29:19 +03:00
|
|
|
defmodule PlausibleWeb.Router do
|
|
|
|
|
use PlausibleWeb, :router
|
|
|
|
|
@two_weeks_in_seconds 60 * 60 * 24 * 14
|
|
|
|
|
|
|
|
|
|
pipeline :browser do
|
|
|
|
|
plug :accepts, ["html"]
|
2020-08-11 11:04:26 +03:00
|
|
|
plug PlausibleWeb.Firewall
|
2019-09-02 14:29:19 +03:00
|
|
|
plug :fetch_session
|
|
|
|
|
plug :fetch_flash
|
|
|
|
|
plug :put_secure_browser_headers
|
2022-11-10 14:42:22 +03:00
|
|
|
plug PlausibleWeb.FirstLaunchPlug, redirect_to: "/register"
|
2019-09-02 14:29:19 +03:00
|
|
|
plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_seconds
|
|
|
|
|
plug PlausibleWeb.AuthPlug
|
|
|
|
|
plug PlausibleWeb.LastSeenPlug
|
|
|
|
|
end
|
|
|
|
|
|
2021-03-02 12:15:43 +03:00
|
|
|
pipeline :shared_link do
|
|
|
|
|
plug :accepts, ["html"]
|
|
|
|
|
plug :put_secure_browser_headers
|
|
|
|
|
end
|
|
|
|
|
|
2020-04-14 14:04:35 +03:00
|
|
|
pipeline :csrf do
|
|
|
|
|
plug :protect_from_forgery
|
|
|
|
|
end
|
|
|
|
|
|
2019-09-02 14:29:19 +03:00
|
|
|
pipeline :api do
|
2019-12-03 12:42:17 +03:00
|
|
|
plug :accepts, ["json"]
|
2020-08-11 11:04:26 +03:00
|
|
|
plug PlausibleWeb.Firewall
|
2019-09-02 14:29:19 +03:00
|
|
|
plug :fetch_session
|
|
|
|
|
plug PlausibleWeb.AuthPlug
|
|
|
|
|
end
|
|
|
|
|
|
2021-02-05 12:23:30 +03:00
|
|
|
pipeline :internal_stats_api do
|
2019-11-19 07:30:42 +03:00
|
|
|
plug :accepts, ["json"]
|
2020-08-11 11:04:26 +03:00
|
|
|
plug PlausibleWeb.Firewall
|
2019-11-19 07:30:42 +03:00
|
|
|
plug :fetch_session
|
2021-06-16 15:00:07 +03:00
|
|
|
plug PlausibleWeb.AuthorizeSiteAccess
|
2021-02-05 12:23:30 +03:00
|
|
|
end
|
|
|
|
|
|
2021-04-09 10:30:51 +03:00
|
|
|
pipeline :public_api do
|
2021-02-05 12:23:30 +03:00
|
|
|
plug :accepts, ["json"]
|
|
|
|
|
plug PlausibleWeb.Firewall
|
2019-11-19 07:30:42 +03:00
|
|
|
end
|
|
|
|
|
|
2022-04-21 16:43:16 +03:00
|
|
|
pipeline :flags do
|
2022-04-21 10:54:08 +03:00
|
|
|
plug :accepts, ["html"]
|
|
|
|
|
plug :put_secure_browser_headers
|
2022-04-21 16:43:16 +03:00
|
|
|
plug :fetch_session
|
|
|
|
|
plug PlausibleWeb.CRMAuthPlug
|
2022-04-21 10:54:08 +03:00
|
|
|
end
|
|
|
|
|
|
2021-10-22 12:26:07 +03:00
|
|
|
if Mix.env() == :dev do
|
2019-09-02 14:29:19 +03:00
|
|
|
forward "/sent-emails", Bamboo.SentEmailViewerPlug
|
|
|
|
|
end
|
|
|
|
|
|
2021-01-07 11:42:45 +03:00
|
|
|
use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug]
|
|
|
|
|
|
2022-04-21 16:43:16 +03:00
|
|
|
scope path: "/flags" do
|
|
|
|
|
pipe_through :flags
|
|
|
|
|
forward "/", FunWithFlags.UI.Router, namespace: "flags"
|
2022-04-21 10:54:08 +03:00
|
|
|
end
|
|
|
|
|
|
2019-11-19 07:30:42 +03:00
|
|
|
scope "/api/stats", PlausibleWeb.Api do
|
2021-02-05 12:23:30 +03:00
|
|
|
pipe_through :internal_stats_api
|
2019-11-19 07:30:42 +03:00
|
|
|
|
|
|
|
|
get "/:domain/current-visitors", StatsController, :current_visitors
|
|
|
|
|
get "/:domain/main-graph", StatsController, :main_graph
|
2022-04-13 10:38:47 +03:00
|
|
|
get "/:domain/top-stats", StatsController, :top_stats
|
2020-09-28 11:29:24 +03:00
|
|
|
get "/:domain/sources", StatsController, :sources
|
|
|
|
|
get "/:domain/utm_mediums", StatsController, :utm_mediums
|
|
|
|
|
get "/:domain/utm_sources", StatsController, :utm_sources
|
|
|
|
|
get "/:domain/utm_campaigns", StatsController, :utm_campaigns
|
2021-12-16 12:02:09 +03:00
|
|
|
get "/:domain/utm_contents", StatsController, :utm_contents
|
|
|
|
|
get "/:domain/utm_terms", StatsController, :utm_terms
|
2019-11-19 07:30:42 +03:00
|
|
|
get "/:domain/referrers/:referrer", StatsController, :referrer_drilldown
|
|
|
|
|
get "/:domain/pages", StatsController, :pages
|
2020-07-30 11:18:28 +03:00
|
|
|
get "/:domain/entry-pages", StatsController, :entry_pages
|
Adds entry and exit pages to Top Pages module (#712)
* Initial Pass
* Adds support for page visits counting by referrer
* Includes goal selection in entry and exit computation
* Adds goal-based entry and exit page stats, formatting, code cleanup
* Changelog
* Format
* Exit rate, visit duration, updated tests
* I keep forgetting to format :/
* Tests, last time
* Fixes double counting, exit rate >100%, relevant tests
* Fixes exit pages on filter and goal states
* Adds entry and exit filters, fixes various bugs
* Fixes discussed issues
* Format
* Fixes impossible case in tests
Originally, there were only 2 pageviews for `test-site.com`,`/` on `2019-01-01`, but that doesn't make sense when there were 3 sessions that exited on the same site/date.
* Format
* Removes boolean function parameter in favor of separate function
* Adds support for queries that use `page` filter as `entry-page`
* Format
* Makes loader/title interaction in sources report consistent
2021-02-26 12:02:37 +03:00
|
|
|
get "/:domain/exit-pages", StatsController, :exit_pages
|
2019-11-19 07:30:42 +03:00
|
|
|
get "/:domain/countries", StatsController, :countries
|
2021-11-23 12:39:09 +03:00
|
|
|
get "/:domain/regions", StatsController, :regions
|
|
|
|
|
get "/:domain/cities", StatsController, :cities
|
2019-11-19 07:30:42 +03:00
|
|
|
get "/:domain/browsers", StatsController, :browsers
|
2020-11-10 16:18:59 +03:00
|
|
|
get "/:domain/browser-versions", StatsController, :browser_versions
|
2019-11-19 07:30:42 +03:00
|
|
|
get "/:domain/operating-systems", StatsController, :operating_systems
|
2020-11-10 16:18:59 +03:00
|
|
|
get "/:domain/operating-system-versions", StatsController, :operating_system_versions
|
2019-11-19 07:30:42 +03:00
|
|
|
get "/:domain/screen-sizes", StatsController, :screen_sizes
|
|
|
|
|
get "/:domain/conversions", StatsController, :conversions
|
2020-10-30 11:49:41 +03:00
|
|
|
get "/:domain/property/:prop_name", StatsController, :prop_breakdown
|
2021-06-21 14:42:16 +03:00
|
|
|
get "/:domain/suggestions/:filter_name", StatsController, :filter_suggestions
|
2019-11-19 07:30:42 +03:00
|
|
|
end
|
|
|
|
|
|
2021-02-05 12:23:30 +03:00
|
|
|
scope "/api/v1/stats", PlausibleWeb.Api do
|
2021-04-09 10:30:51 +03:00
|
|
|
pipe_through [:public_api, PlausibleWeb.AuthorizeStatsApiPlug]
|
2021-02-05 12:23:30 +03:00
|
|
|
|
|
|
|
|
get "/realtime/visitors", ExternalStatsController, :realtime_visitors
|
|
|
|
|
get "/aggregate", ExternalStatsController, :aggregate
|
2021-02-22 11:21:25 +03:00
|
|
|
get "/breakdown", ExternalStatsController, :breakdown
|
2021-02-05 12:23:30 +03:00
|
|
|
get "/timeseries", ExternalStatsController, :timeseries
|
|
|
|
|
end
|
|
|
|
|
|
2021-04-09 10:30:51 +03:00
|
|
|
scope "/api/v1/sites", PlausibleWeb.Api do
|
|
|
|
|
pipe_through [:public_api, PlausibleWeb.AuthorizeSitesApiPlug]
|
|
|
|
|
|
|
|
|
|
post "/", ExternalSitesController, :create_site
|
2022-06-03 10:37:14 +03:00
|
|
|
get "/:site_id", ExternalSitesController, :get_site
|
2021-11-30 12:21:56 +03:00
|
|
|
delete "/:site_id", ExternalSitesController, :delete_site
|
2021-04-15 11:38:44 +03:00
|
|
|
put "/shared-links", ExternalSitesController, :find_or_create_shared_link
|
2021-12-09 17:08:01 +03:00
|
|
|
put "/goals", ExternalSitesController, :find_or_create_goal
|
|
|
|
|
delete "/goals/:goal_id", ExternalSitesController, :delete_goal
|
2021-04-09 10:30:51 +03:00
|
|
|
end
|
|
|
|
|
|
2019-09-02 14:29:19 +03:00
|
|
|
scope "/api", PlausibleWeb do
|
|
|
|
|
pipe_through :api
|
|
|
|
|
|
2019-10-31 06:49:46 +03:00
|
|
|
post "/event", Api.ExternalController, :event
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/error", Api.ExternalController, :error
|
2020-07-20 10:34:35 +03:00
|
|
|
get "/health", Api.ExternalController, :health
|
2022-05-27 16:01:42 +03:00
|
|
|
get "/system", Api.ExternalController, :info
|
2019-09-02 14:29:19 +03:00
|
|
|
|
|
|
|
|
post "/paddle/webhook", Api.PaddleController, :webhook
|
|
|
|
|
|
|
|
|
|
get "/:domain/status", Api.InternalController, :domain_status
|
2020-08-13 13:45:18 +03:00
|
|
|
get "/sites", Api.InternalController, :sites
|
2019-09-02 14:29:19 +03:00
|
|
|
end
|
|
|
|
|
|
|
|
|
|
scope "/", PlausibleWeb do
|
2021-04-01 10:43:32 +03:00
|
|
|
pipe_through [:browser, :csrf]
|
2019-09-02 14:29:19 +03:00
|
|
|
|
|
|
|
|
get "/register", AuthController, :register_form
|
|
|
|
|
post "/register", AuthController, :register
|
2021-06-16 15:00:07 +03:00
|
|
|
get "/register/invitation/:invitation_id", AuthController, :register_from_invitation_form
|
|
|
|
|
post "/register/invitation/:invitation_id", AuthController, :register_from_invitation
|
2020-12-15 12:30:45 +03:00
|
|
|
get "/activate", AuthController, :activate_form
|
|
|
|
|
post "/activate/request-code", AuthController, :request_activation_code
|
|
|
|
|
post "/activate", AuthController, :activate
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/login", AuthController, :login_form
|
|
|
|
|
post "/login", AuthController, :login
|
|
|
|
|
get "/password/request-reset", AuthController, :password_reset_request_form
|
|
|
|
|
post "/password/request-reset", AuthController, :password_reset_request
|
|
|
|
|
get "/password/reset", AuthController, :password_reset_form
|
|
|
|
|
post "/password/reset", AuthController, :password_reset
|
2020-04-14 14:04:35 +03:00
|
|
|
end
|
|
|
|
|
|
2021-03-02 12:15:43 +03:00
|
|
|
scope "/", PlausibleWeb do
|
|
|
|
|
pipe_through [:shared_link]
|
|
|
|
|
|
2022-02-22 22:46:57 +03:00
|
|
|
get "/share/:domain", StatsController, :shared_link
|
2021-03-02 12:15:43 +03:00
|
|
|
post "/share/:slug/authenticate", StatsController, :authenticate_shared_link
|
|
|
|
|
end
|
|
|
|
|
|
2020-04-14 14:04:35 +03:00
|
|
|
scope "/", PlausibleWeb do
|
|
|
|
|
pipe_through [:browser, :csrf]
|
|
|
|
|
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/password", AuthController, :password_form
|
|
|
|
|
post "/password", AuthController, :set_password
|
2021-03-31 16:48:56 +03:00
|
|
|
get "/logout", AuthController, :logout
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/settings", AuthController, :user_settings
|
|
|
|
|
put "/settings", AuthController, :save_settings
|
|
|
|
|
delete "/me", AuthController, :delete_me
|
2021-02-05 12:23:30 +03:00
|
|
|
get "/settings/api-keys/new", AuthController, :new_api_key
|
|
|
|
|
post "/settings/api-keys", AuthController, :create_api_key
|
|
|
|
|
delete "/settings/api-keys/:id", AuthController, :delete_api_key
|
2019-09-02 14:29:19 +03:00
|
|
|
|
|
|
|
|
get "/auth/google/callback", AuthController, :google_auth_callback
|
|
|
|
|
|
|
|
|
|
get "/", PageController, :index
|
|
|
|
|
|
|
|
|
|
get "/billing/change-plan", BillingController, :change_plan_form
|
2020-04-21 14:11:38 +03:00
|
|
|
get "/billing/change-plan/preview/:plan_id", BillingController, :change_plan_preview
|
|
|
|
|
post "/billing/change-plan/:new_plan_id", BillingController, :change_plan
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/billing/upgrade", BillingController, :upgrade
|
2021-03-16 13:03:50 +03:00
|
|
|
get "/billing/upgrade/:plan_id", BillingController, :upgrade_to_plan
|
2021-10-20 17:49:11 +03:00
|
|
|
get "/billing/upgrade/enterprise/:plan_id", BillingController, :upgrade_enterprise_plan
|
|
|
|
|
get "/billing/change-plan/enterprise/:plan_id", BillingController, :change_enterprise_plan
|
2020-08-18 12:48:01 +03:00
|
|
|
get "/billing/upgrade-success", BillingController, :upgrade_success
|
2022-10-04 15:54:54 +03:00
|
|
|
get "/billing/subscription/ping", BillingController, :ping_subscription
|
2019-09-02 14:29:19 +03:00
|
|
|
|
2020-03-31 16:16:21 +03:00
|
|
|
get "/sites", SiteController, :index
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/sites/new", SiteController, :new
|
|
|
|
|
post "/sites", SiteController, :create_site
|
|
|
|
|
post "/sites/:website/make-public", SiteController, :make_public
|
|
|
|
|
post "/sites/:website/make-private", SiteController, :make_private
|
2019-09-09 14:37:57 +03:00
|
|
|
post "/sites/:website/weekly-report/enable", SiteController, :enable_weekly_report
|
|
|
|
|
post "/sites/:website/weekly-report/disable", SiteController, :disable_weekly_report
|
2020-01-22 12:16:53 +03:00
|
|
|
post "/sites/:website/weekly-report/recipients", SiteController, :add_weekly_report_recipient
|
2020-06-08 10:35:13 +03:00
|
|
|
|
|
|
|
|
delete "/sites/:website/weekly-report/recipients/:recipient",
|
|
|
|
|
SiteController,
|
|
|
|
|
:remove_weekly_report_recipient
|
|
|
|
|
|
2019-09-09 14:37:57 +03:00
|
|
|
post "/sites/:website/monthly-report/enable", SiteController, :enable_monthly_report
|
|
|
|
|
post "/sites/:website/monthly-report/disable", SiteController, :disable_monthly_report
|
2020-06-08 10:35:13 +03:00
|
|
|
|
|
|
|
|
post "/sites/:website/monthly-report/recipients",
|
|
|
|
|
SiteController,
|
|
|
|
|
:add_monthly_report_recipient
|
|
|
|
|
|
|
|
|
|
delete "/sites/:website/monthly-report/recipients/:recipient",
|
|
|
|
|
SiteController,
|
|
|
|
|
:remove_monthly_report_recipient
|
2020-01-22 12:16:53 +03:00
|
|
|
|
2020-12-09 16:59:41 +03:00
|
|
|
post "/sites/:website/spike-notification/enable", SiteController, :enable_spike_notification
|
|
|
|
|
post "/sites/:website/spike-notification/disable", SiteController, :disable_spike_notification
|
|
|
|
|
put "/sites/:website/spike-notification", SiteController, :update_spike_notification
|
2020-12-29 16:17:27 +03:00
|
|
|
|
2020-12-09 16:59:41 +03:00
|
|
|
post "/sites/:website/spike-notification/recipients",
|
|
|
|
|
SiteController,
|
|
|
|
|
:add_spike_notification_recipient
|
|
|
|
|
|
|
|
|
|
delete "/sites/:website/spike-notification/recipients/:recipient",
|
|
|
|
|
SiteController,
|
|
|
|
|
:remove_spike_notification_recipient
|
|
|
|
|
|
2020-01-29 12:29:11 +03:00
|
|
|
get "/sites/:website/shared-links/new", SiteController, :new_shared_link
|
|
|
|
|
post "/sites/:website/shared-links", SiteController, :create_shared_link
|
2021-04-06 14:32:38 +03:00
|
|
|
get "/sites/:website/shared-links/:slug/edit", SiteController, :edit_shared_link
|
|
|
|
|
put "/sites/:website/shared-links/:slug", SiteController, :update_shared_link
|
2020-01-29 12:29:11 +03:00
|
|
|
delete "/sites/:website/shared-links/:slug", SiteController, :delete_shared_link
|
|
|
|
|
|
2020-06-30 11:00:19 +03:00
|
|
|
delete "/sites/:website/custom-domains/:id", SiteController, :delete_custom_domain
|
2020-02-26 11:54:21 +03:00
|
|
|
|
2021-06-16 15:00:07 +03:00
|
|
|
get "/sites/:website/memberships/invite", Site.MembershipController, :invite_member_form
|
|
|
|
|
post "/sites/:website/memberships/invite", Site.MembershipController, :invite_member
|
|
|
|
|
|
2022-10-27 09:39:34 +03:00
|
|
|
post "/sites/invitations/:invitation_id/accept", InvitationController, :accept_invitation
|
|
|
|
|
|
|
|
|
|
post "/sites/invitations/:invitation_id/reject", InvitationController, :reject_invitation
|
|
|
|
|
|
|
|
|
|
delete "/sites/:website/invitations/:invitation_id", InvitationController, :remove_invitation
|
2021-06-16 15:00:07 +03:00
|
|
|
|
|
|
|
|
get "/sites/:website/transfer-ownership", Site.MembershipController, :transfer_ownership_form
|
|
|
|
|
post "/sites/:website/transfer-ownership", Site.MembershipController, :transfer_ownership
|
|
|
|
|
|
|
|
|
|
put "/sites/:website/memberships/:id/role/:new_role", Site.MembershipController, :update_role
|
|
|
|
|
delete "/sites/:website/memberships/:id", Site.MembershipController, :remove_member
|
|
|
|
|
|
2020-01-22 12:16:53 +03:00
|
|
|
get "/sites/:website/weekly-report/unsubscribe", UnsubscribeController, :weekly_report
|
|
|
|
|
get "/sites/:website/monthly-report/unsubscribe", UnsubscribeController, :monthly_report
|
|
|
|
|
|
2019-09-02 14:29:19 +03:00
|
|
|
get "/:website/snippet", SiteController, :add_snippet
|
|
|
|
|
get "/:website/settings", SiteController, :settings
|
2020-11-06 14:30:38 +03:00
|
|
|
get "/:website/settings/general", SiteController, :settings_general
|
2021-06-16 15:00:07 +03:00
|
|
|
get "/:website/settings/people", SiteController, :settings_people
|
2020-11-19 15:57:55 +03:00
|
|
|
get "/:website/settings/visibility", SiteController, :settings_visibility
|
2020-11-06 14:30:38 +03:00
|
|
|
get "/:website/settings/goals", SiteController, :settings_goals
|
2020-11-16 16:38:44 +03:00
|
|
|
get "/:website/settings/search-console", SiteController, :settings_search_console
|
|
|
|
|
get "/:website/settings/email-reports", SiteController, :settings_email_reports
|
|
|
|
|
get "/:website/settings/custom-domain", SiteController, :settings_custom_domain
|
2020-11-19 15:57:55 +03:00
|
|
|
get "/:website/settings/danger-zone", SiteController, :settings_danger_zone
|
2019-10-31 08:39:51 +03:00
|
|
|
get "/:website/goals/new", SiteController, :new_goal
|
|
|
|
|
post "/:website/goals", SiteController, :create_goal
|
2019-10-31 09:20:45 +03:00
|
|
|
delete "/:website/goals/:id", SiteController, :delete_goal
|
2019-09-02 14:29:19 +03:00
|
|
|
put "/:website/settings", SiteController, :update_settings
|
2019-10-10 07:12:15 +03:00
|
|
|
put "/:website/settings/google", SiteController, :update_google_auth
|
[Continued] Google Analytics import (#1753)
* Add has_imported_stats boolean to Site
* Add Google Analytics import panel to general settings
* Get GA profiles to display in import settings panel
* Add import_from_google method as entrypoint to import data
* Add imported_visitors table
* Remove conflicting code from migration
* Import visitors data into clickhouse database
* Pass another dataset to main graph for rendering in red
This adds another entry to the JSON data returned via the main graph API
called `imported_plot`, which is similar to `plot` in form but will be
completed with previously imported data. Currently it simply returns
the values from `plot` / 2. The data is rendered in the main graph in
red without fill, and without an indicator for the present. Rationale:
imported data will not continue to grow so there is no projection
forward, only backwards.
* Hook imported GA data to dashboard timeseries plot
* Add settings option to forget imported data
* Import sources from google analytics
* Merge imported sources when queried
* Merge imported source data native data when querying sources
* Start converting metrics to atoms so they can be subqueried
This changes "visitors" and in some places "sources" to atoms. This does
not change the behaviour of the functions - the tests all pass unchanged
following this commit. This is necessary as joining subqueries requires
that the keys in `select` statements be atoms and not strings.
* Convery GA (direct) source to empty string
* Import utm campaign and utm medium from GA
* format
* Import all data types from GA into new tables
* Handle large amounts of more data more safely
* Fix some mistakes in tables
* Make GA requests in chunks of 5 queries
* Only display imported timeseries when there is no filter
* Correctly show last 30 minutes timeseries when 'realtime'
* Add with_imported key to Query struct
* Account for injected :is_not filter on sources from dashboard
* Also add tentative imported_utm_sources table
This needs a bit more work on the google import side, as GA do not
report sources and utm sources as distinct things.
* Return imported data to dashboard for rest of Sources panel
This extends the merge_imported function definition for sources to
utm_sources, utm_mediums and utm_campaigns too. This appears to be
working on the DB side but something is incomplete on the client side.
* Clear imported stats from all tables when requested
* Merge entry pages and exit pages from imported data into unfiltered dashboard view
This requires converting the `"visits"` and `"visit_duration"` metrics
to atoms so that they can be used in ecto subqueries.
* Display imported devices, browsers and OSs on dashboard
* Display imported country data on dashboard
* Add more metrics to entries/exits for modals
* make sure data is returned via API with correct keys
* Import regions and cities from GA
* Capitalize device upon import to match native data
* Leave query limits/offsets until after possibly joining with imported data
* Also import timeOnPage and pageviews for pages from GA
* imported_countries -> imported_locations
* Get timeOnPage and pageviews for pages from GA
These are needed for the pages modal, and for calculating exit rates for
exit pages.
* Add indicator to dashboard when imported data is being used
* Don't show imported data as separately line on main graph
* "bounce_rate" -> :bounce_rate, so it works in subqueries
* Drop imported browser and OS versions
These are not needed.
* Toggle displaying imported data by clicking indicator
* Parse referrers with RefInspector
- Use 'ga:fullReferrer' instead of 'ga:source'. This provides the actual
referrer host + path, whereas 'ga:source' includes utm_mediums and
other values when relevant.
- 'ga:fullReferror' does however include search engine names directly,
so they are manually checked for as RefInspector won't pick up on
these.
* Keep imported data indicator on dashboard and strikethrough when hidden
* Add unlink google button to import panel
* Rename some GA browsers and OSes to plausible versions
* Get main top pages and exit pages panels working correctly with imported data
* mix format
* Fetch time_on_pages for imported data when needed
* entry pages need to fetch bounces from GA
* "sample_percent" -> :sample_percent as only atoms can be used in subqueries
* Calculate bounce_rate for joined native and imported data for top pages modal
* Flip some query bindings around to be less misleading
* Fixup entry page modal visit durations
* mix format
* Fetch bounces and visit_duration for sources from GA
* add more source metrics used for data in modals
* Make sources modals display correct values
* imported_visitors: bounce_rate -> bounces, avg_visit_duration -> visit_duration
* Merge imported data into aggregate stats
* Reformat top graph side icons
* Ensure sample_percent is yielded from aggregate data
* filter event_props should be strings
* Hide imported data from frontend when using filter
* Fix existing tests
* fix tests
* Fix imported indicator appearing when filtering
* comma needed, lost when rebasing
* Import utm_terms and utm_content from GA
* Merge imported utm_term and utm_content
* Rename imported Countries data as Locations
* Set imported city schema field to int
* Remove utm_terms and utm_content when clearing imported
* Clean locations import from Google Analytics
- Country and region should be set to "" when GA provides "(not set)"
- City should be set to 0 for "unknown", as we cannot reliably import
city data from GA.
* Display imported region and city in dashboard
* os -> operating_system in some parts of code
The inconsistency of using os in some places and operating_system in
others causes trouble with subqueries and joins for the native and
imported data, which would require additional logic to account for. The
simplest solution is the just use a consistent word for all uses. This
doesn't make any user-facing or database changes.
* to_atom -> to_existing_atom
* format
* "events" metric -> :events
* ignore imported data when "events" in metrics
* update "bounce_rate"
* atomise some more metrics from new city and region api
* atomise some more metrics for email handlers
* "conversion_rate" -> :conversion_rate during csv export
* Move imported data stats code to own module
* Move imported timeseries function to Stats.Imported
* Use Timex.parse to import dates from GA
* has_imported_stats -> imported_source
* "time_on_page" -> :time_on_page
* Convert imported GA data to UTC
* Clean up GA request code a bit
There was some weird logic here with two separate lists that really
ought to be together, so this merges those.
* Fail sooner if GA timezone can't be identified
* Link imported tables to site by id
* imported_utm_content -> imported_utm_contents
* Imported GA from all of time
* Reorganise GA data fetch logic
- Fetch data from the start of time (2005)
- Check whether no data was fetched, and if so, inform user and don't
consider data to be imported.
* Clarify removal of "visits" data when it isn't in metrics
* Apply location filters from API
This makes it consistent with the sources etc which filter out 'Direct /
None' on the API side. These filters are used by both the native and
imported data handling code, which would otherwise both duplicate the
filters in their `where` clauses.
* Do not use changeset for setting site.imported_source
* Add all metrics to all dimensions
* Run GA import in the background
* Send email when GA import completes
* Add handler to insert imported data into tests and imported_browsers_factory
* Add remaining import data test factories
* Add imported location data to test
* Test main graph with imported data
* Add imported data to operating systems tests
* Add imported data to pages tests
* Add imported data to entry pages tests
* Add imported data to exit pages tests
* Add imported data to devices tests
* Add imported data to sources tests
* Add imported data to UTM tests
* Add new test module for the data import step
* Test import of sources GA data
* Test import of utm_mediums GA data
* Test import of utm_campaigns GA data
* Add tests for UTM terms
* Add tests for UTM contents
* Add test for importing pages and entry pages data from GA
* Add test for importing exit page data
* Fix module file name typo
* Add test for importing location data from GA
* Add test for importing devices data from GA
* Add test for importing browsers data from GA
* Add test for importing OS data from GA
* Paginate GA requests to download all data
* Bump clickhouse_ecto version
* Move RefInspector wrapper function into module
* Drop timezone transform on import
* Order imported by side_id then date
* More strings -> atoms
Also changes a conditional to be a bit nicer
* Remove parallelisation of data import
* Split sources and UTM sources from fetched GA data
GA has only a "source" dimension and no "UTM source" dimension. Instead
it returns these combined. The logic herein to tease these apart is:
1. "(direct)" -> it's a direct source
2. if the source is a domain -> it's a source
3. "google" -> it's from adwords; let's make this a UTM source "adwords"
4. else -> just a UTM source
* Keep prop names in queries as strings
* fix typo
* Fix import
* Insert data to clickhouse in batches
* Fix link when removing imported data
* Merge source tables
* Import hostname as well as pathname
* Record start and end time of imported data
* Track import progress
* Fix month interval with imported data
* Do not JOIN when imported date range has no overlap
* Fix time on page using exits
Co-authored-by: mcol <mcol@posteo.net>
2022-03-11 00:04:59 +03:00
|
|
|
delete "/:website/settings/google-search", SiteController, :delete_google_auth
|
|
|
|
|
delete "/:website/settings/google-import", SiteController, :delete_google_auth
|
2019-09-02 14:29:19 +03:00
|
|
|
delete "/:website", SiteController, :delete_site
|
2020-07-16 13:07:45 +03:00
|
|
|
delete "/:website/stats", SiteController, :reset_stats
|
2019-09-02 14:29:19 +03:00
|
|
|
|
2022-03-24 12:49:45 +03:00
|
|
|
get "/:website/import/google-analytics/view-id",
|
|
|
|
|
SiteController,
|
|
|
|
|
:import_from_google_view_id_form
|
|
|
|
|
|
2022-03-22 17:09:45 +03:00
|
|
|
post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id
|
2022-03-24 12:49:45 +03:00
|
|
|
|
|
|
|
|
get "/:website/import/google-analytics/user-metric",
|
|
|
|
|
SiteController,
|
|
|
|
|
:import_from_google_user_metric_notice
|
|
|
|
|
|
2022-03-22 17:09:45 +03:00
|
|
|
get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm
|
[Continued] Google Analytics import (#1753)
* Add has_imported_stats boolean to Site
* Add Google Analytics import panel to general settings
* Get GA profiles to display in import settings panel
* Add import_from_google method as entrypoint to import data
* Add imported_visitors table
* Remove conflicting code from migration
* Import visitors data into clickhouse database
* Pass another dataset to main graph for rendering in red
This adds another entry to the JSON data returned via the main graph API
called `imported_plot`, which is similar to `plot` in form but will be
completed with previously imported data. Currently it simply returns
the values from `plot` / 2. The data is rendered in the main graph in
red without fill, and without an indicator for the present. Rationale:
imported data will not continue to grow so there is no projection
forward, only backwards.
* Hook imported GA data to dashboard timeseries plot
* Add settings option to forget imported data
* Import sources from google analytics
* Merge imported sources when queried
* Merge imported source data native data when querying sources
* Start converting metrics to atoms so they can be subqueried
This changes "visitors" and in some places "sources" to atoms. This does
not change the behaviour of the functions - the tests all pass unchanged
following this commit. This is necessary as joining subqueries requires
that the keys in `select` statements be atoms and not strings.
* Convery GA (direct) source to empty string
* Import utm campaign and utm medium from GA
* format
* Import all data types from GA into new tables
* Handle large amounts of more data more safely
* Fix some mistakes in tables
* Make GA requests in chunks of 5 queries
* Only display imported timeseries when there is no filter
* Correctly show last 30 minutes timeseries when 'realtime'
* Add with_imported key to Query struct
* Account for injected :is_not filter on sources from dashboard
* Also add tentative imported_utm_sources table
This needs a bit more work on the google import side, as GA do not
report sources and utm sources as distinct things.
* Return imported data to dashboard for rest of Sources panel
This extends the merge_imported function definition for sources to
utm_sources, utm_mediums and utm_campaigns too. This appears to be
working on the DB side but something is incomplete on the client side.
* Clear imported stats from all tables when requested
* Merge entry pages and exit pages from imported data into unfiltered dashboard view
This requires converting the `"visits"` and `"visit_duration"` metrics
to atoms so that they can be used in ecto subqueries.
* Display imported devices, browsers and OSs on dashboard
* Display imported country data on dashboard
* Add more metrics to entries/exits for modals
* make sure data is returned via API with correct keys
* Import regions and cities from GA
* Capitalize device upon import to match native data
* Leave query limits/offsets until after possibly joining with imported data
* Also import timeOnPage and pageviews for pages from GA
* imported_countries -> imported_locations
* Get timeOnPage and pageviews for pages from GA
These are needed for the pages modal, and for calculating exit rates for
exit pages.
* Add indicator to dashboard when imported data is being used
* Don't show imported data as separately line on main graph
* "bounce_rate" -> :bounce_rate, so it works in subqueries
* Drop imported browser and OS versions
These are not needed.
* Toggle displaying imported data by clicking indicator
* Parse referrers with RefInspector
- Use 'ga:fullReferrer' instead of 'ga:source'. This provides the actual
referrer host + path, whereas 'ga:source' includes utm_mediums and
other values when relevant.
- 'ga:fullReferror' does however include search engine names directly,
so they are manually checked for as RefInspector won't pick up on
these.
* Keep imported data indicator on dashboard and strikethrough when hidden
* Add unlink google button to import panel
* Rename some GA browsers and OSes to plausible versions
* Get main top pages and exit pages panels working correctly with imported data
* mix format
* Fetch time_on_pages for imported data when needed
* entry pages need to fetch bounces from GA
* "sample_percent" -> :sample_percent as only atoms can be used in subqueries
* Calculate bounce_rate for joined native and imported data for top pages modal
* Flip some query bindings around to be less misleading
* Fixup entry page modal visit durations
* mix format
* Fetch bounces and visit_duration for sources from GA
* add more source metrics used for data in modals
* Make sources modals display correct values
* imported_visitors: bounce_rate -> bounces, avg_visit_duration -> visit_duration
* Merge imported data into aggregate stats
* Reformat top graph side icons
* Ensure sample_percent is yielded from aggregate data
* filter event_props should be strings
* Hide imported data from frontend when using filter
* Fix existing tests
* fix tests
* Fix imported indicator appearing when filtering
* comma needed, lost when rebasing
* Import utm_terms and utm_content from GA
* Merge imported utm_term and utm_content
* Rename imported Countries data as Locations
* Set imported city schema field to int
* Remove utm_terms and utm_content when clearing imported
* Clean locations import from Google Analytics
- Country and region should be set to "" when GA provides "(not set)"
- City should be set to 0 for "unknown", as we cannot reliably import
city data from GA.
* Display imported region and city in dashboard
* os -> operating_system in some parts of code
The inconsistency of using os in some places and operating_system in
others causes trouble with subqueries and joins for the native and
imported data, which would require additional logic to account for. The
simplest solution is the just use a consistent word for all uses. This
doesn't make any user-facing or database changes.
* to_atom -> to_existing_atom
* format
* "events" metric -> :events
* ignore imported data when "events" in metrics
* update "bounce_rate"
* atomise some more metrics from new city and region api
* atomise some more metrics for email handlers
* "conversion_rate" -> :conversion_rate during csv export
* Move imported data stats code to own module
* Move imported timeseries function to Stats.Imported
* Use Timex.parse to import dates from GA
* has_imported_stats -> imported_source
* "time_on_page" -> :time_on_page
* Convert imported GA data to UTC
* Clean up GA request code a bit
There was some weird logic here with two separate lists that really
ought to be together, so this merges those.
* Fail sooner if GA timezone can't be identified
* Link imported tables to site by id
* imported_utm_content -> imported_utm_contents
* Imported GA from all of time
* Reorganise GA data fetch logic
- Fetch data from the start of time (2005)
- Check whether no data was fetched, and if so, inform user and don't
consider data to be imported.
* Clarify removal of "visits" data when it isn't in metrics
* Apply location filters from API
This makes it consistent with the sources etc which filter out 'Direct /
None' on the API side. These filters are used by both the native and
imported data handling code, which would otherwise both duplicate the
filters in their `where` clauses.
* Do not use changeset for setting site.imported_source
* Add all metrics to all dimensions
* Run GA import in the background
* Send email when GA import completes
* Add handler to insert imported data into tests and imported_browsers_factory
* Add remaining import data test factories
* Add imported location data to test
* Test main graph with imported data
* Add imported data to operating systems tests
* Add imported data to pages tests
* Add imported data to entry pages tests
* Add imported data to exit pages tests
* Add imported data to devices tests
* Add imported data to sources tests
* Add imported data to UTM tests
* Add new test module for the data import step
* Test import of sources GA data
* Test import of utm_mediums GA data
* Test import of utm_campaigns GA data
* Add tests for UTM terms
* Add tests for UTM contents
* Add test for importing pages and entry pages data from GA
* Add test for importing exit page data
* Fix module file name typo
* Add test for importing location data from GA
* Add test for importing devices data from GA
* Add test for importing browsers data from GA
* Add test for importing OS data from GA
* Paginate GA requests to download all data
* Bump clickhouse_ecto version
* Move RefInspector wrapper function into module
* Drop timezone transform on import
* Order imported by side_id then date
* More strings -> atoms
Also changes a conditional to be a bit nicer
* Remove parallelisation of data import
* Split sources and UTM sources from fetched GA data
GA has only a "source" dimension and no "UTM source" dimension. Instead
it returns these combined. The logic herein to tease these apart is:
1. "(direct)" -> it's a direct source
2. if the source is a domain -> it's a source
3. "google" -> it's from adwords; let's make this a UTM source "adwords"
4. else -> just a UTM source
* Keep prop names in queries as strings
* fix typo
* Fix import
* Insert data to clickhouse in batches
* Fix link when removing imported data
* Merge source tables
* Import hostname as well as pathname
* Record start and end time of imported data
* Track import progress
* Fix month interval with imported data
* Do not JOIN when imported date range has no overlap
* Fix time on page using exits
Co-authored-by: mcol <mcol@posteo.net>
2022-03-11 00:04:59 +03:00
|
|
|
post "/:website/settings/google-import", SiteController, :import_from_google
|
|
|
|
|
delete "/:website/settings/forget-imported", SiteController, :forget_imported
|
2022-03-21 13:47:27 +03:00
|
|
|
|
|
|
|
|
get "/:domain/export", StatsController, :csv_export
|
|
|
|
|
get "/:domain/*path", StatsController, :stats
|
2019-09-02 14:29:19 +03:00
|
|
|
end
|
|
|
|
|
end
|
