From 3bedf9281c69b125445b96b87f643dacdfb6403a Mon Sep 17 00:00:00 2001 From: Vini Brasil Date: Thu, 17 Nov 2022 21:46:42 -0300 Subject: [PATCH] Seed database with pageviews (#2449) * Seed database with pageviews This commit adds basic support for database seeding useful for testing, especially dashboard changes, like intervals. It creates two years of pageviews with random timestamps. There is lot of room for improvement, such as adding sources, entry pages, geolocation, devices, custom events, but this already helps us with testing. * Update CONTRIBUTING.md file --- CONTRIBUTING.md | 22 ++++++++++++++++------ mix.exs | 5 ++--- priv/repo/seeds.exs | 28 ++++++++++++++++++++++++++++ test/support/factory.ex | 8 ++++---- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcb4de2c8a..5996d6b6c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,21 +8,31 @@ The easiest way to get up and running is to [install](https://docs.docker.com/ge Make sure Docker, Elixir, Erlang and Node.js are all installed on your development machine. The [`.tool-versions`](https://github.com/plausible/analytics/blob/master/.tool-versions) file is available to use with [asdf](https://github.com/asdf-vm/asdf) or similar tools. -### Start the environment: +### Start the environment 1. Run both `make postgres` and `make clickhouse`. 2. You can set up everything with `make install`, alternatively run each command separately: 1. Run `mix deps.get`. This will download the required Elixir dependencies. 2. Run `mix ecto.create`. This will create the required databases in both Postgres and Clickhouse. 3. Run `mix ecto.migrate` to build the database schema. - 4. Run `npm ci --prefix assets` to install the required client-side dependencies. - 5. Run `npm ci --prefix tracker` to install the required tracker dependencies. - 6. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js` - 7. Run `mix download_country_database` to fetch geolocation database + 4. Run `mix run priv/repo/seeds.exs` to seed the database. Check the [Seeds](#Seeds) section for more. + 5. Run `npm ci --prefix assets` to install the required client-side dependencies. + 6. Run `npm ci --prefix tracker` to install the required tracker dependencies. + 7. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js` + 8. Run `mix download_country_database` to fetch geolocation database 3. Run `make server` or `mix phx.server` to start the Phoenix server. 4. The system is now available on `localhost:8000`. -### Creating an account +### Seeds + +You can optionally seed your database to automatically create an account and a site with stats: + +1. Run `mix run priv/repo/seeds.exs` to seed the database. +2. Start the server with `make server` and navigate to `http://localhost:8000/login`. +3. Log in with the following e-mail and password combination: `user@plausible.test` and `plausible`. +4. You should now have a `dummy.site` site with generated stats. + +Alternatively, you can manually create a new account: 1. Navigate to `http://localhost:8000/register` and fill in the form. 2. Fill in the rest of the forms and for the domain use `dummy.site` diff --git a/mix.exs b/mix.exs index a78b939875..ec13b6efd8 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule Plausible.MixProject do end # Specifies which paths to compile per environment. - defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(env) when env in [:test, :dev], do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] # Specifies your project dependencies. @@ -71,9 +71,8 @@ defmodule Plausible.MixProject do {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:double, "~> 0.8.0", only: :test}, {:ecto_sql, "~> 3.0"}, - {:elixir_uuid, "~> 1.2", only: :test}, {:envy, "~> 1.1.1"}, - {:ex_machina, "~> 2.3", only: :test}, + {:ex_machina, "~> 2.3", only: [:dev, :test]}, {:excoveralls, "~> 0.10", only: :test}, {:exvcr, "~> 0.11", only: :test}, {:finch, "~> 0.12.0", override: true}, diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index b4931d33bb..612dde53fa 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -9,3 +9,31 @@ # # We recommend using the bang functions (`insert!`, `update!` # and so on) as they will fail if something goes wrong. + +user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") + +site = Plausible.Factory.insert(:site, domain: "dummy.site") + +membership = Plausible.Factory.insert(:site_membership, user: user, site: site, role: :owner) + +put_random_time = fn date -> + random_time = Time.new!(:rand.uniform(23), :rand.uniform(59), 0) + + date + |> NaiveDateTime.new!(random_time) + |> NaiveDateTime.truncate(:second) +end + +Enum.flat_map(-720..0, fn day_index -> + number_of_events = :rand.uniform(500) + date = Date.add(Date.utc_today(), day_index) + + attrs = [ + domain: site.domain, + hostname: site.domain, + timestamp: fn -> put_random_time.(date) end + ] + + Plausible.Factory.build_list(number_of_events, :pageview, attrs) +end) +|> Plausible.TestUtils.populate_stats() diff --git a/test/support/factory.ex b/test/support/factory.ex index 30ca268069..9797fb2fb3 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -39,8 +39,8 @@ defmodule Plausible.Factory do %Plausible.ClickhouseSession{ sign: 1, - session_id: SipHash.hash!(hash_key(), UUID.uuid4()), - user_id: SipHash.hash!(hash_key(), UUID.uuid4()), + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), hostname: hostname, domain: hostname, referrer: "", @@ -83,8 +83,8 @@ defmodule Plausible.Factory do domain: hostname, pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), - user_id: SipHash.hash!(hash_key(), UUID.uuid4()), - session_id: SipHash.hash!(hash_key(), UUID.uuid4()), + user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), referrer: "", referrer_source: "", utm_medium: "",