Seed database with pageviews (#2449)

* Seed database with pageviews

This commit adds basic support for database seeding useful for testing,
especially dashboard changes, like intervals.

It creates two years of pageviews with random timestamps. There is lot
of room for improvement, such as adding sources, entry pages,
geolocation, devices, custom events, but this already helps us with
testing.

* Update CONTRIBUTING.md file

CONTRIBUTING.md

@@ -8,21 +8,31 @@ The easiest way to get up and running is to [install](https://docs.docker.com/ge
 
 Make sure Docker, Elixir, Erlang and Node.js are all installed on your development machine. The [`.tool-versions`](https://github.com/plausible/analytics/blob/master/.tool-versions) file is available to use with [asdf](https://github.com/asdf-vm/asdf) or similar tools.
 
-### Start the environment:
+### Start the environment
 
 1. Run both `make postgres` and `make clickhouse`.
 2. You can set up everything with `make install`, alternatively run each command separately:
     1. Run `mix deps.get`. This will download the required Elixir dependencies.
     2. Run `mix ecto.create`. This will create the required databases in both Postgres and Clickhouse.
     3. Run `mix ecto.migrate` to build the database schema.
-    4. Run `npm ci --prefix assets` to install the required client-side dependencies.
-    5. Run `npm ci --prefix tracker` to install the required tracker dependencies.
-    6. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js`
-    7. Run `mix download_country_database` to fetch geolocation database
+    4. Run `mix run priv/repo/seeds.exs` to seed the database. Check the [Seeds](#Seeds) section for more.
+    5. Run `npm ci --prefix assets` to install the required client-side dependencies.
+    6. Run `npm ci --prefix tracker` to install the required tracker dependencies.
+    7. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js`
+    8. Run `mix download_country_database` to fetch geolocation database
 3. Run `make server` or `mix phx.server` to start the Phoenix server.
 4. The system is now available on `localhost:8000`.
 
-### Creating an account
+### Seeds
+
+You can optionally seed your database to automatically create an account and a site with stats:
+
+1. Run `mix run priv/repo/seeds.exs` to seed the database.
+2. Start the server with `make server` and navigate to `http://localhost:8000/login`.
+3. Log in with the following e-mail and password combination: `user@plausible.test` and `plausible`.
+4. You should now have a `dummy.site` site with generated stats.
+
+Alternatively, you can manually create a new account:
 
 1. Navigate to `http://localhost:8000/register` and fill in the form.
 2. Fill in the rest of the forms and for the domain use `dummy.site`

mix.exs

@@ -47,7 +47,7 @@ defmodule Plausible.MixProject do
   end
 
   # Specifies which paths to compile per environment.
-  defp elixirc_paths(:test), do: ["lib", "test/support"]
+  defp elixirc_paths(env) when env in [:test, :dev], do: ["lib", "test/support"]
   defp elixirc_paths(_), do: ["lib"]
 
   # Specifies your project dependencies.
@@ -71,9 +71,8 @@ defmodule Plausible.MixProject do
       {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false},
       {:double, "~> 0.8.0", only: :test},
       {:ecto_sql, "~> 3.0"},
-      {:elixir_uuid, "~> 1.2", only: :test},
       {:envy, "~> 1.1.1"},
-      {:ex_machina, "~> 2.3", only: :test},
+      {:ex_machina, "~> 2.3", only: [:dev, :test]},
       {:excoveralls, "~> 0.10", only: :test},
       {:exvcr, "~> 0.11", only: :test},
       {:finch, "~> 0.12.0", override: true},

priv/repo/seeds.exs

@@ -9,3 +9,31 @@
 #
 # We recommend using the bang functions (`insert!`, `update!`
 # and so on) as they will fail if something goes wrong.
+
+user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible")
+
+site = Plausible.Factory.insert(:site, domain: "dummy.site")
+
+membership = Plausible.Factory.insert(:site_membership, user: user, site: site, role: :owner)
+
+put_random_time = fn date ->
+  random_time = Time.new!(:rand.uniform(23), :rand.uniform(59), 0)
+
+  date
+  |> NaiveDateTime.new!(random_time)
+  |> NaiveDateTime.truncate(:second)
+end
+
+Enum.flat_map(-720..0, fn day_index ->
+  number_of_events = :rand.uniform(500)
+  date = Date.add(Date.utc_today(), day_index)
+
+  attrs = [
+    domain: site.domain,
+    hostname: site.domain,
+    timestamp: fn -> put_random_time.(date) end
+  ]
+
+  Plausible.Factory.build_list(number_of_events, :pageview, attrs)
+end)
+|> Plausible.TestUtils.populate_stats()

test/support/factory.ex

@@ -39,8 +39,8 @@ defmodule Plausible.Factory do
 
     %Plausible.ClickhouseSession{
       sign: 1,
-      session_id: SipHash.hash!(hash_key(), UUID.uuid4()),
-      user_id: SipHash.hash!(hash_key(), UUID.uuid4()),
+      session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+      user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
       hostname: hostname,
       domain: hostname,
       referrer: "",
@@ -83,8 +83,8 @@ defmodule Plausible.Factory do
       domain: hostname,
       pathname: "/",
       timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
-      user_id: SipHash.hash!(hash_key(), UUID.uuid4()),
-      session_id: SipHash.hash!(hash_key(), UUID.uuid4()),
+      user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+      session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
       referrer: "",
       referrer_source: "",
       utm_medium: "",