analytics/test/support/clickhouse_setup.ex

defmodule Plausible.Test.ClickhouseSetup do
  def run() do
    create_events()
    create_sessions()
    load_fixtures()
  end

  def create_events() do
    drop = "DROP TABLE events"
    create = """
    CREATE TABLE events (
      timestamp DateTime,
      name String,
      domain String,
      user_id UInt64,
      session_id UInt64,
      hostname String,
      pathname String,
      referrer String,
      referrer_source String,
      initial_referrer String,
      initial_referrer_source String,
      country_code LowCardinality(FixedString(2)),
      screen_size LowCardinality(String),
      operating_system LowCardinality(String),
      browser LowCardinality(String)
    ) ENGINE = MergeTree()
    PARTITION BY toYYYYMM(timestamp)
    ORDER BY (name, domain, user_id, timestamp)
    SETTINGS index_granularity = 8192
    """

    Clickhousex.query(:clickhouse, drop, [],log: {Plausible.Clickhouse, :log, []})
    Clickhousex.query(:clickhouse, create, [],log: {Plausible.Clickhouse, :log, []})
  end

  def create_sessions() do
    drop = "DROP TABLE sessions"
    create = """
    CREATE TABLE sessions (
      session_id UInt64,
      sign Int8,
      domain String,
      user_id UInt64,
      hostname String,
      timestamp DateTime,
      start DateTime,
      is_bounce UInt8,
      entry_page String,
      exit_page String,
      pageviews Int32,
      events Int32,
      duration UInt32,
      referrer String,
      referrer_source String,
      country_code LowCardinality(FixedString(2)),
      screen_size LowCardinality(String),
      operating_system LowCardinality(String),
      browser LowCardinality(String)
    ) ENGINE = CollapsingMergeTree(sign)
    PARTITION BY toYYYYMM(start)
    ORDER BY (domain, user_id, session_id, start)
    SETTINGS index_granularity = 8192
    """

    Clickhousex.query(:clickhouse, drop, [],log: {Plausible.Clickhouse, :log, []})
    Clickhousex.query(:clickhouse, create, [],log: {Plausible.Clickhouse, :log, []})
  end

  @conversion_1_session_id 123
  @conversion_2_session_id 234

  def load_fixtures() do
    Plausible.TestUtils.create_events([
      %{name: "pageview", domain: "test-site.com", pathname: "/", country_code: "EE", browser: "Chrome", operating_system: "Mac", screen_size: "Desktop", referrer_source: "10words", referrer: "10words.com/page1", timestamp: ~N[2019-01-01 00:00:00]},
      %{name: "pageview", domain: "test-site.com", pathname: "/", country_code: "EE", browser: "Chrome", operating_system: "Mac", screen_size: "Desktop", referrer_source: "10words", referrer: "10words.com/page2", timestamp: ~N[2019-01-01 00:00:00]},
      %{name: "pageview", domain: "test-site.com", pathname: "/contact", country_code: "GB", browser: "Firefox", operating_system: "Android", screen_size: "Mobile", referrer_source: "Bing", timestamp: ~N[2019-01-01 00:00:00]},

      %{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]},

      %{name: "Signup", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 01:00:00]},
      %{name: "Signup", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 02:00:00]},
      %{name: "Signup", domain: "test-site.com", session_id: @conversion_2_session_id, timestamp: ~N[2019-01-01 02:00:00]},

      %{name: "pageview", pathname: "/register", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 23:00:00]},
      %{name: "pageview", pathname: "/register", domain: "test-site.com", session_id: @conversion_2_session_id, timestamp: ~N[2019-01-01 23:00:00]},
      %{name: "pageview", pathname: "/irrelevant", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 23:00:00]},

      %{name: "pageview", domain: "test-site.com", referrer_source: "Google", timestamp: ~N[2019-02-01 01:00:00]},
      %{name: "pageview", domain: "test-site.com", referrer_source: "Google", timestamp: ~N[2019-02-01 02:00:00]},

      %{name: "pageview", domain: "test-site.com", referrer: "t.co/some-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 01:00:00]},
      %{name: "pageview", domain: "test-site.com", referrer: "t.co/some-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 01:00:00]},
      %{name: "pageview", domain: "test-site.com", referrer: "t.co/nonexistent-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 02:00:00]},

      %{name: "pageview", domain: "test-site.com"},
      %{name: "pageview", domain: "test-site.com", timestamp: Timex.now() |> Timex.shift(minutes: -3)},
      %{name: "pageview", domain: "test-site.com", timestamp: Timex.now() |> Timex.shift(minutes: -6)},

      %{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},
      %{name: "pageview", domain: "public-site.io"},
      %{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/a-link", referrer_source: "Twitter"},
      %{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/b-link", referrer_source: "Twitter", timestamp: Timex.now() |> Timex.shift(days: -5)}
    ])

    Plausible.TestUtils.create_sessions([
      %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", session_id: @conversion_1_session_id, is_bounce: true, start: ~N[2019-01-01 02:00:00]},
      %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", session_id: @conversion_2_session_id, is_bounce: false, start: ~N[2019-01-01 02:00:00]}
    ])
  end
end