plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-26 00:24:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

use ecto_ch (#2890)

Browse Source 
* use ecto_ch

* use Ch type in BoolUInt8

* update ecto_ch and ch to latest versions
This commit is contained in:
ruslandoga 2023-05-25 15:26:50 +08:00 committed by GitHub
parent dafb60164a
commit 447ed12155
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 111 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/plausible/clickhouse_event_v2.ex
View File

@ -7,12 +7,12 @@ defmodule Plausible.ClickhouseEventV2 do
@primary_key false
schema "events_v2" do
field :name, :string
field :site_id, Ch.Types.UInt64
field :name, Ch, type: "LowCardinality(String)"
field :site_id, Ch, type: "UInt64"
field :hostname, :string
field :pathname, :string
field :user_id, Ch.Types.UInt64
field :session_id, Ch.Types.UInt64
field :user_id, Ch, type: "UInt64"
field :session_id, Ch, type: "UInt64"
field :timestamp, :naive_datetime
field :referrer, :string
@ -23,16 +23,16 @@ defmodule Plausible.ClickhouseEventV2 do
field :utm_content, :string
field :utm_term, :string
field :country_code, Ch.Types.FixedString, size: 2
field :subdivision1_code, :string
field :subdivision2_code, :string
field :city_geoname_id, Ch.Types.UInt32
field :country_code, Ch, type: "FixedString(2)"
field :subdivision1_code, Ch, type: "LowCardinality(String)"
field :subdivision2_code, Ch, type: "LowCardinality(String)"
field :city_geoname_id, Ch, type: "UInt32"
field :screen_size, :string
field :operating_system, :string
field :operating_system_version, :string
field :browser, :string
field :browser_version, :string
field :screen_size, Ch, type: "LowCardinality(String)"
field :operating_system, Ch, type: "LowCardinality(String)"
field :operating_system_version, Ch, type: "LowCardinality(String)"
field :browser, Ch, type: "LowCardinality(String)"
field :browser_version, Ch, type: "LowCardinality(String)"
field :"meta.key", {:array, :string}
field :"meta.value", {:array, :string}

61
lib/plausible/clickhouse_session_v2.ex
View File

@ -4,21 +4,48 @@ defmodule Plausible.ClickhouseSessionV2 do
"""
use Ecto.Schema
defmodule BoolUInt8 do
@moduledoc """
Custom type to cast Bool as UInt8
"""
use Ecto.Type
u8 = Ecto.ParameterizedType.init(Ch, type: "UInt8")
@impl true
def type, do: unquote(Macro.escape(u8))
@impl true
def cast(true), do: {:ok, 1}
def cast(false), do: {:ok, 0}
def cast(nil), do: {:ok, 0}
@impl true
def load(1), do: {:ok, true}
def load(0), do: {:ok, false}
@impl true
def dump(true), do: {:ok, 1}
def dump(false), do: {:ok, 0}
def dump(nil), do: {:ok, 0}
end
@primary_key false
schema "sessions_v2" do
field :hostname, :string
field :site_id, Ch.Types.UInt64
field :user_id, Ch.Types.UInt64
field :session_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :user_id, Ch, type: "UInt64"
field :session_id, Ch, type: "UInt64"
field :start, :naive_datetime
field :duration, Ch.Types.UInt32
field :is_bounce, :boolean
field :duration, Ch, type: "UInt32"
field :is_bounce, BoolUInt8
field :entry_page, :string
field :exit_page, :string
field :pageviews, Ch.Types.Int32
field :events, Ch.Types.Int32
field :sign, Ch.Types.Int8
field :pageviews, Ch, type: "Int32"
field :events, Ch, type: "Int32"
field :sign, Ch, type: "Int8"
field :"entry_meta.key", {:array, :string}
field :"entry_meta.value", {:array, :string}
@ -31,16 +58,16 @@ defmodule Plausible.ClickhouseSessionV2 do
field :referrer, :string
field :referrer_source, :string
field :country_code, Ch.Types.FixedString, size: 2
field :subdivision1_code, :string
field :subdivision2_code, :string
field :city_geoname_id, Ch.Types.UInt32
field :country_code, Ch, type: "LowCardinality(FixedString(2))"
field :subdivision1_code, Ch, type: "LowCardinality(String)"
field :subdivision2_code, Ch, type: "LowCardinality(String)"
field :city_geoname_id, Ch, type: "UInt32"
field :screen_size, :string
field :operating_system, :string
field :operating_system_version, :string
field :browser, :string
field :browser_version, :string
field :screen_size, Ch, type: "LowCardinality(String)"
field :operating_system, Ch, type: "LowCardinality(String)"
field :operating_system_version, Ch, type: "LowCardinality(String)"
field :browser, Ch, type: "LowCardinality(String)"
field :browser_version, Ch, type: "LowCardinality(String)"
field :timestamp, :naive_datetime
field :transferred_from, :string

2
lib/plausible/data_migration/numeric_ids.ex
View File

@ -13,7 +13,7 @@ defmodule Plausible.DataMigration.NumericIDs do
@primary_key false
schema "domains_lookup" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :domain, :string
end
end

90
lib/plausible/google/schemas.ex
View File

@ -4,13 +4,13 @@ defmodule Plausible.Google.ImportedVisitor do
@primary_key false
schema "imported_visitors" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :visitors, Ch.Types.UInt64
field :pageviews, Ch.Types.UInt64
field :bounces, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
end
end
@ -20,17 +20,17 @@ defmodule Plausible.Google.ImportedSource do
@primary_key false
schema "imported_sources" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :source, :string
field :utm_medium, :string
field :utm_campaign, :string
field :utm_content, :string
field :utm_term, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end
@ -40,14 +40,14 @@ defmodule Plausible.Google.ImportedPage do
@primary_key false
schema "imported_pages" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :hostname, :string
field :page, :string
field :visitors, Ch.Types.UInt64
field :pageviews, Ch.Types.UInt64
field :exits, Ch.Types.UInt64
field :time_on_page, Ch.Types.UInt64
field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
field :time_on_page, Ch, type: "UInt64"
end
end
@ -57,13 +57,13 @@ defmodule Plausible.Google.ImportedEntryPage do
@primary_key false
schema "imported_entry_pages" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :entry_page, :string
field :visitors, Ch.Types.UInt64
field :entrances, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :visitors, Ch, type: "UInt64"
field :entrances, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end
@ -73,11 +73,11 @@ defmodule Plausible.Google.ImportedExitPage do
@primary_key false
schema "imported_exit_pages" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :exit_page, :string
field :visitors, Ch.Types.UInt64
field :exits, Ch.Types.UInt64
field :visitors, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
end
end
@ -87,15 +87,15 @@ defmodule Plausible.Google.ImportedLocation do
@primary_key false
schema "imported_locations" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :country, :string
field :region, :string
field :city, Ch.Types.UInt64
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :city, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end
@ -105,13 +105,13 @@ defmodule Plausible.Google.ImportedDevice do
@primary_key false
schema "imported_devices" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :device, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end
@ -121,13 +121,13 @@ defmodule Plausible.Google.ImportedBrowser do
@primary_key false
schema "imported_browsers" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :browser, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end
@ -137,12 +137,12 @@ defmodule Plausible.Google.ImportedOperatingSystem do
@primary_key false
schema "imported_operating_systems" do
field :site_id, Ch.Types.UInt64
field :site_id, Ch, type: "UInt64"
field :date, :date
field :operating_system, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

8
lib/plausible/ingestion/counters/record.ex
View File

@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do
@primary_key false
schema "ingest_counters" do
field :event_timebucket, :utc_datetime
field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64
field :domain, :string
field :metric, :string
field :value, Ch.Types.UInt64
field :site_id, Ch, type: "Nullable(UInt64)"
field :domain, Ch, type: "LowCardinality(String)"
field :metric, Ch, type: "LowCardinality(String)"
field :value, Ch, type: "UInt64"
end
end

2
mix.exs
View File

@ -61,7 +61,7 @@ defmodule Plausible.MixProject do
{:bcrypt_elixir, "~> 3.0"},
{:bypass, "~> 2.1", only: [:dev, :test]},
{:cachex, "~> 3.4"},
{:chto, github: "plausible/chto"},
{:ecto_ch, "~> 0.1.0"},
{:combination, "~> 0.0.3"},
{:connection, "~> 1.1", override: true},
{:cors_plug, "~> 3.0"},

6
mix.lock
View File

@ -10,9 +10,8 @@
"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
"castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
"ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},
"ch": {:hex, :ch, "0.1.14", "c53489b66eeb83dca63e63155c3e0de74f99ba30a15e90d0cd6b38db86be5891", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "306357bd9a92662713b6b9b4244eb94e1ef93ececa3906dbbef7392c4001c8ef"},
"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
"chto": {:git, "https://github.com/plausible/chto.git", "6afe76281e617c19df9b7a705c8401f604fe091f", []},
"cldr_utils": {:hex, :cldr_utils, "2.22.0", "5df60df2603dfeeffe26e40ab1ee565df34da288a53bb2db0d0dbd243fd646ef", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "ea14e8a6aa89ffd59a5d49baebe7ebf852cc024ac50dc2b3dabcd3786eeed657"},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
@ -26,12 +25,13 @@
"csv": {:hex, :csv, "2.5.0", "c47b5a5221bf2e56d6e8eb79e77884046d7fd516280dc7d9b674251e0ae46246", [:mix], [{:parallel_stream, "~> 1.0.4 or ~> 1.1.0", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "e821f541487045c7591a1963eeb42afff0dfa99bdcdbeb3410795a2f59c77d34"},
"ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
"db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"},
"decimal": {:hex, :decimal, "2.1.0", "4a607736e18d9a77ea44fa18388bf43bb7f7df70ff2e0d23d4238f9d18a3ad7d", [:mix], [], "hexpm", "6396853b6b4bf8b1ff2c2217121f1fad4a1aff999014a10fba60e6009194f91b"},
"decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"},
"digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"},
"double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"},
"earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"},
"ecto": {:hex, :ecto, "3.9.5", "9f0aa7ae44a1577b651c98791c6988cd1b69b21bc724e3fd67090b97f7604263", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d4f3115d8cbacdc0bfa4b742865459fb1371d0715515842a1fb17fe31920b74c"},
"ecto_ch": {:hex, :ecto_ch, "0.1.9", "cb8e7bbe926c73d4160f4a1d9153a7bca5b757678b648ff2d8baf7c0dad11200", [:mix], [{:ch, "~> 0.1.14", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "94c125d335581208c64b76a37d98961393f81131f8ef2a57ea4c4e66bbe42753"},
"ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},
"elixir_make": {:hex, :elixir_make, "0.7.6", "67716309dc5d43e16b5abbd00c01b8df6a0c2ab54a8f595468035a50189f9169", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5a0569756b0f7873a77687800c164cca6dfc03a09418e6fcf853d78991f49940"},
"envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},