use ecto_ch (#2890)

* use ecto_ch

* use Ch type in BoolUInt8

* update ecto_ch and ch to latest versions

lib/plausible/clickhouse_event_v2.ex

@@ -7,12 +7,12 @@ defmodule Plausible.ClickhouseEventV2 do
 
   @primary_key false
   schema "events_v2" do
-    field :name, :string
+    field :name, Ch, type: "LowCardinality(String)"
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :hostname, :string
     field :pathname, :string
-    field :user_id, Ch.Types.UInt64
+    field :user_id, Ch, type: "UInt64"
-    field :session_id, Ch.Types.UInt64
+    field :session_id, Ch, type: "UInt64"
     field :timestamp, :naive_datetime
 
     field :referrer, :string
@@ -23,16 +23,16 @@ defmodule Plausible.ClickhouseEventV2 do
     field :utm_content, :string
     field :utm_term, :string
 
-    field :country_code, Ch.Types.FixedString, size: 2
+    field :country_code, Ch, type: "FixedString(2)"
-    field :subdivision1_code, :string
+    field :subdivision1_code, Ch, type: "LowCardinality(String)"
-    field :subdivision2_code, :string
+    field :subdivision2_code, Ch, type: "LowCardinality(String)"
-    field :city_geoname_id, Ch.Types.UInt32
+    field :city_geoname_id, Ch, type: "UInt32"
 
-    field :screen_size, :string
+    field :screen_size, Ch, type: "LowCardinality(String)"
-    field :operating_system, :string
+    field :operating_system, Ch, type: "LowCardinality(String)"
-    field :operating_system_version, :string
+    field :operating_system_version, Ch, type: "LowCardinality(String)"
-    field :browser, :string
+    field :browser, Ch, type: "LowCardinality(String)"
-    field :browser_version, :string
+    field :browser_version, Ch, type: "LowCardinality(String)"
 
     field :"meta.key", {:array, :string}
     field :"meta.value", {:array, :string}

lib/plausible/clickhouse_session_v2.ex

@@ -4,21 +4,48 @@ defmodule Plausible.ClickhouseSessionV2 do
   """
   use Ecto.Schema
 
+  defmodule BoolUInt8 do
+    @moduledoc """
+    Custom type to cast Bool as UInt8
+    """
+
+    use Ecto.Type
+
+    u8 = Ecto.ParameterizedType.init(Ch, type: "UInt8")
+
+    @impl true
+    def type, do: unquote(Macro.escape(u8))
+
+    @impl true
+    def cast(true), do: {:ok, 1}
+    def cast(false), do: {:ok, 0}
+    def cast(nil), do: {:ok, 0}
+
+    @impl true
+    def load(1), do: {:ok, true}
+    def load(0), do: {:ok, false}
+
+    @impl true
+    def dump(true), do: {:ok, 1}
+    def dump(false), do: {:ok, 0}
+    def dump(nil), do: {:ok, 0}
+  end
+
   @primary_key false
   schema "sessions_v2" do
     field :hostname, :string
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
-    field :user_id, Ch.Types.UInt64
+    field :user_id, Ch, type: "UInt64"
-    field :session_id, Ch.Types.UInt64
+    field :session_id, Ch, type: "UInt64"
 
     field :start, :naive_datetime
-    field :duration, Ch.Types.UInt32
+    field :duration, Ch, type: "UInt32"
-    field :is_bounce, :boolean
+    field :is_bounce, BoolUInt8
     field :entry_page, :string
     field :exit_page, :string
-    field :pageviews, Ch.Types.Int32
+    field :pageviews, Ch, type: "Int32"
-    field :events, Ch.Types.Int32
+    field :events, Ch, type: "Int32"
-    field :sign, Ch.Types.Int8
+    field :sign, Ch, type: "Int8"
 
     field :"entry_meta.key", {:array, :string}
     field :"entry_meta.value", {:array, :string}
@@ -31,16 +58,16 @@ defmodule Plausible.ClickhouseSessionV2 do
     field :referrer, :string
     field :referrer_source, :string
 
-    field :country_code, Ch.Types.FixedString, size: 2
+    field :country_code, Ch, type: "LowCardinality(FixedString(2))"
-    field :subdivision1_code, :string
+    field :subdivision1_code, Ch, type: "LowCardinality(String)"
-    field :subdivision2_code, :string
+    field :subdivision2_code, Ch, type: "LowCardinality(String)"
-    field :city_geoname_id, Ch.Types.UInt32
+    field :city_geoname_id, Ch, type: "UInt32"
 
-    field :screen_size, :string
+    field :screen_size, Ch, type: "LowCardinality(String)"
-    field :operating_system, :string
+    field :operating_system, Ch, type: "LowCardinality(String)"
-    field :operating_system_version, :string
+    field :operating_system_version, Ch, type: "LowCardinality(String)"
-    field :browser, :string
+    field :browser, Ch, type: "LowCardinality(String)"
-    field :browser_version, :string
+    field :browser_version, Ch, type: "LowCardinality(String)"
     field :timestamp, :naive_datetime
 
     field :transferred_from, :string

lib/plausible/data_migration/numeric_ids.ex

@@ -13,7 +13,7 @@ defmodule Plausible.DataMigration.NumericIDs do
 
     @primary_key false
     schema "domains_lookup" do
-      field :site_id, Ch.Types.UInt64
+      field :site_id, Ch, type: "UInt64"
       field :domain, :string
     end
   end

lib/plausible/google/schemas.ex

@@ -4,13 +4,13 @@ defmodule Plausible.Google.ImportedVisitor do
 
   @primary_key false
   schema "imported_visitors" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :pageviews, Ch.Types.UInt64
+    field :pageviews, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt64
+    field :bounces, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
   end
 end
 
@@ -20,17 +20,17 @@ defmodule Plausible.Google.ImportedSource do
 
   @primary_key false
   schema "imported_sources" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :source, :string
     field :utm_medium, :string
     field :utm_campaign, :string
     field :utm_content, :string
     field :utm_term, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end
 
@@ -40,14 +40,14 @@ defmodule Plausible.Google.ImportedPage do
 
   @primary_key false
   schema "imported_pages" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :hostname, :string
     field :page, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :pageviews, Ch.Types.UInt64
+    field :pageviews, Ch, type: "UInt64"
-    field :exits, Ch.Types.UInt64
+    field :exits, Ch, type: "UInt64"
-    field :time_on_page, Ch.Types.UInt64
+    field :time_on_page, Ch, type: "UInt64"
   end
 end
 
@@ -57,13 +57,13 @@ defmodule Plausible.Google.ImportedEntryPage do
 
   @primary_key false
   schema "imported_entry_pages" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :entry_page, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :entrances, Ch.Types.UInt64
+    field :entrances, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end
 
@@ -73,11 +73,11 @@ defmodule Plausible.Google.ImportedExitPage do
 
   @primary_key false
   schema "imported_exit_pages" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :exit_page, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :exits, Ch.Types.UInt64
+    field :exits, Ch, type: "UInt64"
   end
 end
 
@@ -87,15 +87,15 @@ defmodule Plausible.Google.ImportedLocation do
 
   @primary_key false
   schema "imported_locations" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :country, :string
     field :region, :string
-    field :city, Ch.Types.UInt64
+    field :city, Ch, type: "UInt64"
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end
 
@@ -105,13 +105,13 @@ defmodule Plausible.Google.ImportedDevice do
 
   @primary_key false
   schema "imported_devices" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :device, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end
 
@@ -121,13 +121,13 @@ defmodule Plausible.Google.ImportedBrowser do
 
   @primary_key false
   schema "imported_browsers" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :browser, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end
 
@@ -137,12 +137,12 @@ defmodule Plausible.Google.ImportedOperatingSystem do
 
   @primary_key false
   schema "imported_operating_systems" do
-    field :site_id, Ch.Types.UInt64
+    field :site_id, Ch, type: "UInt64"
     field :date, :date
     field :operating_system, :string
-    field :visitors, Ch.Types.UInt64
+    field :visitors, Ch, type: "UInt64"
-    field :visits, Ch.Types.UInt64
+    field :visits, Ch, type: "UInt64"
-    field :visit_duration, Ch.Types.UInt64
+    field :visit_duration, Ch, type: "UInt64"
-    field :bounces, Ch.Types.UInt32
+    field :bounces, Ch, type: "UInt32"
   end
 end

lib/plausible/ingestion/counters/record.ex

@@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do
   @primary_key false
   schema "ingest_counters" do
     field :event_timebucket, :utc_datetime
-    field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64
+    field :site_id, Ch, type: "Nullable(UInt64)"
-    field :domain, :string
+    field :domain, Ch, type: "LowCardinality(String)"
-    field :metric, :string
+    field :metric, Ch, type: "LowCardinality(String)"
-    field :value, Ch.Types.UInt64
+    field :value, Ch, type: "UInt64"
   end
 end

mix.exs

@@ -61,7 +61,7 @@ defmodule Plausible.MixProject do
       {:bcrypt_elixir, "~> 3.0"},
       {:bypass, "~> 2.1", only: [:dev, :test]},
       {:cachex, "~> 3.4"},
-      {:chto, github: "plausible/chto"},
+      {:ecto_ch, "~> 0.1.0"},
       {:combination, "~> 0.0.3"},
       {:connection, "~> 1.1", override: true},
       {:cors_plug, "~> 3.0"},

mix.lock

@@ -10,9 +10,8 @@
   "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
   "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
   "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
-  "ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},
+  "ch": {:hex, :ch, "0.1.14", "c53489b66eeb83dca63e63155c3e0de74f99ba30a15e90d0cd6b38db86be5891", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "306357bd9a92662713b6b9b4244eb94e1ef93ececa3906dbbef7392c4001c8ef"},
   "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
-  "chto": {:git, "https://github.com/plausible/chto.git", "6afe76281e617c19df9b7a705c8401f604fe091f", []},
   "cldr_utils": {:hex, :cldr_utils, "2.22.0", "5df60df2603dfeeffe26e40ab1ee565df34da288a53bb2db0d0dbd243fd646ef", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "ea14e8a6aa89ffd59a5d49baebe7ebf852cc024ac50dc2b3dabcd3786eeed657"},
   "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
@@ -26,12 +25,13 @@
   "csv": {:hex, :csv, "2.5.0", "c47b5a5221bf2e56d6e8eb79e77884046d7fd516280dc7d9b674251e0ae46246", [:mix], [{:parallel_stream, "~> 1.0.4 or ~> 1.1.0", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "e821f541487045c7591a1963eeb42afff0dfa99bdcdbeb3410795a2f59c77d34"},
   "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
   "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"},
-  "decimal": {:hex, :decimal, "2.1.0", "4a607736e18d9a77ea44fa18388bf43bb7f7df70ff2e0d23d4238f9d18a3ad7d", [:mix], [], "hexpm", "6396853b6b4bf8b1ff2c2217121f1fad4a1aff999014a10fba60e6009194f91b"},
+  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"},
   "digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"},
   "double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"},
   "ecto": {:hex, :ecto, "3.9.5", "9f0aa7ae44a1577b651c98791c6988cd1b69b21bc724e3fd67090b97f7604263", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d4f3115d8cbacdc0bfa4b742865459fb1371d0715515842a1fb17fe31920b74c"},
+  "ecto_ch": {:hex, :ecto_ch, "0.1.9", "cb8e7bbe926c73d4160f4a1d9153a7bca5b757678b648ff2d8baf7c0dad11200", [:mix], [{:ch, "~> 0.1.14", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "94c125d335581208c64b76a37d98961393f81131f8ef2a57ea4c4e66bbe42753"},
   "ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},
   "elixir_make": {:hex, :elixir_make, "0.7.6", "67716309dc5d43e16b5abbd00c01b8df6a0c2ab54a8f595468035a50189f9169", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5a0569756b0f7873a77687800c164cca6dfc03a09418e6fcf853d78991f49940"},
   "envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},