Maxmind (#80)

* Add maxmind Geolite2 database

* Look up the actual country from MaxMind

* Fix test for country lookup

.gitignore

@@ -33,6 +33,7 @@ npm-debug.log
 # we ignore priv/static. You may want to comment
 # this depending on your deployment strategy.
 /priv/static/
+/priv/geolix/
 
 # Files matching config/*.secret.exs pattern contain sensitive
 # data and you should not commit them into version control.

config/config.exs

@@ -163,6 +163,15 @@ config :plausible, :custom_domain_server,
   password: System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD"),
   ip: System.get_env("CUSTOM_DOMAIN_SERVER_IP")
 
+config :geolix,
+  databases: [
+    %{
+      id: :geolite2_country,
+      adapter: Geolix.Adapter.MMDB2,
+      source: "priv/geolix/GeoLite2-Country.mmdb"
+    }
+  ]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/releases.exs

@@ -51,6 +51,7 @@ cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false"))
 custom_domain_server_ip = System.get_env("CUSTOM_DOMAIN_SERVER_IP")
 custom_domain_server_user = System.get_env("CUSTOM_DOMAIN_SERVER_USER")
 custom_domain_server_password = System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD")
+geolite2_country_db = System.get_env("GEOLITE2_COUNTRY_DB")
 
 config :plausible,
   admin_user: admin_user,
@@ -171,4 +172,15 @@ config :ref_inspector,
 config :ua_inspector,
   init: {Plausible.Release, :configure_ua_inspector}
 
+if geolite2_country_db do
+  config :geolix,
+    databases: [
+      %{
+        id: :country,
+        adapter: Geolix.Adapter.MMDB2,
+        source: geolite2_country_db
+      }
+    ]
+end
+
 config :logger, level: :warn

config/test.exs

@@ -44,5 +44,14 @@ config :junit_formatter,
   prepend_project_name?: true,
   include_filename?: true
 
+config :geolix,
+  databases: [
+    %{
+      id: :geolite2_country,
+      adapter: Geolix.Adapter.Fake,
+      data: %{{1, 1, 1, 1} => %{country: %{iso_code: "US"}}}
+    }
+  ]
+
 config :plausible,
   session_timeout: 0

lib/plausible_web/controllers/api/external_controller.ex

@@ -29,7 +29,6 @@ defmodule PlausibleWeb.Api.ExternalController do
 
   defp create_event(conn, params) do
     uri = params["url"] && URI.parse(params["url"])
-    country_code = Plug.Conn.get_req_header(conn, "x-country") |> List.first()
     user_agent = Plug.Conn.get_req_header(conn, "user-agent") |> List.first()
 
     if UAInspector.bot?(user_agent) do
@@ -41,6 +40,7 @@ defmodule PlausibleWeb.Api.ExternalController do
         end
 
       ref = parse_referrer(uri, params["referrer"])
+      country_code = visitor_country(conn)
 
       event_attrs = %{
         timestamp: NaiveDateTime.utc_now(),
@@ -71,6 +71,28 @@ defmodule PlausibleWeb.Api.ExternalController do
     end
   end
 
+  defp get_ip(conn) do
+    forwarded_for = List.first(Plug.Conn.get_req_header(conn, "x-forwarded-for"))
+
+    if forwarded_for do
+      String.split(forwarded_for, ",")
+      |> Enum.map(&String.trim/1)
+      |> List.first
+    else
+      to_string(:inet_parse.ntoa(conn.remote_ip))
+    end
+  end
+
+  defp visitor_country(conn) do
+    result = get_ip(conn)
+    |> Geolix.lookup()
+    |> Map.get(:geolite2_country)
+
+    if result do
+      result.country.iso_code
+    end
+  end
+
   defp parse_referrer(_, nil), do: nil
 
   defp parse_referrer(uri, referrer_str) do
@@ -87,8 +109,7 @@ defmodule PlausibleWeb.Api.ExternalController do
       |> binary_part(0, 16)
 
     user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
-    # Netlify sets this header as the remote client IP
-    ip_address = List.first(Plug.Conn.get_req_header(conn, "x-bb-ip")) || ""
+    ip_address = get_ip(conn)
     domain = strip_www(params["domain"]) || ""
 
     SipHash.hash!(hash_key, user_agent <> ip_address <> domain)

mix.exs

@@ -89,6 +89,8 @@ defmodule Plausible.MixProject do
       {:siphash, "~> 3.2"},
       {:oban, "~> 1.2"},
       {:sshex, "2.2.1"},
+      {:geolix, "~> 1.0"},
+      {:geolix_adapter_mmdb2, "~> 0.5.0"},
       {:clickhousex, [git: "https://github.com/atlas-forks/clickhousex.git"]}
     ]
   end

mix.lock

@@ -24,6 +24,8 @@
   "excoveralls": {:hex, :excoveralls, "0.12.3", "2142be7cb978a3ae78385487edda6d1aff0e482ffc6123877bb7270a8ffbcfe0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "568a3e616c264283f5dea5b020783ae40eef3f7ee2163f7a67cbd7b35bcadada"},
   "file_system": {:hex, :file_system, "0.2.8", "f632bd287927a1eed2b718f22af727c5aeaccc9a98d8c2bd7bff709e851dc986", [:mix], [], "hexpm", "97a3b6f8d63ef53bd0113070102db2ce05352ecf0d25390eb8d747c2bde98bca"},
   "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"},
+  "geolix": {:hex, :geolix, "1.1.0", "8b0fe847fef486d9e8b7c21eae6cbc2d998fb249e61d3f4f136f8016b9c1c833", [:mix], [{:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "980854f2aef30c288dc79e86c5267806d704c4525fde1b75de9a92f67fb16300"},
+  "geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.5.0", "5912723d9538ecddc6b29b1d8041b917b735a78fd3c122bfea8c44aa782e3369", [:mix], [{:geolix, "~> 1.1", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "cb1485b6a0a2d3e541949207428a245718dbf1258453a0df0e5fdd925bcecd3e"},
   "gettext": {:hex, :gettext, "0.18.0", "406d6b9e0e3278162c2ae1de0a60270452c553536772167e2d701f028116f870", [:mix], [], "hexpm", "c3f850be6367ebe1a08616c2158affe4a23231c70391050bf359d5f92f66a571"},
   "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"},
   "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"},
@@ -35,6 +37,7 @@
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.0", "54828676a36e75e9a25bc9a0bb0598d4c7fcc767bf0b40674850b22e05b7b6cc", [:mix], [], "hexpm", "359dc9242915538d1dceb9f6d96c72201dca76ce62e49d22e2ed1e86f20bea8e"},
   "nanoid": {:hex, :nanoid, "2.0.2", "f3f7b4bf103ab6667f22beb00b6315825ee3f30100dd2c93d534e5c02164e857", [:mix], [], "hexpm", "3095cb1fac7bbc78843a8ccd99f1af375d0da1d3ebaa8552e846b73438c0c44f"},
   "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm", "9374f4302045321874cccdc57eb975893643bd69c3b22bf1312dab5f06e5788e"},
   "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"},

test/plausible_web/controllers/api/external_controller_test.exs

@@ -17,7 +17,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
   end
 
   @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
-  @country_code "EE"
 
   describe "POST /api/event" do
     test "records the event", %{conn: conn} do
@@ -33,7 +32,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
         conn
         |> put_req_header("content-type", "text/plain")
         |> put_req_header("user-agent", @user_agent)
-        |> put_req_header("x-country", @country_code)
         |> post("/api/event", Jason.encode!(params))
 
       pageview = get_event("external-controller-test-1.com")
@@ -42,7 +40,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       assert pageview["hostname"] == "gigride.live"
       assert pageview["domain"] == "external-controller-test-1.com"
       assert pageview["pathname"] == "/"
-      assert pageview["country_code"] == @country_code
     end
 
     test "www. is stripped from domain", %{conn: conn} do
@@ -363,6 +360,25 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
     assert event["name"] == "custom event"
   end
 
+  # Fake data is set up in config/test.exs
+  test "looks up the country from the ip address", %{conn: conn} do
+    params = %{
+      name: "pageview",
+      domain: "external-controller-test-19.com",
+      url: "http://gigride.live/"
+    }
+
+    conn
+    |> put_req_header("content-type", "text/plain")
+    |> put_req_header("x-forwarded-for", "1.1.1.1")
+    |> post("/api/event", Jason.encode!(params))
+
+    pageview = get_event("external-controller-test-19.com")
+
+    assert pageview["country_code"] == "US"
+  end
+
+
   test "responds 400 when required fields are missing", %{conn: conn} do
     params = %{}