Add DATA_DIR (#4100)

* add DATA_DIR

* add test

* changelog

* fix test in CI where PERSISTENT_CACHE_DIR is always set

* consistent fallback

CHANGELOG.md

@@ -42,6 +42,7 @@ All notable changes to this project will be documented in this file.
 - Add Yesterday as an time range option in the dashboard
 - Add support for importing Google Analytics 4 data
 - Import custom events from Google Analytics 4
+- Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

config/runtime.exs

@@ -229,8 +229,12 @@ ip_geolocation_db = get_var_from_path_or_env(config_dir, "IP_GEOLOCATION_DB", ge
 geonames_source_file = get_var_from_path_or_env(config_dir, "GEONAMES_SOURCE_FILE")
 maxmind_license_key = get_var_from_path_or_env(config_dir, "MAXMIND_LICENSE_KEY")
 maxmind_edition = get_var_from_path_or_env(config_dir, "MAXMIND_EDITION", "GeoLite2-City")
+data_dir = get_var_from_path_or_env(config_dir, "DATA_DIR")
 persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIR")
 
+data_dir = data_dir || persistent_cache_dir
+persistent_cache_dir = persistent_cache_dir || data_dir
+
 enable_email_verification =
   config_dir
   |> get_var_from_path_or_env("ENABLE_EMAIL_VERIFICATION", "false")
@@ -297,7 +301,7 @@ config :plausible,
   custom_script_name: custom_script_name,
   log_failed_login_attempts: log_failed_login_attempts,
   license_key: license_key,
-  persistent_cache_dir: persistent_cache_dir
+  data_dir: data_dir
 
 config :plausible, :selfhost,
   enable_email_verification: enable_email_verification,

lib/plausible/exports.ex

@@ -130,13 +130,8 @@ defmodule Plausible.Exports do
 
   @spec local_export_file(pos_integer) :: Path.t()
   defp local_export_file(site_id) do
-    persistent_cache_dir = Application.get_env(:plausible, :persistent_cache_dir)
-
-    Path.join([
-      persistent_cache_dir || System.tmp_dir!(),
-      "plausible-exports",
-      Integer.to_string(site_id)
-    ])
+    data_dir = Application.get_env(:plausible, :data_dir)
+    Path.join([data_dir || System.tmp_dir!(), "plausible-exports", Integer.to_string(site_id)])
   end
 
   @doc "Gets S3 export for a site. Raises if object storage is unavailable."

lib/plausible/imported/csv_importer.ex

@@ -321,7 +321,7 @@ defmodule Plausible.Imported.CSVImporter do
   @doc """
   Returns local directory for CSV imports storage.
 
-  Builds upon `$PERSISTENT_CACHE_DIR` (if set) and falls back to /tmp
+  Builds upon `$DATA_DIR` or `$PERSISTENT_CACHE_DIR` (if set) and falls back to /tmp
 
   Examples:
 
@@ -331,12 +331,7 @@ defmodule Plausible.Imported.CSVImporter do
 
   """
   def local_dir(site_id) do
-    persistent_cache_dir = Application.get_env(:plausible, :persistent_cache_dir)
-
-    Path.join([
-      persistent_cache_dir || System.tmp_dir!(),
-      "plausible-imports",
-      Integer.to_string(site_id)
-    ])
+    data_dir = Application.get_env(:plausible, :data_dir)
+    Path.join([data_dir || System.tmp_dir!(), "plausible-imports", Integer.to_string(site_id)])
   end
 end

test/plausible/config_test.exs

@@ -278,6 +278,59 @@ defmodule Plausible.ConfigTest do
     end
   end
 
+  describe "storage" do
+    test "with only DATA_DIR set" do
+      env = [
+        {"MAXMIND_LICENSE_KEY", "abc"},
+        {"PERSISTENT_CACHE_DIR", nil},
+        {"DATA_DIR", "/data"}
+      ]
+
+      config = runtime_config(env)
+
+      # exports/imports
+      assert get_in(config, [:plausible, :data_dir]) == "/data"
+      # locus (mmdb cache)
+      assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/data"
+      # tzdata (timezones cache)
+      assert get_in(config, [:tzdata, :data_dir]) == "/data/tzdata_data"
+    end
+
+    test "with only PERSISTENT_CACHE_DIR set" do
+      env = [
+        {"MAXMIND_LICENSE_KEY", "abc"},
+        {"PERSISTENT_CACHE_DIR", "/cache"},
+        {"DATA_DIR", nil}
+      ]
+
+      config = runtime_config(env)
+
+      # exports/imports
+      assert get_in(config, [:plausible, :data_dir]) == "/cache"
+      # locus (mmdb cache)
+      assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"
+      # tzdata (timezones cache)
+      assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"
+    end
+
+    test "with both DATA_DIR and PERSISTENT_CACHE_DIR set" do
+      env = [
+        {"MAXMIND_LICENSE_KEY", "abc"},
+        {"PERSISTENT_CACHE_DIR", "/cache"},
+        {"DATA_DIR", "/data"}
+      ]
+
+      config = runtime_config(env)
+
+      # exports/imports
+      assert get_in(config, [:plausible, :data_dir]) == "/data"
+      # locus (mmdb cache)
+      assert get_in(config, [:plausible, Plausible.Geo, :cache_dir]) == "/cache"
+      # tzdata (timezones cache)
+      assert get_in(config, [:tzdata, :data_dir]) == "/cache/tzdata_data"
+    end
+  end
+
   describe "extra config" do
     test "no-op when no extra path is set" do
       put_system_env_undo({"EXTRA_CONFIG_PATH", nil})