CSV imports (UI) (#3845)

* add basic ui

* remove TODO

* credo

* allow folder upload

* redirect external

* mention folder, use folder icon for file picker

* back to multiple file upload

* mention zip

* escape dots in archive filename

assets/js/liveview/live_socket.js

@@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) {
       })
     }
   }
+  let Uploaders = {}
+  Uploaders.S3 = function (entries, onViewError) {
+    entries.forEach(entry => {
+      let xhr = new XMLHttpRequest()
+      onViewError(() => xhr.abort())
+      xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
+      xhr.onerror = () => entry.error()
+      xhr.upload.addEventListener("progress", (event) => {
+        if (event.lengthComputable) {
+          let percent = Math.round((event.loaded / event.total) * 100)
+          if (percent < 100) { entry.progress(percent) }
+        }
+      })
+      let url = entry.meta.url
+      xhr.open("PUT", url, true)
+      xhr.send(entry.file)
+    })
+  }
   let token = csrfToken.getAttribute("content")
   let url = websocketUrl.getAttribute("content")
   let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
   let liveSocket = new LiveSocket(liveUrl, Socket, {
     heartbeatIntervalMs: 10000,
-    params: { _csrf_token: token }, hooks: Hooks, dom: {
+    params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {
       // for alpinejs integration
       onBeforeElUpdated(from, to) {
         if (from._x_dataStack) {

lib/plausible/exports.ex

@@ -12,14 +12,24 @@ defmodule Plausible.Exports do
   Examples:
 
       iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
-      "plausible.io_20210101_20241231.zip"
+      "plausible_io_20210101_20241231.zip"
 
       iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
-      "Bücher.example_20210101_20241231.zip"
+      "Bücher_example_20210101_20241231.zip"
 
   """
   def archive_filename(domain, min_date, max_date) do
-    "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip"
+    name =
+      Enum.join(
+        [
+          String.replace(domain, ".", "_"),
+          Calendar.strftime(min_date, "%Y%m%d"),
+          Calendar.strftime(max_date, "%Y%m%d")
+        ],
+        "_"
+      )
+
+    name <> ".zip"
   end
 
   @doc """

lib/plausible/imported/csv_importer.ex

@@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do
       Date.range(~D[2019-01-01], ~D[2022-01-01])
 
       iex> date_range([])
-      ** (ArgumentError) empty uploads
+      nil
 
   """
-  @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()
+  @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
   def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
-  def date_range([]), do: raise(ArgumentError, "empty uploads")
+  def date_range([]), do: nil
 
   defp date_range([upload | uploads], prev_start_date, prev_end_date) do
     filename =
@@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter do
   def parse_filename!(_filename) do
     raise ArgumentError, "invalid filename"
   end
+
+  @doc """
+  Checks if the provided filename conforms to the expected format.
+
+  Examples:
+
+      iex> valid_filename?("my_data.csv")
+      false
+
+      iex> valid_filename?("imported_devices_00010101_20250101.csv")
+      true
+
+  """
+  @spec valid_filename?(String.t()) :: boolean
+  def valid_filename?(filename) do
+    try do
+      parse_filename!(filename)
+    else
+      _ -> true
+    rescue
+      _ -> false
+    end
+  end
+
+  @doc """
+  Extracts the table name from the provided filename.
+
+  Raises if the filename doesn't conform to the expected format.
+
+  Examples:
+
+      iex> extract_table("my_data.csv")
+      ** (ArgumentError) invalid filename
+
+      iex> extract_table("imported_devices_00010101_20250101.csv")
+      "imported_devices"
+
+  """
+  @spec extract_table(String.t()) :: String.t()
+  def extract_table(filename) do
+    {table, _start_date, _end_date} = parse_filename!(filename)
+    table
+  end
 end

lib/plausible_web/controllers/site_controller.ex

@@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do
     |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
   end
 
-  def export(conn, _params) do
+  def csv_export(conn, _params) do
     %{site: site, current_user: user} = conn.assigns
 
     Oban.insert!(
@@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do
     |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))
   end
 
+  def csv_import(conn, _params) do
+    conn
+    |> assign(:skip_plausible_tracking, true)
+    |> render("csv_import.html",
+      layout: {PlausibleWeb.LayoutView, "focus.html"},
+      connect_live_socket: true
+    )
+  end
+
   def change_domain(conn, _params) do
     changeset = Plausible.Site.update_changeset(conn.assigns.site)
 

lib/plausible_web/live/csv_import.ex

@@ -0,0 +1,228 @@
+defmodule PlausibleWeb.Live.CSVImport do
+  @moduledoc """
+  LiveView allowing uploading CSVs for imported tables to S3
+  """
+  use PlausibleWeb, :live_view
+  alias Plausible.Imported.CSVImporter
+
+  @impl true
+  def mount(_params, session, socket) do
+    %{"site_id" => site_id, "user_id" => user_id} = session
+
+    socket =
+      socket
+      |> assign(site_id: site_id, user_id: user_id)
+      |> allow_upload(:import,
+        accept: [".csv", "text/csv"],
+        auto_upload: true,
+        max_entries: length(Plausible.Imported.tables()),
+        # 1GB
+        max_file_size: 1_000_000_000,
+        external: &presign_upload/2,
+        progress: &handle_progress/3
+      )
+      |> process_imported_tables()
+
+    {:ok, socket}
+  end
+
+  @impl true
+  def render(assigns) do
+    ~H"""
+    <div>
+      <form action="#" method="post" phx-change="validate-upload-form" phx-submit="submit-upload-form">
+        <.csv_picker upload={@uploads.import} imported_tables={@imported_tables} />
+        <.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} />
+
+        <p :for={error <- upload_errors(@uploads.import)} class="text-red-400">
+          <%= error_to_string(error) %>
+        </p>
+      </form>
+    </div>
+    """
+  end
+
+  defp csv_picker(assigns) do
+    ~H"""
+    <label
+      phx-drop-target={@upload.ref}
+      class="block border-2 dark:border-gray-600 rounded p-4 group hover:border-indigo-500 dark:hover:border-indigo-600 transition cursor-pointer"
+    >
+      <div class="flex items-center">
+        <div class="bg-gray-200 dark:bg-gray-600 rounded p-1 group-hover:bg-indigo-500 dark:group-hover:bg-indigo-600 transition">
+          <Heroicons.document_plus class="w-5 h-5 group-hover:text-white transition" />
+        </div>
+        <span class="ml-2 text-sm text-gray-600 dark:text-gray-500">
+          (or drag-and-drop your unzipped CSVs here)
+        </span>
+        <.live_file_input upload={@upload} class="hidden" />
+      </div>
+
+      <ul id="imported-tables" class="mt-3.5 mb-0.5 space-y-1.5">
+        <.imported_table
+          :for={{table, upload} <- @imported_tables}
+          table={table}
+          upload={upload}
+          errors={if(upload, do: upload_errors(@upload, upload), else: [])}
+        />
+      </ul>
+    </label>
+    """
+  end
+
+  defp confirm_button(assigns) do
+    ~H"""
+    <button
+      type="submit"
+      disabled={not @can_confirm?}
+      class={[
+        "rounded-md w-full bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-gray-400 dark:disabled:bg-gray-700 mt-4",
+        unless(@can_confirm?, do: "cursor-not-allowed")
+      ]}
+    >
+      <%= if @date_range do %>
+        Confirm import from <%= @date_range.first %> to <%= @date_range.last %>
+      <% else %>
+        Confirm import
+      <% end %>
+    </button>
+    """
+  end
+
+  defp imported_table(assigns) do
+    status =
+      cond do
+        assigns.upload && assigns.upload.progress == 100 -> :success
+        assigns.upload && assigns.upload.progress > 0 -> :in_progress
+        not Enum.empty?(assigns.errors) -> :error
+        true -> :empty
+      end
+
+    assigns = assign(assigns, status: status)
+
+    ~H"""
+    <li id={@table} class="ml-1.5">
+      <div class="flex items-center space-x-2">
+        <Heroicons.document_check :if={@status == :success} class="w-4 h-4 text-indigo-600" />
+        <PlausibleWeb.Components.Generic.spinner
+          :if={@status == :in_progress}
+          class="w-4 h-4 text-indigo-600"
+        />
+        <Heroicons.document :if={@status == :empty} class="w-4 h-4 text-gray-400 dark:text-gray-500" />
+        <Heroicons.document :if={@status == :error} class="w-4 h-4 text-red-600 dark:text-red-700" />
+
+        <span class={[
+          "text-sm",
+          if(@upload, do: "dark:text-gray-400", else: "text-gray-400 dark:text-gray-500"),
+          if(@status == :error, do: "text-red-600 dark:text-red-700")
+        ]}>
+          <%= if @upload do %>
+            <%= @upload.client_name %>
+          <% else %>
+            <%= @table %>_YYYYMMDD_YYYYMMDD.csv
+          <% end %>
+        </span>
+      </div>
+
+      <p :for={error <- @errors} class="ml-6 text-sm text-red-600 dark:text-red-700">
+        <%= error_to_string(error) %>
+      </p>
+    </li>
+    """
+  end
+
+  @impl true
+  def handle_event("validate-upload-form", _params, socket) do
+    {:noreply, process_imported_tables(socket)}
+  end
+
+  def handle_event("submit-upload-form", _params, socket) do
+    %{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns
+    site = Plausible.Repo.get!(Plausible.Site, site_id)
+    user = Plausible.Repo.get!(Plausible.Auth.User, user_id)
+
+    uploads =
+      consume_uploaded_entries(socket, :import, fn meta, entry ->
+        {:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}
+      end)
+
+    {:ok, _job} =
+      CSVImporter.new_import(site, user,
+        start_date: date_range.first,
+        end_date: date_range.last,
+        uploads: uploads
+      )
+
+    redirect_to =
+      Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain))
+
+    {:noreply, redirect(socket, external: redirect_to)}
+  end
+
+  defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"
+  defp error_to_string(:too_many_files), do: "too many files"
+  defp error_to_string(:not_accepted), do: "unacceptable file types"
+  defp error_to_string(:external_client_failure), do: "browser upload failed"
+
+  defp presign_upload(entry, socket) do
+    %{s3_url: s3_url, presigned_url: upload_url} =
+      Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name)
+
+    {:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}
+  end
+
+  defp handle_progress(:import, entry, socket) do
+    if entry.done? do
+      {:noreply, process_imported_tables(socket)}
+    else
+      {:noreply, socket}
+    end
+  end
+
+  defp process_imported_tables(socket) do
+    tables = Plausible.Imported.tables()
+    {completed, in_progress} = uploaded_entries(socket, :import)
+
+    {valid_uploads, invalid_uploads} =
+      Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))
+
+    imported_tables_all_uploads =
+      Enum.map(tables, fn table ->
+        uploads =
+          Enum.filter(valid_uploads, fn upload ->
+            CSVImporter.extract_table(upload.client_name) == table
+          end)
+
+        {upload, replaced_uploads} = List.pop_at(uploads, -1)
+        {table, upload, replaced_uploads}
+      end)
+
+    imported_tables =
+      Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->
+        {table, upload}
+      end)
+
+    replaced_uploads =
+      Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->
+        replaced_uploads
+      end)
+
+    date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))
+    all_uploaded? = completed != [] and in_progress == []
+
+    socket
+    |> cancel_uploads(invalid_uploads)
+    |> cancel_uploads(replaced_uploads)
+    |> assign(
+      imported_tables: imported_tables,
+      can_confirm?: all_uploaded?,
+      date_range: date_range
+    )
+  end
+
+  defp cancel_uploads(socket, uploads) do
+    Enum.reduce(uploads, socket, fn upload, socket ->
+      cancel_upload(socket, :import, upload.ref)
+    end)
+  end
+end

lib/plausible_web/router.ex

@@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router do
 
     delete "/:website/settings/forget-imported", SiteController, :forget_imported
     delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
-    post "/:website/settings/export", SiteController, :export
+    post "/:website/settings/export", SiteController, :csv_export
+    get "/:website/settings/import", SiteController, :csv_import
 
     get "/:domain/export", StatsController, :csv_export
     get "/:domain/*path", StatsController, :stats

lib/plausible_web/templates/site/csv_import.html.heex

@@ -0,0 +1,20 @@
+<div class="max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded p-6  mb-4 mt-8">
+  <h2 class="text-xl font-black dark:text-gray-100">Import from CSV files</h2>
+
+  <div class="my-3 space-y-1.5 text-sm text-gray-400">
+    <p>
+      Please ensure each file follows
+      <.styled_link href="https://plausible.io/docs/csv-import">
+        our CSV format guidelines.
+      </.styled_link>
+    </p>
+
+    <p>
+      You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area.
+    </p>
+  </div>
+
+  <%= live_render(@conn, PlausibleWeb.Live.CSVImport,
+    session: %{"site_id" => @site.id, "user_id" => @current_user.id}
+  ) %>
+</div>

lib/plausible_web/templates/site/settings_imports_exports.html.heex

@@ -20,9 +20,9 @@
     </PlausibleWeb.Components.Generic.button_link>
 
     <PlausibleWeb.Components.Generic.button_link
-      class="w-36 h-20 opacity-40 cursor-not-allowed"
+      class="w-36 h-20"
       theme="bright"
-      href=""
+      href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
     >
       <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
     </PlausibleWeb.Components.Generic.button_link>