From c263df5805339405e491787950114e59c3c25c05 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Mar 2024 19:55:14 +0800 Subject: [PATCH] CSV imports (UI) (#3845) * add basic ui * remove TODO * credo * allow folder upload * redirect external * mention folder, use folder icon for file picker * back to multiple file upload * mention zip * escape dots in archive filename --- assets/js/liveview/live_socket.js | 20 +- lib/plausible/exports.ex | 16 +- lib/plausible/imported/csv_importer.ex | 49 +++- .../controllers/site_controller.ex | 11 +- lib/plausible_web/live/csv_import.ex | 228 ++++++++++++++++++ lib/plausible_web/router.ex | 3 +- .../templates/site/csv_import.html.heex | 20 ++ .../site/settings_imports_exports.html.heex | 4 +- 8 files changed, 340 insertions(+), 11 deletions(-) create mode 100644 lib/plausible_web/live/csv_import.ex create mode 100644 lib/plausible_web/templates/site/csv_import.html.heex diff --git a/assets/js/liveview/live_socket.js b/assets/js/liveview/live_socket.js index a6208ff4cf..ed287267f7 100644 --- a/assets/js/liveview/live_socket.js +++ b/assets/js/liveview/live_socket.js @@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) { }) } } + let Uploaders = {} + Uploaders.S3 = function (entries, onViewError) { + entries.forEach(entry => { + let xhr = new XMLHttpRequest() + onViewError(() => xhr.abort()) + xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error() + xhr.onerror = () => entry.error() + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + let percent = Math.round((event.loaded / event.total) * 100) + if (percent < 100) { entry.progress(percent) } + } + }) + let url = entry.meta.url + xhr.open("PUT", url, true) + xhr.send(entry.file) + }) + } let token = csrfToken.getAttribute("content") let url = websocketUrl.getAttribute("content") let liveUrl = (url === "") ? "/live" : new URL("/live", url).href; let liveSocket = new LiveSocket(liveUrl, Socket, { heartbeatIntervalMs: 10000, - params: { _csrf_token: token }, hooks: Hooks, dom: { + params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: { // for alpinejs integration onBeforeElUpdated(from, to) { if (from._x_dataStack) { diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex index dbabcd7ff8..c15e0bb78d 100644 --- a/lib/plausible/exports.ex +++ b/lib/plausible/exports.ex @@ -12,14 +12,24 @@ defmodule Plausible.Exports do Examples: iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31]) - "plausible.io_20210101_20241231.zip" + "plausible_io_20210101_20241231.zip" iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31]) - "Bücher.example_20210101_20241231.zip" + "Bücher_example_20210101_20241231.zip" """ def archive_filename(domain, min_date, max_date) do - "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip" + name = + Enum.join( + [ + String.replace(domain, ".", "_"), + Calendar.strftime(min_date, "%Y%m%d"), + Calendar.strftime(max_date, "%Y%m%d") + ], + "_" + ) + + name <> ".zip" end @doc """ diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 2d10ca32e9..6376d851d3 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do Date.range(~D[2019-01-01], ~D[2022-01-01]) iex> date_range([]) - ** (ArgumentError) empty uploads + nil """ - @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() + @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil) - def date_range([]), do: raise(ArgumentError, "empty uploads") + def date_range([]), do: nil defp date_range([upload | uploads], prev_start_date, prev_end_date) do filename = @@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter do def parse_filename!(_filename) do raise ArgumentError, "invalid filename" end + + @doc """ + Checks if the provided filename conforms to the expected format. + + Examples: + + iex> valid_filename?("my_data.csv") + false + + iex> valid_filename?("imported_devices_00010101_20250101.csv") + true + + """ + @spec valid_filename?(String.t()) :: boolean + def valid_filename?(filename) do + try do + parse_filename!(filename) + else + _ -> true + rescue + _ -> false + end + end + + @doc """ + Extracts the table name from the provided filename. + + Raises if the filename doesn't conform to the expected format. + + Examples: + + iex> extract_table("my_data.csv") + ** (ArgumentError) invalid filename + + iex> extract_table("imported_devices_00010101_20250101.csv") + "imported_devices" + + """ + @spec extract_table(String.t()) :: String.t() + def extract_table(filename) do + {table, _start_date, _end_date} = parse_filename!(filename) + table + end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 64cc4332df..e61a266111 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain)) end - def export(conn, _params) do + def csv_export(conn, _params) do %{site: site, current_user: user} = conn.assigns Oban.insert!( @@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain)) end + def csv_import(conn, _params) do + conn + |> assign(:skip_plausible_tracking, true) + |> render("csv_import.html", + layout: {PlausibleWeb.LayoutView, "focus.html"}, + connect_live_socket: true + ) + end + def change_domain(conn, _params) do changeset = Plausible.Site.update_changeset(conn.assigns.site) diff --git a/lib/plausible_web/live/csv_import.ex b/lib/plausible_web/live/csv_import.ex new file mode 100644 index 0000000000..38c97ca17f --- /dev/null +++ b/lib/plausible_web/live/csv_import.ex @@ -0,0 +1,228 @@ +defmodule PlausibleWeb.Live.CSVImport do + @moduledoc """ + LiveView allowing uploading CSVs for imported tables to S3 + """ + use PlausibleWeb, :live_view + alias Plausible.Imported.CSVImporter + + @impl true + def mount(_params, session, socket) do + %{"site_id" => site_id, "user_id" => user_id} = session + + socket = + socket + |> assign(site_id: site_id, user_id: user_id) + |> allow_upload(:import, + accept: [".csv", "text/csv"], + auto_upload: true, + max_entries: length(Plausible.Imported.tables()), + # 1GB + max_file_size: 1_000_000_000, + external: &presign_upload/2, + progress: &handle_progress/3 + ) + |> process_imported_tables() + + {:ok, socket} + end + + @impl true + def render(assigns) do + ~H""" +
+
+ <.csv_picker upload={@uploads.import} imported_tables={@imported_tables} /> + <.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} /> + +

+ <%= error_to_string(error) %> +

+
+
+ """ + end + + defp csv_picker(assigns) do + ~H""" + + """ + end + + defp confirm_button(assigns) do + ~H""" + + """ + end + + defp imported_table(assigns) do + status = + cond do + assigns.upload && assigns.upload.progress == 100 -> :success + assigns.upload && assigns.upload.progress > 0 -> :in_progress + not Enum.empty?(assigns.errors) -> :error + true -> :empty + end + + assigns = assign(assigns, status: status) + + ~H""" +
  • +
    + + + + + + + <%= if @upload do %> + <%= @upload.client_name %> + <% else %> + <%= @table %>_YYYYMMDD_YYYYMMDD.csv + <% end %> + +
    + +

    + <%= error_to_string(error) %> +

    +
    • + """ + end + + @impl true + def handle_event("validate-upload-form", _params, socket) do + {:noreply, process_imported_tables(socket)} + end + + def handle_event("submit-upload-form", _params, socket) do + %{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns + site = Plausible.Repo.get!(Plausible.Site, site_id) + user = Plausible.Repo.get!(Plausible.Auth.User, user_id) + + uploads = + consume_uploaded_entries(socket, :import, fn meta, entry -> + {:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}} + end) + + {:ok, _job} = + CSVImporter.new_import(site, user, + start_date: date_range.first, + end_date: date_range.last, + uploads: uploads + ) + + redirect_to = + Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain)) + + {:noreply, redirect(socket, external: redirect_to)} + end + + defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)" + defp error_to_string(:too_many_files), do: "too many files" + defp error_to_string(:not_accepted), do: "unacceptable file types" + defp error_to_string(:external_client_failure), do: "browser upload failed" + + defp presign_upload(entry, socket) do + %{s3_url: s3_url, presigned_url: upload_url} = + Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name) + + {:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket} + end + + defp handle_progress(:import, entry, socket) do + if entry.done? do + {:noreply, process_imported_tables(socket)} + else + {:noreply, socket} + end + end + + defp process_imported_tables(socket) do + tables = Plausible.Imported.tables() + {completed, in_progress} = uploaded_entries(socket, :import) + + {valid_uploads, invalid_uploads} = + Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name)) + + imported_tables_all_uploads = + Enum.map(tables, fn table -> + uploads = + Enum.filter(valid_uploads, fn upload -> + CSVImporter.extract_table(upload.client_name) == table + end) + + {upload, replaced_uploads} = List.pop_at(uploads, -1) + {table, upload, replaced_uploads} + end) + + imported_tables = + Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} -> + {table, upload} + end) + + replaced_uploads = + Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} -> + replaced_uploads + end) + + date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name)) + all_uploaded? = completed != [] and in_progress == [] + + socket + |> cancel_uploads(invalid_uploads) + |> cancel_uploads(replaced_uploads) + |> assign( + imported_tables: imported_tables, + can_confirm?: all_uploaded?, + date_range: date_range + ) + end + + defp cancel_uploads(socket, uploads) do + Enum.reduce(uploads, socket, fn upload, socket -> + cancel_upload(socket, :import, upload.ref) + end) + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 6a4fd7660c..914b549fba 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router do delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import - post "/:website/settings/export", SiteController, :export + post "/:website/settings/export", SiteController, :csv_export + get "/:website/settings/import", SiteController, :csv_import get "/:domain/export", StatsController, :csv_export get "/:domain/*path", StatsController, :stats diff --git a/lib/plausible_web/templates/site/csv_import.html.heex b/lib/plausible_web/templates/site/csv_import.html.heex new file mode 100644 index 0000000000..ae9fc4038d --- /dev/null +++ b/lib/plausible_web/templates/site/csv_import.html.heex @@ -0,0 +1,20 @@ +
    +

    Import from CSV files

    + +
    +

    + Please ensure each file follows + <.styled_link href="https://plausible.io/docs/csv-import"> + our CSV format guidelines. + +

    + +

    + You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area. +

    +
    + + <%= live_render(@conn, PlausibleWeb.Live.CSVImport, + session: %{"site_id" => @site.id, "user_id" => @current_user.id} + ) %> +
    diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex index afc374c32e..4011ac033c 100644 --- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex +++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex @@ -20,9 +20,9 @@ New CSV import