Speed up stats

2024-09-11 23:37:08 +03:00 · 2023-12-12 17:46:03 +01:00 · 2023-12-12 17:46:03 +01:00 · 548b5a42bd
commit 548b5a42bd
parent 47c4b8cb16
5 changed files with 215 additions and 143 deletions
--- a/wasp-ai/main.wasp
+++ b/wasp-ai/main.wasp
@ -130,6 +130,13 @@ query getAppGenerationResult {
  ]
 }

+query getProjects {
+  fn: import { getProjects } from "@server/operations.js",
+  entities: [
+    Project
+  ]
+}
+
 query getStats {
  fn: import { getStats } from "@server/operations.js",
  entities: [
--- a/wasp-ai/src/client/components/BarChart.jsx
+++ b/wasp-ai/src/client/components/BarChart.jsx
@ -4,76 +4,12 @@ import { Group } from "@visx/group";
 import { scaleBand, scaleLinear } from "@visx/scale";
 import { AxisBottom, AxisLeft } from "@visx/axis";

-function generateLast24HoursData(projects) {
-  const buckets = [];
-  const now = new Date();
-  const last24Hours = new Date(now.getTime() - 24 * 60 * 60 * 1000);
-  for (let i = 0; i < 24; i++) {
-    const bucketStart = new Date(last24Hours.getTime() + i * 60 * 60 * 1000);
-    const bucket = {
-      date: bucketStart,
-      displayValue: bucketStart.getHours() + 1,
-      count: 0,
-    };
-    buckets.push(bucket);
-  }
-  projects.forEach((project) => {
-    const createdAt = new Date(project.createdAt);
-    // Difference in hours between now and when the project was created
-    const bucketIndex = Math.floor(
-      (now.getTime() - createdAt.getTime()) / (60 * 60 * 1000)
-    );
-    const reverseBucketIndex = buckets.length - bucketIndex - 1;
-    // Count only projects that were created in the last 24 hours
-    if (bucketIndex >= 0 && bucketIndex < 24) {
-      buckets[reverseBucketIndex].count++;
-    }
-  });
-  return buckets;
-}
-
-function generateLast30DaysData(projects) {
-  const buckets = [];
-  const now = new Date();
-  const last30Days = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
-  for (let i = 0; i < 30; i++) {
-    const bucketStart = new Date(
-      last30Days.getTime() + i * 24 * 60 * 60 * 1000
-    );
-    const bucket = {
-      date: bucketStart,
-      displayValue: bucketStart.getDate(),
-      count: 0,
-    };
-    buckets.push(bucket);
-  }
-  projects.forEach((project) => {
-    const createdAt = new Date(project.createdAt);
-    // Difference in days between now and when the project was created
-    const bucketIndex = Math.floor(
-      (now.getTime() - createdAt.getTime()) / (24 * 60 * 60 * 1000)
-    );
-    const reverseBucketIndex = buckets.length - bucketIndex - 1;
-    // Count only projects that were created in the last 30 days
-    if (bucketIndex >= 0 && bucketIndex < 30) {
-      buckets[reverseBucketIndex].count++;
-    }
-  });
-  return buckets;
-}
-
 const verticalMargin = 50;
 const margins = {
  left: 0,
 };

-export function BarChart({ projects, chartType, width, height }) {
-  const data = useMemo(() => {
-    if (chartType === "last24Hours") {
-      return generateLast24HoursData(projects);
-    }
-    return generateLast30DaysData(projects);
-  }, [chartType, projects]);
+export function BarChart({ data, width, height }) {
  // bounds
  const xMax = width - margins.left;
  const yMax = height - verticalMargin;
@ -110,9 +46,8 @@ export function BarChart({ projects, chartType, width, height }) {
          const barY = yMax - barHeight;
          return (
            d.count > 0 && (
-              <Group>
+              <Group key={`bar-${d.date}`}>
                <Bar
-                  key={`bar-${d.date}`}
                  x={barX}
                  y={barY}
                  width={barWidth}
--- a/wasp-ai/src/client/pages/StatsPage.jsx
+++ b/wasp-ai/src/client/pages/StatsPage.jsx
@ -1,4 +1,5 @@
 import { useState, useMemo } from "react";
+import getProjects from "@wasp/queries/getProjects";
 import getStats from "@wasp/queries/getStats";
 import { useQuery } from "@wasp/queries";
 import { Link } from "react-router-dom";
@ -9,7 +10,6 @@ import { BarChart } from "../components/BarChart";
 import ParentSize from "@visx/responsive/lib/components/ParentSize";
 import { exampleIdeas } from "../examples";
 import logout from "@wasp/auth/logout";
-import { WaspIcon } from "../components/WaspIcon";
 import { Header } from "../components/Header";
 import { PiDownloadDuotone, PiUserDuotone } from "react-icons/pi";
 import { MyDropdown } from "../components/Dropdown";
@ -33,62 +33,62 @@ const chartTypes = [

 export function Stats() {
  const [filterOutExampleApps, setFilterOutExampleApps] = useState(false);
-  const [filterOutKnownUsers, setFilterOutKnownUsers] = useState(false);
  const [chartType, setChartType] = useState(chartTypes[0]);

-  const { data: stats, isLoading, error } = useQuery(getStats);
+  const { data: projects, isLoading, error } = useQuery(getProjects);
+  const { data: stats } = useQuery(getStats, {
+    filterOutExampleApps,
+  });

  const logsByProjectId = useMemo(() => {
-    if (!stats) {
+    if (!projects) {
      return {};
    }
-    if (!stats.latestProjectsWithLogs) {
+    if (!projects.latestProjectsWithLogs) {
      return {};
    }
-    return stats.latestProjectsWithLogs.reduce((acc, project) => {
+    return projects.latestProjectsWithLogs.reduce((acc, project) => {
      acc[project.id] = project.logs;
      return acc;
    }, {});
-  }, [stats]);
+  }, [projects]);

  const filteredProjects = useMemo(() => {
    const filters = [];
    if (filterOutExampleApps) {
-      filters.push(
-        (stat) =>
-          !exampleIdeas.some((example) => example.name === stat.name && example.description === stat.description)
-      );
+      filters.push((stat) => !exampleIdeas.some((example) => example.name === stat.name));
    }
-    if (filterOutKnownUsers) {
-      filters.push((stat) => !stat.user);
-    }
-    return stats
-      ? stats.projects.filter((stat) => {
+    return projects
+      ? projects.projects.filter((stat) => {
          return filters.every((filter) => filter(stat));
        })
      : [];
-  }, [stats, stats?.projects, filterOutExampleApps, filterOutKnownUsers]);
+  }, [projects, projects?.projects, filterOutExampleApps]);

-  const limitedFilteredProjects = useMemo(() => {
-    return filteredProjects.slice(0, 1000);
-  }, [filteredProjects]);
+  const barChartData = useMemo(() => {
+    if (!stats) {
+      return [];
+    }
+
+    if (chartType.value === "last24Hours") {
+      return stats.last24Hours;
+    } else {
+      return stats.last30Days;
+    }
+  }, [stats, chartType, filteredProjects]);

  if (isLoading) {
-    return <p>Loading</p>;
+    return <p>Loading...</p>;
  }

  if (error) {
    return <p>Error: {error.message}</p>;
  }

-  if (!stats) {
+  if (!projects || !stats) {
    return <p>Couldn't load stats</p>;
  }

-  const downloadStats = getDownloadStats(filteredProjects);
-
-  const downloadedPercentage = Math.round(downloadStats.downloadRatio * 10000) / 100;
-
  return (
    <>
      <Header>
@ -104,9 +104,11 @@ export function Stats() {
          </div>
        </div>

-        {stats.projects.length === 0 && <p className="text-sm text-slate-500">No projects created yet.</p>}
+        {projects.projects.length === 0 && (
+          <p className="text-sm text-slate-500">No projects created yet.</p>
+        )}

-        {stats.projects.length > 0 && (
+        {projects.projects.length > 0 && (
          <>
            <div className="mb-3 flex justify-between items-end">
              <div>
@ -119,7 +121,12 @@ export function Stats() {
            <div style={{ height: 300, width: "100%" }} className="mb-4">
              <ParentSize>
                {({ width, height }) => (
-                  <BarChart chartType={chartType.value} projects={filteredProjects} width={width} height={height} />
+                  <BarChart
+                    chartType={chartType.value}
+                    data={barChartData}
+                    width={width}
+                    height={height}
+                  />
                )}
              </ParentSize>
            </div>
@ -138,29 +145,19 @@ export function Stats() {
                    Filter out example apps
                  </label>
                </div>
-                <div className="flex items-center mb-4">
-                  <input
-                    id="default-checkbox"
-                    type="checkbox"
-                    checked={filterOutKnownUsers}
-                    onChange={(event) => setFilterOutKnownUsers(event.target.checked)}
-                    className="w-4 h-4 text-sky-600 bg-gray-100 border-gray-300 rounded focus:ring-sky-500"
-                  />
-                  <label htmlFor="default-checkbox" className="ml-2 text-sm font-medium text-gray-900">
-                    Filter out known users
-                  </label>
-                </div>
              </div>

-              <p className="text-sm text-slate-800 flex gap-2">
-                <span className="bg-slate-100 rounded-md px-2 py-1">
-                  Generated: <strong className="text-slate-800">{filteredProjects.length}</strong>
-                </span>
-                <span className="bg-slate-100 rounded-md px-2 py-1">
-                  Downloaded:{" "}
-                  <strong className="text-slate-800">{`${downloadStats.projectsDownloaded} (${downloadedPercentage}%)`}</strong>
-                </span>
-              </p>
+              {stats && (
+                <p className="text-sm text-slate-800 flex gap-2">
+                  <span className="bg-slate-100 rounded-md px-2 py-1">
+                    Generated: <strong className="text-slate-800">{stats.totalGenerated}</strong>
+                  </span>
+                  <span className="bg-slate-100 rounded-md px-2 py-1">
+                    Downloaded:{" "}
+                    <strong className="text-slate-800">{`${stats.totalDownloaded} (${stats.downloadedPercentage}%)`}</strong>
+                  </span>
+                </p>
+              )}
            </div>

            <div className="relative overflow-x-auto shadow-md sm:rounded-lg">
@ -192,7 +189,9 @@ export function Stats() {
                        scope="row"
                        className="px-6 py-4 font-medium text-gray-900 whitespace-nowrap flex items-center gap-2"
                      >
-                        <Color value={getTailwindClassNameForProjectBrandColor(project.primaryColor)} />{" "}
+                        <Color
+                          value={getTailwindClassNameForProjectBrandColor(project.primaryColor)}
+                        />{" "}
                        <span className="max-w-[250px] overflow-hidden overflow-ellipsis">
                          {project.name}
                        </span>{" "}
@ -216,7 +215,10 @@ export function Stats() {
                        </span>
                      </th>
                      <td className="px-6 py-4">
-                        <StatusPill status={getTailwindClassNameForProjectStatus(project.status)} sm>
+                        <StatusPill
+                          status={getTailwindClassNameForProjectStatus(project.status)}
+                          sm
+                        >
                          {projectStatusToDisplayableText(project.status)}
                        </StatusPill>
                      </td>
@ -230,9 +232,14 @@ export function Stats() {
                        {getWaitingInQueueDuration(project, logsByProjectId)} &rarr;{" "}
                        {getDuration(project, logsByProjectId)}
                      </td>
-                      <td className={`px-6 py-4 creativity-${project.creativityLevel}`}>{project.creativityLevel}</td>
+                      <td className={`px-6 py-4 creativity-${project.creativityLevel}`}>
+                        {project.creativityLevel}
+                      </td>
                      <td className="px-6 py-4">
-                        <Link to={`/result/${project.id}`} className="font-medium text-sky-600 hover:underline">
+                        <Link
+                          to={`/result/${project.id}`}
+                          className="font-medium text-sky-600 hover:underline"
+                        >
                          View the app &rarr;
                        </Link>
                      </td>
@ -240,11 +247,9 @@ export function Stats() {
                  ))}
                </tbody>
              </table>
-              {filteredProjects.length > limitedFilteredProjects.length && (
-                <div className="relative px-6 py-3 bg-gray-50 text-sm text-slate-500 text-center">
-                  Showing only the latest 1000 projects
-                </div>
-              )}
+              <div className="relative px-6 py-3 bg-gray-50 text-sm text-slate-500 text-center">
+                Showing only the latest 1000 projects
+              </div>
            </div>
          </>
        )}
@ -253,20 +258,6 @@ export function Stats() {
  );
 }

-function getDownloadStats(projects) {
-  const projectsAfterDownloadTracking = projects.filter(
-    (project) =>
-      // This is the time of the first recorded download (after we rolled out download tracking).
-      project.createdAt > new Date("2023-07-14 10:36:45.12") && project.status === "success"
-  );
-  const downloadedProjects = projectsAfterDownloadTracking.filter((project) => project.zipDownloadedAt !== null);
-  return {
-    projectsDownloaded: downloadedProjects.length,
-    downloadRatio:
-      projectsAfterDownloadTracking.length > 0 ? downloadedProjects.length / projectsAfterDownloadTracking.length : 0,
-  };
-}
-
 function getFormattedDiff(start, end) {
  const diff = (end - start) / 1000;
  const minutes = Math.round(diff / 60);
--- a/wasp-ai/src/server/operations.ts
+++ b/wasp-ai/src/server/operations.ts
@ -7,6 +7,7 @@ import {
 import {
  GetAppGenerationResult,
  GetStats,
+  GetProjects,
  GetFeedback,
  GetNumProjects,
  GetProjectsByUser,
@ -15,6 +16,8 @@ import HttpError from "@wasp/core/HttpError.js";
 import { checkPendingAppsJob } from "@wasp/jobs/checkPendingAppsJob.js";
 import { getNowInUTC } from "./utils.js";
 import type { Project, User } from "@wasp/entities";
+import type { Prisma } from "@prisma/client";
+import { generateLast24HoursData, generateLast30DaysData } from "./stats.js";

 export const startGeneratingNewApp: StartGeneratingNewApp<
  {
@ -172,7 +175,7 @@ export const getFeedback = (async (args, context) => {
  };
 }) satisfies GetFeedback<{}>;

-export const getStats = (async (_args, context) => {
+export const getProjects = (async (_args, context) => {
  const emailsWhitelist = process.env.ADMIN_EMAILS_WHITELIST?.split(",") || [];
  if (!context.user || !emailsWhitelist.includes(context.user.email)) {
    throw new HttpError(401, "Only admins can access stats.");
@ -199,7 +202,7 @@ export const getStats = (async (_args, context) => {
    },
  });

-  // All projects but without logs
+  // Latest 1000 projects but without logs
  const projects = await Project.findMany({
    orderBy: {
      createdAt: "desc",
@ -221,13 +224,86 @@ export const getStats = (async (_args, context) => {
        },
      },
    },
+    take: 1000,
  });

  return {
    projects,
    latestProjectsWithLogs,
  };
-}) satisfies GetStats<{}>;
+}) satisfies GetProjects<{}>;
+
+export const getStats = (async (args, context) => {
+  const emailsWhitelist = process.env.ADMIN_EMAILS_WHITELIST?.split(",") || [];
+  if (!context.user || !emailsWhitelist.includes(context.user.email)) {
+    throw new HttpError(401, "Only admins can access stats.");
+  }
+
+  const { Project } = context.entities;
+
+  const filterOutExampleAppsCondition = args.filterOutExampleApps
+    ? ({
+        name: {
+          not: {
+            in: ["TodoApp", "MyPlants", "Blog"],
+          },
+        },
+      } satisfies Prisma.ProjectWhereInput)
+    : {};
+
+  const projectsAfterDownloadTrackingCondition = {
+    createdAt: {
+      gt: new Date("2023-07-14 10:36:45.12"),
+    },
+    status: "success",
+  };
+  const [totalGenerated, projectsAfterDownloadTracking, downloadedProjects, last30DaysProjects] =
+    await Promise.all([
+      Project.count({
+        where: {
+          ...filterOutExampleAppsCondition,
+        },
+      }),
+      Project.count({
+        where: {
+          ...projectsAfterDownloadTrackingCondition,
+          ...filterOutExampleAppsCondition,
+        },
+      }),
+      Project.count({
+        where: {
+          ...projectsAfterDownloadTrackingCondition,
+          ...filterOutExampleAppsCondition,
+          zipDownloadedAt: {
+            not: null,
+          },
+        },
+      }),
+      Project.findMany({
+        where: {
+          createdAt: {
+            gte: new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000),
+          },
+          ...filterOutExampleAppsCondition,
+        },
+        select: {
+          createdAt: true,
+        },
+      }),
+    ]);
+  const downloadRatio =
+    projectsAfterDownloadTracking > 0 ? downloadedProjects / projectsAfterDownloadTracking : 0;
+
+  return {
+    totalGenerated,
+    totalDownloaded: downloadedProjects,
+    downloadedPercentage: Math.round(downloadRatio * 10000) / 100,
+    last24Hours: generateLast24HoursData(last30DaysProjects),
+    last30Days: generateLast30DaysData(last30DaysProjects),
+  };
+}) satisfies GetStats<{
+  filterOutExampleApps: boolean;
+}>;

 export const getNumProjects = (async (_args, context) => {
  return context.entities.Project.count();
--- a/wasp-ai/src/server/stats.ts
+++ b/wasp-ai/src/server/stats.ts
@ -0,0 +1,63 @@
+import { Project } from "@wasp/entities";
+
+export function generateLast24HoursData(projects: Pick<Project, "createdAt">[]) {
+  const buckets: {
+    date: Date;
+    displayValue: number;
+    count: number;
+  }[] = [];
+
+  const now = new Date();
+  const last24Hours = new Date(now.getTime() - 24 * 60 * 60 * 1000);
+  for (let i = 0; i < 24; i++) {
+    const bucketStart = new Date(last24Hours.getTime() + i * 60 * 60 * 1000);
+    const bucket = {
+      date: bucketStart,
+      displayValue: bucketStart.getHours() + 1,
+      count: 0,
+    };
+    buckets.push(bucket);
+  }
+  projects.forEach((project) => {
+    const createdAt = new Date(project.createdAt);
+    // Difference in hours between now and when the project was created
+    const bucketIndex = Math.floor((now.getTime() - createdAt.getTime()) / (60 * 60 * 1000));
+    const reverseBucketIndex = buckets.length - bucketIndex - 1;
+    // Count only projects that were created in the last 24 hours
+    if (bucketIndex >= 0 && bucketIndex < 24) {
+      buckets[reverseBucketIndex].count++;
+    }
+  });
+  return buckets;
+}
+
+export function generateLast30DaysData(projects: Pick<Project, "createdAt">[]) {
+  const buckets: {
+    date: Date;
+    displayValue: number;
+    count: number;
+  }[] = [];
+
+  const now = new Date();
+  const last30Days = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
+  for (let i = 0; i < 30; i++) {
+    const bucketStart = new Date(last30Days.getTime() + i * 24 * 60 * 60 * 1000);
+    const bucket = {
+      date: bucketStart,
+      displayValue: bucketStart.getDate(),
+      count: 0,
+    };
+    buckets.push(bucket);
+  }
+  projects.forEach((project) => {
+    const createdAt = new Date(project.createdAt);
+    // Difference in days between now and when the project was created
+    const bucketIndex = Math.floor((now.getTime() - createdAt.getTime()) / (24 * 60 * 60 * 1000));
+    const reverseBucketIndex = buckets.length - bucketIndex - 1;
+    // Count only projects that were created in the last 30 days
+    if (bucketIndex >= 0 && bucketIndex < 30) {
+      buckets[reverseBucketIndex].count++;
+    }
+  });
+  return buckets;
+}