From 0a7f82333bea2249ae4ea6428c99d393606ef9e8 Mon Sep 17 00:00:00 2001 From: Ady Beraud <102751374+ady-beraud@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:44:44 +0300 Subject: [PATCH] Make Github stars dynamic and improve database init (#5000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I extracted the init database logic into its own file. You can now run it with yarn database:init. Added database entry for GitHub stars. Do you want me to remove the init route or is it used for something else ? --------- Co-authored-by: Ady Beraud Co-authored-by: Félix Malfait --- package.json | 1 + packages/twenty-website/package.json | 2 + .../ui/layout/header/HeaderDesktop.tsx | 11 +++- .../ui/layout/header/HeaderMobile.tsx | 10 +++- .../_components/ui/layout/header/index.tsx | 15 ++++-- .../src/app/api/github-stars/route.tsx | 26 ++++++++++ .../src/app/contributors/api/init/route.tsx | 46 ----------------- .../src/app/contributors/api/update/route.tsx | 10 ++-- .../src/database/migrate-database.ts | 8 +++ packages/twenty-website/src/database/model.ts | 29 ++++++----- .../migrations/0000_absent_giant_man.sql | 2 +- .../migrations/0001_marvelous_eddie_brock.sql | 4 ++ .../migrations/meta/0000_snapshot.json | 50 +++++-------------- .../postgres/migrations/meta/_journal.json | 7 +++ .../src/database/postgres/schema-postgres.ts | 7 ++- .../sqlite/migrations/meta/_journal.json | 2 +- .../contributors}/fetch-assignable-users.tsx | 2 +- .../contributors}/fetch-issues-prs.tsx | 2 +- .../contributors}/save-issues-to-db.tsx | 2 +- .../contributors}/save-prs-to-db.tsx | 2 +- .../contributors}/search-issues-prs.tsx | 2 +- .../contributors}/types.tsx | 5 ++ .../github-sync/fetch-and-save-github-data.ts | 44 ++++++++++++++++ .../fetch-and-save-github-stars.tsx | 27 ++++++++++ .../src/github-sync/github-sync.ts | 8 +++ .../src/shared-utils/formatNumberOfStars.ts | 3 ++ yarn.lock | 28 ++++++++++- 27 files changed, 237 insertions(+), 118 deletions(-) create mode 100644 packages/twenty-website/src/app/api/github-stars/route.tsx delete mode 100644 packages/twenty-website/src/app/contributors/api/init/route.tsx create mode 100644 packages/twenty-website/src/database/migrate-database.ts create mode 100644 packages/twenty-website/src/database/postgres/migrations/0001_marvelous_eddie_brock.sql rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/fetch-assignable-users.tsx (88%) rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/fetch-issues-prs.tsx (98%) rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/save-issues-to-db.tsx (95%) rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/save-prs-to-db.tsx (95%) rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/search-issues-prs.tsx (98%) rename packages/twenty-website/src/{app/contributors/api => github-sync/contributors}/types.tsx (94%) create mode 100644 packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts create mode 100644 packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx create mode 100644 packages/twenty-website/src/github-sync/github-sync.ts create mode 100644 packages/twenty-website/src/shared-utils/formatNumberOfStars.ts diff --git a/package.json b/package.json index 51629ce86a..d666947d95 100644 --- a/package.json +++ b/package.json @@ -308,6 +308,7 @@ "ts-loader": "^9.2.3", "ts-node": "10.9.1", "tsconfig-paths": "^4.2.0", + "tsx": "^4.7.2", "typescript": "5.3.3", "vite": "^5.0.0", "vite-plugin-checker": "^0.6.2", diff --git a/packages/twenty-website/package.json b/packages/twenty-website/package.json index a28bbba1cd..1aa0823373 100644 --- a/packages/twenty-website/package.json +++ b/packages/twenty-website/package.json @@ -8,6 +8,8 @@ "build": "npx next build", "start": "npx next start", "lint": "npx next lint", + "github:sync": "npx tsx src/github-sync/github-sync.ts", + "database:migrate": "npx tsx src/database/migrate-database.ts", "database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/postgres/drizzle-posgres.config.ts", "database:generate:sqlite": "npx drizzle-kit generate:sqlite --config=src/database/sqlite/drizzle-sqlite.config.ts" }, diff --git a/packages/twenty-website/src/app/_components/ui/layout/header/HeaderDesktop.tsx b/packages/twenty-website/src/app/_components/ui/layout/header/HeaderDesktop.tsx index 7cd3e846b1..5219dcf7d1 100644 --- a/packages/twenty-website/src/app/_components/ui/layout/header/HeaderDesktop.tsx +++ b/packages/twenty-website/src/app/_components/ui/layout/header/HeaderDesktop.tsx @@ -11,8 +11,13 @@ import { LogoContainer, } from '@/app/_components/ui/layout/header/styled'; import { Logo } from '@/app/_components/ui/layout/Logo'; +import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars'; -export const HeaderDesktop = () => { +type Props = { + numberOfStars: number; +}; + +export const HeaderDesktop = ({ numberOfStars }: Props) => { return ( @@ -26,7 +31,9 @@ export const HeaderDesktop = () => { Docs - 8.3k + + {formatNumberOfStars(numberOfStars)} + diff --git a/packages/twenty-website/src/app/_components/ui/layout/header/HeaderMobile.tsx b/packages/twenty-website/src/app/_components/ui/layout/header/HeaderMobile.tsx index aa1a472641..bc3dea86c1 100644 --- a/packages/twenty-website/src/app/_components/ui/layout/header/HeaderMobile.tsx +++ b/packages/twenty-website/src/app/_components/ui/layout/header/HeaderMobile.tsx @@ -18,6 +18,7 @@ import { NavOpen, } from '@/app/_components/ui/layout/header/styled'; import { Logo } from '@/app/_components/ui/layout/Logo'; +import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars'; const IBMPlexMono = IBM_Plex_Mono({ weight: '500', @@ -25,7 +26,11 @@ const IBMPlexMono = IBM_Plex_Mono({ display: 'swap', }); -export const HeaderMobile = () => { +type Props = { + numberOfStars: number; +}; + +export const HeaderMobile = ({ numberOfStars }: Props) => { const isTwentyDev = false; const [menuOpen, setMenuOpen] = useState(false); @@ -64,7 +69,8 @@ export const HeaderMobile = () => { Docs - 8.3k + {' '} + {formatNumberOfStars(numberOfStars)} diff --git a/packages/twenty-website/src/app/_components/ui/layout/header/index.tsx b/packages/twenty-website/src/app/_components/ui/layout/header/index.tsx index 257c42071b..e650895742 100644 --- a/packages/twenty-website/src/app/_components/ui/layout/header/index.tsx +++ b/packages/twenty-website/src/app/_components/ui/layout/header/index.tsx @@ -1,13 +1,20 @@ -'use client'; +import { desc } from 'drizzle-orm'; import { HeaderDesktop } from '@/app/_components/ui/layout/header/HeaderDesktop'; import { HeaderMobile } from '@/app/_components/ui/layout/header/HeaderMobile'; +import { findOne } from '@/database/database'; +import { githubStarsModel } from '@/database/model'; + +export const AppHeader = async () => { + const githubStars = await findOne( + githubStarsModel, + desc(githubStarsModel.timestamp), + ); -export const AppHeader = () => { return ( <> - - + + ); }; diff --git a/packages/twenty-website/src/app/api/github-stars/route.tsx b/packages/twenty-website/src/app/api/github-stars/route.tsx new file mode 100644 index 0000000000..74ceb3124e --- /dev/null +++ b/packages/twenty-website/src/app/api/github-stars/route.tsx @@ -0,0 +1,26 @@ +import { desc } from 'drizzle-orm'; + +import { findOne } from '@/database/database'; +import { githubStarsModel } from '@/database/model'; +import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars'; + +export const dynamic = 'force-dynamic'; + +export async function GET() { + try { + const githubStars = await findOne( + githubStarsModel, + desc(githubStarsModel.timestamp), + ); + + const formattedGithubNumberOfStars = formatNumberOfStars( + githubStars[0].numberOfStars, + ); + + return Response.json(formattedGithubNumberOfStars); + } catch (error: any) { + return new Response(`Github stars error: ${error?.message}`, { + status: 500, + }); + } +} diff --git a/packages/twenty-website/src/app/contributors/api/init/route.tsx b/packages/twenty-website/src/app/contributors/api/init/route.tsx deleted file mode 100644 index e3181c6e47..0000000000 --- a/packages/twenty-website/src/app/contributors/api/init/route.tsx +++ /dev/null @@ -1,46 +0,0 @@ -export const dynamic = 'force-dynamic'; - -import { global } from '@apollo/client/utilities/globals'; -import { graphql } from '@octokit/graphql'; - -import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users'; -import { fetchIssuesPRs } from '@/app/contributors/api/fetch-issues-prs'; -import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db'; -import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db'; -import { IssueNode, PullRequestNode } from '@/app/contributors/api/types'; -import { migrate } from '@/database/database'; - -export async function GET() { - if (!global.process.env.GITHUB_TOKEN) { - return new Response('No GitHub token provided', { status: 500 }); - } - - const query = graphql.defaults({ - headers: { - Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN, - }, - }); - - await migrate(); - - const assignableUsers = await fetchAssignableUsers(query); - const fetchedPRs = (await fetchIssuesPRs( - query, - null, - false, - [], - )) as Array; - const fetchedIssues = (await fetchIssuesPRs( - query, - null, - true, - [], - )) as Array; - - savePRsToDB(fetchedPRs, assignableUsers); - saveIssuesToDB(fetchedIssues, assignableUsers); - - return new Response('Data synced', { - status: 200, - }); -} diff --git a/packages/twenty-website/src/app/contributors/api/update/route.tsx b/packages/twenty-website/src/app/contributors/api/update/route.tsx index e5bcbb2996..ee512aab0d 100644 --- a/packages/twenty-website/src/app/contributors/api/update/route.tsx +++ b/packages/twenty-website/src/app/contributors/api/update/route.tsx @@ -1,13 +1,13 @@ import { graphql } from '@octokit/graphql'; import { desc } from 'drizzle-orm'; -import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users'; -import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db'; -import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db'; -import { searchIssuesPRs } from '@/app/contributors/api/search-issues-prs'; -import { IssueNode, PullRequestNode } from '@/app/contributors/api/types'; import { findOne } from '@/database/database'; import { issueModel, pullRequestModel } from '@/database/model'; +import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users'; +import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db'; +import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db'; +import { searchIssuesPRs } from '@/github-sync/contributors/search-issues-prs'; +import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types'; export async function GET() { if (!global.process.env.GITHUB_TOKEN) { diff --git a/packages/twenty-website/src/database/migrate-database.ts b/packages/twenty-website/src/database/migrate-database.ts new file mode 100644 index 0000000000..4627be9395 --- /dev/null +++ b/packages/twenty-website/src/database/migrate-database.ts @@ -0,0 +1,8 @@ +import { migrate } from '@/database/database'; + +export const migrateDatabase = async () => { + await migrate(); + process.exit(0); +}; + +migrateDatabase(); diff --git a/packages/twenty-website/src/database/model.ts b/packages/twenty-website/src/database/model.ts index a217c91d8c..fb19c139ea 100644 --- a/packages/twenty-website/src/database/model.ts +++ b/packages/twenty-website/src/database/model.ts @@ -1,4 +1,5 @@ import { + pgGithubStars, pgIssueLabels, pgIssues, pgLabels, @@ -31,17 +32,19 @@ export const issueLabelModel = isSqliteDriver ? sqlLiteIssueLabels : pgIssueLabels; -export type User = typeof sqlLiteUsers.$inferSelect; -export type PullRequest = typeof sqlLitePullRequests.$inferSelect; -export type Issue = typeof sqlLiteIssues.$inferSelect; -export type Label = typeof sqlLiteLabels.$inferSelect; -export type PullRequestLabel = typeof sqlLitePullRequestLabels.$inferSelect; -export type IssueLabel = typeof sqlLiteIssueLabels.$inferSelect; +export const githubStarsModel = pgGithubStars; -export type UserInsert = typeof sqlLiteUsers.$inferInsert; -export type PullRequestInsert = typeof sqlLitePullRequests.$inferInsert; -export type IssueInsert = typeof sqlLiteIssues.$inferInsert; -export type LabelInsert = typeof sqlLiteLabels.$inferInsert; -export type PullRequestLabelInsert = - typeof sqlLitePullRequestLabels.$inferInsert; -export type IssueLabelInsert = typeof sqlLiteIssueLabels.$inferInsert; +export type User = typeof pgUsers.$inferSelect; +export type PullRequest = typeof pgPullRequests.$inferSelect; +export type Issue = typeof pgIssues.$inferSelect; +export type Label = typeof pgLabels.$inferSelect; +export type PullRequestLabel = typeof pgPullRequestLabels.$inferSelect; +export type IssueLabel = typeof pgIssueLabels.$inferSelect; + +export type UserInsert = typeof pgUsers.$inferInsert; +export type PullRequestInsert = typeof pgPullRequests.$inferInsert; +export type IssueInsert = typeof pgIssues.$inferInsert; +export type LabelInsert = typeof pgLabels.$inferInsert; +export type PullRequestLabelInsert = typeof pgPullRequestLabels.$inferInsert; +export type IssueLabelInsert = typeof pgIssueLabels.$inferInsert; +export type GithubStars = typeof pgGithubStars.$inferInsert; diff --git a/packages/twenty-website/src/database/postgres/migrations/0000_absent_giant_man.sql b/packages/twenty-website/src/database/postgres/migrations/0000_absent_giant_man.sql index 140920b557..aeb1824e94 100644 --- a/packages/twenty-website/src/database/postgres/migrations/0000_absent_giant_man.sql +++ b/packages/twenty-website/src/database/postgres/migrations/0000_absent_giant_man.sql @@ -81,4 +81,4 @@ DO $$ BEGIN ALTER TABLE "pullRequests" ADD CONSTRAINT "pullRequests_authorId_users_id_fk" FOREIGN KEY ("authorId") REFERENCES "users"("id") ON DELETE no action ON UPDATE no action; EXCEPTION WHEN duplicate_object THEN null; -END $$; +END $$; \ No newline at end of file diff --git a/packages/twenty-website/src/database/postgres/migrations/0001_marvelous_eddie_brock.sql b/packages/twenty-website/src/database/postgres/migrations/0001_marvelous_eddie_brock.sql new file mode 100644 index 0000000000..92c4f965c0 --- /dev/null +++ b/packages/twenty-website/src/database/postgres/migrations/0001_marvelous_eddie_brock.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS "githubStars" ( + "timestamp" timestamp DEFAULT now() NOT NULL, + "numberOfStars" integer +); diff --git a/packages/twenty-website/src/database/postgres/migrations/meta/0000_snapshot.json b/packages/twenty-website/src/database/postgres/migrations/meta/0000_snapshot.json index bd9ec2ceec..a407c9bd32 100644 --- a/packages/twenty-website/src/database/postgres/migrations/meta/0000_snapshot.json +++ b/packages/twenty-website/src/database/postgres/migrations/meta/0000_snapshot.json @@ -27,12 +27,8 @@ "name": "issueLabels_issueId_issues_id_fk", "tableFrom": "issueLabels", "tableTo": "issues", - "columnsFrom": [ - "issueId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["issueId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -40,12 +36,8 @@ "name": "issueLabels_labelId_labels_id_fk", "tableFrom": "issueLabels", "tableTo": "labels", - "columnsFrom": [ - "labelId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["labelId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -118,12 +110,8 @@ "name": "issues_authorId_users_id_fk", "tableFrom": "issues", "tableTo": "users", - "columnsFrom": [ - "authorId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["authorId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -194,12 +182,8 @@ "name": "pullRequestLabels_pullRequestExternalId_pullRequests_id_fk", "tableFrom": "pullRequestLabels", "tableTo": "pullRequests", - "columnsFrom": [ - "pullRequestExternalId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["pullRequestExternalId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -207,12 +191,8 @@ "name": "pullRequestLabels_labelId_labels_id_fk", "tableFrom": "pullRequestLabels", "tableTo": "labels", - "columnsFrom": [ - "labelId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["labelId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -285,12 +265,8 @@ "name": "pullRequests_authorId_users_id_fk", "tableFrom": "pullRequests", "tableTo": "users", - "columnsFrom": [ - "authorId" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["authorId"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -340,4 +316,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/twenty-website/src/database/postgres/migrations/meta/_journal.json b/packages/twenty-website/src/database/postgres/migrations/meta/_journal.json index c4b3f6d0a7..816086f87a 100644 --- a/packages/twenty-website/src/database/postgres/migrations/meta/_journal.json +++ b/packages/twenty-website/src/database/postgres/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1707921820164, "tag": "0000_absent_giant_man", "breakpoints": true + }, + { + "idx": 1, + "version": "5", + "when": 1713792223113, + "tag": "0001_marvelous_eddie_brock", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/twenty-website/src/database/postgres/schema-postgres.ts b/packages/twenty-website/src/database/postgres/schema-postgres.ts index bb5f429f64..d65aca0c5c 100644 --- a/packages/twenty-website/src/database/postgres/schema-postgres.ts +++ b/packages/twenty-website/src/database/postgres/schema-postgres.ts @@ -1,4 +1,4 @@ -import { pgTable, text } from 'drizzle-orm/pg-core'; +import { integer, pgTable, text, timestamp } from 'drizzle-orm/pg-core'; export const pgUsers = pgTable('users', { id: text('id').primaryKey(), @@ -50,3 +50,8 @@ export const pgIssueLabels = pgTable('issueLabels', { issueId: text('issueId').references(() => pgIssues.id), labelId: text('labelId').references(() => pgLabels.id), }); + +export const pgGithubStars = pgTable('githubStars', { + timestamp: timestamp('timestamp').notNull().defaultNow(), + numberOfStars: integer('numberOfStars'), +}); diff --git a/packages/twenty-website/src/database/sqlite/migrations/meta/_journal.json b/packages/twenty-website/src/database/sqlite/migrations/meta/_journal.json index 4eba798dd0..0b83c40ad7 100644 --- a/packages/twenty-website/src/database/sqlite/migrations/meta/_journal.json +++ b/packages/twenty-website/src/database/sqlite/migrations/meta/_journal.json @@ -10,4 +10,4 @@ "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/twenty-website/src/app/contributors/api/fetch-assignable-users.tsx b/packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx similarity index 88% rename from packages/twenty-website/src/app/contributors/api/fetch-assignable-users.tsx rename to packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx index a244980c0f..63d5ebc45b 100644 --- a/packages/twenty-website/src/app/contributors/api/fetch-assignable-users.tsx +++ b/packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx @@ -1,6 +1,6 @@ import { graphql } from '@octokit/graphql'; -import { Repository } from '@/app/contributors/api/types'; +import { Repository } from '@/github-sync/contributors/types'; export async function fetchAssignableUsers( query: typeof graphql, diff --git a/packages/twenty-website/src/app/contributors/api/fetch-issues-prs.tsx b/packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx similarity index 98% rename from packages/twenty-website/src/app/contributors/api/fetch-issues-prs.tsx rename to packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx index 7ae588a7dd..0127eb6c28 100644 --- a/packages/twenty-website/src/app/contributors/api/fetch-issues-prs.tsx +++ b/packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx @@ -4,7 +4,7 @@ import { IssueNode, PullRequestNode, Repository, -} from '@/app/contributors/api/types'; +} from '@/github-sync/contributors/types'; export async function fetchIssuesPRs( query: typeof graphql, diff --git a/packages/twenty-website/src/app/contributors/api/save-issues-to-db.tsx b/packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx similarity index 95% rename from packages/twenty-website/src/app/contributors/api/save-issues-to-db.tsx rename to packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx index 4ab8bdf02e..f56d247843 100644 --- a/packages/twenty-website/src/app/contributors/api/save-issues-to-db.tsx +++ b/packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx @@ -1,4 +1,3 @@ -import { IssueNode } from '@/app/contributors/api/types'; import { insertMany } from '@/database/database'; import { issueLabelModel, @@ -6,6 +5,7 @@ import { labelModel, userModel, } from '@/database/model'; +import { IssueNode } from '@/github-sync/contributors/types'; export async function saveIssuesToDB( issues: Array, diff --git a/packages/twenty-website/src/app/contributors/api/save-prs-to-db.tsx b/packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx similarity index 95% rename from packages/twenty-website/src/app/contributors/api/save-prs-to-db.tsx rename to packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx index 2f5d37f6cc..27ebf2e984 100644 --- a/packages/twenty-website/src/app/contributors/api/save-prs-to-db.tsx +++ b/packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx @@ -1,4 +1,3 @@ -import { PullRequestNode } from '@/app/contributors/api/types'; import { insertMany } from '@/database/database'; import { labelModel, @@ -6,6 +5,7 @@ import { pullRequestModel, userModel, } from '@/database/model'; +import { PullRequestNode } from '@/github-sync/contributors/types'; export async function savePRsToDB( prs: Array, diff --git a/packages/twenty-website/src/app/contributors/api/search-issues-prs.tsx b/packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx similarity index 98% rename from packages/twenty-website/src/app/contributors/api/search-issues-prs.tsx rename to packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx index cf6e150e17..a2f5ed7bc3 100644 --- a/packages/twenty-website/src/app/contributors/api/search-issues-prs.tsx +++ b/packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx @@ -4,7 +4,7 @@ import { IssueNode, PullRequestNode, SearchIssuesPRsQuery, -} from '@/app/contributors/api/types'; +} from '@/github-sync/contributors/types'; export async function searchIssuesPRs( query: typeof graphql, diff --git a/packages/twenty-website/src/app/contributors/api/types.tsx b/packages/twenty-website/src/github-sync/contributors/types.tsx similarity index 94% rename from packages/twenty-website/src/app/contributors/api/types.tsx rename to packages/twenty-website/src/github-sync/contributors/types.tsx index 0f3968d594..3b78b4df1b 100644 --- a/packages/twenty-website/src/app/contributors/api/types.tsx +++ b/packages/twenty-website/src/github-sync/contributors/types.tsx @@ -64,11 +64,16 @@ export interface AssignableUsers { nodes: AssignableUserNode[]; } +export interface Stargazers { + totalCount: number; +} + export interface Repository { repository: { pullRequests: PullRequests; issues: Issues; assignableUsers: AssignableUsers; + stargazers: Stargazers; }; } diff --git a/packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts b/packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts new file mode 100644 index 0000000000..a3e7cb4c9a --- /dev/null +++ b/packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts @@ -0,0 +1,44 @@ +import { global } from '@apollo/client/utilities/globals'; +import { graphql } from '@octokit/graphql'; + +import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users'; +import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs'; +import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db'; +import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db'; +import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types'; +import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars'; + +export const fetchAndSaveGithubData = async () => { + if (!global.process.env.GITHUB_TOKEN) { + return new Error('No GitHub token provided'); + } + + console.log('Synching data..'); + + const query = graphql.defaults({ + headers: { + Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN, + }, + }); + + await fetchAndSaveGithubStars(query); + + const assignableUsers = await fetchAssignableUsers(query); + const fetchedPRs = (await fetchIssuesPRs( + query, + null, + false, + [], + )) as Array; + const fetchedIssues = (await fetchIssuesPRs( + query, + null, + true, + [], + )) as Array; + + savePRsToDB(fetchedPRs, assignableUsers); + saveIssuesToDB(fetchedIssues, assignableUsers); + + console.log('data synched!'); +}; diff --git a/packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx b/packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx new file mode 100644 index 0000000000..46bdfd992f --- /dev/null +++ b/packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx @@ -0,0 +1,27 @@ +import { graphql } from '@octokit/graphql'; + +import { insertMany } from '@/database/database'; +import { githubStarsModel } from '@/database/model'; +import { Repository } from '@/github-sync/contributors/types'; + +export const fetchAndSaveGithubStars = async ( + query: typeof graphql, +): Promise => { + const { repository } = await query(` + query { + repository(owner: "twentyhq", name: "twenty") { + stargazers { + totalCount + } + } + } + `); + + const numberOfStars = repository.stargazers.totalCount; + + await insertMany(githubStarsModel, [ + { + numberOfStars, + }, + ]); +}; diff --git a/packages/twenty-website/src/github-sync/github-sync.ts b/packages/twenty-website/src/github-sync/github-sync.ts new file mode 100644 index 0000000000..5abb0e3f0e --- /dev/null +++ b/packages/twenty-website/src/github-sync/github-sync.ts @@ -0,0 +1,8 @@ +import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data'; + +export const githubSync = async () => { + await fetchAndSaveGithubData(); + process.exit(0); +}; + +githubSync(); diff --git a/packages/twenty-website/src/shared-utils/formatNumberOfStars.ts b/packages/twenty-website/src/shared-utils/formatNumberOfStars.ts new file mode 100644 index 0000000000..a9fad93d24 --- /dev/null +++ b/packages/twenty-website/src/shared-utils/formatNumberOfStars.ts @@ -0,0 +1,3 @@ +export const formatNumberOfStars = (numberOfStars: number) => { + return Math.floor(numberOfStars / 100) / 10 + 'k'; +}; diff --git a/yarn.lock b/yarn.lock index 9408bdd80f..5badb5c178 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25843,7 +25843,7 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.19.7": +"esbuild@npm:^0.19.7, esbuild@npm:~0.19.10": version: 0.19.12 resolution: "esbuild@npm:0.19.12" dependencies: @@ -28372,6 +28372,15 @@ __metadata: languageName: node linkType: hard +"get-tsconfig@npm:^4.7.2": + version: 4.7.3 + resolution: "get-tsconfig@npm:4.7.3" + dependencies: + resolve-pkg-maps: "npm:^1.0.0" + checksum: b15ca9d5d0887ebfccadc9fe88b6ff3827a5691ec90e7608a5e9c74bef959c14aba62f6bb88ac7f50322395731789a2cf654244f00e10f4f76349911b6846d6f + languageName: node + linkType: hard + "getpass@npm:^0.1.1": version: 0.1.7 resolution: "getpass@npm:0.1.7" @@ -46211,6 +46220,22 @@ __metadata: languageName: node linkType: hard +"tsx@npm:^4.7.2": + version: 4.7.2 + resolution: "tsx@npm:4.7.2" + dependencies: + esbuild: "npm:~0.19.10" + fsevents: "npm:~2.3.3" + get-tsconfig: "npm:^4.7.2" + dependenciesMeta: + fsevents: + optional: true + bin: + tsx: dist/cli.mjs + checksum: 0338598cc3b7b01a47939297797dfb77a1d675acb33bf71e816faf2b8cb76da3994d341d2920d105dbe98cd01a4babd80ca4b9a5a36120813dd79c4fc1c32df1 + languageName: node + linkType: hard + "tty-browserify@npm:0.0.1, tty-browserify@npm:^0.0.1": version: 0.0.1 resolution: "tty-browserify@npm:0.0.1" @@ -46647,6 +46672,7 @@ __metadata: tsconfig-paths: "npm:^4.2.0" tslib: "npm:^2.3.0" tsup: "npm:^8.0.1" + tsx: "npm:^4.7.2" type-fest: "npm:4.10.1" typeorm: "npm:^0.3.17" typescript: "npm:5.3.3"