Make Github stars dynamic and improve database init (#5000)

I extracted the init database logic into its own file. 
You can now run it with yarn database:init.
Added database entry for GitHub stars. 

Do you want me to remove the init route or is it used for something else
?

---------

Co-authored-by: Ady Beraud <a.beraud96@gmail.com>
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
Ady Beraud 2024-04-24 10:44:44 +03:00 committed by GitHub
parent fda0c3c93c
commit 0a7f82333b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 237 additions and 118 deletions

View File

@ -308,6 +308,7 @@
"ts-loader": "^9.2.3",
"ts-node": "10.9.1",
"tsconfig-paths": "^4.2.0",
"tsx": "^4.7.2",
"typescript": "5.3.3",
"vite": "^5.0.0",
"vite-plugin-checker": "^0.6.2",

View File

@ -8,6 +8,8 @@
"build": "npx next build",
"start": "npx next start",
"lint": "npx next lint",
"github:sync": "npx tsx src/github-sync/github-sync.ts",
"database:migrate": "npx tsx src/database/migrate-database.ts",
"database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/postgres/drizzle-posgres.config.ts",
"database:generate:sqlite": "npx drizzle-kit generate:sqlite --config=src/database/sqlite/drizzle-sqlite.config.ts"
},

View File

@ -11,8 +11,13 @@ import {
LogoContainer,
} from '@/app/_components/ui/layout/header/styled';
import { Logo } from '@/app/_components/ui/layout/Logo';
import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
export const HeaderDesktop = () => {
type Props = {
numberOfStars: number;
};
export const HeaderDesktop = ({ numberOfStars }: Props) => {
return (
<DesktopNav>
<LogoContainer>
@ -26,7 +31,9 @@ export const HeaderDesktop = () => {
Docs <ExternalArrow />
</ListItem>
<ListItem href="https://github.com/twentyhq/twenty">
<GithubIcon color="rgb(71,71,71)" /> 8.3k <ExternalArrow />
<GithubIcon color="rgb(71,71,71)" />
{formatNumberOfStars(numberOfStars)}
<ExternalArrow />
</ListItem>
</LinkList>
<CallToAction />

View File

@ -18,6 +18,7 @@ import {
NavOpen,
} from '@/app/_components/ui/layout/header/styled';
import { Logo } from '@/app/_components/ui/layout/Logo';
import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
const IBMPlexMono = IBM_Plex_Mono({
weight: '500',
@ -25,7 +26,11 @@ const IBMPlexMono = IBM_Plex_Mono({
display: 'swap',
});
export const HeaderMobile = () => {
type Props = {
numberOfStars: number;
};
export const HeaderMobile = ({ numberOfStars }: Props) => {
const isTwentyDev = false;
const [menuOpen, setMenuOpen] = useState(false);
@ -64,7 +69,8 @@ export const HeaderMobile = () => {
Docs <ExternalArrow />
</ListItem>
<ListItem href="https://github.com/twentyhq/twenty">
<GithubIcon color="rgb(71,71,71)" /> 8.3k <ExternalArrow />
<GithubIcon color="rgb(71,71,71)" />{' '}
{formatNumberOfStars(numberOfStars)} <ExternalArrow />
</ListItem>
</MobileLinkList>
<CallToAction />

View File

@ -1,13 +1,20 @@
'use client';
import { desc } from 'drizzle-orm';
import { HeaderDesktop } from '@/app/_components/ui/layout/header/HeaderDesktop';
import { HeaderMobile } from '@/app/_components/ui/layout/header/HeaderMobile';
import { findOne } from '@/database/database';
import { githubStarsModel } from '@/database/model';
export const AppHeader = async () => {
const githubStars = await findOne(
githubStarsModel,
desc(githubStarsModel.timestamp),
);
export const AppHeader = () => {
return (
<>
<HeaderDesktop />
<HeaderMobile />
<HeaderDesktop numberOfStars={githubStars?.[0]?.numberOfStars} />
<HeaderMobile numberOfStars={githubStars?.[0]?.numberOfStars} />
</>
);
};

View File

@ -0,0 +1,26 @@
import { desc } from 'drizzle-orm';
import { findOne } from '@/database/database';
import { githubStarsModel } from '@/database/model';
import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
export const dynamic = 'force-dynamic';
export async function GET() {
try {
const githubStars = await findOne(
githubStarsModel,
desc(githubStarsModel.timestamp),
);
const formattedGithubNumberOfStars = formatNumberOfStars(
githubStars[0].numberOfStars,
);
return Response.json(formattedGithubNumberOfStars);
} catch (error: any) {
return new Response(`Github stars error: ${error?.message}`, {
status: 500,
});
}
}

View File

@ -1,46 +0,0 @@
export const dynamic = 'force-dynamic';
import { global } from '@apollo/client/utilities/globals';
import { graphql } from '@octokit/graphql';
import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users';
import { fetchIssuesPRs } from '@/app/contributors/api/fetch-issues-prs';
import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db';
import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db';
import { IssueNode, PullRequestNode } from '@/app/contributors/api/types';
import { migrate } from '@/database/database';
export async function GET() {
if (!global.process.env.GITHUB_TOKEN) {
return new Response('No GitHub token provided', { status: 500 });
}
const query = graphql.defaults({
headers: {
Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
},
});
await migrate();
const assignableUsers = await fetchAssignableUsers(query);
const fetchedPRs = (await fetchIssuesPRs(
query,
null,
false,
[],
)) as Array<PullRequestNode>;
const fetchedIssues = (await fetchIssuesPRs(
query,
null,
true,
[],
)) as Array<IssueNode>;
savePRsToDB(fetchedPRs, assignableUsers);
saveIssuesToDB(fetchedIssues, assignableUsers);
return new Response('Data synced', {
status: 200,
});
}

View File

@ -1,13 +1,13 @@
import { graphql } from '@octokit/graphql';
import { desc } from 'drizzle-orm';
import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users';
import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db';
import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db';
import { searchIssuesPRs } from '@/app/contributors/api/search-issues-prs';
import { IssueNode, PullRequestNode } from '@/app/contributors/api/types';
import { findOne } from '@/database/database';
import { issueModel, pullRequestModel } from '@/database/model';
import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
import { searchIssuesPRs } from '@/github-sync/contributors/search-issues-prs';
import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
export async function GET() {
if (!global.process.env.GITHUB_TOKEN) {

View File

@ -0,0 +1,8 @@
import { migrate } from '@/database/database';
export const migrateDatabase = async () => {
await migrate();
process.exit(0);
};
migrateDatabase();

View File

@ -1,4 +1,5 @@
import {
pgGithubStars,
pgIssueLabels,
pgIssues,
pgLabels,
@ -31,17 +32,19 @@ export const issueLabelModel = isSqliteDriver
? sqlLiteIssueLabels
: pgIssueLabels;
export type User = typeof sqlLiteUsers.$inferSelect;
export type PullRequest = typeof sqlLitePullRequests.$inferSelect;
export type Issue = typeof sqlLiteIssues.$inferSelect;
export type Label = typeof sqlLiteLabels.$inferSelect;
export type PullRequestLabel = typeof sqlLitePullRequestLabels.$inferSelect;
export type IssueLabel = typeof sqlLiteIssueLabels.$inferSelect;
export const githubStarsModel = pgGithubStars;
export type UserInsert = typeof sqlLiteUsers.$inferInsert;
export type PullRequestInsert = typeof sqlLitePullRequests.$inferInsert;
export type IssueInsert = typeof sqlLiteIssues.$inferInsert;
export type LabelInsert = typeof sqlLiteLabels.$inferInsert;
export type PullRequestLabelInsert =
typeof sqlLitePullRequestLabels.$inferInsert;
export type IssueLabelInsert = typeof sqlLiteIssueLabels.$inferInsert;
export type User = typeof pgUsers.$inferSelect;
export type PullRequest = typeof pgPullRequests.$inferSelect;
export type Issue = typeof pgIssues.$inferSelect;
export type Label = typeof pgLabels.$inferSelect;
export type PullRequestLabel = typeof pgPullRequestLabels.$inferSelect;
export type IssueLabel = typeof pgIssueLabels.$inferSelect;
export type UserInsert = typeof pgUsers.$inferInsert;
export type PullRequestInsert = typeof pgPullRequests.$inferInsert;
export type IssueInsert = typeof pgIssues.$inferInsert;
export type LabelInsert = typeof pgLabels.$inferInsert;
export type PullRequestLabelInsert = typeof pgPullRequestLabels.$inferInsert;
export type IssueLabelInsert = typeof pgIssueLabels.$inferInsert;
export type GithubStars = typeof pgGithubStars.$inferInsert;

View File

@ -81,4 +81,4 @@ DO $$ BEGIN
ALTER TABLE "pullRequests" ADD CONSTRAINT "pullRequests_authorId_users_id_fk" FOREIGN KEY ("authorId") REFERENCES "users"("id") ON DELETE no action ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
END $$;

View File

@ -0,0 +1,4 @@
CREATE TABLE IF NOT EXISTS "githubStars" (
"timestamp" timestamp DEFAULT now() NOT NULL,
"numberOfStars" integer
);

View File

@ -27,12 +27,8 @@
"name": "issueLabels_issueId_issues_id_fk",
"tableFrom": "issueLabels",
"tableTo": "issues",
"columnsFrom": [
"issueId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["issueId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
},
@ -40,12 +36,8 @@
"name": "issueLabels_labelId_labels_id_fk",
"tableFrom": "issueLabels",
"tableTo": "labels",
"columnsFrom": [
"labelId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["labelId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
}
@ -118,12 +110,8 @@
"name": "issues_authorId_users_id_fk",
"tableFrom": "issues",
"tableTo": "users",
"columnsFrom": [
"authorId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["authorId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
}
@ -194,12 +182,8 @@
"name": "pullRequestLabels_pullRequestExternalId_pullRequests_id_fk",
"tableFrom": "pullRequestLabels",
"tableTo": "pullRequests",
"columnsFrom": [
"pullRequestExternalId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["pullRequestExternalId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
},
@ -207,12 +191,8 @@
"name": "pullRequestLabels_labelId_labels_id_fk",
"tableFrom": "pullRequestLabels",
"tableTo": "labels",
"columnsFrom": [
"labelId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["labelId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
}
@ -285,12 +265,8 @@
"name": "pullRequests_authorId_users_id_fk",
"tableFrom": "pullRequests",
"tableTo": "users",
"columnsFrom": [
"authorId"
],
"columnsTo": [
"id"
],
"columnsFrom": ["authorId"],
"columnsTo": ["id"],
"onDelete": "no action",
"onUpdate": "no action"
}
@ -340,4 +316,4 @@
"schemas": {},
"tables": {}
}
}
}

View File

@ -8,6 +8,13 @@
"when": 1707921820164,
"tag": "0000_absent_giant_man",
"breakpoints": true
},
{
"idx": 1,
"version": "5",
"when": 1713792223113,
"tag": "0001_marvelous_eddie_brock",
"breakpoints": true
}
]
}

View File

@ -1,4 +1,4 @@
import { pgTable, text } from 'drizzle-orm/pg-core';
import { integer, pgTable, text, timestamp } from 'drizzle-orm/pg-core';
export const pgUsers = pgTable('users', {
id: text('id').primaryKey(),
@ -50,3 +50,8 @@ export const pgIssueLabels = pgTable('issueLabels', {
issueId: text('issueId').references(() => pgIssues.id),
labelId: text('labelId').references(() => pgLabels.id),
});
export const pgGithubStars = pgTable('githubStars', {
timestamp: timestamp('timestamp').notNull().defaultNow(),
numberOfStars: integer('numberOfStars'),
});

View File

@ -10,4 +10,4 @@
"breakpoints": true
}
]
}
}

View File

@ -1,6 +1,6 @@
import { graphql } from '@octokit/graphql';
import { Repository } from '@/app/contributors/api/types';
import { Repository } from '@/github-sync/contributors/types';
export async function fetchAssignableUsers(
query: typeof graphql,

View File

@ -4,7 +4,7 @@ import {
IssueNode,
PullRequestNode,
Repository,
} from '@/app/contributors/api/types';
} from '@/github-sync/contributors/types';
export async function fetchIssuesPRs(
query: typeof graphql,

View File

@ -1,4 +1,3 @@
import { IssueNode } from '@/app/contributors/api/types';
import { insertMany } from '@/database/database';
import {
issueLabelModel,
@ -6,6 +5,7 @@ import {
labelModel,
userModel,
} from '@/database/model';
import { IssueNode } from '@/github-sync/contributors/types';
export async function saveIssuesToDB(
issues: Array<IssueNode>,

View File

@ -1,4 +1,3 @@
import { PullRequestNode } from '@/app/contributors/api/types';
import { insertMany } from '@/database/database';
import {
labelModel,
@ -6,6 +5,7 @@ import {
pullRequestModel,
userModel,
} from '@/database/model';
import { PullRequestNode } from '@/github-sync/contributors/types';
export async function savePRsToDB(
prs: Array<PullRequestNode>,

View File

@ -4,7 +4,7 @@ import {
IssueNode,
PullRequestNode,
SearchIssuesPRsQuery,
} from '@/app/contributors/api/types';
} from '@/github-sync/contributors/types';
export async function searchIssuesPRs(
query: typeof graphql,

View File

@ -64,11 +64,16 @@ export interface AssignableUsers {
nodes: AssignableUserNode[];
}
export interface Stargazers {
totalCount: number;
}
export interface Repository {
repository: {
pullRequests: PullRequests;
issues: Issues;
assignableUsers: AssignableUsers;
stargazers: Stargazers;
};
}

View File

@ -0,0 +1,44 @@
import { global } from '@apollo/client/utilities/globals';
import { graphql } from '@octokit/graphql';
import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs';
import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars';
export const fetchAndSaveGithubData = async () => {
if (!global.process.env.GITHUB_TOKEN) {
return new Error('No GitHub token provided');
}
console.log('Synching data..');
const query = graphql.defaults({
headers: {
Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
},
});
await fetchAndSaveGithubStars(query);
const assignableUsers = await fetchAssignableUsers(query);
const fetchedPRs = (await fetchIssuesPRs(
query,
null,
false,
[],
)) as Array<PullRequestNode>;
const fetchedIssues = (await fetchIssuesPRs(
query,
null,
true,
[],
)) as Array<IssueNode>;
savePRsToDB(fetchedPRs, assignableUsers);
saveIssuesToDB(fetchedIssues, assignableUsers);
console.log('data synched!');
};

View File

@ -0,0 +1,27 @@
import { graphql } from '@octokit/graphql';
import { insertMany } from '@/database/database';
import { githubStarsModel } from '@/database/model';
import { Repository } from '@/github-sync/contributors/types';
export const fetchAndSaveGithubStars = async (
query: typeof graphql,
): Promise<void> => {
const { repository } = await query<Repository>(`
query {
repository(owner: "twentyhq", name: "twenty") {
stargazers {
totalCount
}
}
}
`);
const numberOfStars = repository.stargazers.totalCount;
await insertMany(githubStarsModel, [
{
numberOfStars,
},
]);
};

View File

@ -0,0 +1,8 @@
import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data';
export const githubSync = async () => {
await fetchAndSaveGithubData();
process.exit(0);
};
githubSync();

View File

@ -0,0 +1,3 @@
export const formatNumberOfStars = (numberOfStars: number) => {
return Math.floor(numberOfStars / 100) / 10 + 'k';
};

View File

@ -25843,7 +25843,7 @@ __metadata:
languageName: node
linkType: hard
"esbuild@npm:^0.19.7":
"esbuild@npm:^0.19.7, esbuild@npm:~0.19.10":
version: 0.19.12
resolution: "esbuild@npm:0.19.12"
dependencies:
@ -28372,6 +28372,15 @@ __metadata:
languageName: node
linkType: hard
"get-tsconfig@npm:^4.7.2":
version: 4.7.3
resolution: "get-tsconfig@npm:4.7.3"
dependencies:
resolve-pkg-maps: "npm:^1.0.0"
checksum: b15ca9d5d0887ebfccadc9fe88b6ff3827a5691ec90e7608a5e9c74bef959c14aba62f6bb88ac7f50322395731789a2cf654244f00e10f4f76349911b6846d6f
languageName: node
linkType: hard
"getpass@npm:^0.1.1":
version: 0.1.7
resolution: "getpass@npm:0.1.7"
@ -46211,6 +46220,22 @@ __metadata:
languageName: node
linkType: hard
"tsx@npm:^4.7.2":
version: 4.7.2
resolution: "tsx@npm:4.7.2"
dependencies:
esbuild: "npm:~0.19.10"
fsevents: "npm:~2.3.3"
get-tsconfig: "npm:^4.7.2"
dependenciesMeta:
fsevents:
optional: true
bin:
tsx: dist/cli.mjs
checksum: 0338598cc3b7b01a47939297797dfb77a1d675acb33bf71e816faf2b8cb76da3994d341d2920d105dbe98cd01a4babd80ca4b9a5a36120813dd79c4fc1c32df1
languageName: node
linkType: hard
"tty-browserify@npm:0.0.1, tty-browserify@npm:^0.0.1":
version: 0.0.1
resolution: "tty-browserify@npm:0.0.1"
@ -46647,6 +46672,7 @@ __metadata:
tsconfig-paths: "npm:^4.2.0"
tslib: "npm:^2.3.0"
tsup: "npm:^8.0.1"
tsx: "npm:^4.7.2"
type-fest: "npm:4.10.1"
typeorm: "npm:^0.3.17"
typescript: "npm:5.3.3"