3011 fill the messagerecipient table when fetching messages (#3073)

* wip

* trying to parse display names and emails

* add nodemailer mailparser

* mail parsing is working

* add personId and workspaceMemberId

* add date to messages

* Fix PR

* Run tsc on bigger machine

* Fix lint

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
bosiraphael 2023-12-19 17:08:54 +01:00 committed by GitHub
parent b1ec3bdf42
commit 5afcab4e78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 263 additions and 56 deletions

View File

@ -142,6 +142,8 @@ jobs:
restore-keys: root-node_modules-
- name: Front / Run linter
run: yarn nx lint:ci twenty-front
- name: Front / Run Typescript Check
run: yarn nx tsc:ci twenty-front
front-jest:
needs: front-yarn-install
runs-on: ubuntu-latest

View File

@ -2,9 +2,7 @@
"targetDefaults": {
"build": {
"cache": true,
"dependsOn": [
"^build"
]
"dependsOn": ["^build"]
},
"lint": {
"cache": true
@ -21,6 +19,5 @@
},
"affected": {
"defaultBase": "main"
},
"nxCloudAccessToken": "OWU4MWJiZGYtNTIzZi00OTVhLWE5NWEtNDBhNjk0NThhYzRmfHJlYWQtd3JpdGU="
}
}

View File

@ -9,6 +9,7 @@
"build": "tsc && vite build && yarn build:inject-runtime-env",
"build:inject-runtime-env": "sh ./scripts/inject-runtime-env.sh",
"tsc": "tsc --watch",
"tsc:ci": "tsc",
"preview": "vite preview",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"lint:ci": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0 --config .eslintrc-ci.cjs",

View File

@ -1,5 +1,6 @@
import { H2Title } from '@/ui/display/typography/components/H2Title';
import { Section } from '@/ui/layout/section/components/Section';
import { SettingsAccountsEmptyStateCard } from './SettingsAccountsEmptyStateCard';
export const SettingsNewAccountSection = () => {

View File

@ -3,4 +3,5 @@ export type FeatureFlagKey =
| 'IS_NOTE_CREATE_IMAGES_ENABLED'
| 'IS_RELATION_FIELD_TYPE_ENABLED'
| 'IS_SELECT_FIELD_TYPE_ENABLED'
| 'IS_QUICK_ACTIONS_ENABLED'
| 'IS_RATING_FIELD_TYPE_ENABLED';

View File

@ -58,6 +58,7 @@
"@sentry/tracing": "^7.66.0",
"@types/lodash.camelcase": "^4.3.7",
"@types/lodash.merge": "^4.6.7",
"@types/mailparser": "^3.4.4",
"add": "^2.0.6",
"apollo-server-express": "^3.12.0",
"axios": "^1.6.2",
@ -88,6 +89,7 @@
"lodash.merge": "^4.6.2",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"mailparser": "^3.6.5",
"microdiff": "^1.3.2",
"nest-commander": "^3.12.0",
"openapi-types": "^12.1.3",

View File

@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import { simpleParser } from 'mailparser';
@Injectable()
export class FetchBatchMessagesService {
@ -57,7 +58,9 @@ export class FetchBatchMessagesService {
},
);
return this.formatBatchResponse(response);
const formattedResponse = await this.formatBatchResponse(response);
return formattedResponse;
}
createBatchBody(messageQueries, boundary: string): string {
@ -121,40 +124,44 @@ export class FetchBatchMessagesService {
return boundary.replace('boundary=', '').trim('; ');
}
formatBatchResponse(response) {
async formatBatchResponse(response) {
const parsedResponse = this.parseBatch(response);
return parsedResponse
.map((item) => {
const { id, threadId, payload } = item;
return Promise.all(
parsedResponse.map(async (item) => {
const { id, threadId, internalDate, raw } = item;
const headers = payload?.headers;
const message = atob(raw?.replace(/-/g, '+').replace(/_/g, '/'));
const parts = payload?.parts;
const parsed = await simpleParser(message);
if (!parts) {
return;
}
const bodyBase64 = parts[0]?.body?.data;
if (!bodyBase64) {
return;
}
const body = atob(bodyBase64.replace(/-/g, '+').replace(/_/g, '/'));
const {
subject,
messageId,
from,
to,
cc,
bcc,
text,
html,
attachments,
} = parsed;
return {
externalId: id,
headerMessageId: headers?.find(
(header) => header.name === 'Message-ID',
)?.value,
subject: headers?.find((header) => header.name === 'Subject')?.value,
headerMessageId: messageId,
subject: subject,
messageThreadId: threadId,
from: headers?.find((header) => header.name === 'From')?.value,
body,
internalDate,
from,
to,
cc,
bcc,
text,
html,
attachments,
};
})
.filter((item) => item);
}),
);
}
}

View File

@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { google } from 'googleapis';
import { v4 } from 'uuid';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -110,7 +111,7 @@ export class FetchWorkspaceMessagesService {
}
const messageQueries = messagesData.map((message) => ({
uri: '/gmail/v1/users/me/messages/' + message.id,
uri: '/gmail/v1/users/me/messages/' + message.id + '?format=RAW',
}));
const messagesResponse =
@ -123,6 +124,7 @@ export class FetchWorkspaceMessagesService {
messagesResponse,
dataSourceMetadata,
workspaceDataSource,
workspaceMemberId,
);
return messages;
@ -168,33 +170,61 @@ export class FetchWorkspaceMessagesService {
}
}
async saveMessages(messages, dataSourceMetadata, workspaceDataSource) {
async saveMessages(
messages,
dataSourceMetadata,
workspaceDataSource,
workspaceMemberId,
) {
for (const message of messages) {
const {
externalId,
headerMessageId,
subject,
messageThreadId,
internalDate,
from,
body,
text,
} = message;
const date = new Date(parseInt(internalDate));
const messageThread = await workspaceDataSource?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."messageThread" WHERE "externalId" = $1`,
[messageThreadId],
);
await workspaceDataSource?.query(
`INSERT INTO ${dataSourceMetadata.schema}."message" ("externalId", "headerMessageId", "subject", "messageThreadId", "direction", "body") VALUES ($1, $2, $3, $4, $5, $6)`,
[
externalId,
headerMessageId,
subject,
messageThread[0]?.id,
'incoming',
body,
],
const messageId = v4();
const handle = from?.value[0]?.address;
const displayName = from?.value[0]?.name;
const person = await workspaceDataSource?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."person" WHERE "email" = $1`,
[handle],
);
const personId = person[0]?.id;
await workspaceDataSource?.transaction(async (manager) => {
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."message" ("id", "externalId", "headerMessageId", "subject", "date", "messageThreadId", "direction", "body") VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
[
messageId,
externalId,
headerMessageId,
subject,
date,
messageThread[0]?.id,
'incoming',
text,
],
);
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."messageRecipient" ("messageId", "role", "handle", "displayName", "personId", "workspaceMemberId") VALUES ($1, $2, $3, $4, $5, $6)`,
[messageId, 'from', handle, displayName, personId, workspaceMemberId],
);
});
}
}
}

View File

@ -81,6 +81,15 @@ export class MessageObjectMetadata extends BaseObjectMetadata {
@IsNullable()
body: string;
@FieldMetadata({
type: FieldMetadataType.DATE_TIME,
label: 'Date',
description: 'Date',
icon: 'IconCalendar',
})
@IsNullable()
date: string;
@FieldMetadata({
type: FieldMetadataType.RELATION,
label: 'Message Recipients',

181
yarn.lock
View File

@ -9959,6 +9959,16 @@ __metadata:
languageName: node
linkType: hard
"@selderee/plugin-htmlparser2@npm:^0.11.0":
version: 0.11.0
resolution: "@selderee/plugin-htmlparser2@npm:0.11.0"
dependencies:
domhandler: "npm:^5.0.3"
selderee: "npm:^0.11.0"
checksum: e938ba9aeb31a9cf30dcb2977ef41685c598bf744bedc88c57aa9e8b7e71b51781695cf99c08aac50773fd7714eba670bd2a079e46db0788abe40c6d220084eb
languageName: node
linkType: hard
"@sentry-internal/tracing@npm:7.86.0":
version: 7.86.0
resolution: "@sentry-internal/tracing@npm:7.86.0"
@ -13889,6 +13899,16 @@ __metadata:
languageName: node
linkType: hard
"@types/mailparser@npm:^3.4.4":
version: 3.4.4
resolution: "@types/mailparser@npm:3.4.4"
dependencies:
"@types/node": "npm:*"
iconv-lite: "npm:^0.6.3"
checksum: 5d16e87cebff438f9e725ebb4f4cea4e6c55dfa1d5cdda3c56f3f91b915a0801a84675fee2a8d20b6de20ca8be79678a4e99fb5956104e2eb3344dfac387691c
languageName: node
linkType: hard
"@types/mdast@npm:^3.0.0":
version: 3.0.15
resolution: "@types/mdast@npm:3.0.15"
@ -21445,6 +21465,13 @@ __metadata:
languageName: node
linkType: hard
"encoding-japanese@npm:2.0.0":
version: 2.0.0
resolution: "encoding-japanese@npm:2.0.0"
checksum: 453bbca71d3666213a9bc873d5a69441b379f158a2992aa5cd1fc124c915b518e19fce7654f973d1334234f870e8053443a464c8f73ff9d7efe66bbc1ce1f4f6
languageName: node
linkType: hard
"encoding@npm:^0.1.12, encoding@npm:^0.1.13":
version: 0.1.13
resolution: "encoding@npm:0.1.13"
@ -25258,7 +25285,7 @@ __metadata:
languageName: node
linkType: hard
"he@npm:^1.2.0":
"he@npm:1.2.0, he@npm:^1.2.0":
version: 1.2.0
resolution: "he@npm:1.2.0"
bin:
@ -25459,6 +25486,19 @@ __metadata:
languageName: node
linkType: hard
"html-to-text@npm:9.0.5":
version: 9.0.5
resolution: "html-to-text@npm:9.0.5"
dependencies:
"@selderee/plugin-htmlparser2": "npm:^0.11.0"
deepmerge: "npm:^4.3.1"
dom-serializer: "npm:^2.0.0"
htmlparser2: "npm:^8.0.2"
selderee: "npm:^0.11.0"
checksum: 5d2c77b798cf88a81b1da2fc1ea1a3b3e2ff49fe5a3d812392f802fff18ec315cf0969bd7846ef2eb7df8c37f463bc63e8cbdcf84e42696c6f3e15dfa61cdf4f
languageName: node
linkType: hard
"html-void-elements@npm:^2.0.0":
version: 2.0.1
resolution: "html-void-elements@npm:2.0.1"
@ -25528,7 +25568,7 @@ __metadata:
languageName: node
linkType: hard
"htmlparser2@npm:^8.0.1":
"htmlparser2@npm:^8.0.1, htmlparser2@npm:^8.0.2":
version: 8.0.2
resolution: "htmlparser2@npm:8.0.2"
dependencies:
@ -25842,7 +25882,7 @@ __metadata:
languageName: node
linkType: hard
"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2":
"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2, iconv-lite@npm:^0.6.3":
version: 0.6.3
resolution: "iconv-lite@npm:0.6.3"
dependencies:
@ -29017,6 +29057,13 @@ __metadata:
languageName: node
linkType: hard
"leac@npm:^0.6.0":
version: 0.6.0
resolution: "leac@npm:0.6.0"
checksum: 5257781e10791ef8462eb1cbe5e48e3cda7692486f2a775265d6f5216cc088960c62f138163b8df0dcf2119d18673bfe7b050d6b41543d92a7b7ac90e4eb1e8b
languageName: node
linkType: hard
"leven@npm:^3.1.0":
version: 3.1.0
resolution: "leven@npm:3.1.0"
@ -29053,6 +29100,37 @@ __metadata:
languageName: node
linkType: hard
"libbase64@npm:1.2.1":
version: 1.2.1
resolution: "libbase64@npm:1.2.1"
checksum: 908db9dc88cbcd9e1b9355c78b9fefde5034d933a50e823bbbb6008a56908de1e5183e25bf648e9e7fe907f53e10e11676d5ac89fd624a300c46a705556182a5
languageName: node
linkType: hard
"libmime@npm:5.2.0":
version: 5.2.0
resolution: "libmime@npm:5.2.0"
dependencies:
encoding-japanese: "npm:2.0.0"
iconv-lite: "npm:0.6.3"
libbase64: "npm:1.2.1"
libqp: "npm:2.0.1"
checksum: 22a75d7aad8f01bed7d9b32270a40a32c4d4e44070edda1067ea5229df99a09f34aedf3481693394aa998fa8375b6c90d1c651b491655692cb313561c5a48762
languageName: node
linkType: hard
"libmime@npm:5.2.1":
version: 5.2.1
resolution: "libmime@npm:5.2.1"
dependencies:
encoding-japanese: "npm:2.0.0"
iconv-lite: "npm:0.6.3"
libbase64: "npm:1.2.1"
libqp: "npm:2.0.1"
checksum: cf91c78a05824f160e45b36850d52eee9e18073bfd4561ace3b3af8e52a8d551eccc0dcef428505e44d1f2146c16cec84e91e17d9489244451c38572862a857a
languageName: node
linkType: hard
"libphonenumber-js@npm:^1.10.14, libphonenumber-js@npm:^1.10.26, libphonenumber-js@npm:^1.10.48":
version: 1.10.51
resolution: "libphonenumber-js@npm:1.10.51"
@ -29060,6 +29138,13 @@ __metadata:
languageName: node
linkType: hard
"libqp@npm:2.0.1":
version: 2.0.1
resolution: "libqp@npm:2.0.1"
checksum: c52e51c70180fbf0b000036de33ed976da1f8355fd63feffbbf5a9653a816e9169917b1ce28b289a5006b28e44b2d84d234fdedbdfefc0de4802867aa03537df
languageName: node
linkType: hard
"lilconfig@npm:^2.0.3":
version: 2.1.0
resolution: "lilconfig@npm:2.1.0"
@ -29088,6 +29173,15 @@ __metadata:
languageName: node
linkType: hard
"linkify-it@npm:4.0.1, linkify-it@npm:^4.0.1":
version: 4.0.1
resolution: "linkify-it@npm:4.0.1"
dependencies:
uc.micro: "npm:^1.0.1"
checksum: f1949ee2c7c2979c4f80c8c08f507d813f50775ebc5adfdb7ee662f28e0ee53dbd4a329d5231be67414405fc60d4e99b37536d6949702d311fe509a6bcbcf4a6
languageName: node
linkType: hard
"linkify-it@npm:^3.0.1":
version: 3.0.3
resolution: "linkify-it@npm:3.0.3"
@ -29097,15 +29191,6 @@ __metadata:
languageName: node
linkType: hard
"linkify-it@npm:^4.0.1":
version: 4.0.1
resolution: "linkify-it@npm:4.0.1"
dependencies:
uc.micro: "npm:^1.0.1"
checksum: f1949ee2c7c2979c4f80c8c08f507d813f50775ebc5adfdb7ee662f28e0ee53dbd4a329d5231be67414405fc60d4e99b37536d6949702d311fe509a6bcbcf4a6
languageName: node
linkType: hard
"linkifyjs@npm:^4.1.0":
version: 4.1.3
resolution: "linkifyjs@npm:4.1.3"
@ -29736,6 +29821,34 @@ __metadata:
languageName: node
linkType: hard
"mailparser@npm:^3.6.5":
version: 3.6.5
resolution: "mailparser@npm:3.6.5"
dependencies:
encoding-japanese: "npm:2.0.0"
he: "npm:1.2.0"
html-to-text: "npm:9.0.5"
iconv-lite: "npm:0.6.3"
libmime: "npm:5.2.1"
linkify-it: "npm:4.0.1"
mailsplit: "npm:5.4.0"
nodemailer: "npm:6.9.3"
tlds: "npm:1.240.0"
checksum: b3b2d8d7f9f9230f449bf0cfac45c0386b63365929e389e196326496d7bae58377830b4cebb59a4d44c429b349590a8e191704f26e7fb1fddcbd7492482c7d35
languageName: node
linkType: hard
"mailsplit@npm:5.4.0":
version: 5.4.0
resolution: "mailsplit@npm:5.4.0"
dependencies:
libbase64: "npm:1.2.1"
libmime: "npm:5.2.0"
libqp: "npm:2.0.1"
checksum: b0e1ce1866ea44413ca0ee8b7291afb671cb3f7ced2a53c644e3097b64b74079a4cb1ec02c9aaaef6a9927a71187304ac1a809852503aba2f829b67ce2d41496
languageName: node
linkType: hard
"make-dir@npm:^2.0.0, make-dir@npm:^2.1.0":
version: 2.1.0
resolution: "make-dir@npm:2.1.0"
@ -32606,6 +32719,13 @@ __metadata:
languageName: node
linkType: hard
"nodemailer@npm:6.9.3":
version: 6.9.3
resolution: "nodemailer@npm:6.9.3"
checksum: a51f8ca46bf8960915f74b7d0c050e498bc5a76d2e5b376ce7abe638da997c246f4a828d231e8845de5eabf189aa23dd92af2dfe63ed04c9c194f3c014ad955d
languageName: node
linkType: hard
"nopt@npm:^5.0.0":
version: 5.0.0
resolution: "nopt@npm:5.0.0"
@ -33989,6 +34109,16 @@ __metadata:
languageName: node
linkType: hard
"parseley@npm:^0.12.0":
version: 0.12.1
resolution: "parseley@npm:0.12.1"
dependencies:
leac: "npm:^0.6.0"
peberminta: "npm:^0.9.0"
checksum: df3de74172b72305b867298a71e5882c413df75d30f2bafb5fb70779dfd349c5e4db03441fbf8ca83da8e4aa72bd0ef2b5c73086c4825d27d1c649d61bc0bcc0
languageName: node
linkType: hard
"parseurl@npm:^1.3.3, parseurl@npm:~1.3.2, parseurl@npm:~1.3.3":
version: 1.3.3
resolution: "parseurl@npm:1.3.3"
@ -34291,6 +34421,13 @@ __metadata:
languageName: node
linkType: hard
"peberminta@npm:^0.9.0":
version: 0.9.0
resolution: "peberminta@npm:0.9.0"
checksum: 59c2c39269d9f7f559cf44582f1c0503524c6a9bc3478e0309adba2b41c71ab98745a239a4e6f98f46105291256e6d8f12ae9860d9f016b1c9a6f52c0b63bfe7
languageName: node
linkType: hard
"peek-readable@npm:^4.1.0":
version: 4.1.0
resolution: "peek-readable@npm:4.1.0"
@ -38156,6 +38293,15 @@ __metadata:
languageName: node
linkType: hard
"selderee@npm:^0.11.0":
version: 0.11.0
resolution: "selderee@npm:0.11.0"
dependencies:
parseley: "npm:^0.12.0"
checksum: c2ad8313a0dbf3c0b74752a8d03cfbc0931ae77a36679cdb64733eb732c1762f95a5174249bf7e8b8103874cb0e013a030f9c8b72f5d41e62f1d847d4a845d39
languageName: node
linkType: hard
"select-hose@npm:^2.0.0":
version: 2.0.0
resolution: "select-hose@npm:2.0.0"
@ -40232,6 +40378,15 @@ __metadata:
languageName: node
linkType: hard
"tlds@npm:1.240.0":
version: 1.240.0
resolution: "tlds@npm:1.240.0"
bin:
tlds: bin.js
checksum: c9c9f0fd8ed5fffdb7d8fb73575d16fae621b1a6a8a7c0a19c28b05092893696de1dcc670e068965af0296f07abb8a9c96695a0f3e7de4bb70b558841ed49ba9
languageName: node
linkType: hard
"tmp@npm:0.2.1, tmp@npm:~0.2.1":
version: 0.2.1
resolution: "tmp@npm:0.2.1"
@ -41035,6 +41190,7 @@ __metadata:
"@types/lodash.merge": "npm:^4.6.7"
"@types/lodash.snakecase": "npm:^4.1.7"
"@types/lodash.upperfirst": "npm:^4.3.7"
"@types/mailparser": "npm:^3.4.4"
"@types/ms": "npm:^0.7.31"
"@types/node": "npm:^16.0.0"
"@types/passport-google-oauth20": "npm:^2.0.11"
@ -41079,6 +41235,7 @@ __metadata:
lodash.merge: "npm:^4.6.2"
lodash.snakecase: "npm:^4.1.1"
lodash.upperfirst: "npm:^4.3.1"
mailparser: "npm:^3.6.5"
microdiff: "npm:^1.3.2"
nest-commander: "npm:^3.12.0"
openapi-types: "npm:^12.1.3"