upload rework

Signed-off-by: Andrey Platov <andrey@hardcoreeng.com>

common/config/rush/pnpm-lock.yaml

@@ -8202,37 +8202,6 @@ packages:
       yn: 3.1.1
     dev: false
 
-  /ts-node/10.2.1_f50b86b1778cd2aa4e5405c08bb39559:
-    resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==}
-    engines: {node: '>=12.0.0'}
-    hasBin: true
-    peerDependencies:
-      '@swc/core': '>=1.2.50'
-      '@swc/wasm': '>=1.2.50'
-      '@types/node': '*'
-      typescript: '>=2.7'
-    peerDependenciesMeta:
-      '@swc/core':
-        optional: true
-      '@swc/wasm':
-        optional: true
-    dependencies:
-      '@cspotcode/source-map-support': 0.6.1
-      '@tsconfig/node10': 1.0.8
-      '@tsconfig/node12': 1.0.9
-      '@tsconfig/node14': 1.0.1
-      '@tsconfig/node16': 1.0.2
-      '@types/node': 16.7.1
-      acorn: 8.4.1
-      acorn-walk: 8.1.1
-      arg: 4.1.3
-      create-require: 1.1.1
-      diff: 4.0.2
-      make-error: 1.3.6
-      typescript: 4.3.5
-      yn: 3.1.1
-    dev: false
-
   /ts-node/10.2.1_typescript@4.3.5:
     resolution: {integrity: sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==}
     engines: {node: '>=12.0.0'}
@@ -9164,7 +9133,7 @@ packages:
     version: 0.0.0
     dependencies:
       '@types/heft-jest': 1.0.2
-      '@types/node': 16.7.1
+      '@types/node': 16.7.5
       '@types/ws': 7.4.7
       '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea
       eslint: 7.32.0
@@ -9298,23 +9267,23 @@ packages:
     dev: false
 
   file:projects/elastic.tgz_6c259fadfeb3a4b20890aefe87070b8b:
-    resolution: {integrity: sha512-uoPGQmOAopNC8F64rYWrRDU1s3/u70CNQeBYz7EH/PE6mR0QsYIFJW3U6/LgeDWsMXNge+5fKkjwAJh0+EQtBA==, tarball: file:projects/elastic.tgz}
+    resolution: {integrity: sha512-IEQdOVDEl4SLQUo1mu4wppfdnbAyiAMp38TL1yfF+Xh7J99/R7uSqQq0pI+nsUXKAR0oSsmMoHPiEYo7Kc15Ig==, tarball: file:projects/elastic.tgz}
     id: file:projects/elastic.tgz
     name: '@rush-temp/elastic'
     version: 0.0.0
     dependencies:
       '@elastic/elasticsearch': 7.14.0
       '@types/heft-jest': 1.0.2
+      '@types/node': 16.7.5
       '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea
       eslint: 7.32.0
       eslint-plugin-import: 2.23.4_eslint@7.32.0
       eslint-plugin-node: 11.1.0_eslint@7.32.0
       eslint-plugin-promise: 4.3.1
-      ts-node: 10.2.1_typescript@4.3.5
+      ts-node: 10.2.1_eb14afb1492fcd444e277f1fdb668e87
     transitivePeerDependencies:
       - '@swc/core'
       - '@swc/wasm'
-      - '@types/node'
       - '@typescript-eslint/parser'
       - supports-color
       - typescript
@@ -9328,7 +9297,7 @@ packages:
     dependencies:
       '@types/express': 4.17.13
       '@types/heft-jest': 1.0.2
-      '@types/node': 16.7.1
+      '@types/node': 16.7.5
       '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea
       esbuild: 0.12.24
       eslint: 7.32.0
@@ -9395,13 +9364,13 @@ packages:
     name: '@rush-temp/model-all'
     version: 0.0.0
     dependencies:
-      '@types/node': 16.7.1
+      '@types/node': 16.7.5
       '@typescript-eslint/eslint-plugin': 4.28.5_a8e83fcad666e1ba86be4b2e27a20aea
       eslint: 7.32.0
       eslint-plugin-import: 2.23.4_eslint@7.32.0
       eslint-plugin-node: 11.1.0_eslint@7.32.0
       eslint-plugin-promise: 4.3.1
-      ts-node: 10.2.1_f50b86b1778cd2aa4e5405c08bb39559
+      ts-node: 10.2.1_eb14afb1492fcd444e277f1fdb668e87
     transitivePeerDependencies:
       - '@swc/core'
       - '@swc/wasm'
@@ -10167,7 +10136,7 @@ packages:
     dev: false
 
   file:projects/workspace.tgz_6c259fadfeb3a4b20890aefe87070b8b:
-    resolution: {integrity: sha512-xsi1Oqd00ZntGoGAMsxj9vzdN9ff9tEgf//ZmaAH4GqaVT0VF2pzeWwDNlS8cmQR0ECP61Cbu1Jd2Ppyinz8+g==, tarball: file:projects/workspace.tgz}
+    resolution: {integrity: sha512-Mey3vBH1eqNXrxrhsXI3d66AoIW4hqE34OWI7KN7hqY0nqVzUcex0GHBc78dw7vS+UDkW1Cc2J6rCiLvYQkzfw==, tarball: file:projects/workspace.tgz}
     id: file:projects/workspace.tgz
     name: '@rush-temp/workspace'
     version: 0.0.0

deploy/setup-es-attachment-pipeline.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+curl -XPUT "localhost:9200/_ingest/pipeline/attachment?pretty" -H 'Content-Type: application/json' -d'
+{
+  "description" : "Field for processing file attachments",
+  "processors" : [
+    {
+      "attachment" : {
+        "field" : "attachment"
+      }
+    }
+  ]
+}
+'

images/Dockerfile.elastic

@@ -0,0 +1,2 @@
+FROM bitnami/elasticsearch:7.14.0-debian-10-r22
+RUN /opt/bitnami/elasticsearch/bin/elasticsearch-plugin install --batch ingest-attachment

images/build-elastic.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+docker build -t anticrm/elasticsearch:7.14.0-debian-10-r22 -f Dockerfile.elastic .

packages/core/src/tx.ts

@@ -174,9 +174,10 @@ export class TxOperations implements Storage {
   async createDoc<T extends Doc> (
     _class: Ref<Class<T>>,
     space: Ref<Space>,
-    attributes: Data<T>
+    attributes: Data<T>,
+    id?: Ref<T>
   ): Promise<Ref<T>> {
-    const tx = this.txFactory.createTxCreateDoc(_class, space, attributes)
+    const tx = this.txFactory.createTxCreateDoc(_class, space, attributes, id)
     await this.storage.tx(tx)
     return tx.objectId
   }

plugins/recruit-resources/src/components/CreateCandidate.svelte

@@ -49,4 +49,4 @@
   }
 </script>
 
-<DialogHeader />
+<DialogHeader {space}/>

plugins/recruit-resources/src/components/DialogHeader.svelte

@@ -16,8 +16,10 @@
 
 <script lang="ts">
   import { getMetadata } from '@anticrm/platform'
+  import type { Ref, Space, Doc } from '@anticrm/core'
+  import { generateId } from '@anticrm/core'
   import login from '@anticrm/login'
-  import { createQuery } from '@anticrm/presentation'
+  import { createQuery, getClient } from '@anticrm/presentation'
 
   import { EditBox, Button, CircleButton, Grid, Label } from '@anticrm/ui'
   import FileUpload from './icons/FileUpload.svelte'
@@ -27,6 +29,10 @@
 
   import chunter from '@anticrm/chunter'
 
+  import { uploadFile } from '../utils'
+
+  export let space: Ref<Space>
+
   const query = createQuery()
 
   $: query.query(chunter.class.Attachment, {}, result => { console.log('attachments',  result) })  
@@ -34,31 +40,27 @@
   let dragover = false
   let loading = false
 
-  function upload(file: File) {
-    console.log(file)
-    const uploadUrl = getMetadata(login.metadata.UploadUrl)
-    
-    const data = new FormData()
-    data.append('file', file)
-
+  async function createAttachment(file: File) {
     loading = true
-    const url = `${uploadUrl}?collection=resume&name=${file.name}`
-    fetch(url, {
-      method: 'POST',
-      headers: {
-        'Authorization': 'Bearer ' + getMetadata(login.metadata.LoginToken)
-      },
-      body: data
-    })
-    .then(resonse => { console.log(resonse) })
-    .catch(error => { console.log(error) })
-    .finally(() => { loading = false })
+    try {
+      const id = generateId()
+      const uuid = await uploadFile(id, file)
+      console.log('uploaded file uuid', uuid)
+      getClient().createDoc(chunter.class.Attachment, space, {
+        attachmentTo: 'xxxx' as Ref<Doc>,
+        collection: 'resume',
+        name: file.name,
+        file: uuid
+      })
+    } finally {
+      loading = false
+    }
   }
 
   function drop(event: DragEvent) {
     dragover = false
     const droppedFile = event.dataTransfer?.files[0]
-    if (droppedFile !== undefined) { upload(droppedFile) }
+    if (droppedFile !== undefined) { createAttachment(droppedFile) }
   }
 
   let inputFile: HTMLInputElement
@@ -66,7 +68,7 @@
   function fileSelected() {
     console.log(inputFile.files)
     const file = inputFile.files?.[0]
-    if (file !== undefined) { upload(file) }
+    if (file !== undefined) { createAttachment(file) }
   }
 </script>
 

plugins/recruit-resources/src/utils.ts

@@ -0,0 +1,40 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+// 
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// 
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import type { Ref, Doc } from '@anticrm/core'
+import { getMetadata } from '@anticrm/platform'
+
+import login from '@anticrm/login'
+
+export async function uploadFile(id: Ref<Doc>, file: File): Promise<string> {
+  console.log(file)
+  const uploadUrl = getMetadata(login.metadata.UploadUrl)
+  
+  const data = new FormData()
+  data.append('file', file)
+
+  const url = `${uploadUrl}?id=${id}&space=space`
+  const resp = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'Authorization': 'Bearer ' + getMetadata(login.metadata.LoginToken)
+    },
+    body: data
+  })
+  const uuid = await resp.text()
+  console.log(uuid)
+  return uuid
+}

server/core/src/fulltext.ts

@@ -16,7 +16,7 @@
 
 import type { TxCreateDoc, Doc, Ref, Class, Obj, Hierarchy, AnyAttribute } from '@anticrm/core'
 import { TxProcessor, IndexKind } from '@anticrm/core'
-import type { IndexedDoc, FullTextAdapter } from './types'
+import type { IndexedContent, FullTextAdapter } from './types'
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
 const NO_INDEX = {} as AnyAttribute
@@ -52,7 +52,7 @@ export class FullTextIndex extends TxProcessor {
     if (attribute === undefined) return
     const doc = TxProcessor.createDoc2Doc(tx)
     const content = (doc as any)[attribute.name]
-    const indexedDoc: IndexedDoc = {
+    const indexedDoc: IndexedContent = {
       id: doc._id,
       _class: doc._class,
       modifiedBy: doc.modifiedBy,

server/core/src/types.ts

@@ -39,9 +39,22 @@ export interface IndexedDoc {
   space: Ref<Space>
   modifiedOn: Timestamp
   modifiedBy: Ref<Account>
+}
+
+/**
+ * @public
+ */
+export interface IndexedContent extends IndexedDoc {
   content: string
 }
 
+/**
+ * @public
+ */
+export interface IndexedAttachment extends IndexedDoc {
+  attachment: string
+}
+
 /**
  * @public
  */

server/elastic/src/__tests__/adapter.test.ts

@@ -16,18 +16,20 @@
 
 import type { Ref, Doc, Class, Obj, Account, Space } from '@anticrm/core'
 import { createElasticAdapter } from '../adapter'
+import type { IndexedContent } from '@anticrm/server-core'
 
 describe('client', () => {
   it('should create document', async () => {
     const adapter = await createElasticAdapter('http://localhost:9200/', 'ws1')
-    await adapter.index({
+    const doc: IndexedContent = {
       id: 'doc1' as Ref<Doc>,
       _class: 'class1' as Ref<Class<Obj>>,
       modifiedBy: 'andrey' as Ref<Account>,
       modifiedOn: 0,
       space: 'space1' as Ref<Space>,
       content: 'hey there!'
-    })
+    }
+    await adapter.index(doc)
     const hits = await adapter.search({})
     console.log(hits)
   })

server/upload/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#  Copyright © 2020, 2021 Anticrm Platform Contributors.
+#  Copyright © 2021 Hardcore Engineering Inc.
+#  
+#  Licensed under the Eclipse Public License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License. You may
+#  obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+rushx bundle
+rushx docker:build
+rushx docker:push

server/upload/package.json

@@ -21,7 +21,6 @@
     "eslint-plugin-promise":"4",
     "eslint-plugin-node":"11",
     "eslint":"^7.32.0",
-    "@types/ws":"^7.4.7",
     "@types/express":"^4.17.13",
     "ts-node":"^10.2.0",
     "@types/express-fileupload":"^1.1.7",
@@ -34,7 +33,6 @@
     "@anticrm/platform": "~0.6.5",
     "express": "^4.17.1",
     "express-fileupload":"^1.2.1",
-    "aws-sdk":"^2.969.0",
     "uuid":"^8.3.2",
     "cors":"^2.8.5",
     "@anticrm/elastic":"~0.6.0",

server/upload/src/app.ts

@@ -20,11 +20,12 @@ import cors from 'cors'
 import { v4 as uuid } from 'uuid'
 import { decode } from 'jwt-simple'
 
-import type { Space, Ref, Account, Doc } from '@anticrm/core'
-import { TxFactory } from '@anticrm/core'
-import type { Token } from '@anticrm/server-core'
+import type { Space, Ref, Doc, Account } from '@anticrm/core'
+// import { TxFactory } from '@anticrm/core'
+import type { Token, IndexedAttachment } from '@anticrm/server-core'
+import { createElasticAdapter } from '@anticrm/elastic'
 import chunter from '@anticrm/chunter'
-import { createContributingClient } from '@anticrm/contrib'
+// import { createContributingClient } from '@anticrm/contrib'
 
 import { Client } from 'minio'
 
@@ -55,19 +56,19 @@ async function minioUpload (minio: Client, workspace: string, file: UploadedFile
   return id
 }
 
-async function createAttachment (endpoint: string, token: string, account: Ref<Account>, space: Ref<Space>, attachmentTo: Ref<Doc>, collection: string, name: string, file: string): Promise<void> {
-  const txFactory = new TxFactory(account)
-  const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, {
-    attachmentTo,
-    collection,
-    name,
-    file
-  })
-  const url = new URL(`/${token}`, endpoint)
-  const client = await createContributingClient(url.href)
-  await client.tx(tx)
-  client.close()
-}
+// async function createAttachment (endpoint: string, token: string, account: Ref<Account>, space: Ref<Space>, attachmentTo: Ref<Doc>, collection: string, name: string, file: string): Promise<void> {
+//   const txFactory = new TxFactory(account)
+//   const tx = txFactory.createTxCreateDoc(chunter.class.Attachment, space, {
+//     attachmentTo,
+//     collection,
+//     name,
+//     file
+//   })
+//   const url = new URL(`/${token}`, endpoint)
+//   const client = await createContributingClient(url.href)
+//   await client.tx(tx)
+//   client.close()
+// }
 
 /**
  * @public
@@ -81,7 +82,7 @@ export function start (transactorEndpoint: string, elasticUrl: string, minio: Cl
 
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
   app.post('/', async (req, res) => {
-    const file = req.files?.file
+    const file = req.files?.file as UploadedFile
 
     if (file === undefined) {
       res.status(400).send()
@@ -98,26 +99,37 @@ export function start (transactorEndpoint: string, elasticUrl: string, minio: Cl
       const token = authHeader.split(' ')[1]
       const payload = decode(token ?? '', 'secret', false) as Token
       // const fileId = await awsUpload(file as UploadedFile)
-      const fileId = await minioUpload(minio, payload.workspace, file as UploadedFile)
+      const uuid = await minioUpload(minio, payload.workspace, file)
 
+      const id = req.query.id as Ref<Doc>
       const space = req.query.space as Ref<Space>
-      const attachmentTo = req.query.attachmentTo as Ref<Doc>
-      const name = req.query.name as string
-      const collection = req.query.collection as string
-      console.log('name', name)
+      // const name = req.query.name as string
 
-      await createAttachment(
-        transactorEndpoint,
-        token,
-        'core:account:System' as Ref<Account>,
+      // await createAttachment(
+      //   transactorEndpoint,
+      //   token,
+      //   'core:account:System' as Ref<Account>,
+      //   space,
+      //   attachmentTo,
+      //   collection,
+      //   name,
+      //   fileId
+      // )
+
+      const elastic = await createElasticAdapter(elasticUrl, payload.workspace)
+
+      const indexedDoc: IndexedAttachment = {
+        id,
+        _class: chunter.class.Attachment,
         space,
-        attachmentTo,
-        collection,
-        name,
-        fileId
-      )
+        modifiedOn: Date.now(),
+        modifiedBy: 'core:account:System' as Ref<Account>,
+        attachment: file.data.toString('base64')
+      }
 
-      res.status(200).send()
+      await elastic.index(indexedDoc)
+
+      res.status(200).send(uuid)
     } catch (error) {
       console.log(error)
       res.status(500).send()