hcengineering/platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2024-12-22 19:11:33 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: remove prefixes from listStream method parameters (#6480)

Browse Source 
Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov 2024-09-04 08:47:56 +07:00 committed by GitHub
parent 3be2de033a
commit 60444ebdab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 15 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
models/core/src/migration.ts
View File

@ -337,7 +337,7 @@ async function migrateBlobData (exAdapter: StorageAdapterEx, client: MigrationCl
if (!(await adapter.exists(ctx, client.workspaceId))) {
continue
}
const blobs = await adapter.listStream(ctx, client.workspaceId, '')
const blobs = await adapter.listStream(ctx, client.workspaceId)
const bulk = new Map<Ref<Blob>, Blob>()
try {
const push = async (force: boolean): Promise<void> => {

15
packages/storage/src/index.ts
View File

@ -45,7 +45,7 @@ export interface StorageAdapter {
listBuckets: (ctx: MeasureContext) => Promise<BucketInfo[]>
remove: (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]) => Promise<void>
listStream: (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string) => Promise<BlobStorageIterator>
listStream: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise<BlobStorageIterator>
stat: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise<Blob | undefined>
get: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise<Readable>
put: (
@ -114,15 +114,11 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx {
async remove (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]): Promise<void> {}
async list (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string | undefined): Promise<ListBlobResult[]> {
async list (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<ListBlobResult[]> {
return []
}
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
return {
next: async (): Promise<ListBlobResult | undefined> => {
return undefined
@ -203,11 +199,10 @@ export async function removeAllObjects (
export async function objectsToArray (
ctx: MeasureContext,
storage: StorageAdapter,
workspaceId: WorkspaceId,
prefix?: string
workspaceId: WorkspaceId
): Promise<ListBlobResult[]> {
// We need to list all files and delete them
const iterator = await storage.listStream(ctx, workspaceId, prefix)
const iterator = await storage.listStream(ctx, workspaceId)
const bulk: ListBlobResult[] = []
while (true) {
const obj = await iterator.next()

6
server/core/src/__tests__/memAdapters.ts
View File

@ -52,11 +52,7 @@ export class MemStorageAdapter implements StorageAdapter {
}
}
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
const files = Array.from(this.files.values()).filter((it) => it.workspace === workspaceId.name)
return {
next: async () => {

10
server/core/src/server/aggregator.ts
View File

@ -224,14 +224,8 @@ export class AggregatorStorageAdapter implements StorageAdapter, StorageAdapterE
await this.dbAdapter.clean(ctx, workspaceId, DOMAIN_BLOB, objectNames as Ref<Blob>[])
}
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
const data = await this.dbAdapter.findStream<Blob>(ctx, workspaceId, DOMAIN_BLOB, {
_id: { $regex: `${prefix ?? ''}.*` }
})
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
const data = await this.dbAdapter.findStream<Blob>(ctx, workspaceId, DOMAIN_BLOB, {})
return {
next: async (): Promise<ListBlobResult | undefined> => {
return await data.next()

6
server/datalake/src/index.ts
View File

@ -73,11 +73,7 @@ export class DatalakeService implements StorageAdapter {
}
@withContext('listStream')
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
throw new Error('not supported')
}

8
server/minio/src/index.ts
View File

@ -182,11 +182,7 @@ export class MinioService implements StorageAdapter {
}
@withContext('listStream')
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
let hasMore = true
let stream: BucketStream<BucketItem> | undefined
let done = false
@ -199,7 +195,7 @@ export class MinioService implements StorageAdapter {
next: async (): Promise<ListBlobResult | undefined> => {
try {
if (stream === undefined && !done) {
const rprefix = rootPrefix !== undefined ? rootPrefix + (prefix ?? '') : prefix ?? ''
const rprefix = rootPrefix ?? ''
stream = this.client.listObjects(this.getBucketId(workspaceId), rprefix, true)
stream.on('end', () => {
stream?.destroy()

10
server/s3/src/index.ts
View File

@ -232,11 +232,7 @@ export class S3Service implements StorageAdapter {
}
@withContext('listStream')
async listStream (
ctx: MeasureContext,
workspaceId: WorkspaceId,
prefix?: string | undefined
): Promise<BlobStorageIterator> {
async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<BlobStorageIterator> {
let hasMore = true
const buffer: ListBlobResult[] = []
let token: string | undefined
@ -248,7 +244,7 @@ export class S3Service implements StorageAdapter {
if (hasMore && buffer.length === 0) {
const res = await this.client.listObjectsV2({
Bucket: this.getBucketId(workspaceId),
Prefix: rootPrefix !== undefined ? rootPrefix + (prefix ?? '') : prefix ?? '',
Prefix: rootPrefix ?? '',
ContinuationToken: token
})
if (res.IsTruncated === true) {
@ -273,7 +269,7 @@ export class S3Service implements StorageAdapter {
}
}
} catch (err: any) {
ctx.error('Failed to get list', { error: err, workspaceId: workspaceId.name, prefix })
ctx.error('Failed to get list', { error: err, workspaceId: workspaceId.name })
}
if (buffer.length > 0) {
return buffer.shift()