universo-platform-3d

1
import { storage as firebaseStorage } from 'firebase-admin'
2
import {
3
  File,
4
  GetFilesResponse,
5
  MakeFilePublicResponse,
6
  PredefinedAcl
7
} from '@google-cloud/storage'
8
import { HttpException, Inject, Injectable, forwardRef } from '@nestjs/common'
9
import { FileUploadInterface } from './file-upload.interface'
10
import { FileUploadDto } from './dto/file-upload.dto'
11
import * as fs from 'fs'
12
import { String } from 'lodash'
13
import { DownloadResponse, Storage } from '@google-cloud/storage'
14
import { AssetService, AssetServiceType } from '../../asset/asset.service'
15
import { ASSET_TYPE } from '../../option-sets/asset-type'
16
import { ASSET_MANAGER_UID } from '../../mirror-server-config/asset-manager-uid'
17
import * as path from 'path'
18

19
interface StreamFinishResponse {
20
  data: any // undefined on success - can we remove this?
21
  fileObject: File
22
}
23

24
@Injectable()
25
export class FileUploadService implements FileUploadInterface {
26
  constructor(
27
    @Inject(forwardRef(() => AssetService))
28
    private readonly assetService: AssetServiceType // circular dependency fix. The suffixed -Type type is used to solve circular dependency issue with swc https://github.com/swc-project/swc/issues/5047#issuecomment-1302444311
29
  ) {}
30

31
  private readonly mimeTypeFileMap = {
32
    'image/webp': '.webp',
33
    'image/svg+xml': '.svg',
34
    'image/png': '.png',
35
    'image/jpeg': '.jpg',
36
    'image/gif': '.gif',
37
    'image/bmp': '.bmp',
38
    'image/tiff': '.tiff',
39
    'image/x-exr': '.exr',
40
    'model/gltf-binary': '.glb',
41
    'model/gltf+json': '.gltf',
42
    'model/mibm': '.mibm',
43
    'application/scene-binary': '.pck',
44
    'audio/ogg': '.ogg',
45
    'audio/mpeg': '.mp3',
46
    'audio/wav': '.wav',
47
    'script/gdscript': '.gd',
48
    'script/mirror-visual-script+json': '.vs.json',
49
    'application/json': 'json'
50
  }
51

52
  public async uploadFilePublic({ file, path }: FileUploadDto) {
53
    if (!file) {
54
      throw new HttpException('File upload error: File not provided', 400)
55
    }
56

57
    try {
58
      const pathWithFileType = `${path}${this._getFileTypeEnding(
59
        file.mimetype
60
      )}`
61

62
      if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
63
        await this.streamFile(
64
          process.env.GCS_BUCKET_PUBLIC,
65
          pathWithFileType,
66
          file,
67
          'publicRead'
68
        )
69
        return {
70
          publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${pathWithFileType}`
71
        }
72
      }
73

74
      // If we're using local asset storage, we'll just save the file to the local storage
75
      if (
76
        !process.env.ASSET_STORAGE_DRIVER ||
77
        process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
78
      ) {
79
        console.log('Uploading file to local storage')
80
        await this.uploadFileLocal(file, pathWithFileType)
81
        return {
82
          publicUrl: process.env.ASSET_STORAGE_URL + pathWithFileType
83
        }
84
      }
85
    } catch (error: any) {
86
      const message: string = error?.message
87
      throw new HttpException(`File upload error: ${message}`, 400)
88
    }
89
  }
90

91
  // This is mocked up based on old implementation
92
  // We currently have no definition on how private asset uploads should work
93
  public async uploadFilePrivate({ file, path }: FileUploadDto) {
94
    if (!file) {
95
      throw new HttpException('File upload error: File not provided', 400)
96
    }
97

98
    try {
99
      const pathWithFileType = `${path}${this._getFileTypeEnding(
100
        file.mimetype
101
      )}`
102

103
      if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
104
        await this.streamFile(
105
          process.env.GCS_BUCKET,
106
          pathWithFileType,
107
          file,
108
          'private'
109
        )
110

111
        return { relativePath: pathWithFileType }
112
      }
113

114
      // If we're using local asset storage, we'll just save the file to the local storage
115
      if (
116
        !process.env.ASSET_STORAGE_DRIVER ||
117
        process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
118
      ) {
119
        await this.uploadFileLocal(file, pathWithFileType)
120
        return { relativePath: pathWithFileType }
121
      }
122
    } catch (error: any) {
123
      const message: string = error?.message
124
      throw new HttpException(`File upload error: ${message}`, 400)
125
    }
126
  }
127

128
  public async uploadThumbnail({ file, path }: FileUploadDto) {
129
    if (!file) {
130
      throw new HttpException('File upload error: File not provided', 400)
131
    }
132

133
    try {
134
      const thumbnailPath = path + this._getFileTypeEnding(file.mimetype)
135

136
      // If we're using local asset storage, we'll just save the file to the local storage
137
      if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
138
        await this.uploadFileLocal(file, thumbnailPath)
139
        return {
140
          publicUrl: process.env.ASSET_STORAGE_URL + thumbnailPath
141
        }
142
      }
143

144
      await this.streamFile(
145
        process.env.GCS_BUCKET_PUBLIC,
146
        thumbnailPath,
147
        file,
148
        'publicRead' // we do want the thumbnail to be public by default. We may modify this in the future though to be more sophisticated
149
      )
150
      return {
151
        publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${thumbnailPath}`
152
      }
153
    } catch (error: any) {
154
      const message: string = error?.message
155
      throw new HttpException(`File upload error: ${message}`, 400)
156
    }
157
  }
158

159
  public copyFileInBucket(
160
    bucketName: string,
161
    fromPath: string,
162
    toPath: string
163
  ) {
164
    const storage = firebaseStorage()
165
    const destination = storage.bucket(bucketName).file(toPath)
166
    const options = { predefinedAcl: 'publicRead' }
167
    return storage
168
      .bucket(bucketName)
169
      .file(fromPath)
170
      .copy(destination, options) as Promise<any> // conflicting types issue
171
  }
172

173
  // the file location is this (starting from root of bucket): <userid>/assets/<assetid>/<fileid.[jpg|png|jpeg|gif|etc]>
174
  public async streamFile(
175
    bucketName: string,
176
    relativePath: string,
177
    file: Express.Multer.File,
178
    acl: PredefinedAcl = 'publicRead'
179
  ): Promise<StreamFinishResponse> {
180
    return await this.streamData(
181
      bucketName,
182
      relativePath,
183
      file.mimetype,
184
      file.buffer,
185
      acl
186
    )
187
  }
188

189
  public async streamData(
190
    bucketName: string,
191
    relativePath: string,
192
    mimeType: string,
193
    buffer: Buffer,
194
    acl: PredefinedAcl = 'publicRead'
195
  ): Promise<StreamFinishResponse> {
196
    return await new Promise<StreamFinishResponse>((resolve, reject) => {
197
      const storage = firebaseStorage()
198
      const theRemoteFile = storage
199
        .bucket(bucketName)
200
        .file(relativePath) as unknown as File // conflicting types issue when typed as File (GCS ServiceObject)
201
      const stream = theRemoteFile.createWriteStream({
202
        metadata: {
203
          contentType: mimeType
204
        },
205
        predefinedAcl: acl,
206
        resumable: false
207
      })
208

209
      stream.on('error', (err) => reject(err))
210
      stream.on('finish', (data) =>
211
        resolve({
212
          data: data,
213
          fileObject: theRemoteFile
214
        })
215
      )
216
      stream.end(buffer)
217
    })
218
  }
219

220
  public async getFiles(
221
    bucketName: string,
222
    directoryRelativePath: string
223
  ): Promise<GetFilesResponse> {
224
    const theBucket = firebaseStorage().bucket(bucketName)
225
    // 2022-06-10 00:18:50 v low priority issue, but there's a weird type incompatability between firebase-admin consuming @google-cloud storage but the types being slightly out of sync, so force typing this to be the GCS type here
226
    // the error shows:  Property 'crc32cGenerator' is missing in type 'import("/Users/jared/Documents/GitHub/mirror-server/node_modules/firebase-admin/node_modules/@google-cloud/storage/build/src/file").File' but required in type 'import("/Users/jared/Documents/GitHub/mirror-server/node_modules/@google-cloud/storage/build/src/file").File'.
227
    return (await theBucket.getFiles({
228
      prefix: directoryRelativePath
229
    })) as unknown as GetFilesResponse
230
  }
231

232
  private _getFileTypeEnding(mimeType: string): string {
233
    if (mimeType in this.mimeTypeFileMap) {
234
      return this.mimeTypeFileMap[mimeType]
235
    }
236

237
    const supportedMimes = Object.keys(this.mimeTypeFileMap).join(', ')
238
    const supportedMessage = `Supported MIME types: ${supportedMimes}`
239
    throw new Error(
240
      `MIME type ${mimeType} is not supported. ${supportedMessage}`
241
    )
242
  }
243
  // Inverse of _getFileTypeEnding
244
  private _getMimeTypeFromFileNameEnding(name: string): string {
245
    const fileEnding = '.' + name.split('.').pop()
246
    if (
247
      fileEnding &&
248
      Object.values(this.mimeTypeFileMap).includes(fileEnding)
249
    ) {
250
      return Object.keys(this.mimeTypeFileMap).find(
251
        (key) => this.mimeTypeFileMap[key] === fileEnding
252
      )
253
    }
254

255
    const supportedFileEndings = Object.keys(this.mimeTypeFileMap).join(', ')
256
    const supportedMessage = `Supported file endings: ${supportedFileEndings}`
257
    throw new Error(
258
      `File ending ${fileEnding} is not supported. ${supportedMessage}`
259
    )
260
  }
261

262
  /**
263
   * START Section: Batch file upload  ------------------------------------------------------
264
   */
265
  /**
266
   * @description This should be run as one-off from HTTP request, ideally from a Retool dashboard or something similar
267
   * @date 2023-09-02 17:39
268
   */
269
  async batchAssetUploadFromQueueBucket(
270
    useCloudQueueBucket = true,
271
    inputFolderNameForLocal = 'inputs'
272
  ) {
273
    const QUEUE_BUCKET_NAME = 'the-mirror-asset-queue'
274

275
    // Find files
276

277
    if (useCloudQueueBucket) {
278
      // ensure that we're using remote mongo
279
      if (
280
        !process.env.MONGODB_URL ||
281
        (process.env.MONGODB_URL as string).includes('local')
282
      ) {
283
        throw new Error('Trying to use local mongo for a remote upload')
284
      }
285
      // ensure that this is only on dev
286
      if (!(process.env.MONGODB_URL as string).includes('dev')) {
287
        throw new Error('This script should only be run on dev')
288
      }
289
      const storage = new Storage()
290
      const bucket = storage.bucket(QUEUE_BUCKET_NAME)
291
      const [cloudFiles] = await bucket.getFiles()
292
      const uploadPromises = cloudFiles.map(async (file) => {
293
        const name = file.name
294
        // ensure that it's not the directory
295
        if (!name.includes(QUEUE_BUCKET_NAME)) {
296
          const [contents]: DownloadResponse = await bucket
297
            .file(name)
298
            .download()
299
          return this.runUploadFileForBatchProcess(name, contents)
300
        }
301
      })
302
      await Promise.all(uploadPromises)
303
      console.log('Finished batchAssetUploadFromQueueBucket from cloud bucket')
304
    } else {
305
      const files = fs.readdirSync(inputFolderNameForLocal)
306
      const uploadPromises = files.map((fileName) => {
307
        const inputFolderPath = __dirname + `/${inputFolderNameForLocal}/`
308
        const name = this.capitalizeFirstLetter(
309
          fileName.slice(0, fileName.length - 4)
310
        )
311
        const fileBuffer = fs.readFileSync(inputFolderPath + fileName)
312
        return this.runUploadFileForBatchProcess(name, fileBuffer)
313
      })
314

315
      await Promise.all(uploadPromises)
316
    }
317
  }
318

319
  public async moveAllObjectsFromQueueBucketToQueueCompletedBucket() {
320
    const sourceBucketName = 'the-mirror-asset-queue'
321
    const destinationBucketName = 'the-mirror-asset-queue-completed'
322
    const storage = new Storage()
323
    const sourceBucket = storage.bucket(sourceBucketName)
324
    const [files] = await sourceBucket.getFiles({
325
      delimiter: '/'
326
    })
327

328
    files.forEach(async (file) => {
329
      await file.move(destinationBucketName + '/' + file.name)
330
    })
331
  }
332

333
  private async runUploadFileForBatchProcess(name: string, fileBuffer) {
334
    const asset = await this.assetService.createAsset({
335
      ownerId: ASSET_MANAGER_UID,
336
      name,
337
      assetType: ASSET_TYPE.MESH,
338
      mirrorPublicLibrary: true
339
    })
340
    console.log('Created asset' + name)
341

342
    const { publicUrl: currentFile } =
343
      await this.assetService.uploadAssetFilePublicWithRolesCheck({
344
        assetId: asset._id,
345
        userId: ASSET_MANAGER_UID,
346
        file: {
347
          buffer: fileBuffer,
348
          mimetype: this._getMimeTypeFromFileNameEnding(name)
349
        } as Express.Multer.File
350
      })
351

352
    await this.assetService.updateOneWithRolesCheck(
353
      ASSET_MANAGER_UID,
354
      asset._id,
355
      {
356
        currentFile
357
      }
358
    )
359
    console.log('Uploaded ' + name + ' ' + asset._id)
360
  }
361

362
  private capitalizeFirstLetter(string) {
363
    return string.charAt(0).toUpperCase() + string.slice(1)
364
  }
365
  /**
366
   * END Section: Batch file upload  ------------------------------------------------------
367
   */
368

369
  /**
370
   * START Section: Local driver  ------------------------------------------------------
371
   */
372
  async uploadFileLocal(
373
    file: Express.Multer.File,
374
    pathWithFileType: string
375
  ): Promise<string> {
376
    const directoryPath = this.getLocalStoragePath()
377
    const filePath = path.join(directoryPath, pathWithFileType)
378
    console.log('Uploading file to local storage:', this.getLocalStoragePath())
379
    try {
380
      await fs.promises.mkdir(path.dirname(filePath), { recursive: true }) // Create directory recursively if it doesn't exist
381
      await fs.promises.writeFile(filePath, file.buffer)
382
      console.log('File uploaded to local storage:', filePath)
383
      return filePath
384
    } catch (error) {
385
      console.error('Error uploading file:', error)
386
      throw error
387
    }
388
  }
389

390
  async copyFileLocal(fromPath: string, toPath: string): Promise<string> {
391
    if (!fs.existsSync(fromPath)) {
392
      throw new Error(`File does not exist at path: ${fromPath}`)
393
    }
394
    const directoryPath = this.getLocalStoragePath()
395
    const toFilePath = path.join(directoryPath, toPath)
396
    await fs.promises.mkdir(path.dirname(toFilePath), { recursive: true }) // Create directory recursively if it doesn't exist
397
    await fs.promises.copyFile(fromPath, toFilePath)
398
    return toFilePath
399
  }
400

401
  getLocalStoragePath(): string {
402
    const localStorage = 'localStorage' // Define the relative path to the local storage directory
403
    return path.join(__dirname, '..', '..', '..', localStorage)
404
  }
405
  /**
406
   * END Section: Local driver  ------------------------------------------------------
407
   */
408
}
409