Flowise

1
import { DocumentStore } from './database/entities/DocumentStore'
2

3
export enum DocumentStoreStatus {
4
    EMPTY_SYNC = 'EMPTY',
5
    SYNC = 'SYNC',
6
    SYNCING = 'SYNCING',
7
    STALE = 'STALE',
8
    NEW = 'NEW'
9
}
10

11
export interface IDocumentStore {
12
    id: string
13
    name: string
14
    description: string
15
    loaders: string // JSON string
16
    whereUsed: string // JSON string
17
    updatedDate: Date
18
    createdDate: Date
19
    status: DocumentStoreStatus
20
}
21

22
export interface IDocumentStoreFileChunk {
23
    id: string
24
    chunkNo: number
25
    docId: string
26
    storeId: string
27
    pageContent: string
28
    metadata: string
29
}
30

31
export interface IDocumentStoreFileChunkPagedResponse {
32
    chunks: IDocumentStoreFileChunk[]
33
    count: number
34
    file?: IDocumentStoreLoader
35
    currentPage: number
36
    storeName: string
37
    description: string
38
}
39

40
export interface IDocumentStoreLoader {
41
    id: string
42
    loaderId: string
43
    loaderName: string
44
    loaderConfig: any // JSON string
45
    splitterId: string
46
    splitterName: string
47
    splitterConfig: any // JSON string
48
    totalChunks: number
49
    totalChars: number
50
    status: DocumentStoreStatus
51
    storeId?: string
52
    files?: IDocumentStoreLoaderFile[]
53
    source?: string
54
    credential?: string
55
}
56

57
export interface IDocumentStoreLoaderForPreview extends IDocumentStoreLoader {
58
    rehydrated: boolean
59
    preview: boolean
60
    previewChunkCount: number
61
}
62

63
export interface IDocumentStoreLoaderFile {
64
    id: string
65
    name: string
66
    mimePrefix: string
67
    size: number
68
    status: DocumentStoreStatus
69
    uploaded: Date
70
}
71

72
export interface IDocumentStoreWhereUsed {
73
    id: string
74
    name: string
75
}
76

77
export class DocumentStoreDTO {
78
    id: string
79
    name: string
80
    description: string
81
    files: IDocumentStoreLoaderFile[]
82
    whereUsed: IDocumentStoreWhereUsed[]
83
    createdDate: Date
84
    updatedDate: Date
85
    status: DocumentStoreStatus
86
    chunkOverlap: number
87
    splitter: string
88
    totalChunks: number
89
    totalChars: number
90
    chunkSize: number
91
    loaders: IDocumentStoreLoader[]
92

93
    constructor() {}
94

95
    static fromEntity(entity: DocumentStore): DocumentStoreDTO {
96
        let documentStoreDTO = new DocumentStoreDTO()
97

98
        Object.assign(documentStoreDTO, entity)
99
        documentStoreDTO.id = entity.id
100
        documentStoreDTO.name = entity.name
101
        documentStoreDTO.description = entity.description
102
        documentStoreDTO.status = entity.status
103
        documentStoreDTO.totalChars = 0
104
        documentStoreDTO.totalChunks = 0
105

106
        if (entity.whereUsed) {
107
            documentStoreDTO.whereUsed = JSON.parse(entity.whereUsed)
108
        } else {
109
            documentStoreDTO.whereUsed = []
110
        }
111

112
        if (entity.loaders) {
113
            documentStoreDTO.loaders = JSON.parse(entity.loaders)
114
            documentStoreDTO.loaders.map((loader) => {
115
                documentStoreDTO.totalChars += loader.totalChars
116
                documentStoreDTO.totalChunks += loader.totalChunks
117
                switch (loader.loaderId) {
118
                    case 'pdfFile':
119
                        loader.source = loader.loaderConfig.pdfFile.replace('FILE-STORAGE::', '')
120
                        break
121
                    case 'apiLoader':
122
                        loader.source = loader.loaderConfig.url + ' (' + loader.loaderConfig.method + ')'
123
                        break
124
                    case 'cheerioWebScraper':
125
                        loader.source = loader.loaderConfig.url
126
                        break
127
                    case 'jsonFile':
128
                        loader.source = loader.loaderConfig.jsonFile.replace('FILE-STORAGE::', '')
129
                        break
130
                    case 'docxFile':
131
                        loader.source = loader.loaderConfig.docxFile.replace('FILE-STORAGE::', '')
132
                        break
133
                    case 'textFile':
134
                        loader.source = loader.loaderConfig.txtFile.replace('FILE-STORAGE::', '')
135
                        break
136
                    case 'unstructuredFileLoader':
137
                        loader.source = loader.loaderConfig.filePath
138
                        break
139
                    default:
140
                        loader.source = 'None'
141
                        break
142
                }
143
                if (loader.status !== 'SYNC') {
144
                    documentStoreDTO.status = DocumentStoreStatus.STALE
145
                }
146
            })
147
        }
148

149
        return documentStoreDTO
150
    }
151

152
    static fromEntities(entities: DocumentStore[]): DocumentStoreDTO[] {
153
        return entities.map((entity) => this.fromEntity(entity))
154
    }
155

156
    static toEntity(body: any): DocumentStore {
157
        const docStore = new DocumentStore()
158
        Object.assign(docStore, body)
159
        docStore.loaders = '[]'
160
        docStore.whereUsed = '[]'
161
        // when a new document store is created, it is empty and in sync
162
        docStore.status = DocumentStoreStatus.EMPTY_SYNC
163
        return docStore
164
    }
165
}
166