idlize

1
/*
2
 * Copyright (c) 2024 Huawei Device Co., Ltd.
3
 * Licensed under the Apache License, Version 2.0 (the "License");
4
 * you may not use this file except in compliance with the License.
5
 * You may obtain a copy of the License at
6
 *
7
 * http://www.apache.org/licenses/LICENSE-2.0
8
 *
9
 * Unless required by applicable law or agreed to in writing, software
10
 * distributed under the License is distributed on an "AS IS" BASIS,
11
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
 * See the License for the specific language governing permissions and
13
 * limitations under the License.
14
 */
15

16
import { program } from "commander"
17
import * as fs from "fs"
18
import * as path from "path"
19
import { fromIDL, scanIDL } from "./from-idl/common"
20
import { idlToString } from "./from-idl/DtsPrinter"
21
import { generate } from "./idlize"
22
import { IDLEntry, forEachChild, toIDLString } from "./idl"
23
import { printHeader, toHeaderString, wrapWithPrologueAndEpilogue } from "./idl2h"
24
import { LinterMessage, LinterVisitor, toLinterString } from "./linter"
25
import { CompileContext, IDLVisitor } from "./IDLVisitor"
26
import { TestGeneratorVisitor } from "./TestGeneratorVisitor"
27
import {
28
    bridgeCcDeclaration,
29
    copyPeerLib,
30
    dummyImplementations,
31
    dummyModifierList,
32
    dummyModifiers,
33
    makeApiHeaders,
34
    makeApiModifiers,
35
    makeCDeserializer,
36
    makeTSSerializer,
37
    nativeModuleDeclaration,
38
    nativeModuleEmptyDeclaration
39
} from "./peer-generation/FileGenerators"
40
import {
41
    PeerGeneratorVisitor
42
} from "./peer-generation/PeerGeneratorVisitor"
43
import { defaultCompilerOptions, isDefined, renameDtsToPeer, stringOrNone, toSet } from "./util"
44
import { TypeChecker  } from "./typecheck"
45
import { SortingEmitter } from "./peer-generation/SortingEmitter"
46
import { initRNG } from "./rand_utils"
47

48
const options = program
49
    .option('--dts2idl', 'Convert .d.ts to IDL definitions')
50
    .option('--dts2h', 'Convert .d.ts to .h definitions')
51
    .option('--dts2test', 'Generate tests from .d.ts to .h')
52
    .option('--dts2peer', 'Convert .d.ts to peer drafts')
53
    .option('--ets2ts', 'Convert .ets to .ts')
54
    .option('--input-dir <path>', 'Path to input dir')
55
    .option('--output-dir <path>', 'Path to output dir')
56
    .option('--input-file <name>', 'Name of file to convert, all files in input-dir if none')
57
    .option('--idl2dts', 'Convert IDL to .d.ts definitions')
58
    .option('--idl2h', 'Convert IDL to .h definitions')
59
    .option('--linter', 'Run linter')
60
    .option('--linter-suppress-errors <suppress>', 'Error codes to suppress, comma separated, no space')
61
    .option('--linter-whitelist <whitelist.json>', 'Whitelist for linter')
62
    .option('--verbose', 'Verbose processing')
63
    .option('--verify-idl', 'Verify produced IDL')
64
    .option('--common-to-attributes', 'Transform common attributes as IDL attributes')
65
    .option('--test-interface <name>', 'Interfaces to test (comma separated)')
66
    .option('--test-method <name>', 'Methods to test (comma separated)')
67
    .option('--test-property <name>', 'Properties to test (comma separated)')
68
    .option('--generate-interface <name>', 'Interfaces to generate (comma separated)')
69
    .option('--disable-enum-initializers', "Don't include enum member initializers in the interface")
70
    .option('--native-bridge-path <name>', "Path to native bridge")
71
    .option('--dump-serialized', "Dump serialized data")
72
    .option('--docs [all|opt|none]', 'How to handle documentation: include, optimize, or skip')
73
    .option('--version')
74
    .parse()
75
    .opts()
76

77
function findVersion() {
78
    if (process.env.npm_package_version) return process.env.npm_package_version
79
    let packageJson = path.join(__dirname, '..', 'package.json')
80
    try {
81
        let json = fs.readFileSync(packageJson).toString()
82
        return json ? JSON.parse(json).version : undefined
83
    } catch(e) {
84
        return undefined
85
    }
86
}
87

88
if (process.env.npm_package_version) {
89
    console.log(`IDLize version ${findVersion()}`)
90
}
91

92
let didJob = false
93

94
if (options.dts2idl) {
95
    const tsCompileContext = new CompileContext()
96
    generate(
97
        options.inputDir,
98
        options.inputFile,
99
        options.outputDir ?? "./idl",
100
        (sourceFile, typeChecker) => new IDLVisitor(sourceFile, typeChecker, tsCompileContext, options),
101
        {
102
            compilerOptions: defaultCompilerOptions,
103
            onSingleFile: (entries: IDLEntry[], outputDir, sourceFile) => {
104
                const outFile = path.join(outputDir,
105
                    path.basename(sourceFile.fileName).replace(".d.ts", ".idl"))
106
                console.log("producing", outFile)
107
                if (options.skipDocs) {
108
                    entries.forEach(it => forEachChild(
109
                        it, (it) => it.documentation = undefined))
110
                }
111
                let generated = toIDLString(entries, {
112
                    verifyIdl: options.verifyIdl ?? false,
113
                    disableEnumInitializers: options.disableEnumInitializers ?? false
114
                })
115
                if (options.verbose) console.log(generated)
116
                fs.writeFileSync(outFile, generated)
117
            }
118
        }
119
    )
120
    didJob = true
121
}
122

123
if (options.dts2h) {
124
    const allEntries = new Array<IDLEntry[]>()
125
    const tsCompileContext = new CompileContext()
126
    generate(
127
        options.inputDir,
128
        options.inputFile,
129
        options.outputDir ?? "./headers",
130
        (sourceFile, typeChecker) => new IDLVisitor(sourceFile, typeChecker, tsCompileContext, options.commonToAttributes ?? false),
131
        {
132
            compilerOptions: defaultCompilerOptions,
133
            onSingleFile: (entries: IDLEntry[]) => allEntries.push(entries),
134
        }
135
    )
136
    const outFile = path.join(options.outputDir ?? "./headers", "arkoala_api.h")
137
    console.log("producing", outFile)
138

139
    const generated = toHeaderString(new TypeChecker(allEntries.flat()), allEntries, options.generateInterface)
140
    if (options.verbose) console.log(generated)
141
    fs.writeFileSync(outFile, generated)
142
    didJob = true
143
}
144

145
if (options.linter) {
146
    const allEntries = new Array<LinterMessage[]>()
147
    generate(
148
        options.inputDir,
149
        options.inputFile,
150
        options.outputDir,
151
        (sourceFile, typeChecker) => new LinterVisitor(sourceFile, typeChecker),
152
        {
153
            compilerOptions: defaultCompilerOptions,
154
            onSingleFile: (entries: LinterMessage[]) => allEntries.push(entries),
155
            onBegin: () => {},
156
            onEnd: (outputDir) => {
157
                const outFile = options.outputDir ? path.join(outputDir, "linter.txt") : undefined
158
                let [generated, exitCode, histogram] = toLinterString(allEntries, options.linterSuppressErrors, options.linterWhitelist)
159
                console.log(histogram)
160
                if (!outFile || options.verbose) console.log(generated)
161
                if (outFile) fs.writeFileSync(outFile, generated)
162
                process.exit(exitCode)
163
            }
164
        }
165
    )
166
    didJob = true
167
}
168

169
if (options.dts2test) {
170
    initRNG()
171
    let testInterfaces = options.testInterface
172
    if (testInterfaces === undefined) {
173
        function fileNameToClass(name: string): string {
174
            return name
175
                .split('_')
176
                .map(s => s.charAt(0).toUpperCase() + s.slice(1))
177
                .join(``)
178
        }
179

180
        let inDir = path.resolve(options.inputDir)
181
        testInterfaces = fs.readdirSync(inDir)
182
            .filter(file => file.endsWith("d.ts"))
183
            .map(file => file.substring(0, file.length - 5))
184
            .map(fileNameToClass)
185
            .join(',')
186
    }
187

188
    let lines: string[] = []
189
    generate(
190
        options.inputDir,
191
        options.inputFile,
192
        options.outputDir ?? "./tests",
193
        (sourceFile, typeChecker) => new TestGeneratorVisitor(sourceFile, typeChecker, testInterfaces, options.testMethod, options.testProperties),
194
        {
195
            compilerOptions: defaultCompilerOptions,
196
            onBegin: (outDir: string) => {
197
            },
198
            onSingleFile: (entries: string[], outputDir, sourceFile) => {
199
                lines = lines.concat(entries)
200
            },
201
            onEnd: (outDir: string) => {
202
                let generated = lines.join("\n")
203
                const outFile = path.join(outDir, "index.ts")
204
                if (options.verbose) {
205
                    console.log(generated)
206
                }
207
                console.log(`Write fuzzing peers to file ${outFile}`)
208
                fs.writeFileSync(outFile, lines.join("\n"))
209
            }
210
        }
211
    )
212
    didJob = true
213
}
214

215
if (options.idl2dts) {
216
    fromIDL(
217
        options.inputDir,
218
        options.inputFile,
219
        options.outputDir ?? "./dts/",
220
        ".d.ts",
221
        options.verbose ?? false,
222
        idlToString,
223
    )
224
    didJob = true
225
}
226

227
if (options.idl2h) {
228
    const idlFiles = scanIDL(
229
        options.inputDir,
230
        options.inputFile
231
    )
232
    const typeChecker = new TypeChecker(idlFiles.flat())
233
    const body = idlFiles
234
        .flatMap(it => printHeader(typeChecker, it, toSet(options.generateInterface)))
235
        .filter(isDefined)
236
        .filter(it => it.length > 0)
237
        .join("\n")
238
    const generatedHeader = wrapWithPrologueAndEpilogue(body)
239
    if (options.verbose) {
240
        console.log(body)
241
    }
242
    const outputDir = options.outputDir ?? "./headers"
243
    if (!fs.existsSync(outputDir)) {
244
        fs.mkdirSync(outputDir, { recursive: true })
245
    }
246
    const outFile = path.join(outputDir, "arkoala_api.h")
247
    console.log("producing", outFile)
248
    fs.writeFileSync(outFile, generatedHeader)
249
    didJob = true
250
}
251

252
if (options.dts2peer) {
253
    const nativeMethods: string[] = []
254
    const nativeEmptyMethods: string[] = []
255
    const bridgeCcArray: string[] = []
256
    const deserializerC: string[] = []
257
    const structsC = new SortingEmitter()
258
    const structsForwardC: string[] = []
259
    const serializerTS: string[] = []
260
    const apiHeaders: string[] = []
261
    const apiHeadersList: string[] = []
262
    const dummyImpl: string[] = []
263
    const dummyImplModifiers: string[] = []
264
    const dummyImplModifierList: string[] = []
265

266
    generate(
267
        options.inputDir,
268
        undefined,
269
        options.outputDir ?? "./peers",
270
        (sourceFile, typeChecker) => new PeerGeneratorVisitor({
271
            sourceFile: sourceFile,
272
            typeChecker: typeChecker,
273
            interfacesToGenerate: toSet(options.generateInterface),
274
            nativeModuleMethods: nativeMethods,
275
            nativeModuleEmptyMethods: nativeEmptyMethods,
276
            outputC: bridgeCcArray,
277
            outputSerializersTS: serializerTS,
278
            outputSerializersC: deserializerC,
279
            outputStructsForwardC: structsForwardC,
280
            outputStructsC: structsC,
281
            apiHeaders: apiHeaders,
282
            apiHeadersList: apiHeadersList,
283
            dummyImpl: dummyImpl,
284
            dummyImplModifiers: dummyImplModifiers,
285
            dummyImplModifierList: dummyImplModifierList,
286
            dumpSerialized: options.dumpSerialized ?? false,
287
        }),
288
        {
289
            compilerOptions: defaultCompilerOptions,
290
            onSingleFile: (entries: stringOrNone[], outputDir, sourceFile) => {
291
                const outFile = path.join(
292
                    outputDir,
293
                    renameDtsToPeer(path.basename(sourceFile.fileName))
294
                )
295
                if (entries.length > 0) {
296
                    console.log("producing", outFile)
297
                    let generated = entries
298
                        .filter(element => (element?.length ?? 0) > 0)
299
                        .join("\n")
300
                    if (options.verbose) console.log(generated)
301
                    fs.writeFileSync(outFile, generated)
302
                }
303
            },
304
            onEnd(outDir: string) {
305
                fs.writeFileSync(
306
                    path.join(outDir, 'NativeModule.ts'),
307
                    nativeModuleDeclaration(nativeMethods, options.nativeBridgeDir ?? "../../../../native/NativeBridge", false)
308
                )
309
                fs.writeFileSync(
310
                    path.join(outDir, 'NativeModuleEmpty.ts'),
311
                    nativeModuleEmptyDeclaration(nativeEmptyMethods)
312
                )
313
                const bridgeCc = bridgeCcDeclaration(bridgeCcArray)
314
                fs.writeFileSync(path.join(outDir, 'bridge.cc'), bridgeCc)
315
                fs.writeFileSync(path.join(outDir, 'Serializer.ts'), makeTSSerializer(serializerTS))
316
                fs.writeFileSync(path.join(outDir, 'Deserializer.h'), makeCDeserializer(structsForwardC, structsC.getOutput(), deserializerC))
317
                fs.writeFileSync(path.join(outDir, 'arkoala_api.h'), makeApiHeaders(apiHeaders) + makeApiModifiers(apiHeadersList))
318
                const dummyImplCc =
319
                    dummyImplementations(dummyImpl) +
320
                    dummyModifiers(dummyImplModifiers) +
321
                    dummyModifierList(dummyImplModifierList)
322
                fs.writeFileSync(path.join(outDir, 'dummy_impl.cc'), dummyImplCc)
323
                copyPeerLib(path.join(__dirname, '..', 'peer_lib'), outDir)
324
            }
325
        }
326
    )
327
    didJob = true
328
}
329

330
if (!didJob) {
331
    program.help()
332
}
333