githubmirror/webpack

lib/serialization/FileMiddleware.js
813 строк24 KB
Sebastian Beltran
feat: add support for zstd compression in cache options (#21438)
21 июл 2026, 20:45
21 июл 2026, 20:45508e10f
/*
	MIT License http://www.opensource.org/licenses/mit-license.php
*/

"use strict";

const { constants } = require("buffer");
const { pipeline } = require("stream");
const {
	constants: zConstants,
	// eslint-disable-next-line n/no-unsupported-features/node-builtins
	createBrotliCompress,
	// eslint-disable-next-line n/no-unsupported-features/node-builtins
	createBrotliDecompress,
	createGunzip,
	createGzip,
	// zstd is only available on Node.js >= 22.15; guarded at use sites
	// eslint-disable-next-line n/no-unsupported-features/node-builtins
	createZstdCompress,
	// eslint-disable-next-line n/no-unsupported-features/node-builtins
	createZstdDecompress
} = require("zlib");
const { DEFAULTS } = require("../config/defaults");
const createHash = require("../util/createHash");
const { dirname, join, mkdirp } = require("../util/fs");
const memoize = require("../util/memoize");
const SerializerMiddleware = require("./SerializerMiddleware");

/** @typedef {import("../util/Hash").HashFunction} HashFunction */
/** @typedef {import("../util/fs").IStats} IStats */
/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
/** @typedef {import("./types").BufferSerializableType} BufferSerializableType */

/*
Format:

File -> Header Section*

Version -> u32
AmountOfSections -> u32
SectionSize -> i32 (if less than zero represents lazy value)

Header -> Version AmountOfSections SectionSize*

Buffer -> n bytes
Section -> Buffer

*/

// "wpc" + 1 in little-endian
const VERSION = 0x01637077;
const WRITE_LIMIT_TOTAL = 0x7fff0000;
const WRITE_LIMIT_CHUNK = 511 * 1024 * 1024;

/**
 * Returns hash.
 * @param {Buffer[]} buffers buffers
 * @param {HashFunction} hashFunction hash function to use
 * @returns {string} hash
 */
const hashForName = (buffers, hashFunction) => {
	const hash = createHash(hashFunction);
	for (const buf of buffers) hash.update(buf);
	return hash.digest("hex");
};

const COMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
const DECOMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;

/** @type {(buffer: Buffer, value: number, offset: number) => void} */
const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
	? (buf, value, offset) => {
			buf.writeBigUInt64LE(BigInt(value), offset);
		}
	: (buf, value, offset) => {
			const low = value % 0x100000000;
			const high = (value - low) / 0x100000000;
			buf.writeUInt32LE(low, offset);
			buf.writeUInt32LE(high, offset + 4);
		};

/** @type {(buffer: Buffer, offset: number) => void} */
const readUInt64LE = Buffer.prototype.readBigUInt64LE
	? (buf, offset) => Number(buf.readBigUInt64LE(offset))
	: (buf, offset) => {
			const low = buf.readUInt32LE(offset);
			const high = buf.readUInt32LE(offset + 4);
			return high * 0x100000000 + low;
		};

/** @typedef {Promise<void | void[]>} BackgroundJob */

/**
 * Defines the serialize result type used by this module.
 * @typedef {object} SerializeResult
 * @property {string | false} name
 * @property {number} size
 * @property {BackgroundJob=} backgroundJob
 */

/** @typedef {{ name: string, size: number }} LazyOptions */
/**
 * Defines the lazy function type used by this module.
 * @typedef {import("./SerializerMiddleware").LazyFunction<BufferSerializableType[], Buffer, FileMiddleware, LazyOptions>} LazyFunction
 */

/**
 * Serializes this instance into the provided serializer context.
 * @param {FileMiddleware} middleware this
 * @param {(BufferSerializableType | LazyFunction)[]} data data to be serialized
 * @param {string | boolean} name file base name
 * @param {(name: string | false, buffers: Buffer[], size: number) => Promise<void>} writeFile writes a file
 * @param {HashFunction=} hashFunction hash function to use
 * @returns {Promise<SerializeResult>} resulting file pointer and promise
 */
const serialize = async (
	middleware,
	data,
	name,
	writeFile,
	hashFunction = DEFAULTS.HASH_FUNCTION
) => {
	/** @type {(Buffer[] | Buffer | Promise<SerializeResult>)[]} */
	const processedData = [];
	/** @type {WeakMap<SerializeResult, LazyFunction>} */
	const resultToLazy = new WeakMap();
	/** @type {Buffer[] | undefined} */
	let lastBuffers;
	for (const item of await data) {
		if (typeof item === "function") {
			if (!SerializerMiddleware.isLazy(item)) {
				throw new Error("Unexpected function");
			}
			if (!SerializerMiddleware.isLazy(item, middleware)) {
				throw new Error(
					"Unexpected lazy value with non-this target (can't pass through lazy values)"
				);
			}
			lastBuffers = undefined;
			const serializedInfo = SerializerMiddleware.getLazySerializedValue(item);
			if (serializedInfo) {
				if (typeof serializedInfo === "function") {
					throw new Error(
						"Unexpected lazy value with non-this target (can't pass through lazy values)"
					);
				} else {
					processedData.push(serializedInfo);
				}
			} else {
				const content = item();
				if (content) {
					const options = SerializerMiddleware.getLazyOptions(item);
					processedData.push(
						serialize(
							middleware,
							/** @type {BufferSerializableType[]} */
							(content),
							(options && options.name) || true,
							writeFile,
							hashFunction
						).then((result) => {
							/** @type {LazyOptions} */
							(item.options).size = result.size;
							resultToLazy.set(result, item);
							return result;
						})
					);
				} else {
					throw new Error(
						"Unexpected falsy value returned by lazy value function"
					);
				}
			}
		} else if (item) {
			if (lastBuffers) {
				lastBuffers.push(item);
			} else {
				lastBuffers = [item];
				processedData.push(lastBuffers);
			}
		} else {
			throw new Error("Unexpected falsy value in items array");
		}
	}
	/** @type {BackgroundJob[]} */
	const backgroundJobs = [];
	const resolvedData = (await Promise.all(processedData)).map((item) => {
		if (Array.isArray(item) || Buffer.isBuffer(item)) return item;

		backgroundJobs.push(
			/** @type {BackgroundJob} */
			(item.backgroundJob)
		);
		// create pointer buffer from size and name
		const name = /** @type {string} */ (item.name);
		const nameBuffer = Buffer.from(name);
		const buf = Buffer.allocUnsafe(8 + nameBuffer.length);
		writeUInt64LE(buf, item.size, 0);
		nameBuffer.copy(buf, 8, 0);
		const lazy =
			/** @type {LazyFunction} */
			(resultToLazy.get(item));
		SerializerMiddleware.setLazySerializedValue(lazy, buf);
		return buf;
	});
	/** @type {number[]} */
	const lengths = [];
	for (const item of resolvedData) {
		if (Array.isArray(item)) {
			let l = 0;
			for (const b of item) l += b.length;
			while (l > 0x7fffffff) {
				lengths.push(0x7fffffff);
				l -= 0x7fffffff;
			}
			lengths.push(l);
		} else if (item) {
			lengths.push(-item.length);
		} else {
			throw new Error(`Unexpected falsy value in resolved data ${item}`);
		}
	}
	const header = Buffer.allocUnsafe(8 + lengths.length * 4);
	header.writeUInt32LE(VERSION, 0);
	header.writeUInt32LE(lengths.length, 4);
	for (let i = 0; i < lengths.length; i++) {
		header.writeInt32LE(lengths[i], 8 + i * 4);
	}
	/** @type {Buffer[]} */
	const buf = [header];
	for (const item of resolvedData) {
		if (Array.isArray(item)) {
			for (const b of item) buf.push(b);
		} else if (item) {
			buf.push(item);
		}
	}
	if (name === true) {
		name = hashForName(buf, hashFunction);
	}
	let size = 0;
	for (const b of buf) size += b.length;
	backgroundJobs.push(writeFile(name, buf, size));
	return {
		size,
		name,
		backgroundJob:
			backgroundJobs.length === 1
				? backgroundJobs[0]
				: /** @type {BackgroundJob} */ (Promise.all(backgroundJobs))
	};
};

/**
 * Restores this instance from the provided deserializer context.
 * @param {FileMiddleware} middleware this
 * @param {string | false} name filename
 * @param {(name: string | false) => Promise<Buffer[]>} readFile read content of a file
 * @returns {Promise<BufferSerializableType[]>} deserialized data
 */
const deserialize = async (middleware, name, readFile) => {
	const contents = await readFile(name);
	if (contents.length === 0) throw new Error(`Empty file ${name}`);
	let contentsIndex = 0;
	let contentItem = contents[0];
	let contentItemLength = contentItem.length;
	let contentPosition = 0;
	if (contentItemLength === 0) throw new Error(`Empty file ${name}`);
	const nextContent = () => {
		contentsIndex++;
		contentItem = contents[contentsIndex];
		contentItemLength = contentItem.length;
		contentPosition = 0;
	};
	/**
	 * Processes the provided n.
	 * @param {number} n number of bytes to ensure
	 */
	const ensureData = (n) => {
		if (contentPosition === contentItemLength) {
			nextContent();
		}
		while (contentItemLength - contentPosition < n) {
			const remaining = contentItem.subarray(contentPosition);
			let lengthFromNext = n - remaining.length;
			/** @type {Buffer[]} */
			const buffers = [remaining];
			for (let i = contentsIndex + 1; i < contents.length; i++) {
				const l = contents[i].length;
				if (l > lengthFromNext) {
					buffers.push(contents[i].subarray(0, lengthFromNext));
					contents[i] = contents[i].subarray(lengthFromNext);
					lengthFromNext = 0;
					break;
				} else {
					buffers.push(contents[i]);
					contentsIndex = i;
					lengthFromNext -= l;
				}
			}
			if (lengthFromNext > 0) throw new Error("Unexpected end of data");
			contentItem = Buffer.concat(buffers, n);
			contentItemLength = n;
			contentPosition = 0;
		}
	};
	/**
	 * Returns value value.
	 * @returns {number} value value
	 */
	const readUInt32LE = () => {
		ensureData(4);
		const value = contentItem.readUInt32LE(contentPosition);
		contentPosition += 4;
		return value;
	};
	/**
	 * Returns value value.
	 * @returns {number} value value
	 */
	const readInt32LE = () => {
		ensureData(4);
		const value = contentItem.readInt32LE(contentPosition);
		contentPosition += 4;
		return value;
	};
	/**
	 * Returns buffer.
	 * @param {number} l length
	 * @returns {Buffer} buffer
	 */
	const readSlice = (l) => {
		ensureData(l);
		if (contentPosition === 0 && contentItemLength === l) {
			const result = contentItem;
			if (contentsIndex + 1 < contents.length) {
				nextContent();
			} else {
				contentPosition = l;
			}
			return result;
		}
		const result = contentItem.subarray(contentPosition, contentPosition + l);
		contentPosition += l;
		// we clone the buffer here to allow the original content to be garbage collected
		return l * 2 < contentItem.buffer.byteLength ? Buffer.from(result) : result;
	};
	const version = readUInt32LE();
	if (version !== VERSION) {
		throw new Error("Invalid file version");
	}
	const sectionCount = readUInt32LE();
	/** @type {number[]} */
	const lengths = [];
	let lastLengthPositive = false;
	for (let i = 0; i < sectionCount; i++) {
		const value = readInt32LE();
		const valuePositive = value >= 0;
		if (lastLengthPositive && valuePositive) {
			lengths[lengths.length - 1] += value;
		} else {
			lengths.push(value);
			lastLengthPositive = valuePositive;
		}
	}
	/** @type {BufferSerializableType[]} */
	const result = [];
	for (let length of lengths) {
		if (length < 0) {
			const slice = readSlice(-length);
			const size = Number(readUInt64LE(slice, 0));
			const nameBuffer = slice.subarray(8);
			const name = nameBuffer.toString();
			const lazy =
				/** @type {LazyFunction} */
				(
					SerializerMiddleware.createLazy(
						memoize(() => deserialize(middleware, name, readFile)),
						middleware,
						{ name, size },
						slice
					)
				);
			result.push(lazy);
		} else {
			// A section may start exactly at a content-buffer boundary; advance
			// first, then read from the fresh buffer (don't fall through to the
			// `while` below, which would skip it).
			if (contentPosition === contentItemLength) {
				nextContent();
			}
			if (contentPosition !== 0) {
				if (length <= contentItemLength - contentPosition) {
					result.push(
						Buffer.from(
							contentItem.buffer,
							contentItem.byteOffset + contentPosition,
							length
						)
					);
					contentPosition += length;
					length = 0;
				} else {
					const l = contentItemLength - contentPosition;
					result.push(
						Buffer.from(
							contentItem.buffer,
							contentItem.byteOffset + contentPosition,
							l
						)
					);
					length -= l;
					contentPosition = contentItemLength;
				}
			} else if (length >= contentItemLength) {
				result.push(contentItem);
				length -= contentItemLength;
				contentPosition = contentItemLength;
			} else {
				result.push(
					Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
				);
				contentPosition += length;
				length = 0;
			}
			while (length > 0) {
				nextContent();
				if (length >= contentItemLength) {
					result.push(contentItem);
					length -= contentItemLength;
					contentPosition = contentItemLength;
				} else {
					result.push(
						Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
					);
					contentPosition += length;
					length = 0;
				}
			}
		}
	}
	return result;
};

/** @typedef {BufferSerializableType[]} DeserializedType */
/** @typedef {true} SerializedType */
/** @typedef {{ filename: string, extension?: string }} Context */

/**
 * Represents FileMiddleware.
 * @extends {SerializerMiddleware<DeserializedType, SerializedType, Context>}
 */
class FileMiddleware extends SerializerMiddleware {
	/**
	 * Creates an instance of FileMiddleware.
	 * @param {IntermediateFileSystem} fs filesystem
	 * @param {HashFunction} hashFunction hash function to use
	 */
	constructor(fs, hashFunction = DEFAULTS.HASH_FUNCTION) {
		super();
		/** @type {IntermediateFileSystem} */
		this.fs = fs;
		/** @type {HashFunction} */
		this._hashFunction = hashFunction;
	}

	/**
	 * Serializes this instance into the provided serializer context.
	 * @param {DeserializedType} data data
	 * @param {Context} context context object
	 * @returns {SerializedType | Promise<SerializedType> | null} serialized data
	 */
	serialize(data, context) {
		const { filename, extension = "" } = context;
		return new Promise((resolve, reject) => {
			mkdirp(this.fs, dirname(this.fs, filename), (err) => {
				if (err) return reject(err);

				// It's important that we don't touch existing files during serialization
				// because serialize may read existing files (when deserializing)
				/** @type {Set<string>} */
				const allWrittenFiles = new Set();
				/**
				 * Processes the provided name.
				 * @param {string | false} name name
				 * @param {Buffer[]} content content
				 * @param {number} size size
				 * @returns {Promise<void>}
				 */
				const writeFile = async (name, content, size) => {
					const file = name
						? join(this.fs, filename, `../${name}${extension}`)
						: filename;
					await new Promise(
						/**
						 * Handles the callback logic for this hook.
						 * @param {(value?: undefined) => void} resolve resolve
						 * @param {(reason?: Error | null) => void} reject reject
						 */
						(resolve, reject) => {
							let stream = this.fs.createWriteStream(`${file}_`);
							/** @type {undefined | import("zlib").Gzip | import("zlib").BrotliCompress | import("zlib").ZstdCompress} */
							let compression;
							if (file.endsWith(".gz")) {
								compression = createGzip({
									chunkSize: COMPRESSION_CHUNK_SIZE,
									level: zConstants.Z_BEST_SPEED
								});
							} else if (file.endsWith(".br")) {
								compression = createBrotliCompress({
									chunkSize: COMPRESSION_CHUNK_SIZE,
									params: {
										[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
										[zConstants.BROTLI_PARAM_QUALITY]: 2,
										[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
										[zConstants.BROTLI_PARAM_SIZE_HINT]: size
									}
								});
							} else if (file.endsWith(".zst")) {
								// eslint-disable-next-line n/no-unsupported-features/node-builtins
								const levelParam = zConstants.ZSTD_c_compressionLevel;
								// default level 3; some runtimes (e.g. Deno) don't expose ZSTD_CLEVEL_DEFAULT
								// eslint-disable-next-line n/no-unsupported-features/node-builtins
								const defaultLevel = zConstants.ZSTD_CLEVEL_DEFAULT;
								const level = defaultLevel === undefined ? 3 : defaultLevel;
								compression = createZstdCompress({
									chunkSize: COMPRESSION_CHUNK_SIZE,
									params: { [levelParam]: level }
								});
							}
							if (compression) {
								pipeline(compression, stream, reject);
								stream = compression;
								stream.on("finish", () => resolve());
							} else {
								stream.on("error", (err) => reject(err));
								stream.on("finish", () => resolve());
							}
							// split into chunks for WRITE_LIMIT_CHUNK size
							/** @type {Buffer[]} */
							const chunks = [];
							for (const b of content) {
								if (b.length < WRITE_LIMIT_CHUNK) {
									chunks.push(b);
								} else {
									for (let i = 0; i < b.length; i += WRITE_LIMIT_CHUNK) {
										chunks.push(b.subarray(i, i + WRITE_LIMIT_CHUNK));
									}
								}
							}

							const len = chunks.length;
							let i = 0;
							/**
							 * Processes the provided err.
							 * @param {(Error | null)=} err err
							 */
							const batchWrite = (err) => {
								// will be handled in "on" error handler
								if (err) return;

								if (i === len) {
									stream.end();
									return;
								}

								// queue up a batch of chunks up to the write limit
								// end is exclusive
								let end = i;
								let sum = chunks[end++].length;
								while (end < len) {
									sum += chunks[end].length;
									if (sum > WRITE_LIMIT_TOTAL) break;
									end++;
								}
								while (i < end - 1) {
									stream.write(chunks[i++]);
								}
								stream.write(chunks[i++], batchWrite);
							};
							batchWrite();
						}
					);
					if (name) allWrittenFiles.add(file);
				};

				resolve(
					serialize(this, data, false, writeFile, this._hashFunction).then(
						async ({ backgroundJob }) => {
							await backgroundJob;

							// Rename the index file to disallow access during inconsistent file state
							await new Promise(
								/**
								 * Handles the callback logic for this hook.
								 * @param {(value?: undefined) => void} resolve resolve
								 */
								(resolve) => {
									this.fs.rename(filename, `${filename}.old`, (_err) => {
										resolve();
									});
								}
							);

							// update all written files
							await Promise.all(
								Array.from(
									allWrittenFiles,
									(file) =>
										new Promise(
											/**
											 * Handles the callback logic for this hook.
											 * @param {(value?: undefined) => void} resolve resolve
											 * @param {(reason?: Error | null) => void} reject reject
											 * @returns {void}
											 */
											(resolve, reject) => {
												this.fs.rename(`${file}_`, file, (err) => {
													if (err) return reject(err);
													resolve();
												});
											}
										)
								)
							);

							// As final step automatically update the index file to have a consistent pack again
							await new Promise(
								/**
								 * Handles the callback logic for this hook.
								 * @param {(value?: undefined) => void} resolve resolve
								 * @returns {void}
								 */
								(resolve) => {
									this.fs.rename(`${filename}_`, filename, (err) => {
										if (err) return reject(err);
										resolve();
									});
								}
							);
							return /** @type {true} */ (true);
						}
					)
				);
			});
		});
	}

	/**
	 * Restores this instance from the provided deserializer context.
	 * @param {SerializedType} data data
	 * @param {Context} context context object
	 * @returns {DeserializedType | Promise<DeserializedType>} deserialized data
	 */
	deserialize(data, context) {
		const { filename, extension = "" } = context;
		/**
		 * Returns result.
		 * @param {string | boolean} name name
		 * @returns {Promise<Buffer[]>} result
		 */
		const readFile = (name) =>
			new Promise((resolve, reject) => {
				const file = name
					? join(this.fs, filename, `../${name}${extension}`)
					: filename;
				this.fs.stat(file, (err, stats) => {
					if (err) {
						reject(err);
						return;
					}
					let remaining = /** @type {IStats} */ (stats).size;
					/** @type {Buffer | undefined} */
					let currentBuffer;
					/** @type {number | undefined} */
					let currentBufferUsed;
					/** @type {Buffer[]} */
					const buf = [];
					/** @type {import("zlib").Zlib & import("stream").Transform | undefined} */
					let decompression;
					if (file.endsWith(".gz")) {
						decompression = createGunzip({
							chunkSize: DECOMPRESSION_CHUNK_SIZE
						});
					} else if (file.endsWith(".br")) {
						decompression = createBrotliDecompress({
							chunkSize: DECOMPRESSION_CHUNK_SIZE
						});
					} else if (file.endsWith(".zst")) {
						decompression = createZstdDecompress({
							chunkSize: DECOMPRESSION_CHUNK_SIZE
						});
					}
					if (decompression) {
						/** @typedef {(value: Buffer[] | PromiseLike<Buffer[]>) => void} NewResolve */
						/** @typedef {(reason?: Error) => void} NewReject */

						/** @type {NewResolve | undefined} */
						let newResolve;
						/** @type {NewReject | undefined} */
						let newReject;
						resolve(
							Promise.all([
								new Promise((rs, rj) => {
									newResolve = rs;
									newReject = rj;
								}),
								new Promise(
									/**
									 * Handles the chunk size callback for this hook.
									 * @param {(value?: undefined) => void} resolve resolve
									 * @param {(reason?: Error) => void} reject reject
									 */
									(resolve, reject) => {
										decompression.on("data", (chunk) => buf.push(chunk));
										decompression.on("end", () => resolve());
										decompression.on("error", (err) => reject(err));
									}
								)
							]).then(() => buf)
						);
						resolve = /** @type {NewResolve} */ (newResolve);
						reject = /** @type {NewReject} */ (newReject);
					}
					this.fs.open(file, "r", (err, _fd) => {
						if (err) {
							reject(err);
							return;
						}
						const fd = /** @type {number} */ (_fd);
						const read = () => {
							if (currentBuffer === undefined) {
								currentBuffer = Buffer.allocUnsafeSlow(
									Math.min(
										constants.MAX_LENGTH,
										remaining,
										decompression ? DECOMPRESSION_CHUNK_SIZE : Infinity
									)
								);
								currentBufferUsed = 0;
							}
							let readBuffer = currentBuffer;
							let readOffset = /** @type {number} */ (currentBufferUsed);
							let readLength =
								currentBuffer.length -
								/** @type {number} */ (currentBufferUsed);
							// values passed to fs.read must be valid int32 values
							if (readOffset > 0x7fffffff) {
								readBuffer = currentBuffer.subarray(readOffset);
								readOffset = 0;
							}
							if (readLength > 0x7fffffff) {
								readLength = 0x7fffffff;
							}
							this.fs.read(
								fd,
								readBuffer,
								readOffset,
								readLength,
								null,
								(err, bytesRead) => {
									if (err) {
										this.fs.close(fd, () => {
											reject(err);
										});
										return;
									}
									/** @type {number} */
									(currentBufferUsed) += bytesRead;
									remaining -= bytesRead;
									if (
										currentBufferUsed ===
										/** @type {Buffer} */
										(currentBuffer).length
									) {
										if (decompression) {
											decompression.write(currentBuffer);
										} else {
											buf.push(
												/** @type {Buffer} */
												(currentBuffer)
											);
										}
										currentBuffer = undefined;
										if (remaining === 0) {
											if (decompression) {
												decompression.end();
											}
											this.fs.close(fd, (err) => {
												if (err) {
													reject(err);
													return;
												}
												resolve(buf);
											});
											return;
										}
									}
									read();
								}
							);
						};
						read();
					});
				});
			});
		return deserialize(this, false, readFile);
	}
}

module.exports = FileMiddleware;
// Exposed for testing the content-buffer boundary handling in `deserialize`.
module.exports._deserialize = deserialize;