tooling/generate-html-entities.js - main - githubmirror/webpack

tooling/generate-html-entities.js
177 строк6 KB
Alexander Akait
refactor: consolidate HTML/CSS tokenizer modules into syntax.js (#21168)
11 июн 2026, 16:02
11 июн 2026, 16:0291de841
/*
	MIT License http://www.opensource.org/licenses/mit-license.php
*/

"use strict";

// cspell:disable

// Generates the `// #region html entities` block inside
// `lib/html/syntax.js` from the vendored WHATWG named character
// references table at `tooling/html-entities.json`.
//
// Run as part of `yarn lint:special` to verify the inlined block is in
// sync, or with `--write` (via `yarn fix:special`) to update it in place.
//
// To refresh `tooling/html-entities.json` against the current spec, run
// with `--fetch` (one-off, requires network access). Source URL:
// https://html.spec.whatwg.org/entities.json (WHATWG HTML Standard).

const fs = require("fs");
const https = require("https");
const path = require("path");

const SPEC_URL = "https://html.spec.whatwg.org/entities.json";
const FALLBACK_URL =
	"https://raw.githubusercontent.com/w3c/html/master/entities.json";
const DATA_PATH = path.resolve(__dirname, "html-entities.json");
const TARGET_PATH = path.resolve(__dirname, "..", "lib", "html", "syntax.js");

// Tolerate both LF and CRLF so the generator's `check` mode doesn't
// false-fail on Windows checkouts where git normalized line endings to CRLF.
const REGION_REGEXP =
	/\/\/ #region html entities\r?\n[\s\S]+?\/\/ #endregion\r?\n/;

const doWrite = process.argv.includes("--write");
const doFetch = process.argv.includes("--fetch");

/**
 * @param {string} url URL to fetch
 * @returns {Promise<string>} fetched body
 */
const fetchUrl = (url) =>
	new Promise((resolve, reject) => {
		https
			.get(
				url,
				{ headers: { "user-agent": "webpack/generate-html-entities" } },
				(res) => {
					if (
						res.statusCode === 301 ||
						res.statusCode === 302 ||
						res.statusCode === 307 ||
						res.statusCode === 308
					) {
						// Drain the response body so the socket is released
						// before we open the follow-up request.
						res.resume();
						const location = res.headers.location;
						if (!location) {
							return reject(
								new Error(
									`Redirect from ${url} with no Location header (HTTP ${res.statusCode})`
								)
							);
						}
						// `Location` may be relative; resolve against the current
						// request URL so `https.get` receives an absolute URL.
						return fetchUrl(new URL(location, url).toString()).then(
							resolve,
							reject
						);
					}
					if (res.statusCode !== 200) {
						// Drain so the response body doesn't hold the socket open.
						res.resume();
						return reject(
							new Error(`Failed to fetch ${url}: HTTP ${res.statusCode}`)
						);
					}
					/** @type {Buffer[]} */
					const chunks = [];
					res.on("data", (chunk) => chunks.push(chunk));
					res.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
					res.on("error", reject);
				}
			)
			.on("error", reject);
	});

/**
 * @param {Record<string, { characters: string }>} entities raw WHATWG table
 * @returns {Record<string, string>} name -> decoded characters
 */
const buildMap = (entities) => {
	/** @type {Record<string, string>} */
	const map = {};
	// Sort alphabetically so the emitted literal is deterministic.
	for (const k of Object.keys(entities).sort()) {
		map[k.slice(1)] = entities[k].characters;
	}
	return map;
};

/**
 * Render the `// #region html entities` … `// #endregion` block. The entity
 * table is emitted as a single-line frozen Object literal so the JS engine
 * builds it once at module-load time (no per-call hash construction). The
 * `// prettier-ignore` and `// cspell:disable-next-line` directives above
 * the literal keep prettier from wrapping the line and cspell from flagging
 * entity-name spellings.
 * @param {Record<string, string>} map entity name to characters
 * @returns {string} replacement block (newline-terminated)
 */
const renderRegion = (map) =>
	`// #region html entities
// The contents of this region are auto-generated by
// \`tooling/generate-html-entities.js\` from \`tooling/html-entities.json\`.
// Do not edit by hand — re-run the generator (via \`yarn fix:special\`) to refresh.
//
// WHATWG named character references. Keys are entity names WITHOUT the
// leading \`&\` (some end with \`;\`, others omit it for legacy entities that
// match without a closing semicolon). Values are the decoded character
// strings (1–2 UTF-16 code units).
// Built on a null prototype so bracket lookups (\`HTML_ENTITIES[name]\`)
// can't be poisoned by inherited \`Object.prototype\` keys like \`toString\`,
// \`constructor\`, or \`__proto__\` — without this, \`&toString;\` would falsely
// look like a matched named character reference.
// prettier-ignore
// cspell:disable-next-line
const HTML_ENTITIES = /** @type {Readonly<Record<string, string>>} */ (Object.freeze(Object.assign(Object.create(null), ${JSON.stringify(map)})));
// #endregion
`;

(async () => {
	if (doFetch) {
		let body;
		try {
			body = await fetchUrl(SPEC_URL);
		} catch (_err) {
			body = await fetchUrl(FALLBACK_URL);
		}
		const parsed = JSON.parse(body);
		fs.writeFileSync(DATA_PATH, `${JSON.stringify(parsed, null, 2)}\n`);
		console.error(`${path.relative(process.cwd(), DATA_PATH)} updated`);
	}

	const entities = JSON.parse(fs.readFileSync(DATA_PATH, "utf8"));
	const map = buildMap(entities);

	const currentContent = fs.readFileSync(TARGET_PATH, "utf8");
	if (!REGION_REGEXP.test(currentContent)) {
		throw new Error(
			`Could not find the \`// #region html entities\` block in ${TARGET_PATH}`
		);
	}
	// Preserve the file's existing EOL style (CRLF on Windows, LF elsewhere)
	// so writing the regenerated region doesn't introduce mixed line endings.
	const eol = currentContent.includes("\r\n") ? "\r\n" : "\n";
	const region = renderRegion(map).replace(/\n/g, eol);
	const newContent = currentContent.replace(REGION_REGEXP, region);

	if (newContent !== currentContent) {
		if (doWrite) {
			fs.writeFileSync(TARGET_PATH, newContent);
			console.error(
				`${path.relative(process.cwd(), TARGET_PATH)} updated (${Object.keys(entities).length} entities)`
			);
		} else {
			console.error(
				`${path.relative(process.cwd(), TARGET_PATH)} needs to be updated`
			);
			process.exitCode = 1;
		}
	}
})().catch((err) => {
	throw err;
});