src/common/unicode_case.c - master - githubmirror/postgres

src/common/unicode_case.c
481 строка13 KB
Jeff Davis
Generate unicode_limits.h.
18 июл 2026, 02:41
18 июл 2026, 02:412cf212d
/*-------------------------------------------------------------------------
 * unicode_case.c
 *		Unicode case mapping and case conversion.
 *
 * Portions Copyright (c) 2017-2026, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *	  src/common/unicode_case.c
 *
 *-------------------------------------------------------------------------
 */
#ifndef FRONTEND
#include "postgres.h"
#else
#include "postgres_fe.h"
#endif

#include "common/unicode_case.h"
#include "common/unicode_case_table.h"
#include "common/unicode_category.h"
#include "mb/pg_wchar.h"

enum CaseMapResult
{
	CASEMAP_SELF,
	CASEMAP_SIMPLE,
	CASEMAP_SPECIAL,
};

/*
 * Map for each case kind.
 */
static const char32_t *const casekind_map[NCaseKind] =
{
	[CaseLower] = case_map_lower,
	[CaseTitle] = case_map_title,
	[CaseUpper] = case_map_upper,
	[CaseFold] = case_map_fold,
};

static char32_t find_case_map(char32_t ucs, const char32_t *map);
static size_t convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
						   size_t *pconsumed, CaseKind str_casekind, bool full,
						   WordBoundaryNext wbnext, void *wbstate);
static enum CaseMapResult casemap(char32_t u1, CaseKind casekind, bool full,
								  const char *src, size_t srclen, size_t srcoff,
								  char32_t *simple, const char32_t **special);

char32_t
unicode_lowercase_simple(char32_t code)
{
	char32_t	cp = find_case_map(code, case_map_lower);

	return cp != 0 ? cp : code;
}

char32_t
unicode_titlecase_simple(char32_t code)
{
	char32_t	cp = find_case_map(code, case_map_title);

	return cp != 0 ? cp : code;
}

char32_t
unicode_uppercase_simple(char32_t code)
{
	char32_t	cp = find_case_map(code, case_map_upper);

	return cp != 0 ? cp : code;
}

char32_t
unicode_casefold_simple(char32_t code)
{
	char32_t	cp = find_case_map(code, case_map_fold);

	return cp != 0 ? cp : code;
}

/*
 * unicode_strlower()
 *
 * Convert src to lowercase, and return the result length (not including
 * terminating NUL). Sets *pconsumed to the amount of src successfully
 * consumed; if less than srclen, indicates a decoding error.
 *
 * String src must be encoded in UTF-8.
 *
 * Result string is stored in dst, truncating if larger than dstsize. If
 * dstsize is greater than the result length, dst will be NUL-terminated;
 * otherwise not.
 *
 * If dstsize is zero, dst may be NULL. This is useful for calculating the
 * required buffer size before allocating.
 *
 * If full is true, use special case mappings if available and if the
 * conditions are satisfied.
 */
size_t
unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
				 size_t *pconsumed, bool full)
{
	return convert_case(dst, dstsize, src, srclen, pconsumed, CaseLower, full,
						NULL, NULL);
}

/*
 * unicode_strtitle()
 *
 * Convert src to titlecase, and return the result length (not including
 * terminating NUL). Sets *pconsumed to the amount of src successfully
 * consumed; if less than srclen, indicates a decoding error.
 *
 * String src must be encoded in UTF-8.
 *
 * Result string is stored in dst, truncating if larger than dstsize. If
 * dstsize is greater than the result length, dst will be NUL-terminated;
 * otherwise not.
 *
 * If dstsize is zero, dst may be NULL. This is useful for calculating the
 * required buffer size before allocating.
 *
 * If full is true, use special case mappings if available and if the
 * conditions are satisfied. Otherwise, use only simple mappings and use
 * uppercase instead of titlecase.
 *
 * Titlecasing requires knowledge about word boundaries, which is provided by
 * the callback wbnext. A word boundary is the offset of the start of a word
 * or the offset of the character immediately following a word.
 *
 * The caller is expected to initialize and free the callback state
 * wbstate. The callback should first return offset 0 for the first boundary;
 * then the offset of each subsequent word boundary; then the total length of
 * the string to indicate the final boundary.
 */
size_t
unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
				 size_t *pconsumed, bool full, WordBoundaryNext wbnext,
				 void *wbstate)
{
	return convert_case(dst, dstsize, src, srclen, pconsumed, CaseTitle, full,
						wbnext, wbstate);
}

/*
 * unicode_strupper()
 *
 * Convert src to uppercase, and return the result length (not including
 * terminating NUL). Sets *pconsumed to the amount of src successfully
 * consumed; if less than srclen, indicates a decoding error.
 *
 * String src must be encoded in UTF-8.
 *
 * Result string is stored in dst, truncating if larger than dstsize. If
 * dstsize is greater than the result length, dst will be NUL-terminated;
 * otherwise not.
 *
 * If dstsize is zero, dst may be NULL. This is useful for calculating the
 * required buffer size before allocating.
 *
 * If full is true, use special case mappings if available and if the
 * conditions are satisfied.
 */
size_t
unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
				 size_t *pconsumed, bool full)
{
	return convert_case(dst, dstsize, src, srclen, pconsumed, CaseUpper, full,
						NULL, NULL);
}

/*
 * unicode_strfold()
 *
 * Case fold src, and return the result length (not including terminating
 * NUL). Sets *pconsumed to the amount of src successfully consumed; if less
 * than srclen, indicates a decoding error.
 *
 * String src must be encoded in UTF-8.
 *
 * Result string is stored in dst, truncating if larger than dstsize. If
 * dstsize is greater than the result length, dst will be NUL-terminated;
 * otherwise not.
 *
 * If dstsize is zero, dst may be NULL. This is useful for calculating the
 * required buffer size before allocating.
 */
size_t
unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
				size_t *pconsumed, bool full)
{
	return convert_case(dst, dstsize, src, srclen, pconsumed, CaseFold, full,
						NULL, NULL);
}

/* local version of pg_utf_mblen() to be inlinable */
static int
utf8_mblen(const unsigned char *s)
{
	if ((*s & 0x80) == 0)
		return 1;
	else if ((*s & 0xe0) == 0xc0)
		return 2;
	else if ((*s & 0xf0) == 0xe0)
		return 3;
	else if ((*s & 0xf8) == 0xf0)
		return 4;
	else
		return -1;
}

/*
 * Implement Unicode Default Case Conversion algorithm.
 *
 * If str_casekind is CaseLower or CaseUpper, map each character in the string
 * for which a mapping is available.
 *
 * If str_casekind is CaseTitle, maps characters found on a word boundary to
 * titlecase (or uppercase if full is false) and other characters to
 * lowercase. NB: does not currently implement the Unicode behavior in which
 * the word boundary is adjusted to the next Cased character. That behavior
 * could be implemented as an option, but it doesn't match the default
 * behavior of ICU, nor does it match the documented behavior of INITCAP().
 *
 * If full is true, use special mappings for relevant characters, which can
 * map a single codepoint to multiple codepoints, or depend on conditions.
 */
static size_t
convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
			 size_t *pconsumed, CaseKind str_casekind, bool full,
			 WordBoundaryNext wbnext, void *wbstate)
{
	/* character CaseKind varies while titlecasing */
	CaseKind	chr_casekind = str_casekind;
	size_t		srcoff = 0;
	size_t		result_len = 0;
	size_t		boundary = 0;

	/*
	 * Must be guaranteed by caller to avoid overflow (text values limited to
	 * MaxAllocSize anyway).
	 */
	Assert(srclen < SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION);

	Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
		   (str_casekind != CaseTitle && !wbnext && !wbstate));

	if (str_casekind == CaseTitle)
	{
		boundary = wbnext(wbstate);
		Assert(boundary == 0);	/* start of text is always a boundary */
	}

	while (srcoff < srclen)
	{
		int			u1len = utf8_mblen((const unsigned char *) src + srcoff);
		char32_t	u1;
		char32_t	simple = 0;
		const char32_t *special = NULL;
		enum CaseMapResult casemap_result;

		/* invalid UTF8 */
		if (u1len < 0 || srcoff + u1len > srclen)
			break;

		u1 = utf8_to_unicode((const unsigned char *) src + srcoff);

		if (str_casekind == CaseTitle)
		{
			if (srcoff == boundary)
			{
				chr_casekind = full ? CaseTitle : CaseUpper;
				boundary = wbnext(wbstate);
			}
			else
				chr_casekind = CaseLower;
		}

		casemap_result = casemap(u1, chr_casekind, full, src, srclen, srcoff,
								 &simple, &special);

		switch (casemap_result)
		{
			case CASEMAP_SELF:
				/* no mapping; copy bytes from src */
				Assert(simple == 0);
				Assert(special == NULL);
				if (result_len + u1len <= dstsize)
					memcpy(dst + result_len, src + srcoff, u1len);

				result_len += u1len;
				break;
			case CASEMAP_SIMPLE:
				{
					/* replace with single character */
					char32_t	u2 = simple;
					char32_t	u2len = unicode_utf8len(u2);

					Assert(special == NULL);
					if (result_len + u2len <= dstsize)
						unicode_to_utf8(u2, (unsigned char *) dst + result_len);

					result_len += u2len;
				}
				break;
			case CASEMAP_SPECIAL:
				/* replace with up to UNICODE_MAX_CASEMAP_CODEPOINTS */
				Assert(simple == 0);
				for (int i = 0; i < UNICODE_MAX_CASEMAP_CODEPOINTS && special[i]; i++)
				{
					char32_t	u2 = special[i];
					size_t		u2len = unicode_utf8len(u2);

					if (result_len + u2len <= dstsize)
						unicode_to_utf8(u2, (unsigned char *) dst + result_len);

					result_len += u2len;
				}
				break;
		}

		srcoff += u1len;
	}

	if (result_len < dstsize)
		dst[result_len] = '\0';

	*pconsumed = srcoff;
	return result_len;
}

/*
 * Check that the condition matches Final_Sigma, described in Unicode Table
 * 3-17. The character at the given offset must be directly preceded by a
 * Cased character, and must not be directly followed by a Cased character.
 *
 * Case_Ignorable characters are ignored. Neither beginning of string nor end
 * of string are considered Cased characters. NB: some characters may be both
 * Cased and Case_Ignorable, in which case they are ignored.
 */
static bool
check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
	bool		preceded_by_cased = false;
	bool		followed_by_cased = false;
	char32_t	curr;
	int			ulen;

	/* iterate backwards looking for preceding character */
	for (size_t i = offset; i > 0;)
	{
		/* skip backwards through continuation bytes */
		i--;
		if ((str[i] & 0xC0) == 0x80)
			continue;

		/* now at leading byte of previous sequence */
		Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);

		ulen = utf8_mblen((const unsigned char *) str + i);

		/* invalid UTF8 */
		if (ulen < 0 || i + ulen > len)
			return false;

		curr = utf8_to_unicode((const unsigned char *) str + i);

		if (!pg_u_prop_case_ignorable(curr))
		{
			preceded_by_cased = pg_u_prop_cased(curr);
			break;
		}
	}

	ulen = utf8_mblen((const unsigned char *) str + offset);

	/* iterate forward looking for following character */
	for (size_t i = offset + ulen; i < len;)
	{
		ulen = utf8_mblen((const unsigned char *) str + i);

		/* invalid UTF8 */
		if (ulen < 0 || i + ulen > len)
			return false;

		curr = utf8_to_unicode((const unsigned char *) str + i);

		if (!pg_u_prop_case_ignorable(curr))
		{
			followed_by_cased = pg_u_prop_cased(curr);
			break;
		}

		i += ulen;
	}

	return (preceded_by_cased && !followed_by_cased);
}

/*
 * Unicode allows for special casing to be applied only under certain
 * circumstances. The only currently-supported condition is Final_Sigma.
 */
static bool
check_special_conditions(int conditions, const char *str, size_t len,
						 size_t offset)
{
	if (conditions == 0)
		return true;
	else if (conditions == PG_U_FINAL_SIGMA)
		return check_final_sigma((const unsigned char *) str, len, offset);

	/* no other conditions supported */
	Assert(false);
	return false;
}

/*
 * Map the given character to the requested case.
 *
 * If full is true, and a special case mapping is found and the conditions are
 * met, 'special' is set to the mapping result (which is an array of up to
 * UNICODE_MAX_CASEMAP_CODEPOINTS) and CASEMAP_SPECIAL is returned.
 *
 * Otherwise, search for a simple mapping, and if found, set 'simple' to the
 * result and return CASEMAP_SIMPLE.
 *
 * If no mapping is found, return CASEMAP_SELF, and the caller should copy the
 * character without modification.
 */
static enum CaseMapResult
casemap(char32_t u1, CaseKind casekind, bool full,
		const char *src, size_t srclen, size_t srcoff,
		char32_t *simple, const char32_t **special)
{
	uint16		idx;

	/* Fast path for codepoints < 0x80 */
	if (u1 < 0x80)
	{
		/*
		 * The first elements in all tables are reserved as 0 (as NULL). The
		 * data starts at index 1, not 0.
		 */
		*simple = casekind_map[casekind][u1 + 1];

		return CASEMAP_SIMPLE;
	}

	idx = case_index(u1);

	if (idx == 0)
		return CASEMAP_SELF;

	if (full && case_map_special[idx] &&
		check_special_conditions(special_case[case_map_special[idx]].conditions,
								 src, srclen, srcoff))
	{
		*special = special_case[case_map_special[idx]].map[casekind];
		return CASEMAP_SPECIAL;
	}

	*simple = casekind_map[casekind][idx];

	return CASEMAP_SIMPLE;
}

/*
 * Find entry in simple case map.
 * If the entry does not exist, 0 will be returned.
 */
static char32_t
find_case_map(char32_t ucs, const char32_t *map)
{
	/* Fast path for codepoints < 0x80 */
	if (ucs < 0x80)
		/* The first elements in all tables are reserved as 0 (as NULL). */
		return map[ucs + 1];
	return map[case_index(ucs)];
}