src/common/unicode/case_test.c - master - githubmirror/postgres

src/common/unicode/case_test.c
391 строка10 KB
Jeff Davis
unicode_case.c: change API to signal UTF8 decoding error.
08 июл 2026, 01:34
08 июл 2026, 01:3407211f6
/*-------------------------------------------------------------------------
 * case_test.c
 *		Program to test Unicode case mapping functions.
 *
 * Portions Copyright (c) 2017-2026, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *	  src/common/unicode/case_test.c
 *
 *-------------------------------------------------------------------------
 */
#include "postgres_fe.h"

#include <locale.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wctype.h>

#ifdef USE_ICU
#include <unicode/ucasemap.h>
#include <unicode/uchar.h>
#endif
#include "common/unicode_case.h"
#include "common/unicode_category.h"
#include "common/unicode_version.h"
#include "mb/pg_wchar.h"

/* enough to hold largest source or result string, including NUL */
#define BUFSZ 256

#ifdef USE_ICU
static UCaseMap *casemap = NULL;
#endif

typedef size_t (*TestFunc) (char *dst, size_t dstsize, const char *src,
							size_t srclen);

/* simple boundary iterator copied from pg_locale_builtin.c */
struct WordBoundaryState
{
	const char *str;
	size_t		len;
	size_t		offset;
	bool		posix;
	bool		init;
	bool		prev_alnum;
};

static size_t
initcap_wbnext(void *state)
{
	struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;

	while (wbstate->offset < wbstate->len &&
		   wbstate->str[wbstate->offset] != '\0')
	{
		char32_t	u = utf8_to_unicode((const unsigned char *) wbstate->str +
										wbstate->offset);
		bool		curr_alnum = pg_u_isalnum(u, wbstate->posix);

		if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
		{
			size_t		prev_offset = wbstate->offset;

			wbstate->init = true;
			wbstate->offset += unicode_utf8len(u);
			wbstate->prev_alnum = curr_alnum;
			return prev_offset;
		}

		wbstate->offset += unicode_utf8len(u);
	}

	return wbstate->len;
}

#ifdef USE_ICU

static void
icu_test_simple(char32_t code)
{
	char32_t	lower = unicode_lowercase_simple(code);
	char32_t	title = unicode_titlecase_simple(code);
	char32_t	upper = unicode_uppercase_simple(code);
	char32_t	fold = unicode_casefold_simple(code);
	char32_t	iculower = u_tolower(code);
	char32_t	icutitle = u_totitle(code);
	char32_t	icuupper = u_toupper(code);
	char32_t	icufold = u_foldCase(code, U_FOLD_CASE_DEFAULT);

	if (lower != iculower || title != icutitle || upper != icuupper ||
		fold != icufold)
	{
		printf("case_test: FAILURE for codepoint 0x%06x\n", code);
		printf("case_test: Postgres lower/title/upper/fold:	0x%06x/0x%06x/0x%06x/0x%06x\n",
			   lower, title, upper, fold);
		printf("case_test: ICU lower/title/upper/fold:		0x%06x/0x%06x/0x%06x/0x%06x\n",
			   iculower, icutitle, icuupper, icufold);
		printf("\n");
		exit(1);
	}
}

static void
icu_test_full(char *str)
{
	char		lower[BUFSZ];
	char		title[BUFSZ];
	char		upper[BUFSZ];
	char		fold[BUFSZ];
	char		icu_lower[BUFSZ];
	char		icu_title[BUFSZ];
	char		icu_upper[BUFSZ];
	char		icu_fold[BUFSZ];
	UErrorCode	status;
	size_t		len = strlen(str);
	size_t		consumed;

	/* full case mapping doesn't use posix semantics */
	struct WordBoundaryState wbstate = {
		.str = str,
		.len = strlen(str),
		.offset = 0,
		.posix = false,
		.init = false,
		.prev_alnum = false,
	};

	unicode_strlower(lower, BUFSZ, str, len, &consumed, true);
	unicode_strtitle(title, BUFSZ, str, len, &consumed, true, initcap_wbnext, &wbstate);
	unicode_strupper(upper, BUFSZ, str, len, &consumed, true);
	unicode_strfold(fold, BUFSZ, str, len, &consumed, true);
	status = U_ZERO_ERROR;
	ucasemap_utf8ToLower(casemap, icu_lower, BUFSZ, str, len, &status);
	status = U_ZERO_ERROR;
	ucasemap_utf8ToTitle(casemap, icu_title, BUFSZ, str, len, &status);
	status = U_ZERO_ERROR;
	ucasemap_utf8ToUpper(casemap, icu_upper, BUFSZ, str, len, &status);
	status = U_ZERO_ERROR;
	ucasemap_utf8FoldCase(casemap, icu_fold, BUFSZ, str, len, &status);

	if (strcmp(lower, icu_lower) != 0)
	{
		printf("case_test: str='%s' lower='%s' icu_lower='%s'\n", str, lower,
			   icu_lower);
		exit(1);
	}
	if (strcmp(title, icu_title) != 0)
	{
		printf("case_test: str='%s' title='%s' icu_title='%s'\n", str, title,
			   icu_title);
		exit(1);
	}
	if (strcmp(upper, icu_upper) != 0)
	{
		printf("case_test: str='%s' upper='%s' icu_upper='%s'\n", str, upper,
			   icu_upper);
		exit(1);
	}
	if (strcmp(fold, icu_fold) != 0)
	{
		printf("case_test: str='%s' fold='%s' icu_fold='%s'\n", str, fold,
			   icu_fold);
		exit(1);
	}
}

/*
 * Exhaustively compare case mappings with the results from ICU.
 */
static void
test_icu(void)
{
	int			successful = 0;
	int			skipped_mismatch = 0;

	for (char32_t code = 0; code <= 0x10ffff; code++)
	{
		pg_unicode_category category = unicode_category(code);

		if (category != PG_U_UNASSIGNED)
		{
			uint8_t		icu_category = u_charType(code);
			char		code_str[5] = {0};

			if (icu_category == PG_U_UNASSIGNED)
			{
				skipped_mismatch++;
				continue;
			}

			icu_test_simple(code);
			unicode_to_utf8(code, (unsigned char *) code_str);
			icu_test_full(code_str);

			successful++;
		}
	}

	if (skipped_mismatch > 0)
		printf("case_test: skipped %d codepoints unassigned in ICU due to Unicode version mismatch\n",
			   skipped_mismatch);

	printf("case_test: ICU simple mapping test: %d codepoints successful\n",
		   successful);
}
#endif

static void
test_convert(TestFunc tfunc, const char *test_string, const char *expected)
{
	size_t		src1len = strlen(test_string);
	size_t		dst1len = strlen(expected);
	size_t		dst2len = strlen(expected) + 1; /* NUL-terminated */
	char	   *src1 = malloc(src1len);
	char	   *dst1 = malloc(dst1len);
	char	   *dst2 = malloc(dst2len);
	size_t		needed;

	memcpy(src1, test_string, src1len); /* not NUL-terminated */

	/* destination is not NUL-terminated */
	memset(dst1, 0x7F, dst1len);
	needed = tfunc(dst1, dst1len, src1, src1len);
	if (needed != strlen(expected))
	{
		printf("case_test: convert_case test1 FAILURE: '%s' needed %zu expected %zu\n",
			   test_string, needed, strlen(expected));
		exit(1);
	}
	if (memcmp(dst1, expected, dst1len) != 0)
	{
		printf("case_test: convert_case test1 FAILURE: test: '%s' result: '%.*s' expected: '%s'\n",
			   test_string, (int) dst1len, dst1, expected);
		exit(1);
	}

	/* destination is NUL-terminated */
	memset(dst2, 0x7F, dst2len);
	needed = tfunc(dst2, dst2len, src1, src1len);
	if (needed != strlen(expected))
	{
		printf("case_test: convert_case test2 FAILURE: '%s' needed %zu expected %zu\n",
			   test_string, needed, strlen(expected));
		exit(1);
	}
	if (strcmp(dst2, expected) != 0)
	{
		printf("case_test: convert_case test2 FAILURE: test: '%s' result: '%s' expected: '%s'\n",
			   test_string, dst2, expected);
		exit(1);
	}

	free(src1);
	free(dst1);
	free(dst2);
}

static size_t
tfunc_lower(char *dst, size_t dstsize, const char *src,
			size_t srclen)
{
	size_t		consumed;

	return unicode_strlower(dst, dstsize, src, srclen, &consumed, true);
}

static size_t
tfunc_title(char *dst, size_t dstsize, const char *src,
			size_t srclen)
{
	size_t		consumed;
	struct WordBoundaryState wbstate = {
		.str = src,
		.len = srclen,
		.offset = 0,
		.init = false,
		.prev_alnum = false,
	};

	return unicode_strtitle(dst, dstsize, src, srclen, &consumed, true,
							initcap_wbnext, &wbstate);
}

static size_t
tfunc_upper(char *dst, size_t dstsize, const char *src,
			size_t srclen)
{
	size_t		consumed;

	return unicode_strupper(dst, dstsize, src, srclen, &consumed, true);
}

static size_t
tfunc_fold(char *dst, size_t dstsize, const char *src,
		   size_t srclen)
{
	size_t		consumed;

	return unicode_strfold(dst, dstsize, src, srclen, &consumed, true);
}

static void
test_convert_case(void)
{
	size_t		needed;
	size_t		consumed;

	/* test string with no case changes */
	test_convert(tfunc_lower, "√∞", "√∞");
	/* test adjust-to-cased behavior */
	test_convert(tfunc_title, "abc 123xyz", "Abc 123xyz");
	/* test string with case changes */
	test_convert(tfunc_upper, "abc", "ABC");
	/* test string with case changes and byte length changes */
	test_convert(tfunc_lower, "ȺȺȺ", "ⱥⱥⱥ");
	/* test special case conversions */
	test_convert(tfunc_upper, "ß", "SS");
	test_convert(tfunc_lower, "ıiIİ", "ıiii\u0307");
	test_convert(tfunc_upper, "ıiIİ", "IIIİ");
	test_convert(tfunc_fold, "ıiIİ", "ıiii\u0307");
	/* test final sigma */
	test_convert(tfunc_lower, "σςΣ ΣΣΣ", "σςς σσς");
	test_convert(tfunc_lower, "σς'Σ' ΣΣ'Σ'", "σς'ς' σσ'ς'");
	test_convert(tfunc_title, "σςΣ ΣΣΣ", "Σςς Σσς");
	test_convert(tfunc_fold, "σςΣ ΣΣΣ", "σσσ σσσ");
	/* test that alphanumerics are word characters */
	test_convert(tfunc_title, "λλ", "Λλ");
	test_convert(tfunc_title, "1a", "1a");
	/* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
	test_convert(tfunc_title, "\uFF11a", "\uFF11a");

	/* invalid UTF8: truncated multibyte sequence */
	needed = unicode_strfold(NULL, 0, "abc\xCE", 4, &consumed, false);
	Assert(needed == 3 && consumed == 3);
	/* invalid UTF8: leading byte invalid length */
	needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
	Assert(needed == 3 && consumed == 3);

#ifdef USE_ICU
	icu_test_full("");
	icu_test_full("ȺȺȺ");
	icu_test_full("ßßß");
	icu_test_full("√∞");
	icu_test_full("a b");
	icu_test_full("abc 123xyz");
	icu_test_full("σςΣ ΣΣΣ");
	icu_test_full("ıiIİ");
	icu_test_full("\uFF11a");
	/* test <alpha><iota_subscript><acute> */
	icu_test_full("\u0391\u0345\u0301");
#endif

	printf("case_test: convert_case: success\n");
}

int
main(int argc, char **argv)
{
#ifdef USE_ICU
	UErrorCode	status = U_ZERO_ERROR;

	/*
	 * Disable ICU's word break adjustment for titlecase to match the expected
	 * behavior of unicode_strtitle().
	 */
	casemap = ucasemap_open("und", U_TITLECASE_NO_BREAK_ADJUSTMENT, &status);
	if (U_FAILURE(status))
	{
		printf("case_test: failure opening UCaseMap: %s\n",
			   u_errorName(status));
		exit(1);
	}
#endif

	printf("case_test: Postgres Unicode version:\t%s\n", PG_UNICODE_VERSION);
#ifdef USE_ICU
	printf("case_test: ICU Unicode version:\t\t%s\n", U_UNICODE_VERSION);
	test_icu();
#else
	printf("case_test: ICU not available; skipping\n");
#endif

	test_convert_case();

#ifdef USE_ICU
	ucasemap_close(casemap);
#endif
	exit(0);
}