src/lib/test-unicode-data.c - main - niceSOFT/dovecot

src/lib/test-unicode-data.c
848 строк24 KB
Stephan Bosch
lib: unicode-data - Add data for IDNA code point context rules
13 июл 2026, 21:22
13 июл 2026, 21:22b895689
/* Copyright (c) Dovecot authors, see top-level COPYING file */

#include "test-lib.h"
#include "strnum.h"
#include "str.h"
#include "unichar.h"
#include "istream.h"
#include "unicode-data.h"

#include <fcntl.h>

#define UCD_CASE_FOLDING_TXT "CaseFolding.txt"
#define UCD_COMPOSITION_EXCLUSIONS_TXT "CompositionExclusions.txt"
#define UCD_DERIVED_BIDI_CLASS_TXT "DerivedBidiClass.txt"
#define UCD_DERIVED_JOINING_TYPE_TXT "DerivedJoiningType.txt"
#define UCD_DERIVED_NORMALIZATION_PROPS_TXT "DerivedNormalizationProps.txt"
#define UCD_GRAPHEME_BREAK_PROPERTY_TXT "GraphemeBreakProperty.txt"
#define UCD_PROP_LIST_TXT "PropList.txt"
#define UCD_SCRIPTS_TXT "Scripts.txt"
#define UCD_SPECIAL_CASING_TXT "SpecialCasing.txt"
#define UCD_UNICODE_DATA_TXT "UnicodeData.txt"
#define UCD_WORD_BREAK_PROPERTY_TXT "WordBreakProperty.txt"
#define UCD_IDNA_MAPPING_TABLE_TXT "IdnaMappingTable.txt"

static bool
parse_prop_file_line(const char *line, const char *file, unsigned int line_num,
		     uint32_t *cp_first_r, uint32_t *cp_last_r,
		     const char **prop_r, const char **value_r)
{
	unsigned int expected_columns = 1;

	if (*line == '\0')
		return FALSE;

	if (prop_r != NULL)
		expected_columns++;

	const char *const *columns = t_strsplit(line, ";");
	unsigned int columns_count = str_array_length(columns);
	if (columns_count == 0)
		return FALSE;
	if (columns_count < expected_columns) {
		test_failed(t_strdup_printf(
			"Invalid data at %s:%u", file, line_num));
		return FALSE;
	}

	const char *p = strstr(columns[0], "..");
	const char *cp_first_hex, *cp_last_hex;

	cp_last_hex = NULL;
	if (p == NULL) {
		cp_first_hex = t_str_trim(columns[0], " \t");
	} else {
		cp_first_hex = t_str_trim(t_strdup_until(columns[0], p), " \t");
		cp_last_hex = t_str_trim(p + 2, " \t");
	}
	if (str_to_uint32_hex(cp_first_hex, cp_first_r) < 0) {
		test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad first code point", file, line_num));
		return FALSE;
	}
	if (cp_last_hex == NULL)
		*cp_last_r = *cp_first_r;
	else if (str_to_uint32_hex(cp_last_hex, cp_last_r) < 0) {
		test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad first code point", file, line_num));
		return FALSE;
	}

	if (prop_r != NULL) {
		*prop_r = t_str_trim(columns[1], " \t");
		if (value_r != NULL) {
			if (columns_count >= 3 && columns[2] != NULL)
				*value_r = t_str_trim(columns[2], " \t");
			else
				*value_r = NULL;
		}
	} else if (value_r != NULL) {
		if (columns_count >= 2 && columns[1] != NULL)
			*value_r = t_str_trim(columns[1], " \t");
		else
			*value_r = NULL;
	}
	return !test_has_failed();
}

static void
test_case_mapping(uint32_t cp, const char *const *parsed_mapping,
		  const uint32_t *case_map, unsigned int case_map_len)
{
	unsigned int case_map_idx;
	unsigned int parsed_mapping_len = str_array_length(parsed_mapping);

	if (parsed_mapping_len == 1 && case_map_len == 0) {
		/* Maps to itself (compiled as len == 0) */
		uint32_t mcp;

		test_assert_idx(str_to_uint32_hex(*parsed_mapping, &mcp) >= 0, cp);
		if (test_has_failed())
			return;
		test_assert_idx(mcp == cp, cp);
		return;
	}

	/* Explicit mapping */
	test_assert(parsed_mapping_len == case_map_len);
	if (test_has_failed())
		return;

	case_map_idx = 0;
	while (*parsed_mapping != NULL && !test_has_failed()) {
		uint32_t mcp;

		test_assert_idx(str_to_uint32_hex(*parsed_mapping, &mcp) >= 0, cp);
		if (test_has_failed())
			return;
		test_assert_idx(uni_is_valid_ucs4(mcp), cp);
		test_assert_idx(mcp == case_map[case_map_idx], cp);

		case_map_idx++;
		parsed_mapping++;
	}
}

static void
test_case_folding_line(const char *line, unsigned int line_num,
		       const char *comment_data ATTR_UNUSED)
{
	if (*line == '\0')
		return;

	const char *const *columns = t_strsplit(line, ";");
	size_t num_columns = str_array_length(columns);

	/* <code>; <status>; <mapping>; # <name> */

	if (num_columns == 0)
		return;
	if (num_columns < 4) {
		test_failed(t_strdup_printf(
			"Invalid data at %s:%u",
			UCD_CASE_FOLDING_TXT, line_num));
		return;
	}

	if (num_columns > 4 && strlen(t_str_trim(columns[4], " ")) > 0) {
		/* Skip lines with condition list */
		return;
	}

	const char *cp_hex = t_str_trim(columns[0], " ");
	uint32_t cp;

	if (str_to_uint32_hex(cp_hex, &cp) < 0) {
		test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad code point",
				UCD_CASE_FOLDING_TXT, line_num));
		return;
	}

	/* Parse Decomposition_* */

	const char *status = t_str_trim(columns[1], " ");

	if (strcmp(status, "C") != 0 && strcmp(status, "F") != 0)
		return;

	const char *mapping = t_str_trim(columns[2], " ");
	const char *const *map = t_strsplit(mapping, " ");

	/* Check data */

	const struct unicode_code_point_data *cp_data =
		unicode_code_point_get_data(cp);
	const uint32_t *case_map;
	size_t case_map_len;

	case_map_len = unicode_code_point_data_get_casefold_mapping(
		cp_data, &case_map);
	test_case_mapping(cp, map, case_map, case_map_len);
}

static void
test_composition_exclusions_line(const char *line, unsigned int line_num,
				 const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;

	if (!parse_prop_file_line(line, UCD_COMPOSITION_EXCLUSIONS_TXT,
				  line_num, &cp_first, &cp_last, NULL, NULL))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		test_assert_idx(cp_data->composition_count == 0, cp);
	}
}

static void
test_derived_bidi_class_line(const char *line, unsigned int line_num,
			     const char *comment_data)
{
	uint32_t cp_first, cp_last, cp;
	const char *value, *comment_line = NULL;

	if (str_begins(comment_data, "@missing: ", &comment_line))
		line = comment_line;

	if (!parse_prop_file_line(line, UCD_DERIVED_BIDI_CLASS_TXT, line_num,
				  &cp_first, &cp_last, NULL, &value))
		return;

	enum unicode_bidi_class bidi_class;
	if (comment_line == NULL)
		bidi_class = unicode_bidi_class_from_string(value);
	else {
		if (cp_first == 0x0000 && cp_last == 0x10ffff) {
			/* In this test, we currently assume the specified
			   @missing ranges are disjoint apart from the first
			   one. */
			return;
		}
		bidi_class = unicode_bidi_class_from_string_long(value);
	}

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);
		if (comment_line != NULL &&
		    cp_data->general_category != UNICODE_GENERAL_CATEGORY_CN)
			continue;
		test_assert_idx(cp_data->bidi_class == bidi_class, cp);
	}
}

static void
test_derived_normalization_props_line(const char *line, unsigned int line_num,
				      const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *prop, *value;

	if (!parse_prop_file_line(line, UCD_DERIVED_NORMALIZATION_PROPS_TXT,
				  line_num, &cp_first, &cp_last, &prop, &value))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);
		uint8_t qc, qc_no, qc_maybe;

		if (strcmp(prop, "NFD_QC") == 0) {
			qc = (cp_data->nf_quick_check &
			      UNICODE_NFD_QUICK_CHECK_MASK);
			qc_no = UNICODE_NFD_QUICK_CHECK_NO;
			qc_maybe = UNICODE_NFD_QUICK_CHECK_MAYBE;
		} else if (strcmp(prop, "NFKD_QC") == 0) {
			qc = (cp_data->nf_quick_check &
			      UNICODE_NFKD_QUICK_CHECK_MASK);
			qc_no = UNICODE_NFKD_QUICK_CHECK_NO;
			qc_maybe = UNICODE_NFKD_QUICK_CHECK_MAYBE;
		} else if (strcmp(prop, "NFC_QC") == 0) {
			qc = (cp_data->nf_quick_check &
			      UNICODE_NFC_QUICK_CHECK_MASK);
			qc_no = UNICODE_NFC_QUICK_CHECK_NO;
			qc_maybe = UNICODE_NFC_QUICK_CHECK_MAYBE;
		} else if (strcmp(prop, "NFKC_QC") == 0) {
			qc = (cp_data->nf_quick_check &
			      UNICODE_NFKC_QUICK_CHECK_MASK);
			qc_no = UNICODE_NFKC_QUICK_CHECK_NO;
			qc_maybe = UNICODE_NFKC_QUICK_CHECK_MAYBE;
		} else {
			continue;
		}

		i_assert(value != NULL);
		if (strcmp(value, "N") == 0)
			test_assert_idx(qc == qc_no, cp);
		else if (strcmp(value, "M") == 0)
			test_assert_idx(qc == qc_maybe, cp);
	}
}

static void
test_derived_joining_type_line(const char *line, unsigned int line_num,
			       const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *value;

	if (!parse_prop_file_line(line, UCD_DERIVED_JOINING_TYPE_TXT, line_num,
				  &cp_first, &cp_last, NULL, &value))
		return;

	enum unicode_joining_type jtype =
		unicode_joining_type_from_string(value);

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		test_assert_idx(cp_data->joining_type == jtype, cp);
	}
}

static void
test_grapheme_break_property_line(const char *line, unsigned int line_num,
				  const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *prop;

	if (!parse_prop_file_line(line, UCD_GRAPHEME_BREAK_PROPERTY_TXT,
				  line_num, &cp_first, &cp_last, &prop, NULL))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		if (strcmp(prop, "CR") == 0)
			test_assert_idx(cp_data->pb_b_cr, cp);
		else if (strcmp(prop, "LF") == 0)
			test_assert_idx(cp_data->pb_b_lf, cp);
		else if (strcmp(prop, "Control") == 0)
			test_assert_idx(cp_data->pb_gcb_control, cp);
		else if (strcmp(prop, "Extend") == 0)
			test_assert_idx(cp_data->pb_gcb_extend, cp);
		else if (strcmp(prop, "ZWJ") == 0)
			test_assert_idx(cp_data->pb_b_zwj, cp);
		else if (strcmp(prop, "Regional_Indicator") == 0)
			test_assert_idx(cp_data->pb_b_regional_indicator, cp);
		else if (strcmp(prop, "Prepend") == 0)
			test_assert_idx(cp_data->pb_gcb_prepend, cp);
		else if (strcmp(prop, "SpacingMark") == 0)
			test_assert_idx(cp_data->pb_gcb_spacingmark, cp);
		else if (strcmp(prop, "L") == 0)
			test_assert_idx(cp_data->pb_gcb_l, cp);
		else if (strcmp(prop, "V") == 0)
			test_assert_idx(cp_data->pb_gcb_v, cp);
		else if (strcmp(prop, "T") == 0)
			test_assert_idx(cp_data->pb_gcb_t, cp);
		else if (strcmp(prop, "LV") == 0)
			test_assert_idx(cp_data->pb_gcb_lv, cp);
		else if (strcmp(prop, "LVT") == 0)
			test_assert_idx(cp_data->pb_gcb_lvt, cp);
	}
}

static void
test_prop_list_line(const char *line, unsigned int line_num,
		    const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *prop;

	if (!parse_prop_file_line(line, UCD_PROP_LIST_TXT, line_num,
				  &cp_first, &cp_last, &prop, NULL))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		if (strcmp(prop, "White_Space") == 0)
			test_assert_idx(cp_data->pb_g_white_space, cp);
		else if (strcmp(prop, "Join_Control") == 0)
			test_assert_idx(cp_data->pb_sr_join_control, cp);
		else if (strcmp(prop, "Pattern_White_Space") == 0)
			test_assert_idx(cp_data->pb_i_pattern_white_space, cp);
		else if (strcmp(prop, "Quotation_Mark") == 0)
			test_assert_idx(cp_data->pb_m_quotation_mark, cp);
		else if (strcmp(prop, "Dash") == 0)
			test_assert_idx(cp_data->pb_m_dash, cp);
		else if (strcmp(prop, "Sentence_Terminal") == 0)
			test_assert_idx(cp_data->pb_m_sentence_terminal, cp);
		else if (strcmp(prop, "Terminal_Punctuation") == 0)
			test_assert_idx(cp_data->pb_m_terminal_punctuation, cp);
	}
}

static void
test_scripts_line(const char *line, unsigned int line_num,
		  const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *value;

	if (!parse_prop_file_line(line, UCD_SCRIPTS_TXT, line_num,
				  &cp_first, &cp_last, NULL, &value))
		return;

	enum unicode_script script = unicode_script_from_string(value);

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		test_assert_idx(cp_data->script == script, cp);
	}
}

static void
test_special_casing_line(const char *line, unsigned int line_num,
			 const char *comment_data ATTR_UNUSED)
{
	if (*line == '\0')
		return;

	const char *const *columns = t_strsplit(line, ";");
	size_t num_columns = str_array_length(columns);

	/* <code>; <lower>; <title>; <upper>; (<condition_list>;)? */

	if (num_columns == 0)
		return;
	if (num_columns < 4) {
		test_failed(t_strdup_printf(
			"Invalid data at %s:%u",
			UCD_SPECIAL_CASING_TXT, line_num));
		return;
	}

	if (num_columns > 4 && strlen(t_str_trim(columns[4], " ")) > 0) {
		/* Skip lines with condition list */
		return;
	}

	const char *cp_hex = t_str_trim(columns[0], " ");
	uint32_t cp;

	if (str_to_uint32_hex(cp_hex, &cp) < 0) {
		test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad code point",
				UCD_SPECIAL_CASING_TXT, line_num));
		return;
	}

	/* Parse Decomposition_* */

	const char *lower = t_str_trim(columns[1], " ");
	const char *upper = t_str_trim(columns[3], " ");
	const char *const *lower_map = t_strsplit(lower, " ");
	const char *const *upper_map = t_strsplit(upper, " ");

	/* Check data */

	const struct unicode_code_point_data *cp_data =
		unicode_code_point_get_data(cp);
	const uint32_t *case_map;
	size_t case_map_len;

	case_map_len = unicode_code_point_data_get_uppercase_mapping(
		cp_data, &case_map);
	test_case_mapping(cp, upper_map, case_map, case_map_len);
	case_map_len = unicode_code_point_data_get_lowercase_mapping(
		cp_data, &case_map);
	test_case_mapping(cp, lower_map, case_map, case_map_len);
}

static void
test_unicode_data_line(const char *line, unsigned int line_num,
		       const char *comment_data ATTR_UNUSED)
{
	static uint32_t cp_first = 0;

	const char *const *columns = t_strsplit(line, ";");
	if (str_array_length(columns) < 15) {
		test_failed(t_strdup_printf(
			"Invalid data at %s:%u",
			UCD_UNICODE_DATA_TXT, line_num));
		return;
	}

	const char *cp_hex = columns[0];
	uint32_t cp;

	if (str_to_uint32_hex(cp_hex, &cp) < 0) {
		test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad code point",
				UCD_UNICODE_DATA_TXT, line_num));
		return;
	}

	/* Parse Name */

	const char *cp_name = columns[1];
	size_t cp_name_len = strlen(cp_name);
	const char *p;

	if (cp_name[0] == '<' && cp_name[cp_name_len - 1] == '>') {
		p = strchr(cp_name + 1, ',');
		if (p != NULL) {
			if (strcmp(p, ", First>") == 0) {
				cp_first = cp;
				return;
			} else if (strcmp(p, ", Last>") != 0) {
				test_failed(t_strdup_printf(
					"Invalid data at %s:%u: "
					"Bad code point range: %s",
					UCD_UNICODE_DATA_TXT, line_num, cp_name));
				return;
			}
		}
	}

	/* Parse General_Category */

	uint8_t general_category =
		(uint8_t)unicode_general_category_from_string(columns[2]);
	if (general_category == UNICODE_GENERAL_CATEGORY_INVALID) {
		test_failed(t_strdup_printf(
			    "Invalid data at %s:%u: "
			    "Bad General_Category for code point %"PRIu32": %s",
			    UCD_UNICODE_DATA_TXT, line_num, cp, columns[2]));
		return;
	}
	test_assert(!unicode_general_category_is_group(general_category));

	/* Parse Canonical_Combining_Class */

	unsigned int ccc = 0;
	if (*columns[3] != '\0' &&
	    (str_to_uint(columns[3], &ccc) < 0 || ccc > UINT8_MAX)) {
		test_failed(t_strdup_printf(
			    "Invalid data at %s:%u: "
			    "Bad Canonical_Combining_Class for code point %"PRIu32": %s",
			    UCD_UNICODE_DATA_TXT, line_num, cp, columns[3]));
		return;
	}

	/* Parse Decomposition_* */

	const char *decomp_spec = columns[5];
	enum unicode_decomposition_type decomp_type =
		UNICODE_DECOMPOSITION_TYPE_CANONICAL;

	if (*decomp_spec == '<') {
		const char *p = strchr(decomp_spec + 1, '>');

		if (p == NULL || *(p + 1) != ' ') {
			test_failed(t_strdup_printf(
				"Invalid data at %s:%u: "
				"Bad Decomposition for code point %"PRIu32": %s",
				UCD_UNICODE_DATA_TXT, line_num, cp, columns[5]));
			return;
		}
		decomp_type = unicode_decomposition_type_from_string(
			t_strdup_until(decomp_spec + 1, p));
		decomp_spec = p + 2;
	}

	const char *const *decomp = t_strsplit(decomp_spec, " ");

	/* Parse Simple_*case_Mapping */

	uint32_t simple_uppercase_mapping = 0;
	uint32_t simple_lowercase_mapping = 0;
	uint32_t simple_titlecase_mapping = 0;

	if (*columns[12] != '\0' &&
	    str_to_uint32_hex(columns[12], &simple_uppercase_mapping) < 0) {
		test_failed(t_strdup_printf(
			    "Invalid data at %s:%u: "
			    "Bad Simple_Uppercase_Mapping for code point %"PRIu32": %s",
			    UCD_UNICODE_DATA_TXT, line_num, cp, columns[12]));
		return;
	}
	if (*columns[13] != '\0' &&
	    str_to_uint32_hex(columns[13], &simple_lowercase_mapping) < 0) {
		test_failed(t_strdup_printf(
			    "Invalid data at %s:%u: "
			    "Bad Simple_Lowercase_Mapping for code point %"PRIu32": %s",
			    UCD_UNICODE_DATA_TXT, line_num, cp, columns[13]));
		return;
	}
	if (*columns[14] != '\0' &&
	    str_to_uint32_hex(columns[14], &simple_titlecase_mapping) < 0) {
		test_failed(t_strdup_printf(
			    "Invalid data at %s:%u: "
			    "Bad Simple_Titlecase_Mapping for code point %"PRIu32": %s",
			    UCD_UNICODE_DATA_TXT, line_num, cp, columns[14]));
		return;
	}

	/* Check data */

	uint32_t cp_last = cp;

	if (cp_first == 0)
		cp_first = cp;
	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		test_assert_idx(
			cp_data->general_category == general_category, cp);
		test_assert_idx(
			cp_data->canonical_combining_class == ccc, cp);

		const uint32_t *cp_decomp;
		size_t cp_decomp_len, cp_decomp_idx;
		uint8_t cp_decomp_type;

		cp_decomp_len =
			unicode_code_point_data_get_first_decomposition(
				cp_data, &cp_decomp_type, &cp_decomp);
		test_assert(str_array_length(decomp) == cp_decomp_len);
		if (test_has_failed())
			break;

		test_assert_idx(
			(cp_decomp_type == decomp_type ||
			 cp_decomp_type == UNICODE_DECOMPOSITION_TYPE_COMPAT),
			cp);
		cp_decomp_idx = 0;
		while (*decomp != NULL && !test_has_failed()) {
			uint32_t dcp;

			test_assert_idx(str_to_uint32_hex(*decomp, &dcp) >= 0, cp);
			if (test_has_failed())
				break;
			test_assert_idx(uni_is_valid_ucs4(dcp), cp);
			test_assert_idx(dcp == cp_decomp[cp_decomp_idx], cp);

			cp_decomp_idx++;
			decomp++;
		}

		if (cp_data->uppercase_mapping_length == 1) {
			const uint32_t *map;
			size_t map_len =
				unicode_code_point_data_get_uppercase_mapping(
					cp_data, &map);
			test_assert_idx(map_len == 1 &&
					map[0] == simple_uppercase_mapping, cp);
		}
		if (cp_data->lowercase_mapping_length == 1) {
			const uint32_t *map;
			size_t map_len =
				unicode_code_point_data_get_lowercase_mapping(
					cp_data, &map);
			test_assert_idx(map_len == 1 &&
					map[0] == simple_lowercase_mapping, cp);
		}
		test_assert_idx(
			cp_data->simple_titlecase_mapping == simple_titlecase_mapping,
			cp);
	}

	cp_first = 0;
}

static void
test_word_break_property_line(const char *line, unsigned int line_num,
			      const char *comment_data ATTR_UNUSED)
{
	uint32_t cp_first, cp_last, cp;
	const char *prop;

	if (!parse_prop_file_line(line, UCD_WORD_BREAK_PROPERTY_TXT, line_num,
				  &cp_first, &cp_last, &prop, NULL))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		if (strcmp(prop, "CR") == 0)
			test_assert_idx(cp_data->pb_b_cr, cp);
		else if (strcmp(prop, "LF") == 0)
			test_assert_idx(cp_data->pb_b_lf, cp);
		else if (strcmp(prop, "Newline") == 0)
			test_assert_idx(cp_data->pb_wb_newline, cp);
		else if (strcmp(prop, "Extend") == 0)
			test_assert_idx(cp_data->pb_wb_extend, cp);
		else if (strcmp(prop, "ZWJ") == 0)
			test_assert_idx(cp_data->pb_b_zwj, cp);
		else if (strcmp(prop, "Regional_Indicator") == 0)
			test_assert_idx(cp_data->pb_b_regional_indicator, cp);
		else if (strcmp(prop, "Format") == 0)
			test_assert_idx(cp_data->pb_wb_format, cp);
		else if (strcmp(prop, "Katakana") == 0)
			test_assert_idx(cp_data->pb_wb_katakana, cp);
		else if (strcmp(prop, "Hebrew_Letter") == 0)
			test_assert_idx(cp_data->pb_wb_hebrew_letter, cp);
		else if (strcmp(prop, "ALetter") == 0)
			test_assert_idx(cp_data->pb_wb_aletter, cp);
		else if (strcmp(prop, "Single_Quote") == 0)
			test_assert_idx(cp_data->pb_wb_single_quote, cp);
		else if (strcmp(prop, "Double_Quote") == 0)
			test_assert_idx(cp_data->pb_wb_double_quote, cp);
		else if (strcmp(prop, "MidNumLet") == 0)
			test_assert_idx(cp_data->pb_wb_midnumlet, cp);
		else if (strcmp(prop, "MidLetter") == 0)
			test_assert_idx(cp_data->pb_wb_midletter, cp);
		else if (strcmp(prop, "MidNum") == 0)
			test_assert_idx(cp_data->pb_wb_midnum, cp);
		else if (strcmp(prop, "Numeric") == 0)
			test_assert_idx(cp_data->pb_wb_numeric, cp);
		else if (strcmp(prop, "ExtendNumLet") == 0)
			test_assert_idx(cp_data->pb_wb_extendnumlet, cp);
	}
}

static void
test_idna_mapping_table_line(const char *line, unsigned int line_num,
			     const char *comment_data ATTR_UNUSED)
{
	if (*line == '\0')
		return;

	const char *const *columns = t_strsplit(line, ";");
	size_t num_columns = str_array_length(columns);

	if (num_columns == 0)
		return;
	if (num_columns < 2) {
		test_failed(t_strdup_printf(
			"Invalid data at %s:%u",
			UCD_IDNA_MAPPING_TABLE_TXT, line_num));
		return;
	}

	uint32_t cp_first, cp_last, cp;
	const char *status, *value;

	if (!parse_prop_file_line(line, UCD_IDNA_MAPPING_TABLE_TXT, line_num,
				  &cp_first, &cp_last, &status, &value))
		return;

	for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
		const struct unicode_code_point_data *cp_data =
			unicode_code_point_get_data(cp);

		switch (cp_data->idna_status) {
		case UNICODE_IDNA_STATUS_DISALLOWED:
			test_assert_strcmp_idx(status, "disallowed", line_num);
			break;
		case UNICODE_IDNA_STATUS_VALID:
			test_assert_strcmp_idx(status, "valid", line_num);
			break;
		case UNICODE_IDNA_STATUS_IGNORED:
			test_assert_strcmp_idx(status, "ignored", line_num);
			break;
		case UNICODE_IDNA_STATUS_MAPPED:
			test_assert_strcmp_idx(status, "mapped", line_num);
			break;
		case UNICODE_IDNA_STATUS_DEVIATION:
			test_assert_strcmp_idx(status, "deviation", line_num);
			break;
		}

		if (strcmp(status, "mapping") != 0 &&
		    strcmp(status, "deviation") != 0)
			continue;

		const char *mapping = t_str_trim(value, " ");
		const char *const *map = t_strsplit(mapping, " ");

		/* Check data */

		const uint32_t *idna_map =
			&unicode_idna_mappings[cp_data->idna_mapping_offset];
		unsigned int idna_map_len = cp_data->idna_mapping_length;

		test_case_mapping(cp, map, idna_map, idna_map_len);
	}
}

static void
test_ucd_file(const char *filename,
	      void (*test_line)(const char *line, unsigned int line_num,
				const char *comment_data))
{
	const char *file_path = t_strconcat(UCD_DIR, "/", filename, NULL);
	struct istream *input;
	int fd;

	fd = open(file_path, O_RDONLY);
	if (fd < 0)
		i_fatal("Failed to open '%s': %m", file_path);

	test_begin(t_strdup_printf("unicode_data - %s", filename));

	input = i_stream_create_fd_autoclose(&fd, 1024);

	unsigned int line_num = 0;

	while (!test_has_failed()) {
		char *line = i_stream_read_next_line(input);

		if (line == NULL)
			break;
		line_num++;

		char *comment = strchr(line, '#');
		const char *comment_data = NULL;

		if (comment != NULL) {
			comment_data = t_str_trim(comment + 1, " ");
			*comment = '\0';
		}
		if (*line == '\0' && comment_data == NULL)
			continue;

		T_BEGIN {
			test_line(line, line_num, comment_data);
		} T_END;
	}

	i_stream_destroy(&input);
	test_end();
}

void test_unicode_data(void)
{
	/* Check that UCD data files match with what is compiled. For the
	   property files only the positive assignment of properties to the
	   code points mentioned in the files is tested, and notably not their
	   absence for other code points. */
	test_ucd_file(UCD_CASE_FOLDING_TXT, test_case_folding_line);
	test_ucd_file(UCD_COMPOSITION_EXCLUSIONS_TXT,
		      test_composition_exclusions_line);
	test_ucd_file(UCD_DERIVED_BIDI_CLASS_TXT, test_derived_bidi_class_line);
	test_ucd_file(UCD_DERIVED_JOINING_TYPE_TXT,
		      test_derived_joining_type_line);
	test_ucd_file(UCD_DERIVED_NORMALIZATION_PROPS_TXT,
		      test_derived_normalization_props_line);
	test_ucd_file(UCD_GRAPHEME_BREAK_PROPERTY_TXT,
		      test_grapheme_break_property_line);
	test_ucd_file(UCD_PROP_LIST_TXT, test_prop_list_line);
	test_ucd_file(UCD_SCRIPTS_TXT, test_scripts_line);
	test_ucd_file(UCD_SPECIAL_CASING_TXT, test_special_casing_line);
	test_ucd_file(UCD_UNICODE_DATA_TXT, test_unicode_data_line);
	test_ucd_file(UCD_WORD_BREAK_PROPERTY_TXT,
		      test_word_break_property_line);
	test_ucd_file(UCD_IDNA_MAPPING_TABLE_TXT,
		      test_idna_mapping_table_line);
}