generic/tclEncoding.c - main - niceSOFT/tcl

generic/tclEncoding.c
5 740 строк165 KB
dkf
Minor change: Some more indentation corrections
21 июн 2026, 19:33
21 июн 2026, 19:331cfaa53
/*
 * tclEncoding.c --
 *
 *	Contains the implementation of the encoding conversion package.
 *
 * Copyright © 1996-1998 Sun Microsystems, Inc.
 *
 * See the file "license.terms" for information on usage and redistribution of
 * this file, and for a DISCLAIMER OF ALL WARRANTIES.
 */

#include "tclInt.h"
#include "../utf8proc/utf8proc.h" /* Relative path to ignore system include */

typedef size_t (LengthProc)(const char *src);

/*
 * The following data structure represents an encoding, which describes how to
 * convert between various character sets and UTF-8.
 */

typedef struct {
    char *name;			/* Name of encoding. Malloced because (1) hash
				 * table entry that owns this encoding may be
				 * freed prior to this encoding being freed,
				 * (2) string passed in the Tcl_EncodingType
				 * structure may not be persistent. */
    Tcl_EncodingConvertProc *toUtfProc;
				/* Function to convert from external encoding
				 * into UTF-8. */
    Tcl_EncodingConvertProc *fromUtfProc;
				/* Function to convert from UTF-8 into
				 * external encoding. */
    Tcl_EncodingFreeProc *freeProc;
				/* If non-NULL, function to call when this
				 * encoding is deleted. */
    void *clientData;		/* Arbitrary value associated with encoding
				 * type. Passed to conversion functions. */
    Tcl_Size nullSize;		/* Number of 0x00 bytes that signify
				 * end-of-string in this encoding. This number
				 * is used to determine the source string
				 * length when the srcLen argument is
				 * negative. This number can be 1, 2, or 4. */
    LengthProc *lengthProc;	/* Function to compute length of
				 * null-terminated strings in this encoding.
				 * If nullSize is 1, this is strlen; if
				 * nullSize is 2, this is a function that
				 * returns the number of bytes in a 0x0000
				 * terminated string; if nullSize is 4, this
				 * is a function that returns the number of
				 * bytes in a 0x00000000 terminated string. */
    size_t refCount;		/* Number of uses of this structure. */
    Tcl_HashEntry *hPtr;	/* Hash table entry that owns this encoding. */
} Encoding;

/*
 * The following structure is the clientData for a dynamically-loaded,
 * table-driven encoding created by LoadTableEncoding(). It maps between
 * Unicode and a single-byte, double-byte, or multibyte (1 or 2 bytes only)
 * encoding.
 */

typedef struct {
    int fallback;		/* Character (in this encoding) to substitute
				 * when this encoding cannot represent a UTF-8
				 * character. */
    char prefixBytes[256];	/* If a byte in the input stream is a lead
				 * byte for a 2-byte sequence, the
				 * corresponding entry in this array is 1,
				 * otherwise it is 0. */
    unsigned short **toUnicode;	/* Two dimensional sparse matrix to map
				 * characters from the encoding to Unicode.
				 * Each element of the toUnicode array points
				 * to an array of 256 shorts. If there is no
				 * corresponding character in Unicode, the
				 * value in the matrix is 0x0000.
				 * malloc'd. */
    unsigned short **fromUnicode;
				/* Two dimensional sparse matrix to map
				 * characters from Unicode to the encoding.
				 * Each element of the fromUnicode array
				 * points to an array of 256 shorts. If there
				 * is no corresponding character the encoding,
				 * the value in the matrix is 0x0000.
				 * malloc'd. */
} TableEncodingData;

/*
 * Each of the following structures is the clientData for a dynamically-loaded
 * escape-driven encoding that is itself comprised of other simpler encodings.
 * An example is "iso-2022-jp", which uses escape sequences to switch between
 * ascii, jis0208, jis0212, gb2312, and ksc5601. Note that "escape-driven"
 * does not necessarily mean that the ESCAPE character is the character used
 * for switching character sets.
 */

typedef struct {
    unsigned sequenceLen;	/* Length of following string. */
    char sequence[16];		/* Escape code that marks this encoding. */
    char name[32];		/* Name for encoding. */
    Encoding *encodingPtr;	/* Encoding loaded using above name, or NULL
				 * if this sub-encoding has not been needed
				 * yet. */
} EscapeSubTable;

typedef struct {
    int fallback;		/* Character (in this encoding) to substitute
				 * when this encoding cannot represent a UTF-8
				 * character. */
    unsigned initLen;		/* Length of following string. */
    char init[16];		/* String to emit or expect before first char
				 * in conversion. */
    unsigned finalLen;		/* Length of following string. */
    char final[16];		/* String to emit or expect after last char in
				 * conversion. */
    char prefixBytes[256];	/* If a byte in the input stream is the first
				 * character of one of the escape sequences in
				 * the following array, the corresponding
				 * entry in this array is 1, otherwise it is
				 * 0. */
    int numSubTables;		/* Length of following array. */
    EscapeSubTable subTables[TCLFLEXARRAY];
				/* Information about each EscapeSubTable used
				 * by this encoding type. The actual size is
				 * as large as necessary to hold all
				 * EscapeSubTables. */
} EscapeEncodingData;

/*
 * Values used when loading an encoding file to identify the type of the
 * file.
 */
enum EncodingTypes {
    ENCODING_SINGLEBYTE = 0,	/* Encoding is single byte per character. */
    ENCODING_DOUBLEBYTE = 1,	/* Encoding is two bytes per character. */
    ENCODING_MULTIBYTE = 2,	/* Encoding is variable bytes per character. */
    ENCODING_ESCAPE = 3		/* Encoding has modes with escapes to move
				 * between them. */
};

/*
 * A list of directories in which Tcl should look for *.enc files. This list
 * is shared by all threads. Access is governed by a mutex lock.
 */

static TclInitProcessGlobalValueProc InitializeEncodingSearchPath;
static ProcessGlobalValue encodingSearchPath = {
    0, 0, NULL, NULL, InitializeEncodingSearchPath, NULL, NULL
};

/*
 * A map from encoding names to the directories in which their data files have
 * been seen. The string value of the map is shared by all threads. Access to
 * the shared string is governed by a mutex lock.
 */

static ProcessGlobalValue encodingFileMap = {
    0, 0, NULL, NULL, NULL, NULL, NULL
};

/*
 * A list of directories making up the "library path". Historically this
 * search path has served many uses, but the only one remaining is a base for
 * the encodingSearchPath above. If the application does not explicitly set
 * the encodingSearchPath, then it is initialized by appending /encoding
 * to each directory in this "libraryPath".
 */

static ProcessGlobalValue libraryPath = {
    0, 0, NULL, NULL, TclpInitLibraryPath, NULL, NULL
};

static int encodingsInitialized = 0;

/*
 * Hash table that keeps track of all loaded Encodings. Keys are the string
 * names that represent the encoding, values are (Encoding *).
 */

static Tcl_HashTable encodingTable;
TCL_DECLARE_MUTEX(encodingMutex)

/*
 * The following are used to hold the default and current system encodings.
 * If NULL is passed to one of the conversion routines, the current setting of
 * the system encoding is used to perform the conversion.
 */

static Tcl_Encoding defaultEncoding = NULL;
static Tcl_Encoding systemEncoding = NULL;
Tcl_Encoding tclIdentityEncoding = NULL;
Tcl_Encoding tclUtf8Encoding = NULL;

/*
 * Names of encoding profiles and corresponding integer values.
 * Keep alphabetical order for error messages.
 */
static const struct TclEncodingProfiles {
    const char *name;
    int value;
} encodingProfiles[] = {
    {"replace", TCL_ENCODING_PROFILE_REPLACE},
    {"strict", TCL_ENCODING_PROFILE_STRICT},
    {"tcl8", TCL_ENCODING_PROFILE_TCL8},
};

#define PROFILE_TCL8(flags_) \
    (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8)

#define PROFILE_REPLACE(flags_) \
    (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE)

#define PROFILE_STRICT(flags_) \
    (!PROFILE_TCL8(flags_) && !PROFILE_REPLACE(flags_))

#define UNICODE_REPLACE_CHAR 0xFFFD
#define SURROGATE(c_)		(((c_) & ~0x7FF) == 0xD800)
#define HIGH_SURROGATE(c_)	(((c_) & ~0x3FF) == 0xD800)
#define LOW_SURROGATE(c_)	(((c_) & ~0x3FF) == 0xDC00)

/*
 * The following variable is used in the sparse matrix code for a
 * TableEncoding to represent a page in the table that has no entries.
 */

static unsigned short emptyPage[256];

/*
 * Functions used only in this module.
 */

static Tcl_EncodingConvertProc	BinaryProc;
static Tcl_DupInternalRepProc	DupEncodingInternalRep;
static Tcl_EncodingFreeProc	EscapeFreeProc;
static Tcl_EncodingConvertProc	EscapeFromUtfProc;
static Tcl_EncodingConvertProc	EscapeToUtfProc;
static void			FillEncodingFileMap(void);
static void			FreeEncoding(Tcl_Encoding encoding);
static Tcl_FreeInternalRepProc	FreeEncodingInternalRep;
static Encoding *		GetTableEncoding(EscapeEncodingData *dataPtr,
				    int state);
static Tcl_Encoding		LoadEncodingFile(Tcl_Interp *interp,
				    const char *name);
static Tcl_Encoding		LoadTableEncoding(const char *name, int type,
				    Tcl_Channel chan);
static Tcl_Encoding		LoadEscapeEncoding(const char *name,
				    Tcl_Channel chan);
static Tcl_Channel		OpenEncodingFileChannel(Tcl_Interp *interp,
				    const char *name);
static Tcl_EncodingFreeProc	TableFreeProc;
static Tcl_EncodingConvertProc	TableFromUtfProc;
static Tcl_EncodingConvertProc	TableToUtfProc;
static size_t		unilen(const char *src);
static size_t		unilen4(const char *src);
static Tcl_EncodingConvertProc	Utf32ToUtfProc;
static Tcl_EncodingConvertProc	UtfToUtf32Proc;
static Tcl_EncodingConvertProc	Utf16ToUtfProc;
static Tcl_EncodingConvertProc	UtfToUtf16Proc;
static Tcl_EncodingConvertProc	UtfToUcs2Proc;
static Tcl_EncodingConvertProc	UtfToUtfProc;
static Tcl_EncodingConvertProc	UtfToCesu8Proc;
static Tcl_EncodingConvertProc	Cesu8ToUtfProc;
static Tcl_EncodingConvertProc	Iso88591FromUtfProc;
static Tcl_EncodingConvertProc	Iso88591ToUtfProc;

/*
 * A Tcl_ObjType for holding a cached Tcl_Encoding in the twoPtrValue.ptr1
 * field of the internalrep. This should help the lifetime of encodings be more
 * useful. See concerns raised in [Bug 1077262].
 */

static const Tcl_ObjType encodingType = {
    "encoding",
    FreeEncodingInternalRep,
    DupEncodingInternalRep,
    NULL,			// UpdateString
    NULL,			// SetFromAny
    TCL_OBJTYPE_V0
};

#define EncodingSetInternalRep(objPtr, encoding) \
    do {								\
	Tcl_ObjInternalRep ir;						\
	ir.twoPtrValue.ptr1 = (encoding);				\
	ir.twoPtrValue.ptr2 = NULL;					\
	Tcl_StoreInternalRep((objPtr), &encodingType, &ir);		\
    } while (0)

#define EncodingGetInternalRep(objPtr, encoding) \
    do {								\
	const Tcl_ObjInternalRep *irPtr;				\
	irPtr = TclFetchInternalRep ((objPtr), &encodingType);		\
	(encoding) = irPtr ? (Tcl_Encoding)irPtr->twoPtrValue.ptr1 : NULL; \
    } while (0)

/*
 *----------------------------------------------------------------------
 *
 * Tcl_GetEncodingFromObj --
 *
 *	Writes to (*encodingPtr) the Tcl_Encoding value of (*objPtr), if
 *	possible, and returns TCL_OK. If no such encoding exists, TCL_ERROR is
 *	returned, and if interp is non-NULL, an error message is written
 *	there.
 *
 * Results:
 *	Standard Tcl return code.
 *
 * Side effects:
 *	Caches the Tcl_Encoding value as the internal rep of (*objPtr).
 *
 *----------------------------------------------------------------------
 */

int
Tcl_GetEncodingFromObj(
    Tcl_Interp *interp,
    Tcl_Obj *objPtr,
    Tcl_Encoding *encodingPtr)
{
    Tcl_Encoding encoding;
    const char *name = TclGetString(objPtr);

    EncodingGetInternalRep(objPtr, encoding);
    if (encoding == NULL) {
	encoding = Tcl_GetEncoding(interp, name);
	if (encoding == NULL) {
	    return TCL_ERROR;
	}
	EncodingSetInternalRep(objPtr, encoding);
    }
    *encodingPtr = Tcl_GetEncoding(NULL, name);
    return TCL_OK;
}

/*
 *----------------------------------------------------------------------
 *
 * FreeEncodingInternalRep --
 *
 *	The Tcl_FreeInternalRepProc for the "encoding" Tcl_ObjType.
 *
 *----------------------------------------------------------------------
 */

static void
FreeEncodingInternalRep(
    Tcl_Obj *objPtr)
{
    Tcl_Encoding encoding;

    EncodingGetInternalRep(objPtr, encoding);
    Tcl_FreeEncoding(encoding);
}

/*
 *----------------------------------------------------------------------
 *
 * DupEncodingInternalRep --
 *
 *	The Tcl_DupInternalRepProc for the "encoding" Tcl_ObjType.
 *
 *----------------------------------------------------------------------
 */

static void
DupEncodingInternalRep(
    Tcl_Obj *srcPtr,
    Tcl_Obj *dupPtr)
{
    Tcl_Encoding encoding = Tcl_GetEncoding(NULL, TclGetString(srcPtr));
    EncodingSetInternalRep(dupPtr, encoding);
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_GetEncodingSearchPath --
 *
 *	Keeps the per-thread copy of the encoding search path current with
 *	changes to the global copy.
 *
 * Results:
 *	Returns a "list" (Tcl_Obj *) that contains the encoding search path.
 *
 *----------------------------------------------------------------------
 */

Tcl_Obj *
Tcl_GetEncodingSearchPath(void)
{
    return TclGetProcessGlobalValue(&encodingSearchPath);
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_SetEncodingSearchPath --
 *
 *	Keeps the per-thread copy of the encoding search path current with
 *	changes to the global copy.
 *
 *----------------------------------------------------------------------
 */

int
Tcl_SetEncodingSearchPath(
    Tcl_Obj *searchPath)
{
    Tcl_Size dummy;

    if (TCL_ERROR == TclListObjLength(NULL, searchPath, &dummy)) {
	return TCL_ERROR;
    }
    TclSetProcessGlobalValue(&encodingSearchPath, searchPath);
    return TCL_OK;
}

/*
 *---------------------------------------------------------------------------
 *
 * FillEncodingFileMap --
 *
 *	Called to update the encoding file map with the current value
 *	of the encoding search path.
 *
 *	Finds *.end files in the directories on the encoding search path and
 *	stores the found pathnames in a map associated with the encoding name.
 *
 *	If $dir is on the encoding search path and the file $dir/foo.enc is
 *	found, stores a "foo" -> $dir entry in the map.  if the "foo" encoding
 *	is needed later, the $dir/foo.enc name can be quickly constructed in
 *	order to read the encoding data.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Entries are added to the encoding file map.
 *
 *---------------------------------------------------------------------------
 */

static void
FillEncodingFileMap(void)
{
    Tcl_Size i, numDirs = 0;
    Tcl_Obj *map, *searchPath;

    searchPath = Tcl_GetEncodingSearchPath();
    Tcl_IncrRefCount(searchPath);
    TclListObjLength(NULL, searchPath, &numDirs);
    map = Tcl_NewDictObj();
    Tcl_IncrRefCount(map);

    for (i = numDirs-1; i != TCL_INDEX_NONE; i--) {
	/*
	 * Iterate backwards through the search path so as we overwrite
	 * entries found, we favor files earlier on the search path.
	 */

	Tcl_Size j, numFiles;
	Tcl_Obj *directory, *matchFileList;
	Tcl_Obj **filev;
	Tcl_GlobTypeData readableFiles = {
	    TCL_GLOB_TYPE_FILE, TCL_GLOB_PERM_R, NULL, NULL
	};

	TclNewObj(matchFileList);
	Tcl_ListObjIndex(NULL, searchPath, i, &directory);
	Tcl_IncrRefCount(directory);
	Tcl_IncrRefCount(matchFileList);
	Tcl_FSMatchInDirectory(NULL, matchFileList, directory, "*.enc",
		&readableFiles);

	TclListObjGetElements(NULL, matchFileList, &numFiles, &filev);
	for (j=0; j<numFiles; j++) {
	    Tcl_Obj *encoding, *fileObj;

	    fileObj = TclPathPart(NULL, filev[j], TCL_PATH_TAIL);
	    encoding = TclPathPart(NULL, fileObj, TCL_PATH_ROOT);
	    Tcl_DictObjPut(NULL, map, encoding, directory);
	    Tcl_DecrRefCount(fileObj);
	    Tcl_DecrRefCount(encoding);
	}
	Tcl_DecrRefCount(matchFileList);
	Tcl_DecrRefCount(directory);
    }
    Tcl_DecrRefCount(searchPath);
    TclSetProcessGlobalValue(&encodingFileMap, map);
    Tcl_DecrRefCount(map);
}

/*
 *---------------------------------------------------------------------------
 *
 * TclInitEncodingSubsystem --
 *
 *	Initialize all resources used by this subsystem on a per-process
 *	basis.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Depends on the memory, object, and IO subsystems.
 *
 *---------------------------------------------------------------------------
 */

/*
 * NOTE: THESE BIT DEFINITIONS SHOULD NOT OVERLAP WITH INTERNAL USE BITS
 * DEFINED IN tcl.h (TCL_ENCODING_* et al). Be cognizant of this
 * when adding bits. TODO - should really be defined in a single file.
 *
 * To prevent conflicting bits, only define bits within 0xff00 mask here.
 */
enum InternalEncodingFlags {
    TCL_ENCODING_LE = 0x100,	/* Used to distinguish LE/BE variants */
    ENCODING_INPUT = 0x400	/* Only affects UTF-8 encoder. If set,
				 * external (standard UTF-8) ->
				 * internal (Tcl TUTF-8). If unset, the
				 * reverse */
};

void
TclInitEncodingSubsystem(void)
{
    Tcl_EncodingType type;
    TableEncodingData *dataPtr;
    unsigned size;
    unsigned short i;
    union {
	char c;
	short s;
    } isLe;
    int leFlags;

    if (encodingsInitialized) {
	return;
    }

    /* Note: This DEPENDS on TCL_ENCODING_LE being defined in least sig byte */
    isLe.s = 1;
    leFlags = isLe.c ? TCL_ENCODING_LE : 0;

    Tcl_MutexLock(&encodingMutex);
    Tcl_InitHashTable(&encodingTable, TCL_STRING_KEYS);
    Tcl_MutexUnlock(&encodingMutex);

    /*
     * Create a few initial encodings.  UTF-8 to UTF-8 translation is not a
     * no-op because it turns a stream of improperly formed UTF-8 into a
     * properly formed stream.
     */

    type.encodingName	= NULL;
    type.toUtfProc	= BinaryProc;
    type.fromUtfProc	= BinaryProc;
    type.freeProc	= NULL;
    type.nullSize	= 1;
    type.clientData	= NULL;
    tclIdentityEncoding = Tcl_CreateEncoding(&type);

    type.encodingName	= "utf-8";
    type.toUtfProc	= UtfToUtfProc;
    type.fromUtfProc	= UtfToUtfProc;
    type.freeProc	= NULL;
    type.nullSize	= 1;
    type.clientData	= NULL;
    tclUtf8Encoding = Tcl_CreateEncoding(&type);

    type.encodingName	= "cesu-8";
    type.toUtfProc	= Cesu8ToUtfProc;
    type.fromUtfProc	= UtfToCesu8Proc;
    type.freeProc	= NULL;
    type.nullSize	= 1;
    type.clientData	= NULL;
    Tcl_CreateEncoding(&type);

    type.toUtfProc	= Utf16ToUtfProc;
    type.fromUtfProc	= UtfToUcs2Proc;
    type.freeProc	= NULL;
    type.nullSize	= 2;
    type.encodingName	= "ucs-2le";
    type.clientData	= INT2PTR(TCL_ENCODING_LE);
    Tcl_CreateEncoding(&type);
    type.encodingName	= "ucs-2be";
    type.clientData	= NULL;
    Tcl_CreateEncoding(&type);
    type.encodingName	= "ucs-2";
    type.clientData	= INT2PTR(leFlags);
    Tcl_CreateEncoding(&type);

    type.toUtfProc	= Utf32ToUtfProc;
    type.fromUtfProc	= UtfToUtf32Proc;
    type.freeProc	= NULL;
    type.nullSize	= 4;
    type.encodingName	= "utf-32le";
    type.clientData	= INT2PTR(TCL_ENCODING_LE);
    Tcl_CreateEncoding(&type);
    type.encodingName	= "utf-32be";
    type.clientData	= NULL;
    Tcl_CreateEncoding(&type);
    type.encodingName	= "utf-32";
    type.clientData	= INT2PTR(leFlags);
    Tcl_CreateEncoding(&type);

    type.toUtfProc	= Utf16ToUtfProc;
    type.fromUtfProc    = UtfToUtf16Proc;
    type.freeProc	= NULL;
    type.nullSize	= 2;
    type.encodingName	= "utf-16le";
    type.clientData	= INT2PTR(TCL_ENCODING_LE);
    Tcl_CreateEncoding(&type);
    type.encodingName	= "utf-16be";
    type.clientData	= NULL;
    Tcl_CreateEncoding(&type);
    type.encodingName	= "utf-16";
    type.clientData	= INT2PTR(leFlags);
    Tcl_CreateEncoding(&type);

#ifndef TCL_NO_DEPRECATED
    type.encodingName	= "unicode";
    Tcl_CreateEncoding(&type);
#endif

    /*
     * Need the iso8859-1 encoding in order to process binary data, so force
     * it to always be embedded. Note that this encoding *must* be a proper
     * table encoding or some of the escape encodings crash! Hence the ugly
     * code to duplicate the structure of a table encoding here.
     */

    dataPtr = (TableEncodingData *)Tcl_Alloc(sizeof(TableEncodingData));
    memset(dataPtr, 0, sizeof(TableEncodingData));
    dataPtr->fallback = '?';

    size = 256*(sizeof(unsigned short *) + sizeof(unsigned short));
    dataPtr->toUnicode = (unsigned short **)Tcl_Alloc(size);
    memset(dataPtr->toUnicode, 0, size);
    dataPtr->fromUnicode = (unsigned short **)Tcl_Alloc(size);
    memset(dataPtr->fromUnicode, 0, size);

    dataPtr->toUnicode[0] = (unsigned short *) (dataPtr->toUnicode + 256);
    dataPtr->fromUnicode[0] = (unsigned short *) (dataPtr->fromUnicode + 256);
    for (i=1 ; i<256 ; i++) {
	dataPtr->toUnicode[i] = emptyPage;
	dataPtr->fromUnicode[i] = emptyPage;
    }

    for (i=0 ; i<256 ; i++) {
	dataPtr->toUnicode[0][i] = i;
	dataPtr->fromUnicode[0][i] = i;
    }

    type.encodingName	= "iso8859-1";
    type.toUtfProc	= Iso88591ToUtfProc;
    type.fromUtfProc	= Iso88591FromUtfProc;
    type.freeProc	= TableFreeProc;
    type.nullSize	= 1;
    type.clientData	= dataPtr;
    defaultEncoding	= Tcl_CreateEncoding(&type);
    systemEncoding	= Tcl_GetEncoding(NULL, type.encodingName);

    encodingsInitialized = 1;
}

/*
 *----------------------------------------------------------------------
 *
 * TclFinalizeEncodingSubsystem --
 *
 *	Release the state associated with the encoding subsystem.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Frees all of the encodings.
 *
 *----------------------------------------------------------------------
 */

void
TclFinalizeEncodingSubsystem(void)
{
    Tcl_HashSearch search;
    Tcl_HashEntry *hPtr;

    Tcl_MutexLock(&encodingMutex);
    encodingsInitialized = 0;
    FreeEncoding(systemEncoding);
    systemEncoding = NULL;
    defaultEncoding = NULL;
    FreeEncoding(tclIdentityEncoding);
    tclIdentityEncoding = NULL;
    FreeEncoding(tclUtf8Encoding);
    tclUtf8Encoding = NULL;

    hPtr = Tcl_FirstHashEntry(&encodingTable, &search);
    while (hPtr != NULL) {
	/*
	 * Call FreeEncoding instead of doing it directly to handle refcounts
	 * like escape encodings use. [Bug 524674] Make sure to call
	 * Tcl_FirstHashEntry repeatedly so that all encodings are eventually
	 * cleaned up.
	 */

	FreeEncoding((Tcl_Encoding)Tcl_GetHashValue(hPtr));
	hPtr = Tcl_FirstHashEntry(&encodingTable, &search);
    }

    Tcl_DeleteHashTable(&encodingTable);
    Tcl_MutexUnlock(&encodingMutex);
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_GetEncoding --
 *
 *	Given the name of a encoding, find the corresponding Tcl_Encoding
 *	token. If the encoding did not already exist, Tcl attempts to
 *	dynamically load an encoding by that name.
 *
 * Results:
 *	Returns a token that represents the encoding. If the name didn't refer
 *	to any known or loadable encoding, NULL is returned. If NULL was
 *	returned, an error message is left in interp's result object, unless
 *	interp was NULL.
 *
 * Side effects:
 *	LoadEncodingFile is called if necessary.
 *
 *-------------------------------------------------------------------------
 */

Tcl_Encoding
Tcl_GetEncoding(
    Tcl_Interp *interp,		/* Interp for error reporting, if not NULL. */
    const char *name)		/* The name of the desired encoding. */
{
    Tcl_HashEntry *hPtr;
    Encoding *encodingPtr;

    Tcl_MutexLock(&encodingMutex);
    if (name == NULL) {
	encodingPtr = (Encoding *) systemEncoding;
	encodingPtr->refCount++;
	Tcl_MutexUnlock(&encodingMutex);
	return systemEncoding;
    }

    hPtr = Tcl_FindHashEntry(&encodingTable, name);
    if (hPtr != NULL) {
	encodingPtr = (Encoding *)Tcl_GetHashValue(hPtr);
	encodingPtr->refCount++;
	Tcl_MutexUnlock(&encodingMutex);
	return (Tcl_Encoding) encodingPtr;
    }
    Tcl_MutexUnlock(&encodingMutex);

    return LoadEncodingFile(interp, name);
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_FreeEncoding --
 *
 *	Releases an encoding allocated by Tcl_CreateEncoding() or
 *	Tcl_GetEncoding().
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	The reference count associated with the encoding is decremented and
 *	the encoding is deleted if nothing is using it anymore.
 *
 *---------------------------------------------------------------------------
 */

void
Tcl_FreeEncoding(
    Tcl_Encoding encoding)
{
    Tcl_MutexLock(&encodingMutex);
    FreeEncoding(encoding);
    Tcl_MutexUnlock(&encodingMutex);
}

/*
 *----------------------------------------------------------------------
 *
 * FreeEncoding --
 *
 *	Decrements the reference count of an encoding.  The caller must hold
 *	encodingMutes.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Releases the resource for an encoding if it is now unused.
 *	The reference count associated with the encoding is decremented and
 *	the encoding may be deleted if nothing is using it anymore.
 *
 *----------------------------------------------------------------------
 */

static void
FreeEncoding(
    Tcl_Encoding encoding)
{
    Encoding *encodingPtr = (Encoding *) encoding;

    if (encodingPtr == NULL) {
	return;
    }
    if (encodingPtr->refCount-- <= 1) {
	if (encodingPtr->freeProc != NULL) {
	    encodingPtr->freeProc(encodingPtr->clientData);
	}
	if (encodingPtr->hPtr != NULL) {
	    Tcl_DeleteHashEntry(encodingPtr->hPtr);
	}
	if (encodingPtr->name) {
	    Tcl_Free(encodingPtr->name);
	}
	Tcl_Free(encodingPtr);
    }
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_GetEncodingName --
 *
 *	Given an encoding, return the name that was used to construct the
 *	encoding.
 *
 * Results:
 *	The name of the encoding.
 *
 * Side effects:
 *	None.
 *
 *---------------------------------------------------------------------------
 */

const char *
Tcl_GetEncodingName(
    Tcl_Encoding encoding)	/* The encoding whose name to fetch. */
{
    if (encoding == NULL) {
	encoding = systemEncoding;
    }

    return ((Encoding *) encoding)->name;
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_GetEncodingNames --
 *
 *	Get the list of all known encodings, including the ones stored as
 *	files on disk in the encoding path.
 *
 * Results:
 *	Modifies interp's result object to hold a list of all the available
 *	encodings.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

void
Tcl_GetEncodingNames(
    Tcl_Interp *interp)		/* Interp to hold result. */
{
    Tcl_HashTable table;
    Tcl_HashSearch search;
    Tcl_HashEntry *hPtr;
    Tcl_Obj *map, *name, *result;
    Tcl_DictSearch mapSearch;
    int done = 0;

    TclNewObj(result);
    Tcl_InitObjHashTable(&table);

    /*
     * Copy encoding names from loaded encoding table to table.
     */

    Tcl_MutexLock(&encodingMutex);
    for (hPtr = Tcl_FirstHashEntry(&encodingTable, &search); hPtr != NULL;
	    hPtr = Tcl_NextHashEntry(&search)) {
	Encoding *encodingPtr = (Encoding *)Tcl_GetHashValue(hPtr);

	Tcl_CreateHashEntry(&table,
		Tcl_NewStringObj(encodingPtr->name, TCL_INDEX_NONE), NULL);
    }
    Tcl_MutexUnlock(&encodingMutex);

    FillEncodingFileMap();
    map = TclGetProcessGlobalValue(&encodingFileMap);

    /*
     * Copy encoding names from encoding file map to table.
     */

    Tcl_DictObjFirst(NULL, map, &mapSearch, &name, NULL, &done);
    for (; !done; Tcl_DictObjNext(&mapSearch, &name, NULL, &done)) {
	Tcl_CreateHashEntry(&table, name, NULL);
    }

    /*
     * Pull all encoding names from table into the result list.
     */

    for (hPtr = Tcl_FirstHashEntry(&table, &search); hPtr != NULL;
	    hPtr = Tcl_NextHashEntry(&search)) {
	Tcl_ListObjAppendElement(NULL, result,
		(Tcl_Obj *) Tcl_GetHashKey(&table, hPtr));
    }
    Tcl_SetObjResult(interp, result);
    Tcl_DeleteHashTable(&table);
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_GetEncodingNulLength --
 *
 *	Given an encoding, return the number of nul bytes used for the
 *	string termination.
 *
 * Results:
 *	The number of nul bytes used for the string termination.
 *
 * Side effects:
 *	None.
 *
 *---------------------------------------------------------------------------
 */
Tcl_Size
Tcl_GetEncodingNulLength(
    Tcl_Encoding encoding)
{
    if (encoding == NULL) {
	encoding = systemEncoding;
    }

    return ((Encoding *) encoding)->nullSize;
}

/*
 *------------------------------------------------------------------------
 *
 * Tcl_SetSystemEncoding --
 *
 *	Sets the default encoding that should be used whenever the user passes
 *	a NULL value in to one of the conversion routines. If the supplied
 *	name is NULL, the system encoding is reset to the default system
 *	encoding.
 *
 * Results:
 *	The return value is TCL_OK if the system encoding was successfully set
 *	to the encoding specified by name, TCL_ERROR otherwise. If TCL_ERROR
 *	is returned, an error message is left in interp's result object,
 *	unless interp was NULL.
 *
 * Side effects:
 *	If the passed encoding is the same as the current system
 *	encoding, the call is effectively a no-op. Otherwise, the reference
 *	count of the new system encoding is incremented. The reference count
 *	of the old system encoding is decremented and it may be freed. All
 *	VFS cached information is invalidated.
 *
 *------------------------------------------------------------------------
 */

int
Tcl_SetSystemEncoding(
    Tcl_Interp *interp,		/* Interp for error reporting, if not NULL. */
    const char *name)		/* The name of the desired encoding, or NULL/""
				 * to reset to default encoding. */
{
    Tcl_Encoding encoding = NULL;

    if (name && *name) {
	encoding = Tcl_GetEncoding(interp, name); /* this increases refCount */
	if (encoding == NULL) {
	    return TCL_ERROR;
	}
    }

    /* Don't lock (and change anything, bump epoch) if it remains unchanged. */

    if ((encoding ? encoding : defaultEncoding) == systemEncoding) {
	if (encoding) {
	    Tcl_FreeEncoding(encoding); /* paired to Tcl_GetEncoding */
	}
	return TCL_OK;
    }

    /* Checks above ensure this is only called when system encoding changes */
    Tcl_MutexLock(&encodingMutex);
    if (!encoding) {
	encoding = defaultEncoding; /* need increase its refCount */
	((Encoding *)encoding)->refCount++;
    }

    FreeEncoding(systemEncoding);
    systemEncoding = encoding;
    Tcl_MutexUnlock(&encodingMutex);

    Tcl_FSMountsChanged(NULL);

    return TCL_OK;
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_CreateEncoding --
 *
 *	Defines a new encoding, along with the functions that are used to
 *	convert to and from Unicode.
 *
 * Results:
 *	Returns a token that represents the encoding. If an encoding with the
 *	same name already existed, the old encoding token remains valid and
 *	continues to behave as it used to, and is eventually garbage collected
 *	when the last reference to it goes away. Any subsequent calls to
 *	Tcl_GetEncoding with the specified name retrieve the most recent
 *	encoding token.
 *
 * Side effects:
 *	A new record having the name of the encoding is entered into a table of
 *	encodings visible to all interpreters.  For each call to this function,
 *	there should eventually be a call to Tcl_FreeEncoding, which cleans
 *	deletes the record in the table when an encoding is no longer needed.
 *
 *---------------------------------------------------------------------------
 */

Tcl_Encoding
Tcl_CreateEncoding(
    const Tcl_EncodingType *typePtr)
				/* The encoding type. */
{
    Encoding *encodingPtr = (Encoding *)Tcl_Alloc(sizeof(Encoding));
    encodingPtr->name		= NULL;
    encodingPtr->toUtfProc	= typePtr->toUtfProc;
    encodingPtr->fromUtfProc	= typePtr->fromUtfProc;
    encodingPtr->freeProc	= typePtr->freeProc;
    encodingPtr->nullSize	= typePtr->nullSize;
    encodingPtr->clientData	= typePtr->clientData;
    if (typePtr->nullSize == 2) {
	encodingPtr->lengthProc = (LengthProc *) unilen;
    } else if (typePtr->nullSize == 4) {
	encodingPtr->lengthProc = (LengthProc *) unilen4;
    } else {
	encodingPtr->lengthProc = (LengthProc *) strlen;
    }
    encodingPtr->refCount	= 1;
    encodingPtr->hPtr		= NULL;

    if (typePtr->encodingName) {
	Tcl_HashEntry *hPtr;
	int isNew;
	char *name;

	Tcl_MutexLock(&encodingMutex);
	hPtr = Tcl_CreateHashEntry(&encodingTable, typePtr->encodingName, &isNew);
	if (isNew == 0) {
	    /*
	     * Remove old encoding from hash table, but don't delete it until last
	     * reference goes away.
	     */

	    Encoding *replaceMe = (Encoding *)Tcl_GetHashValue(hPtr);
	    replaceMe->hPtr = NULL;
	}

	name = (char *)Tcl_Alloc(strlen(typePtr->encodingName) + 1);
	encodingPtr->name	= strcpy(name, typePtr->encodingName);
	encodingPtr->hPtr	= hPtr;
	Tcl_SetHashValue(hPtr, encodingPtr);

	Tcl_MutexUnlock(&encodingMutex);
    }
    return (Tcl_Encoding) encodingPtr;
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_ExternalToUtfDString --
 *
 *	Convert a source buffer from the specified encoding into UTF-8. If any
 *	of the bytes in the source buffer are invalid or cannot be represented
 *	in the target encoding, a default fallback character will be
 *	substituted.
 *
 * Results:
 *	The converted bytes are stored in the DString, which is then NULL
 *	terminated. The return value is a pointer to the value stored in the
 *	DString.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

#undef Tcl_ExternalToUtfDString
char *
Tcl_ExternalToUtfDString(
    Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
				 * for the default system encoding. */
    const char *src,		/* Source string in specified encoding. */
    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
				 * encoding-specific string length. */
    Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
				 * converted string is stored. */
{
    Tcl_ExternalToUtfDStringEx(
	NULL, encoding, src, srcLen, TCL_ENCODING_PROFILE_TCL8, dstPtr, NULL);
    return Tcl_DStringValue(dstPtr);
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_ExternalToUtfDStringEx --
 *
 *	Convert a source buffer from the specified encoding into UTF-8.
 *	"flags" controls the behavior if any of the bytes in
 *	the source buffer are invalid or cannot be represented in utf-8.
 *	Possible flags values:
 *	target encoding. It should be composed by OR-ing the following:
 *	- *At most one* of TCL_ENCODING_PROFILE{DEFAULT,TCL8,STRICT}
 *
 * Results:
 *	The return value is one of
 *	  TCL_OK: success. Converted string in *dstPtr
 *	  TCL_ERROR: error in passed parameters. Error message in interp
 *	  TCL_CONVERT_MULTIBYTE: source ends in truncated multibyte sequence
 *	  TCL_CONVERT_SYNTAX: source is not conformant to encoding definition
 *	  TCL_CONVERT_UNKNOWN: source contained a character that could not
 *	      be represented in target encoding.
 *
 * Side effects:
 *	TCL_OK: The converted bytes are stored in the DString and NUL
 *	    terminated in an encoding-specific manner.
 *	TCL_ERROR: an error, message is stored in the interp if not NULL.
 *	TCL_CONVERT_*: if errorLocPtr is NULL, an error message is stored
 *	    in the interpreter (if not NULL). If errorLocPtr is not NULL,
 *	    no error message is stored as it is expected the caller is
 *	    interested in whatever is decoded so far and not treating this
 *	    as an error condition.
 *
 *	In addition, *dstPtr is always initialized and must be cleared
 *	by the caller irrespective of the return code.
 *
 *-------------------------------------------------------------------------
 */

int
Tcl_ExternalToUtfDStringEx(
    Tcl_Interp *interp,		/* For error messages. May be NULL. */
    Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
				 * for the default system encoding. */
    const char *src,		/* Source string in specified encoding. */
    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
				 * encoding-specific string length. */
    int flags,			/* Conversion control flags. */
    Tcl_DString *dstPtr,	/* Uninitialized or free DString in which the
				 * converted string is stored. */
    Tcl_Size *errorLocPtr)	/* Where to store the error location
				 * (or TCL_INDEX_NONE if no error). May
				 * be NULL. */
{
    char *dst;
    Tcl_EncodingState state;
    const Encoding *encodingPtr;
    int result;
    Tcl_Size dstLen, soFar;
    const char *srcStart = src;

    /* DO FIRST - Must always be initialized before returning */
    Tcl_DStringInit(dstPtr);

    dst = Tcl_DStringValue(dstPtr);
    dstLen = dstPtr->spaceAvl - 1;

    if (encoding == NULL) {
	encoding = systemEncoding;
    }
    encodingPtr = (Encoding *)encoding;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	srcLen = encodingPtr->lengthProc(src);
    }

    flags &= ~TCL_ENCODING_END;
    flags |= TCL_ENCODING_START;
    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	flags |= ENCODING_INPUT;
    }

    while (1) {
	int srcChunkLen, srcChunkRead;
	int dstChunkLen, dstChunkWrote, dstChunkChars;

	if (srcLen > INT_MAX) {
	    srcChunkLen = INT_MAX;
	} else {
	    srcChunkLen = srcLen;
	    flags |= TCL_ENCODING_END; /* Last chunk */
	}
	dstChunkLen = dstLen > INT_MAX ? INT_MAX : dstLen;

	result = encodingPtr->toUtfProc(encodingPtr->clientData, src,
		srcChunkLen, flags, &state, dst, dstChunkLen,
		&srcChunkRead, &dstChunkWrote, &dstChunkChars);
	soFar = dst + dstChunkWrote - Tcl_DStringValue(dstPtr);

	src += srcChunkRead;

	/*
	 * Keep looping in two case -
	 *   - our destination buffer did not have enough room
	 *   - we had not passed in all the data and error indicated fragment
	 *     of a multibyte character
	 * In both cases we have to grow buffer, move the input source pointer
	 * and loop. Otherwise, return the result we got.
	 */
	if ((result != TCL_CONVERT_NOSPACE) &&
		(result != TCL_CONVERT_MULTIBYTE || (flags & TCL_ENCODING_END))) {
	    Tcl_Size nBytesProcessed = (src - srcStart);

	    Tcl_DStringSetLength(dstPtr, soFar);
	    if (errorLocPtr) {
		/*
		 * Do not write error message into interpreter if caller
		 * wants to know error location.
		 */
		*errorLocPtr = result == TCL_OK
			? TCL_INDEX_NONE : nBytesProcessed;
	    } else {
		/* Caller wants error message on failure */
		if (result != TCL_OK && interp != NULL) {
		    char buf[TCL_INTEGER_SPACE];
		    snprintf(buf, sizeof(buf), "%" TCL_SIZE_MODIFIER "d",
			    nBytesProcessed);
		    Tcl_SetObjResult(interp, Tcl_ObjPrintf(
			    "unexpected byte sequence starting at index %"
			    TCL_SIZE_MODIFIER "d: '\\x%02X'",
			    nBytesProcessed, UCHAR(srcStart[nBytesProcessed])));
		    Tcl_SetErrorCode(
			    interp, "TCL", "ENCODING", "ILLEGALSEQUENCE", buf,
			    (char *)NULL);
		}
	    }
	    if (result != TCL_OK) {
		errno = (result == TCL_CONVERT_NOSPACE) ? ENOMEM : EILSEQ;
	    }
	    return result;
	}

	/* Expand space and continue */
	flags &= ~TCL_ENCODING_START;
	srcLen -= srcChunkRead;
	if (Tcl_DStringLength(dstPtr) == 0) {
	    Tcl_DStringSetLength(dstPtr, dstLen);
	}
	Tcl_DStringSetLength(dstPtr, 2 * Tcl_DStringLength(dstPtr) + 1);
	dst = Tcl_DStringValue(dstPtr) + soFar;
	dstLen = Tcl_DStringLength(dstPtr) - soFar - 1;
    }
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_ExternalToUtf --
 *
 *	Convert a source buffer from the specified encoding into UTF-8.
 *
 * Results:
 *	The return value is one of TCL_OK, TCL_CONVERT_MULTIBYTE,
 *	TCL_CONVERT_SYNTAX, TCL_CONVERT_UNKNOWN, or TCL_CONVERT_NOSPACE, as
 *	documented in tcl.h.
 *
 * Side effects:
 *	The converted bytes are stored in the output buffer.
 *
 *-------------------------------------------------------------------------
 */

int
Tcl_ExternalToUtf(
    Tcl_Interp *interp,		/* For error messages. May be NULL. */
    Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
				 * for the default system encoding. */
    const char *src,		/* Source string in specified encoding. */
    Tcl_Size srcLen,		/* Source string length in bytes, or
				 * negative for encoding-specific string
				 * length. */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    Tcl_Size dstLen,		/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    int result;
    Tcl_Size srcRead, dstWrote, dstChars;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	Encoding *encodingPtr = (Encoding *)encoding;
	if (encodingPtr == NULL) {
	    encodingPtr = (Encoding *)systemEncoding;
	}
	srcLen = encodingPtr->lengthProc(src);
    }

    if (dstCharsPtr && (flags & TCL_ENCODING_CHAR_LIMIT)) {
	dstChars = *dstCharsPtr;
    }
    result = Tcl_ExternalToUtfEx(interp, encoding, src, srcLen, flags,
	    statePtr, dst, dstLen,
	    srcReadPtr ? &srcRead : NULL,
	    dstWrotePtr ? &dstWrote : NULL,
	    dstCharsPtr ? &dstChars : NULL);

    if ((srcReadPtr && srcRead > INT_MAX) ||
	    (dstWrotePtr && dstWrote > INT_MAX) ||
	    (dstCharsPtr && dstChars > INT_MAX)) {
	/* An output count is too large for integer width */
	srcRead = 0;
	dstWrote = 0;
	dstChars = 0;
	if (statePtr) {
	    statePtr = 0;
	}
	if (interp) {
	    Tcl_SetResult(interp,
		    "Tcl_ExternalToUtf does not support lengths greater than "
		    "INT_MAX. Use Tcl_ExternalToUtfEx instead.",
		    TCL_STATIC);
	}
	/*
	 * This is a incompatibility with 9.0 as TCL_ERROR is not documented
	 * as a return value but no choice as no other value would be correct.
	 * This is a very low probability event in practice.
	 */
	result = TCL_ERROR;
    }

    if (srcReadPtr) {
	*srcReadPtr = (int)srcRead;
    }
    if (dstWrotePtr) {
	*dstWrotePtr = (int)dstWrote;
    }
    if (dstCharsPtr) {
	*dstCharsPtr = (int)dstChars;
    }
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_ExternalToUtfEx --
 *
 *	Convert a source buffer from the specified encoding into Tcl internal
 *	UTF-8 representation, using 64-bit counters for lengths.
 *
 * Results:
 *	The return value is one of TCL_OK, TCL_CONVERT_MULTIBYTE,
 *	TCL_CONVERT_SYNTAX, TCL_CONVERT_UNKNOWN, or TCL_CONVERT_NOSPACE, as
 *	documented in tcl.h.
 *
 * Side effects:
 *	The converted bytes are stored in the output buffer.
 *
 *-------------------------------------------------------------------------
 */

int
Tcl_ExternalToUtfEx(
    TCL_UNUSED(Tcl_Interp *),
    Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
				 * for the default system encoding. */
    const char *src,		/* Source string in specified encoding. */
    Tcl_Size srcLen,		/* Source string length in bytes, or
				 * TCL_INDEX_NONE for encoding-specific string
				 * length. */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    Tcl_Size dstLen,		/* The maximum length of output buffer in
				 * bytes. */
    Tcl_Size *srcReadPtr,	/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    Tcl_Size *dstWrotePtr,	/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    Tcl_Size *dstCharsPtr)	/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const Encoding *encodingPtr;
    int result = TCL_OK;
    int terminatorLength = (flags & TCL_ENCODING_NO_TERMINATE) == 0;
    Tcl_EncodingState localState;
    char *dstStart = dst;
    Tcl_Size srcBytesRead = 0;
    Tcl_Size dstBytesWritten = 0;
    Tcl_Size dstCharsWritten = 0;

    if (encoding == NULL) {
	encoding = systemEncoding;
    }
    encodingPtr = (Encoding *) encoding;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	srcLen = encodingPtr->lengthProc(src);
    }

    /* Need to initialize these before any jumps to storeResult */
    Tcl_Size srcBytesLeft = srcLen;
    Tcl_Size dstSpaceLeft = dstLen;

    /* If caller cannot preserve state, assume not a fragment */
    if (statePtr == NULL) {
	flags |= TCL_ENCODING_START | TCL_ENCODING_END;
	statePtr = &localState;
    }

    bool charLimited = ((flags & TCL_ENCODING_CHAR_LIMIT) != 0) && (dstCharsPtr != NULL);

    /* Note two separate checks to handle intermediate underflow */
    if (terminatorLength > dstLen) {
	terminatorLength = 0; /* So we do not try to store before returning */
	result = TCL_CONVERT_NOSPACE;
	goto storeResult;
    }
    dstLen -= terminatorLength;
    dstSpaceLeft = dstLen;

    if (dstLen <= 0 && srcLen > 0) {
	result = TCL_CONVERT_NOSPACE;
	goto storeResult;
    }

    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	/* UTF-8 -> TUTF-8 */
	flags |= ENCODING_INPUT;
    }

    /*
     * The low level encoding routines only take int arguments. Since they
     * are part of the public API, that cannot be changed easily. So we are
     * forced to loop over chunks of max size INT_MAX.
     */

    while (1) {
	int chunkSrcLen;
	int chunkDstLen;
	int chunkSrcRead = 0, chunkDstWritten = 0, chunkDstChars = 0;
	int chunkFlags = flags;
	int chunkCharLimit = INT_MAX;

	/* Determine chunk sizes */
	chunkDstLen = dstSpaceLeft > INT_MAX ? INT_MAX : (int)dstSpaceLeft;
	if (srcBytesLeft > INT_MAX) {
	    chunkSrcLen = INT_MAX;
	    /* not last chunk: ensure END not set */
	    chunkFlags &= ~TCL_ENCODING_END;
	} else {
	    /* For the last chunk we pass on the original TCL_ENCODING_END flag */
	    chunkSrcLen = (int)srcBytesLeft;
	}
	if (srcBytesRead > 0) {
	    chunkFlags &= ~TCL_ENCODING_START;
	}

	if (charLimited) {
	    assert(chunkFlags & TCL_ENCODING_CHAR_LIMIT);
	    assert(dstCharsPtr != NULL);
	    assert(*dstCharsPtr >= dstCharsWritten);
	    Tcl_Size remainingChars = *dstCharsPtr - dstCharsWritten;
	    if (remainingChars <= 0) {
		/* Limit already reached */
		break;
	    }
	    if (remainingChars > INT_MAX) {
		chunkCharLimit = INT_MAX;
	    } else {
		chunkCharLimit = (int)remainingChars;
	    }
	    chunkDstChars = chunkCharLimit; /* Limit for this iteration */
	}

	result = encodingPtr->toUtfProc(encodingPtr->clientData, src,
		chunkSrcLen, chunkFlags, statePtr, dst, chunkDstLen,
		&chunkSrcRead, &chunkDstWritten, &chunkDstChars);

	assert(chunkSrcRead <= srcBytesLeft);
	srcBytesLeft -= chunkSrcRead;
	srcBytesRead += chunkSrcRead;
	src += chunkSrcRead;

	assert(chunkDstWritten <= dstSpaceLeft);
	dstSpaceLeft -= chunkDstWritten;
	dstBytesWritten += chunkDstWritten;
	dst += chunkDstWritten;

	assert(chunkDstChars <= chunkCharLimit); /* NOT necessarily true in 9.0! */
	dstCharsWritten += chunkDstChars;

	/*
	 * We know the entire input data has been SEEN by the encoder when
	 *    chunkSrcLen == srcLen (input fit in single chunk)
	 *    chunkSrcLen < INT_MAX (last chunk was passed to encoder)
	 * Note this does NOT mean entire input data has been
	 * PROCESSED by the encoder as it may be forced to abort due to
	 * encoding errors, insufficient output space, or incomplete
	 * multibyte at end of the data. This condition of all input having
	 * been PROCESSED is indicated by srcBytesLeft == 0.
	 *
	 * Similarly, we know the entire output buffer has been made available
	 * to the encoder when
	 *    chunkDstLen == dstLen (entire output buffer in single pass)
	 *    chunkDstLen < INT_MAX (last part of output buffer was passed)
	 */
	int allInputSeen = (chunkSrcLen < INT_MAX) || (chunkSrcLen == srcLen);
	int allOutputSeen = (chunkDstLen < INT_MAX) || (chunkDstLen == dstLen);

	 /*
	 * The encoder may return any of the following result codes.
	 *
	 * TCL_OK - successfully processed current chunk.
	 *
	 *    1a If entire input has been seen by the encoder or character
	 *       limit has been reached, we are done and return TCL_OK.
	 *    1b If entire input has not been seen by the encoder,
	 *       continue with the loop to process next chunk.
	 *
	 * TCL_CONVERT_MULTIBYTE - end of chunk had an incomplete character
	 *
	 *    2a If entire input has been processed by the encoder
	 *       AND caller has indicated there is no more
	 *       data (TCL_ENCODING_END), return TCL_CONVERT_SYNTAX. The
	 *       encoder returns are not consistent in handling this case.
	 *       Some return *_MULTIBYTE and some return *_SYNTAX so we map
	 *       the former to the latter.
	 *    2b If entire input has been processed by the encoder AND
	 *       caller has indicated there is more data (TCL_ENCODING_END
	 *       not set) return TCL_CONVERT_MULTIBYTE.
	 *    2c If entire input has not been processed, continue the loop
	 *       and pass next chunk which would complete the multibyte unit.
	 *
	 * TCL_CONVERT_NOSPACE - Insufficient space in destination chunk
	 *    3a If we had passed in the entire original caller buffer,
	 *       return TCL_CONVERT_NOSPACE.
	 *    3c Else loop around again with the additional space.
	 *
	 * TCL_CONVERT_SYNTAX - encoding error
	 *    4 Return the error
	 *
	 * TCL_CONVERT_UNKNOWN - unencodable character
	 *    5 Return the error
	 */

	/* Handle the most likely cases first */
	if (result == TCL_OK) {
	    if (allInputSeen ||
		    (charLimited && dstCharsWritten >= *dstCharsPtr)) {
		break; /* 1a */
	    }
	    /* else 1b continue loop */
	} else if (result == TCL_CONVERT_MULTIBYTE) {
	    if (allInputSeen) {
		/* 2a, 2b */
		if (flags & TCL_ENCODING_END) {
		    result = TCL_CONVERT_SYNTAX; /* 2a */
		}
		break;
	    }
	    /* else 2c continue loop */
	} else if (result == TCL_CONVERT_NOSPACE) {
	    if (allOutputSeen) {
		break; /* 3a */
	    }
	    /* else 3b continue loop */
	} else {
	    assert(result == TCL_CONVERT_SYNTAX ||
		   result == TCL_CONVERT_UNKNOWN);
	    break; /* 4, 5 */
	}
    } /* End while (1) */

storeResult:
    assert(dstBytesWritten >= 0 && dstBytesWritten <= dstLen);
    if (terminatorLength != 0) {
	dstStart[dstBytesWritten] = '\0'; /* Already reserved space earlier */
    }

    assert(srcBytesRead >= 0 && srcBytesRead <= srcLen);
    assert((srcBytesRead + srcBytesLeft) == srcLen);
    if (srcReadPtr) {
	*srcReadPtr = srcBytesRead;
    }

    assert((dstBytesWritten + dstSpaceLeft) == dstLen);
    if (dstWrotePtr) {
	*dstWrotePtr = dstBytesWritten;
    }

    if (dstCharsPtr) {
	assert(dstCharsWritten >= 0);
	assert(!charLimited || dstCharsWritten <= *dstCharsPtr);
	*dstCharsPtr = dstCharsWritten;
    }

    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_UtfToExternalDString --
 *
 *	Convert a source buffer from UTF-8 to the specified encoding. If any
 *	of the bytes in the source buffer are invalid or cannot be represented
 *	in the target encoding, a default fallback character is substituted.
 *
 * Results:
 *	The converted bytes are stored in the DString, which is then NULL
 *	terminated in an encoding-specific manner. The return value is a
 *	pointer to the value stored in the DString.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */
#undef Tcl_UtfToExternalDString
char *
Tcl_UtfToExternalDString(
    Tcl_Encoding encoding,	/* The encoding for the converted string, or
				 * NULL for the default system encoding. */
    const char *src,		/* Source string in UTF-8. */
    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
				 * strlen(). */
    Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
				 * converted string is stored. */
{
    Tcl_UtfToExternalDStringEx(
	NULL, encoding, src, srcLen, TCL_ENCODING_PROFILE_TCL8, dstPtr, NULL);
    return Tcl_DStringValue(dstPtr);
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_UtfToExternalDStringEx --
 *
 *	Convert a source buffer from UTF-8 to the specified encoding.
 *	The parameter flags controls the behavior, if any of the bytes in
 *	the source buffer are invalid or cannot be represented in the
 *	target encoding. It should be composed by OR-ing the following:
 *	- *At most one* of TCL_ENCODING_PROFILE_*
 *
 * Results:
 *	The return value is one of
 *	  TCL_OK: success. Converted string in *dstPtr
 *	  TCL_ERROR: error in passed parameters. Error message in interp
 *	  TCL_CONVERT_MULTIBYTE: source ends in truncated multibyte sequence
 *	  TCL_CONVERT_SYNTAX: source is not conformant to encoding definition
 *	  TCL_CONVERT_UNKNOWN: source contained a character that could not
 *	      be represented in target encoding.
 *
 * Side effects:
 *	TCL_OK: The converted bytes are stored in the DString and NUL
 *	    terminated in an encoding-specific manner
 *	TCL_ERROR: an error, message is stored in the interp if not NULL.
 *	TCL_CONVERT_*: if errorLocPtr is NULL, an error message is stored
 *	    in the interpreter (if not NULL). If errorLocPtr is not NULL,
 *	    no error message is stored as it is expected the caller is
 *	    interested in whatever is decoded so far and not treating this
 *	    as an error condition.
 *
 *	In addition, *dstPtr is always initialized and must be cleared
 *	by the caller irrespective of the return code.
 *
 *-------------------------------------------------------------------------
 */

int
Tcl_UtfToExternalDStringEx(
    Tcl_Interp *interp,		/* For error messages. May be NULL. */
    Tcl_Encoding encoding,	/* The encoding for the converted string, or
				 * NULL for the default system encoding. */
    const char *src,		/* Source string in UTF-8. */
    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
				 * strlen(). */
    int flags,			/* Conversion control flags. */
    Tcl_DString *dstPtr,	/* Uninitialized or free DString in which the
				 * converted string is stored. */
    Tcl_Size *errorLocPtr)	/* Where to store the error location
				 * (or TCL_INDEX_NONE if no error). May
				 * be NULL. */
{
    char *dst;
    Tcl_EncodingState state;
    const Encoding *encodingPtr;
    int result;
    const char *srcStart = src;
    Tcl_Size dstLen, soFar;

    /* DO FIRST - must always be initialized on return */
    Tcl_DStringInit(dstPtr);

    dst = Tcl_DStringValue(dstPtr);
    dstLen = dstPtr->spaceAvl - 1;

    if (encoding == NULL) {
	encoding = systemEncoding;
    }
    encodingPtr = (Encoding *) encoding;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	srcLen = strlen(src);
    }

    flags &= ~TCL_ENCODING_END;
    flags |= TCL_ENCODING_START;
    while (1) {
	int srcChunkLen, srcChunkRead;
	int dstChunkLen, dstChunkWrote, dstChunkChars;

	if (srcLen > INT_MAX) {
	    srcChunkLen = INT_MAX;
	} else {
	    srcChunkLen = srcLen;
	    flags |= TCL_ENCODING_END; /* Last chunk */
	}
	dstChunkLen = dstLen > INT_MAX ? INT_MAX : dstLen;

	result = encodingPtr->fromUtfProc(encodingPtr->clientData, src,
		srcChunkLen, flags, &state, dst, dstChunkLen,
		&srcChunkRead, &dstChunkWrote, &dstChunkChars);
	soFar = dst + dstChunkWrote - Tcl_DStringValue(dstPtr);

	/* Move past the part processed in this go around */
	src += srcChunkRead;

	/*
	 * Keep looping in two case -
	 *   - our destination buffer did not have enough room
	 *   - we had not passed in all the data and error indicated fragment
	 *     of a multibyte character
	 * In both cases we have to grow buffer, move the input source pointer
	 * and loop. Otherwise, return the result we got.
	 */
	if ((result != TCL_CONVERT_NOSPACE) &&
		(result != TCL_CONVERT_MULTIBYTE || (flags & TCL_ENCODING_END))) {
	    Tcl_Size nBytesProcessed = (src - srcStart);
	    Tcl_Size i = soFar + encodingPtr->nullSize - 1;
	    /* Loop as DStringSetLength only stores one nul byte at a time */
	    while (i >= soFar) {
		Tcl_DStringSetLength(dstPtr, i--);
	    }
	    if (errorLocPtr) {
		/*
		 * Do not write error message into interpreter if caller
		 * wants to know error location.
		 */
		*errorLocPtr = result == TCL_OK
			? TCL_INDEX_NONE : nBytesProcessed;
	    } else {
		/* Caller wants error message on failure */
		if (result != TCL_OK && interp != NULL) {
		    Tcl_Size pos = Tcl_NumUtfChars(srcStart, nBytesProcessed);
		    int ucs4;
		    char buf[TCL_INTEGER_SPACE];

		    TclUtfToUniChar(&srcStart[nBytesProcessed], &ucs4);
		    snprintf(buf, sizeof(buf), "%" TCL_SIZE_MODIFIER "d",
			    nBytesProcessed);
		    Tcl_SetObjResult(interp, Tcl_ObjPrintf(
			    "unexpected character at index %" TCL_SIZE_MODIFIER
			    "u: 'U+%06X'",
			    pos, ucs4));
		    Tcl_SetErrorCode(interp, "TCL", "ENCODING", "ILLEGALSEQUENCE",
			    buf, (char *)NULL);
		}
	    }
	    if (result != TCL_OK) {
		errno = (result == TCL_CONVERT_NOSPACE) ? ENOMEM : EILSEQ;
	    }
	    return result;
	}

	flags &= ~TCL_ENCODING_START;
	srcLen -= srcChunkRead;

	if (Tcl_DStringLength(dstPtr) == 0) {
	    Tcl_DStringSetLength(dstPtr, dstLen);
	}
	Tcl_DStringSetLength(dstPtr, 2 * Tcl_DStringLength(dstPtr) + 1);
	dst = Tcl_DStringValue(dstPtr) + soFar;
	dstLen = Tcl_DStringLength(dstPtr) - soFar - 1;
    }
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_UtfToExternal --
 *
 *	Convert a buffer from UTF-8 into the specified encoding.
 *
 * Results:
 *	The return value is one of TCL_OK, TCL_CONVERT_MULTIBYTE,
 *	TCL_CONVERT_SYNTAX, TCL_CONVERT_UNKNOWN, or TCL_CONVERT_NOSPACE, as
 *	documented in tcl.h.
 *
 * Side effects:
 *	The converted bytes are stored in the output buffer.
 *
 *-------------------------------------------------------------------------
 */

int
Tcl_UtfToExternal(
    Tcl_Interp *interp,		/* TODO: Re-examine this. */
    Tcl_Encoding encoding,	/* The encoding for the converted string, or
				 * NULL for the default system encoding. */
    const char *src,		/* Source string in UTF-8. */
    Tcl_Size srcLen,		/* Source string length in bytes, or
				 * < 0 for strlen(). */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string
				 * is stored. */
    Tcl_Size dstLen,		/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    int result;
    Tcl_Size srcRead, dstWrote, dstChars;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	srcLen = strlen(src);
    }

    /* Low level encoders do not support CHAR_LIMIT. Future-proofing */
    if (dstCharsPtr && (flags & TCL_ENCODING_CHAR_LIMIT)) {
	dstChars = *dstCharsPtr;
    }

    result = Tcl_UtfToExternalEx(interp, encoding, src, srcLen, flags,
	    statePtr, dst, dstLen,
	    srcReadPtr ? &srcRead : NULL,
	    dstWrotePtr ? &dstWrote : NULL,
	    dstCharsPtr ? &dstChars : NULL);

    if ((srcReadPtr && srcRead > INT_MAX) ||
	    (dstWrotePtr && dstWrote > INT_MAX) ||
	    (dstCharsPtr && dstChars > INT_MAX)) {
	/* An output count is too large for integer width */
	srcRead = 0;
	dstWrote = 0;
	dstChars = 0;
	if (statePtr) {
	    statePtr = 0;
	}
	if (interp) {
	    Tcl_SetResult(interp,
		    "Tcl_UtfToExternal does not support lengths greater than "
		    "INT_MAX. Use Tcl_UtfToExternalEx instead.",
		    TCL_STATIC);
	}
	/*
	 * This is a incompatibility with 9.0 as TCL_ERROR is not documented
	 * as a return value but no choice as no other value would be correct.
	 * This is a very low probability event in practice.
	 */
	result = TCL_ERROR;
    }

    if (srcReadPtr) {
	assert(srcRead >= 0 && srcRead <= srcLen);
	*srcReadPtr = (int)srcRead;
    }
    if (dstWrotePtr) {
	assert(dstWrote >= 0 && dstWrote <= dstLen);
	*dstWrotePtr = (int)dstWrote;
    }
    if (dstCharsPtr) {
	assert(dstChars >= 0 && dstChars <= INT_MAX);
	*dstCharsPtr = (int)dstChars;
    }

    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Tcl_UtfToExternalEx --
 *
 *	Convert a source buffer from the Tcl internal UTF-8 encoding to
 *	the specified encoding, using 64-bit counters for lengths.
 *
 * Results:
 *	The return value is one of TCL_OK, TCL_CONVERT_MULTIBYTE,
 *	TCL_CONVERT_SYNTAX, TCL_CONVERT_UNKNOWN, or TCL_CONVERT_NOSPACE, as
 *	documented in tcl.h.
 *
 * Side effects:
 *	The converted bytes are stored in the output buffer.
 *
 *-------------------------------------------------------------------------
 */
int
Tcl_UtfToExternalEx(
    TCL_UNUSED(Tcl_Interp *),
    Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
				 * for the default system encoding. */
    const char *src,		/* Source string in specified encoding. */
    Tcl_Size srcLen,		/* Source string length in bytes, or
				 * TCL_INDEX_NONE for encoding-specific string
				 * length. */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    Tcl_Size dstLen,		/* The maximum length of output buffer in
				 * bytes. */
    Tcl_Size *srcReadPtr,	/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    Tcl_Size *dstWrotePtr,	/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    Tcl_Size *dstCharsPtr)	/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const Encoding *encodingPtr;
    int result = TCL_OK;
    int terminatorLength = (flags & TCL_ENCODING_NO_TERMINATE) == 0;
    Tcl_EncodingState localState;
    char *dstStart = dst;
    Tcl_Size srcBytesRead = 0;
    Tcl_Size dstBytesWritten = 0;
    Tcl_Size dstCharsWritten = 0;

    if (encoding == NULL) {
	encoding = systemEncoding;
    }
    encodingPtr = (Encoding *) encoding;
    terminatorLength =
	(flags & TCL_ENCODING_NO_TERMINATE) ? 0 : encodingPtr->nullSize;

    if (src == NULL) {
	srcLen = 0;
	src = "";	/* Avoid ABSAN warnings about ops on NULL pointers */
    } else if (srcLen < 0) {
	srcLen = strlen(src); /* strlen works for TUTF-8 */
    }

    /* Need to initialize these before any jumps to storeResult */
    Tcl_Size dstSpaceLeft = dstLen;
    Tcl_Size srcBytesLeft = srcLen;

    /* If caller cannot preserve state, assume not a fragment */
    if (statePtr == NULL) {
	flags |= TCL_ENCODING_START | TCL_ENCODING_END;
	statePtr = &localState;
    }

    /*
     * TODO - The fromUtf routines do not support the
     * TCL_ENCODING_CHAR_LIMIT flag. Ignore if present. However, for future
     * support, the looping code is prepared to handle it.
     */
    flags &= ~TCL_ENCODING_CHAR_LIMIT;
    bool charLimited = ((flags & TCL_ENCODING_CHAR_LIMIT) != 0) && (dstCharsPtr != NULL);

    /* Note two separate checks to handle intermediate underflow */
    if (terminatorLength > dstLen) {
	terminatorLength = 0; /* So we do not try to store before returning */
	result = TCL_CONVERT_NOSPACE;
	goto storeResult;
    }
    dstLen -= terminatorLength;
    dstSpaceLeft = dstLen;

    if (dstLen <= 0 && srcLen > 0) {
	result = TCL_CONVERT_NOSPACE;
	goto storeResult;
    }

    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	/* TUTF-8 -> UTF-8 */
	flags &= ~ENCODING_INPUT; /* Ensure bit not set */
    }

    /*
     * The low level encoding routines only take int arguments. Since they
     * are part of the public API, that cannot be changed easily. So we are
     * forced to loop over chunks of max size INT_MAX.
     */
    while (1) {
	int chunkSrcLen;
	int chunkDstLen;
	int chunkSrcRead = 0, chunkDstWritten = 0, chunkDstChars = 0;
	int chunkFlags = flags;
	int chunkCharLimit = INT_MAX;

	/* Determine chunk sizes */
	chunkDstLen = dstSpaceLeft > INT_MAX ? INT_MAX : (int)dstSpaceLeft;
	if (srcBytesLeft > INT_MAX) {
	    chunkSrcLen = INT_MAX;
	    /* not last chunk: ensure END not set */
	    chunkFlags &= ~TCL_ENCODING_END;
	} else {
	    /* For last chunk we pass on the original TCL_ENCODING_END flag */
	    chunkSrcLen = (int)srcBytesLeft;
	}
	if (srcBytesRead > 0) {
	    chunkFlags &= ~TCL_ENCODING_START;
	}

	if (charLimited) {
	    assert(chunkFlags & TCL_ENCODING_CHAR_LIMIT);
	    assert(dstCharsPtr != NULL);
	    assert(*dstCharsPtr >= dstCharsWritten);
	    Tcl_Size remainingChars = *dstCharsPtr - dstCharsWritten;
	    if (remainingChars <= 0) {
		/* Limit already reached */
		break;
	    }
	    if (remainingChars > INT_MAX) {
		chunkCharLimit = INT_MAX;
	    } else {
		chunkCharLimit = (int)remainingChars;
	    }
	    chunkDstChars = chunkCharLimit; /* Limit for this iteration */
	}

	result = encodingPtr->fromUtfProc(encodingPtr->clientData, src,
		chunkSrcLen, chunkFlags, statePtr, dst, chunkDstLen,
		&chunkSrcRead, &chunkDstWritten, &chunkDstChars);

	assert(chunkSrcRead <= srcBytesLeft);
	srcBytesLeft -= chunkSrcRead;
	srcBytesRead += chunkSrcRead;
	src += chunkSrcRead;

	assert(chunkDstWritten <= dstSpaceLeft);
	dstSpaceLeft -= chunkDstWritten;
	dstBytesWritten += chunkDstWritten;
	dst += chunkDstWritten;

	assert(chunkDstChars <= chunkCharLimit); /* NOT necessarily true in 9.0! */
	dstCharsWritten += chunkDstChars;

	/*
	 * We know the entire input data has been SEEN by the encoder when
	 *    chunkSrcLen == srcLen (input fit in single chunk)
	 *    chunkSrcLen < INT_MAX (last chunk was passed to encoder)
	 * Note this does NOT mean entire input data has been
	 * PROCESSED by the encoder as it may be forced to abort due to
	 * encoding errors, insufficient output space, or incomplete
	 * multibyte at end of the data. This condition of all input having
	 * been PROCESSED is indicated by srcBytesLeft == 0.
	 *
	 * Similarly, we know the entire output buffer has been made available
	 * to the encoder when
	 *    chunkDstLen == dstLen (entire output buffer in single pass)
	 *    chunkDstLen < INT_MAX (last part of output buffer was passed)
	 */
	int allInputSeen = (chunkSrcLen < INT_MAX) || (chunkSrcLen == srcLen);
	int allOutputSeen = (chunkDstLen < INT_MAX) || (chunkDstLen == dstLen);

	 /*
	 * The encoder may return any of the following result codes.
	 *
	 * TCL_OK - successfully processed current chunk.
	 *
	 *    1a If entire input has been seen by the encoder or character
	 *       limit has been reached, we are done and return TCL_OK.
	 *    1b If entire input has not been seen by the encoder,
	 *       continue with the loop to process next chunk.
	 *
	 * TCL_CONVERT_MULTIBYTE - end of chunk had an incomplete character
	 *
	 *    2a If entire input has been processed by the encoder
	 *       AND caller has indicated there is no more
	 *       data (TCL_ENCODING_END), return TCL_CONVERT_SYNTAX. The
	 *       encoder returns are not consistent in handling this case.
	 *       Some return *_MULTIBYTE and some return *_SYNTAX so we map
	 *       the former to the latter.
	 *    2b If entire input has been processed by the encoder AND
	 *       caller has indicated there is more data (TCL_ENCODING_END
	 *       not set) return TCL_CONVERT_MULTIBYTE.
	 *    2c If entire input has not been processed, continue the loop
	 *       and pass next chunk which would complete the multibyte unit.
	 *
	 * TCL_CONVERT_NOSPACE - Insufficient space in destination chunk
	 *    3a If we had passed in the entire original caller buffer,
	 *       return TCL_CONVERT_NOSPACE.
	 *    3c Else loop around again with the additional space.
	 *
	 * TCL_CONVERT_SYNTAX - encoding error
	 *    4 Return the error
	 *
	 * TCL_CONVERT_UNKNOWN - unencodable character
	 *    5 Return the error
	 */

	/* Handle the most likely cases first */
	if (result == TCL_OK) {
	    if (allInputSeen ||
		    (charLimited && dstCharsWritten >= *dstCharsPtr)) {
		break; /* 1a */
	    }
	    /* else 1b continue loop */
	} else if (result == TCL_CONVERT_MULTIBYTE) {
	    if (allInputSeen) {
		/* 2a, 2b */
		if (flags & TCL_ENCODING_END) {
		    result = TCL_CONVERT_SYNTAX; /* 2a */
		}
		break;
	    }
	    /* else 2c continue loop */
	} else if (result == TCL_CONVERT_NOSPACE) {
	    if (allOutputSeen) {
		break; /* 3a */
	    }
	    /* else 3b continue loop */
	} else {
	    assert(result == TCL_CONVERT_SYNTAX ||
		   result == TCL_CONVERT_UNKNOWN);
	    break; /* 4, 5 */
	}
    } /* End while (1) */

storeResult:
    assert(dstBytesWritten >= 0 && dstBytesWritten <= dstLen);
    if (terminatorLength != 0) {
	/* Space was already preserved before the loop */
	memset(&dstStart[dstBytesWritten], '\0', terminatorLength);
    }

    assert(srcBytesRead >= 0 && srcBytesRead <= srcLen);
    assert((srcBytesRead + srcBytesLeft) == srcLen);
    if (srcReadPtr) {
	*srcReadPtr = srcBytesRead;
    }

    assert(dstBytesWritten >= 0 && dstBytesWritten <= dstLen);
    assert((dstBytesWritten + dstSpaceLeft) == dstLen);
    if (dstWrotePtr) {
	*dstWrotePtr = dstBytesWritten;
    }

    if (dstCharsPtr) {
	assert(dstCharsWritten >= 0);
	assert(!charLimited || dstCharsWritten <= *dstCharsPtr);
	*dstCharsPtr = dstCharsWritten;
    }

    return result;
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_FindExecutable --
 *
 *	This function computes the absolute path name of the current
 *	application, given its argv[0] value.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	The absolute pathname for the application is computed and stored to be
 *	returned later by [info nameofexecutable].
 *
 *---------------------------------------------------------------------------
 */
#undef Tcl_FindExecutable
const char *
Tcl_FindExecutable(
    const char *argv0)		/* The value of the application's argv[0]
				 * (native). */
{
    const char *version = Tcl_InitSubsystems();
    TclpSetInitialEncodings();
    TclpFindExecutable(argv0);
#ifdef STATIC_BUILD
    Tcl_RegisterPostInitProc(TclInitStaticPackages, NULL);
#endif
    return version;
}

/*
 *---------------------------------------------------------------------------
 *
 * OpenEncodingFileChannel --
 *
 *	Open the file believed to hold data for the encoding, "name".
 *
 * Results:
 *	Returns the readable Tcl_Channel from opening the file, or NULL if the
 *	file could not be successfully opened. If NULL was returned, an error
 *	message is left in interp's result object, unless interp was NULL.
 *
 * Side effects:
 *	Channel may be opened. Information about the filesystem may be cached
 *	to speed later calls.
 *
 *---------------------------------------------------------------------------
 */

static Tcl_Channel
OpenEncodingFileChannel(
    Tcl_Interp *interp,		/* Interp for error reporting, if not NULL. */
    const char *name)		/* The name of the encoding file on disk and
				 * also the name for new encoding. */
{
    Tcl_Obj *fileNameObj = Tcl_ObjPrintf("%s.enc", name);
    Tcl_Obj *searchPath = Tcl_DuplicateObj(Tcl_GetEncodingSearchPath());
    Tcl_Obj *map = TclGetProcessGlobalValue(&encodingFileMap);
    Tcl_Obj **dir, *path, *directory = NULL;
    Tcl_Channel chan = NULL;
    Tcl_Size i, numDirs;

    TclListObjGetElements(NULL, searchPath, &numDirs, &dir);
    Tcl_IncrRefCount(fileNameObj);
    TclDictGet(NULL, map, name, &directory);

    /*
     * Check that any cached directory is still on the encoding search path.
     */

    if (NULL != directory) {
	int verified = 0;

	for (i=0; i<numDirs && !verified; i++) {
	    if (dir[i] == directory) {
		verified = 1;
	    }
	}
	if (!verified) {
	    const char *dirString = TclGetString(directory);

	    for (i=0; i<numDirs && !verified; i++) {
		if (strcmp(dirString, TclGetString(dir[i])) == 0) {
		    verified = 1;
		}
	    }
	}
	if (!verified) {
	    /*
	     * Directory no longer on the search path. Remove from cache.
	     */

	    map = Tcl_DuplicateObj(map);
	    TclDictRemove(NULL, map, name);
	    TclSetProcessGlobalValue(&encodingFileMap, map);
	    directory = NULL;
	}
    }

    if (NULL != directory) {
	/*
	 * Got a directory from the cache. Try to use it first.
	 */

	Tcl_IncrRefCount(directory);
	path = Tcl_FSJoinToPath(directory, 1, &fileNameObj);
	Tcl_IncrRefCount(path);
	Tcl_DecrRefCount(directory);
	chan = Tcl_FSOpenFileChannel(NULL, path, "r", 0);
	Tcl_DecrRefCount(path);
    }

    /*
     * Scan the search path until we find it.
     */

    for (i=0; i<numDirs && (chan == NULL); i++) {
	path = Tcl_FSJoinToPath(dir[i], 1, &fileNameObj);
	Tcl_IncrRefCount(path);
	chan = Tcl_FSOpenFileChannel(NULL, path, "r", 0);
	Tcl_DecrRefCount(path);
	if (chan != NULL) {
	    /*
	     * Save directory in the cache.
	     */

	    map = Tcl_DuplicateObj(TclGetProcessGlobalValue(&encodingFileMap));
	    TclDictPut(NULL, map, name, dir[i]);
	    TclSetProcessGlobalValue(&encodingFileMap, map);
	}
    }

    if ((NULL == chan) && (interp != NULL)) {
	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		"unknown encoding \"%s\"", name));
	Tcl_SetErrorCode(interp, "TCL", "LOOKUP", "ENCODING", name, (char *)NULL);
    }
    Tcl_DecrRefCount(fileNameObj);
    Tcl_DecrRefCount(searchPath);

    return chan;
}

/*
 *---------------------------------------------------------------------------
 *
 * LoadEncodingFile --
 *
 *	Read a file that describes an encoding and create a new Encoding from
 *	the data.
 *
 * Results:
 *	The return value is the newly loaded Tcl_Encoding or NULL if the file
 *	didn't exist or could not be processed. If NULL is returned and interp
 *	is not NULL, an error message is left in interp's result object.
 *
 * Side effects:
 *	A corresponding encoding file might be read from persistent storage, in
 *	which case LoadTableEncoding is called.
 *
 *---------------------------------------------------------------------------
 */

static Tcl_Encoding
LoadEncodingFile(
    Tcl_Interp *interp,		/* Interp for error reporting, if not NULL. */
    const char *name)		/* The name of both the encoding file
				 * and the new encoding. */
{
    Tcl_Channel chan = NULL;
    Tcl_Encoding encoding = NULL;
    int ch;

    chan = OpenEncodingFileChannel(interp, name);
    if (chan == NULL) {
	return NULL;
    }

    Tcl_SetChannelOption(NULL, chan, "-encoding", "utf-8");

    while (1) {
	Tcl_DString ds;

	Tcl_DStringInit(&ds);
	Tcl_Gets(chan, &ds);
	ch = Tcl_DStringValue(&ds)[0];
	Tcl_DStringFree(&ds);
	if (ch != '#') {
	    break;
	}
    }

    switch (ch) {
    case 'S':
	encoding = LoadTableEncoding(name, ENCODING_SINGLEBYTE, chan);
	break;
    case 'D':
	encoding = LoadTableEncoding(name, ENCODING_DOUBLEBYTE, chan);
	break;
    case 'M':
	encoding = LoadTableEncoding(name, ENCODING_MULTIBYTE, chan);
	break;
    case 'E':
	encoding = LoadEscapeEncoding(name, chan);
	break;
    }
    if ((encoding == NULL) && (interp != NULL)) {
	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		"invalid encoding file \"%s\"", name));
	Tcl_SetErrorCode(interp, "TCL", "LOOKUP", "ENCODING", name, (char *)NULL);
    }
    Tcl_CloseEx(NULL, chan, 0);

    return encoding;
}

/*
 *-------------------------------------------------------------------------
 *
 * LoadTableEncoding --
 *
 *	Helper function for LoadEncodingFile().  Creates a Tcl_EncodingType
 *	structure along with its corresponding TableEncodingData structure, and
 *	passes it to Tcl_Createncoding.
 *
 *	The file contains binary data but begins with a marker to indicate
 *	byte-ordering so a single binary file can be read on big or
 *	little-endian systems.
 *
 * Results:
 *	Returns the new Tcl_Encoding,  or NULL if it could
 *	not be created because the file contained invalid data.
 *
 * Side effects:
 *	See Tcl_CreateEncoding().
 *
 *-------------------------------------------------------------------------
 */

static Tcl_Encoding
LoadTableEncoding(
    const char *name,		/* Name of the new encoding. */
    int type,			/* Type of encoding (ENCODING_?????). */
    Tcl_Channel chan)		/* File containing new encoding. */
{
    Tcl_DString lineString;
    Tcl_Obj *objPtr;
    char *line;
    int i, hi, lo, numPages, symbol, fallback, len;
    unsigned char used[256];
    unsigned size;
    TableEncodingData *dataPtr;
    unsigned short *pageMemPtr, *page;
    Tcl_EncodingType encType;

    /*
     * Speed over memory. Use a full 256 character table to decode hex
     * sequences in the encoding files.
     */

    static const char staticHex[] = {
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*   0 ...  15 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*  16 ...  31 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*  32 ...  47 */
	0,  1,  2,  3,  4,  5,  6, 7, 8, 9, 0, 0, 0, 0, 0, 0, /*  48 ...  63 */
	0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*  64 ...  79 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*  80 ...  95 */
	0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*  96 ... 111 */
	0,  1,  2,  3,  4,  5,  6, 7, 8, 9, 0, 0, 0, 0, 0, 0, /* 112 ... 127 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 128 ... 143 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 144 ... 159 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 160 ... 175 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 176 ... 191 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 192 ... 207 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 208 ... 223 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 224 ... 239 */
	0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 240 ... 255 */
    };

    Tcl_DStringInit(&lineString);
    if (Tcl_Gets(chan, &lineString) < 0) {
	return NULL;
    }
    line = Tcl_DStringValue(&lineString);

    fallback = (int) strtol(line, &line, 16);
    symbol = (int) strtol(line, &line, 10);
    numPages = (int) strtol(line, &line, 10);
    Tcl_DStringFree(&lineString);

    if (numPages < 0) {
	numPages = 0;
    } else if (numPages > 256) {
	numPages = 256;
    }

    memset(used, 0, sizeof(used));

#undef PAGESIZE
#define PAGESIZE    (256 * sizeof(unsigned short))

    dataPtr = (TableEncodingData *)Tcl_Alloc(sizeof(TableEncodingData));
    memset(dataPtr, 0, sizeof(TableEncodingData));

    dataPtr->fallback = fallback;

    /*
     * Read the table that maps characters to Unicode. Performs a single
     * malloc to get the memory for the array and all the pages needed by the
     * array.
     */

    size = 256 * sizeof(unsigned short *) + numPages * PAGESIZE;
    dataPtr->toUnicode = (unsigned short **)Tcl_Alloc(size);
    memset(dataPtr->toUnicode, 0, size);
    pageMemPtr = (unsigned short *) (dataPtr->toUnicode + 256);

    TclNewObj(objPtr);
    Tcl_IncrRefCount(objPtr);
    for (i = 0; i < numPages; i++) {
	int ch;
	const char *p;
	Tcl_Size expected = 3 + 16 * (16 * 4 + 1);

	if (Tcl_ReadChars(chan, objPtr, expected, 0) != expected) {
	    return NULL;
	}
	p = TclGetString(objPtr);
	hi = (staticHex[UCHAR(p[0])] << 4) + staticHex[UCHAR(p[1])];
	dataPtr->toUnicode[hi] = pageMemPtr;
	p += 2;
	for (lo = 0; lo < 256; lo++) {
	    if ((lo & 0x0F) == 0) {
		p++;
	    }
	    ch = (staticHex[UCHAR(p[0])] << 12) + (staticHex[UCHAR(p[1])] << 8)
		    + (staticHex[UCHAR(p[2])] << 4) + staticHex[UCHAR(p[3])];
	    if (ch != 0) {
		used[ch >> 8] = 1;
	    }
	    *pageMemPtr = (unsigned short) ch;
	    pageMemPtr++;
	    p += 4;
	}
    }
    TclDecrRefCount(objPtr);

    if (type == ENCODING_DOUBLEBYTE) {
	memset(dataPtr->prefixBytes, 1, sizeof(dataPtr->prefixBytes));
    } else {
	for (hi = 1; hi < 256; hi++) {
	    if (dataPtr->toUnicode[hi] != NULL) {
		dataPtr->prefixBytes[hi] = 1;
	    }
	}
    }

    /*
     * Invert the toUnicode array to produce the fromUnicode array. Performs a
     * single malloc to get the memory for the array and all the pages needed
     * by the array. While reading in the toUnicode array remember what
     * pages are needed for the fromUnicode array.
     */

    if (symbol) {
	used[0] = 1;
    }
    numPages = 0;
    for (hi = 0; hi < 256; hi++) {
	if (used[hi]) {
	    numPages++;
	}
    }
    size = 256 * sizeof(unsigned short *) + numPages * PAGESIZE;
    dataPtr->fromUnicode = (unsigned short **)Tcl_Alloc(size);
    memset(dataPtr->fromUnicode, 0, size);
    pageMemPtr = (unsigned short *) (dataPtr->fromUnicode + 256);

    for (hi = 0; hi < 256; hi++) {
	if (dataPtr->toUnicode[hi] == NULL) {
	    dataPtr->toUnicode[hi] = emptyPage;
	    continue;
	}
	for (lo = 0; lo < 256; lo++) {
	    int ch = dataPtr->toUnicode[hi][lo];

	    if (ch != 0) {
		page = dataPtr->fromUnicode[ch >> 8];
		if (page == NULL) {
		    page = pageMemPtr;
		    pageMemPtr += 256;
		    dataPtr->fromUnicode[ch >> 8] = page;
		}
		page[ch & 0xFF] = (unsigned short) ((hi << 8) + lo);
	    }
	}
    }
    if (type == ENCODING_MULTIBYTE) {
	/*
	 * If multibyte encodings don't have a backslash character, define
	 * one. Otherwise, on Windows, native file names don't work because
	 * the backslash in the file name maps to the unknown character
	 * (question mark) when converting from UTF-8 to external encoding.
	 */

	if (dataPtr->fromUnicode[0] != NULL) {
	    if (dataPtr->fromUnicode[0][(int)'\\'] == '\0') {
		dataPtr->fromUnicode[0][(int)'\\'] = '\\';
	    }
	}
    }
    if (symbol) {
	/*
	 * Make a special symbol encoding that maps each symbol character from
	 * its Unicode code point down into page 0, and also ensure that each
	 * characters on page 0 maps to itself so that a symbol font can be
	 * used to display a simple string like "abcd" and have alpha, beta,
	 * chi, delta show up, rather than have "unknown" chars show up because
	 * strictly speaking the symbol font doesn't have glyphs for those low
	 * ASCII chars.
	 */

	page = dataPtr->fromUnicode[0];
	if (page == NULL) {
	    page = pageMemPtr;
	    dataPtr->fromUnicode[0] = page;
	}
	for (lo = 0; lo < 256; lo++) {
	    if (dataPtr->toUnicode[0][lo] != 0) {
		page[lo] = (unsigned short) lo;
	    }
	}
    }
    for (hi = 0; hi < 256; hi++) {
	if (dataPtr->fromUnicode[hi] == NULL) {
	    dataPtr->fromUnicode[hi] = emptyPage;
	}
    }

    /*
     * For trailing 'R'everse encoding, see [Patch 689341]
     */

    Tcl_DStringInit(&lineString);

    /*
     * Skip leading empty lines.
     */

    while ((len = Tcl_Gets(chan, &lineString)) == 0) {
	/* empty body */
    }
    if (len < 0) {
	goto doneParse;
    }

    /*
     * Require that it starts with an 'R'.
     */

    line = Tcl_DStringValue(&lineString);
    if (line[0] != 'R') {
	goto doneParse;
    }

    /*
     * Read lines until EOF.
     */

    for (TclDStringClear(&lineString);
	    (len = Tcl_Gets(chan, &lineString)) != -1;
	    TclDStringClear(&lineString)) {
	const unsigned char *p;
	int to, from;

	/*
	 * Skip short lines.
	 */

	if (len < 5) {
	    continue;
	}

	/*
	 * Parse the line as a sequence of hex digits.
	 */

	p = (const unsigned char *) Tcl_DStringValue(&lineString);
	to = (staticHex[p[0]] << 12) + (staticHex[p[1]] << 8)
		+ (staticHex[p[2]] << 4) + staticHex[p[3]];
	if (to == 0) {
	    continue;
	}
	for (p += 5, len -= 5; len >= 0 && *p; p += 5, len -= 5) {
	    from = (staticHex[p[0]] << 12) + (staticHex[p[1]] << 8)
		    + (staticHex[p[2]] << 4) + staticHex[p[3]];
	    if (from == 0) {
		continue;
	    }
	    dataPtr->fromUnicode[from >> 8][from & 0xFF] = to;
	}
    }
  doneParse:
    Tcl_DStringFree(&lineString);

    /*
     * Package everything into an encoding structure.
     */

    encType.encodingName    = name;
    encType.toUtfProc	    = TableToUtfProc;
    encType.fromUtfProc	    = TableFromUtfProc;
    encType.freeProc	    = TableFreeProc;
    encType.nullSize	    = (type == ENCODING_DOUBLEBYTE) ? 2 : 1;
    encType.clientData	    = dataPtr;

    return Tcl_CreateEncoding(&encType);
}

/*
 *-------------------------------------------------------------------------
 *
 * LoadEscapeEncoding --
 *
 *	Helper function for LoadEncodingTable(). Loads a state machine that
 *	converts between Unicode and some other encoding.
 *
 *	File contains text data that describes the escape sequences that are
 *	used to choose an encoding and the associated names for the
 *	sub-encodings.
 *
 * Results:
 *	The return value is the new encoding, or NULL if the encoding could
 *	not be created (because the file contained invalid data).
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static Tcl_Encoding
LoadEscapeEncoding(
    const char *name,		/* Name of the new encoding. */
    Tcl_Channel chan)		/* File containing new encoding. */
{
    int i;
    unsigned size;
    Tcl_DString escapeData;
    char init[16], final[16];
    EscapeEncodingData *dataPtr;
    Tcl_EncodingType type;

    init[0] = '\0';
    final[0] = '\0';
    Tcl_DStringInit(&escapeData);

    while (1) {
	Tcl_Size argc;
	const char **argv;
	char *line;
	Tcl_DString lineString;

	Tcl_DStringInit(&lineString);
	if (Tcl_Gets(chan, &lineString) < 0) {
	    break;
	}
	line = Tcl_DStringValue(&lineString);
	if (Tcl_SplitList(NULL, line, &argc, &argv) != TCL_OK) {
	    Tcl_DStringFree(&lineString);
	    continue;
	}
	if (argc >= 2) {
	    if (strcmp(argv[0], "name") == 0) {
		/* do nothing */
	    } else if (strcmp(argv[0], "init") == 0) {
		strncpy(init, argv[1], sizeof(init));
		init[sizeof(init) - 1] = '\0';
	    } else if (strcmp(argv[0], "final") == 0) {
		strncpy(final, argv[1], sizeof(final));
		final[sizeof(final) - 1] = '\0';
	    } else {
		EscapeSubTable est;
		Encoding *e;

		strncpy(est.sequence, argv[1], sizeof(est.sequence));
		est.sequence[sizeof(est.sequence) - 1] = '\0';
		est.sequenceLen = strlen(est.sequence);

		strncpy(est.name, argv[0], sizeof(est.name));
		est.name[sizeof(est.name) - 1] = '\0';

		/*
		 * To avoid infinite recursion in [encoding system iso2022-*]
		 */

		e = (Encoding *) Tcl_GetEncoding(NULL, est.name);
		if ((e != NULL) && (e->toUtfProc != TableToUtfProc)
			&& (e->toUtfProc != Iso88591ToUtfProc)) {
		    Tcl_FreeEncoding((Tcl_Encoding) e);
		    e = NULL;
		}
		est.encodingPtr = e;
		Tcl_DStringAppend(&escapeData, (char *) &est, sizeof(est));
	    }
	}
	Tcl_Free(argv);
	Tcl_DStringFree(&lineString);
    }

    size = offsetof(EscapeEncodingData, subTables)
	    + Tcl_DStringLength(&escapeData);
    dataPtr = (EscapeEncodingData *)Tcl_Alloc(size);
    dataPtr->initLen = strlen(init);
    memcpy(dataPtr->init, init, dataPtr->initLen + 1);
    dataPtr->finalLen = strlen(final);
    memcpy(dataPtr->final, final, dataPtr->finalLen + 1);
    dataPtr->numSubTables =
	    Tcl_DStringLength(&escapeData) / sizeof(EscapeSubTable);
    memcpy(dataPtr->subTables, Tcl_DStringValue(&escapeData),
	    Tcl_DStringLength(&escapeData));
    Tcl_DStringFree(&escapeData);

    memset(dataPtr->prefixBytes, 0, sizeof(dataPtr->prefixBytes));
    for (i = 0; i < dataPtr->numSubTables; i++) {
	dataPtr->prefixBytes[UCHAR(dataPtr->subTables[i].sequence[0])] = 1;
    }
    if (dataPtr->init[0] != '\0') {
	dataPtr->prefixBytes[UCHAR(dataPtr->init[0])] = 1;
    }
    if (dataPtr->final[0] != '\0') {
	dataPtr->prefixBytes[UCHAR(dataPtr->final[0])] = 1;
    }

    /*
     * Package everything into an encoding structure.
     */

    type.encodingName	= name;
    type.toUtfProc	= EscapeToUtfProc;
    type.fromUtfProc    = EscapeFromUtfProc;
    type.freeProc	= EscapeFreeProc;
    type.nullSize	= 1;
    type.clientData	= dataPtr;

    return Tcl_CreateEncoding(&type);
}

/*
 *-------------------------------------------------------------------------
 *
 * BinaryProc --
 *
 *	The default conversion when no other conversion is specified. No
 *	translation is done; source bytes are copied directly to destination
 *	bytes.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
BinaryProc(
    TCL_UNUSED(void *),
    const char *src,		/* Source string (unknown encoding). */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    int result;

    result = TCL_OK;
    dstLen -= TCL_UTF_MAX - 1;
    if (dstLen < 0) {
	dstLen = 0;
    }
    if ((flags & TCL_ENCODING_CHAR_LIMIT) && srcLen > *dstCharsPtr) {
	srcLen = *dstCharsPtr;
    }
    if (srcLen > dstLen) {
	srcLen = dstLen;
	result = TCL_CONVERT_NOSPACE;
    }

    *srcReadPtr = srcLen;
    *dstWrotePtr = srcLen;
    *dstCharsPtr = srcLen;
    memcpy(dst, src, srcLen);
    return result;
}

static int
UtfToUtfProc(
    void *clientData,
    const char *src,
    int srcLen,
    int flags,
    Tcl_EncodingState *statePtr,
    char *dst,
    int dstLen,
    int *srcReadPtr,
    int *dstWrotePtr,
    int *dstCharsPtr)
{
    const char *srcStart, *srcEnd, *srcClose;
    const char *dstStart, *dstEnd;
    int result, numChars = 0, charLimit = INT_MAX;
    int ch;
    int profile;

    *statePtr = 0;
    result = TCL_OK;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }

    dstStart = dst;
    flags |= PTR2INT(clientData);

    dstEnd = dst + dstLen - TCL_UTF_MAX;

    /* Checks if a source byte can be copied directly to destination */
#define BYTE_COPYABLE(byte_) \
    (UCHAR(byte_) < 0x80 && ((UCHAR(byte_) != 0)))

    profile = ENCODING_PROFILE_GET(flags);

    for (numChars = 0; src < srcEnd && numChars < charLimit; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}

	if (BYTE_COPYABLE(*src)) {
	    /*
	     * Common case fast path for non-nul ASCII bytes.
	     *
	     * Because we resolved any pending surrogate above (before the
	     * loop), *statePtr is guaranteed to be 0 here, so no
	     * CHECK_ISOLATEDSURROGATE() call is needed.
	     */
	    const char *p = src;
	    ptrdiff_t copyCount = srcEnd - src;
	    if (charLimit != INT_MAX) {
		if (copyCount > (charLimit - numChars)) {
		    copyCount = charLimit - numChars;
		}
	    }
	    if (copyCount > (dstEnd - dst + 1)) {
		copyCount = dstEnd - dst + 1;
	    }
	    const char *srcStop = src + copyCount;
	    while (p < srcStop && BYTE_COPYABLE(*p)) {
		*dst++ = *p++;
	    }
	    numChars += (int)(p - src) - 1;
	    src = p;
	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd) &&
		(UCHAR(src[1]) == 0x80) &&
		(!(flags & ENCODING_INPUT) || !PROFILE_TCL8(profile))) {
	    /* Special sequence \xC0\x80 */
	    if (!PROFILE_TCL8(profile) && (flags & ENCODING_INPUT)) {
		if (PROFILE_REPLACE(profile)) {
		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		    src += 2;
		} else {
		    /* PROFILE_STRICT */
		    result = TCL_CONVERT_SYNTAX;
		    break;
		}
	    } else {
		*dst++ = 0;
		src += 2;
	    }
	} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
	    /*
	     * Incomplete byte sequence (truncated UTF-8, not just end of
	     * source buffer — that is caught by the srcClose check above).
	     */
	    if (flags & ENCODING_INPUT) {
		if (PROFILE_STRICT(profile)) {
		    result = (flags & TCL_ENCODING_CHAR_LIMIT)
			    ? TCL_CONVERT_MULTIBYTE
			    : TCL_CONVERT_SYNTAX;
		    break;
		}
	    }
	    if (PROFILE_REPLACE(profile)) {
		ch = UNICODE_REPLACE_CHAR;
		++src;
	    } else {
		/* TCL_ENCODING_PROFILE_TCL8 */
		char chbuf[2];
		chbuf[0] = UCHAR(*src++);
		chbuf[1] = 0;
		TclUtfToUniChar(chbuf, &ch);
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	} else {
	    /* Have a complete character */
	    size_t len = TclUtfToUniChar(src, &ch);

	    /*
	     * For invalid inputs, Tcl_UtfToUniChar will return the byte
	     * itself and len == 1. We know this is invalid because valid
	     * single byte ASCII is already handled above except for 0.
	     * For the output case, Tcl should not have invalid byte sequences
	     * internally but if it does, garbage in, garbage out. This is
	     * historical behavior that should be changed imho. See
	     * ticket [b69e00ecf6]. TODO.
	     */
	    if (flags & ENCODING_INPUT) {
		if ((len < 2) && (ch != 0)) {
		    if (PROFILE_STRICT(profile)) {
			result = TCL_CONVERT_SYNTAX;
			break;
		    } else if (PROFILE_REPLACE(profile)) {
			ch = UNICODE_REPLACE_CHAR;
		    }
		}
	    }

	    const char *saveSrc = src;
	    src += len;

	    if (SURROGATE(ch)) {
		if (PROFILE_STRICT(profile)) {
		    result = (flags & ENCODING_INPUT)
			    ? TCL_CONVERT_SYNTAX : TCL_CONVERT_UNKNOWN;
		    src = saveSrc;
		    break;
		} else if (PROFILE_REPLACE(profile)) {
		    ch = UNICODE_REPLACE_CHAR;
		}
		/* PROFILE_TCL8: fall through and output as-is */
	    }
	    /* Normal character (or surrogate resolved to replacement/as-is) */
	    assert(ch >= 0 && ch <= 0x10FFFF);
	    if (ch == 0) {
		if (flags & ENCODING_INPUT) {
		    *dst++ = (char)0xC0;
		    *dst++ = (char)0x80;
		} else {
		    *dst++ = 0;
		}
	    } else if ((unsigned)ch < 0x800) {
		assert(ch >= 0x80);
		*dst++ = (char)(0xC0 | (ch >> 6));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    } else if ((unsigned)ch < 0x10000) {
		*dst++ = (char)(0xE0 | (ch >> 12));
		*dst++ = (char)(0x80 | ((ch >> 6) & 0x3F));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    } else {
		*dst++ = (char)(0xF0 | (ch >> 18));
		*dst++ = (char)(0x80 | ((ch >> 12) & 0x3F));
		*dst++ = (char)(0x80 | ((ch >> 6)  & 0x3F));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    }
	}
    }

    *srcReadPtr  = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;

#undef BYTE_COPYABLE
}

/*
 *----------------------------------------------------------------------
 *
 * UtfToCesu8Proc --
 *
 *	Converts TUTF-8 to CESU-8 unless ENCODING_INPUT is set in flags
 *	in which case it converts TUTF-8 to TUTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static int
UtfToCesu8Proc(
    void *clientData,		/* additional flags */
    const char *src,		/* Source string in CESU-8 if ENCODING_INPUT
				 * is set in flags, and TUTF-8 otherwise. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* TCL_ENCODING_* conversion control flags
				 * and ENCODING_INPUT as described above. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose;
    const char *dstStart, *dstEnd;
    int result, numChars, charLimit = INT_MAX;
    int ch;
    int profile;

    if (flags & TCL_ENCODING_START) {
	/* *statePtr will hold high surrogate in a split surrogate pair */
	*statePtr = 0;
    }
    result = TCL_OK;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= 6;
    }
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }

    dstStart = dst;
    flags |= PTR2INT(clientData);

    /*
     * Max space for one char is TCL_UTF_MAX for TUTF-8 output, 6 for CESU-8.
     */
    dstEnd = dst + dstLen - ((flags & ENCODING_INPUT) ? TCL_UTF_MAX : 6);

    /*
     * Macro to output an isolated high surrogate when it is not followed
     * by a low surrogate. NOT to be called for strict profile since
     * that should raise an error.
     */
#define OUTPUT_ISOLATEDSURROGATE() \
    do {								\
	Tcl_UniChar high;						\
	if (PROFILE_REPLACE(profile)) {					\
	    high = UNICODE_REPLACE_CHAR;				\
	} else {							\
	    high = (Tcl_UniChar)(ptrdiff_t) *statePtr;			\
	}								\
	assert(HIGH_SURROGATE(high));					\
	assert(!PROFILE_STRICT(profile));				\
	dst += Tcl_UniCharToUtf(high, dst);				\
	*statePtr = 0; /* Reset state */				\
    } while (0)

    /* Checks if a source byte can be copied directly to destination */
#define BYTE_COPYABLE(byte_) \
    (UCHAR(byte_) < 0x80 && ((UCHAR(byte_) != 0)))

    /*
     * Macro to check for isolated surrogate and either break with
     * an error if profile is strict, or output an appropriate
     * character for replace and tcl8 profiles and continue.
     */
#define CHECK_ISOLATEDSURROGATE() \
    if (*statePtr) {							\
	if (PROFILE_STRICT(profile)) {					\
	    result = TCL_CONVERT_SYNTAX;				\
	    break;							\
	}								\
	OUTPUT_ISOLATEDSURROGATE();					\
	continue; /* Rerun loop so length checks etc. repeated */	\
    } else								\
	(void) 0

    profile = ENCODING_PROFILE_GET(flags);
    for (numChars = 0; src < srcEnd && numChars < charLimit; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	if (BYTE_COPYABLE(*src)) {
	    CHECK_ISOLATEDSURROGATE();
	    /*
	     * Common case fast path for non-0 ASCII. Condition to be met
	     *  - ASCII byte
	     *  - not nul byte when in ENCODING_INPUT (nul must be converted)
	     *  - charLimit not reached
	     *  - destination not full
	     */
	    const char *p = src;
	    ptrdiff_t copyCount = srcEnd - src;
	    /* Limit to caller-specified character count */
	    if (charLimit != INT_MAX) {
		if (copyCount > (charLimit - numChars)) {
		    copyCount = charLimit - numChars;
		}
	    }
	    /*
	     * Limit to remaining destination space. dstEnd is the last
	     * location we can write to, not the upper bound.
	     */
	    if (copyCount > (dstEnd - dst + 1)) {
		copyCount = dstEnd - dst + 1;
	    }
	    const char *srcStop = src + copyCount;
	    while (p < srcStop && BYTE_COPYABLE(*p)) {
		*dst++ = *p++;
	    }
	    numChars += (int)(p - src) - 1; /* -1 to adjust loop increment */
	    src = p;
	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd) &&
		(UCHAR(src[1]) == 0x80) &&
		(!(flags & ENCODING_INPUT) || !PROFILE_TCL8(profile))) {
	    /* Special sequence \xC0\x80 */

	    CHECK_ISOLATEDSURROGATE();
	    if (!PROFILE_TCL8(profile) && (flags & ENCODING_INPUT)) {
		if (PROFILE_REPLACE(profile)) {
		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		    src += 2;
		} else {
		    /* PROFILE_STRICT */
		    result = TCL_CONVERT_SYNTAX;
		    break;
		}
	    } else {
		/*
		 * Convert 0xC080 to real nulls when we are in output mode,
		 * irrespective of the profile.
		 */
		*dst++ = 0;
		src += 2;
	    }
	} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
	    /*
	     * Incomplete byte sequence not because there are insufficient
	     * bytes in source buffer (have already checked that above) but
	     * because the UTF-8 sequence is truncated.
	     */

	    CHECK_ISOLATEDSURROGATE();

	    if (flags & ENCODING_INPUT) {
		/* Incomplete bytes for modified UTF-8 target */
		if (PROFILE_STRICT(profile)) {
		    result = (flags & TCL_ENCODING_CHAR_LIMIT)
			    ? TCL_CONVERT_MULTIBYTE
			    : TCL_CONVERT_SYNTAX;
		    break;
		}
	    }
	    if (PROFILE_REPLACE(profile)) {
		ch = UNICODE_REPLACE_CHAR;
		++src;
	    } else {
		/* TCL_ENCODING_PROFILE_TCL8 */
		char chbuf[2];
		chbuf[0] = UCHAR(*src++);
		chbuf[1] = 0;
		TclUtfToUniChar(chbuf, &ch);
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	} else {
	    /* Have a complete character */
	    size_t len = TclUtfToUniChar(src, &ch);

	    Tcl_UniChar savedSurrogate = (Tcl_UniChar) (ptrdiff_t)*statePtr;
	    *statePtr = 0; /* Reset surrogate */

	    if (flags & ENCODING_INPUT) {
		if (((len < 2) && (ch != 0)) || (ch > 0xFFFF)) {
		    if (PROFILE_STRICT(profile)) {
			result = TCL_CONVERT_SYNTAX;
			break;
		    } else if (PROFILE_REPLACE(profile)) {
			ch = UNICODE_REPLACE_CHAR;
		    }
		}
	    }

	    const char *saveSrc = src;
	    src += len;
	    if (!(flags & ENCODING_INPUT) && (ch > 0x7FF)) {
		assert(savedSurrogate == 0);	/* Since this flag combo
						 * will never set *statePtr */
		if (ch > 0xFFFF) {
		    /* CESU-8 6-byte sequence for chars > U+FFFF */
		    ch -= 0x10000;
		    *dst++ = 0xED;
		    *dst++ = (char) (((ch >> 16) & 0x0F) | 0xA0);
		    *dst++ = (char) (((ch >> 10) & 0x3F) | 0x80);
		    ch = (ch & 0x03FF) | 0xDC00;
		}
		*dst++ = (char)(((ch >> 12) | 0xE0) & 0xEF);
		*dst++ = (char)(((ch >> 6) | 0x80) & 0xBF);
		*dst++ = (char)((ch | 0x80) & 0xBF);
		continue;
	    } else if (SURROGATE(ch)) {
		/* CESU-8 */
		if (LOW_SURROGATE(ch)) {
		    if (savedSurrogate) {
			assert(HIGH_SURROGATE(savedSurrogate));
			ch = 0x10000 + ((savedSurrogate - 0xd800) << 10) +
				(ch - 0xdc00);
		    } else {
			/* Isolated low surrogate */
			if (PROFILE_STRICT(profile)) {
			    result = (flags & ENCODING_INPUT)
				    ? TCL_CONVERT_SYNTAX
				    : TCL_CONVERT_UNKNOWN;
			    src = saveSrc;
			    break;
			} else if (PROFILE_REPLACE(profile)) {
			    ch = UNICODE_REPLACE_CHAR;
			} else {
			    /* Tcl8 profile. Output low surrogate as is */
			}
		    }
		} else {
		    assert(HIGH_SURROGATE(ch));
		    /* Save the high surrogate */
		    *statePtr = (Tcl_EncodingState)(ptrdiff_t)ch;
		    if (savedSurrogate) {
			assert(HIGH_SURROGATE(savedSurrogate));
			if (PROFILE_STRICT(profile)) {
			    result = (flags & ENCODING_INPUT)
				    ? TCL_CONVERT_SYNTAX
				    : TCL_CONVERT_UNKNOWN;
			    src = saveSrc;
			    break;
			} else if (PROFILE_REPLACE(profile)) {
			    ch = UNICODE_REPLACE_CHAR;
			} else {
			    /* Output the isolated high surrogate */
			    ch = savedSurrogate;
			}
		    } else {
			/* High surrogate saved in *statePtr. Do not output
			 * anything just yet. */
			--numChars; /* Cancel the increment at end of loop */
			continue;
		    }
		}
	    } else {
		/* Normal character */
		CHECK_ISOLATEDSURROGATE();
	    }

	    assert(ch >= 0 && ch <= 0x10FFFF);
	    if (ch == 0) {
		/* Nul character must be encoded as \xC0\x80 in Tcl's UTF-8 */
		if (flags & ENCODING_INPUT) {
		    *dst++ = (char)0xC0;
		    *dst++ = (char)0x80;
		} else {
		    /*
		     * Can only happen if Tcl's internal string rep contains a
		     * nul byte (which is an invalid case but protect anyways)
		     */
		    *dst++ = 0;
		}
	    } else if ((unsigned)ch < 0x800) {
		/* 0 < ch < 0x80 case already handled in ASCII path earlier */
		assert(ch >= 0x80);
		*dst++ = (char)(0xC0 | (ch >> 6));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    } else if ((unsigned)ch < 0x10000) {
		*dst++ = (char)(0xE0 | (ch >> 12));
		*dst++ = (char)(0x80 | ((ch >> 6) & 0x3F));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    } else {
		/* Here, always ch < 0x110000 */
		*dst++ = (char)(0xF0 | (ch >> 18));
		*dst++ = (char)(0x80 | ((ch >> 12) & 0x3F));
		*dst++ = (char)(0x80 | ((ch >> 6) & 0x3F));
		*dst++ = (char)(0x80 | (ch & 0x3F));
	    }
	}
    }

    /* Check if an high surrogate left over */
    if (*statePtr) {
	if (!(flags & TCL_ENCODING_END)) {
	    /* More data coming */
	} else {
	    /* No more data coming */
	    if (PROFILE_STRICT(profile)) {
		result = (flags & ENCODING_INPUT)
			? TCL_CONVERT_SYNTAX : TCL_CONVERT_UNKNOWN;
	    } else {
		if (PROFILE_REPLACE(profile)) {
		    ch = UNICODE_REPLACE_CHAR;
		} else {
		    ch = (Tcl_UniChar) (ptrdiff_t) *statePtr;
		}
		if (dst < dstEnd) {
		    dst += Tcl_UniCharToUtf(ch, dst);
		    ++numChars;
		} else {
		    /* No room in destination */
		    result = TCL_CONVERT_NOSPACE;
		}
	    }
	}
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;

#undef BYTE_COPYABLE
}

/*
 *----------------------------------------------------------------------
 *
 * Cesu8ToUtfProc --
 *
 *	Converts CESU-8 to TUTF-8.
 *	See UtfToCesu8Proc for all parameter details.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */
static int
Cesu8ToUtfProc(
    void *clientData,
    const char *src, int srcLen,
    int flags,
    Tcl_EncodingState *statePtr,
    char *dst, int dstLen,
    int *srcReadPtr,
    int *dstWrotePtr,
    int *dstCharsPtr)
{
    return UtfToCesu8Proc(clientData,
	    src, srcLen, flags | ENCODING_INPUT, statePtr,
	    dst, dstLen, srcReadPtr, dstWrotePtr, dstCharsPtr);
}

/*
 *-------------------------------------------------------------------------
 *
 * Utf32ToUtfProc --
 *
 *	Convert from UTF-32 to UTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
Utf32ToUtfProc(
    void *clientData,		/* additional flags, e.g. TCL_ENCODING_LE */
    const char *src,		/* Source string in Unicode. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd;
    const char *dstEnd, *dstStart;
    int result, numChars, charLimit = INT_MAX;
    int ch = 0, bytesLeft = srcLen % 4;

    flags |= PTR2INT(clientData);
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }
    result = TCL_OK;

    /*
     * Check alignment with utf-32 (4 == sizeof(UTF-32))
     */
    if (bytesLeft != 0) {
	/* We have a truncated code unit */
	result = TCL_CONVERT_MULTIBYTE;
	srcLen -= bytesLeft;
    }

    srcStart = src;
    srcEnd = src + srcLen;

    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

    for (numChars = 0; src < srcEnd && numChars < charLimit; numChars++) {
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}

	if (flags & TCL_ENCODING_LE) {
	    ch = (unsigned int)(src[3] & 0xFF) << 24 | (src[2] & 0xFF) << 16
		    | (src[1] & 0xFF) << 8 | (src[0] & 0xFF);
	} else {
	    ch = (unsigned int)(src[0] & 0xFF) << 24 | (src[1] & 0xFF) << 16
		    | (src[2] & 0xFF) << 8 | (src[3] & 0xFF);
	}
	if ((unsigned)ch > 0x10FFFF) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    ch = UNICODE_REPLACE_CHAR;
	} else if (SURROGATE(ch)) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    if (PROFILE_REPLACE(flags)) {
		ch = UNICODE_REPLACE_CHAR;
	    }
	}

	/*
	 * Special case for 1-byte utf chars for speed. Make sure we work with
	 * unsigned short-size data.
	 */

	if ((unsigned)ch - 1 < 0x7F) {
	    *dst++ = (ch & 0xFF);
	} else {
	    dst += Tcl_UniCharToUtf(ch, dst);
	}
	src += 4;
    }

    if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
	/* We have a code fragment left-over at the end */
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	} else {
	    /* destination is not full, so we really are at the end now */
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
	    } else {
		/* PROFILE_REPLACE or PROFILE_TCL8 */
		result = TCL_OK;
		dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		numChars++;
		src += bytesLeft; /* Go past truncated code unit */
	    }
	}
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * UtfToUtf32Proc --
 *
 *	Convert from UTF-8 to UTF-32.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
UtfToUtf32Proc(
    void *clientData,		/* additional flags, e.g. TCL_ENCODING_LE */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
    int result, numChars;
    int ch, len;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd = dst + dstLen - sizeof(Tcl_UniChar);
    flags |= PTR2INT(clientData);

    result = TCL_OK;
    for (numChars = 0; src < srcEnd; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);
	if (SURROGATE(ch)) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_UNKNOWN;
		break;
	    }
	    if (PROFILE_REPLACE(flags)) {
		ch = UNICODE_REPLACE_CHAR;
	    }
	}
	src += len;
	if (flags & TCL_ENCODING_LE) {
	    *dst++ = (ch & 0xFF);
	    *dst++ = ((ch >> 8) & 0xFF);
	    *dst++ = ((ch >> 16) & 0xFF);
	    *dst++ = ((ch >> 24) & 0xFF);
	} else {
	    *dst++ = ((ch >> 24) & 0xFF);
	    *dst++ = ((ch >> 16) & 0xFF);
	    *dst++ = ((ch >> 8) & 0xFF);
	    *dst++ = (ch & 0xFF);
	}
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Utf16ToUtfProc --
 *
 *	Convert from UTF-16 to UTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
Utf16ToUtfProc(
    void *clientData,		/* additional flags, e.g. TCL_ENCODING_LE */
    const char *src,		/* Source string in Unicode. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd;
    const char *dstEnd, *dstStart;
    int result, numChars, charLimit = INT_MAX;
    unsigned short ch = 0;

    flags |= PTR2INT(clientData);
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }
    result = TCL_OK;

    /*
     * Check alignment with utf-16 (2 == sizeof(UTF-16))
     */

    if ((srcLen % 2) != 0) {
	result = TCL_CONVERT_MULTIBYTE;
	srcLen--;
    }

#if 0
    /*
     * If last code point is a high surrogate, we cannot handle that yet,
     * unless we are at the end.
     */

    if (!(flags & TCL_ENCODING_END) && (srcLen >= 2) &&
	    ((src[srcLen - ((flags & TCL_ENCODING_LE)?1:2)] & 0xFC) == 0xD8)) {
	result = TCL_CONVERT_MULTIBYTE;
	srcLen-= 2;
    }
#endif

    srcStart = src;
    srcEnd = src + srcLen;

    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

    for (numChars = 0; src < srcEnd && numChars < charLimit; src += 2, numChars++) {
	if (dst > dstEnd && !HIGH_SURROGATE(ch)) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}

	unsigned short prev = ch;
	if (flags & TCL_ENCODING_LE) {
	    ch = (src[1] & 0xFF) << 8 | (src[0] & 0xFF);
	} else {
	    ch = (src[0] & 0xFF) << 8 | (src[1] & 0xFF);
	}
	if (HIGH_SURROGATE(prev)) {
	    if (LOW_SURROGATE(ch)) {
		/*
		 * High surrogate was followed by a low surrogate.
		 * Tcl_UniCharToUtf would have stashed away the state in dst.
		 * Call it again to combine that state with the low surrogate.
		 * We also have to compensate the numChars as two UTF-16 units
		 * have been combined into one character.
		 */
		dst += Tcl_UniCharToUtf(ch | TCL_COMBINE, dst);
	    } else {
		/* High surrogate was not followed by a low surrogate */
		if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_SYNTAX;
		    src -= 2;	/* Go back to beginning of high surrogate */
		    dst--;	/* Also undo writing a single byte too much */
		    break;
		}
		if (PROFILE_REPLACE(flags)) {
		    /*
		     * Previous loop wrote a single byte to mark the high surrogate.
		     * Replace it with the replacement character.
		     */
		    ch = UNICODE_REPLACE_CHAR;
		    dst--;
		    numChars++;
		    dst += Tcl_UniCharToUtf(ch, dst);
		} else {
		    /*
		     * Bug [10c2c17c32]. If Hi surrogate not followed by Lo
		     * surrogate, finish 3-byte UTF-8
		     */
		    dst += Tcl_UniCharToUtf(-1, dst);
		}
		/* Loop around again so destination space and other checks are done */
		prev = 0; /* Reset high surrogate tracker */
		src -= 2;
	    }
	} else {
	    /* Previous char was not a high surrogate */

	    /*
	     * Special case for 1-byte utf chars for speed. Make sure we work with
	     * unsigned short-size data. Order checks based on expected frequency.
	     */
	    if ((unsigned)ch - 1 < 0x7F) {
		/* ASCII except nul */
		*dst++ = (ch & 0xFF);
	    } else if (!SURROGATE(ch)) {
		/* Not ASCII, not surrogate */
		dst += Tcl_UniCharToUtf(ch, dst);
	    } else if (HIGH_SURROGATE(ch)) {
		dst += Tcl_UniCharToUtf(ch | TCL_COMBINE, dst);
		/* Do not count this just yet. Compensate for numChars++ in loop counter */
		numChars--;
	    } else {
		assert(LOW_SURROGATE(ch));
		if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_SYNTAX;
		    break;
		}
		if (PROFILE_REPLACE(flags)) {
		    ch = UNICODE_REPLACE_CHAR;
		}
		dst += Tcl_UniCharToUtf(ch, dst);
	    }
	}
    }

    /*
     * When the above loop ends, result may have the following values:
     * 1. TCL_OK - full source buffer was completely processed.
     *    src, dst, numChars will hold values up to that point BUT
     *    there may be a leftover high surrogate we need to deal with.
     * 2. TCL_CONVERT_NOSPACE - Ran out of room in the destination buffer.
     *    Same considerations as (1)
     * 3. TCL_CONVERT_SYNTAX - decoding error.
     * 4. TCL_CONVERT_MULTIBYTE - the buffer passed in was not fully
     *    processed, because there was a trailing single byte. However,
     *    we *may* have processed the requested number of characters already
     *    in which case the trailing byte does not matter. We still
     *    *may* still be a leftover high surrogate as in (1) and (2).
     */
    switch (result) {
    case TCL_CONVERT_MULTIBYTE: /* FALLTHRU */
    case TCL_OK: /* FALLTHRU */
    case TCL_CONVERT_NOSPACE:
	if (HIGH_SURROGATE(ch)) {
	    if (flags & TCL_ENCODING_END) {
		/*
		 * No more data expected. There will be space for output of
		 * one character (essentially overwriting the dst area holding
		 * high surrogate state)
		 */
		assert((dst-1) <= dstEnd);
		if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_SYNTAX;
		    src -= 2;
		    dst--;
		} else if (PROFILE_REPLACE(flags)) {
		    dst--;
		    numChars++;
		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		} else {
		    /* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */
		    numChars++;
		    dst += Tcl_UniCharToUtf(-1, dst);
		}
	    } else {
		/* More data is expected. Revert the surrogate state */
		src -= 2;
		dst--;
		/* Note: leave result of TCL_CONVERT_NOSPACE as is */
		if (result == TCL_OK) {
		    result = TCL_CONVERT_MULTIBYTE;
		}
	    }
	} else if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
	    /*
	     * If we had a trailing byte at the end AND this is the last
	     * fragment AND profile is not "strict", stick FFFD in its place.
	     * Note in this case we DO need to check for room in dst.
	     */
	    if (dst > dstEnd) {
		result = TCL_CONVERT_NOSPACE;
	    } else {
		if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_SYNTAX;
		} else {
		    /* PROFILE_REPLACE or PROFILE_TCL8 */
		    result = TCL_OK;
		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		    numChars++;
		    src++;
		}
	    }
	}
	break;
    case TCL_CONVERT_SYNTAX:
	break; /* Nothing to do */
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * UtfToUtf16Proc --
 *
 *	Convert from UTF-8 to UTF-16.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
UtfToUtf16Proc(
    void *clientData,		/* additional flags, e.g. TCL_ENCODING_LE */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
    int result, numChars;
    int ch, len;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd   = dst + dstLen - 2; /* 2 -> sizeof a UTF-16 code unit */
    flags |= PTR2INT(clientData);

    result = TCL_OK;
    for (numChars = 0; src < srcEnd; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);
	if (SURROGATE(ch)) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_UNKNOWN;
		break;
	    }
	    if (PROFILE_REPLACE(flags)) {
		ch = UNICODE_REPLACE_CHAR;
	    }
	}
	if (ch <= 0xFFFF) {
	    if (flags & TCL_ENCODING_LE) {
		*dst++ = (ch & 0xFF);
		*dst++ = (ch >> 8);
	    } else {
		*dst++ = (ch >> 8);
		*dst++ = (ch & 0xFF);
	    }
	} else {
	    if ((dst+2) > dstEnd) {
		/* Surrogates need 2 more bytes! Bug [66da4d4228] */
		result = TCL_CONVERT_NOSPACE;
		break;
	    }
	    if (flags & TCL_ENCODING_LE) {
		*dst++ = (((ch - 0x10000) >> 10) & 0xFF);
		*dst++ = (((ch - 0x10000) >> 18) & 0x3) | 0xD8;
		*dst++ = (ch & 0xFF);
		*dst++ = ((ch >> 8) & 0x3) | 0xDC;
	    } else {
		*dst++ = (((ch - 0x10000) >> 18) & 0x3) | 0xD8;
		*dst++ = (((ch - 0x10000) >> 10) & 0xFF);
		*dst++ = ((ch >> 8) & 0x3) | 0xDC;
		*dst++ = (ch & 0xFF);
	    }
	}
	src += len;
    }
    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * UtfToUcs2Proc --
 *
 *	Convert from UTF-8 to UCS-2.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
UtfToUcs2Proc(
    void *clientData,		/* additional flags, e.g. TCL_ENCODING_LE */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
    int result, numChars, len;
    Tcl_UniChar ch = 0;

    flags |= PTR2INT(clientData);
    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd   = dst + dstLen - 2; /* 2 - size of UCS code unit */

    result = TCL_OK;
    for (numChars = 0; src < srcEnd; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);
	if (ch > 0xFFFF) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_UNKNOWN;
		break;
	    }
	    ch = UNICODE_REPLACE_CHAR;
	}
	if (PROFILE_STRICT(flags) && SURROGATE(ch)) {
	    result = TCL_CONVERT_SYNTAX;
	    break;
	}

	src += len;

	/*
	 * Need to handle this in a way that won't cause misalignment by
	 * casting dst to a Tcl_UniChar. [Bug 1122671]
	 */

	if (flags & TCL_ENCODING_LE) {
	    *dst++ = (ch & 0xFF);
	    *dst++ = (ch >> 8);
	} else {
	    *dst++ = (ch >> 8);
	    *dst++ = (ch & 0xFF);
	}
    }
    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * TableToUtfProc --
 *
 *	Convert from the encoding specified by the TableEncodingData into
 *	UTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
TableToUtfProc(
    void *clientData,		/* TableEncodingData that specifies
				 * encoding. */
    const char *src,		/* Source string in specified encoding. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd;
    const char *dstEnd, *dstStart, *prefixBytes;
    int result, byte, numChars, charLimit = INT_MAX;
    Tcl_UniChar ch = 0;
    const unsigned short *const *toUnicode;
    const unsigned short *pageZero;
    TableEncodingData *dataPtr = (TableEncodingData *)clientData;

    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }
    srcStart = src;
    srcEnd = src + srcLen;

    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

    toUnicode = (const unsigned short *const *) dataPtr->toUnicode;
    prefixBytes = dataPtr->prefixBytes;
    pageZero = toUnicode[0];

    result = TCL_OK;
    for (numChars = 0; src < srcEnd && numChars < charLimit; numChars++) {
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	byte = *((unsigned char *) src);
	if (prefixBytes[byte]) {
	    if (src >= srcEnd-1) {
		/* Prefix byte but nothing after it */
		if (!(flags & TCL_ENCODING_END)) {
		    /* More data to come */
		    result = TCL_CONVERT_MULTIBYTE;
		    break;
		} else if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_SYNTAX;
		    break;
		} else if (PROFILE_REPLACE(flags)) {
		    ch = UNICODE_REPLACE_CHAR;
		} else {
		    /* For prefix bytes, we don't fallback to cp1252, see [1355b9a874] */
		    ch = byte;
		}
	    } else {
		ch = toUnicode[byte][*((unsigned char *)++src)];
	    }
	} else {
	    ch = pageZero[byte];
	}
	if ((ch == 0) && (byte != 0)) {
	    /* Prefix+suffix pair is invalid */
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    if (prefixBytes[byte]) {
		src--;
	    }
	    if (PROFILE_REPLACE(flags)) {
		ch = UNICODE_REPLACE_CHAR;
	    } else {
		char chbuf[2];
		chbuf[0] = byte;
		chbuf[1] = 0;
		TclUtfToUniChar(chbuf, &ch);
	    }
	}

	/*
	 * Special case for 1-byte Utf chars for speed.
	 */

	if ((unsigned)ch - 1 < 0x7F) {
	    *dst++ = (char) ch;
	} else {
	    dst += Tcl_UniCharToUtf(ch, dst);
	}
	src++;
    }

    assert(src <= srcEnd);
    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * TableFromUtfProc --
 *
 *	Convert from UTF-8 into the encoding specified by the
 *	TableEncodingData.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
TableFromUtfProc(
    void *clientData,		/* TableEncodingData that specifies
				 * encoding. */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose;
    const char *dstStart, *dstEnd, *prefixBytes;
    Tcl_UniChar ch = 0;
    int result, len, word, numChars;
    TableEncodingData *dataPtr = (TableEncodingData *)clientData;
    const unsigned short *const *fromUnicode;

    result = TCL_OK;

    prefixBytes = dataPtr->prefixBytes;
    fromUnicode = (const unsigned short *const *) dataPtr->fromUnicode;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd = dst + dstLen - 1;

    for (numChars = 0; src < srcEnd; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);

	/* Unicode chars > +U0FFFF cannot be represented in any table encoding */
	if (ch & 0xFFFF0000) {
	    word = 0;
	} else {
	    word = fromUnicode[(ch >> 8)][ch & 0xFF];
	}

	if ((word == 0) && (ch != 0)) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_UNKNOWN;
		break;
	    }
	    word = dataPtr->fallback; /* Both profiles REPLACE and TCL8 */
	}
	if (prefixBytes[(word >> 8)] != 0) {
	    if (dst + 1 > dstEnd) {
		result = TCL_CONVERT_NOSPACE;
		break;
	    }
	    dst[0] = (char) (word >> 8);
	    dst[1] = (char) word;
	    dst += 2;
	} else {
	    if (dst > dstEnd) {
		result = TCL_CONVERT_NOSPACE;
		break;
	    }
	    dst[0] = (char) word;
	    dst++;
	}
	src += len;
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Iso88591ToUtfProc --
 *
 *	Convert from the "iso8859-1" encoding into UTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
Iso88591ToUtfProc(
    TCL_UNUSED(void *),
    const char *src,		/* Source string in specified encoding. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd;
    const char *dstEnd, *dstStart;
    int result, numChars, charLimit = INT_MAX;

    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }
    srcStart = src;
    srcEnd = src + srcLen;

    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

    result = TCL_OK;
    for (numChars = 0; src < srcEnd && numChars < charLimit; numChars++) {
	Tcl_UniChar ch = 0;

	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	ch = *((unsigned char *) src);

	/*
	 * Special case for 1-byte utf chars for speed.
	 */

	if ((unsigned)ch - 1 < 0x7F) {
	    *dst++ = (char) ch;
	} else {
	    dst += Tcl_UniCharToUtf(ch, dst);
	}
	src++;
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * Iso88591FromUtfProc --
 *
 *	Convert from UTF-8 into the encoding "iso8859-1".
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
Iso88591FromUtfProc(
    TCL_UNUSED(void *),
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    TCL_UNUSED(Tcl_EncodingState *),
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    const char *srcStart, *srcEnd, *srcClose;
    const char *dstStart, *dstEnd;
    int result = TCL_OK, numChars;
    Tcl_UniChar ch = 0;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd = dst + dstLen - 1;

    for (numChars = 0; src < srcEnd; numChars++) {
	int len;

	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);

	/*
	 * Check for illegal characters.
	 */

	if (ch > 0xFF) {
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_UNKNOWN;
		break;
	    }
	    /*
	     * Plunge on, using '?' as a fallback character.
	     */

	    ch = '?'; /* Profiles TCL8 and REPLACE */
	}

	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	*(dst++) = (char) ch;
	src += len;
    }

    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *---------------------------------------------------------------------------
 *
 * TableFreeProc --
 *
 *	This function is invoked when an encoding is deleted. It deletes the
 *	memory used by the TableEncodingData.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Memory freed.
 *
 *---------------------------------------------------------------------------
 */

static void
TableFreeProc(
    void *clientData)		/* TableEncodingData that specifies
				 * encoding. */
{
    TableEncodingData *dataPtr = (TableEncodingData *)clientData;

    /*
     * Make sure we aren't freeing twice on shutdown. [Bug 219314]
     */

    Tcl_Free(dataPtr->toUnicode);
    dataPtr->toUnicode = NULL;
    Tcl_Free(dataPtr->fromUnicode);
    dataPtr->fromUnicode = NULL;
    Tcl_Free(dataPtr);
}

/*
 *-------------------------------------------------------------------------
 *
 * EscapeToUtfProc --
 *
 *	Convert from the encoding specified by the EscapeEncodingData into
 *	UTF-8.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
EscapeToUtfProc(
    void *clientData,		/* EscapeEncodingData that specifies
				 * encoding. */
    const char *src,		/* Source string in specified encoding. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
    const char *prefixBytes, *tablePrefixBytes, *srcStart, *srcEnd;
    const unsigned short *const *tableToUnicode;
    const Encoding *encodingPtr;
    int state, result, numChars, charLimit = INT_MAX;
    const char *dstStart, *dstEnd;

    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }
    result = TCL_OK;
    tablePrefixBytes = NULL;
    tableToUnicode = NULL;
    prefixBytes = dataPtr->prefixBytes;
    encodingPtr = NULL;

    srcStart = src;
    srcEnd = src + srcLen;

    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

    state = PTR2INT(*statePtr);
    if (flags & TCL_ENCODING_START) {
	state = 0;
    }

    for (numChars = 0; src < srcEnd && numChars < charLimit; ) {
	int byte, hi, lo, ch;

	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	byte = *((unsigned char *) src);
	if (prefixBytes[byte]) {
	    unsigned left, len, longest;
	    int checked, i;
	    const EscapeSubTable *subTablePtr;

	    /*
	     * Saw the beginning of an escape sequence.
	     */

	    left = srcEnd - src;
	    len = dataPtr->initLen;
	    longest = len;
	    checked = 0;

	    if (len <= left) {
		checked++;
		if ((len > 0) && (memcmp(src, dataPtr->init, len) == 0)) {
		    /*
		     * If we see initialization string, skip it, even if we're
		     * not at the beginning of the buffer.
		     */

		    src += len;
		    continue;
		}
	    }

	    len = dataPtr->finalLen;
	    if (len > longest) {
		longest = len;
	    }

	    if (len <= left) {
		checked++;
		if ((len > 0) && (memcmp(src, dataPtr->final, len) == 0)) {
		    /*
		     * If we see finalization string, skip it, even if we're
		     * not at the end of the buffer.
		     */

		    src += len;
		    continue;
		}
	    }

	    subTablePtr = dataPtr->subTables;
	    for (i = 0; i < dataPtr->numSubTables; i++) {
		len = subTablePtr->sequenceLen;
		if (len > longest) {
		    longest = len;
		}
		if (len <= left) {
		    checked++;
		    if ((len > 0) &&
			    (memcmp(src, subTablePtr->sequence, len) == 0)) {
			state = i;
			encodingPtr = NULL;
			subTablePtr = NULL;
			src += len;
			break;
		    }
		}
		subTablePtr++;
	    }

	    if (subTablePtr == NULL) {
		/*
		 * A match was found, the escape sequence was consumed, and
		 * the state was updated.
		 */

		continue;
	    }

	    /*
	     * We have a split-up or unrecognized escape sequence. If we
	     * checked all the sequences, then it's a syntax error, otherwise
	     * we need more bytes to determine a match.
	     */

	    if ((checked == dataPtr->numSubTables + 2)
		    || (flags & TCL_ENCODING_END)) {
		if (!PROFILE_STRICT(flags)) {
		    unsigned skip = longest > left ? left : longest;
		    /* Unknown escape sequence */
		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		    src += skip;
		    continue;
		}
		result = TCL_CONVERT_SYNTAX;
	    } else {
		result = TCL_CONVERT_MULTIBYTE;
	    }
	    break;
	}

	if (encodingPtr == NULL) {
	    TableEncodingData *tableDataPtr;

	    encodingPtr = GetTableEncoding(dataPtr, state);
	    tableDataPtr = (TableEncodingData *)encodingPtr->clientData;
	    tablePrefixBytes = tableDataPtr->prefixBytes;
	    tableToUnicode = (const unsigned short *const*)
		    tableDataPtr->toUnicode;
	}

	if (tablePrefixBytes[byte]) {
	    src++;
	    if (src >= srcEnd) {
		src--;
		result = TCL_CONVERT_MULTIBYTE;
		break;
	    }
	    hi = byte;
	    lo = *((unsigned char *) src);
	} else {
	    hi = 0;
	    lo = byte;
	}

	ch = tableToUnicode[hi][lo];
	dst += Tcl_UniCharToUtf(ch, dst);
	src++;
	numChars++;
    }

    if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
	/* We have a code fragment left-over at the end */
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	} else {
	    /* destination is not full, so we really are at the end now */
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
	    } else {
		/*
		 * PROFILE_REPLACE or PROFILE_TCL8. The latter is treated
		 * similar to former because Tcl8 was broken in this regard
		 * as it just ignored the byte and truncated which is really
		 * a no-no as per Unicode recommendations.
		 */
		result = TCL_OK;
		dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
		numChars++;
		/* TCL_CONVERT_MULTIBYTE means all source consumed */
		src = srcEnd;
	    }
	}
    }

    *statePtr = (Tcl_EncodingState) INT2PTR(state);
    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *-------------------------------------------------------------------------
 *
 * EscapeFromUtfProc --
 *
 *	Convert from UTF-8 into the encoding specified by the
 *	EscapeEncodingData.
 *
 * Results:
 *	Returns TCL_OK if conversion was successful.
 *
 * Side effects:
 *	None.
 *
 *-------------------------------------------------------------------------
 */

static int
EscapeFromUtfProc(
    void *clientData,		/* EscapeEncodingData that specifies
				 * encoding. */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes. */
    int flags,			/* Conversion control flags. */
    Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
				 * information used during a piecewise
				 * conversion. Contents of statePtr are
				 * initialized and/or reset by conversion
				 * routine under control of flags argument. */
    char *dst,			/* Output buffer in which converted string is
				 * stored. */
    int dstLen,			/* The maximum length of output buffer in
				 * bytes. */
    int *srcReadPtr,		/* Filled with the number of bytes from the
				 * source string that were converted. This may
				 * be less than the original source length if
				 * there was a problem converting some source
				 * characters. */
    int *dstWrotePtr,		/* Filled with the number of bytes that were
				 * stored in the output buffer as a result of
				 * the conversion. */
    int *dstCharsPtr)		/* Filled with the number of characters that
				 * correspond to the bytes stored in the
				 * output buffer. */
{
    EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
    const Encoding *encodingPtr;
    const char *srcStart, *srcEnd, *srcClose;
    const char *dstStart, *dstEnd;
    int state, result, numChars;
    const TableEncodingData *tableDataPtr;
    const char *tablePrefixBytes;
    const unsigned short *const *tableFromUnicode;
    Tcl_UniChar ch = 0;

    result = TCL_OK;

    srcStart = src;
    srcEnd = src + srcLen;
    srcClose = srcEnd;
    if ((flags & TCL_ENCODING_END) == 0) {
	srcClose -= TCL_UTF_MAX;
    }

    dstStart = dst;
    dstEnd = dst + dstLen - 1;

    /*
     * RFC 1468 states that the text starts in ASCII, and switches to Japanese
     * characters, and that the text must end in ASCII. [Patch 474358]
     */

    if (flags & TCL_ENCODING_START) {
	state = 0;
	if ((dst + dataPtr->initLen) > dstEnd) {
	    *srcReadPtr = 0;
	    *dstWrotePtr = 0;
	    return TCL_CONVERT_NOSPACE;
	}
	memcpy(dst, dataPtr->init, dataPtr->initLen);
	dst += dataPtr->initLen;
    } else {
	state = PTR2INT(*statePtr);
    }

    encodingPtr = GetTableEncoding(dataPtr, state);
    tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
    tablePrefixBytes = tableDataPtr->prefixBytes;
    tableFromUnicode = (const unsigned short *const *)
	    tableDataPtr->fromUnicode;

    for (numChars = 0; src < srcEnd; numChars++) {
	unsigned len;
	int word;

	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	len = TclUtfToUniChar(src, &ch);
	if (ch > 0xFFFF) {
	    /* Bug 201c7a3aa6 crash - tables are 256x256 (64K) */
	    if (PROFILE_STRICT(flags)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    /* Will be encoded as encoding specific replacement below */
	    ch = UNICODE_REPLACE_CHAR;
	}
	word = tableFromUnicode[(ch >> 8)][ch & 0xFF];

	if ((word == 0) && (ch != 0)) {
	    int oldState;
	    const EscapeSubTable *subTablePtr;

	    oldState = state;
	    for (state = 0; state < dataPtr->numSubTables; state++) {
		encodingPtr = GetTableEncoding(dataPtr, state);
		tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
		word = tableDataPtr->fromUnicode[(ch >> 8)][ch & 0xFF];
		if (word != 0) {
		    break;
		}
	    }

	    if (word == 0) {
		state = oldState;
		if (PROFILE_STRICT(flags)) {
		    result = TCL_CONVERT_UNKNOWN;
		    break;
		}
		encodingPtr = GetTableEncoding(dataPtr, state);
		tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
		word = tableDataPtr->fallback;
	    }

	    tablePrefixBytes = (const char *) tableDataPtr->prefixBytes;
	    tableFromUnicode = (const unsigned short *const *)
		    tableDataPtr->fromUnicode;

	    /*
	     * The state variable has the value of oldState when word is 0.
	     * In this case, the escape sequence should not be copied to dst
	     * because the current character set is not changed.
	     */

	    if (state != oldState) {
		subTablePtr = &dataPtr->subTables[state];
		if ((dst + subTablePtr->sequenceLen) > dstEnd) {
		    /*
		     * If there is no space to write the escape sequence, the
		     * state variable must be changed to the value of oldState
		     * variable because this escape sequence must be written
		     * in the next conversion.
		     */

		    state = oldState;
		    result = TCL_CONVERT_NOSPACE;
		    break;
		}
		memcpy(dst, subTablePtr->sequence, subTablePtr->sequenceLen);
		dst += subTablePtr->sequenceLen;
	    }
	}

	if (tablePrefixBytes[(word >> 8)] != 0) {
	    if (dst + 1 > dstEnd) {
		result = TCL_CONVERT_NOSPACE;
		break;
	    }
	    dst[0] = (char) (word >> 8);
	    dst[1] = (char) word;
	    dst += 2;
	} else {
	    if (dst > dstEnd) {
		result = TCL_CONVERT_NOSPACE;
		break;
	    }
	    dst[0] = (char) word;
	    dst++;
	}
	src += len;
    }

    if ((result == TCL_OK) && (flags & TCL_ENCODING_END)) {
	unsigned len = dataPtr->subTables[0].sequenceLen;

	/*
	 * Certain encodings like iso2022-jp need to write an escape sequence
	 * after all characters have been converted. This logic checks that
	 * enough room is available in the buffer for the escape bytes. The
	 * TCL_ENCODING_END flag is cleared after a final escape sequence has
	 * been added to the buffer so that another call to this method does
	 * not attempt to append escape bytes a second time.
	 */

	if ((dst + dataPtr->finalLen + (state?len:0)) > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	} else {
	    if (state) {
		memcpy(dst, dataPtr->subTables[0].sequence, len);
		dst += len;
	    }
	    memcpy(dst, dataPtr->final, dataPtr->finalLen);
	    dst += dataPtr->finalLen;
	    state &= ~TCL_ENCODING_END;
	}
    }

    *statePtr = (Tcl_EncodingState) INT2PTR(state);
    *srcReadPtr = src - srcStart;
    *dstWrotePtr = dst - dstStart;
    *dstCharsPtr = numChars;
    return result;
}

/*
 *---------------------------------------------------------------------------
 *
 * EscapeFreeProc --
 *
 *	Frees resources used by the encoding.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Memory is freed.
 *
 *---------------------------------------------------------------------------
 */

static void
EscapeFreeProc(
    void *clientData)		/* EscapeEncodingData that specifies
				 * encoding. */
{
    EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
    EscapeSubTable *subTablePtr;
    int i;

    if (dataPtr == NULL) {
	return;
    }

    /*
     * The subTables should be freed recursively in normal operation but not
     * during TclFinalizeEncodingSubsystem because they are also present as a
     * weak reference in the toplevel encodingTable (i.e., they don't have a
     * +1 refcount for this), and unpredictable nuking order could remove them
     * from under the following loop's feet. [Bug 2891556]
     *
     * The encodingsInitialized flag, being reset on entry to TFES, can serve
     * as a "not in finalization" test.
     */

    if (encodingsInitialized) {
	subTablePtr = dataPtr->subTables;
	for (i = 0; i < dataPtr->numSubTables; i++) {
	    FreeEncoding((Tcl_Encoding) subTablePtr->encodingPtr);
	    subTablePtr->encodingPtr = NULL;
	    subTablePtr++;
	}
    }
    Tcl_Free(dataPtr);
}

/*
 *---------------------------------------------------------------------------
 *
 * GetTableEncoding --
 *
 *	Helper function for the EscapeEncodingData conversions. Gets the
 *	encoding (of type TextEncodingData) that represents the specified
 *	state.
 *
 * Results:
 *	The return value is the encoding.
 *
 * Side effects:
 *	If the encoding that represents the specified state has not already
 *	been used by this EscapeEncoding, it will be loaded and cached in the
 *	dataPtr.
 *
 *---------------------------------------------------------------------------
 */

static Encoding *
GetTableEncoding(
    EscapeEncodingData *dataPtr,/* Contains names of encodings. */
    int state)			/* Index in dataPtr of desired Encoding. */
{
    EscapeSubTable *subTablePtr = &dataPtr->subTables[state];
    Encoding *encodingPtr = subTablePtr->encodingPtr;

    if (encodingPtr == NULL) {
	encodingPtr = (Encoding *) Tcl_GetEncoding(NULL, subTablePtr->name);
	if ((encodingPtr == NULL)
		|| (encodingPtr->toUtfProc != TableToUtfProc
		&& encodingPtr->toUtfProc != Iso88591ToUtfProc)) {
	    Tcl_Panic("EscapeToUtfProc: invalid sub table");
	}
	subTablePtr->encodingPtr = encodingPtr;
    }

    return encodingPtr;
}

/*
 *---------------------------------------------------------------------------
 *
 * unilen, unilen4 --
 *
 *	A helper function for the Tcl_ExternalToUtf functions. This function
 *	is similar to strlen for double-byte characters: it returns the number
 *	of bytes in a 0x0000 terminated string.
 *
 * Results:
 *	As above.
 *
 * Side effects:
 *	None.
 *
 *---------------------------------------------------------------------------
 */

static size_t
unilen(
    const char *src)
{
    unsigned short *p;

    p = (unsigned short *) src;
    while (*p != 0x0000) {
	p++;
    }
    return (char *) p - src;
}

static size_t
unilen4(
    const char *src)
{
    unsigned int *p;

    p = (unsigned int *) src;
    while (*p != 0x00000000) {
	p++;
    }
    return (char *) p - src;
}

/*
 *-------------------------------------------------------------------------
 *
 * InitializeEncodingSearchPath --
 *
 *	This is the fallback routine that sets the default value of the
 *	encoding search path if the application has not set one via a call to
 *	Tcl_SetEncodingSearchPath() by the first time the search path is needed
 *	to load encoding data.
 *
 *	The default encoding search path is produced by taking each directory
 *	in the library path, appending a subdirectory named "encoding", and if
 *	the resulting directory exists, adding it to the encoding search path.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Sets the encoding search path to an initial value.
 *
 *-------------------------------------------------------------------------
 */

static void
InitializeEncodingSearchPath(
    char **valuePtr,
    size_t *lengthPtr,
    Tcl_Encoding *encodingPtr)
{
    const char *bytes;
    Tcl_Size i, numDirs, numBytes;
    Tcl_Obj *libPathObj, *encodingObj, *searchPathObj;

    TclNewLiteralStringObj(encodingObj, "encoding");
    TclNewObj(searchPathObj);
    Tcl_IncrRefCount(encodingObj);
    Tcl_IncrRefCount(searchPathObj);
    libPathObj = TclGetProcessGlobalValue(&libraryPath);
    Tcl_IncrRefCount(libPathObj);
    TclListObjLength(NULL, libPathObj, &numDirs);

    for (i = 0; i < numDirs; i++) {
	Tcl_Obj *directoryObj, *pathObj;
	Tcl_StatBuf stat;

	Tcl_ListObjIndex(NULL, libPathObj, i, &directoryObj);
	pathObj = Tcl_FSJoinToPath(directoryObj, 1, &encodingObj);
	Tcl_IncrRefCount(pathObj);
	if ((0 == Tcl_FSStat(pathObj, &stat)) && S_ISDIR(stat.st_mode)) {
	    Tcl_ListObjAppendElement(NULL, searchPathObj, pathObj);
	}
	Tcl_DecrRefCount(pathObj);
    }

    Tcl_DecrRefCount(libPathObj);
    Tcl_DecrRefCount(encodingObj);
    *encodingPtr = libraryPath.encoding;
    if (*encodingPtr) {
	((Encoding *)(*encodingPtr))->refCount++;
    }
    bytes = TclGetStringFromObj(searchPathObj, &numBytes);

    *lengthPtr = numBytes;
    *valuePtr = (char *)Tcl_Alloc(numBytes + 1);
    memcpy(*valuePtr, bytes, numBytes + 1);
    Tcl_DecrRefCount(searchPathObj);
}

/*
 *------------------------------------------------------------------------
 *
 * TclEncodingProfileParseName --
 *
 *	Maps an encoding profile name to its integer equivalent.
 *
 * Results:
 *	TCL_OK on success or TCL_ERROR on failure.
 *
 * Side effects:
 *	Returns the profile enum value in *profilePtr
 *
 *------------------------------------------------------------------------
 */
int
TclEncodingProfileNameToId(
    Tcl_Interp *interp,		/* For error messages. May be NULL */
    const char *profileName,	/* Name of profile */
    int *profilePtr)		/* Output */
{
    size_t i;
    size_t numProfiles = sizeof(encodingProfiles) / sizeof(encodingProfiles[0]);

    for (i = 0; i < numProfiles; ++i) {
	if (!strcmp(profileName, encodingProfiles[i].name)) {
	    *profilePtr = encodingProfiles[i].value;
	    return TCL_OK;
	}
    }
    if (interp) {
	/* This code assumes at least two profiles :-) */
	Tcl_Obj *errorObj = Tcl_ObjPrintf("bad profile name \"%s\": must be",
		profileName);
	for (i = 0; i < (numProfiles - 1); ++i) {
	    Tcl_AppendStringsToObj(
		    errorObj, " ", encodingProfiles[i].name, ",", (char *)NULL);
	}
	Tcl_AppendStringsToObj(
		errorObj, " or ", encodingProfiles[numProfiles-1].name, (char *)NULL);

	Tcl_SetObjResult(interp, errorObj);
	Tcl_SetErrorCode(
		interp, "TCL", "ENCODING", "PROFILE", profileName, (char *)NULL);
    }
    return TCL_ERROR;
}

/*
 *------------------------------------------------------------------------
 *
 * TclEncodingProfileValueToName --
 *
 *	Maps an encoding profile value to its name.
 *
 * Results:
 *	Pointer to the name or NULL on failure. Caller must not make
 *	not modify the string and must make a copy to hold on to it.
 *
 * Side effects:
 *	None.
 *------------------------------------------------------------------------
 */
const char *
TclEncodingProfileIdToName(
    Tcl_Interp *interp,		/* For error messages. May be NULL */
    int profileValue)		/* Profile #define value */
{
    size_t i;

    for (i = 0; i < sizeof(encodingProfiles) / sizeof(encodingProfiles[0]); ++i) {
	if (profileValue == encodingProfiles[i].value) {
	    return encodingProfiles[i].name;
	}
    }
    if (interp) {
	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		"Internal error. Bad profile id \"%d\".", profileValue));
	Tcl_SetErrorCode(
		interp, "TCL", "ENCODING", "PROFILEID", (char *)NULL);
    }
    return NULL;
}

/*
 *------------------------------------------------------------------------
 *
 * TclGetEncodingProfiles --
 *
 *	Get the list of supported encoding profiles.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	The list of profile names is stored in the interpreter result.
 *
 *------------------------------------------------------------------------
 */
void
TclGetEncodingProfiles(
    Tcl_Interp *interp)
{
    size_t i, n;
    Tcl_Obj *objPtr;
    n = sizeof(encodingProfiles) / sizeof(encodingProfiles[0]);
    objPtr = Tcl_NewListObj(n, NULL);
    for (i = 0; i < n; ++i) {
	Tcl_ListObjAppendElement(interp, objPtr,
		Tcl_NewStringObj(encodingProfiles[i].name, TCL_INDEX_NONE));
    }
    Tcl_SetObjResult(interp, objPtr);
}

/*
 *------------------------------------------------------------------------
 *
 * Utf8procErrorToTclError --
 *
 *	Converts an error from the utf8proc library into a Tcl error
 *	message/code.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	The interpreter result and error code are set.
 *
 *------------------------------------------------------------------------
 */
static inline void
Utf8procErrorToTclError(
    Tcl_Interp *interp,		// Interpreter to put error description into.
    utf8proc_ssize_t errcode)	// Error code to convert.
{
    const char *errorMsg = utf8proc_errmsg(errcode);
    Tcl_SetObjResult(interp, Tcl_NewStringObj(
	    errorMsg ? errorMsg : "Unicode normalization failed.",
	    TCL_AUTO_LENGTH));
    switch (errcode) {
    case UTF8PROC_ERROR_NOMEM:
	// Memory allocation failure can use the standard Tcl code.
	Tcl_SetErrorCode(interp, "TCL", "MEMORY", NULL);
	break;
    case UTF8PROC_ERROR_OVERFLOW:
	Tcl_SetErrorCode(interp, "TCL", "UNICODE", "OVERFLOW", NULL);
	break;
    case UTF8PROC_ERROR_INVALIDUTF8:
	Tcl_SetErrorCode(interp, "TCL", "UNICODE", "INVALIDUTF8", NULL);
	break;
    case UTF8PROC_ERROR_NOTASSIGNED:
	Tcl_SetErrorCode(interp, "TCL", "UNICODE", "NOTASSIGNED", NULL);
	break;
    case UTF8PROC_ERROR_INVALIDOPTS:
	Tcl_SetErrorCode(interp, "TCL", "UNICODE", "INVALIDOPTS", NULL);
	break;
    default:
	// Shouldn't happen...
	Tcl_SetErrorCode(interp, "TCL", "UNICODE", "UNKNOWN", NULL);
	break;
    }
}

/*
 *------------------------------------------------------------------------
 *
 * TclUtfNormalize --
 *
 *	Apply a normalization rule to a string.
 *
 * Results:
 *	The length of output string. If negative, an error occurred.
 *
 * Side effects:
 *	The interpreter may be updated on error.
 *
 *------------------------------------------------------------------------
 */
static utf8proc_ssize_t
TclUtfNormalize(
    Tcl_Interp *interp,		// Error messages. May be NULL.
    const char *bytes,		// Operand encoded in Tcl internal UTF8.
    Tcl_Size numBytes,		// Length bytes[], or -1 if NUL terminated.
    Tcl_Encoding encoding,	// Encoding - must be UTF-8. Caller passed for reuse
    Tcl_UnicodeNormalizationForm normForm, // TCL_{NFC,NFD,NFKC,NFKC}
    int profile,		// TCL_ENCODING_PROFILE_{STRICT,REPLACE}
    utf8proc_uint8_t **bufPtrPtr) // On success, output length excluding nul.
{
    if (profile != TCL_ENCODING_PROFILE_REPLACE &&
	    profile != TCL_ENCODING_PROFILE_STRICT) {
	if (interp) {
	    Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		    "Invalid value %d passed for encoding profile.",
		    profile));
	    Tcl_SetErrorCode(
		    interp, "TCL", "ENCODING", "PROFILEID", (char *)NULL);
	}
	return -1;
    }

    unsigned options = UTF8PROC_STABLE;
    switch (normForm) {
    case TCL_NFC:
	options |= UTF8PROC_COMPOSE;
	break;
    case TCL_NFD:
	options |= UTF8PROC_DECOMPOSE;
	break;
    case TCL_NFKC:
	options |= UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
	break;
    case TCL_NFKD:
	options |= UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
	break;
    default:
	if (interp) {
	    Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		    "Invalid value %d passed for normalization form.",
		    normForm));
	    Tcl_SetErrorCode(
		    interp, "TCL", "ENCODING", "NORMFORM", (char *)NULL);
	}
	return -1;
    }

    if (numBytes < 0) {
	numBytes = -1;
    }
    Tcl_DString dsExt;
    int result = Tcl_UtfToExternalDStringEx(interp, encoding, bytes, numBytes,
	    profile, &dsExt, NULL);
    /* !!! dsExt needs to be freed even in case of error returns */

    utf8proc_ssize_t normLength = -1;
    if (result == TCL_OK) {
	normLength = utf8proc_map_custom(
		(utf8proc_uint8_t *)Tcl_DStringValue(&dsExt),
		Tcl_DStringLength(&dsExt), bufPtrPtr,
		(utf8proc_option_t) options, NULL, NULL);

	if (normLength < 0 && interp) {
	    // There was an error and we want to transfer it to the interpreter.
	    Utf8procErrorToTclError(interp, normLength);
	}
    }

    Tcl_DStringFree(&dsExt);
    return normLength;
}

/*
 *------------------------------------------------------------------------
 *
 * Tcl_UtfToNormalizedDString --
 *
 *	Converts the passed string to a Unicode normalization form storing
 *	it in dsPtr.
 *
 * Results:
 *	A standard Tcl error code.
 *
 * Side effects:
 *	The output string is stored in dsPtr, which is initialized.
 *
 *------------------------------------------------------------------------
 */
int
Tcl_UtfToNormalizedDString(
    Tcl_Interp *interp,		/* Used for error messages. May be NULL */
    const char *bytes,		/* Operand encoded in Tcl internal UTF8 */
    Tcl_Size numBytes,		/* Length of bytes[], or -1 if NUL terminated */
    Tcl_UnicodeNormalizationForm normForm, /* TCL_{NFC,NFD,NFKC,NFKC} */
    int profile,		/* TCL_ENCODING_PROFILE_{STRICT,REPLACE} */
    Tcl_DString *dsPtr)		/* Converted output string in Tcl internal
				 * UTF8 encoding. Init'ed by function */
{
    Tcl_DStringInit(dsPtr);
    Tcl_Encoding encoding = Tcl_GetEncoding(interp, "utf-8");
    if (encoding == NULL) {
	return TCL_ERROR;
    }

    utf8proc_uint8_t *normUtf8;
    utf8proc_ssize_t normLength;

    normLength = TclUtfNormalize(interp, bytes, numBytes, encoding, normForm,
	    profile, &normUtf8);
    if (normLength >= 0) {
	assert(normUtf8);
	/* Convert standard UTF8 to internal UTF8 */
	int result = Tcl_ExternalToUtfDStringEx(interp, encoding,
		(const char *)normUtf8, normLength, profile, dsPtr, NULL);
	if (result != TCL_OK) {
	    normLength = -1;
	}
	utf8proc_free(normUtf8); /* NOT Tcl_Free! */
    }
    Tcl_FreeEncoding(encoding);
    return normLength >= 0 ? TCL_OK : TCL_ERROR;
}

/*
 *------------------------------------------------------------------------
 *
 * Tcl_UtfToNormalized --
 *
 *	Converts the passed string to a Unicode normalization form storing
 *	it in the caller provided buffer.
 *
 * Results:
 *	A standard Tcl error code.
 *
 * Side effects:
 *	The output string is stored in bufPtr.
 *
 *------------------------------------------------------------------------
 */
int
Tcl_UtfToNormalized(
    Tcl_Interp *interp,		/* Used for error messages. May be NULL */
    const char *bytes,		/* Operand encoded in Tcl internal UTF8 */
    Tcl_Size numBytes,		/* Length of bytes[], or -1 if NUL terminated */
    Tcl_UnicodeNormalizationForm normForm, /* TCL_{NFC,NFD,NFKC,NFKC} */
    int profile,		/* TCL_ENCODING_PROFILE_{STRICT,REPLACE} */
    char *bufPtr,		/* Pointer to output buffer. Must not be NULL.  */
    Tcl_Size bufLen,		/* Size of bufPtr storage. */
    Tcl_Size *lengthPtr)	/* Length of the output string in bytes excluding
				 * the trailing NUL byte. May be NULL */
{
    Tcl_Encoding encoding = Tcl_GetEncoding(interp, "utf-8");
    if (encoding == NULL) {
	return TCL_ERROR;
    }

    utf8proc_uint8_t *normUtf8;
    utf8proc_ssize_t normLength;
    normLength = TclUtfNormalize(interp, bytes, numBytes, encoding, normForm,
	    profile, &normUtf8);
    Tcl_FreeEncoding(encoding);
    if (normLength < 0) {
	return TCL_ERROR;
    }
    assert(normUtf8);

    /* Convert standard UTF8 to internal UTF8 */
    int result;
    utf8proc_uint8_t *from = normUtf8;
    utf8proc_uint8_t *fromEnd = from + normLength;
    char *to = bufPtr;
    char *toEnd = to + bufLen - 1; /* -1 for trailing NUL */
    while (from < fromEnd && to < toEnd) {
	/*
	 * The only difference between UTF-8 and internal UTF-8 is that
	 * internal UTF-8 does not allow NUL bytes in the middle of the string.
	 */
	if (*from) {
	    *to++ = *from++;
	} else {
	    *to++ = 0xC0; /* NUL byte in internal UTF-8 is encoded as C080 */
	    if (to == toEnd) {
		break;
	    }
	    *to++ = 0x80;
	    from++;
	}
    }
    if (from < fromEnd) {
	Tcl_SetObjResult(interp, Tcl_NewStringObj(
		"Output buffer too small.", -1));
	result = TCL_CONVERT_NOSPACE;
    } else {
	assert(to <= toEnd);
	*to = '\0'; /* NUL terminate the output */
	if (lengthPtr) {
	    *lengthPtr = to-bufPtr;
	}
	result = TCL_OK;
    }

    utf8proc_free(normUtf8); /* NOT Tcl_Free! */
    return result;
}

/*
 * Local Variables:
 * mode: c
 * c-basic-offset: 4
 * fill-column: 78
 * End:
 */