SandboXP
1#define malloc v86_malloc2#define free v86_free3#include <stddef.h>4void *calloc(size_t nmemb, size_t size);5void *memset(void *s, int c, size_t n);6void *memcpy(void *dest, const void *src, size_t n);7void *memmove(void *dest, const void *src, size_t n);8void *malloc(size_t size);9void free(void *ptr);10/**11* \file zstddeclib.c12* Single-file Zstandard decompressor.13*14* Generate using:15* \code16* combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c17* \endcode18*/19/*20* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.21* All rights reserved.22*23* This source code is licensed under both the BSD-style license (found in the24* LICENSE file in the root directory of this source tree) and the GPLv2 (found25* in the COPYING file in the root directory of this source tree).26* You may select, at your option, one of the above-listed licenses.27*/28/*29* Settings to bake for the standalone decompressor.30*31* Note: It's important that none of these affects 'zstd.h' (only the32* implementation files we're amalgamating).33*34* Note: MEM_MODULE stops xxhash redefining BYTE, U16, etc., which are also35* defined in mem.h (breaking C99 compatibility).36*37* Note: the undefs for xxHash allow Zstd's implementation to coinside with with38* standalone xxHash usage (with global defines).39*/40#define DEBUGLEVEL 041#define MEM_MODULE42#undef XXH_NAMESPACE43#define XXH_NAMESPACE ZSTD_44#undef XXH_PRIVATE_API45#define XXH_PRIVATE_API46#undef XXH_INLINE_ALL47#define XXH_INLINE_ALL48#define ZSTD_LEGACY_SUPPORT 049#define ZSTD_LIB_COMPRESSION 050#define ZSTD_LIB_DEPRECATED 051#define ZSTD_NOBENCH52#define ZSTD_STRIP_ERROR_STRINGS53/**** start inlining common/debug.c ****/55/* ******************************************************************56* debug57* Part of FSE library58* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.59*60* You can contact the author at :61* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy62*63* This source code is licensed under both the BSD-style license (found in the64* LICENSE file in the root directory of this source tree) and the GPLv2 (found65* in the COPYING file in the root directory of this source tree).66* You may select, at your option, one of the above-listed licenses.67****************************************************************** */68/*71* This module only hosts one global variable72* which can be used to dynamically influence the verbosity of traces,73* such as DEBUGLOG and RAWLOG74*/75/**** start inlining debug.h ****/77/* ******************************************************************78* debug79* Part of FSE library80* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.81*82* You can contact the author at :83* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy84*85* This source code is licensed under both the BSD-style license (found in the86* LICENSE file in the root directory of this source tree) and the GPLv2 (found87* in the COPYING file in the root directory of this source tree).88* You may select, at your option, one of the above-listed licenses.89****************************************************************** */90/*93* The purpose of this header is to enable debug functions.94* They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,95* and DEBUG_STATIC_ASSERT() for compile-time.96*97* By default, DEBUGLEVEL==0, which means run-time debug is disabled.98*99* Level 1 enables assert() only.100* Starting level 2, traces can be generated and pushed to stderr.101* The higher the level, the more verbose the traces.102*103* It's possible to dynamically adjust level using variable g_debug_level,104* which is only declared if DEBUGLEVEL>=2,105* and is a global variable, not multi-thread protected (use with care)106*/107/* static assert is triggered at compile time, leaving no runtime artefact.117* static assert only works with compile-time constants.118* Also, this variant can only be used inside a function. */119#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])120/* DEBUGLEVEL is expected to be defined externally,123* typically through compiler command line.124* Value must be a number. */125#ifndef DEBUGLEVEL126# define DEBUGLEVEL 0127#endif128/* DEBUGFILE can be defined externally,131* typically through compiler command line.132* note : currently useless.133* Value must be stderr or stdout */134#ifndef DEBUGFILE135# define DEBUGFILE stderr136#endif137/* recommended values for DEBUGLEVEL :140* 0 : release mode, no debug, all run-time checks disabled141* 1 : enables assert() only, no display142* 2 : reserved, for currently active debug path143* 3 : events once per object lifetime (CCtx, CDict, etc.)144* 4 : events once per frame145* 5 : events once per block146* 6 : events once per sequence (verbose)147* 7+: events at every position (*very* verbose)148*149* It's generally inconvenient to output traces > 5.150* In which case, it's possible to selectively trigger high verbosity levels151* by modifying g_debug_level.152*/153it actually lives in debug.c,166and is shared by the whole process.167It's not thread-safe.168It's useful when enabling very verbose levels169on selective conditions (such as position in src) */170}188#endif189/**** ended inlining debug.h ****/192/**** ended inlining common/debug.c ****/195/**** start inlining common/entropy_common.c ****/196/* ******************************************************************197* Common functions of New Generation Entropy library198* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.199*200* You can contact the author at :201* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy202* - Public forum : https://groups.google.com/forum/#!forum/lz4c203*204* This source code is licensed under both the BSD-style license (found in the205* LICENSE file in the root directory of this source tree) and the GPLv2 (found206* in the COPYING file in the root directory of this source tree).207* You may select, at your option, one of the above-listed licenses.208****************************************************************** */209/* *************************************211* Dependencies212***************************************/213/**** start inlining mem.h ****/214/*215* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.216* All rights reserved.217*218* This source code is licensed under both the BSD-style license (found in the219* LICENSE file in the root directory of this source tree) and the GPLv2 (found220* in the COPYING file in the root directory of this source tree).221* You may select, at your option, one of the above-listed licenses.222*/223/*-****************************************232* Dependencies233******************************************/234#include <stddef.h> /* size_t, ptrdiff_t */235/*-****************************************238* Compiler specifics239******************************************/240#if defined(_MSC_VER) /* Visual Studio */241# include <intrin.h> /* _byteswap_* */242#endif243#if defined(__GNUC__)244# define MEM_STATIC static __inline __attribute__((unused))245#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)246# define MEM_STATIC static inline247#elif defined(_MSC_VER)248# define MEM_STATIC static __inline249#else250# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */251#endif252/* code only tested on 32 and 64 bits systems */258#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }259MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }260/* detects whether we are being compiled under msan */262#if defined (__has_feature)263# if __has_feature(memory_sanitizer)264# define MEMORY_SANITIZER 1265# endif266#endif267/* Not all platforms that support msan provide sanitizers/msan_interface.h.270* We therefore declare the functions we need ourselves, rather than trying to271* include the header file... */272/* Make memory region fully initialized (without changing its contents). */276void __msan_unpoison(const volatile void *a, size_t size);277/* Make memory region fully uninitialized (without changing its contents).279This is a legacy interface that does not update origin information. Use280__msan_allocated_memory() instead. */281void __msan_poison(const volatile void *a, size_t size);282/* Returns the offset of the first (at least partially) poisoned byte in the284memory range, or -1 if the whole range is good. */285intptr_t __msan_test_shadow(const volatile void *x, size_t size);286#endif287/* detects whether we are being compiled under asan */289#if defined (__has_feature)290# if __has_feature(address_sanitizer)291# define ADDRESS_SANITIZER 1292# endif293#elif defined(__SANITIZE_ADDRESS__)294# define ADDRESS_SANITIZER 1295#endif296/* Not all platforms that support asan provide sanitizers/asan_interface.h.299* We therefore declare the functions we need ourselves, rather than trying to300* include the header file... */301/**303* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.304*305* This memory must be previously allocated by your program. Instrumented306* code is forbidden from accessing addresses in this region until it is307* unpoisoned. This function is not guaranteed to poison the entire region -308* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan309* alignment restrictions.310*311* \note This function is not thread-safe because no two threads can poison or312* unpoison memory in the same memory region simultaneously.313*314* \param addr Start of memory region.315* \param size Size of memory region. */316void __asan_poison_memory_region(void const volatile *addr, size_t size);317/**319* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.320*321* This memory must be previously allocated by your program. Accessing322* addresses in this region is allowed until this region is poisoned again.323* This function could unpoison a super-region of <c>[addr, addr+size)</c> due324* to ASan alignment restrictions.325*326* \note This function is not thread-safe because no two threads can327* poison or unpoison memory in the same memory region simultaneously.328*329* \param addr Start of memory region.330* \param size Size of memory region. */331void __asan_unpoison_memory_region(void const volatile *addr, size_t size);332#endif333/*-**************************************************************336* Basic Types337*****************************************************************/338#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )339# include <stdint.h>340typedef uint8_t BYTE;341typedef uint16_t U16;342typedef int16_t S16;343typedef uint32_t U32;344typedef int32_t S32;345typedef uint64_t U64;346typedef int64_t S64;347#else348# include <limits.h>349#if CHAR_BIT != 8350# error "this implementation requires char to be exactly 8-bit type"351#endif352typedef unsigned char BYTE;353#if USHRT_MAX != 65535354# error "this implementation requires short to be exactly 16-bit type"355#endif356typedef unsigned short U16;357typedef signed short S16;358#if UINT_MAX != 4294967295359# error "this implementation requires int to be exactly 32-bit type"360#endif361typedef unsigned int U32;362typedef signed int S32;363/* note : there are no limits defined for long long type in C90.364* limits exist in C99, however, in such case, <stdint.h> is preferred */365typedef unsigned long long U64;366typedef signed long long S64;367#endif368/*-**************************************************************371* Memory I/O372*****************************************************************/373/* MEM_FORCE_MEMORY_ACCESS :374* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.375* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.376* The below switch allow to select different access method for improved performance.377* Method 0 (default) : use `memcpy()`. Safe and portable.378* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).379* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.380* Method 2 : direct access. This method is portable but violate C standard.381* It can generate buggy code on targets depending on alignment.382* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)383* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.384* Prefer these methods in priority order (0 > 1 > 2)385*/386#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */387# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )388# define MEM_FORCE_MEMORY_ACCESS 2389# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)390# define MEM_FORCE_MEMORY_ACCESS 1391# endif392#endif393{399const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */400return one.c[0];401}402/* violates C standard, by lying on structure alignment.406Only use if no other choice to achieve best performance on target platform */407MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }408MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }409MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }410MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }411/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */419/* currently only defined for gcc and icc */420#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))421__pragma( pack(push, 1) )422typedef struct { U16 v; } unalign16;423typedef struct { U32 v; } unalign32;424typedef struct { U64 v; } unalign64;425typedef struct { size_t v; } unalignArch;426__pragma( pack(pop) )427#else428typedef struct { U16 v; } __attribute__((packed)) unalign16;429typedef struct { U32 v; } __attribute__((packed)) unalign32;430typedef struct { U64 v; } __attribute__((packed)) unalign64;431typedef struct { size_t v; } __attribute__((packed)) unalignArch;432#endif433/* default method, safe and standard.446can sometimes prove slower */447{450U16 val; memcpy(&val, memPtr, sizeof(val)); return val;451}452{455U32 val; memcpy(&val, memPtr, sizeof(val)); return val;456}457{460U64 val; memcpy(&val, memPtr, sizeof(val)); return val;461}462{465size_t val; memcpy(&val, memPtr, sizeof(val)); return val;466}467{470memcpy(memPtr, &value, sizeof(value));471}472{475memcpy(memPtr, &value, sizeof(value));476}477{480memcpy(memPtr, &value, sizeof(value));481}482{487#if defined(_MSC_VER) /* Visual Studio */488return _byteswap_ulong(in);489#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \490|| (defined(__clang__) && __has_builtin(__builtin_bswap32))491return __builtin_bswap32(in);492#else493return ((in << 24) & 0xff000000 ) |494((in << 8) & 0x00ff0000 ) |495((in >> 8) & 0x0000ff00 ) |496((in >> 24) & 0x000000ff );497#endif498}499{502#if defined(_MSC_VER) /* Visual Studio */503return _byteswap_uint64(in);504#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \505|| (defined(__clang__) && __has_builtin(__builtin_bswap64))506return __builtin_bswap64(in);507#else508return ((in << 56) & 0xff00000000000000ULL) |509((in << 40) & 0x00ff000000000000ULL) |510((in << 24) & 0x0000ff0000000000ULL) |511((in << 8) & 0x000000ff00000000ULL) |512((in >> 8) & 0x00000000ff000000ULL) |513((in >> 24) & 0x0000000000ff0000ULL) |514((in >> 40) & 0x000000000000ff00ULL) |515((in >> 56) & 0x00000000000000ffULL);516#endif517}518{521if (MEM_32bits())522return (size_t)MEM_swap32((U32)in);523else524return (size_t)MEM_swap64((U64)in);525}526/*=== Little endian r/w ===*/528{531if (MEM_isLittleEndian())532return MEM_read16(memPtr);533else {534const BYTE* p = (const BYTE*)memPtr;535return (U16)(p[0] + (p[1]<<8));536}537}538{541if (MEM_isLittleEndian()) {542MEM_write16(memPtr, val);543} else {544BYTE* p = (BYTE*)memPtr;545p[0] = (BYTE)val;546p[1] = (BYTE)(val>>8);547}548}549{552return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);553}554{557MEM_writeLE16(memPtr, (U16)val);558((BYTE*)memPtr)[2] = (BYTE)(val>>16);559}560{563if (MEM_isLittleEndian())564return MEM_read32(memPtr);565else566return MEM_swap32(MEM_read32(memPtr));567}568{571if (MEM_isLittleEndian())572MEM_write32(memPtr, val32);573else574MEM_write32(memPtr, MEM_swap32(val32));575}576{579if (MEM_isLittleEndian())580return MEM_read64(memPtr);581else582return MEM_swap64(MEM_read64(memPtr));583}584{587if (MEM_isLittleEndian())588MEM_write64(memPtr, val64);589else590MEM_write64(memPtr, MEM_swap64(val64));591}592{595if (MEM_32bits())596return (size_t)MEM_readLE32(memPtr);597else598return (size_t)MEM_readLE64(memPtr);599}600{603if (MEM_32bits())604MEM_writeLE32(memPtr, (U32)val);605else606MEM_writeLE64(memPtr, (U64)val);607}608/*=== Big endian r/w ===*/610{613if (MEM_isLittleEndian())614return MEM_swap32(MEM_read32(memPtr));615else616return MEM_read32(memPtr);617}618{621if (MEM_isLittleEndian())622MEM_write32(memPtr, MEM_swap32(val32));623else624MEM_write32(memPtr, val32);625}626{629if (MEM_isLittleEndian())630return MEM_swap64(MEM_read64(memPtr));631else632return MEM_read64(memPtr);633}634{637if (MEM_isLittleEndian())638MEM_write64(memPtr, MEM_swap64(val64));639else640MEM_write64(memPtr, val64);641}642{645if (MEM_32bits())646return (size_t)MEM_readBE32(memPtr);647else648return (size_t)MEM_readBE64(memPtr);649}650{653if (MEM_32bits())654MEM_writeBE32(memPtr, (U32)val);655else656MEM_writeBE64(memPtr, (U64)val);657}658}662#endif663/**** ended inlining mem.h ****/666/**** start inlining error_private.h ****/667/*668* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.669* All rights reserved.670*671* This source code is licensed under both the BSD-style license (found in the672* LICENSE file in the root directory of this source tree) and the GPLv2 (found673* in the COPYING file in the root directory of this source tree).674* You may select, at your option, one of the above-listed licenses.675*/676/* Note : this module is expected to remain private, do not expose it */678/* ****************************************688* Dependencies689******************************************/690#include <stddef.h> /* size_t */691/**** start inlining zstd_errors.h ****/692/*693* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.694* All rights reserved.695*696* This source code is licensed under both the BSD-style license (found in the697* LICENSE file in the root directory of this source tree) and the GPLv2 (found698* in the COPYING file in the root directory of this source tree).699* You may select, at your option, one of the above-listed licenses.700*/701/*===== dependency =====*/710#include <stddef.h> /* size_t */711/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */714#ifndef ZSTDERRORLIB_VISIBILITY715# if defined(__GNUC__) && (__GNUC__ >= 4)716# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))717# else718# define ZSTDERRORLIB_VISIBILITY719# endif720#endif721#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)722# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY723#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)724# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/725#else726# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY727#endif728/*-*********************************************730* Error codes list731*-*********************************************732* Error codes _values_ are pinned down since v1.3.1 only.733* Therefore, don't rely on values if you may link to any version < v1.3.1.734*735* Only values < 100 are considered stable.736*737* note 1 : this API shall be used with static linking only.738* dynamic linking is not yet officially supported.739* note 2 : Prefer relying on the enum than on its value whenever possible740* This is the only supported way to use the error list < v1.3.1741* note 3 : ZSTD_isError() is always correct, whatever the library version.742**********************************************/743typedef enum {744ZSTD_error_no_error = 0,745ZSTD_error_GENERIC = 1,746ZSTD_error_prefix_unknown = 10,747ZSTD_error_version_unsupported = 12,748ZSTD_error_frameParameter_unsupported = 14,749ZSTD_error_frameParameter_windowTooLarge = 16,750ZSTD_error_corruption_detected = 20,751ZSTD_error_checksum_wrong = 22,752ZSTD_error_dictionary_corrupted = 30,753ZSTD_error_dictionary_wrong = 32,754ZSTD_error_dictionaryCreation_failed = 34,755ZSTD_error_parameter_unsupported = 40,756ZSTD_error_parameter_outOfBound = 42,757ZSTD_error_tableLog_tooLarge = 44,758ZSTD_error_maxSymbolValue_tooLarge = 46,759ZSTD_error_maxSymbolValue_tooSmall = 48,760ZSTD_error_stage_wrong = 60,761ZSTD_error_init_missing = 62,762ZSTD_error_memory_allocation = 64,763ZSTD_error_workSpace_tooSmall= 66,764ZSTD_error_dstSize_tooSmall = 70,765ZSTD_error_srcSize_wrong = 72,766ZSTD_error_dstBuffer_null = 74,767/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */768ZSTD_error_frameIndex_tooLarge = 100,769ZSTD_error_seekableIO = 102,770ZSTD_error_dstBuffer_wrong = 104,771ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */772} ZSTD_ErrorCode;773/*! ZSTD_getErrorCode() :775convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,776which can be used to compare with enum list published above */777ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);778ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */779}783#endif784/**** ended inlining zstd_errors.h ****/787/* ****************************************790* Compiler-specific791******************************************/792#if defined(__GNUC__)793# define ERR_STATIC static __attribute__((unused))794#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)795# define ERR_STATIC static inline796#elif defined(_MSC_VER)797# define ERR_STATIC static __inline798#else799# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */800#endif801/*-****************************************804* Customization (error_public.h)805******************************************/806typedef ZSTD_ErrorCode ERR_enum;807#define PREFIX(name) ZSTD_error_##name808/*-****************************************811* Error codes handling812******************************************/813#undef ERROR /* already defined on Visual Studio */814#define ERROR(name) ZSTD_ERROR(name)815#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))816/* check and forward error code */822#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e823#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }824/*-****************************************827* Error Strings828******************************************/829{834return ERR_getErrorString(ERR_getErrorCode(code));835}836}839#endif840/**** ended inlining error_private.h ****/843#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */844/**** start inlining fse.h ****/845/* ******************************************************************846* FSE : Finite State Entropy codec847* Public Prototypes declaration848* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.849*850* You can contact the author at :851* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy852*853* This source code is licensed under both the BSD-style license (found in the854* LICENSE file in the root directory of this source tree) and the GPLv2 (found855* in the COPYING file in the root directory of this source tree).856* You may select, at your option, one of the above-listed licenses.857****************************************************************** */858/*-*****************************************868* Dependencies869******************************************/870#include <stddef.h> /* size_t, ptrdiff_t */871/*-*****************************************874* FSE_PUBLIC_API : control library symbols visibility875******************************************/876#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)877# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))878#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */879# define FSE_PUBLIC_API __declspec(dllexport)880#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)881# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/882#else883# define FSE_PUBLIC_API884#endif885/*------ Version ------*/887#define FSE_VERSION_MAJOR 0888#define FSE_VERSION_MINOR 9889#define FSE_VERSION_RELEASE 0890/*-****************************************901* FSE simple functions902******************************************/903/*! FSE_compress() :904Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.905'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).906@return : size of compressed data (<= dstCapacity).907Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!908if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.909if FSE_isError(return), compression failed (more details using FSE_getErrorName())910*/911FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,912const void* src, size_t srcSize);913/*! FSE_decompress():915Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',916into already allocated destination buffer 'dst', of size 'dstCapacity'.917@return : size of regenerated data (<= maxDstSize),918or an error code, which can be tested using FSE_isError() .919** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!921Why ? : making this distinction requires a header.922Header management is intentionally delegated to the user layer, which can better manage special cases.923*/924FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,925const void* cSrc, size_t cSrcSize);926/*-*****************************************929* Tool functions930******************************************/931FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */932/* Error Management */934FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */935FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */936/*-*****************************************939* FSE advanced functions940******************************************/941/*! FSE_compress2() :942Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'943Both parameters can be defined as '0' to mean : use default value944@return : size of compressed data945Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!946if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.947if FSE_isError(return), it's an error code.948*/949FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);950/*-*****************************************953* FSE detailed API954******************************************/955/*!956FSE_compress() does the following:9571. count symbol occurrence from source[] into table count[] (see hist.h)9582. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)9593. save normalized counters to memory buffer using writeNCount()9604. build encoding table 'CTable' from normalized counters9615. encode the data stream using encoding table 'CTable'962FSE_decompress() does the following:9641. read normalized counters with readNCount()9652. build decoding table 'DTable' from normalized counters9663. decode the data stream using decoding table 'DTable'967The following API allows targeting specific sub-functions for advanced tasks.969For example, it's possible to compress several blocks using the same 'CTable',970or to save and provide normalized distribution using external method.971*/972/* *** COMPRESSION *** */974/*! FSE_optimalTableLog():976dynamically downsize 'tableLog' when conditions are met.977It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.978@return : recommended tableLog (necessarily <= 'maxTableLog') */979FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);980/*! FSE_normalizeCount():982normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)983'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).984@return : tableLog,985or an errorCode, which can be tested using FSE_isError() */986FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,987const unsigned* count, size_t srcSize, unsigned maxSymbolValue);988/*! FSE_NCountWriteBound():990Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.991Typically useful for allocation purpose. */992FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);993/*! FSE_writeNCount():995Compactly save 'normalizedCounter' into 'buffer'.996@return : size of the compressed table,997or an errorCode, which can be tested using FSE_isError(). */998FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,999const short* normalizedCounter,1000unsigned maxSymbolValue, unsigned tableLog);1001/*! Constructor and Destructor of FSE_CTable.1003Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */1004typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */1005FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);1006FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);1007/*! FSE_buildCTable():1009Builds `ct`, which must be already allocated, using FSE_createCTable().1010@return : 0, or an errorCode, which can be tested using FSE_isError() */1011FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);1012/*! FSE_compress_usingCTable():1014Compress `src` using `ct` into `dst` which must be already allocated.1015@return : size of compressed data (<= `dstCapacity`),1016or 0 if compressed data could not fit into `dst`,1017or an errorCode, which can be tested using FSE_isError() */1018FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);1019/*!1021Tutorial :1022----------1023The first step is to count all symbols. FSE_count() does this job very fast.1024Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.1025'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]1026maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)1027FSE_count() will return the number of occurrence of the most frequent symbol.1028This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.1029If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).1030The next step is to normalize the frequencies.1032FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.1033It also guarantees a minimum of 1 to any Symbol with frequency >= 1.1034You can use 'tableLog'==0 to mean "use default tableLog value".1035If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),1036which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").1037The result of FSE_normalizeCount() will be saved into a table,1039called 'normalizedCounter', which is a table of signed short.1040'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.1041The return value is tableLog if everything proceeded as expected.1042It is 0 if there is a single symbol within distribution.1043If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).1044'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().1046'buffer' must be already allocated.1047For guaranteed success, buffer size must be at least FSE_headerBound().1048The result of the function is the number of bytes written into 'buffer'.1049If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).1050'normalizedCounter' can then be used to create the compression table 'CTable'.1052The space required by 'CTable' must be already allocated, using FSE_createCTable().1053You can then use FSE_buildCTable() to fill 'CTable'.1054If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).1055'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().1057Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'1058The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.1059If it returns '0', compressed data could not fit into 'dst'.1060If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).1061*/1062/* *** DECOMPRESSION *** */1065/*! FSE_readNCount():1067Read compactly saved 'normalizedCounter' from 'rBuffer'.1068@return : size read from 'rBuffer',1069or an errorCode, which can be tested using FSE_isError().1070maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */1071FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,1072unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,1073const void* rBuffer, size_t rBuffSize);1074/*! Constructor and Destructor of FSE_DTable.1076Note that its size depends on 'tableLog' */1077typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */1078FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);1079FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);1080/*! FSE_buildDTable():1082Builds 'dt', which must be already allocated, using FSE_createDTable().1083return : 0, or an errorCode, which can be tested using FSE_isError() */1084FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);1085/*! FSE_decompress_usingDTable():1087Decompress compressed source `cSrc` of size `cSrcSize` using `dt`1088into `dst` which must be already allocated.1089@return : size of regenerated data (necessarily <= `dstCapacity`),1090or an errorCode, which can be tested using FSE_isError() */1091FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);1092/*!1094Tutorial :1095----------1096(Note : these functions only decompress FSE-compressed blocks.1097If block is uncompressed, use memcpy() instead1098If block is a single repeated byte, use memset() instead )1099The first step is to obtain the normalized frequencies of symbols.1101This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().1102'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.1103In practice, that means it's necessary to know 'maxSymbolValue' beforehand,1104or size the table to handle worst case situations (typically 256).1105FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.1106The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.1107Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.1108If there is an error, the function will return an error code, which can be tested using FSE_isError().1109The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.1111This is performed by the function FSE_buildDTable().1112The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().1113If there is an error, the function will return an error code, which can be tested using FSE_isError().1114`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().1116`cSrcSize` must be strictly correct, otherwise decompression will fail.1117FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).1118If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)1119*/1120/* *** Dependency *** */1127/**** start inlining bitstream.h ****/1128/* ******************************************************************1129* bitstream1130* Part of FSE library1131* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.1132*1133* You can contact the author at :1134* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy1135*1136* This source code is licensed under both the BSD-style license (found in the1137* LICENSE file in the root directory of this source tree) and the GPLv2 (found1138* in the COPYING file in the root directory of this source tree).1139* You may select, at your option, one of the above-listed licenses.1140****************************************************************** */1141#ifndef BITSTREAM_H_MODULE1142#define BITSTREAM_H_MODULE1143/*1149* This API consists of small unitary functions, which must be inlined for best performance.1150* Since link-time-optimization is not available for all compilers,1151* these functions are defined into a .h to be included.1152*/1153/*-****************************************1155* Dependencies1156******************************************/1157/**** skipping file: mem.h ****/1158/**** start inlining compiler.h ****/1159/*1160* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.1161* All rights reserved.1162*1163* This source code is licensed under both the BSD-style license (found in the1164* LICENSE file in the root directory of this source tree) and the GPLv2 (found1165* in the COPYING file in the root directory of this source tree).1166* You may select, at your option, one of the above-listed licenses.1167*/1168/*-*******************************************************1173* Compiler specifics1174*********************************************************/1175/* force inlining */1176/**1200On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).1201This explictly marks such functions as __cdecl so that the code will still compile1202if a CC other than __cdecl has been made the default.1203*/1204#if defined(_MSC_VER)1205# define WIN_CDECL __cdecl1206#else1207# define WIN_CDECL1208#endif1209/**1211* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant1212* parameters. They must be inlined for the compiler to eliminate the constant1213* branches.1214*/1215#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR1216/**1217* HINT_INLINE is used to help the compiler generate better code. It is *not*1218* used for "templates", so it can be tweaked based on the compilers1219* performance.1220*1221* gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the1222* always_inline attribute.1223*1224* clang up to 5.0.0 (trunk) benefit tremendously from the always_inline1225* attribute.1226*/1227#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 51228# define HINT_INLINE static INLINE_KEYWORD1229#else1230# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR1231#endif1232/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */1234#if defined(__GNUC__)1235# define UNUSED_ATTR __attribute__((unused))1236#else1237# define UNUSED_ATTR1238#endif1239/* force no inlining */1241#ifdef _MSC_VER1242# define FORCE_NOINLINE static __declspec(noinline)1243#else1244# if defined(__GNUC__) || defined(__ICCARM__)1245# define FORCE_NOINLINE static __attribute__((__noinline__))1246# else1247# define FORCE_NOINLINE static1248# endif1249#endif1250/* target attribute */1252#ifndef __has_attribute1253#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */1254#endif1255#if defined(__GNUC__) || defined(__ICCARM__)1256# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))1257#else1258# define TARGET_ATTRIBUTE(target)1259#endif1260/* Enable runtime BMI2 dispatch based on the CPU.1262* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.1263*/1264#ifndef DYNAMIC_BMI21265#if ((defined(__clang__) && __has_attribute(__target__)) \1266|| (defined(__GNUC__) \1267&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \1268&& (defined(__x86_64__) || defined(_M_X86)) \1269&& !defined(__BMI2__)1270# define DYNAMIC_BMI2 11271#else1272# define DYNAMIC_BMI2 01273#endif1274#endif1275/* prefetch1277* can be disabled, by declaring NO_PREFETCH build macro */1278#if defined(NO_PREFETCH)1279# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */1280# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */1281#else1282# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */1283# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */1284# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)1285# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)1286# elif defined(__aarch64__)1287# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))1288# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))1289# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )1290# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)1291# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)1292# else1293# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */1294# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */1295# endif1296#endif /* NO_PREFETCH */1297}1308/* vectorization1310* older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */1311#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__)1312# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)1313# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))1314# else1315# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")1316# endif1317#else1318# define DONT_VECTORIZE1319#endif1320/* Tell the compiler that a branch is likely or unlikely.1322* Only use these macros if it causes the compiler to generate better code.1323* If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc1324* and clang, please do.1325*/1326#if defined(__GNUC__)1327#define LIKELY(x) (__builtin_expect((x), 1))1328#define UNLIKELY(x) (__builtin_expect((x), 0))1329#else1330#define LIKELY(x) (x)1331#define UNLIKELY(x) (x)1332#endif1333/* disable warnings */1335#ifdef _MSC_VER /* Visual Studio */1336# include <intrin.h> /* For Visual 2005 */1337# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */1338# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */1339# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */1340# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */1341# pragma warning(disable : 4324) /* disable: C4324: padded structure */1342#endif1343/**** ended inlining compiler.h ****/1346/**** skipping file: debug.h ****/1347/**** skipping file: error_private.h ****/1348/*=========================================1351* Target specific1352=========================================*/1353#if defined(__BMI__) && defined(__GNUC__)1354# include <immintrin.h> /* support for bextr (experimental) */1355#elif defined(__ICCARM__)1356# include <intrinsics.h>1357#endif1358/*-******************************************1365* bitStream encoding API (write forward)1366********************************************/1367/* bitStream can mix input from multiple sources.1368* A critical property of these streams is that they encode and decode in **reverse** direction.1369* So the first bit sequence you add will be the last to be read, like a LIFO stack.1370*/1371typedef struct {1372size_t bitContainer;1373unsigned bitPos;1374char* startPtr;1375char* ptr;1376char* endPtr;1377} BIT_CStream_t;1378/* Start with initCStream, providing the size of buffer to write into.1385* bitStream will never write outside of this buffer.1386* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.1387*1388* bits are first added to a local register.1389* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.1390* Writing data into memory is an explicit operation, performed by the flushBits function.1391* Hence keep track how many bits are potentially stored into local register to avoid register overflow.1392* After a flushBits, a maximum of 7 bits might still be stored into local register.1393*1394* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.1395*1396* Last operation is to close the bitStream.1397* The function returns the final size of CStream in bytes.1398* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)1399*/1400/*-********************************************1403* bitStream decoding API (read backward)1404**********************************************/1405typedef struct {1406size_t bitContainer;1407unsigned bitsConsumed;1408const char* ptr;1409const char* start;1410const char* limitPtr;1411} BIT_DStream_t;1412/* Start by invoking BIT_initDStream().1426* A chunk of the bitStream is then stored into a local register.1427* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).1428* You can then retrieve bitFields stored into the local register, **in reverse order**.1429* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.1430* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.1431* Otherwise, it can be less than that, so proceed accordingly.1432* Checking if DStream has reached its end can be performed with BIT_endOfDStream().1433*/1434/*-****************************************1437* unsafe API1438******************************************/1439MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);1440/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */1441/* unsafe version; does not check buffer overflow */1444/* faster, but works only if nbBits >= 1 */1447/*-**************************************************************1451* Internal functions1452****************************************************************/1453MEM_STATIC unsigned BIT_highbit32 (U32 val)1454{1455assert(val != 0);1456{1457# if defined(_MSC_VER) /* Visual */1458unsigned long r=0;1459return _BitScanReverse ( &r, val ) ? (unsigned)r : 0;1460# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */1461return __builtin_clz (val) ^ 31;1462# elif defined(__ICCARM__) /* IAR Intrinsic */1463return 31 - __CLZ(val);1464# else /* Software version */1465static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,146611, 14, 16, 18, 22, 25, 3, 30,14678, 12, 20, 28, 15, 17, 24, 7,146819, 27, 23, 6, 26, 5, 4, 31 };1469U32 v = val;1470v |= v >> 1;1471v |= v >> 2;1472v |= v >> 4;1473v |= v >> 8;1474v |= v >> 16;1475return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];1476# endif1477}1478}1479/*===== Local Constants =====*/1481static const unsigned BIT_mask[] = {14820, 1, 3, 7, 0xF, 0x1F,14830x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,14840xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,14850x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,14860xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,14870x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */1488#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))1489/*-**************************************************************1491* bitStream encoding1492****************************************************************/1493/*! BIT_initCStream() :1494* `dstCapacity` must be > sizeof(size_t)1495* @return : 0 if success,1496* otherwise an error code (can be tested using ERR_isError()) */1497MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,1498void* startPtr, size_t dstCapacity)1499{1500bitC->bitContainer = 0;1501bitC->bitPos = 0;1502bitC->startPtr = (char*)startPtr;1503bitC->ptr = bitC->startPtr;1504bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);1505if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);1506return 0;1507}1508/*! BIT_addBits() :1510* can add up to 31 bits into `bitC`.1511* Note : does not check for register overflow ! */1512MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,1513size_t value, unsigned nbBits)1514{1515MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);1516assert(nbBits < BIT_MASK_SIZE);1517assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);1518bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;1519bitC->bitPos += nbBits;1520}1521/*! BIT_addBitsFast() :1523* works only if `value` is _clean_,1524* meaning all high bits above nbBits are 0 */1525MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,1526size_t value, unsigned nbBits)1527{1528assert((value>>nbBits) == 0);1529assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);1530bitC->bitContainer |= value << bitC->bitPos;1531bitC->bitPos += nbBits;1532}1533/*! BIT_flushBitsFast() :1535* assumption : bitContainer has not overflowed1536* unsafe version; does not check buffer overflow */1537MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)1538{1539size_t const nbBytes = bitC->bitPos >> 3;1540assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);1541assert(bitC->ptr <= bitC->endPtr);1542MEM_writeLEST(bitC->ptr, bitC->bitContainer);1543bitC->ptr += nbBytes;1544bitC->bitPos &= 7;1545bitC->bitContainer >>= nbBytes*8;1546}1547/*! BIT_flushBits() :1549* assumption : bitContainer has not overflowed1550* safe version; check for buffer overflow, and prevents it.1551* note : does not signal buffer overflow.1552* overflow will be revealed later on using BIT_closeCStream() */1553MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)1554{1555size_t const nbBytes = bitC->bitPos >> 3;1556assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);1557assert(bitC->ptr <= bitC->endPtr);1558MEM_writeLEST(bitC->ptr, bitC->bitContainer);1559bitC->ptr += nbBytes;1560if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;1561bitC->bitPos &= 7;1562bitC->bitContainer >>= nbBytes*8;1563}1564/*! BIT_closeCStream() :1566* @return : size of CStream, in bytes,1567* or 0 if it could not fit into dstBuffer */1568MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)1569{1570BIT_addBitsFast(bitC, 1, 1); /* endMark */1571BIT_flushBits(bitC);1572if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */1573return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);1574}1575/*-********************************************************1578* bitStream decoding1579**********************************************************/1580/*! BIT_initDStream() :1581* Initialize a BIT_DStream_t.1582* `bitD` : a pointer to an already allocated BIT_DStream_t structure.1583* `srcSize` must be the *exact* size of the bitStream, in bytes.1584* @return : size of stream (== srcSize), or an errorCode if a problem is detected1585*/1586MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)1587{1588if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }1589}1633{1636return bitContainer >> start;1637}1638{1641U32 const regMask = sizeof(bitContainer)*8 - 1;1642/* if start > regMask, bitstream is corrupted, and result is undefined */1643assert(nbBits < BIT_MASK_SIZE);1644return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];1645}1646{1649assert(nbBits < BIT_MASK_SIZE);1650return bitContainer & BIT_mask[nbBits];1651}1652/*! BIT_lookBits() :1654* Provides next n bits from local register.1655* local register is not modified.1656* On 32-bits, maxNbBits==24.1657* On 64-bits, maxNbBits==56.1658* @return : value extracted */1659MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)1660{1661/* arbitrate between double-shift and shift+mask */1662#if 11663/* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,1664* bitstream is likely corrupted, and result is undefined */1665return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);1666#else1667/* this code path is slower on my os-x laptop */1668U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;1669return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);1670#endif1671}1672/*! BIT_lookBitsFast() :1674* unsafe version; only works if nbBits >= 1 */1675MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)1676{1677U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;1678assert(nbBits >= 1);1679return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);1680}1681{1684bitD->bitsConsumed += nbBits;1685}1686/*! BIT_readBits() :1688* Read (consume) next n bits from local register and update.1689* Pay attention to not read more than nbBits contained into local register.1690* @return : extracted value. */1691MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)1692{1693size_t const value = BIT_lookBits(bitD, nbBits);1694BIT_skipBits(bitD, nbBits);1695return value;1696}1697/*! BIT_readBitsFast() :1699* unsafe version; only works only if nbBits >= 1 */1700MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)1701{1702size_t const value = BIT_lookBitsFast(bitD, nbBits);1703assert(nbBits >= 1);1704BIT_skipBits(bitD, nbBits);1705return value;1706}1707/*! BIT_reloadDStreamFast() :1709* Similar to BIT_reloadDStream(), but with two differences:1710* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!1711* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this1712* point you must use BIT_reloadDStream() to reload.1713*/1714MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)1715{1716if (UNLIKELY(bitD->ptr < bitD->limitPtr))1717return BIT_DStream_overflow;1718assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);1719bitD->ptr -= bitD->bitsConsumed >> 3;1720bitD->bitsConsumed &= 7;1721bitD->bitContainer = MEM_readLEST(bitD->ptr);1722return BIT_DStream_unfinished;1723}1724/*! BIT_reloadDStream() :1726* Refill `bitD` from buffer previously set in BIT_initDStream() .1727* This function is safe, it guarantees it will not read beyond src buffer.1728* @return : status of `BIT_DStream_t` internal register.1729* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */1730MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)1731{1732if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */1733return BIT_DStream_overflow;1734}1755/*! BIT_endOfDStream() :1757* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).1758*/1759MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)1760{1761return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));1762}1763}1766#endif1767/**** ended inlining bitstream.h ****/1770/* *****************************************1773* Static allocation1774*******************************************/1775/* FSE buffer bounds */1776#define FSE_NCOUNTBOUND 5121777#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */)1778#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */1779/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */1781#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))1782#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))1783/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */1785#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))1786#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))1787/* *****************************************1790* FSE advanced API1791***************************************** */1792/**< same as FSE_optimalTableLog(), which used `minus==2` */1795/* FSE_compress_wksp() :1797* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).1798* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.1799*/1800#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )1801size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);1802/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */1805/**< build a fake FSE_CTable, designed to compress always the same symbolValue */1808/* FSE_buildCTable_wksp() :1810* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).1811* `wkspSize` must be >= `(1<<tableLog)`.1812*/1813size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);1814/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */1817/**< build a fake FSE_DTable, designed to always generate the same symbolValue */1820/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */1823/* *****************************************1831* FSE symbol compression API1832*******************************************/1833/*!1834This API consists of small unitary functions, which highly benefit from being inlined.1835Hence their body are included in next section.1836*/1837typedef struct {1838ptrdiff_t value;1839const void* stateTable;1840const void* symbolTT;1841unsigned stateLog;1842} FSE_CState_t;1843/**<1851These functions are inner components of FSE_compress_usingCTable().1852They allow the creation of custom streams, mixing multiple tables and bit sources.1853A key property to keep in mind is that encoding and decoding are done **in reverse direction**.1855So the first symbol you will encode is the last you will decode, like a LIFO stack.1856You will need a few variables to track your CStream. They are :1858FSE_CTable ct; // Provided by FSE_buildCTable()1860BIT_CStream_t bitStream; // bitStream tracking structure1861FSE_CState_t state; // State tracking structure (can have several)1862The first thing to do is to init bitStream and state.1865size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);1866FSE_initCState(&state, ct);1867Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();1869You can then encode your input data, byte after byte.1870FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.1871Remember decoding will be done in reverse direction.1872FSE_encodeByte(&bitStream, &state, symbol);1873At any time, you can also add any bit sequence.1875Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders1876BIT_addBits(&bitStream, bitField, nbBits);1877The above methods don't commit data to memory, they just store it into local register, for speed.1879Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).1880Writing data to memory is a manual operation, performed by the flushBits function.1881BIT_flushBits(&bitStream);1882Your last FSE encoding operation shall be to flush your last state value(s).1884FSE_flushState(&bitStream, &state);1885Finally, you must close the bitStream.1887The function returns the size of CStream in bytes.1888If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)1889If there is an error, it returns an errorCode (which can be tested using FSE_isError()).1890size_t size = BIT_closeCStream(&bitStream);1891*/1892/* *****************************************1895* FSE symbol decompression API1896*******************************************/1897typedef struct {1898size_t state;1899const void* table; /* precise table may vary, depending on U16 */1900} FSE_DState_t;1901/**<1910Let's now decompose FSE_decompress_usingDTable() into its unitary components.1911You will decode FSE-encoded symbols from the bitStream,1912and also any other bitFields you put in, **in reverse order**.1913You will need a few variables to track your bitStream. They are :1915BIT_DStream_t DStream; // Stream context1917FSE_DState_t DState; // State context. Multiple ones are possible1918FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()1919The first thing to do is to init the bitStream.1921errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);1922You should then retrieve your initial state(s)1924(in reverse flushing order if you have several ones) :1925errorCode = FSE_initDState(&DState, &DStream, DTablePtr);1926You can then decode your data, symbol after symbol.1928For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.1929Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).1930unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);1931You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)1933Note : maximum allowed nbBits is 25, for 32-bits compatibility1934size_t bitField = BIT_readBits(&DStream, nbBits);1935All above operations only read from local register (which size depends on size_t).1937Refueling the register from memory is manually performed by the reload method.1938endSignal = FSE_reloadDStream(&DStream);1939BIT_reloadDStream() result tells if there is still some more data to read from DStream.1941BIT_DStream_unfinished : there is still some data left into the DStream.1942BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.1943BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.1944BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.1945When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,1947to properly detect the exact end of stream.1948After each decoded symbol, check if DStream is fully consumed using this simple test :1949BIT_reloadDStream(&DStream) >= BIT_DStream_completed1950When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.1952Checking if DStream has reached its end is performed by :1953BIT_endOfDStream(&DStream);1954Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.1955FSE_endOfDState(&DState);1956*/1957/* *****************************************1960* FSE unsafe API1961*******************************************/1962static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);1963/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */1964/* *****************************************1967* Implementation of inlined functions1968*******************************************/1969typedef struct {1970int deltaFindState;1971U32 deltaNbBits;1972} FSE_symbolCompressionTransform; /* total 8 bytes */1973{1976const void* ptr = ct;1977const U16* u16ptr = (const U16*) ptr;1978const U32 tableLog = MEM_read16(ptr);1979statePtr->value = (ptrdiff_t)1<<tableLog;1980statePtr->stateTable = u16ptr+2;1981statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);1982statePtr->stateLog = tableLog;1983}1984/*! FSE_initCState2() :1987* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)1988* uses the smallest state value possible, saving the cost of this symbol */1989MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)1990{1991FSE_initCState(statePtr, ct);1992{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];1993const U16* stateTable = (const U16*)(statePtr->stateTable);1994U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);1995statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;1996statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];1997}1998}1999{2002FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];2003const U16* const stateTable = (const U16*)(statePtr->stateTable);2004U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);2005BIT_addBits(bitC, statePtr->value, nbBitsOut);2006statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];2007}2008{2011BIT_addBits(bitC, statePtr->value, statePtr->stateLog);2012BIT_flushBits(bitC);2013}2014/* FSE_getMaxNbBits() :2017* Approximate maximum cost of a symbol, in bits.2018* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)2019* note 1 : assume symbolValue is valid (<= maxSymbolValue)2020* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */2021MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)2022{2023const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;2024return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;2025}2026/* FSE_bitCost() :2028* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)2029* note 1 : assume symbolValue is valid (<= maxSymbolValue)2030* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */2031MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)2032{2033const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;2034U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;2035U32 const threshold = (minNbBits+1) << 16;2036assert(tableLog < 16);2037assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */2038{ U32 const tableSize = 1 << tableLog;2039U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);2040U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */2041U32 const bitMultiplier = 1 << accuracyLog;2042assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);2043assert(normalizedDeltaFromThreshold <= bitMultiplier);2044return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;2045}2046}2047/* ====== Decompression ====== */2050{2058unsigned short newState;2059unsigned char symbol;2060unsigned char nbBits;2061} FSE_decode_t; /* size == U32 */2062{2065const void* ptr = dt;2066const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;2067DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);2068BIT_reloadDStream(bitD);2069DStatePtr->table = dt + 1;2070}2071{2074FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];2075return DInfo.symbol;2076}2077{2080FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];2081U32 const nbBits = DInfo.nbBits;2082size_t const lowBits = BIT_readBits(bitD, nbBits);2083DStatePtr->state = DInfo.newState + lowBits;2084}2085{2088FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];2089U32 const nbBits = DInfo.nbBits;2090BYTE const symbol = DInfo.symbol;2091size_t const lowBits = BIT_readBits(bitD, nbBits);2092}2096/*! FSE_decodeSymbolFast() :2098unsafe, only works if no symbol has a probability > 50% */2099MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)2100{2101FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];2102U32 const nbBits = DInfo.nbBits;2103BYTE const symbol = DInfo.symbol;2104size_t const lowBits = BIT_readBitsFast(bitD, nbBits);2105}2109{2112return DStatePtr->state == 0;2113}2114/* **************************************************************2120* Tuning parameters2121****************************************************************/2122/*!MEMORY_USAGE :2123* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)2124* Increasing memory usage improves compression ratio2125* Reduced memory usage can improve speed, due to cache effect2126* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */2127#ifndef FSE_MAX_MEMORY_USAGE2128# define FSE_MAX_MEMORY_USAGE 142129#endif2130#ifndef FSE_DEFAULT_MEMORY_USAGE2131# define FSE_DEFAULT_MEMORY_USAGE 132132#endif2133/*!FSE_MAX_SYMBOL_VALUE :2135* Maximum symbol value authorized.2136* Required for proper stack allocation */2137#ifndef FSE_MAX_SYMBOL_VALUE2138# define FSE_MAX_SYMBOL_VALUE 2552139#endif2140/* **************************************************************2142* template functions type & suffix2143****************************************************************/2144#define FSE_FUNCTION_TYPE BYTE2145#define FSE_FUNCTION_EXTENSION2146#define FSE_DECODE_TYPE FSE_decode_t2147/* ***************************************************************2153* Constants2154*****************************************************************/2155#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)2156#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)2157#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)2158#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)2159#define FSE_MIN_TABLELOG 52160}2174#endif2175/**** ended inlining fse.h ****/2176#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */2177/**** start inlining huf.h ****/2178/* ******************************************************************2179* huff0 huffman codec,2180* part of Finite State Entropy library2181* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.2182*2183* You can contact the author at :2184* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy2185*2186* This source code is licensed under both the BSD-style license (found in the2187* LICENSE file in the root directory of this source tree) and the GPLv2 (found2188* in the COPYING file in the root directory of this source tree).2189* You may select, at your option, one of the above-listed licenses.2190****************************************************************** */2191/* *** Dependencies *** */2200#include <stddef.h> /* size_t */2201/* *** library symbols visibility *** */2204/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,2205* HUF symbols remain "private" (internal symbols for library only).2206* Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */2207#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)2208# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))2209#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */2210# define HUF_PUBLIC_API __declspec(dllexport)2211#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)2212# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */2213#else2214# define HUF_PUBLIC_API2215#endif2216/* ========================== */2219/* *** simple functions *** */2220/* ========================== */2221/** HUF_compress() :2223* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.2224* 'dst' buffer must be already allocated.2225* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).2226* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.2227* @return : size of compressed data (<= `dstCapacity`).2228* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!2229* if HUF_isError(return), compression failed (more details using HUF_getErrorName())2230*/2231HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,2232const void* src, size_t srcSize);2233/** HUF_decompress() :2235* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',2236* into already allocated buffer 'dst', of minimum size 'dstSize'.2237* `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.2238* Note : in contrast with FSE, HUF_decompress can regenerate2239* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,2240* because it knows size to regenerate (originalSize).2241* @return : size of regenerated data (== originalSize),2242* or an error code, which can be tested using HUF_isError()2243*/2244HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,2245const void* cSrc, size_t cSrcSize);2246/* *** Tool functions *** */2249#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */2250HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */2251/* Error Management */2253HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */2254HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */2255/* *** Advanced function *** */2258/** HUF_compress2() :2260* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.2261* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .2262* `tableLog` must be `<= HUF_TABLELOG_MAX` . */2263HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,2264const void* src, size_t srcSize,2265unsigned maxSymbolValue, unsigned tableLog);2266/** HUF_compress4X_wksp() :2268* Same as HUF_compress2(), but uses externally allocated `workSpace`.2269* `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */2270#define HUF_WORKSPACE_SIZE ((6 << 10) + 256)2271#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))2272HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,2273const void* src, size_t srcSize,2274unsigned maxSymbolValue, unsigned tableLog,2275void* workSpace, size_t wkspSize);2276/* ******************************************************************2280* WARNING !!2281* The following section contains advanced and experimental definitions2282* which shall never be used in the context of a dynamic library,2283* because they are not guaranteed to remain stable in the future.2284* Only consider them in association with static linking.2285* *****************************************************************/2286#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)2287#define HUF_H_HUF_STATIC_LINKING_ONLY2288/* *** Dependencies *** */2290/**** skipping file: mem.h ****/2291/* *** Constants *** */2294#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */2295#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */2296#define HUF_SYMBOLVALUE_MAX 2552297/* ****************************************2305* Static allocation2306******************************************/2307/* HUF buffer bounds */2308#define HUF_CTABLEBOUND 1292309#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */2310#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */2311/* static allocation of HUF's Compression Table */2313#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */2314#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))2315#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \2316U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \2317void* name##hv = &(name##hb); \2318HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */2319/* static allocation of HUF's DTable */2321typedef U32 HUF_DTable;2322#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))2323#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \2324HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }2325#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \2326HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }2327/* ****************************************2330* Advanced decompression functions2331******************************************/2332size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */2333#ifndef HUF_FORCE_DECOMPRESS_X12334size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */2335#endif2336/* ****************************************2349* HUF detailed API2350* ****************************************/2351/*! HUF_compress() does the following:2353* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")2354* 2. (optional) refine tableLog using HUF_optimalTableLog()2355* 3. build Huffman table from count using HUF_buildCTable()2356* 4. save Huffman table to memory buffer using HUF_writeCTable()2357* 5. encode the data stream using HUF_compress4X_usingCTable()2358*2359* The following API allows targeting specific sub-functions for advanced tasks.2360* For example, it's possible to compress several blocks using the same 'CTable',2361* or to save and regenerate 'CTable' using external methods.2362*/2363unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);2364typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */2365size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */2366size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);2367size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);2368size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);2369int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);2370/** HUF_compress4X_repeat() :2377* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.2378* If it uses hufTable it does not modify hufTable or repeat.2379* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.2380* If preferRepeat then the old table will always be used if valid. */2381size_t HUF_compress4X_repeat(void* dst, size_t dstSize,2382const void* src, size_t srcSize,2383unsigned maxSymbolValue, unsigned tableLog,2384void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */2385HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);2386/** HUF_buildCTable_wksp() :2388* Same as HUF_buildCTable(), but using externally allocated scratch buffer.2389* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.2390*/2391#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)2392#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))2393size_t HUF_buildCTable_wksp (HUF_CElt* tree,2394const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,2395void* workSpace, size_t wkspSize);2396/*! HUF_readStats() :2398* Read compact Huffman tree, saved by HUF_writeCTable().2399* `huffWeight` is destination buffer.2400* @return : size read from `src` , or an error Code .2401* Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */2402size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,2403U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,2404const void* src, size_t srcSize);2405/** HUF_readCTable() :2407* Loading a CTable saved with HUF_writeCTable() */2408size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);2409/** HUF_getNbBits() :2411* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX2412* Note 1 : is not inlined, as HUF_CElt definition is private2413* Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */2414U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue);2415/*2417* HUF_decompress() does the following:2418* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics2419* 2. build Huffman table from save, using HUF_readDTableX?()2420* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()2421*/2422/** HUF_selectDecoder() :2424* Tells which decoder is likely to decode faster,2425* based on a set of pre-computed metrics.2426* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .2427* Assumption : 0 < dstSize <= 128 KB */2428U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);2429/**2431* The minimum workspace size for the `workSpace` used in2432* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().2433*2434* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when2435* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.2436* Buffer overflow errors may potentially occur if code modifications result in2437* a required workspace size greater than that specified in the following2438* macro.2439*/2440#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)2441#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))2442/* ====================== */2462/* single stream variants */2463/* ====================== */2464/** HUF_compress1X_repeat() :2469* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.2470* If it uses hufTable it does not modify hufTable or repeat.2471* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.2472* If preferRepeat then the old table will always be used if valid. */2473size_t HUF_compress1X_repeat(void* dst, size_t dstSize,2474const void* src, size_t srcSize,2475unsigned maxSymbolValue, unsigned tableLog,2476void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */2477HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);2478/* BMI2 variants.2504* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.2505*/2506size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);2507#ifndef HUF_FORCE_DECOMPRESS_X22508size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);2509#endif2510size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);2511size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);2512}2517#endif2518/**** ended inlining huf.h ****/2519/*=== Version ===*/2522unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }2523/*=== Error Management ===*/2526unsigned FSE_isError(size_t code) { return ERR_isError(code); }2527const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }2528/*-**************************************************************2534* FSE NCount encoding-decoding2535****************************************************************/2536size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,2537const void* headerBuffer, size_t hbSize)2538{2539const BYTE* const istart = (const BYTE*) headerBuffer;2540const BYTE* const iend = istart + hbSize;2541const BYTE* ip = istart;2542int nbBits;2543int remaining;2544int threshold;2545U32 bitStream;2546int bitCount;2547unsigned charnum = 0;2548int previous0 = 0;2549}2641/*! HUF_readStats() :2644Read compact Huffman tree, saved by HUF_writeCTable().2645`huffWeight` is destination buffer.2646`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.2647@return : size read from `src` , or an error Code .2648Note : Needed by HUF_readCTable() and HUF_readDTableX?() .2649*/2650size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,2651U32* nbSymbolsPtr, U32* tableLogPtr,2652const void* src, size_t srcSize)2653{2654U32 weightTotal;2655const BYTE* ip = (const BYTE*) src;2656size_t iSize;2657size_t oSize;2658}2712/**** ended inlining common/entropy_common.c ****/2713/**** start inlining common/error_private.c ****/2714/*2715* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.2716* All rights reserved.2717*2718* This source code is licensed under both the BSD-style license (found in the2719* LICENSE file in the root directory of this source tree) and the GPLv2 (found2720* in the COPYING file in the root directory of this source tree).2721* You may select, at your option, one of the above-listed licenses.2722*/2723/* The purpose of this file is to have a single list of error strings embedded in binary */2725/**** skipping file: error_private.h ****/2727{2730#ifdef ZSTD_STRIP_ERROR_STRINGS2731(void)code;2732return "Error strings stripped";2733#else2734static const char* const notErrorCode = "Unspecified error code";2735switch( code )2736{2737case PREFIX(no_error): return "No error detected";2738case PREFIX(GENERIC): return "Error (generic)";2739case PREFIX(prefix_unknown): return "Unknown frame descriptor";2740case PREFIX(version_unsupported): return "Version not supported";2741case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";2742case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";2743case PREFIX(corruption_detected): return "Corrupted block detected";2744case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";2745case PREFIX(parameter_unsupported): return "Unsupported parameter";2746case PREFIX(parameter_outOfBound): return "Parameter is out of bound";2747case PREFIX(init_missing): return "Context should be init first";2748case PREFIX(memory_allocation): return "Allocation error : not enough memory";2749case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";2750case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";2751case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";2752case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";2753case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";2754case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";2755case PREFIX(dictionary_wrong): return "Dictionary mismatch";2756case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";2757case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";2758case PREFIX(srcSize_wrong): return "Src size is incorrect";2759case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";2760/* following error codes are not stable and may be removed or changed in a future version */2761case PREFIX(frameIndex_tooLarge): return "Frame index is too large";2762case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";2763case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";2764case PREFIX(maxCode):2765default: return notErrorCode;2766}2767#endif2768}2769/**** ended inlining common/error_private.c ****/2770/**** start inlining common/fse_decompress.c ****/2771/* ******************************************************************2772* FSE : Finite State Entropy decoder2773* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.2774*2775* You can contact the author at :2776* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy2777* - Public forum : https://groups.google.com/forum/#!forum/lz4c2778*2779* This source code is licensed under both the BSD-style license (found in the2780* LICENSE file in the root directory of this source tree) and the GPLv2 (found2781* in the COPYING file in the root directory of this source tree).2782* You may select, at your option, one of the above-listed licenses.2783****************************************************************** */2784/* **************************************************************2787* Includes2788****************************************************************/2789/**** skipping file: bitstream.h ****/2790/**** skipping file: compiler.h ****/2791#define FSE_STATIC_LINKING_ONLY2792/**** skipping file: fse.h ****/2793/**** skipping file: error_private.h ****/2794/* **************************************************************2797* Error Management2798****************************************************************/2799#define FSE_isError ERR_isError2800#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */2801/* **************************************************************2804* Templates2805****************************************************************/2806/*2807designed to be included2808for type-specific functions (template emulation in C)2809Objective is to write these functions only once, for improved maintenance2810*/2811/* safety checks */2813#ifndef FSE_FUNCTION_EXTENSION2814# error "FSE_FUNCTION_EXTENSION must be defined"2815#endif2816#ifndef FSE_FUNCTION_TYPE2817# error "FSE_FUNCTION_TYPE must be defined"2818#endif2819/* Function names */2821#define FSE_CAT(X,Y) X##Y2822#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)2823#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)2824/* Function templates */2827FSE_DTable* FSE_createDTable (unsigned tableLog)2828{2829if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;2830return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );2831}2832{2835free(dt);2836}2837{2840void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */2841FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);2842U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];2843}2894/*-*******************************************************2899* Decompression (Byte symbols)2900*********************************************************/2901size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)2902{2903void* ptr = dt;2904FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;2905void* dPtr = dt + 1;2906FSE_decode_t* const cell = (FSE_decode_t*)dPtr;2907}2917{2921void* ptr = dt;2922FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;2923void* dPtr = dt + 1;2924FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;2925const unsigned tableSize = 1 << nbBits;2926const unsigned tableMask = tableSize - 1;2927const unsigned maxSV1 = tableMask+1;2928unsigned s;2929}2944{2950BYTE* const ostart = (BYTE*) dst;2951BYTE* op = ostart;2952BYTE* const omax = op + maxDstSize;2953BYTE* const olimit = omax-3;2954}3006{3012const void* ptr = dt;3013const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;3014const U32 fastMode = DTableH->fastMode;3015}3020{3024const BYTE* const istart = (const BYTE*)cSrc;3025const BYTE* ip = istart;3026short counting[FSE_MAX_SYMBOL_VALUE+1];3027unsigned tableLog;3028unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;3029}3042{3048DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */3049return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);3050}3051/**** ended inlining common/fse_decompress.c ****/3056/**** start inlining common/zstd_common.c ****/3057/*3058* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.3059* All rights reserved.3060*3061* This source code is licensed under both the BSD-style license (found in the3062* LICENSE file in the root directory of this source tree) and the GPLv2 (found3063* in the COPYING file in the root directory of this source tree).3064* You may select, at your option, one of the above-listed licenses.3065*/3066/*-*************************************3070* Dependencies3071***************************************/3072/**** skipping file: error_private.h ****/3073/**** start inlining zstd_internal.h ****/3074/*3075* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.3076* All rights reserved.3077*3078* This source code is licensed under both the BSD-style license (found in the3079* LICENSE file in the root directory of this source tree) and the GPLv2 (found3080* in the COPYING file in the root directory of this source tree).3081* You may select, at your option, one of the above-listed licenses.3082*/3083/* this module contains definitions which must be identical3088* across compression, decompression and dictBuilder.3089* It also contains a few functions useful to at least 2 of them3090* and which benefit from being inlined */3091/*-*************************************3093* Dependencies3094***************************************/3095#ifdef __aarch64__3096#include <arm_neon.h>3097#endif3098/**** skipping file: compiler.h ****/3099/**** skipping file: mem.h ****/3100/**** skipping file: debug.h ****/3101/**** skipping file: error_private.h ****/3102#define ZSTD_STATIC_LINKING_ONLY3103/**** start inlining ../zstd.h ****/3104/*3105* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.3106* All rights reserved.3107*3108* This source code is licensed under both the BSD-style license (found in the3109* LICENSE file in the root directory of this source tree) and the GPLv2 (found3110* in the COPYING file in the root directory of this source tree).3111* You may select, at your option, one of the above-listed licenses.3112*/3113#if defined (__cplusplus)3114extern "C" {3115#endif3116/* ====== Dependency ======*/3121#include <limits.h> /* INT_MAX */3122#include <stddef.h> /* size_t */3123/* ===== ZSTDLIB_API : control library symbols visibility ===== */3126#ifndef ZSTDLIB_VISIBILITY3127# if defined(__GNUC__) && (__GNUC__ >= 4)3128# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))3129# else3130# define ZSTDLIB_VISIBILITY3131# endif3132#endif3133#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)3134# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY3135#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)3136# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/3137#else3138# define ZSTDLIB_API ZSTDLIB_VISIBILITY3139#endif3140/*******************************************************************************3143Introduction3144zstd, short for Zstandard, is a fast lossless compression algorithm, targeting3146real-time compression scenarios at zlib-level and better compression ratios.3147The zstd compression library provides in-memory compression and decompression3148functions.3149The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),3151which is currently 22. Levels >= 20, labeled `--ultra`, should be used with3152caution, as they require more memory. The library also offers negative3153compression levels, which extend the range of speed vs. ratio preferences.3154The lower the level, the faster the speed (at the cost of compression).3155Compression can be done in:3157- a single step (described as Simple API)3158- a single step, reusing a context (described as Explicit context)3159- unbounded multiple steps (described as Streaming compression)3160The compression ratio achievable on small data can be highly improved using3162a dictionary. Dictionary compression can be performed in:3163- a single step (described as Simple dictionary API)3164- a single step, reusing a dictionary (described as Bulk-processing3165dictionary API)3166Advanced experimental functions can be accessed using3168`#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.3169Advanced experimental APIs should never be used with a dynamically-linked3171library. They are not "stable"; their definitions or signatures may change in3172the future. Only static linking is allowed.3173*******************************************************************************/3174/*------ Version ------*/3176#define ZSTD_VERSION_MAJOR 13177#define ZSTD_VERSION_MINOR 43178#define ZSTD_VERSION_RELEASE 53179/* *************************************3190* Default constant3191***************************************/3192#ifndef ZSTD_CLEVEL_DEFAULT3193# define ZSTD_CLEVEL_DEFAULT 33194#endif3195/* *************************************3197* Constants3198***************************************/3199/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */3201#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */3202#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */3203#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */3204#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF03205/***************************************3212* Simple API3213***************************************/3214/*! ZSTD_compress() :3215* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.3216* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.3217* @return : compressed size written into `dst` (<= `dstCapacity),3218* or an error code if it fails (which can be tested using ZSTD_isError()). */3219ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,3220const void* src, size_t srcSize,3221int compressionLevel);3222/*! ZSTD_decompress() :3224* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.3225* `dstCapacity` is an upper bound of originalSize to regenerate.3226* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.3227* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),3228* or an errorCode if it fails (which can be tested using ZSTD_isError()). */3229ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,3230const void* src, size_t compressedSize);3231/*! ZSTD_getFrameContentSize() : requires v1.3.0+3233* `src` should point to the start of a ZSTD encoded frame.3234* `srcSize` must be at least as large as the frame header.3235* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.3236* @return : - decompressed size of `src` frame content, if known3237* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined3238* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)3239* note 1 : a 0 return value means the frame is valid but "empty".3240* note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.3241* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.3242* In which case, it's necessary to use streaming mode to decompress data.3243* Optionally, application can rely on some implicit limit,3244* as ZSTD_decompress() only needs an upper bound of decompressed size.3245* (For example, data could be necessarily cut into blocks <= 16 KB).3246* note 3 : decompressed size is always present when compression is completed using single-pass functions,3247* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().3248* note 4 : decompressed size can be very large (64-bits value),3249* potentially larger than what local system can handle as a single memory segment.3250* In which case, it's necessary to use streaming mode to decompress data.3251* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.3252* Always ensure return value fits within application's authorized limits.3253* Each application can set its own limits.3254* note 6 : This function replaces ZSTD_getDecompressedSize() */3255#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)3256#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)3257ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);3258/*! ZSTD_getDecompressedSize() :3260* NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().3261* Both functions work the same way, but ZSTD_getDecompressedSize() blends3262* "empty", "unknown" and "error" results to the same return value (0),3263* while ZSTD_getFrameContentSize() gives them separate return values.3264* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */3265ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);3266/*! ZSTD_findFrameCompressedSize() :3268* `src` should point to the start of a ZSTD frame or skippable frame.3269* `srcSize` must be >= first frame size3270* @return : the compressed size of the first frame starting at `src`,3271* suitable to pass as `srcSize` to `ZSTD_decompress` or similar,3272* or an error code if input is invalid */3273ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);3274/*====== Helper functions ======*/3277#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */3278ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */3279ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */3280ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */3281ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */3282ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */3283/***************************************3286* Explicit context3287***************************************/3288/*= Compression context3289* When compressing many times,3290* it is recommended to allocate a context just once,3291* and re-use it for each successive compression operation.3292* This will make workload friendlier for system's memory.3293* Note : re-using context is just a speed / resource optimization.3294* It doesn't change the compression ratio, which remains identical.3295* Note 2 : In multi-threaded environments,3296* use one different context per thread for parallel execution.3297*/3298typedef struct ZSTD_CCtx_s ZSTD_CCtx;3299ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);3300ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);3301/*! ZSTD_compressCCtx() :3303* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.3304* Important : in order to behave similarly to `ZSTD_compress()`,3305* this function compresses at requested compression level,3306* __ignoring any other parameter__ .3307* If any advanced parameter was set using the advanced API,3308* they will all be reset. Only `compressionLevel` remains.3309*/3310ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,3311void* dst, size_t dstCapacity,3312const void* src, size_t srcSize,3313int compressionLevel);3314/*= Decompression context3316* When decompressing many times,3317* it is recommended to allocate a context only once,3318* and re-use it for each successive compression operation.3319* This will make workload friendlier for system's memory.3320* Use one context per thread for parallel execution. */3321typedef struct ZSTD_DCtx_s ZSTD_DCtx;3322ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);3323ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);3324/*! ZSTD_decompressDCtx() :3326* Same as ZSTD_decompress(),3327* requires an allocated ZSTD_DCtx.3328* Compatible with sticky parameters.3329*/3330ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,3331void* dst, size_t dstCapacity,3332const void* src, size_t srcSize);3333/***************************************3336* Advanced compression API3337***************************************/3338/* API design :3340* Parameters are pushed one by one into an existing context,3341* using ZSTD_CCtx_set*() functions.3342* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.3343* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !3344* __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .3345*3346* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().3347*3348* This API supercedes all other "advanced" API entry points in the experimental section.3349* In the future, we expect to remove from experimental API entry points which are redundant with this API.3350*/3351/* Compression strategies, listed from fastest to strongest */3354typedef enum { ZSTD_fast=1,3355ZSTD_dfast=2,3356ZSTD_greedy=3,3357ZSTD_lazy=4,3358ZSTD_lazy2=5,3359ZSTD_btlazy2=6,3360ZSTD_btopt=7,3361ZSTD_btultra=8,3362ZSTD_btultra2=93363/* note : new strategies _might_ be added in the future.3364Only the order (from fast to strong) is guaranteed */3365} ZSTD_strategy;3366* Note: When compressing with a ZSTD_CDict these parameters are superseded3372* by the parameters used to construct the ZSTD_CDict.3373* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */3374ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.3375* Note that exact compression parameters are dynamically determined,3376* depending on both compression level and srcSize (when known).3377* Default level is ZSTD_CLEVEL_DEFAULT==3.3378* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.3379* Note 1 : it's possible to pass a negative compression level.3380* Note 2 : setting a level does not automatically set all other compression parameters3381* to default. Setting this will however eventually dynamically impact the compression3382* parameters which have not been manually set. The manually set3383* ones will 'stick'. */3384/* Advanced compression parameters :3385* It's possible to pin down compression parameters to some specific values.3386* In which case, these values are no longer dynamically selected by the compressor */3387ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.3388* This will set a memory budget for streaming decompression,3389* with larger values requiring more memory3390* and typically compressing more.3391* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.3392* Special: value 0 means "use default windowLog".3393* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT3394* requires explicitly allowing such size at streaming decompression stage. */3395ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.3396* Resulting memory usage is (1 << (hashLog+2)).3397* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.3398* Larger tables improve compression ratio of strategies <= dFast,3399* and improve speed of strategies > dFast.3400* Special: value 0 means "use default hashLog". */3401ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.3402* Resulting memory usage is (1 << (chainLog+2)).3403* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.3404* Larger tables result in better and slower compression.3405* This parameter is useless for "fast" strategy.3406* It's still useful when using "dfast" strategy,3407* in which case it defines a secondary probe table.3408* Special: value 0 means "use default chainLog". */3409ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.3410* More attempts result in better and slower compression.3411* This parameter is useless for "fast" and "dFast" strategies.3412* Special: value 0 means "use default searchLog". */3413ZSTD_c_minMatch=105, /* Minimum size of searched matches.3414* Note that Zstandard can still find matches of smaller size,3415* it just tweaks its search algorithm to look for this size and larger.3416* Larger values increase compression and decompression speed, but decrease ratio.3417* Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.3418* Note that currently, for all strategies < btopt, effective minimum is 4.3419* , for all strategies > fast, effective maximum is 6.3420* Special: value 0 means "use default minMatchLength". */3421ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.3422* For strategies btopt, btultra & btultra2:3423* Length of Match considered "good enough" to stop search.3424* Larger values make compression stronger, and slower.3425* For strategy fast:3426* Distance between match sampling.3427* Larger values make compression faster, and weaker.3428* Special: value 0 means "use default targetLength". */3429ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.3430* The higher the value of selected strategy, the more complex it is,3431* resulting in stronger and slower compression.3432* Special: value 0 means "use default strategy". */3433* This parameter is designed to improve compression ratio3437* for large inputs, by finding large matches at long distance.3438* It increases memory usage and window size.3439* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB3440* except when expressly set to a different value. */3441ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.3442* Larger values increase memory usage and compression ratio,3443* but decrease compression speed.3444* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX3445* default: windowlog - 7.3446* Special: value 0 means "automatically determine hashlog". */3447ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.3448* Larger/too small values usually decrease compression ratio.3449* Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.3450* Special: value 0 means "use default value" (default: 64). */3451ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.3452* Larger values improve collision resolution but decrease compression speed.3453* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.3454* Special: value 0 means "use default value" (default: 3). */3455ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.3456* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).3457* Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.3458* Larger values improve compression speed.3459* Deviating far from default value will likely result in a compression ratio decrease.3460* Special: value 0 means "automatically determine hashRateLog". */3461* Content size must be known at the beginning of compression.3465* This is automatically the case when using ZSTD_compress2(),3466* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */3467ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */3468ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */3469* They return an error otherwise. */3473ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.3474* When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :3475* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,3476* while compression work is performed in parallel, within worker threads.3477* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :3478* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).3479* More workers improve speed, but also increase memory usage.3480* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */3481ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.3482* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.3483* 0 means default, which is dynamically determined based on compression parameters.3484* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.3485* The minimum size is automatically and transparently enforced. */3486ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.3487* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.3488* It helps preserve compression ratio, while each job is compressed in parallel.3489* This value is enforced only when nbWorkers >= 1.3490* Larger values increase compression ratio, but decrease speed.3491* Possible values range from 0 to 9 :3492* - 0 means "default" : value will be determined by the library, depending on strategy3493* - 1 means "no overlap"3494* - 9 means "full overlap", using a full window size.3495* Each intermediate rank increases/decreases load size by a factor 2 :3496* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default3497* default value varies between 6 and 9, depending on strategy */3498* within the experimental section of the API.3501* At the time of this writing, they include :3502* ZSTD_c_rsyncable3503* ZSTD_c_format3504* ZSTD_c_forceMaxWindow3505* ZSTD_c_forceAttachDict3506* ZSTD_c_literalCompressionMode3507* ZSTD_c_targetCBlockSize3508* ZSTD_c_srcSizeHint3509* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.3510* note : never ever use experimentalParam? names directly;3511* also, the enums values themselves are unstable and can still change.3512*/3513ZSTD_c_experimentalParam1=500,3514ZSTD_c_experimentalParam2=10,3515ZSTD_c_experimentalParam3=1000,3516ZSTD_c_experimentalParam4=1001,3517ZSTD_c_experimentalParam5=1002,3518ZSTD_c_experimentalParam6=1003,3519ZSTD_c_experimentalParam7=10043520} ZSTD_cParameter;3521/*! ZSTD_cParam_getBounds() :3529* All parameters must belong to an interval with lower and upper bounds,3530* otherwise they will either trigger an error or be automatically clamped.3531* @return : a structure, ZSTD_bounds, which contains3532* - an error status field, which must be tested using ZSTD_isError()3533* - lower and upper bounds, both inclusive3534*/3535ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);3536/*! ZSTD_CCtx_setParameter() :3538* Set one compression parameter, selected by enum ZSTD_cParameter.3539* All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().3540* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).3541* Setting a parameter is generally only possible during frame initialization (before starting compression).3542* Exception : when using multi-threading mode (nbWorkers >= 1),3543* the following parameters can be updated _during_ compression (within same frame):3544* => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.3545* new parameters will be active for next job only (after a flush()).3546* @return : an error code (which can be tested using ZSTD_isError()).3547*/3548ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);3549/*! ZSTD_CCtx_setPledgedSrcSize() :3551* Total input data size to be compressed as a single frame.3552* Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.3553* This value will also be controlled at end of frame, and trigger an error if not respected.3554* @result : 0, or an error code (which can be tested with ZSTD_isError()).3555* Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.3556* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.3557* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.3558* Note 2 : pledgedSrcSize is only valid once, for the next frame.3559* It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.3560* Note 3 : Whenever all input data is provided and consumed in a single round,3561* for example with ZSTD_compress2(),3562* or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),3563* this value is automatically overridden by srcSize instead.3564*/3565ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);3566/*! ZSTD_CCtx_reset() :3574* There are 2 different things that can be reset, independently or jointly :3575* - The session : will stop compressing current frame, and make CCtx ready to start a new one.3576* Useful after an error, or to interrupt any ongoing compression.3577* Any internal data not yet flushed is cancelled.3578* Compression parameters and dictionary remain unchanged.3579* They will be used to compress next frame.3580* Resetting session never fails.3581* - The parameters : changes all parameters back to "default".3582* This removes any reference to any dictionary too.3583* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)3584* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())3585* - Both : similar to resetting the session, followed by resetting parameters.3586*/3587ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);3588/*! ZSTD_compress2() :3590* Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.3591* ZSTD_compress2() always starts a new frame.3592* Should cctx hold data from a previously unfinished frame, everything about it is forgotten.3593* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()3594* - The function is always blocking, returns when compression is completed.3595* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.3596* @return : compressed size written into `dst` (<= `dstCapacity),3597* or an error code if it fails (which can be tested using ZSTD_isError()).3598*/3599ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,3600void* dst, size_t dstCapacity,3601const void* src, size_t srcSize);3602/***************************************3605* Advanced decompression API3606***************************************/3607/* The advanced API pushes parameters one by one into an existing DCtx context.3609* Parameters are sticky, and remain valid for all following frames3610* using the same DCtx context.3611* It's possible to reset parameters to default values using ZSTD_DCtx_reset().3612* Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().3613* Therefore, no new decompression function is necessary.3614*/3615* the streaming API will refuse to allocate memory buffer3620* in order to protect the host from unreasonable memory requirements.3621* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.3622* By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).3623* Special: value 0 means "use default maximum windowLog". */3624* within the experimental section of the API.3627* At the time of this writing, they include :3628* ZSTD_d_format3629* ZSTD_d_stableOutBuffer3630* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.3631* note : never ever use experimentalParam? names directly3632*/3633ZSTD_d_experimentalParam1=1000,3634ZSTD_d_experimentalParam2=10013635/*! ZSTD_dParam_getBounds() :3639* All parameters must belong to an interval with lower and upper bounds,3640* otherwise they will either trigger an error or be automatically clamped.3641* @return : a structure, ZSTD_bounds, which contains3642* - an error status field, which must be tested using ZSTD_isError()3643* - both lower and upper bounds, inclusive3644*/3645ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);3646/*! ZSTD_DCtx_setParameter() :3648* Set one compression parameter, selected by enum ZSTD_dParameter.3649* All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().3650* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).3651* Setting a parameter is only possible during frame initialization (before starting decompression).3652* @return : 0, or an error code (which can be tested using ZSTD_isError()).3653*/3654ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);3655/*! ZSTD_DCtx_reset() :3657* Return a DCtx to clean state.3658* Session and parameters can be reset jointly or separately.3659* Parameters can only be reset when no active frame is being decompressed.3660* @return : 0, or an error code, which can be tested with ZSTD_isError()3661*/3662ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);3663/****************************3666* Streaming3667****************************/3668/*-***********************************************************************3684* Streaming compression - HowTo3685*3686* A ZSTD_CStream object is required to track streaming operation.3687* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.3688* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.3689* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.3690*3691* For parallel execution, use one separate ZSTD_CStream per thread.3692*3693* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.3694*3695* Parameters are sticky : when starting a new compression on the same context,3696* it will re-use the same sticky parameters as previous compression session.3697* When in doubt, it's recommended to fully initialize the context before usage.3698* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),3699* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to3700* set more specific parameters, the pledged source size, or load a dictionary.3701*3702* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to3703* consume input stream. The function will automatically update both `pos`3704* fields within `input` and `output`.3705* Note that the function may not consume the entire input, for example, because3706* the output buffer is already full, in which case `input.pos < input.size`.3707* The caller must check if input has been entirely consumed.3708* If not, the caller must make some room to receive more compressed data,3709* and then present again remaining input data.3710* note: ZSTD_e_continue is guaranteed to make some forward progress when called,3711* but doesn't guarantee maximal forward progress. This is especially relevant3712* when compressing with multiple threads. The call won't block if it can3713* consume some input, but if it can't it will wait for some, but not all,3714* output to be flushed.3715* @return : provides a minimum amount of data remaining to be flushed from internal buffers3716* or an error code, which can be tested using ZSTD_isError().3717*3718* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,3719* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.3720* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).3721* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.3722* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the3723* operation.3724* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will3725* block until the flush is complete or the output buffer is full.3726* @return : 0 if internal buffers are entirely flushed,3727* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),3728* or an error code, which can be tested using ZSTD_isError().3729*3730* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.3731* It will perform a flush and write frame epilogue.3732* The epilogue is required for decoders to consider a frame completed.3733* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.3734* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to3735* start a new frame.3736* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will3737* block until the flush is complete or the output buffer is full.3738* @return : 0 if frame fully completed and fully flushed,3739* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),3740* or an error code, which can be tested using ZSTD_isError().3741*3742* *******************************************************************/3743/*===== ZSTD_CStream management functions =====*/3747ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);3748ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);3749/*===== Streaming compression functions =====*/3751typedef enum {3752ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */3753ZSTD_e_flush=1, /* flush any data provided so far,3754* it creates (at least) one new block, that can be decoded immediately on reception;3755* frame will continue: any future data can still reference previously compressed data, improving compression.3756* note : multithreaded compression will block to flush as much output as possible. */3757ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.3758* note that frame is only closed after compressed data is fully flushed (return value == 0).3759* After that point, any additional data starts a new frame.3760* note : each frame is independent (does not reference any content from previous frame).3761: note : multithreaded compression will block to flush as much output as possible. */3762} ZSTD_EndDirective;3763/*! ZSTD_compressStream2() :3765* Behaves about the same as ZSTD_compressStream, with additional control on end directive.3766* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()3767* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)3768* - output->pos must be <= dstCapacity, input->pos must be <= srcSize3769* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.3770* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.3771* - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,3772* and then immediately returns, just indicating that there is some data remaining to be flushed.3773* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.3774* - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.3775* - @return provides a minimum amount of data remaining to be flushed from internal buffers3776* or an error code, which can be tested using ZSTD_isError().3777* if @return != 0, flush is not fully completed, there is still some data left within internal buffers.3778* This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.3779* For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.3780* - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),3781* only ZSTD_e_end or ZSTD_e_flush operations are allowed.3782* Before starting a new compression job, or changing compression parameters,3783* it is required to fully flush internal buffers.3784*/3785ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,3786ZSTD_outBuffer* output,3787ZSTD_inBuffer* input,3788ZSTD_EndDirective endOp);3789/* These buffer sizes are softly recommended.3792* They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.3793* Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),3794* reducing the amount of memory shuffling and buffering, resulting in minor performance savings.3795*3796* However, note that these recommendations are from the perspective of a C caller program.3797* If the streaming interface is invoked from some other language,3798* especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,3799* a major performance rule is to reduce crossing such interface to an absolute minimum.3800* It's not rare that performance ends being spent more into the interface, rather than compression itself.3801* In which cases, prefer using large buffers, as large as practical,3802* for both input and output, to reduce the nb of roundtrips.3803*/3804ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */3805ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */3806/* *****************************************************************************3809* This following is a legacy streaming API.3810* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().3811* It is redundant, but remains fully supported.3812* Advanced parameters and dictionary compression can only be used through the3813* new API.3814******************************************************************************/3815/*!3817* Equivalent to:3818*3819* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);3820* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)3821* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);3822*/3823ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);3824/*!3825* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).3826* NOTE: The return value is different. ZSTD_compressStream() returns a hint for3827* the next read size (if non-zero and not an error). ZSTD_compressStream2()3828* returns the minimum nb of bytes left to flush (if non-zero and not an error).3829*/3830ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);3831/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */3832ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);3833/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */3834ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);3835/*-***************************************************************************3838* Streaming decompression - HowTo3839*3840* A ZSTD_DStream object is required to track streaming operations.3841* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.3842* ZSTD_DStream objects can be re-used multiple times.3843*3844* Use ZSTD_initDStream() to start a new decompression operation.3845* @return : recommended first input size3846* Alternatively, use advanced API to set specific properties.3847*3848* Use ZSTD_decompressStream() repetitively to consume your input.3849* The function will update both `pos` fields.3850* If `input.pos < input.size`, some input has not been consumed.3851* It's up to the caller to present again remaining data.3852* The function tries to flush all data decoded immediately, respecting output buffer size.3853* If `output.pos < output.size`, decoder has flushed everything it could.3854* But if `output.pos == output.size`, there might be some data left within internal buffers.,3855* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.3856* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.3857* @return : 0 when a frame is completely decoded and fully flushed,3858* or an error code, which can be tested using ZSTD_isError(),3859* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :3860* the return value is a suggested next input size (just a hint for better latency)3861* that will never request more than the remaining frame size.3862* *******************************************************************************/3863/*===== ZSTD_DStream management functions =====*/3867ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);3868ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);3869/*===== Streaming decompression functions =====*/3871/* This function is redundant with the advanced API and equivalent to:3873*3874* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);3875* ZSTD_DCtx_refDDict(zds, NULL);3876*/3877ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);3878/**************************3886* Simple dictionary API3887***************************/3888/*! ZSTD_compress_usingDict() :3889* Compression at an explicit compression level using a Dictionary.3890* A dictionary can be any arbitrary data segment (also called a prefix),3891* or a buffer with specified information (see dictBuilder/zdict.h).3892* Note : This function loads the dictionary, resulting in significant startup delay.3893* It's intended for a dictionary used only once.3894* Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */3895ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,3896void* dst, size_t dstCapacity,3897const void* src, size_t srcSize,3898const void* dict,size_t dictSize,3899int compressionLevel);3900/*! ZSTD_decompress_usingDict() :3902* Decompression using a known Dictionary.3903* Dictionary must be identical to the one used during compression.3904* Note : This function loads the dictionary, resulting in significant startup delay.3905* It's intended for a dictionary used only once.3906* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */3907ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,3908void* dst, size_t dstCapacity,3909const void* src, size_t srcSize,3910const void* dict,size_t dictSize);3911/***********************************3914* Bulk processing dictionary API3915**********************************/3916typedef struct ZSTD_CDict_s ZSTD_CDict;3917/*! ZSTD_createCDict() :3919* When compressing multiple messages or blocks using the same dictionary,3920* it's recommended to digest the dictionary only once, since it's a costly operation.3921* ZSTD_createCDict() will create a state from digesting a dictionary.3922* The resulting state can be used for future compression operations with very limited startup cost.3923* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.3924* @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.3925* Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.3926* Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,3927* in which case the only thing that it transports is the @compressionLevel.3928* This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,3929* expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */3930ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,3931int compressionLevel);3932/*! ZSTD_freeCDict() :3934* Function frees memory allocated by ZSTD_createCDict(). */3935ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);3936/*! ZSTD_compress_usingCDict() :3938* Compression using a digested Dictionary.3939* Recommended when same dictionary is used multiple times.3940* Note : compression level is _decided at dictionary creation time_,3941* and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */3942ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,3943void* dst, size_t dstCapacity,3944const void* src, size_t srcSize,3945const ZSTD_CDict* cdict);3946/*! ZSTD_createDDict() :3951* Create a digested dictionary, ready to start decompression operation without startup delay.3952* dictBuffer can be released after DDict creation, as its content is copied inside DDict. */3953ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);3954/*! ZSTD_freeDDict() :3956* Function frees memory allocated with ZSTD_createDDict() */3957ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);3958/*! ZSTD_decompress_usingDDict() :3960* Decompression using a digested Dictionary.3961* Recommended when same dictionary is used multiple times. */3962ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,3963void* dst, size_t dstCapacity,3964const void* src, size_t srcSize,3965const ZSTD_DDict* ddict);3966/********************************3969* Dictionary helper functions3970*******************************/3971/*! ZSTD_getDictID_fromDict() :3973* Provides the dictID stored within dictionary.3974* if @return == 0, the dictionary is not conformant with Zstandard specification.3975* It can still be loaded, but as a content-only dictionary. */3976ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);3977/*! ZSTD_getDictID_fromDDict() :3979* Provides the dictID of the dictionary loaded into `ddict`.3980* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.3981* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */3982ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);3983/*! ZSTD_getDictID_fromFrame() :3985* Provides the dictID required to decompressed the frame stored within `src`.3986* If @return == 0, the dictID could not be decoded.3987* This could for one of the following reasons :3988* - The frame does not require a dictionary to be decoded (most common case).3989* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.3990* Note : this use case also happens when using a non-conformant dictionary.3991* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).3992* - This is not a Zstandard frame.3993* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */3994ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);3995/*******************************************************************************3998* Advanced dictionary and prefix API3999*4000* This API allows dictionaries to be used with ZSTD_compress2(),4001* ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and4002* only reset with the context is reset with ZSTD_reset_parameters or4003* ZSTD_reset_session_and_parameters. Prefixes are single-use.4004******************************************************************************/4005/*! ZSTD_CCtx_loadDictionary() :4008* Create an internal CDict from `dict` buffer.4009* Decompression will have to use same dictionary.4010* @result : 0, or an error code (which can be tested with ZSTD_isError()).4011* Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,4012* meaning "return to no-dictionary mode".4013* Note 1 : Dictionary is sticky, it will be used for all future compressed frames.4014* To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).4015* Note 2 : Loading a dictionary involves building tables.4016* It's also a CPU consuming operation, with non-negligible impact on latency.4017* Tables are dependent on compression parameters, and for this reason,4018* compression parameters can no longer be changed after loading a dictionary.4019* Note 3 :`dict` content will be copied internally.4020* Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.4021* In such a case, dictionary buffer must outlive its users.4022* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()4023* to precisely select how dictionary content must be interpreted. */4024ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);4025/*! ZSTD_CCtx_refCDict() :4027* Reference a prepared dictionary, to be used for all next compressed frames.4028* Note that compression parameters are enforced from within CDict,4029* and supersede any compression parameter previously set within CCtx.4030* The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.4031* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.4032* The dictionary will remain valid for future compressed frames using same CCtx.4033* @result : 0, or an error code (which can be tested with ZSTD_isError()).4034* Special : Referencing a NULL CDict means "return to no-dictionary mode".4035* Note 1 : Currently, only one dictionary can be managed.4036* Referencing a new dictionary effectively "discards" any previous one.4037* Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */4038ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);4039/*! ZSTD_CCtx_refPrefix() :4041* Reference a prefix (single-usage dictionary) for next compressed frame.4042* A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).4043* Decompression will need same prefix to properly regenerate data.4044* Compressing with a prefix is similar in outcome as performing a diff and compressing it,4045* but performs much faster, especially during decompression (compression speed is tunable with compression level).4046* @result : 0, or an error code (which can be tested with ZSTD_isError()).4047* Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary4048* Note 1 : Prefix buffer is referenced. It **must** outlive compression.4049* Its content must remain unmodified during compression.4050* Note 2 : If the intention is to diff some large src data blob with some prior version of itself,4051* ensure that the window size is large enough to contain the entire source.4052* See ZSTD_c_windowLog.4053* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.4054* It's a CPU consuming operation, with non-negligible impact on latency.4055* If there is a need to use the same prefix multiple times, consider loadDictionary instead.4056* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).4057* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */4058ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,4059const void* prefix, size_t prefixSize);4060/*! ZSTD_DCtx_loadDictionary() :4062* Create an internal DDict from dict buffer,4063* to be used to decompress next frames.4064* The dictionary remains valid for all future frames, until explicitly invalidated.4065* @result : 0, or an error code (which can be tested with ZSTD_isError()).4066* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,4067* meaning "return to no-dictionary mode".4068* Note 1 : Loading a dictionary involves building tables,4069* which has a non-negligible impact on CPU usage and latency.4070* It's recommended to "load once, use many times", to amortize the cost4071* Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.4072* Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.4073* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of4074* how dictionary content is loaded and interpreted.4075*/4076ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);4077/*! ZSTD_DCtx_refDDict() :4079* Reference a prepared dictionary, to be used to decompress next frames.4080* The dictionary remains active for decompression of future frames using same DCtx.4081* @result : 0, or an error code (which can be tested with ZSTD_isError()).4082* Note 1 : Currently, only one dictionary can be managed.4083* Referencing a new dictionary effectively "discards" any previous one.4084* Special: referencing a NULL DDict means "return to no-dictionary mode".4085* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.4086*/4087ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);4088/*! ZSTD_DCtx_refPrefix() :4090* Reference a prefix (single-usage dictionary) to decompress next frame.4091* This is the reverse operation of ZSTD_CCtx_refPrefix(),4092* and must use the same prefix as the one used during compression.4093* Prefix is **only used once**. Reference is discarded at end of frame.4094* End of frame is reached when ZSTD_decompressStream() returns 0.4095* @result : 0, or an error code (which can be tested with ZSTD_isError()).4096* Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary4097* Note 2 : Prefix buffer is referenced. It **must** outlive decompression.4098* Prefix buffer must remain unmodified up to the end of frame,4099* reached when ZSTD_decompressStream() returns 0.4100* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).4101* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)4102* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.4103* A full dictionary is more costly, as it requires building tables.4104*/4105ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,4106const void* prefix, size_t prefixSize);4107/* === Memory management === */4109/*! ZSTD_sizeof_*() :4111* These functions give the _current_ memory usage of selected object.4112* Note that object memory usage can evolve (increase or decrease) over time. */4113ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);4114ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);4115ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);4116ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);4117ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);4118ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);4119/* **************************************************************************************4124* ADVANCED AND EXPERIMENTAL FUNCTIONS4125****************************************************************************************4126* The definitions in the following section are considered experimental.4127* They are provided for advanced scenarios.4128* They should never be used with a dynamic library, as prototypes may change in the future.4129* Use them only in association with static linking.4130* ***************************************************************************************/4131/****************************************************************************************4136* experimental API (static linking only)4137****************************************************************************************4138* The following symbols and constants4139* are not planned to join "stable API" status in the near future.4140* They can still change in future versions.4141* Some of them are planned to remain in the static_only section indefinitely.4142* Some of them might be removed in the future (especially when redundant with existing stable functions)4143* ***************************************************************************************/4144/* compression parameter bounds */4151#define ZSTD_WINDOWLOG_MAX_32 304152#define ZSTD_WINDOWLOG_MAX_64 314153#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))4154#define ZSTD_WINDOWLOG_MIN 104155#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)4156#define ZSTD_HASHLOG_MIN 64157#define ZSTD_CHAINLOG_MAX_32 294158#define ZSTD_CHAINLOG_MAX_64 304159#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))4160#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN4161#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)4162#define ZSTD_SEARCHLOG_MIN 14163#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */4164#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */4165#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX4166#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */4167#define ZSTD_STRATEGY_MIN ZSTD_fast4168#define ZSTD_STRATEGY_MAX ZSTD_btultra24169* requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,4176* to preserve host's memory from unreasonable requirements.4177* This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).4178* The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */4179/* LDM parameter bounds */4182#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN4183#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX4184#define ZSTD_LDM_MINMATCH_MIN 44185#define ZSTD_LDM_MINMATCH_MAX 40964186#define ZSTD_LDM_BUCKETSIZELOG_MIN 14187#define ZSTD_LDM_BUCKETSIZELOG_MAX 84188#define ZSTD_LDM_HASHRATELOG_MIN 04189#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)4190/* Advanced parameter bounds */4192#define ZSTD_TARGETCBLOCKSIZE_MIN 644193#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX4194#define ZSTD_SRCSIZEHINT_MIN 04195#define ZSTD_SRCSIZEHINT_MAX INT_MAX4196/* internal */4198#define ZSTD_HASHLOG3_MAX 174199/* --- Advanced types --- */4202* If seqDef.offset < 3, then this is the corresponding repeat offset4209* But if seqDef.offset < 3 and litLength == 0, this is the4210* repeat offset before the corresponding repeat offset4211* And if seqDef.offset == 3 and litLength == 0, this is the4212* most recent repeat offset - 14213*/4214unsigned int offset;4215unsigned int litLength; /* Literal length */4216unsigned int matchLength; /* Match length */4217/* 0 when seq not rep and seqDef.offset otherwise4218* when litLength == 0 this will be <= 4, otherwise <= 3 like normal4219*/4220unsigned int rep;4221} ZSTD_Sequence;4222* Useful to save 4 bytes per generated frame.4259* Decoder cannot recognise automatically this format, requiring this instruction. */4260} ZSTD_format_e;4261* implementation details of the compressor. They are expected to continue4265* to evolve and should be considered only in the context of extremely4266* advanced performance tuning.4267*4268* Zstd currently supports the use of a CDict in three ways:4269*4270* - The contents of the CDict can be copied into the working context. This4271* means that the compression can search both the dictionary and input4272* while operating on a single set of internal tables. This makes4273* the compression faster per-byte of input. However, the initial copy of4274* the CDict's tables incurs a fixed cost at the beginning of the4275* compression. For small compressions (< 8 KB), that copy can dominate4276* the cost of the compression.4277*4278* - The CDict's tables can be used in-place. In this model, compression is4279* slower per input byte, because the compressor has to search two sets of4280* tables. However, this model incurs no start-up cost (as long as the4281* working context's tables can be reused). For small inputs, this can be4282* faster than copying the CDict's tables.4283*4284* - The CDict's tables are not used at all, and instead we use the working4285* context alone to reload the dictionary and use params based on the source4286* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().4287* This method is effective when the dictionary sizes are very small relative4288* to the input size, and the input size is fairly large to begin with.4289*4290* Zstd has a simple internal heuristic that selects which strategy to use4291* at the beginning of a compression. However, if experimentation shows that4292* Zstd is making poor choices, it is possible to override that choice with4293* this enum.4294*/4295ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */4296ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */4297ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */4298ZSTD_dictForceLoad = 3 /* Always reload the dictionary */4299} ZSTD_dictAttachPref_e;4300* Negative compression levels will be uncompressed, and positive compression4304* levels will be compressed. */4305ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be4306* emitted if Huffman compression is not profitable. */4307ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */4308} ZSTD_literalCompressionMode_e;4309/***************************************4312* Frame size functions4313***************************************/4314/*! ZSTD_findDecompressedSize() :4316* `src` should point to the start of a series of ZSTD encoded and/or skippable frames4317* `srcSize` must be the _exact_ size of this series4318* (i.e. there should be a frame boundary at `src + srcSize`)4319* @return : - decompressed size of all data in all successive frames4320* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN4321* - if an error occurred: ZSTD_CONTENTSIZE_ERROR4322*4323* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.4324* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.4325* In which case, it's necessary to use streaming mode to decompress data.4326* note 2 : decompressed size is always present when compression is done with ZSTD_compress()4327* note 3 : decompressed size can be very large (64-bits value),4328* potentially larger than what local system can handle as a single memory segment.4329* In which case, it's necessary to use streaming mode to decompress data.4330* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.4331* Always ensure result fits within application's authorized limits.4332* Each application can set its own limits.4333* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to4334* read each contained frame header. This is fast as most of the data is skipped,4335* however it does mean that all frame data must be present and valid. */4336ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);4337/*! ZSTD_decompressBound() :4339* `src` should point to the start of a series of ZSTD encoded and/or skippable frames4340* `srcSize` must be the _exact_ size of this series4341* (i.e. there should be a frame boundary at `src + srcSize`)4342* @return : - upper-bound for the decompressed size of all data in all successive frames4343* - if an error occured: ZSTD_CONTENTSIZE_ERROR4344*4345* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.4346* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.4347* in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.4348* note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:4349* upper-bound = # blocks * min(128 KB, Window_Size)4350*/4351ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);4352/*! ZSTD_frameHeaderSize() :4354* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.4355* @return : size of the Frame Header,4356* or an error code (if srcSize is too small) */4357ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);4358/*! ZSTD_getSequences() :4360* Extract sequences from the sequence store4361* zc can be used to insert custom compression params.4362* This function invokes ZSTD_compress24363* @return : number of sequences extracted4364*/4365ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,4366size_t outSeqsSize, const void* src, size_t srcSize);4367/***************************************4370* Memory management4371***************************************/4372/*! ZSTD_estimate*() :4374* These functions make it possible to estimate memory usage4375* of a future {D,C}Ctx, before its creation.4376*4377* ZSTD_estimateCCtxSize() will provide a memory budget large enough4378* for any compression level up to selected one.4379* Note : Unlike ZSTD_estimateCStreamSize*(), this estimate4380* does not include space for a window buffer.4381* Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.4382* The estimate will assume the input may be arbitrarily large,4383* which is the worst case.4384*4385* When srcSize can be bound by a known and rather "small" value,4386* this fact can be used to provide a tighter estimation4387* because the CCtx compression context will need less memory.4388* This tighter estimation can be provided by more advanced functions4389* ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),4390* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().4391* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.4392*4393* Note 2 : only single-threaded compression is supported.4394* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.4395*/4396ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);4397ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);4398ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);4399ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);4400/*! ZSTD_estimateCStreamSize() :4402* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.4403* It will also consider src size to be arbitrarily "large", which is worst case.4404* If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.4405* ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.4406* ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.4407* Note : CStream size estimation is only correct for single-threaded compression.4408* ZSTD_DStream memory budget depends on window Size.4409* This information can be passed manually, using ZSTD_estimateDStreamSize,4410* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();4411* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),4412* an internal ?Dict will be created, which additional size is not estimated here.4413* In this case, get total size by adding ZSTD_estimate?DictSize */4414ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);4415ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);4416ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);4417ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);4418ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);4419/*! ZSTD_estimate?DictSize() :4421* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().4422* ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().4423* Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.4424*/4425ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);4426ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);4427ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);4428/*! ZSTD_initStatic*() :4430* Initialize an object using a pre-allocated fixed-size buffer.4431* workspace: The memory area to emplace the object into.4432* Provided pointer *must be 8-bytes aligned*.4433* Buffer must outlive object.4434* workspaceSize: Use ZSTD_estimate*Size() to determine4435* how large workspace must be to support target scenario.4436* @return : pointer to object (same address as workspace, just different type),4437* or NULL if error (size too small, incorrect alignment, etc.)4438* Note : zstd will never resize nor malloc() when using a static buffer.4439* If the object requires more memory than available,4440* zstd will just error out (typically ZSTD_error_memory_allocation).4441* Note 2 : there is no corresponding "free" function.4442* Since workspace is allocated externally, it must be freed externally too.4443* Note 3 : cParams : use ZSTD_getCParams() to convert a compression level4444* into its associated cParams.4445* Limitation 1 : currently not compatible with internal dictionary creation, triggered by4446* ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().4447* Limitation 2 : static cctx currently not compatible with multi-threading.4448* Limitation 3 : static dctx is incompatible with legacy support.4449*/4450ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);4451ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */4452/*! Custom memory allocation :4471* These prototypes make it possible to pass your own allocation/free functions.4472* ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.4473* All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.4474*/4475typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);4476typedef void (*ZSTD_freeFunction) (void* opaque, void* address);4477typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;4478static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */4479/***************************************4499* Advanced compression functions4500***************************************/4501/*! ZSTD_createCDict_byReference() :4503* Create a digested dictionary for compression4504* Dictionary content is just referenced, not duplicated.4505* As a consequence, `dictBuffer` **must** outlive CDict,4506* and its content must remain unmodified throughout the lifetime of CDict.4507* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */4508ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);4509/*! ZSTD_getCParams() :4511* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.4512* `estimatedSrcSize` value is optional, select 0 if not known */4513ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);4514/*! ZSTD_getParams() :4516* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.4517* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */4518ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);4519/*! ZSTD_checkCParams() :4521* Ensure param values remain within authorized range.4522* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */4523ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);4524/*! ZSTD_adjustCParams() :4526* optimize params for a given `srcSize` and `dictSize`.4527* `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.4528* `dictSize` must be `0` when there is no dictionary.4529* cPar can be invalid : all parameters will be clamped within valid range in the @return struct.4530* This function never fails (wide contract) */4531ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);4532/*! ZSTD_compress_advanced() :4534* Note : this function is now DEPRECATED.4535* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.4536* This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */4537ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,4538void* dst, size_t dstCapacity,4539const void* src, size_t srcSize,4540const void* dict,size_t dictSize,4541ZSTD_parameters params);4542/*! ZSTD_compress_usingCDict_advanced() :4544* Note : this function is now REDUNDANT.4545* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.4546* This prototype will be marked as deprecated and generate compilation warning in some future version */4547ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,4548void* dst, size_t dstCapacity,4549const void* src, size_t srcSize,4550const ZSTD_CDict* cdict,4551ZSTD_frameParameters fParams);4552/*! ZSTD_CCtx_loadDictionary_byReference() :4555* Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.4556* It saves some memory, but also requires that `dict` outlives its usage within `cctx` */4557ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);4558/*! ZSTD_CCtx_loadDictionary_advanced() :4560* Same as ZSTD_CCtx_loadDictionary(), but gives finer control over4561* how to load the dictionary (by copy ? by reference ?)4562* and how to interpret it (automatic ? force raw mode ? full mode only ?) */4563ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);4564/*! ZSTD_CCtx_refPrefix_advanced() :4566* Same as ZSTD_CCtx_refPrefix(), but gives finer control over4567* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */4568ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);4569/* === experimental parameters === */4571/* these parameters can be used with ZSTD_setParameter()4572* they are not guaranteed to remain supported in the future */4573* which makes compressed files more rsync friendly4576* by adding periodic synchronization points to the compressed data.4577* The target average block size is ZSTD_c_jobSize / 2.4578* It's possible to modify the job size to increase or decrease4579* the granularity of the synchronization point.4580* Once the jobSize is smaller than the window size,4581* it will result in compression ratio degradation.4582* NOTE 1: rsyncable mode only works when multithreading is enabled.4583* NOTE 2: rsyncable performs poorly in combination with long range mode,4584* since it will decrease the effectiveness of synchronization points,4585* though mileage may vary.4586* NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.4587* If the selected compression level is already running significantly slower,4588* the overall speed won't be significantly impacted.4589*/4590#define ZSTD_c_rsyncable ZSTD_c_experimentalParam14591/* Select a compression format.4593* The value must be of type ZSTD_format_e.4594* See ZSTD_format_e enum definition for details */4595#define ZSTD_c_format ZSTD_c_experimentalParam24596/* Force back-reference distances to remain < windowSize,4598* even when referencing into Dictionary content (default:0) */4599#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam34600/* Controls whether the contents of a CDict4602* are used in place, or copied into the working context.4603* Accepts values from the ZSTD_dictAttachPref_e enum.4604* See the comments on that enum for an explanation of the feature. */4605#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam44606/* Controls how the literals are compressed (default is auto).4608* The value must be of type ZSTD_literalCompressionMode_e.4609* See ZSTD_literalCompressionMode_t enum definition for details.4610*/4611#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam54612/* Tries to fit compressed block size to be around targetCBlockSize.4614* No target when targetCBlockSize == 0.4615* There is no guarantee on compressed block size (default:0) */4616#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam64617/* User's best guess of source size.4619* Hint is not valid when srcSizeHint == 0.4620* There is no guarantee that hint is close to actual source size,4621* but compression ratio may regress significantly if guess considerably underestimates */4622#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam74623/*! ZSTD_CCtx_getParameter() :4625* Get the requested compression parameter value, selected by enum ZSTD_cParameter,4626* and store it into int* value.4627* @return : 0, or an error code (which can be tested with ZSTD_isError()).4628*/4629ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);4630/*! ZSTD_CCtx_params :4633* Quick howto :4634* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure4635* - ZSTD_CCtxParams_setParameter() : Push parameters one by one into4636* an existing ZSTD_CCtx_params structure.4637* This is similar to4638* ZSTD_CCtx_setParameter().4639* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to4640* an existing CCtx.4641* These parameters will be applied to4642* all subsequent frames.4643* - ZSTD_compressStream2() : Do compression using the CCtx.4644* - ZSTD_freeCCtxParams() : Free the memory.4645*4646* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()4647* for static allocation of CCtx for single-threaded compression.4648*/4649ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);4650ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);4651/*! ZSTD_CCtxParams_reset() :4653* Reset params to default values.4654*/4655ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);4656/*! ZSTD_CCtxParams_init() :4658* Initializes the compression parameters of cctxParams according to4659* compression level. All other parameters are reset to their default values.4660*/4661ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);4662/*! ZSTD_CCtxParams_init_advanced() :4664* Initializes the compression and frame parameters of cctxParams according to4665* params. All other parameters are reset to their default values.4666*/4667ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);4668/*! ZSTD_CCtxParams_setParameter() :4670* Similar to ZSTD_CCtx_setParameter.4671* Set one compression parameter, selected by enum ZSTD_cParameter.4672* Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().4673* @result : 0, or an error code (which can be tested with ZSTD_isError()).4674*/4675ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);4676/*! ZSTD_CCtxParams_getParameter() :4678* Similar to ZSTD_CCtx_getParameter.4679* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.4680* @result : 0, or an error code (which can be tested with ZSTD_isError()).4681*/4682ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);4683/*! ZSTD_CCtx_setParametersUsingCCtxParams() :4685* Apply a set of ZSTD_CCtx_params to the compression context.4686* This can be done even after compression is started,4687* if nbWorkers==0, this will have no impact until a new compression is started.4688* if nbWorkers>=1, new parameters will be picked up at next job,4689* with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).4690*/4691ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(4692ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);4693/*! ZSTD_compressStream2_simpleArgs() :4695* Same as ZSTD_compressStream2(),4696* but using only integral types as arguments.4697* This variant might be helpful for binders from dynamic languages4698* which have troubles handling structures containing memory pointers.4699*/4700ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs (4701ZSTD_CCtx* cctx,4702void* dst, size_t dstCapacity, size_t* dstPos,4703const void* src, size_t srcSize, size_t* srcPos,4704ZSTD_EndDirective endOp);4705/***************************************4708* Advanced decompression functions4709***************************************/4710/*! ZSTD_isFrame() :4712* Tells if the content of `buffer` starts with a valid Frame Identifier.4713* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.4714* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.4715* Note 3 : Skippable Frame Identifiers are considered valid. */4716ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);4717/*! ZSTD_createDDict_byReference() :4719* Create a digested dictionary, ready to start decompression operation without startup delay.4720* Dictionary content is referenced, and therefore stays in dictBuffer.4721* It is important that dictBuffer outlives DDict,4722* it must remain read accessible throughout the lifetime of DDict */4723ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);4724/*! ZSTD_DCtx_loadDictionary_byReference() :4726* Same as ZSTD_DCtx_loadDictionary(),4727* but references `dict` content instead of copying it into `dctx`.4728* This saves memory if `dict` remains around.,4729* However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */4730ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);4731/*! ZSTD_DCtx_loadDictionary_advanced() :4733* Same as ZSTD_DCtx_loadDictionary(),4734* but gives direct control over4735* how to load the dictionary (by copy ? by reference ?)4736* and how to interpret it (automatic ? force raw mode ? full mode only ?). */4737ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);4738/*! ZSTD_DCtx_refPrefix_advanced() :4740* Same as ZSTD_DCtx_refPrefix(), but gives finer control over4741* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */4742ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);4743/*! ZSTD_DCtx_setMaxWindowSize() :4745* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.4746* This protects a decoder context from reserving too much memory for itself (potential attack scenario).4747* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.4748* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)4749* @return : 0, or an error code (which can be tested using ZSTD_isError()).4750*/4751ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);4752/* ZSTD_d_format4754* experimental parameter,4755* allowing selection between ZSTD_format_e input compression formats4756*/4757#define ZSTD_d_format ZSTD_d_experimentalParam14758/* ZSTD_d_stableOutBuffer4759* Experimental parameter.4760* Default is 0 == disabled. Set to 1 to enable.4761*4762* Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same4763* between calls, except for the modifications that zstd makes to pos (the4764* caller must not modify pos). This is checked by the decompressor, and4765* decompression will fail if it ever changes. Therefore the ZSTD_outBuffer4766* MUST be large enough to fit the entire decompressed frame. This will be4767* checked when the frame content size is known. The data in the ZSTD_outBuffer4768* in the range [dst, dst + pos) MUST not be modified during decompression4769* or you will get data corruption.4770*4771* When this flags is enabled zstd won't allocate an output buffer, because4772* it can write directly to the ZSTD_outBuffer, but it will still allocate4773* an input buffer large enough to fit any compressed block. This will also4774* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.4775* If you need to avoid the input buffer allocation use the buffer-less4776* streaming API.4777*4778* NOTE: So long as the ZSTD_outBuffer always points to valid memory, using4779* this flag is ALWAYS memory safe, and will never access out-of-bounds4780* memory. However, decompression WILL fail if you violate the preconditions.4781*4782* WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST4783* not be modified during decompression or you will get data corruption. This4784* is because zstd needs to reference data in the ZSTD_outBuffer to regenerate4785* matches. Normally zstd maintains its own buffer for this purpose, but passing4786* this flag tells zstd to use the user provided buffer.4787*/4788#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam24789/*! ZSTD_DCtx_setFormat() :4791* Instruct the decoder context about what kind of data to decode next.4792* This instruction is mandatory to decode data without a fully-formed header,4793* such ZSTD_f_zstd1_magicless for example.4794* @return : 0, or an error code (which can be tested using ZSTD_isError()). */4795ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);4796/*! ZSTD_decompressStream_simpleArgs() :4798* Same as ZSTD_decompressStream(),4799* but using only integral types as arguments.4800* This can be helpful for binders from dynamic languages4801* which have troubles handling structures containing memory pointers.4802*/4803ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (4804ZSTD_DCtx* dctx,4805void* dst, size_t dstCapacity, size_t* dstPos,4806const void* src, size_t srcSize, size_t* srcPos);4807/********************************************************************4810* Advanced streaming functions4811* Warning : most of these functions are now redundant with the Advanced API.4812* Once Advanced API reaches "stable" status,4813* redundant functions will be deprecated, and then at some point removed.4814********************************************************************/4815/*===== Advanced Streaming compression functions =====*/4817/**! ZSTD_initCStream_srcSize() :4818* This function is deprecated, and equivalent to:4819* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4820* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)4821* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);4822* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);4823*4824* pledgedSrcSize must be correct. If it is not known at init time, use4825* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,4826* "0" also disables frame content size field. It may be enabled in the future.4827* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4828*/4829ZSTDLIB_API size_t4830ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,4831int compressionLevel,4832unsigned long long pledgedSrcSize);4833/**! ZSTD_initCStream_usingDict() :4835* This function is deprecated, and is equivalent to:4836* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4837* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);4838* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);4839*4840* Creates of an internal CDict (incompatible with static CCtx), except if4841* dict == NULL or dictSize < 8, in which case no dict is used.4842* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if4843* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.4844* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4845*/4846ZSTDLIB_API size_t4847ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,4848const void* dict, size_t dictSize,4849int compressionLevel);4850/**! ZSTD_initCStream_advanced() :4852* This function is deprecated, and is approximately equivalent to:4853* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4854* // Pseudocode: Set each zstd parameter and leave the rest as-is.4855* for ((param, value) : params) {4856* ZSTD_CCtx_setParameter(zcs, param, value);4857* }4858* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);4859* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);4860*4861* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.4862* pledgedSrcSize must be correct.4863* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.4864* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4865*/4866ZSTDLIB_API size_t4867ZSTD_initCStream_advanced(ZSTD_CStream* zcs,4868const void* dict, size_t dictSize,4869ZSTD_parameters params,4870unsigned long long pledgedSrcSize);4871/**! ZSTD_initCStream_usingCDict() :4873* This function is deprecated, and equivalent to:4874* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4875* ZSTD_CCtx_refCDict(zcs, cdict);4876*4877* note : cdict will just be referenced, and must outlive compression session4878* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4879*/4880ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);4881/**! ZSTD_initCStream_usingCDict_advanced() :4883* This function is DEPRECATED, and is approximately equivalent to:4884* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4885* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.4886* for ((fParam, value) : fParams) {4887* ZSTD_CCtx_setParameter(zcs, fParam, value);4888* }4889* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);4890* ZSTD_CCtx_refCDict(zcs, cdict);4891*4892* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.4893* pledgedSrcSize must be correct. If srcSize is not known at init time, use4894* value ZSTD_CONTENTSIZE_UNKNOWN.4895* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4896*/4897ZSTDLIB_API size_t4898ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,4899const ZSTD_CDict* cdict,4900ZSTD_frameParameters fParams,4901unsigned long long pledgedSrcSize);4902/*! ZSTD_resetCStream() :4904* This function is deprecated, and is equivalent to:4905* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);4906* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);4907*4908* start a new frame, using same parameters from previous frame.4909* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.4910* Note that zcs must be init at least once before using ZSTD_resetCStream().4911* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.4912* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.4913* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,4914* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.4915* @return : 0, or an error code (which can be tested using ZSTD_isError())4916* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4917*/4918ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);4919/* ZSTD_getFrameProgression() :4931* tells how much data has been ingested (read from input)4932* consumed (input actually compressed) and produced (output) for current frame.4933* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.4934* Aggregates progression inside active worker threads.4935*/4936ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);4937/*! ZSTD_toFlushNow() :4939* Tell how many bytes are ready to be flushed immediately.4940* Useful for multithreading scenarios (nbWorkers >= 1).4941* Probe the oldest active job, defined as oldest job not yet entirely flushed,4942* and check its output buffer.4943* @return : amount of data stored in oldest job and ready to be flushed immediately.4944* if @return == 0, it means either :4945* + there is no active job (could be checked with ZSTD_frameProgression()), or4946* + oldest job is still actively compressing data,4947* but everything it has produced has also been flushed so far,4948* therefore flush speed is limited by production speed of oldest job4949* irrespective of the speed of concurrent (and newer) jobs.4950*/4951ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);4952/*===== Advanced Streaming decompression functions =====*/4955/**4956* This function is deprecated, and is equivalent to:4957*4958* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);4959* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);4960*4961* note: no dictionary will be used if dict == NULL or dictSize < 84962* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4963*/4964ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);4965/**4967* This function is deprecated, and is equivalent to:4968*4969* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);4970* ZSTD_DCtx_refDDict(zds, ddict);4971*4972* note : ddict is referenced, it must outlive decompression session4973* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4974*/4975ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);4976/**4978* This function is deprecated, and is equivalent to:4979*4980* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);4981*4982* re-use decompression parameters from previous init; saves dictionary loading4983* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x4984*/4985ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);4986/*********************************************************************4989* Buffer-less and synchronous inner streaming functions4990*4991* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.4992* But it's also a complex one, with several restrictions, documented below.4993* Prefer normal streaming API for an easier experience.4994********************************************************************* */4995/**4997Buffer-less streaming compression (synchronous mode)4998A ZSTD_CCtx object is required to track streaming operations.5000Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.5001ZSTD_CCtx object can be re-used multiple times within successive compression operations.5002Start by initializing a context.5004Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,5005or ZSTD_compressBegin_advanced(), for finer parameter control.5006It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()5007Then, consume your input using ZSTD_compressContinue().5009There are some important considerations to keep in mind when using this advanced function :5010- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.5011- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.5012- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.5013Worst case evaluation is provided by ZSTD_compressBound().5014ZSTD_compressContinue() doesn't guarantee recover after a failed compression.5015- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).5016It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)5017- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.5018In which case, it will "discard" the relevant memory section from its history.5019Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.5021It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.5022Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.5023`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.5025*/5026/*===== Buffer-less streaming compression functions =====*/5028ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);5029ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);5030ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */5031ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */5032ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */5033ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */5034/*-5040Buffer-less streaming decompression (synchronous mode)5041A ZSTD_DCtx object is required to track streaming operations.5043Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.5044A ZSTD_DCtx object can be re-used multiple times.5045First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().5047Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.5048Data fragment must be large enough to ensure successful decoding.5049`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.5050@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.5051>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.5052errorCode, which can be tested using ZSTD_isError().5053It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,5055such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).5056Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.5057As a consequence, check that values remain within valid application range.5058For example, do not allocate memory blindly, check that `windowSize` is within expectation.5059Each application can set its own limits, depending on local restrictions.5060For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.5061ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.5063ZSTD_decompressContinue() is very sensitive to contiguity,5064if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,5065or that previous contiguous segment is large enough to properly handle maximum back-reference distance.5066There are multiple ways to guarantee this condition.5067The most memory efficient way is to use a round buffer of sufficient size.5069Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),5070which can @return an error code if required value is too large for current system (in 32-bits mode).5071In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,5072up to the moment there is not enough room left in the buffer to guarantee decoding another full block,5073which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.5074At which point, decoding can resume from the beginning of the buffer.5075Note that already decoded data stored in the buffer should be flushed before being overwritten.5076There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.5078Finally, if you control the compression process, you can also ignore all buffer size rules,5080as long as the encoder and decoder progress in "lock-step",5081aka use exactly the same buffer sizes, break contiguity at the same place, etc.5082Once buffers are setup, start decompression, with ZSTD_decompressBegin().5084If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().5085Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.5087ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().5088ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.5089@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).5091It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.5092It can also be an error code, which can be tested with ZSTD_isError().5093A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.5095Context can then be reset to start a new decompression.5096Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().5098This information is not required to properly decode a frame.5099== Special case : skippable frames ==5101Skippable frames allow integration of user-defined data into a flow of concatenated frames.5103Skippable frames will be ignored (skipped) by decompressor.5104The format of skippable frames is as follows :5105a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F5106b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits5107c) Frame Content - any content (User Data) of length equal to Frame Size5108For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.5109For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.5110*/5111/*===== Buffer-less streaming decompression functions =====*/5113typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;5114typedef struct {5115unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */5116unsigned long long windowSize; /* can be very large, up to <= frameContentSize */5117unsigned blockSizeMax;5118ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */5119unsigned headerSize;5120unsigned dictID;5121unsigned checksumFlag;5122} ZSTD_frameHeader;5123/*! ZSTD_getFrameHeader() :5125* decode Frame Header, or requires larger `srcSize`.5126* @return : 0, `zfhPtr` is correctly filled,5127* >0, `srcSize` is too small, value is wanted `srcSize` amount,5128* or an error code, which can be tested using ZSTD_isError() */5129ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */5130/*! ZSTD_getFrameHeader_advanced() :5131* same as ZSTD_getFrameHeader(),5132* with added capability to select a format (like ZSTD_f_zstd1_magicless) */5133ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);5134ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */5135/* misc */5144ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);5145typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;5146ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);5147/* ============================ */5152/** Block level API */5153/* ============================ */5154/*!5156Block functions produce and decode raw zstd blocks, without frame metadata.5157Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).5158But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.5159A few rules to respect :5161- Compressing and decompressing require a context structure5162+ Use ZSTD_createCCtx() and ZSTD_createDCtx()5163- It is necessary to init context before starting5164+ compression : any ZSTD_compressBegin*() variant, including with dictionary5165+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary5166+ copyCCtx() and copyDCtx() can be used too5167- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB5168+ If input is larger than a block size, it's necessary to split input data into multiple blocks5169+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.5170Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.5171- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !5172===> In which case, nothing is produced into `dst` !5173+ User __must__ test for such outcome and deal directly with uncompressed data5174+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.5175Doing so would mess up with statistics history, leading to potential data corruption.5176+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!5177+ In case of multiple successive blocks, should some of them be uncompressed,5178decoder must be informed of their existence in order to follow proper history.5179Use ZSTD_insertBlock() for such a case.5180*/5181/*===== Raw zstd block functions =====*/5183ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);5184ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);5185ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);5186ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */5187}5193#endif5194/**** ended inlining ../zstd.h ****/5195#define FSE_STATIC_LINKING_ONLY5196/**** skipping file: fse.h ****/5197#define HUF_STATIC_LINKING_ONLY5198/**** skipping file: huf.h ****/5199#ifndef XXH_STATIC_LINKING_ONLY5200# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */5201#endif5202/**** start inlining xxhash.h ****/5203/*5204* xxHash - Extremely Fast Hash algorithm5205* Header File5206* Copyright (c) 2012-2020, Yann Collet, Facebook, Inc.5207*5208* You can contact the author at :5209* - xxHash source repository : https://github.com/Cyan4973/xxHash5210*5211* This source code is licensed under both the BSD-style license (found in the5212* LICENSE file in the root directory of this source tree) and the GPLv2 (found5213* in the COPYING file in the root directory of this source tree).5214* You may select, at your option, one of the above-listed licenses.5215*/5216/* Notice extracted from xxHash homepage :5218xxHash is an extremely fast Hash algorithm, running at RAM speed limits.5220It also successfully passes all tests from the SMHasher suite.5221Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)5223Name Speed Q.Score Author5225xxHash 5.4 GB/s 105226CrapWow 3.2 GB/s 2 Andrew5227MumurHash 3a 2.7 GB/s 10 Austin Appleby5228SpookyHash 2.0 GB/s 10 Bob Jenkins5229SBox 1.4 GB/s 9 Bret Mulvey5230Lookup3 1.2 GB/s 9 Bob Jenkins5231SuperFastHash 1.2 GB/s 1 Paul Hsieh5232CityHash64 1.05 GB/s 10 Pike & Alakuijala5233FNV 0.55 GB/s 5 Fowler, Noll, Vo5234CRC32 0.43 GB/s 95235MD5-32 0.33 GB/s 10 Ronald L. Rivest5236SHA1-32 0.28 GB/s 105237Q.Score is a measure of quality of the hash function.5239It depends on successfully passing SMHasher test set.524010 is a perfect score.5241A 64-bits version, named XXH64, is available since r35.5243It offers much better speed, but for 64-bits applications only.5244Name Speed on 64 bits Speed on 32 bits5245XXH64 13.8 GB/s 1.9 GB/s5246XXH32 6.8 GB/s 6.0 GB/s5247*/5248/* ****************************5258* Definitions5259******************************/5260#include <stddef.h> /* size_t */5261typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;5262/* ****************************5265* API modifier5266******************************/5267/** XXH_PRIVATE_API5268* This is useful if you want to include xxhash functions in `static` mode5269* in order to inline them, and remove their symbol from the public list.5270* Methodology :5271* #define XXH_PRIVATE_API5272* #include "xxhash.h"5273* `xxhash.c` is automatically included.5274* It's not useful to compile and link it as a separate module anymore.5275*/5276#ifdef XXH_PRIVATE_API5277# ifndef XXH_STATIC_LINKING_ONLY5278# define XXH_STATIC_LINKING_ONLY5279# endif5280# if defined(__GNUC__)5281# define XXH_PUBLIC_API static __inline __attribute__((unused))5282# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)5283# define XXH_PUBLIC_API static inline5284# elif defined(_MSC_VER)5285# define XXH_PUBLIC_API static __inline5286# else5287# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */5288# endif5289#else5290# define XXH_PUBLIC_API /* do nothing */5291#endif /* XXH_PRIVATE_API */5292/*!XXH_NAMESPACE, aka Namespace Emulation :5294If you want to include _and expose_ xxHash functions from within your own library,5296but also want to avoid symbol collisions with another library which also includes xxHash,5297you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library5299with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).5300Note that no change is required within the calling program as long as it includes `xxhash.h` :5302regular symbol name will be automatically translated by this header.5303*/5304#ifdef XXH_NAMESPACE5305# define XXH_CAT(A,B) A##B5306# define XXH_NAME2(A,B) XXH_CAT(A,B)5307# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)5308# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)5309# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)5310# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)5311# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)5312# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)5313# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)5314# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)5315# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)5316# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)5317# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)5318# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)5319# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)5320# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)5321# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)5322# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)5323# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)5324# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)5325# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)5326#endif5327/* *************************************5330* Version5331***************************************/5332#define XXH_VERSION_MAJOR 05333#define XXH_VERSION_MINOR 65334#define XXH_VERSION_RELEASE 25335#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)5336XXH_PUBLIC_API unsigned XXH_versionNumber (void);5337/* ****************************5340* Simple Hash Functions5341******************************/5342typedef unsigned int XXH32_hash_t;5343typedef unsigned long long XXH64_hash_t;5344/*!5349XXH32() :5350Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".5351The memory between input & input+length must be valid (allocated and read-accessible).5352"seed" can be used to alter the result predictably.5353Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s5354XXH64() :5355Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".5356"seed" can be used to alter the result predictably.5357This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).5358*/5359/* ****************************5362* Streaming Hash Functions5363******************************/5364typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */5365typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */5366/*! State allocation, compatible with dynamic libraries */5368/* hash streaming */5377/*5387These functions generate the xxHash of an input provided in multiple segments.5388Note that, for small input, they are slower than single-call functions, due to state management.5389For small input, prefer `XXH32()` and `XXH64()` .5390XXH state must first be allocated, using XXH*_createState() .5392Start a new hash by initializing state with a seed, using XXH*_reset().5394Then, feed the hash state by calling XXH*_update() as many times as necessary.5396Obviously, input must be allocated and read accessible.5397The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.5398Finally, a hash value can be produced anytime, by using XXH*_digest().5400This function returns the nn-bits hash as an int or long long.5401It's still possible to continue inserting input into the hash state after a digest,5403and generate some new hashes later on, by calling again XXH*_digest().5404When done, free XXH state space if it was allocated dynamically.5406*/5407/* **************************5410* Utils5411****************************/5412#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */5413# define restrict /* disable restrict */5414#endif5415/* **************************5421* Canonical representation5422****************************/5423/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.5424* The canonical representation uses human-readable write convention, aka big-endian (large digits first).5425* These functions allow transformation of hash result into and from its canonical format.5426* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.5427*/5428typedef struct { unsigned char digest[4]; } XXH32_canonical_t;5429typedef struct { unsigned char digest[8]; } XXH64_canonical_t;5430/* ================================================================================================5442This section contains definitions which are not guaranteed to remain stable.5443They may change in future versions, becoming incompatible with a different version of the library.5444They shall only be used with static linking.5445Never use these definitions in association with dynamic linking !5446=================================================================================================== */5447#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)5448#define XXH_STATIC_H_35436876873455449/* These definitions are only meant to allow allocation of XXH state5451statically, on stack, or in a struct for example.5452Do not use members directly. */5453/**** start inlining xxhash.c ****/5480/*5481* xxHash - Fast Hash algorithm5482* Copyright (c) 2012-2020, Yann Collet, Facebook, Inc.5483*5484* You can contact the author at :5485* - xxHash homepage: http://www.xxhash.com5486* - xxHash source repository : https://github.com/Cyan4973/xxHash5487*5488* This source code is licensed under both the BSD-style license (found in the5489* LICENSE file in the root directory of this source tree) and the GPLv2 (found5490* in the COPYING file in the root directory of this source tree).5491* You may select, at your option, one of the above-listed licenses.5492*/5493/* *************************************5496* Tuning parameters5497***************************************/5498/*!XXH_FORCE_MEMORY_ACCESS :5499* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.5500* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.5501* The below switch allow to select different access method for improved performance.5502* Method 0 (default) : use `memcpy()`. Safe and portable.5503* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).5504* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.5505* Method 2 : direct access. This method doesn't depend on compiler but violate C standard.5506* It can generate buggy code on targets which do not support unaligned memory accesses.5507* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)5508* See http://stackoverflow.com/a/32095106/646947 for details.5509* Prefer these methods in priority order (0 > 1 > 2)5510*/5511#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */5512# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )5513# define XXH_FORCE_MEMORY_ACCESS 25514# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \5515(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \5516defined(__ICCARM__)5517# define XXH_FORCE_MEMORY_ACCESS 15518# endif5519#endif5520/*!XXH_ACCEPT_NULL_INPUT_POINTER :5522* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.5523* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.5524* By default, this option is disabled. To enable it, uncomment below define :5525*/5526/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */5527/*!XXH_FORCE_NATIVE_FORMAT :5529* By default, xxHash library provides endian-independent Hash values, based on little-endian convention.5530* Results are therefore identical for little-endian and big-endian CPU.5531* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.5532* Should endian-independence be of no importance for your application, you may set the #define below to 1,5533* to improve speed for Big-endian CPU.5534* This option has no impact on Little_Endian CPU.5535*/5536#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */5537# define XXH_FORCE_NATIVE_FORMAT 05538#endif5539/*!XXH_FORCE_ALIGN_CHECK :5541* This is a minor performance trick, only useful with lots of very small keys.5542* It means : check for aligned/unaligned input.5543* The check costs one initial branch per hash; set to 0 when the input data5544* is guaranteed to be aligned.5545*/5546#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */5547# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)5548# define XXH_FORCE_ALIGN_CHECK 05549# else5550# define XXH_FORCE_ALIGN_CHECK 15551# endif5552#endif5553/* *************************************5556* Includes & Memory related functions5557***************************************/5558/* Modify the local functions below should you wish to use some other memory routines */5559/* for malloc(), free() */5560#include <stddef.h> /* size_t */5561static void* XXH_malloc(size_t s) { return malloc(s); }5562static void XXH_free (void* p) { free(p); }5563/* for memcpy() */5564static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }5565/**** skipping file: xxhash.h ****/5570/* *************************************5573* Compiler Specific Options5574***************************************/5575#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */5576# define INLINE_KEYWORD inline5577#else5578# define INLINE_KEYWORD5579#endif5580/* *************************************5598* Basic Types5599***************************************/5600#ifndef MEM_MODULE5601# define MEM_MODULE5602# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )5603# include <stdint.h>5604typedef uint8_t BYTE;5605typedef uint16_t U16;5606typedef uint32_t U32;5607typedef int32_t S32;5608typedef uint64_t U64;5609# else5610typedef unsigned char BYTE;5611typedef unsigned short U16;5612typedef unsigned int U32;5613typedef signed int S32;5614typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */5615# endif5616#endif5617/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */5622static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }5623static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }5624/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */5628/* currently only defined for gcc and icc */5629typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign;5630/* portable and safe solution. Generally efficient.5637* see : http://stackoverflow.com/a/32095106/6469475638*/5639{5642U32 val;5643memcpy(&val, memPtr, sizeof(val));5644return val;5645}5646{5649U64 val;5650memcpy(&val, memPtr, sizeof(val));5651return val;5652}5653/* ****************************************5658* Compiler-specific Functions and Macros5659******************************************/5660#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)5661/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */5663#if defined(_MSC_VER)5664# define XXH_rotl32(x,r) _rotl(x,r)5665# define XXH_rotl64(x,r) _rotl64(x,r)5666#else5667#if defined(__ICCARM__)5668# include <intrinsics.h>5669# define XXH_rotl32(x,r) __ROR(x,(32 - r))5670#else5671# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))5672#endif5673# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))5674#endif5675{5685return ((x << 24) & 0xff000000 ) |5686((x << 8) & 0x00ff0000 ) |5687((x >> 8) & 0x0000ff00 ) |5688((x >> 24) & 0x000000ff );5689}5690static U64 XXH_swap64 (U64 x)5691{5692return ((x << 56) & 0xff00000000000000ULL) |5693((x << 40) & 0x00ff000000000000ULL) |5694((x << 24) & 0x0000ff0000000000ULL) |5695((x << 8) & 0x000000ff00000000ULL) |5696((x >> 8) & 0x00000000ff000000ULL) |5697((x >> 24) & 0x0000000000ff0000ULL) |5698((x >> 40) & 0x000000000000ff00ULL) |5699((x >> 56) & 0x00000000000000ffULL);5700}5701#endif5702/* *************************************5705* Architecture Macros5706***************************************/5707typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;5708/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */5710#ifndef XXH_CPU_LITTLE_ENDIAN5711static const int g_one = 1;5712# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))5713#endif5714/* ***************************5717* Memory reads5718*****************************/5719typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;5720{5723if (align==XXH_unaligned)5724return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));5725else5726return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);5727}5728{5731return XXH_readLE32_align(ptr, endian, XXH_unaligned);5732}5733{5736return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);5737}5738{5741if (align==XXH_unaligned)5742return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));5743else5744return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);5745}5746{5749return XXH_readLE64_align(ptr, endian, XXH_unaligned);5750}5751{5754return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);5755}5756/* *************************************5759* Macros5760***************************************/5761#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */5762/* *************************************5765* Constants5766***************************************/5767static const U32 PRIME32_1 = 2654435761U;5768static const U32 PRIME32_2 = 2246822519U;5769static const U32 PRIME32_3 = 3266489917U;5770static const U32 PRIME32_4 = 668265263U;5771static const U32 PRIME32_5 = 374761393U;5772/* **************************5783* Utils5784****************************/5785XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)5786{5787memcpy(dstState, srcState, sizeof(*dstState));5788}5789{5792memcpy(dstState, srcState, sizeof(*dstState));5793}5794/* ***************************5797* Simple Hash Functions5798*****************************/5799{5802seed += input * PRIME32_2;5803seed = XXH_rotl32(seed, 13);5804seed *= PRIME32_1;5805return seed;5806}5807{5810const BYTE* p = (const BYTE*)input;5811const BYTE* bEnd = p + len;5812U32 h32;5813#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)5814}5863{5867#if 05868/* Simple version, good for code maintenance, but unfortunately slow for small inputs */5869XXH32_CREATESTATE_STATIC(state);5870XXH32_reset(state, seed);5871XXH32_update(state, input, len);5872return XXH32_digest(state);5873#else5874XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;5875}5890{5894acc += input * PRIME64_2;5895acc = XXH_rotl64(acc, 31);5896acc *= PRIME64_1;5897return acc;5898}5899{5902val = XXH64_round(0, val);5903acc ^= val;5904acc = acc * PRIME64_1 + PRIME64_4;5905return acc;5906}5907{5910const BYTE* p = (const BYTE*)input;5911const BYTE* const bEnd = p + len;5912U64 h64;5913#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)5914}5975{5979#if 05980/* Simple version, good for code maintenance, but unfortunately slow for small inputs */5981XXH64_CREATESTATE_STATIC(state);5982XXH64_reset(state, seed);5983XXH64_update(state, input, len);5984return XXH64_digest(state);5985#else5986XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;5987}6002/* **************************************************6005* Advanced Hash Functions6006****************************************************/6007{6010return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));6011}6012XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)6013{6014XXH_free(statePtr);6015return XXH_OK;6016}6017{6020return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));6021}6022XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)6023{6024XXH_free(statePtr);6025return XXH_OK;6026}6027/*** Hash feed ***/6030{6033XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */6034memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */6035state.v1 = seed + PRIME32_1 + PRIME32_2;6036state.v2 = seed + PRIME32_2;6037state.v3 = seed + 0;6038state.v4 = seed - PRIME32_1;6039memcpy(statePtr, &state, sizeof(state));6040return XXH_OK;6041}6042{6046XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */6047memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */6048state.v1 = seed + PRIME64_1 + PRIME64_2;6049state.v2 = seed + PRIME64_2;6050state.v3 = seed + 0;6051state.v4 = seed - PRIME64_1;6052memcpy(statePtr, &state, sizeof(state));6053return XXH_OK;6054}6055{6059const BYTE* p = (const BYTE*)input;6060const BYTE* const bEnd = p + len;6061}6114{6117XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;6118}6124{6129const BYTE * p = (const BYTE*)state->mem32;6130const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;6131U32 h32;6132}6161{6165XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;6166}6172/* **** XXH64 **** */6176{6179const BYTE* p = (const BYTE*)input;6180const BYTE* const bEnd = p + len;6181}6233{6236XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;6237}6243{6248const BYTE * p = (const BYTE*)state->mem64;6249const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;6250U64 h64;6251}6296{6300XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;6301}6307/* **************************6310* Canonical representation6311****************************/6312/*! Default XXH result types are basic unsigned 32 and 64 bits.6314* The canonical representation follows human-readable write convention, aka big-endian (large digits first).6315* These functions allow transformation of hash result into and from its canonical format.6316* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.6317*/6318{6321XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));6322if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);6323memcpy(dst, &hash, sizeof(*dst));6324}6325{6328XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));6329if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);6330memcpy(dst, &hash, sizeof(*dst));6331}6332{6335return XXH_readBE32(src);6336}6337{6340return XXH_readBE64(src);6341}6342/**** ended inlining xxhash.c ****/6343# endif6344}6350#endif6351/**** ended inlining xxhash.h ****/6352/* ---- static assert (debug) --- */6358#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)6359#define ZSTD_isError ERR_isError /* for inlining */6360#define FSE_isError ERR_isError6361#define HUF_isError ERR_isError6362/*-*************************************6365* shared macros6366***************************************/6367#undef MIN6368#undef MAX6369#define MIN(a,b) ((a)<(b) ? (a) : (b))6370#define MAX(a,b) ((a)>(b) ? (a) : (b))6371/**6373* Ignore: this is an internal helper.6374*6375* This is a helper function to help force C99-correctness during compilation.6376* Under strict compilation modes, variadic macro arguments can't be empty.6377* However, variadic function arguments can be. Using a function therefore lets6378* us statically check that at least one (string) argument was passed,6379* independent of the compilation flags.6380*/6381static INLINE_KEYWORD UNUSED_ATTR6382void _force_has_format_string(const char *format, ...) {6383(void)format;6384}6385/**6387* Ignore: this is an internal helper.6388*6389* We want to force this function invocation to be syntactically correct, but6390* we don't want to force runtime evaluation of its arguments.6391*/6392#define _FORCE_HAS_FORMAT_STRING(...) \6393if (0) { \6394_force_has_format_string(__VA_ARGS__); \6395}6396/**6398* Return the specified error if the condition evaluates to true.6399*6400* In debug modes, prints additional information.6401* In order to do that (particularly, printing the conditional that failed),6402* this can't just wrap RETURN_ERROR().6403*/6404#define RETURN_ERROR_IF(cond, err, ...) \6405if (cond) { \6406RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \6407__FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \6408_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \6409RAWLOG(3, ": " __VA_ARGS__); \6410RAWLOG(3, "\n"); \6411return ERROR(err); \6412}6413/**6415* Unconditionally return the specified error.6416*6417* In debug modes, prints additional information.6418*/6419#define RETURN_ERROR(err, ...) \6420do { \6421RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \6422__FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \6423_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \6424RAWLOG(3, ": " __VA_ARGS__); \6425RAWLOG(3, "\n"); \6426return ERROR(err); \6427} while(0);6428/**6430* If the provided expression evaluates to an error code, returns that error code.6431*6432* In debug modes, prints additional information.6433*/6434#define FORWARD_IF_ERROR(err, ...) \6435do { \6436size_t const err_code = (err); \6437if (ERR_isError(err_code)) { \6438RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \6439__FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \6440_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \6441RAWLOG(3, ": " __VA_ARGS__); \6442RAWLOG(3, "\n"); \6443return err_code; \6444} \6445} while(0);6446/*-*************************************6449* Common constants6450***************************************/6451#define ZSTD_OPT_NUM (1<<12)6452/*-*******************************************6541* Shared functions to include for inlining6542*********************************************/6543static void ZSTD_copy8(void* dst, const void* src) {6544#ifdef __aarch64__6545vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));6546#else6547memcpy(dst, src, 8);6548#endif6549}6550}6559#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }6560ZSTD_overlap_src_before_dst6567/* ZSTD_overlap_dst_before_src, */6568} ZSTD_overlap_e;6569/*! ZSTD_wildcopy() :6571* Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)6572* @param ovtype controls the overlap detection6573* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.6574* - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.6575* The src buffer must be before the dst buffer.6576*/6577MEM_STATIC FORCE_INLINE_ATTR6578void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)6579{6580ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;6581const BYTE* ip = (const BYTE*)src;6582BYTE* op = (BYTE*)dst;6583BYTE* const oend = op + length;6584* almost certain to be short, so the branches have different6596* probabilities. Since it is almost certain to be short, only do6597* one COPY16() in the first call. Then, do two calls per loop since6598* at that point it is more likely to have a high trip count.6599*/6600#ifndef __aarch64__6601do {6602COPY16(op, ip);6603}6604while (op < oend);6605#else6606COPY16(op, ip);6607if (op >= oend) return;6608do {6609COPY16(op, ip);6610COPY16(op, ip);6611}6612while (op < oend);6613#endif6614}6615}6616{6619size_t const length = MIN(dstCapacity, srcSize);6620if (length > 0) {6621memcpy(dst, src, length);6622}6623return length;6624}6625/* define "workspace is too large" as this number of times larger than needed */6627#define ZSTD_WORKSPACETOOLARGE_FACTOR 36628/* when workspace is continuously too large6630* during at least this number of times,6631* context's memory usage is considered wasteful,6632* because it's sized to handle a worst case scenario which rarely happens.6633* In which case, resize it down to free some memory */6634#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 1286635/*-*******************************************6638* Private declarations6639*********************************************/6640typedef struct seqDef_s {6641U32 offset;6642U16 litLength;6643U16 matchLength;6644} seqDef;6645/**6666* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences6667* indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength.6668*/6669MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)6670{6671ZSTD_sequenceLength seqLen;6672seqLen.litLength = seq->litLength;6673seqLen.matchLength = seq->matchLength + MINMATCH;6674if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {6675if (seqStore->longLengthID == 1) {6676seqLen.litLength += 0xFFFF;6677}6678if (seqStore->longLengthID == 2) {6679seqLen.matchLength += 0xFFFF;6680}6681}6682return seqLen;6683}6684/**6686* Contains the compressed frame size and an upper-bound for the decompressed frame size.6687* Note: before using `compressedSize`, check for errors using ZSTD_isError().6688* similarly, before using `decompressedBound`, check for errors using:6689* `decompressedBound != ZSTD_CONTENTSIZE_ERROR`6690*/6691typedef struct {6692size_t compressedSize;6693unsigned long long decompressedBound;6694} ZSTD_frameSizeInfo; /* decompress & legacy */6695/* custom memory allocation functions */6700void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);6701void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);6702void ZSTD_free(void* ptr, ZSTD_customMem customMem);6703{6707assert(val != 0);6708{6709# if defined(_MSC_VER) /* Visual */6710unsigned long r=0;6711return _BitScanReverse(&r, val) ? (unsigned)r : 0;6712# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */6713return __builtin_clz (val) ^ 31;6714# elif defined(__ICCARM__) /* IAR Intrinsic */6715return 31 - __CLZ(val);6716# else /* Software version */6717static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };6718U32 v = val;6719v |= v >> 1;6720v |= v >> 2;6721v |= v >> 4;6722v |= v >> 8;6723v |= v >> 16;6724return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];6725# endif6726}6727}6728/* ZSTD_invalidateRepCodes() :6731* ensures next compression will not use repcodes from previous block.6732* Note : only works with regular variant;6733* do not use with extDict variant ! */6734void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */6735/*! ZSTD_getcBlockSize() :6744* Provides the size of compressed block from block header `src` */6745/* Used by: decompress, fullbench (does not get its definition from here) */6746size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,6747blockProperties_t* bpPtr);6748/*! ZSTD_decodeSeqHeaders() :6750* decode sequence header from src */6751/* Used by: decompress, fullbench (does not get its definition from here) */6752size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,6753const void* src, size_t srcSize);6754}6758#endif6759/**** ended inlining zstd_internal.h ****/6762/*-****************************************6765* Version6766******************************************/6767unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }6768/*-****************************************6773* ZSTD Error Management6774******************************************/6775#undef ZSTD_isError /* defined within zstd_internal.h */6776/*! ZSTD_isError() :6777* tells if a return value is an error code6778* symbol is required for external callers */6779unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }6780/*! ZSTD_getErrorName() :6782* provides error code string from function result (useful for debugging) */6783const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }6784/*! ZSTD_getError() :6786* convert a `size_t` function result into a proper ZSTD_errorCode enum */6787ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }6788/*! ZSTD_getErrorString() :6790* provides error code string from enum */6791const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }6792/*=**************************************************************6796* Custom allocator6797****************************************************************/6798void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)6799{6800if (customMem.customAlloc)6801return customMem.customAlloc(customMem.opaque, size);6802return malloc(size);6803}6804{6807if (customMem.customAlloc) {6808/* calloc implemented as malloc+memset;6809* not as efficient as calloc, but next best guess for custom malloc */6810void* const ptr = customMem.customAlloc(customMem.opaque, size);6811memset(ptr, 0, size);6812return ptr;6813}6814return calloc(1, size);6815}6816{6819if (ptr!=NULL) {6820if (customMem.customFree)6821customMem.customFree(customMem.opaque, ptr);6822else6823free(ptr);6824}6825}6826/**** ended inlining common/zstd_common.c ****/6827/**** start inlining decompress/huf_decompress.c ****/6829/* ******************************************************************6830* huff0 huffman decoder,6831* part of Finite State Entropy library6832* Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.6833*6834* You can contact the author at :6835* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy6836*6837* This source code is licensed under both the BSD-style license (found in the6838* LICENSE file in the root directory of this source tree) and the GPLv2 (found6839* in the COPYING file in the root directory of this source tree).6840* You may select, at your option, one of the above-listed licenses.6841****************************************************************** */6842/* **************************************************************6844* Dependencies6845****************************************************************/6846/**** skipping file: ../common/compiler.h ****/6847/**** skipping file: ../common/bitstream.h ****/6848/**** skipping file: ../common/fse.h ****/6849#define HUF_STATIC_LINKING_ONLY6850/**** skipping file: ../common/huf.h ****/6851/**** skipping file: ../common/error_private.h ****/6852/* **************************************************************6854* Macros6855****************************************************************/6856/* These two optional macros force the use one way or another of the two6858* Huffman decompression implementations. You can't force in both directions6859* at the same time.6860*/6861#if defined(HUF_FORCE_DECOMPRESS_X1) && \6862defined(HUF_FORCE_DECOMPRESS_X2)6863#error "Cannot force the use of the X1 and X2 decoders at the same time!"6864#endif6865/* **************************************************************6868* Error Management6869****************************************************************/6870#define HUF_isError ERR_isError6871/* **************************************************************6874* Byte alignment for workSpace management6875****************************************************************/6876#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)6877#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))6878/* **************************************************************6881* BMI2 Variant Wrappers6882****************************************************************/6883#if DYNAMIC_BMI26884/*-***************************/6926/* generic DTableDesc */6927/*-***************************/6928typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;6929{6932DTableDesc dtd;6933memcpy(&dtd, table, sizeof(dtd));6934return dtd;6935}6936/*-***************************/6941/* single-symbol decoding */6942/*-***************************/6943typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX1; /* single-symbol decoding */6944{6947U32 tableLog = 0;6948U32 nbSymbols = 0;6949size_t iSize;6950void* const dtPtr = DTable + 1;6951HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;6952}7013{7016U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7017return HUF_readDTableX1_wksp(DTable, src, srcSize,7018workSpace, sizeof(workSpace));7019}7020FORCE_INLINE_TEMPLATE BYTE7022HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)7023{7024size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */7025BYTE const c = dt[val].byte;7026BIT_skipBits(Dstream, dt[val].nbBits);7027return c;7028}7029{7044BYTE* const pStart = p;7045}7065{7072BYTE* op = (BYTE*)dst;7073BYTE* const oend = op + dstSize;7074const void* dtPtr = DTable + 1;7075const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;7076BIT_DStream_t bitD;7077DTableDesc const dtd = HUF_getDTableDesc(DTable);7078U32 const dtLog = dtd.tableLog;7079}7088{7095/* Check */7096if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */7097* but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */7163if (op1 > opStart2) return ERROR(corruption_detected);7164if (op2 > opStart3) return ERROR(corruption_detected);7165if (op3 > opStart4) return ERROR(corruption_detected);7166/* note : op4 supposed already verified within main loop */7167}7182{7199DTableDesc dtd = HUF_getDTableDesc(DTable);7200if (dtd.tableType != 0) return ERROR(GENERIC);7201return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7202}7203{7208const BYTE* ip = (const BYTE*) cSrc;7209}7217{7222U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7223return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,7224workSpace, sizeof(workSpace));7225}7226{7229HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);7230return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);7231}7232{7238DTableDesc dtd = HUF_getDTableDesc(DTable);7239if (dtd.tableType != 0) return ERROR(GENERIC);7240return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7241}7242{7247const BYTE* ip = (const BYTE*) cSrc;7248}7257{7262return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);7263}7264{7268U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7269return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,7270workSpace, sizeof(workSpace));7271}7272size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)7273{7274HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);7275return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);7276}7277/* *************************/7284/* double-symbols decoding */7285/* *************************/7286/* HUF_fillDTableX2Level2() :7294* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */7295static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed,7296const U32* rankValOrigin, const int minWeight,7297const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,7298U32 nbBitsBaseline, U16 baseSeq)7299{7300HUF_DEltX2 DElt;7301U32 rankVal[HUF_TABLELOG_MAX + 1];7302}7334{7341U32 rankVal[HUF_TABLELOG_MAX + 1];7342const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */7343const U32 minBits = nbBitsBaseline - maxWeight;7344U32 s;7345}7377{7382U32 tableLog, maxW, sizeOfSort, nbSymbols;7383DTableDesc dtd = HUF_getDTableDesc(DTable);7384U32 const maxTableLog = dtd.maxTableLog;7385size_t iSize;7386void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */7387HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;7388U32 *rankStart;7389}7477{7480U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7481return HUF_readDTableX2_wksp(DTable, src, srcSize,7482workSpace, sizeof(workSpace));7483}7484FORCE_INLINE_TEMPLATE U327487HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)7488{7489size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */7490memcpy(op, dt+val, 2);7491BIT_skipBits(DStream, dt[val].nbBits);7492return dt[val].length;7493}7494FORCE_INLINE_TEMPLATE U327496HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)7497{7498size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */7499memcpy(op, dt+val, 1);7500if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);7501else {7502if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {7503BIT_skipBits(DStream, dt[val].nbBits);7504if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))7505/* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */7506DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);7507} }7508return 1;7509}7510{7526BYTE* const pStart = p;7527}7548{7555BIT_DStream_t bitD;7556}7575{7582if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */7583}7690{7699DTableDesc dtd = HUF_getDTableDesc(DTable);7700if (dtd.tableType != 1) return ERROR(GENERIC);7701return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7702}7703{7708const BYTE* ip = (const BYTE*) cSrc;7709}7718{7723U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7724return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,7725workSpace, sizeof(workSpace));7726}7727{7730HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);7731return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);7732}7733{7739DTableDesc dtd = HUF_getDTableDesc(DTable);7740if (dtd.tableType != 1) return ERROR(GENERIC);7741return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7742}7743{7748const BYTE* ip = (const BYTE*) cSrc;7749}7758{7763return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);7764}7765{7770U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7771return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,7772workSpace, sizeof(workSpace));7773}7774{7777HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);7778return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);7779}7780/* ***********************************/7785/* Universal decompression selectors */7786/* ***********************************/7787{7792DTableDesc const dtd = HUF_getDTableDesc(DTable);7793#if defined(HUF_FORCE_DECOMPRESS_X1)7794(void)dtd;7795assert(dtd.tableType == 0);7796return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7797#elif defined(HUF_FORCE_DECOMPRESS_X2)7798(void)dtd;7799assert(dtd.tableType == 1);7800return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7801#else7802return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :7803HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7804#endif7805}7806{7811DTableDesc const dtd = HUF_getDTableDesc(DTable);7812#if defined(HUF_FORCE_DECOMPRESS_X1)7813(void)dtd;7814assert(dtd.tableType == 0);7815return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7816#elif defined(HUF_FORCE_DECOMPRESS_X2)7817(void)dtd;7818assert(dtd.tableType == 1);7819return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7820#else7821return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :7822HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);7823#endif7824}7825{7831/* single, double, quad */7832{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */7833{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */7834{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */7835{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */7836{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */7837{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */7838{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */7839{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */7840{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */7841{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */7842{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */7843{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */7844{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */7845{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */7846{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */7847{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */7848};7849#endif7850/** HUF_selectDecoder() :7852* Tells which decoder is likely to decode faster,7853* based on a set of pre-computed metrics.7854* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .7855* Assumption : 0 < dstSize <= 128 KB */7856U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)7857{7858assert(dstSize > 0);7859assert(dstSize <= 128*1024);7860#if defined(HUF_FORCE_DECOMPRESS_X1)7861(void)dstSize;7862(void)cSrcSize;7863return 0;7864#elif defined(HUF_FORCE_DECOMPRESS_X2)7865(void)dstSize;7866(void)cSrcSize;7867return 1;7868#else7869/* decoder timing evaluation */7870{ U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */7871U32 const D256 = (U32)(dstSize >> 8);7872U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);7873U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);7874DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */7875return DTime1 < DTime0;7876}7877#endif7878}7879{7885#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)7886static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };7887#endif7888}7909{7912/* validation checks */7913if (dstSize == 0) return ERROR(dstSize_tooSmall);7914if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */7915if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */7916if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */7917}7933{7936U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];7937return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,7938workSpace, sizeof(workSpace));7939}7940{7947/* validation checks */7948if (dstSize == 0) return ERROR(dstSize_tooSmall);7949if (cSrcSize == 0) return ERROR(corruption_detected);7950}7967{7972/* validation checks */7973if (dstSize == 0) return ERROR(dstSize_tooSmall);7974if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */7975if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */7976if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */7977}7997{8001U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];8002return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,8003workSpace, sizeof(workSpace));8004}8005{8009DTableDesc const dtd = HUF_getDTableDesc(DTable);8010#if defined(HUF_FORCE_DECOMPRESS_X1)8011(void)dtd;8012assert(dtd.tableType == 0);8013return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8014#elif defined(HUF_FORCE_DECOMPRESS_X2)8015(void)dtd;8016assert(dtd.tableType == 1);8017return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8018#else8019return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :8020HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8021#endif8022}8023{8027const BYTE* ip = (const BYTE*) cSrc;8028}8036#endif8037{8040DTableDesc const dtd = HUF_getDTableDesc(DTable);8041#if defined(HUF_FORCE_DECOMPRESS_X1)8042(void)dtd;8043assert(dtd.tableType == 0);8044return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8045#elif defined(HUF_FORCE_DECOMPRESS_X2)8046(void)dtd;8047assert(dtd.tableType == 1);8048return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8049#else8050return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :8051HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);8052#endif8053}8054{8057/* validation checks */8058if (dstSize == 0) return ERROR(dstSize_tooSmall);8059if (cSrcSize == 0) return ERROR(corruption_detected);8060}8076/**** ended inlining decompress/huf_decompress.c ****/8077/**** start inlining decompress/zstd_ddict.c ****/8078/*8079* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8080* All rights reserved.8081*8082* This source code is licensed under both the BSD-style license (found in the8083* LICENSE file in the root directory of this source tree) and the GPLv2 (found8084* in the COPYING file in the root directory of this source tree).8085* You may select, at your option, one of the above-listed licenses.8086*/8087/* zstd_ddict.c :8089* concentrates all logic that needs to know the internals of ZSTD_DDict object */8090/*-*******************************************************8092* Dependencies8093*********************************************************/8094/**** start inlining ../common/cpu.h ****/8095/*8096* Copyright (c) 2018-2020, Facebook, Inc.8097* All rights reserved.8098*8099* This source code is licensed under both the BSD-style license (found in the8100* LICENSE file in the root directory of this source tree) and the GPLv2 (found8101* in the COPYING file in the root directory of this source tree).8102* You may select, at your option, one of the above-listed licenses.8103*/8104/**8109* Implementation taken from folly/CpuId.h8110* https://github.com/facebook/folly/blob/master/folly/CpuId.h8111*/8112/**** skipping file: mem.h ****/8115* reserves ebx for use of its pic register so we must specially8151* handle the save and restore to avoid clobbering the register8152*/8153U32 n;8154__asm__(8155"pushl %%ebx\n\t"8156"cpuid\n\t"8157"popl %%ebx\n\t"8158: "=a"(n)8159: "a"(0)8160: "ecx", "edx");8161if (n >= 1) {8162U32 f1a;8163__asm__(8164"pushl %%ebx\n\t"8165"cpuid\n\t"8166"popl %%ebx\n\t"8167: "=a"(f1a), "=c"(f1c), "=d"(f1d)8168: "a"(1));8169}8170if (n >= 7) {8171__asm__(8172"pushl %%ebx\n\t"8173"cpuid\n\t"8174"movl %%ebx, %%eax\n\t"8175"popl %%ebx"8176: "=a"(f7b), "=c"(f7c)8177: "a"(7), "c"(0)8178: "edx");8179}8180#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)8181U32 n;8182__asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");8183if (n >= 1) {8184U32 f1a;8185__asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");8186}8187if (n >= 7) {8188U32 f7a;8189__asm__("cpuid"8190: "=a"(f7a), "=b"(f7b), "=c"(f7c)8191: "a"(7), "c"(0)8192: "edx");8193}8194#endif8195{8196ZSTD_cpuid_t cpuid;8197cpuid.f1c = f1c;8198cpuid.f1d = f1d;8199cpuid.f7b = f7b;8200cpuid.f7c = f7c;8201return cpuid;8202}8203}8204/* cpuid(1): Processor Info and Feature Bits. */8211#define C(name, bit) X(name, f1c, bit)8212C(sse3, 0)8213C(pclmuldq, 1)8214C(dtes64, 2)8215C(monitor, 3)8216C(dscpl, 4)8217C(vmx, 5)8218C(smx, 6)8219C(eist, 7)8220C(tm2, 8)8221C(ssse3, 9)8222C(cnxtid, 10)8223C(fma, 12)8224C(cx16, 13)8225C(xtpr, 14)8226C(pdcm, 15)8227C(pcid, 17)8228C(dca, 18)8229C(sse41, 19)8230C(sse42, 20)8231C(x2apic, 21)8232C(movbe, 22)8233C(popcnt, 23)8234C(tscdeadline, 24)8235C(aes, 25)8236C(xsave, 26)8237C(osxsave, 27)8238C(avx, 28)8239C(f16c, 29)8240C(rdrand, 30)8241#undef C8242#define D(name, bit) X(name, f1d, bit)8243D(fpu, 0)8244D(vme, 1)8245D(de, 2)8246D(pse, 3)8247D(tsc, 4)8248D(msr, 5)8249D(pae, 6)8250D(mce, 7)8251D(cx8, 8)8252D(apic, 9)8253D(sep, 11)8254D(mtrr, 12)8255D(pge, 13)8256D(mca, 14)8257D(cmov, 15)8258D(pat, 16)8259D(pse36, 17)8260D(psn, 18)8261D(clfsh, 19)8262D(ds, 21)8263D(acpi, 22)8264D(mmx, 23)8265D(fxsr, 24)8266D(sse, 25)8267D(sse2, 26)8268D(ss, 27)8269D(htt, 28)8270D(tm, 29)8271D(pbe, 31)8272#undef D8273/* cpuid(7): Extended Features. */8275#define B(name, bit) X(name, f7b, bit)8276B(bmi1, 3)8277B(hle, 4)8278B(avx2, 5)8279B(smep, 7)8280B(bmi2, 8)8281B(erms, 9)8282B(invpcid, 10)8283B(rtm, 11)8284B(mpx, 14)8285B(avx512f, 16)8286B(avx512dq, 17)8287B(rdseed, 18)8288B(adx, 19)8289B(smap, 20)8290B(avx512ifma, 21)8291B(pcommit, 22)8292B(clflushopt, 23)8293B(clwb, 24)8294B(avx512pf, 26)8295B(avx512er, 27)8296B(avx512cd, 28)8297B(sha, 29)8298B(avx512bw, 30)8299B(avx512vl, 31)8300#undef B8301#define C(name, bit) X(name, f7c, bit)8302C(prefetchwt1, 0)8303C(avx512vbmi, 1)8304#undef C8305/**** ended inlining ../common/cpu.h ****/8310/**** skipping file: ../common/mem.h ****/8311#define FSE_STATIC_LINKING_ONLY8312/**** skipping file: ../common/fse.h ****/8313#define HUF_STATIC_LINKING_ONLY8314/**** skipping file: ../common/huf.h ****/8315/**** start inlining zstd_decompress_internal.h ****/8316/*8317* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8318* All rights reserved.8319*8320* This source code is licensed under both the BSD-style license (found in the8321* LICENSE file in the root directory of this source tree) and the GPLv2 (found8322* in the COPYING file in the root directory of this source tree).8323* You may select, at your option, one of the above-listed licenses.8324*/8325/* zstd_decompress_internal:8328* objects and definitions shared within lib/decompress modules */8329/*-*******************************************************8335* Dependencies8336*********************************************************/8337/**** skipping file: ../common/mem.h ****/8338/**** skipping file: ../common/zstd_internal.h ****/8339/*-*******************************************************8343* Constants8344*********************************************************/8345static const U32 LL_base[MaxLL+1] = {83460, 1, 2, 3, 4, 5, 6, 7,83478, 9, 10, 11, 12, 13, 14, 15,834816, 18, 20, 22, 24, 28, 32, 40,834948, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,83500x2000, 0x4000, 0x8000, 0x10000 };8351/*-*******************************************************8375* Decompression types8376*********************************************************/8377typedef struct {8378U32 fastMode;8379U32 tableLog;8380} ZSTD_seqSymbol_header;8381{8420const ZSTD_seqSymbol* LLTptr;8421const ZSTD_seqSymbol* MLTptr;8422const ZSTD_seqSymbol* OFTptr;8423const HUF_DTable* HUFptr;8424ZSTD_entropyDTables_t entropy;8425U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */8426const void* previousDstEnd; /* detect continuity */8427const void* prefixStart; /* start of current segment */8428const void* virtualStart; /* virtual start of previous segment if it was just before current one */8429const void* dictEnd; /* end of previous segment */8430size_t expected;8431ZSTD_frameHeader fParams;8432U64 decodedSize;8433blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */8434ZSTD_dStage stage;8435U32 litEntropy;8436U32 fseEntropy;8437XXH64_state_t xxhState;8438size_t headerSize;8439ZSTD_format_e format;8440const BYTE* litPtr;8441ZSTD_customMem customMem;8442size_t litSize;8443size_t rleSize;8444size_t staticSize;8445int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */8446/*-*******************************************************8487* Shared internal functions8488*********************************************************/8489/*! ZSTD_loadDEntropy() :8491* dict : must point at beginning of a valid zstd dictionary.8492* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */8493size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,8494const void* const dict, size_t const dictSize);8495/*! ZSTD_checkContinuity() :8497* check if next `dst` follows previous position, where decompression ended.8498* If yes, do nothing (continue on current segment).8499* If not, classify previous segment as "external dictionary", and start a new segment.8500* This function cannot fail. */8501void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);8502/**** ended inlining zstd_decompress_internal.h ****/8506/**** start inlining zstd_ddict.h ****/8507/*8508* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8509* All rights reserved.8510*8511* This source code is licensed under both the BSD-style license (found in the8512* LICENSE file in the root directory of this source tree) and the GPLv2 (found8513* in the COPYING file in the root directory of this source tree).8514* You may select, at your option, one of the above-listed licenses.8515*/8516/*-*******************************************************8522* Dependencies8523*********************************************************/8524#include <stddef.h> /* size_t */8525/**** skipping file: ../zstd.h ****/8526/*-*******************************************************8529* Interface8530*********************************************************/8531/* note: several prototypes are already published in `zstd.h` :8533* ZSTD_createDDict()8534* ZSTD_createDDict_byReference()8535* ZSTD_createDDict_advanced()8536* ZSTD_freeDDict()8537* ZSTD_initStaticDDict()8538* ZSTD_sizeof_DDict()8539* ZSTD_estimateDDictSize()8540* ZSTD_getDictID_fromDict()8541*/8542/**** ended inlining zstd_ddict.h ****/8552/**** start inlining ../legacy/zstd_legacy.h ****/8555/*8556* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8557* All rights reserved.8558*8559* This source code is licensed under both the BSD-style license (found in the8560* LICENSE file in the root directory of this source tree) and the GPLv2 (found8561* in the COPYING file in the root directory of this source tree).8562* You may select, at your option, one of the above-listed licenses.8563*/8564/* *************************************8573* Includes8574***************************************/8575/**** skipping file: ../common/mem.h ****/8576/**** skipping file: ../common/error_private.h ****/8577/**** skipping file: ../common/zstd_internal.h ****/8578/**** start inlining zstd_v01.h ****/8586/*8587* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8588* All rights reserved.8589*8590* This source code is licensed under both the BSD-style license (found in the8591* LICENSE file in the root directory of this source tree) and the GPLv2 (found8592* in the COPYING file in the root directory of this source tree).8593* You may select, at your option, one of the above-listed licenses.8594*/8595/* *************************************8604* Includes8605***************************************/8606#include <stddef.h> /* size_t */8607/* *************************************8610* Simple one-step function8611***************************************/8612/**8613ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format8614compressedSize : is the exact source size8615maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.8616It must be equal or larger than originalSize, otherwise decompression will fail.8617return : the number of bytes decompressed into destination buffer (originalSize)8618or an errorCode if it fails (which can be tested using ZSTDv01_isError())8619*/8620size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,8621const void* src, size_t compressedSize);8622ZSTDv01_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.1.x format8625srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'8626cSize (output parameter) : the number of bytes that would be read to decompress this frame8627or an error code if it fails (which can be tested using ZSTDv01_isError())8628dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame8629or ZSTD_CONTENTSIZE_ERROR if an error occurs8630note : assumes `cSize` and `dBound` are _not_ NULL.8632*/8633void ZSTDv01_findFrameSizeInfoLegacy(const void *src, size_t srcSize,8634size_t* cSize, unsigned long long* dBound);8635/**8637ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error8638*/8639unsigned ZSTDv01_isError(size_t code);8640/* *************************************8643* Advanced functions8644***************************************/8645typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx;8646ZSTDv01_Dctx* ZSTDv01_createDCtx(void);8647size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx);8648/* *************************************8654* Streaming functions8655***************************************/8656size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx);8657/**8661Use above functions alternatively.8662ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().8663ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.8664Result is the number of bytes regenerated within 'dst'.8665It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.8666*/8667/* *************************************8669* Prefix - version detection8670***************************************/8671#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */8672#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */8673}8677#endif8678/**** ended inlining zstd_v01.h ****/8681#endif8682#if (ZSTD_LEGACY_SUPPORT <= 2)8683/**** start inlining zstd_v02.h ****/8684/*8685* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8686* All rights reserved.8687*8688* This source code is licensed under both the BSD-style license (found in the8689* LICENSE file in the root directory of this source tree) and the GPLv2 (found8690* in the COPYING file in the root directory of this source tree).8691* You may select, at your option, one of the above-listed licenses.8692*/8693/* *************************************8702* Includes8703***************************************/8704#include <stddef.h> /* size_t */8705/* *************************************8708* Simple one-step function8709***************************************/8710/**8711ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format8712compressedSize : is the exact source size8713maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.8714It must be equal or larger than originalSize, otherwise decompression will fail.8715return : the number of bytes decompressed into destination buffer (originalSize)8716or an errorCode if it fails (which can be tested using ZSTDv01_isError())8717*/8718size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,8719const void* src, size_t compressedSize);8720ZSTDv02_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.2.x format8723srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'8724cSize (output parameter) : the number of bytes that would be read to decompress this frame8725or an error code if it fails (which can be tested using ZSTDv01_isError())8726dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame8727or ZSTD_CONTENTSIZE_ERROR if an error occurs8728note : assumes `cSize` and `dBound` are _not_ NULL.8730*/8731void ZSTDv02_findFrameSizeInfoLegacy(const void *src, size_t srcSize,8732size_t* cSize, unsigned long long* dBound);8733/**8735ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error8736*/8737unsigned ZSTDv02_isError(size_t code);8738/* *************************************8741* Advanced functions8742***************************************/8743typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx;8744ZSTDv02_Dctx* ZSTDv02_createDCtx(void);8745size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx);8746/* *************************************8752* Streaming functions8753***************************************/8754size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx);8755/**8759Use above functions alternatively.8760ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().8761ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.8762Result is the number of bytes regenerated within 'dst'.8763It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.8764*/8765/* *************************************8767* Prefix - version detection8768***************************************/8769#define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */8770}8774#endif8775/**** ended inlining zstd_v02.h ****/8778#endif8779#if (ZSTD_LEGACY_SUPPORT <= 3)8780/**** start inlining zstd_v03.h ****/8781/*8782* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8783* All rights reserved.8784*8785* This source code is licensed under both the BSD-style license (found in the8786* LICENSE file in the root directory of this source tree) and the GPLv2 (found8787* in the COPYING file in the root directory of this source tree).8788* You may select, at your option, one of the above-listed licenses.8789*/8790/* *************************************8799* Includes8800***************************************/8801#include <stddef.h> /* size_t */8802/* *************************************8805* Simple one-step function8806***************************************/8807/**8808ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format8809compressedSize : is the exact source size8810maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.8811It must be equal or larger than originalSize, otherwise decompression will fail.8812return : the number of bytes decompressed into destination buffer (originalSize)8813or an errorCode if it fails (which can be tested using ZSTDv01_isError())8814*/8815size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,8816const void* src, size_t compressedSize);8817ZSTDv03_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.3.x format8820srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'8821cSize (output parameter) : the number of bytes that would be read to decompress this frame8822or an error code if it fails (which can be tested using ZSTDv01_isError())8823dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame8824or ZSTD_CONTENTSIZE_ERROR if an error occurs8825note : assumes `cSize` and `dBound` are _not_ NULL.8827*/8828void ZSTDv03_findFrameSizeInfoLegacy(const void *src, size_t srcSize,8829size_t* cSize, unsigned long long* dBound);8830ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error8833*/8834unsigned ZSTDv03_isError(size_t code);8835/* *************************************8838* Advanced functions8839***************************************/8840typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx;8841ZSTDv03_Dctx* ZSTDv03_createDCtx(void);8842size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx);8843/* *************************************8849* Streaming functions8850***************************************/8851size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx);8852/**8856Use above functions alternatively.8857ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().8858ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.8859Result is the number of bytes regenerated within 'dst'.8860It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.8861*/8862/* *************************************8864* Prefix - version detection8865***************************************/8866#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */8867}8871#endif8872/**** ended inlining zstd_v03.h ****/8875#endif8876#if (ZSTD_LEGACY_SUPPORT <= 4)8877/**** start inlining zstd_v04.h ****/8878/*8879* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.8880* All rights reserved.8881*8882* This source code is licensed under both the BSD-style license (found in the8883* LICENSE file in the root directory of this source tree) and the GPLv2 (found8884* in the COPYING file in the root directory of this source tree).8885* You may select, at your option, one of the above-listed licenses.8886*/8887/* *************************************8896* Includes8897***************************************/8898#include <stddef.h> /* size_t */8899/* *************************************8902* Simple one-step function8903***************************************/8904/**8905ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format8906compressedSize : is the exact source size8907maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.8908It must be equal or larger than originalSize, otherwise decompression will fail.8909return : the number of bytes decompressed into destination buffer (originalSize)8910or an errorCode if it fails (which can be tested using ZSTDv01_isError())8911*/8912size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,8913const void* src, size_t compressedSize);8914ZSTDv04_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.4.x format8917srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'8918cSize (output parameter) : the number of bytes that would be read to decompress this frame8919or an error code if it fails (which can be tested using ZSTDv01_isError())8920dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame8921or ZSTD_CONTENTSIZE_ERROR if an error occurs8922note : assumes `cSize` and `dBound` are _not_ NULL.8924*/8925void ZSTDv04_findFrameSizeInfoLegacy(const void *src, size_t srcSize,8926size_t* cSize, unsigned long long* dBound);8927/**8929ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error8930*/8931unsigned ZSTDv04_isError(size_t code);8932/* *************************************8935* Advanced functions8936***************************************/8937typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx;8938ZSTDv04_Dctx* ZSTDv04_createDCtx(void);8939size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx);8940/* *************************************8947* Direct Streaming8948***************************************/8949size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx);8950/**8954Use above functions alternatively.8955ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().8956ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.8957Result is the number of bytes regenerated within 'dst'.8958It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.8959*/8960/* *************************************8963* Buffered Streaming8964***************************************/8965typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx;8966ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void);8967size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);8968/** ************************************************8975* Streaming decompression8976*8977* A ZBUFF_DCtx object is required to track streaming operation.8978* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.8979* Use ZBUFF_decompressInit() to start a new decompression operation.8980* ZBUFF_DCtx objects can be reused multiple times.8981*8982* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary()8983* It must be the same content as the one set during compression phase.8984* Dictionary content must remain accessible during the decompression process.8985*8986* Use ZBUFF_decompressContinue() repetitively to consume your input.8987* *srcSizePtr and *maxDstSizePtr can be any size.8988* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.8989* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.8990* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst.8991* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)8992* or 0 when a frame is completely decoded8993* or an error code, which can be tested using ZBUFF_isError().8994*8995* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize8996* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.8997* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .8998* **************************************************/8999unsigned ZBUFFv04_isError(size_t errorCode);9000const char* ZBUFFv04_getErrorName(size_t errorCode);9001/** The below functions provide recommended buffer sizes for Compression or Decompression operations.9004* These sizes are not compulsory, they just tend to offer better latency */9005size_t ZBUFFv04_recommendedDInSize(void);9006size_t ZBUFFv04_recommendedDOutSize(void);9007/* *************************************9010* Prefix - version detection9011***************************************/9012#define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */9013}9017#endif9018/**** ended inlining zstd_v04.h ****/9021#endif9022#if (ZSTD_LEGACY_SUPPORT <= 5)9023/**** start inlining zstd_v05.h ****/9024/*9025* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.9026* All rights reserved.9027*9028* This source code is licensed under both the BSD-style license (found in the9029* LICENSE file in the root directory of this source tree) and the GPLv2 (found9030* in the COPYING file in the root directory of this source tree).9031* You may select, at your option, one of the above-listed licenses.9032*/9033/*-*************************************9042* Dependencies9043***************************************/9044#include <stddef.h> /* size_t */9045/**** skipping file: ../common/mem.h ****/9046/* *************************************9049* Simple functions9050***************************************/9051/*! ZSTDv05_decompress() :9052`compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.9053`dstCapacity` must be large enough, equal or larger than originalSize.9054@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),9055or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */9056size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,9057const void* src, size_t compressedSize);9058ZSTDv05_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.5.x format9061srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'9062cSize (output parameter) : the number of bytes that would be read to decompress this frame9063or an error code if it fails (which can be tested using ZSTDv01_isError())9064dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame9065or ZSTD_CONTENTSIZE_ERROR if an error occurs9066note : assumes `cSize` and `dBound` are _not_ NULL.9068*/9069void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize,9070size_t* cSize, unsigned long long* dBound);9071/* *************************************9073* Helper functions9074***************************************/9075/* Error Management */9076unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */9077const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */9078/* *************************************9081* Explicit memory management9082***************************************/9083/** Decompression context */9084typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx;9085ZSTDv05_DCtx* ZSTDv05_createDCtx(void);9086size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */9087/** ZSTDv05_decompressDCtx() :9089* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */9090size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);9091/*-***********************9094* Simple Dictionary API9095*************************/9096/*! ZSTDv05_decompress_usingDict() :9097* Decompression using a pre-defined Dictionary content (see dictBuilder).9098* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.9099* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */9100size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,9101void* dst, size_t dstCapacity,9102const void* src, size_t srcSize,9103const void* dict,size_t dictSize);9104/*-************************9106* Advanced Streaming API9107***************************/9108typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy;9109typedef struct {9110U64 srcSize;9111U32 windowLog; /* the only useful information to retrieve */9112U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy;9113} ZSTDv05_parameters;9114size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize);9115/*-***********************9123* ZBUFF API9124*************************/9125typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;9126ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);9127size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);9128/*-***************************************************************************9137* Streaming decompression9138*9139* A ZBUFFv05_DCtx object is required to track streaming operations.9140* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources.9141* Use ZBUFFv05_decompressInit() to start a new decompression operation,9142* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary.9143* Note that ZBUFFv05_DCtx objects can be reused multiple times.9144*9145* Use ZBUFFv05_decompressContinue() repetitively to consume your input.9146* *srcSizePtr and *dstCapacityPtr can be any size.9147* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.9148* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.9149* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst.9150* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency)9151* or 0 when a frame is completely decoded9152* or an error code, which can be tested using ZBUFFv05_isError().9153*9154* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize()9155* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.9156* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .9157* *******************************************************************************/9158/* *************************************9161* Tool functions9162***************************************/9163unsigned ZBUFFv05_isError(size_t errorCode);9164const char* ZBUFFv05_getErrorName(size_t errorCode);9165/** Functions below provide recommended buffer sizes for Compression or Decompression operations.9167* These sizes are just hints, and tend to offer better latency */9168size_t ZBUFFv05_recommendedDInSize(void);9169size_t ZBUFFv05_recommendedDOutSize(void);9170/*-*************************************9174* Constants9175***************************************/9176#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */9177}9183#endif9184/**** ended inlining zstd_v05.h ****/9187#endif9188#if (ZSTD_LEGACY_SUPPORT <= 6)9189/**** start inlining zstd_v06.h ****/9190/*9191* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.9192* All rights reserved.9193*9194* This source code is licensed under both the BSD-style license (found in the9195* LICENSE file in the root directory of this source tree) and the GPLv2 (found9196* in the COPYING file in the root directory of this source tree).9197* You may select, at your option, one of the above-listed licenses.9198*/9199/*====== Dependency ======*/9208#include <stddef.h> /* size_t */9209/*====== Export for Windows ======*/9212/*!9213* ZSTDv06_DLL_EXPORT :9214* Enable exporting of functions when building a Windows DLL9215*/9216#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1)9217# define ZSTDLIBv06_API __declspec(dllexport)9218#else9219# define ZSTDLIBv06_API9220#endif9221/* *************************************9224* Simple functions9225***************************************/9226/*! ZSTDv06_decompress() :9227`compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.9228`dstCapacity` must be large enough, equal or larger than originalSize.9229@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),9230or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */9231ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,9232const void* src, size_t compressedSize);9233/**9235ZSTDv06_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.6.x format9236srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'9237cSize (output parameter) : the number of bytes that would be read to decompress this frame9238or an error code if it fails (which can be tested using ZSTDv01_isError())9239dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame9240or ZSTD_CONTENTSIZE_ERROR if an error occurs9241note : assumes `cSize` and `dBound` are _not_ NULL.9243*/9244void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize,9245size_t* cSize, unsigned long long* dBound);9246/* *************************************9248* Helper functions9249***************************************/9250ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */9251/* Error Management */9253ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */9254ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */9255/* *************************************9258* Explicit memory management9259***************************************/9260/** Decompression context */9261typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx;9262ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);9263ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */9264/** ZSTDv06_decompressDCtx() :9266* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */9267ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);9268/*-***********************9271* Dictionary API9272*************************/9273/*! ZSTDv06_decompress_usingDict() :9274* Decompression using a pre-defined Dictionary content (see dictBuilder).9275* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.9276* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */9277ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,9278void* dst, size_t dstCapacity,9279const void* src, size_t srcSize,9280const void* dict,size_t dictSize);9281/*-************************9284* Advanced Streaming API9285***************************/9286struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; };9287typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;9288/* *************************************9299* ZBUFF API9300***************************************/9301/*-***************************************************************************9314* Streaming decompression howto9315*9316* A ZBUFFv06_DCtx object is required to track streaming operations.9317* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources.9318* Use ZBUFFv06_decompressInit() to start a new decompression operation,9319* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary.9320* Note that ZBUFFv06_DCtx objects can be re-init multiple times.9321*9322* Use ZBUFFv06_decompressContinue() repetitively to consume your input.9323* *srcSizePtr and *dstCapacityPtr can be any size.9324* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.9325* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.9326* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.9327* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),9328* or 0 when a frame is completely decoded,9329* or an error code, which can be tested using ZBUFFv06_isError().9330*9331* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize()9332* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.9333* input : ZBUFFv06_recommendedDInSize == 128KB + 3;9334* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .9335* *******************************************************************************/9336/* *************************************9339* Tool functions9340***************************************/9341ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode);9342ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode);9343/** Functions below provide recommended buffer sizes for Compression or Decompression operations.9345* These sizes are just hints, they tend to offer better latency */9346ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void);9347ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void);9348/*-*************************************9351* Constants9352***************************************/9353#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */9354}9359#endif9360/**** ended inlining zstd_v06.h ****/9363#endif9364#if (ZSTD_LEGACY_SUPPORT <= 7)9365/**** start inlining zstd_v07.h ****/9366/*9367* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.9368* All rights reserved.9369*9370* This source code is licensed under both the BSD-style license (found in the9371* LICENSE file in the root directory of this source tree) and the GPLv2 (found9372* in the COPYING file in the root directory of this source tree).9373* You may select, at your option, one of the above-listed licenses.9374*/9375/*====== Dependency ======*/9384#include <stddef.h> /* size_t */9385/*====== Export for Windows ======*/9388/*!9389* ZSTDv07_DLL_EXPORT :9390* Enable exporting of functions when building a Windows DLL9391*/9392#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1)9393# define ZSTDLIBv07_API __declspec(dllexport)9394#else9395# define ZSTDLIBv07_API9396#endif9397/* *************************************9400* Simple API9401***************************************/9402/*! ZSTDv07_getDecompressedSize() :9403* @return : decompressed size if known, 0 otherwise.9404note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause.9405note 2 : decompressed size could be wrong or intentionally modified !9406always ensure results fit within application's authorized limits */9407unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);9408/*! ZSTDv07_decompress() :9410`compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail.9411`dstCapacity` must be equal or larger than originalSize.9412@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),9413or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */9414ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,9415const void* src, size_t compressedSize);9416/**9418ZSTDv07_findFrameSizeInfoLegacy() : get the source length and decompressed bound of a ZSTD frame compliant with v0.7.x format9419srcSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'9420cSize (output parameter) : the number of bytes that would be read to decompress this frame9421or an error code if it fails (which can be tested using ZSTDv01_isError())9422dBound (output parameter) : an upper-bound for the decompressed size of the data in the frame9423or ZSTD_CONTENTSIZE_ERROR if an error occurs9424note : assumes `cSize` and `dBound` are _not_ NULL.9426*/9427void ZSTDv07_findFrameSizeInfoLegacy(const void *src, size_t srcSize,9428size_t* cSize, unsigned long long* dBound);9429/*====== Helper functions ======*/9431ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */9432ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */9433/*-*************************************9436* Explicit memory management9437***************************************/9438/** Decompression context */9439typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx;9440ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);9441ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */9442/** ZSTDv07_decompressDCtx() :9444* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */9445ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);9446/*-************************9449* Simple dictionary API9450***************************/9451/*! ZSTDv07_decompress_usingDict() :9452* Decompression using a pre-defined Dictionary content (see dictBuilder).9453* Dictionary must be identical to the one used during compression.9454* Note : This function load the dictionary, resulting in a significant startup time */9455ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,9456void* dst, size_t dstCapacity,9457const void* src, size_t srcSize,9458const void* dict,size_t dictSize);9459/*-**************************9462* Advanced Dictionary API9463****************************/9464/*! ZSTDv07_createDDict() :9465* Create a digested dictionary, ready to start decompression operation without startup delay.9466* `dict` can be released after creation */9467typedef struct ZSTDv07_DDict_s ZSTDv07_DDict;9468ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);9469ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);9470/*! ZSTDv07_decompress_usingDDict() :9472* Decompression using a pre-digested Dictionary9473* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */9474ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,9475void* dst, size_t dstCapacity,9476const void* src, size_t srcSize,9477const ZSTDv07_DDict* ddict);9478/* *************************************9492* Streaming functions9493***************************************/9494typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx;9495ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);9496ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);9497/*-***************************************************************************9506* Streaming decompression howto9507*9508* A ZBUFFv07_DCtx object is required to track streaming operations.9509* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources.9510* Use ZBUFFv07_decompressInit() to start a new decompression operation,9511* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary.9512* Note that ZBUFFv07_DCtx objects can be re-init multiple times.9513*9514* Use ZBUFFv07_decompressContinue() repetitively to consume your input.9515* *srcSizePtr and *dstCapacityPtr can be any size.9516* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.9517* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.9518* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.9519* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),9520* or 0 when a frame is completely decoded,9521* or an error code, which can be tested using ZBUFFv07_isError().9522*9523* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize()9524* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.9525* input : ZBUFFv07_recommendedDInSize == 128KB + 3;9526* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .9527* *******************************************************************************/9528/* *************************************9531* Tool functions9532***************************************/9533ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode);9534ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode);9535/** Functions below provide recommended buffer sizes for Compression or Decompression operations.9537* These sizes are just hints, they tend to offer better latency */9538ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void);9539ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void);9540/*-*************************************9543* Constants9544***************************************/9545#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */9546}9550#endif9551/**** ended inlining zstd_v07.h ****/9554#endif9555/** ZSTD_isLegacy() :9557@return : > 0 if supported by legacy decoder. 0 otherwise.9558return value is the version.9559*/9560MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)9561{9562U32 magicNumberLE;9563if (srcSize<4) return 0;9564magicNumberLE = MEM_readLE32(src);9565switch(magicNumberLE)9566{9567#if (ZSTD_LEGACY_SUPPORT <= 1)9568case ZSTDv01_magicNumberLE:return 1;9569#endif9570#if (ZSTD_LEGACY_SUPPORT <= 2)9571case ZSTDv02_magicNumber : return 2;9572#endif9573#if (ZSTD_LEGACY_SUPPORT <= 3)9574case ZSTDv03_magicNumber : return 3;9575#endif9576#if (ZSTD_LEGACY_SUPPORT <= 4)9577case ZSTDv04_magicNumber : return 4;9578#endif9579#if (ZSTD_LEGACY_SUPPORT <= 5)9580case ZSTDv05_MAGICNUMBER : return 5;9581#endif9582#if (ZSTD_LEGACY_SUPPORT <= 6)9583case ZSTDv06_MAGICNUMBER : return 6;9584#endif9585#if (ZSTD_LEGACY_SUPPORT <= 7)9586case ZSTDv07_MAGICNUMBER : return 7;9587#endif9588default : return 0;9589}9590}9591{9595U32 const version = ZSTD_isLegacy(src, srcSize);9596if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */9597#if (ZSTD_LEGACY_SUPPORT <= 5)9598if (version==5) {9599ZSTDv05_parameters fParams;9600size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);9601if (frResult != 0) return 0;9602return fParams.srcSize;9603}9604#endif9605#if (ZSTD_LEGACY_SUPPORT <= 6)9606if (version==6) {9607ZSTDv06_frameParams fParams;9608size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);9609if (frResult != 0) return 0;9610return fParams.frameContentSize;9611}9612#endif9613#if (ZSTD_LEGACY_SUPPORT <= 7)9614if (version==7) {9615ZSTDv07_frameParams fParams;9616size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);9617if (frResult != 0) return 0;9618return fParams.frameContentSize;9619}9620#endif9621return 0; /* should not be possible */9622}9623{9630U32 const version = ZSTD_isLegacy(src, compressedSize);9631(void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */9632switch(version)9633{9634#if (ZSTD_LEGACY_SUPPORT <= 1)9635case 1 :9636return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);9637#endif9638#if (ZSTD_LEGACY_SUPPORT <= 2)9639case 2 :9640return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);9641#endif9642#if (ZSTD_LEGACY_SUPPORT <= 3)9643case 3 :9644return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);9645#endif9646#if (ZSTD_LEGACY_SUPPORT <= 4)9647case 4 :9648return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);9649#endif9650#if (ZSTD_LEGACY_SUPPORT <= 5)9651case 5 :9652{ size_t result;9653ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();9654if (zd==NULL) return ERROR(memory_allocation);9655result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);9656ZSTDv05_freeDCtx(zd);9657return result;9658}9659#endif9660#if (ZSTD_LEGACY_SUPPORT <= 6)9661case 6 :9662{ size_t result;9663ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();9664if (zd==NULL) return ERROR(memory_allocation);9665result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);9666ZSTDv06_freeDCtx(zd);9667return result;9668}9669#endif9670#if (ZSTD_LEGACY_SUPPORT <= 7)9671case 7 :9672{ size_t result;9673ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();9674if (zd==NULL) return ERROR(memory_allocation);9675result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);9676ZSTDv07_freeDCtx(zd);9677return result;9678}9679#endif9680default :9681return ERROR(prefix_unknown);9682}9683}9684{9687ZSTD_frameSizeInfo frameSizeInfo;9688U32 const version = ZSTD_isLegacy(src, srcSize);9689switch(version)9690{9691#if (ZSTD_LEGACY_SUPPORT <= 1)9692case 1 :9693ZSTDv01_findFrameSizeInfoLegacy(src, srcSize,9694&frameSizeInfo.compressedSize,9695&frameSizeInfo.decompressedBound);9696break;9697#endif9698#if (ZSTD_LEGACY_SUPPORT <= 2)9699case 2 :9700ZSTDv02_findFrameSizeInfoLegacy(src, srcSize,9701&frameSizeInfo.compressedSize,9702&frameSizeInfo.decompressedBound);9703break;9704#endif9705#if (ZSTD_LEGACY_SUPPORT <= 3)9706case 3 :9707ZSTDv03_findFrameSizeInfoLegacy(src, srcSize,9708&frameSizeInfo.compressedSize,9709&frameSizeInfo.decompressedBound);9710break;9711#endif9712#if (ZSTD_LEGACY_SUPPORT <= 4)9713case 4 :9714ZSTDv04_findFrameSizeInfoLegacy(src, srcSize,9715&frameSizeInfo.compressedSize,9716&frameSizeInfo.decompressedBound);9717break;9718#endif9719#if (ZSTD_LEGACY_SUPPORT <= 5)9720case 5 :9721ZSTDv05_findFrameSizeInfoLegacy(src, srcSize,9722&frameSizeInfo.compressedSize,9723&frameSizeInfo.decompressedBound);9724break;9725#endif9726#if (ZSTD_LEGACY_SUPPORT <= 6)9727case 6 :9728ZSTDv06_findFrameSizeInfoLegacy(src, srcSize,9729&frameSizeInfo.compressedSize,9730&frameSizeInfo.decompressedBound);9731break;9732#endif9733#if (ZSTD_LEGACY_SUPPORT <= 7)9734case 7 :9735ZSTDv07_findFrameSizeInfoLegacy(src, srcSize,9736&frameSizeInfo.compressedSize,9737&frameSizeInfo.decompressedBound);9738break;9739#endif9740default :9741frameSizeInfo.compressedSize = ERROR(prefix_unknown);9742frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;9743break;9744}9745if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {9746frameSizeInfo.compressedSize = ERROR(srcSize_wrong);9747frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;9748}9749return frameSizeInfo;9750}9751{9754ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfoLegacy(src, srcSize);9755return frameSizeInfo.compressedSize;9756}9757{9760switch(version)9761{9762default :9763case 1 :9764case 2 :9765case 3 :9766(void)legacyContext;9767return ERROR(version_unsupported);9768#if (ZSTD_LEGACY_SUPPORT <= 4)9769case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);9770#endif9771#if (ZSTD_LEGACY_SUPPORT <= 5)9772case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);9773#endif9774#if (ZSTD_LEGACY_SUPPORT <= 6)9775case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);9776#endif9777#if (ZSTD_LEGACY_SUPPORT <= 7)9778case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);9779#endif9780}9781}9782{9787DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion);9788if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);9789switch(newVersion)9790{9791default :9792case 1 :9793case 2 :9794case 3 :9795(void)dict; (void)dictSize;9796return 0;9797#if (ZSTD_LEGACY_SUPPORT <= 4)9798case 4 :9799{9800ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;9801if (dctx==NULL) return ERROR(memory_allocation);9802ZBUFFv04_decompressInit(dctx);9803ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize);9804*legacyContext = dctx;9805return 0;9806}9807#endif9808#if (ZSTD_LEGACY_SUPPORT <= 5)9809case 5 :9810{9811ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;9812if (dctx==NULL) return ERROR(memory_allocation);9813ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize);9814*legacyContext = dctx;9815return 0;9816}9817#endif9818#if (ZSTD_LEGACY_SUPPORT <= 6)9819case 6 :9820{9821ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;9822if (dctx==NULL) return ERROR(memory_allocation);9823ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize);9824*legacyContext = dctx;9825return 0;9826}9827#endif9828#if (ZSTD_LEGACY_SUPPORT <= 7)9829case 7 :9830{9831ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;9832if (dctx==NULL) return ERROR(memory_allocation);9833ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize);9834*legacyContext = dctx;9835return 0;9836}9837#endif9838}9839}9840{9846DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version);9847switch(version)9848{9849default :9850case 1 :9851case 2 :9852case 3 :9853(void)legacyContext; (void)output; (void)input;9854return ERROR(version_unsupported);9855#if (ZSTD_LEGACY_SUPPORT <= 4)9856case 4 :9857{9858ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;9859const void* src = (const char*)input->src + input->pos;9860size_t readSize = input->size - input->pos;9861void* dst = (char*)output->dst + output->pos;9862size_t decodedSize = output->size - output->pos;9863size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize);9864output->pos += decodedSize;9865input->pos += readSize;9866return hintSize;9867}9868#endif9869#if (ZSTD_LEGACY_SUPPORT <= 5)9870case 5 :9871{9872ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;9873const void* src = (const char*)input->src + input->pos;9874size_t readSize = input->size - input->pos;9875void* dst = (char*)output->dst + output->pos;9876size_t decodedSize = output->size - output->pos;9877size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize);9878output->pos += decodedSize;9879input->pos += readSize;9880return hintSize;9881}9882#endif9883#if (ZSTD_LEGACY_SUPPORT <= 6)9884case 6 :9885{9886ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;9887const void* src = (const char*)input->src + input->pos;9888size_t readSize = input->size - input->pos;9889void* dst = (char*)output->dst + output->pos;9890size_t decodedSize = output->size - output->pos;9891size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize);9892output->pos += decodedSize;9893input->pos += readSize;9894return hintSize;9895}9896#endif9897#if (ZSTD_LEGACY_SUPPORT <= 7)9898case 7 :9899{9900ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;9901const void* src = (const char*)input->src + input->pos;9902size_t readSize = input->size - input->pos;9903void* dst = (char*)output->dst + output->pos;9904size_t decodedSize = output->size - output->pos;9905size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize);9906output->pos += decodedSize;9907input->pos += readSize;9908return hintSize;9909}9910#endif9911}9912}9913}9917#endif9918/**** ended inlining ../legacy/zstd_legacy.h ****/9921#endif9922/*-*******************************************************9926* Types9927*********************************************************/9928struct ZSTD_DDict_s {9929void* dictBuffer;9930const void* dictContent;9931size_t dictSize;9932ZSTD_entropyDTables_t entropy;9933U32 dictID;9934U32 entropyPresent;9935ZSTD_customMem cMem;9936}; /* typedef'd to ZSTD_DDict within "zstd.h" */9937{9940assert(ddict != NULL);9941return ddict->dictContent;9942}9943{9946assert(ddict != NULL);9947return ddict->dictSize;9948}9949{9952DEBUGLOG(4, "ZSTD_copyDDictParameters");9953assert(dctx != NULL);9954assert(ddict != NULL);9955dctx->dictID = ddict->dictID;9956dctx->prefixStart = ddict->dictContent;9957dctx->virtualStart = ddict->dictContent;9958dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;9959dctx->previousDstEnd = dctx->dictEnd;9960#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION9961dctx->dictContentBeginForFuzzing = dctx->prefixStart;9962dctx->dictContentEndForFuzzing = dctx->previousDstEnd;9963#endif9964if (ddict->entropyPresent) {9965dctx->litEntropy = 1;9966dctx->fseEntropy = 1;9967dctx->LLTptr = ddict->entropy.LLTable;9968dctx->MLTptr = ddict->entropy.MLTable;9969dctx->OFTptr = ddict->entropy.OFTable;9970dctx->HUFptr = ddict->entropy.hufTable;9971dctx->entropy.rep[0] = ddict->entropy.rep[0];9972dctx->entropy.rep[1] = ddict->entropy.rep[1];9973dctx->entropy.rep[2] = ddict->entropy.rep[2];9974} else {9975dctx->litEntropy = 0;9976dctx->fseEntropy = 0;9977}9978}9979{9985ddict->dictID = 0;9986ddict->entropyPresent = 0;9987if (dictContentType == ZSTD_dct_rawContent) return 0;9988}10010{10017if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {10018ddict->dictBuffer = NULL;10019ddict->dictContent = dict;10020if (!dict) dictSize = 0;10021} else {10022void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);10023ddict->dictBuffer = internalBuffer;10024ddict->dictContent = internalBuffer;10025if (!internalBuffer) return ERROR(memory_allocation);10026memcpy(internalBuffer, dict, dictSize);10027}10028ddict->dictSize = dictSize;10029ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */10030}10036{10042if (!customMem.customAlloc ^ !customMem.customFree) return NULL;10043}10057/*! ZSTD_createDDict() :10059* Create a digested dictionary, to start decompression without startup delay.10060* `dict` content is copied inside DDict.10061* Consequently, `dict` can be released after `ZSTD_DDict` creation */10062ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)10063{10064ZSTD_customMem const allocator = { NULL, NULL, NULL };10065return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);10066}10067/*! ZSTD_createDDict_byReference() :10069* Create a digested dictionary, to start decompression without startup delay.10070* Dictionary content is simply referenced, it will be accessed during decompression.10071* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */10072ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)10073{10074ZSTD_customMem const allocator = { NULL, NULL, NULL };10075return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);10076}10077{10085size_t const neededSpace = sizeof(ZSTD_DDict)10086+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);10087ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;10088assert(sBuffer != NULL);10089assert(dict != NULL);10090if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */10091if (sBufferSize < neededSpace) return NULL;10092if (dictLoadMethod == ZSTD_dlm_byCopy) {10093memcpy(ddict+1, dict, dictSize); /* local copy */10094dict = ddict+1;10095}10096if (ZSTD_isError( ZSTD_initDDict_internal(ddict,10097dict, dictSize,10098ZSTD_dlm_byRef, dictContentType) ))10099return NULL;10100return ddict;10101}10102{10106if (ddict==NULL) return 0; /* support free on NULL */10107{ ZSTD_customMem const cMem = ddict->cMem;10108ZSTD_free(ddict->dictBuffer, cMem);10109ZSTD_free(ddict, cMem);10110return 0;10111}10112}10113/*! ZSTD_estimateDDictSize() :10115* Estimate amount of memory that will be needed to create a dictionary for decompression.10116* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */10117size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)10118{10119return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);10120}10121{10124if (ddict==NULL) return 0; /* support sizeof on NULL */10125return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;10126}10127/*! ZSTD_getDictID_fromDDict() :10129* Provides the dictID of the dictionary loaded into `ddict`.10130* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.10131* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */10132unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)10133{10134if (ddict==NULL) return 0;10135return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);10136}10137/**** ended inlining decompress/zstd_ddict.c ****/10138/**** start inlining decompress/zstd_decompress.c ****/10139/*10140* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.10141* All rights reserved.10142*10143* This source code is licensed under both the BSD-style license (found in the10144* LICENSE file in the root directory of this source tree) and the GPLv2 (found10145* in the COPYING file in the root directory of this source tree).10146* You may select, at your option, one of the above-listed licenses.10147*/10148/* ***************************************************************10151* Tuning parameters10152*****************************************************************/10153/*!10154* HEAPMODE :10155* Select how default decompression function ZSTD_decompress() allocates its context,10156* on stack (0), or into heap (1, default; requires malloc()).10157* Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.10158*/10159#ifndef ZSTD_HEAPMODE10160# define ZSTD_HEAPMODE 110161#endif10162/*!10164* LEGACY_SUPPORT :10165* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)10166*/10167#ifndef ZSTD_LEGACY_SUPPORT10168# define ZSTD_LEGACY_SUPPORT 010169#endif10170/*!10172* MAXWINDOWSIZE_DEFAULT :10173* maximum window size accepted by DStream __by default__.10174* Frames requiring more memory will be rejected.10175* It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().10176*/10177#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT10178# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)10179#endif10180/*!10182* NO_FORWARD_PROGRESS_MAX :10183* maximum allowed nb of calls to ZSTD_decompressStream()10184* without any forward progress10185* (defined as: no byte read from input, and no byte flushed to output)10186* before triggering an error.10187*/10188#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX10189# define ZSTD_NO_FORWARD_PROGRESS_MAX 1610190#endif10191/*-*******************************************************10194* Dependencies10195*********************************************************/10196/**** skipping file: ../common/cpu.h ****/10197/**** skipping file: ../common/mem.h ****/10198#define FSE_STATIC_LINKING_ONLY10199/**** skipping file: ../common/fse.h ****/10200#define HUF_STATIC_LINKING_ONLY10201/**** skipping file: ../common/huf.h ****/10202/**** skipping file: ../common/zstd_internal.h ****/10203/**** skipping file: zstd_decompress_internal.h ****/10204/**** skipping file: zstd_ddict.h ****/10205/**** start inlining zstd_decompress_block.h ****/10206/*10207* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.10208* All rights reserved.10209*10210* This source code is licensed under both the BSD-style license (found in the10211* LICENSE file in the root directory of this source tree) and the GPLv2 (found10212* in the COPYING file in the root directory of this source tree).10213* You may select, at your option, one of the above-listed licenses.10214*/10215/*-*******************************************************10221* Dependencies10222*********************************************************/10223#include <stddef.h> /* size_t */10224/**** skipping file: ../zstd.h ****/10225/**** skipping file: ../common/zstd_internal.h ****/10226/**** skipping file: zstd_decompress_internal.h ****/10227/* === Prototypes === */10230/* note: prototypes already published within `zstd.h` :10232* ZSTD_decompressBlock()10233*/10234/* note: prototypes already published within `zstd_internal.h` :10236* ZSTD_getcBlockSize()10237* ZSTD_decodeSeqHeaders()10238*/10239/* ZSTD_decompressBlock_internal() :10242* decompress block, starting at `src`,10243* into destination buffer `dst`.10244* @return : decompressed block size,10245* or an error code (which can be tested using ZSTD_isError())10246*/10247size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,10248void* dst, size_t dstCapacity,10249const void* src, size_t srcSize, const int frame);10250/* ZSTD_buildFSETable() :10252* generate FSE decoding table for one symbol (ll, ml or off)10253* this function must be called with valid parameters only10254* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)10255* in which case it cannot fail.10256* Internal use only.10257*/10258void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,10259const short* normalizedCounter, unsigned maxSymbolValue,10260const U32* baseValue, const U32* nbAdditionalBits,10261unsigned tableLog);10262/**** ended inlining zstd_decompress_block.h ****/10266/**** skipping file: ../legacy/zstd_legacy.h ****/10269#endif10270/*-*************************************************************10273* Context management10274***************************************************************/10275size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)10276{10277if (dctx==NULL) return 0; /* support sizeof NULL */10278return sizeof(*dctx)10279+ ZSTD_sizeof_DDict(dctx->ddictLocal)10280+ dctx->inBuffSize + dctx->outBuffSize;10281}10282{10288size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);10289/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */10290assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );10291return startingInputLength;10292}10293{10296dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */10297dctx->staticSize = 0;10298dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;10299dctx->ddict = NULL;10300dctx->ddictLocal = NULL;10301dctx->dictEnd = NULL;10302dctx->ddictIsCold = 0;10303dctx->dictUses = ZSTD_dont_use;10304dctx->inBuff = NULL;10305dctx->inBuffSize = 0;10306dctx->outBuffSize = 0;10307dctx->streamStage = zdss_init;10308dctx->legacyContext = NULL;10309dctx->previousLegacyVersion = 0;10310dctx->noForwardProgress = 0;10311dctx->oversizedDuration = 0;10312dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());10313dctx->outBufferMode = ZSTD_obm_buffered;10314#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION10315dctx->dictContentEndForFuzzing = NULL;10316#endif10317}10318{10321ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;10322}10331{10334if (!customMem.customAlloc ^ !customMem.customFree) return NULL;10335}10343{10346DEBUGLOG(3, "ZSTD_createDCtx");10347return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);10348}10349{10352ZSTD_freeDDict(dctx->ddictLocal);10353dctx->ddictLocal = NULL;10354dctx->ddict = NULL;10355dctx->dictUses = ZSTD_dont_use;10356}10357{10360if (dctx==NULL) return 0; /* support free on NULL */10361RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");10362{ ZSTD_customMem const cMem = dctx->customMem;10363ZSTD_clearDict(dctx);10364ZSTD_free(dctx->inBuff, cMem);10365dctx->inBuff = NULL;10366#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)10367if (dctx->legacyContext)10368ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);10369#endif10370ZSTD_free(dctx, cMem);10371return 0;10372}10373}10374/* no longer useful */10376void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)10377{10378size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);10379memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */10380}10381/*-*************************************************************10384* Frame header decoding10385***************************************************************/10386/*! ZSTD_isFrame() :10388* Tells if the content of `buffer` starts with a valid Frame Identifier.10389* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.10390* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.10391* Note 3 : Skippable Frame Identifiers are considered valid. */10392unsigned ZSTD_isFrame(const void* buffer, size_t size)10393{10394if (size < ZSTD_FRAMEIDSIZE) return 0;10395{ U32 const magic = MEM_readLE32(buffer);10396if (magic == ZSTD_MAGICNUMBER) return 1;10397if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;10398}10399#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)10400if (ZSTD_isLegacy(buffer, size)) return 1;10401#endif10402return 0;10403}10404/** ZSTD_frameHeaderSize_internal() :10406* srcSize must be large enough to reach header size fields.10407* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.10408* @return : size of the Frame Header10409* or an error code, which can be tested with ZSTD_isError() */10410static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)10411{10412size_t const minInputSize = ZSTD_startingInputLength(format);10413RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");10414}10424/** ZSTD_frameHeaderSize() :10426* srcSize must be >= ZSTD_frameHeaderSize_prefix.10427* @return : size of the Frame Header,10428* or an error code (if srcSize is too small) */10429size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)10430{10431return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);10432}10433/** ZSTD_getFrameHeader_advanced() :10436* decode Frame Header, or require larger `srcSize`.10437* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless10438* @return : 0, `zfhPtr` is correctly filled,10439* >0, `srcSize` is too small, value is wanted `srcSize` amount,10440* or an error code, which can be tested using ZSTD_isError() */10441size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)10442{10443const BYTE* ip = (const BYTE*)src;10444size_t const minInputSize = ZSTD_startingInputLength(format);10445}10516/** ZSTD_getFrameHeader() :10518* decode Frame Header, or require larger `srcSize`.10519* note : this function does not consume input, it only reads it.10520* @return : 0, `zfhPtr` is correctly filled,10521* >0, `srcSize` is too small, value is wanted `srcSize` amount,10522* or an error code, which can be tested using ZSTD_isError() */10523size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)10524{10525return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);10526}10527/** ZSTD_getFrameContentSize() :10530* compatible with legacy mode10531* @return : decompressed size of the single frame pointed to be `src` if known, otherwise10532* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined10533* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */10534unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)10535{10536#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)10537if (ZSTD_isLegacy(src, srcSize)) {10538unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);10539return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;10540}10541#endif10542{ ZSTD_frameHeader zfh;10543if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)10544return ZSTD_CONTENTSIZE_ERROR;10545if (zfh.frameType == ZSTD_skippableFrame) {10546return 0;10547} else {10548return zfh.frameContentSize;10549} }10550}10551{10554size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;10555U32 sizeU32;10556}10568/** ZSTD_findDecompressedSize() :10570* compatible with legacy mode10571* `srcSize` must be the exact length of some number of ZSTD compressed and/or10572* skippable frames10573* @return : decompressed size of the frames contained */10574unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)10575{10576unsigned long long totalDstSize = 0;10577}10614/** ZSTD_getDecompressedSize() :10616* compatible with legacy mode10617* @return : decompressed size if known, 0 otherwise10618note : 0 can mean any of the following :10619- frame content is empty10620- decompressed size field is not present in frame header10621- frame header unknown / not supported10622- frame header not complete (`srcSize` too small) */10623unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)10624{10625unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);10626ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);10627return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;10628}10629/** ZSTD_decodeFrameHeader() :10632* `headerSize` must be the size provided by ZSTD_frameHeaderSize().10633* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */10634static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)10635{10636size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);10637if (ZSTD_isError(result)) return result; /* invalid header */10638RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");10639#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION10640/* Skip the dictID check in fuzzing mode, because it makes the search10641* harder.10642*/10643RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),10644dictionary_wrong, "");10645#endif10646if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);10647return 0;10648}10649{10652ZSTD_frameSizeInfo frameSizeInfo;10653frameSizeInfo.compressedSize = ret;10654frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;10655return frameSizeInfo;10656}10657{10660ZSTD_frameSizeInfo frameSizeInfo;10661memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));10662}10723/** ZSTD_findFrameCompressedSize() :10725* compatible with legacy mode10726* `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame10727* `srcSize` must be at least as large as the frame contained10728* @return : the compressed size of the frame starting at `src` */10729size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)10730{10731ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);10732return frameSizeInfo.compressedSize;10733}10734/** ZSTD_decompressBound() :10736* compatible with legacy mode10737* `src` must point to the start of a ZSTD frame or a skippeable frame10738* `srcSize` must be at least as large as the frame contained10739* @return : the maximum decompressed size of the compressed source10740*/10741unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)10742{10743unsigned long long bound = 0;10744/* Iterate over each frame */10745while (srcSize > 0) {10746ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);10747size_t const compressedSize = frameSizeInfo.compressedSize;10748unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;10749if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)10750return ZSTD_CONTENTSIZE_ERROR;10751assert(srcSize >= compressedSize);10752src = (const BYTE*)src + compressedSize;10753srcSize -= compressedSize;10754bound += decompressedBound;10755}10756return bound;10757}10758/*-*************************************************************10761* Frame decoding10762***************************************************************/10763/** ZSTD_insertBlock() :10765* insert `src` block into `dctx` history. Useful to track uncompressed blocks. */10766size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)10767{10768DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);10769ZSTD_checkContinuity(dctx, blockStart);10770dctx->previousDstEnd = (const char*)blockStart + blockSize;10771return blockSize;10772}10773{10778DEBUGLOG(5, "ZSTD_copyRawBlock");10779if (dst == NULL) {10780if (srcSize == 0) return 0;10781RETURN_ERROR(dstBuffer_null, "");10782}10783RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");10784memcpy(dst, src, srcSize);10785return srcSize;10786}10787{10792if (dst == NULL) {10793if (regenSize == 0) return 0;10794RETURN_ERROR(dstBuffer_null, "");10795}10796RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");10797memset(dst, b, regenSize);10798return regenSize;10799}10800/*! ZSTD_decompressFrame() :10803* @dctx must be properly initialized10804* will update *srcPtr and *srcSizePtr,10805* to make *srcPtr progress by one frame. */10806static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,10807void* dst, size_t dstCapacity,10808const void** srcPtr, size_t *srcSizePtr)10809{10810const BYTE* ip = (const BYTE*)(*srcPtr);10811BYTE* const ostart = (BYTE* const)dst;10812BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;10813BYTE* op = ostart;10814size_t remainingSrcSize = *srcSizePtr;10815}10890{10897void* const dststart = dst;10898int moreThan1Frame = 0;10899* use this in all cases but ddict */10951FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");10952}10953ZSTD_checkContinuity(dctx, dst);10954}10983{10989return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);10990}10991{10995switch (dctx->dictUses) {10996default:10997assert(0 /* Impossible */);10998/* fall-through */10999case ZSTD_dont_use:11000ZSTD_clearDict(dctx);11001return NULL;11002case ZSTD_use_indefinitely:11003return dctx->ddict;11004case ZSTD_use_once:11005dctx->dictUses = ZSTD_dont_use;11006return dctx->ddict;11007}11008}11009{11012return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));11013}11014{11018#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)11019size_t regenSize;11020ZSTD_DCtx* const dctx = ZSTD_createDCtx();11021RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");11022regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);11023ZSTD_freeDCtx(dctx);11024return regenSize;11025#else /* stack mode */11026ZSTD_DCtx dctx;11027ZSTD_initDCtx_internal(&dctx);11028return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);11029#endif11030}11031/*-**************************************11034* Advanced Streaming Decompression API11035* Bufferless and synchronous11036****************************************/11037size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }11038/**11040* Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,11041* we allow taking a partial block as the input. Currently only raw uncompressed blocks can11042* be streamed.11043*11044* For blocks that can be streamed, this allows us to reduce the latency until we produce11045* output, and avoid copying the input.11046*11047* @param inputSize - The total amount of input that the caller currently has.11048*/11049static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {11050if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock))11051return dctx->expected;11052if (dctx->bType != bt_raw)11053return dctx->expected;11054return MIN(MAX(inputSize, 1), dctx->expected);11055}11056}11078/** ZSTD_decompressContinue() :11082* srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())11083* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)11084* or an error code, which can be tested using ZSTD_isError() */11085size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)11086{11087DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);11088/* Sanity check */11089RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");11090if (dstCapacity) ZSTD_checkContinuity(dctx, dst);11091}11234{11238dctx->dictEnd = dctx->previousDstEnd;11239dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));11240dctx->prefixStart = dict;11241dctx->previousDstEnd = (const char*)dict + dictSize;11242#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION11243dctx->dictContentBeginForFuzzing = dctx->prefixStart;11244dctx->dictContentEndForFuzzing = dctx->previousDstEnd;11245#endif11246return 0;11247}11248/*! ZSTD_loadDEntropy() :11250* dict : must point at beginning of a valid zstd dictionary.11251* @return : size of entropy tables read */11252size_t11253ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,11254const void* const dict, size_t const dictSize)11255{11256const BYTE* dictPtr = (const BYTE*)dict;11257const BYTE* const dictEnd = dictPtr + dictSize;11258}11333{11336if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);11337{ U32 const magic = MEM_readLE32(dict);11338if (magic != ZSTD_MAGIC_DICTIONARY) {11339return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */11340} }11341dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);11342}11354{11357assert(dctx != NULL);11358dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */11359dctx->stage = ZSTDds_getFrameHeaderSize;11360dctx->decodedSize = 0;11361dctx->previousDstEnd = NULL;11362dctx->prefixStart = NULL;11363dctx->virtualStart = NULL;11364dctx->dictEnd = NULL;11365dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */11366dctx->litEntropy = dctx->fseEntropy = 0;11367dctx->dictID = 0;11368dctx->bType = bt_reserved;11369ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));11370memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */11371dctx->LLTptr = dctx->entropy.LLTable;11372dctx->MLTptr = dctx->entropy.MLTable;11373dctx->OFTptr = dctx->entropy.OFTable;11374dctx->HUFptr = dctx->entropy.hufTable;11375return 0;11376}11377{11380FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");11381if (dict && dictSize)11382RETURN_ERROR_IF(11383ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),11384dictionary_corrupted, "");11385return 0;11386}11387/* ====== ZSTD_DDict ====== */11390{11393DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");11394assert(dctx != NULL);11395if (ddict) {11396const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);11397size_t const dictSize = ZSTD_DDict_dictSize(ddict);11398const void* const dictEnd = dictStart + dictSize;11399dctx->ddictIsCold = (dctx->dictEnd != dictEnd);11400DEBUGLOG(4, "DDict is %s",11401dctx->ddictIsCold ? "~cold~" : "hot!");11402}11403FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");11404if (ddict) { /* NULL ddict is equivalent to no dictionary */11405ZSTD_copyDDictParameters(dctx, ddict);11406}11407return 0;11408}11409/*! ZSTD_getDictID_fromDict() :11411* Provides the dictID stored within dictionary.11412* if @return == 0, the dictionary is not conformant with Zstandard specification.11413* It can still be loaded, but as a content-only dictionary. */11414unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)11415{11416if (dictSize < 8) return 0;11417if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;11418return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);11419}11420/*! ZSTD_getDictID_fromFrame() :11422* Provides the dictID required to decompress frame stored within `src`.11423* If @return == 0, the dictID could not be decoded.11424* This could for one of the following reasons :11425* - The frame does not require a dictionary (most common case).11426* - The frame was built with dictID intentionally removed.11427* Needed dictionary is a hidden information.11428* Note : this use case also happens when using a non-conformant dictionary.11429* - `srcSize` is too small, and as a result, frame header could not be decoded.11430* Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.11431* - This is not a Zstandard frame.11432* When identifying the exact failure cause, it's possible to use11433* ZSTD_getFrameHeader(), which will provide a more precise error code. */11434unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)11435{11436ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };11437size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);11438if (ZSTD_isError(hError)) return 0;11439return zfp.dictID;11440}11441/*! ZSTD_decompress_usingDDict() :11444* Decompression using a pre-digested Dictionary11445* Use dictionary without significant overhead. */11446size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,11447void* dst, size_t dstCapacity,11448const void* src, size_t srcSize,11449const ZSTD_DDict* ddict)11450{11451/* pass content and size in case legacy frames are encountered */11452return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,11453NULL, 0,11454ddict);11455}11456/*=====================================11459* Streaming decompression11460*====================================*/11461{11464DEBUGLOG(3, "ZSTD_createDStream");11465return ZSTD_createDStream_advanced(ZSTD_defaultCMem);11466}11467{11470return ZSTD_initStaticDCtx(workspace, workspaceSize);11471}11472{11475return ZSTD_createDCtx_advanced(customMem);11476}11477{11480return ZSTD_freeDCtx(zds);11481}11482/* *** Initialization *** */11485{11494RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");11495ZSTD_clearDict(dctx);11496if (dict && dictSize != 0) {11497dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);11498RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");11499dctx->ddict = dctx->ddictLocal;11500dctx->dictUses = ZSTD_use_indefinitely;11501}11502return 0;11503}11504{11507return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);11508}11509{11512return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);11513}11514{11517FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");11518dctx->dictUses = ZSTD_use_once;11519return 0;11520}11521{11524return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);11525}11526/* ZSTD_initDStream_usingDict() :11529* return : expected size, aka ZSTD_startingInputLength().11530* this function cannot fail */11531size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)11532{11533DEBUGLOG(4, "ZSTD_initDStream_usingDict");11534FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");11535FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");11536return ZSTD_startingInputLength(zds->format);11537}11538/* note : this variant can't fail */11540size_t ZSTD_initDStream(ZSTD_DStream* zds)11541{11542DEBUGLOG(4, "ZSTD_initDStream");11543return ZSTD_initDStream_usingDDict(zds, NULL);11544}11545/* ZSTD_initDStream_usingDDict() :11547* ddict will just be referenced, and must outlive decompression session11548* this function cannot fail */11549size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)11550{11551FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");11552FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");11553return ZSTD_startingInputLength(dctx->format);11554}11555/* ZSTD_resetDStream() :11557* return : expected size, aka ZSTD_startingInputLength().11558* this function cannot fail */11559size_t ZSTD_resetDStream(ZSTD_DStream* dctx)11560{11561FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");11562return ZSTD_startingInputLength(dctx->format);11563}11564{11568RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");11569ZSTD_clearDict(dctx);11570if (ddict) {11571dctx->ddict = ddict;11572dctx->dictUses = ZSTD_use_indefinitely;11573}11574return 0;11575}11576/* ZSTD_DCtx_setMaxWindowSize() :11578* note : no direct equivalence in ZSTD_DCtx_setParameter,11579* since this version sets windowSize, and the other sets windowLog */11580size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)11581{11582ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);11583size_t const min = (size_t)1 << bounds.lowerBound;11584size_t const max = (size_t)1 << bounds.upperBound;11585RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");11586RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");11587RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");11588dctx->maxWindowSize = maxWindowSize;11589return 0;11590}11591{11594return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format);11595}11596{11599ZSTD_bounds bounds = { 0, 0, 0 };11600switch(dParam) {11601case ZSTD_d_windowLogMax:11602bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;11603bounds.upperBound = ZSTD_WINDOWLOG_MAX;11604return bounds;11605case ZSTD_d_format:11606bounds.lowerBound = (int)ZSTD_f_zstd1;11607bounds.upperBound = (int)ZSTD_f_zstd1_magicless;11608ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);11609return bounds;11610case ZSTD_d_stableOutBuffer:11611bounds.lowerBound = (int)ZSTD_obm_buffered;11612bounds.upperBound = (int)ZSTD_obm_stable;11613return bounds;11614default:;11615}11616bounds.error = ERROR(parameter_unsupported);11617return bounds;11618}11619/* ZSTD_dParam_withinBounds:11621* @return 1 if value is within dParam bounds,11622* 0 otherwise */11623static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)11624{11625ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);11626if (ZSTD_isError(bounds.error)) return 0;11627if (value < bounds.lowerBound) return 0;11628if (value > bounds.upperBound) return 0;11629return 1;11630}11631}11635{11638RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");11639switch(dParam) {11640case ZSTD_d_windowLogMax:11641if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;11642CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);11643dctx->maxWindowSize = ((size_t)1) << value;11644return 0;11645case ZSTD_d_format:11646CHECK_DBOUNDS(ZSTD_d_format, value);11647dctx->format = (ZSTD_format_e)value;11648return 0;11649case ZSTD_d_stableOutBuffer:11650CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);11651dctx->outBufferMode = (ZSTD_outBufferMode_e)value;11652return 0;11653default:;11654}11655RETURN_ERROR(parameter_unsupported, "");11656}11657{11660if ( (reset == ZSTD_reset_session_only)11661|| (reset == ZSTD_reset_session_and_parameters) ) {11662dctx->streamStage = zdss_init;11663dctx->noForwardProgress = 0;11664}11665if ( (reset == ZSTD_reset_parameters)11666|| (reset == ZSTD_reset_session_and_parameters) ) {11667RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");11668ZSTD_clearDict(dctx);11669dctx->format = ZSTD_f_zstd1;11670dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;11671}11672return 0;11673}11674{11678return ZSTD_sizeof_DCtx(dctx);11679}11680{11683size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);11684unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);11685unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);11686size_t const minRBSize = (size_t) neededSize;11687RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,11688frameParameter_windowTooLarge, "");11689return minRBSize;11690}11691{11694size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);11695size_t const inBuffSize = blockSize; /* no block can be larger */11696size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);11697return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;11698}11699{11702U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */11703ZSTD_frameHeader zfh;11704size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);11705if (ZSTD_isError(err)) return err;11706RETURN_ERROR_IF(err>0, srcSize_wrong, "");11707RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,11708frameParameter_windowTooLarge, "");11709return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);11710}11711/* ***** Decompression ***** */11714{11717return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;11718}11719{11722if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))11723zds->oversizedDuration++;11724else11725zds->oversizedDuration = 0;11726}11727{11730return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;11731}11732/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */11734static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)11735{11736ZSTD_outBuffer const expect = zds->expectedOutBuffer;11737/* No requirement when ZSTD_obm_stable is not enabled. */11738if (zds->outBufferMode != ZSTD_obm_stable)11739return 0;11740/* Any buffer is allowed in zdss_init, this must be the same for every other call until11741* the context is reset.11742*/11743if (zds->streamStage == zdss_init)11744return 0;11745/* The buffer must match our expectation exactly. */11746if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)11747return 0;11748RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");11749}11750/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()11752* and updates the stage and the output buffer state. This call is extracted so it can be11753* used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.11754* NOTE: You must break after calling this function since the streamStage is modified.11755*/11756static size_t ZSTD_decompressContinueStream(11757ZSTD_DStream* zds, char** op, char* oend,11758void const* src, size_t srcSize) {11759int const isSkipFrame = ZSTD_isSkipFrame(zds);11760if (zds->outBufferMode == ZSTD_obm_buffered) {11761size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;11762size_t const decodedSize = ZSTD_decompressContinue(zds,11763zds->outBuff + zds->outStart, dstSize, src, srcSize);11764FORWARD_IF_ERROR(decodedSize, "");11765if (!decodedSize && !isSkipFrame) {11766zds->streamStage = zdss_read;11767} else {11768zds->outEnd = zds->outStart + decodedSize;11769zds->streamStage = zdss_flush;11770}11771} else {11772/* Write directly into the output buffer */11773size_t const dstSize = isSkipFrame ? 0 : oend - *op;11774size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);11775FORWARD_IF_ERROR(decodedSize, "");11776*op += decodedSize;11777/* Flushing is not needed. */11778zds->streamStage = zdss_read;11779assert(*op <= oend);11780assert(zds->outBufferMode == ZSTD_obm_stable);11781}11782return 0;11783}11784{11787const char* const src = (const char*)input->src;11788const char* const istart = input->pos != 0 ? src + input->pos : src;11789const char* const iend = input->size != 0 ? src + input->size : src;11790const char* ip = istart;11791char* const dst = (char*)output->dst;11792char* const ostart = output->pos != 0 ? dst + output->pos : dst;11793char* const oend = output->size != 0 ? dst + output->size : dst;11794char* op = ostart;11795U32 someMoreWork = 1;11796}12069{12075ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };12076ZSTD_inBuffer input = { src, srcSize, *srcPos };12077/* ZSTD_compress_generic() will check validity of dstPos and srcPos */12078size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);12079*dstPos = output.pos;12080*srcPos = input.pos;12081return cErr;12082}12083/**** ended inlining decompress/zstd_decompress.c ****/12084/**** start inlining decompress/zstd_decompress_block.c ****/12085/*12086* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.12087* All rights reserved.12088*12089* This source code is licensed under both the BSD-style license (found in the12090* LICENSE file in the root directory of this source tree) and the GPLv2 (found12091* in the COPYING file in the root directory of this source tree).12092* You may select, at your option, one of the above-listed licenses.12093*/12094/* zstd_decompress_block :12096* this module takes care of decompressing _compressed_ block */12097/*-*******************************************************12099* Dependencies12100*********************************************************/12101/**** skipping file: ../common/compiler.h ****/12102/**** skipping file: ../common/cpu.h ****/12103/**** skipping file: ../common/mem.h ****/12104#define FSE_STATIC_LINKING_ONLY12105/**** skipping file: ../common/fse.h ****/12106#define HUF_STATIC_LINKING_ONLY12107/**** skipping file: ../common/huf.h ****/12108/**** skipping file: ../common/zstd_internal.h ****/12109/**** skipping file: zstd_decompress_internal.h ****/12110/**** skipping file: zstd_ddict.h ****/12111/**** skipping file: zstd_decompress_block.h ****/12112/*_*******************************************************12114* Macros12115**********************************************************/12116/* These two optional macros force the use one way or another of the two12118* ZSTD_decompressSequences implementations. You can't force in both directions12119* at the same time.12120*/12121#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \12122defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)12123#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"12124#endif12125/*_*******************************************************12128* Memory operations12129**********************************************************/12130static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }12131/*-*************************************************************12134* Block decoding12135***************************************************************/12136/*! ZSTD_getcBlockSize() :12138* Provides the size of compressed block from block header `src` */12139size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,12140blockProperties_t* bpPtr)12141{12142RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");12143}12154/* Hidden declaration for fullbench */12157size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,12158const void* src, size_t srcSize);12159/*! ZSTD_decodeLiteralsBlock() :12160* @return : nb of bytes read from src (< srcSize )12161* note : symbol not declared but exposed for fullbench */12162size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,12163const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */12164{12165DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");12166RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");12167}12319/* Default FSE distribution tables.12321* These are pre-calculated FSE decoding tables using default distributions as defined in specification :12322* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions12323* They were generated programmatically with following method :12324* - start from default distributions, present in /lib/common/zstd_internal.h12325* - generate tables normally, using ZSTD_buildFSETable()12326* - printout the content of tables12327* - pretify output, report below, test with fuzzer to ensure it's correct */12328/* Default FSE distribution table for Literal Lengths */12330static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {12331{ 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */12332/* nextState, nbAddBits, nbBits, baseVal */12333{ 0, 0, 4, 0}, { 16, 0, 4, 0},12334{ 32, 0, 5, 1}, { 0, 0, 5, 3},12335{ 0, 0, 5, 4}, { 0, 0, 5, 6},12336{ 0, 0, 5, 7}, { 0, 0, 5, 9},12337{ 0, 0, 5, 10}, { 0, 0, 5, 12},12338{ 0, 0, 6, 14}, { 0, 1, 5, 16},12339{ 0, 1, 5, 20}, { 0, 1, 5, 22},12340{ 0, 2, 5, 28}, { 0, 3, 5, 32},12341{ 0, 4, 5, 48}, { 32, 6, 5, 64},12342{ 0, 7, 5, 128}, { 0, 8, 6, 256},12343{ 0, 10, 6, 1024}, { 0, 12, 6, 4096},12344{ 32, 0, 4, 0}, { 0, 0, 4, 1},12345{ 0, 0, 5, 2}, { 32, 0, 5, 4},12346{ 0, 0, 5, 5}, { 32, 0, 5, 7},12347{ 0, 0, 5, 8}, { 32, 0, 5, 10},12348{ 0, 0, 5, 11}, { 0, 0, 6, 13},12349{ 32, 1, 5, 16}, { 0, 1, 5, 18},12350{ 32, 1, 5, 22}, { 0, 2, 5, 24},12351{ 32, 3, 5, 32}, { 0, 3, 5, 40},12352{ 0, 6, 4, 64}, { 16, 6, 4, 64},12353{ 32, 7, 5, 128}, { 0, 9, 6, 512},12354{ 0, 11, 6, 2048}, { 48, 0, 4, 0},12355{ 16, 0, 4, 1}, { 32, 0, 5, 2},12356{ 32, 0, 5, 3}, { 32, 0, 5, 5},12357{ 32, 0, 5, 6}, { 32, 0, 5, 8},12358{ 32, 0, 5, 9}, { 32, 0, 5, 11},12359{ 32, 0, 5, 12}, { 0, 0, 6, 15},12360{ 32, 1, 5, 18}, { 32, 1, 5, 20},12361{ 32, 2, 5, 24}, { 32, 2, 5, 28},12362{ 32, 3, 5, 40}, { 32, 4, 5, 48},12363{ 0, 16, 6,65536}, { 0, 15, 6,32768},12364{ 0, 14, 6,16384}, { 0, 13, 6, 8192},12365}; /* LL_defaultDTable */12366/* Default FSE distribution table for Offset Codes */12368static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {12369{ 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */12370/* nextState, nbAddBits, nbBits, baseVal */12371{ 0, 0, 5, 0}, { 0, 6, 4, 61},12372{ 0, 9, 5, 509}, { 0, 15, 5,32765},12373{ 0, 21, 5,2097149}, { 0, 3, 5, 5},12374{ 0, 7, 4, 125}, { 0, 12, 5, 4093},12375{ 0, 18, 5,262141}, { 0, 23, 5,8388605},12376{ 0, 5, 5, 29}, { 0, 8, 4, 253},12377{ 0, 14, 5,16381}, { 0, 20, 5,1048573},12378{ 0, 2, 5, 1}, { 16, 7, 4, 125},12379{ 0, 11, 5, 2045}, { 0, 17, 5,131069},12380{ 0, 22, 5,4194301}, { 0, 4, 5, 13},12381{ 16, 8, 4, 253}, { 0, 13, 5, 8189},12382{ 0, 19, 5,524285}, { 0, 1, 5, 1},12383{ 16, 6, 4, 61}, { 0, 10, 5, 1021},12384{ 0, 16, 5,65533}, { 0, 28, 5,268435453},12385{ 0, 27, 5,134217725}, { 0, 26, 5,67108861},12386{ 0, 25, 5,33554429}, { 0, 24, 5,16777213},12387}; /* OF_defaultDTable */12388/* Default FSE distribution table for Match Lengths */12391static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {12392{ 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */12393/* nextState, nbAddBits, nbBits, baseVal */12394{ 0, 0, 6, 3}, { 0, 0, 4, 4},12395{ 32, 0, 5, 5}, { 0, 0, 5, 6},12396{ 0, 0, 5, 8}, { 0, 0, 5, 9},12397{ 0, 0, 5, 11}, { 0, 0, 6, 13},12398{ 0, 0, 6, 16}, { 0, 0, 6, 19},12399{ 0, 0, 6, 22}, { 0, 0, 6, 25},12400{ 0, 0, 6, 28}, { 0, 0, 6, 31},12401{ 0, 0, 6, 34}, { 0, 1, 6, 37},12402{ 0, 1, 6, 41}, { 0, 2, 6, 47},12403{ 0, 3, 6, 59}, { 0, 4, 6, 83},12404{ 0, 7, 6, 131}, { 0, 9, 6, 515},12405{ 16, 0, 4, 4}, { 0, 0, 4, 5},12406{ 32, 0, 5, 6}, { 0, 0, 5, 7},12407{ 32, 0, 5, 9}, { 0, 0, 5, 10},12408{ 0, 0, 6, 12}, { 0, 0, 6, 15},12409{ 0, 0, 6, 18}, { 0, 0, 6, 21},12410{ 0, 0, 6, 24}, { 0, 0, 6, 27},12411{ 0, 0, 6, 30}, { 0, 0, 6, 33},12412{ 0, 1, 6, 35}, { 0, 1, 6, 39},12413{ 0, 2, 6, 43}, { 0, 3, 6, 51},12414{ 0, 4, 6, 67}, { 0, 5, 6, 99},12415{ 0, 8, 6, 259}, { 32, 0, 4, 4},12416{ 48, 0, 4, 4}, { 16, 0, 4, 5},12417{ 32, 0, 5, 7}, { 32, 0, 5, 8},12418{ 32, 0, 5, 10}, { 32, 0, 5, 11},12419{ 0, 0, 6, 14}, { 0, 0, 6, 17},12420{ 0, 0, 6, 20}, { 0, 0, 6, 23},12421{ 0, 0, 6, 26}, { 0, 0, 6, 29},12422{ 0, 0, 6, 32}, { 0, 16, 6,65539},12423{ 0, 15, 6,32771}, { 0, 14, 6,16387},12424{ 0, 13, 6, 8195}, { 0, 12, 6, 4099},12425{ 0, 11, 6, 2051}, { 0, 10, 6, 1027},12426}; /* ML_defaultDTable */12427{12431void* ptr = dt;12432ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;12433ZSTD_seqSymbol* const cell = dt + 1;12434}12444/* ZSTD_buildFSETable() :12447* generate FSE decoding table for one symbol (ll, ml or off)12448* cannot fail if input is valid =>12449* all inputs are presumed validated at this stage */12450void12451ZSTD_buildFSETable(ZSTD_seqSymbol* dt,12452const short* normalizedCounter, unsigned maxSymbolValue,12453const U32* baseValue, const U32* nbAdditionalBits,12454unsigned tableLog)12455{12456ZSTD_seqSymbol* const tableDecode = dt+1;12457U16 symbolNext[MaxSeq+1];12458}12511/*! ZSTD_buildSeqTable() :12514* @return : nb bytes read from src,12515* or an error code if it fails */12516static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,12517symbolEncodingType_e type, unsigned max, U32 maxLog,12518const void* src, size_t srcSize,12519const U32* baseValue, const U32* nbAdditionalBits,12520const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,12521int ddictIsCold, int nbSeq)12522{12523switch(type)12524{12525case set_rle :12526RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");12527RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");12528{ U32 const symbol = *(const BYTE*)src;12529U32 const baseline = baseValue[symbol];12530U32 const nbBits = nbAdditionalBits[symbol];12531ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);12532}12533*DTablePtr = DTableSpace;12534return 1;12535case set_basic :12536*DTablePtr = defaultTable;12537return 0;12538case set_repeat:12539RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");12540/* prefetch FSE table if used */12541if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {12542const void* const pStart = *DTablePtr;12543size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));12544PREFETCH_AREA(pStart, pSize);12545}12546return 0;12547case set_compressed :12548{ unsigned tableLog;12549S16 norm[MaxSeq+1];12550size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);12551RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");12552RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");12553ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog);12554*DTablePtr = DTableSpace;12555return headerSize;12556}12557default :12558assert(0);12559RETURN_ERROR(GENERIC, "impossible");12560}12561}12562{12566const BYTE* const istart = (const BYTE* const)src;12567const BYTE* const iend = istart + srcSize;12568const BYTE* ip = istart;12569int nbSeq;12570DEBUGLOG(5, "ZSTD_decodeSeqHeaders");12571}12634/*! ZSTD_overlapCopy8() :12660* Copies 8 bytes from ip to op and updates op and ip where ip <= op.12661* If the offset is < 8 then the offset is spread to at least 8 bytes.12662*12663* Precondition: *ip <= *op12664* Postcondition: *op - *op >= 812665*/12666HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {12667assert(*ip <= *op);12668if (offset < 8) {12669/* close range match, overlap */12670static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */12671static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */12672int const sub2 = dec64table[offset];12673(*op)[0] = (*ip)[0];12674(*op)[1] = (*ip)[1];12675(*op)[2] = (*ip)[2];12676(*op)[3] = (*ip)[3];12677*ip += dec32table[offset];12678ZSTD_copy4(*op+4, *ip);12679*ip -= sub2;12680} else {12681ZSTD_copy8(*op, *ip);12682}12683*ip += 8;12684*op += 8;12685assert(*op - *ip >= 8);12686}12687/*! ZSTD_safecopy() :12689* Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer12690* and write up to 16 bytes past oend_w (op >= oend_w is allowed).12691* This function is only called in the uncommon case where the sequence is near the end of the block. It12692* should be fast for a single long sequence, but can be slow for several short sequences.12693*12694* @param ovtype controls the overlap detection12695* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.12696* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.12697* The src buffer must be before the dst buffer.12698*/12699static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {12700ptrdiff_t const diff = op - ip;12701BYTE* const oend = op + length;12702}12734/* ZSTD_execSequenceEnd():12736* This version handles cases that are near the end of the output buffer. It requires12737* more careful checks to make sure there is no overflow. By separating out these hard12738* and unlikely cases, we can speed up the common cases.12739*12740* NOTE: This function needs to be fast for a single long sequence, but doesn't need12741* to be optimized for many small sequences, since those fall into ZSTD_execSequence().12742*/12743FORCE_NOINLINE12744size_t ZSTD_execSequenceEnd(BYTE* op,12745BYTE* const oend, seq_t sequence,12746const BYTE** litPtr, const BYTE* const litLimit,12747const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)12748{12749BYTE* const oLitEnd = op + sequence.litLength;12750size_t const sequenceLength = sequence.litLength + sequence.matchLength;12751const BYTE* const iLitEnd = *litPtr + sequence.litLength;12752const BYTE* match = oLitEnd - sequence.offset;12753BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;12754}12785HINT_INLINE12787size_t ZSTD_execSequence(BYTE* op,12788BYTE* const oend, seq_t sequence,12789const BYTE** litPtr, const BYTE* const litLimit,12790const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)12791{12792BYTE* const oLitEnd = op + sequence.litLength;12793size_t const sequenceLength = sequence.litLength + sequence.matchLength;12794BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */12795BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */12796const BYTE* const iLitEnd = *litPtr + sequence.litLength;12797const BYTE* match = oLitEnd - sequence.offset;12798* - Read beyond end of literals12803* - Match end is within WILDCOPY_OVERLIMIT of oend12804* - 32-bit mode and the match length overflows12805*/12806if (UNLIKELY(12807iLitEnd > litLimit ||12808oMatchEnd > oend_w ||12809(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))12810return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);12811* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.12822* We likely don't need the full 32-byte wildcopy.12823*/12824assert(WILDCOPY_OVERLENGTH >= 16);12825ZSTD_copy16(op, (*litPtr));12826if (UNLIKELY(sequence.litLength > 16)) {12827ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);12828}12829op = oLitEnd;12830*litPtr = iLitEnd; /* update for next sequence */12831* without overlap checking.12856*/12857if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {12858/* We bet on a full wildcopy for matches, since we expect matches to be12859* longer than literals (in general). In silesia, ~10% of matches are longer12860* than 16 bytes.12861*/12862ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);12863return sequenceLength;12864}12865assert(sequence.offset < WILDCOPY_VECLEN);12866}12877{12881const void* ptr = dt;12882const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;12883DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);12884DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",12885(U32)DStatePtr->state, DTableH->tableLog);12886BIT_reloadDStream(bitD);12887DStatePtr->table = dt + 1;12888}12889{12893ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state];12894U32 const nbBits = DInfo.nbBits;12895size_t const lowBits = BIT_readBits(bitD, nbBits);12896DStatePtr->state = DInfo.nextState + lowBits;12897}12898{12902U32 const nbBits = DInfo.nbBits;12903size_t const lowBits = BIT_readBits(bitD, nbBits);12904DStatePtr->state = DInfo.nextState + lowBits;12905}12906/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum12908* offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)12909* bits before reloading. This value is the maximum number of bytes we read12910* after reloading when we are decoding long offsets.12911*/12912#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \12913(ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \12914? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \12915: 0)12916{12923seq_t seq;12924ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state];12925ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state];12926ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state];12927U32 const llBase = llDInfo.baseValue;12928U32 const mlBase = mlDInfo.baseValue;12929U32 const ofBase = ofDInfo.baseValue;12930BYTE const llBits = llDInfo.nbAdditionalBits;12931BYTE const mlBits = mlDInfo.nbAdditionalBits;12932BYTE const ofBits = ofDInfo.nbAdditionalBits;12933BYTE const totalBits = llBits+mlBits+ofBits;12934* No consequence though : no memory access will occur, offset is only used for prefetching */13001seqState->pos = pos + seq.matchLength;13002}13003* gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo().13006* clang-9.2.0 does 7% worse with ZSTD_updateFseState().13007* Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the13008* better option, so it is the default for other compilers. But, if you13009* measure that it is worse, please put up a pull request.13010*/13011{13012#if defined(__GNUC__) && !defined(__clang__)13013const int kUseUpdateFseState = 1;13014#else13015const int kUseUpdateFseState = 0;13016#endif13017if (kUseUpdateFseState) {13018ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */13019ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */13020if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */13021ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */13022} else {13023ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */13024ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */13025if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */13026ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */13027}13028}13029}13032{13036size_t const windowSize = dctx->fParams.windowSize;13037/* No dictionary used. */13038if (dctx->dictContentEndForFuzzing == NULL) return 0;13039/* Dictionary is our prefix. */13040if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;13041/* Dictionary is not our ext-dict. */13042if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;13043/* Dictionary is not within our window size. */13044if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;13045/* Dictionary is active. */13046return 1;13047}13048{13055#if DEBUGLEVEL >= 113056size_t const windowSize = dctx->fParams.windowSize;13057size_t const sequenceSize = seq.litLength + seq.matchLength;13058BYTE const* const oLitEnd = op + seq.litLength;13059DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",13060(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);13061assert(op <= oend);13062assert((size_t)(oend - op) >= sequenceSize);13063assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX);13064if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {13065size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);13066/* Offset must be within the dictionary. */13067assert(seq.offset <= (size_t)(oLitEnd - virtualStart));13068assert(seq.offset <= windowSize + dictSize);13069} else {13070/* Offset must be within our window. */13071assert(seq.offset <= windowSize);13072}13073#else13074(void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;13075#endif13076}13077#endif13078DONT_VECTORIZE13082ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,13083void* dst, size_t maxDstSize,13084const void* seqStart, size_t seqSize, int nbSeq,13085const ZSTD_longOffset_e isLongOffset,13086const int frame)13087{13088const BYTE* ip = (const BYTE*)seqStart;13089const BYTE* const iend = ip + seqSize;13090BYTE* const ostart = (BYTE* const)dst;13091BYTE* const oend = ostart + maxDstSize;13092BYTE* op = ostart;13093const BYTE* litPtr = dctx->litPtr;13094const BYTE* const litEnd = litPtr + dctx->litSize;13095const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);13096const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);13097const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);13098DEBUGLOG(5, "ZSTD_decompressSequences_body");13099(void)frame;13100*13123* zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression13124* speed swings based on the alignment of the decompression loop. This13125* performance swing is caused by parts of the decompression loop falling13126* out of the DSB. The entire decompression loop should fit in the DSB,13127* when it can't we get much worse performance. You can measure if you've13128* hit the good case or the bad case with this perf command for some13129* compressed file test.zst:13130*13131* perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \13132* -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst13133*13134* If you see most cycles served out of the MITE you've hit the bad case.13135* If you see most cycles served out of the DSB you've hit the good case.13136* If it is pretty even then you may be in an okay case.13137*13138* I've been able to reproduce this issue on the following CPUs:13139* - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i913140* Use Instruments->Counters to get DSB/MITE cycles.13141* I never got performance swings, but I was able to13142* go from the good case of mostly DSB to half of the13143* cycles served from MITE.13144* - Coffeelake: Intel i9-9900k13145*13146* I haven't been able to reproduce the instability or DSB misses on any13147* of the following CPUS:13148* - Haswell13149* - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH13150* - Skylake13151*13152* If you are seeing performance stability this script can help test.13153* It tests on 4 commits in zstd where I saw performance change.13154*13155* https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f413156*/13157__asm__(".p2align 5");13158__asm__("nop");13159__asm__(".p2align 4");13160#endif13161for ( ; ; ) {13162seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch);13163size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);13164#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)13165assert(!ZSTD_isError(oneSeqSize));13166if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);13167#endif13168DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);13169BIT_reloadDStream(&(seqState.DStream));13170/* gcc and clang both don't like early returns in this loop.13171* gcc doesn't like early breaks either.13172* Instead save an error and report it at the end.13173* When there is an error, don't increment op, so we don't13174* overwrite.13175*/13176if (UNLIKELY(ZSTD_isError(oneSeqSize))) error = oneSeqSize;13177else op += oneSeqSize;13178if (UNLIKELY(!--nbSeq)) break;13179}13180}13201{13209return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13210}13211#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */13212{13222const BYTE* ip = (const BYTE*)seqStart;13223const BYTE* const iend = ip + seqSize;13224BYTE* const ostart = (BYTE* const)dst;13225BYTE* const oend = ostart + maxDstSize;13226BYTE* op = ostart;13227const BYTE* litPtr = dctx->litPtr;13228const BYTE* const litEnd = litPtr + dctx->litSize;13229const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);13230const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);13231const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);13232(void)frame;13233}13306{13314return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13315}13316#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */13317DONT_VECTORIZE13325ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,13326void* dst, size_t maxDstSize,13327const void* seqStart, size_t seqSize, int nbSeq,13328const ZSTD_longOffset_e isLongOffset,13329const int frame)13330{13331return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13332}13333#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */13334{13343return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13344}13345#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */13346{13363DEBUGLOG(5, "ZSTD_decompressSequences");13364#if DYNAMIC_BMI213365if (dctx->bmi2) {13366return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13367}13368#endif13369return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13370}13371#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */13372/* ZSTD_decompressSequencesLong() :13376* decompression function triggered when a minimum share of offsets is considered "long",13377* aka out of cache.13378* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".13379* This function will try to mitigate main memory latency through the use of prefetching */13380static size_t13381ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,13382void* dst, size_t maxDstSize,13383const void* seqStart, size_t seqSize, int nbSeq,13384const ZSTD_longOffset_e isLongOffset,13385const int frame)13386{13387DEBUGLOG(5, "ZSTD_decompressSequencesLong");13388#if DYNAMIC_BMI213389if (dctx->bmi2) {13390return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13391}13392#endif13393return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);13394}13395#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */13396/* ZSTD_getLongOffsetsShare() :13402* condition : offTable must be valid13403* @return : "share" of long offsets (arbitrarily defined as > (1<<23))13404* compared to maximum possible of (1<<OffFSELog) */13405static unsigned13406ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)13407{13408const void* ptr = offTable;13409U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;13410const ZSTD_seqSymbol* table = offTable + 1;13411U32 const max = 1 << tableLog;13412U32 u, total = 0;13413DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);13414}13425#endif13426size_t13428ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,13429void* dst, size_t dstCapacity,13430const void* src, size_t srcSize, const int frame)13431{ /* blockType == blockCompressed */13432const BYTE* ip = (const BYTE*)src;13433/* isLongOffset must be true if there are long offsets.13434* Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.13435* We don't expect that to be the case in 64-bit mode.13436* In block mode, window size is not known, so we have to be conservative.13437* (note: but it could be evaluated from current-lowLimit)13438*/13439ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));13440DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);13441* we use. If neither is defined, we do some inspection and dispatch at13456* runtime.13457*/13458#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \13459!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)13460int usePrefetchDecoder = dctx->ddictIsCold;13461#endif13462int nbSeq;13463size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);13464if (ZSTD_isError(seqHSize)) return seqHSize;13465ip += seqHSize;13466srcSize -= seqHSize;13467}13497{13501if (dst != dctx->previousDstEnd) { /* not contiguous */13502dctx->dictEnd = dctx->previousDstEnd;13503dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));13504dctx->prefixStart = dst;13505dctx->previousDstEnd = dst;13506}13507}13508{13514size_t dSize;13515ZSTD_checkContinuity(dctx, dst);13516dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0);13517dctx->previousDstEnd = (char*)dst + dSize;13518return dSize;13519}13520/**** ended inlining decompress/zstd_decompress_block.c ****/13521