src/port/pg_crc32c_armv8.c - master - githubmirror/postgres

src/port/pg_crc32c_armv8.c
217 строк5 KB
Peter Eisentraut
Use C11 alignas instead of pg_attribute_aligned
01 июл 2026, 09:36
01 июл 2026, 09:368061bfd
/*-------------------------------------------------------------------------
 *
 * pg_crc32c_armv8.c
 *	  Compute CRC-32C checksum using ARMv8 CRC Extension instructions
 *
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 *
 * IDENTIFICATION
 *	  src/port/pg_crc32c_armv8.c
 *
 *-------------------------------------------------------------------------
 */
#include "c.h"

#ifdef _MSC_VER
#include <intrin.h>
#else
#include <arm_acle.h>
#endif

#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
#include <arm_neon.h>
#endif

#include "port/pg_crc32c.h"

pg_crc32c
pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len)
{
	const unsigned char *p = data;
	const unsigned char *pend = p + len;

	/*
	 * ARMv8 doesn't require alignment, but aligned memory access is
	 * significantly faster. Process leading bytes so that the loop below
	 * starts with a pointer aligned to eight bytes.
	 */
	if (!PointerIsAligned(p, uint16) &&
		p + 1 <= pend)
	{
		crc = __crc32cb(crc, *p);
		p += 1;
	}
	if (!PointerIsAligned(p, uint32) &&
		p + 2 <= pend)
	{
		crc = __crc32ch(crc, *(const uint16 *) p);
		p += 2;
	}
	if (!PointerIsAligned(p, uint64) &&
		p + 4 <= pend)
	{
		crc = __crc32cw(crc, *(const uint32 *) p);
		p += 4;
	}

	/* Process eight bytes at a time, as far as we can. */
	while (p + 8 <= pend)
	{
		crc = __crc32cd(crc, *(const uint64 *) p);
		p += 8;
	}

	/* Process remaining 0-7 bytes. */
	if (p + 4 <= pend)
	{
		crc = __crc32cw(crc, *(const uint32 *) p);
		p += 4;
	}
	if (p + 2 <= pend)
	{
		crc = __crc32ch(crc, *(const uint16 *) p);
		p += 2;
	}
	if (p < pend)
	{
		crc = __crc32cb(crc, *p);
	}

	return crc;
}

#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK

/*
 * Note: There is no copyright notice in the following generated code.
 *
 * We have modified the output to
 *   - match our function declaration
 *   - match whitespace to our project style
 *   - be more friendly for pgindent
 *   - exit early for small inputs
 */

/* Generated by https://github.com/corsix/fast-crc32/ using: */
/* ./generate -i neon -p crc32c -a v4e */
/* MIT licensed */

pg_attribute_target("+crypto")
static inline
uint64x2_t
clmul_lo_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
{
	uint64x2_t	r;

__asm("pmull %0.1q, %2.1d, %3.1d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
	return r;
}

pg_attribute_target("+crypto")
static inline
uint64x2_t
clmul_hi_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
{
	uint64x2_t	r;

__asm("pmull2 %0.1q, %2.2d, %3.2d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
	return r;
}

pg_attribute_target("+crypto")
pg_crc32c
pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len)
{
	/* adjust names to match generated code */
	pg_crc32c	crc0 = crc;
	const char *buf = data;

	/*
	 * Immediately fall back to the scalar path if the vector path is not
	 * guaranteed to perform at least one iteration after the alignment
	 * preamble.
	 */
	if (len < 5 * sizeof(uint64x2_t))
		return pg_comp_crc32c_armv8(crc, data, len);

	/* align to 16 bytes */
	for (; (uintptr_t) buf & 7; --len)
	{
		crc0 = __crc32cb(crc0, *buf++);
	}
	if ((uintptr_t) buf & 8)
	{
		crc0 = __crc32cd(crc0, *(const uint64_t *) buf);
		buf += 8;
		len -= 8;
	}

	Assert(len >= 64);

	{
		const char *end = buf + len;
		const char *limit = buf + len - 64;

		/* First vector chunk. */
		uint64x2_t	x0 = vld1q_u64((const uint64_t *) buf),
					y0;
		uint64x2_t	x1 = vld1q_u64((const uint64_t *) (buf + 16)),
					y1;
		uint64x2_t	x2 = vld1q_u64((const uint64_t *) (buf + 32)),
					y2;
		uint64x2_t	x3 = vld1q_u64((const uint64_t *) (buf + 48)),
					y3;
		uint64x2_t	k;

		{
			static const alignas(16) uint64_t k_[] = {0x740eef02, 0x9e4addf8};

			k = vld1q_u64(k_);
		}

		/*
		 * pgindent complained of unmatched parens, so the following has been
		 * re-written with intrinsics:
		 *
		 * x0 = veorq_u64((uint64x2_t) {crc0, 0}, x0);
		 */
		x0 = veorq_u64((uint64x2_t) vsetq_lane_u64(crc0, vdupq_n_u64(0), 0), x0);
		buf += 64;

		/* Main loop. */
		while (buf <= limit)
		{
			y0 = clmul_lo_e(x0, k, vld1q_u64((const uint64_t *) buf)), x0 = clmul_hi_e(x0, k, y0);
			y1 = clmul_lo_e(x1, k, vld1q_u64((const uint64_t *) (buf + 16))), x1 = clmul_hi_e(x1, k, y1);
			y2 = clmul_lo_e(x2, k, vld1q_u64((const uint64_t *) (buf + 32))), x2 = clmul_hi_e(x2, k, y2);
			y3 = clmul_lo_e(x3, k, vld1q_u64((const uint64_t *) (buf + 48))), x3 = clmul_hi_e(x3, k, y3);
			buf += 64;
		}

		/* Reduce x0 ... x3 to just x0. */
		{
			static const alignas(16) uint64_t k_[] = {0xf20c0dfe, 0x493c7d27};

			k = vld1q_u64(k_);
		}
		y0 = clmul_lo_e(x0, k, x1), x0 = clmul_hi_e(x0, k, y0);
		y2 = clmul_lo_e(x2, k, x3), x2 = clmul_hi_e(x2, k, y2);
		{
			static const alignas(16) uint64_t k_[] = {0x3da6d0cb, 0xba4fc28e};

			k = vld1q_u64(k_);
		}
		y0 = clmul_lo_e(x0, k, x2), x0 = clmul_hi_e(x0, k, y0);

		/* Reduce 128 bits to 32 bits, and multiply by x^32. */
		crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0));
		crc0 = __crc32cd(crc0, vgetq_lane_u64(x0, 1));
		len = end - buf;
	}

	return pg_comp_crc32c_armv8(crc0, buf, len);
}

#endif