cipher/poly1305.c - master - niceSOFT/libgcrypt

cipher/poly1305.c
846 строк24 KB
Jussi Kivilinna
Fix carry flag clobber for powerpc inline assembly with clang
13 май 2026, 08:31
13 май 2026, 08:3192a2b41
/* poly1305.c  -  Poly1305 internals and generic implementation
 * Copyright (C) 2014,2017,2018 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This file is part of Libgcrypt.
 *
 * Libgcrypt is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * Libgcrypt is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "types.h"
#include "g10lib.h"
#include "cipher.h"
#include "bufhelp.h"
#include "poly1305-internal.h"

#include "mpi-internal.h"
#include "longlong.h"


static const char *selftest (void);


#undef HAVE_ASM_POLY1305_BLOCKS


#undef USE_MPI_64BIT
#undef USE_MPI_32BIT
#if BYTES_PER_MPI_LIMB == 8 && defined(HAVE_U64)
# define USE_MPI_64BIT 1
#elif BYTES_PER_MPI_LIMB == 4
# define USE_MPI_32BIT 1
#else
# error please implement for this limb size.
#endif


/* USE_S390X_ASM indicates whether to enable zSeries code. */
#undef USE_S390X_ASM
#if BYTES_PER_MPI_LIMB == 8
# if defined (__s390x__) && __GNUC__ >= 4 && __ARCH__ >= 9
#  if defined(HAVE_GCC_INLINE_ASM_S390X)
#   define USE_S390X_ASM 1
#  endif /* USE_S390X_ASM */
# endif
#endif


/* AMD64 Assembly implementations use SystemV ABI, ABI conversion and
 * additional stack to store XMM6-XMM15 needed on Win64. */
#undef ASM_FUNC_ABI
#undef ASM_FUNC_WRAPPER_ATTR
#if defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)
# define ASM_FUNC_ABI __attribute__((sysv_abi))
# define ASM_FUNC_WRAPPER_ATTR __attribute__((noinline))
#else
# define ASM_FUNC_ABI
# define ASM_FUNC_WRAPPER_ATTR
#endif


#ifdef USE_S390X_ASM

#define HAVE_ASM_POLY1305_BLOCKS 1

extern unsigned int _gcry_poly1305_s390x_blocks1(void *state,
						 const byte *buf, size_t len,
						 byte high_pad);

static unsigned int
poly1305_blocks (poly1305_context_t *ctx, const byte *buf, size_t len,
		 byte high_pad)
{
  return _gcry_poly1305_s390x_blocks1(&ctx->state, buf, len, high_pad);
}

#endif /* USE_S390X_ASM */


#ifdef POLY1305_USE_AVX512

extern unsigned int
_gcry_poly1305_amd64_avx512_blocks(const void *msg, const u64 msg_len,
				   void *hash, const void *key) ASM_FUNC_ABI;

ASM_FUNC_WRAPPER_ATTR static unsigned int
poly1305_amd64_avx512_blocks(poly1305_context_t *ctx, const byte *buf,
			     size_t len)
{
  POLY1305_STATE *st = &ctx->state;
  return _gcry_poly1305_amd64_avx512_blocks(buf, len, st->h, st->r);
}

#endif /* POLY1305_USE_AVX512 */


#ifdef POLY1305_USE_PPC_VEC

extern unsigned int
gcry_poly1305_p10le_4blocks(unsigned char *key, const byte *m, size_t len);

#endif /* POLY1305_USE_PPC_VEC */


static void poly1305_init (poly1305_context_t *ctx,
			   const byte key[POLY1305_KEYLEN])
{
  POLY1305_STATE *st = &ctx->state;
  unsigned int features = _gcry_get_hw_features ();

#ifdef POLY1305_USE_AVX512
  ctx->use_avx512 = (features & HWF_INTEL_AVX512) != 0;
#endif

#ifdef POLY1305_USE_PPC_VEC
  ctx->use_p10 = (features & HWF_PPC_ARCH_3_10) != 0;
# ifdef ENABLE_FORCE_SOFT_HWFEATURES
  /* HWF_PPC_ARCH_3_10 above is used as soft HW-feature indicator for P10.
   * Actual implementation works with HWF_PPC_ARCH_3_00 also. */
  ctx->use_p10 |= (features & HWF_PPC_ARCH_3_00) != 0;
# endif
#endif

  (void)features;

  ctx->leftover = 0;

  st->h[0] = 0;
  st->h[1] = 0;
  st->h[2] = 0;
  st->h[3] = 0;
  st->h[4] = 0;

  st->r[0] = buf_get_le32(key + 0)  & 0x0fffffff;
  st->r[1] = buf_get_le32(key + 4)  & 0x0ffffffc;
  st->r[2] = buf_get_le32(key + 8)  & 0x0ffffffc;
  st->r[3] = buf_get_le32(key + 12) & 0x0ffffffc;

  st->k[0] = buf_get_le32(key + 16);
  st->k[1] = buf_get_le32(key + 20);
  st->k[2] = buf_get_le32(key + 24);
  st->k[3] = buf_get_le32(key + 28);
}


#ifdef USE_MPI_64BIT

#if defined (__aarch64__) && defined(HAVE_CPU_ARCH_ARM) && __GNUC__ >= 4

/* A += B (armv8/aarch64) */
#define ADD_1305_64(A2, A1, A0, B2, B1, B0) \
      __asm__ ("adds %0, %3, %0\n" \
	       "adcs %1, %4, %1\n" \
	       "adc  %2, %5, %2\n" \
	       : "+r" (A0), "+r" (A1), "+r" (A2) \
	       : "r" (B0), "r" (B1), "r" (B2) \
	       : "cc" )

#endif /* __aarch64__ */

#if defined (__x86_64__) && defined(HAVE_CPU_ARCH_X86) && __GNUC__ >= 4

/* A += B (x86-64) */
#define ADD_1305_64(A2, A1, A0, B2, B1, B0) \
      __asm__ ("addq %3, %0\n" \
	       "adcq %4, %1\n" \
	       "adcq %5, %2\n" \
	       : "+r" (A0), "+r" (A1), "+r" (A2) \
	       : "g" (B0), "g" (B1), "g" (B2) \
	       : "cc" )

#endif /* __x86_64__ */

#if defined (__powerpc__) && defined(HAVE_CPU_ARCH_PPC) && __GNUC__ >= 4

/* A += B (ppc64) */
#define ADD_1305_64(A2, A1, A0, B2, B1, B0) \
      __asm__ ("addc %0, %3, %0\n" \
	       "adde %1, %4, %1\n" \
	       "adde %2, %5, %2\n" \
	       : "+r" (A0), "+r" (A1), "+r" (A2) \
	       : "r" (B0), "r" (B1), "r" (B2) \
	       : "cc", "xer" )

#endif /* __powerpc__ */

#ifndef ADD_1305_64
/* A += B (generic, mpi) */
#  define ADD_1305_64(A2, A1, A0, B2, B1, B0) do { \
    u64 carry; \
    add_ssaaaa(carry, A0, 0, A0, 0, B0); \
    add_ssaaaa(A2, A1, A2, A1, B2, B1); \
    add_ssaaaa(A2, A1, A2, A1, 0, carry); \
  } while (0)
#endif

/* H = H * R mod 2¹³⁰-5 */
#define MUL_MOD_1305_64(H2, H1, H0, R1, R0, R1_MULT5) do { \
    u64 x0_lo, x0_hi, x1_lo, x1_hi; \
    u64 t0_lo, t0_hi, t1_lo, t1_hi; \
    \
    /* x = a * r (partial mod 2^130-5) */ \
    umul_ppmm(x0_hi, x0_lo, H0, R0);  /* h0 * r0 */ \
    umul_ppmm(x1_hi, x1_lo, H0, R1);  /* h0 * r1 */ \
    \
    umul_ppmm(t0_hi, t0_lo, H1, R1_MULT5); /* h1 * r1 mod 2^130-5 */ \
    add_ssaaaa(x0_hi, x0_lo, x0_hi, x0_lo, t0_hi, t0_lo); \
    umul_ppmm(t1_hi, t1_lo, H1, R0);       /* h1 * r0 */ \
    add_ssaaaa(x1_hi, x1_lo, x1_hi, x1_lo, t1_hi, t1_lo); \
    \
    t1_lo = H2 * R1_MULT5; /* h2 * r1 mod 2^130-5 */ \
    t1_hi = H2 * R0;       /* h2 * r0 */ \
    add_ssaaaa(H0, H1, x1_hi, x1_lo, t1_hi, t1_lo); \
    \
    /* carry propagation */ \
    H2 = H0 & 3; \
    H0 = (H0 >> 2) * 5; /* msb mod 2^130-5 */ \
    ADD_1305_64(H2, H1, H0, (u64)0, x0_hi, x0_lo); \
  } while (0)

#ifndef HAVE_ASM_POLY1305_BLOCKS

static unsigned int
poly1305_blocks_generic (poly1305_context_t *ctx, const byte *buf, size_t len,
			 byte high_pad)
{
  POLY1305_STATE *st = &ctx->state;
  u64 r0, r1, r1_mult5;
  u64 h0, h1, h2;
  u64 m0, m1, m2;

  m2 = high_pad;

  h0 = st->h[0] + ((u64)st->h[1] << 32);
  h1 = st->h[2] + ((u64)st->h[3] << 32);
  h2 = st->h[4];

  r0 = st->r[0] + ((u64)st->r[1] << 32);
  r1 = st->r[2] + ((u64)st->r[3] << 32);

  r1_mult5 = (r1 >> 2) + r1;

  m0 = buf_get_le64(buf + 0);
  m1 = buf_get_le64(buf + 8);
  buf += POLY1305_BLOCKSIZE;
  len -= POLY1305_BLOCKSIZE;

  while (len >= POLY1305_BLOCKSIZE)
    {
      /* a = h + m */
      ADD_1305_64(h2, h1, h0, m2, m1, m0);

      m0 = buf_get_le64(buf + 0);
      m1 = buf_get_le64(buf + 8);

      /* h = a * r (partial mod 2^130-5) */
      MUL_MOD_1305_64(h2, h1, h0, r1, r0, r1_mult5);

      buf += POLY1305_BLOCKSIZE;
      len -= POLY1305_BLOCKSIZE;
    }

  /* a = h + m */
  ADD_1305_64(h2, h1, h0, m2, m1, m0);

  /* h = a * r (partial mod 2^130-5) */
  MUL_MOD_1305_64(h2, h1, h0, r1, r0, r1_mult5);

  st->h[0] = h0;
  st->h[1] = h0 >> 32;
  st->h[2] = h1;
  st->h[3] = h1 >> 32;
  st->h[4] = h2;

  return 6 * sizeof (void *) + 18 * sizeof (u64);
}

static unsigned int
poly1305_blocks (poly1305_context_t *ctx, const byte *buf, size_t len,
		 byte high_pad)
{
#ifdef POLY1305_USE_AVX512
  if ((high_pad & ctx->use_avx512) != 0)
    return poly1305_amd64_avx512_blocks(ctx, buf, len);
#endif

  return poly1305_blocks_generic(ctx, buf, len, high_pad);
}

#endif /* !HAVE_ASM_POLY1305_BLOCKS */

static unsigned int poly1305_final (poly1305_context_t *ctx,
				    byte mac[POLY1305_TAGLEN])
{
  POLY1305_STATE *st = &ctx->state;
  unsigned int burn = 0;
  u64 u, carry;
  u64 k0, k1;
  u64 h0, h1;
  u64 h2;

  /* process the remaining block */
  if (ctx->leftover)
    {
      ctx->buffer[ctx->leftover++] = 1;
      if (ctx->leftover < POLY1305_BLOCKSIZE)
	{
	  memset (&ctx->buffer[ctx->leftover], 0,
		  POLY1305_BLOCKSIZE - ctx->leftover);
	  ctx->leftover = POLY1305_BLOCKSIZE;
	}
      burn = poly1305_blocks (ctx, ctx->buffer, POLY1305_BLOCKSIZE, 0);
    }

  h0 = st->h[0] + ((u64)st->h[1] << 32);
  h1 = st->h[2] + ((u64)st->h[3] << 32);
  h2 = st->h[4];

  k0 = st->k[0] + ((u64)st->k[1] << 32);
  k1 = st->k[2] + ((u64)st->k[3] << 32);

  /* check if h is more than 2^130-5, by adding 5. */
  add_ssaaaa(carry, u, 0, h0, 0, 5);
  add_ssaaaa(carry, u, 0, carry, 0, h1);
  u = (carry + h2) >> 2; /* u == 0 or 1 */

  /* minus 2^130-5 ... (+5) */
  u = (-u) & 5;
  add_ssaaaa(h1, h0, h1, h0, 0, u);

  /* add high part of key + h */
  add_ssaaaa(h1, h0, h1, h0, k1, k0);
  buf_put_le64(mac + 0, h0);
  buf_put_le64(mac + 8, h1);

  /* burn_stack */
  return 4 * sizeof (void *) + 7 * sizeof (u64) + burn;
}

#endif /* USE_MPI_64BIT */

#ifdef USE_MPI_32BIT

#ifdef HAVE_COMPATIBLE_GCC_ARM_PLATFORM_AS

/* HI:LO += A * B (arm) */
#define UMUL_ADD_32(HI, LO, A, B) \
      __asm__ ("umlal %1, %0, %4, %5" \
	       : "=r" (HI), "=r" (LO) \
	       : "0" (HI), "1" (LO), "r" (A), "r" (B) )

/* A += B (arm) */
#ifdef __GCC_ASM_FLAG_OUTPUTS__
#  define ADD_1305_32(A4, A3, A2, A1, A0, B4, B3, B2, B1, B0) do { \
      u32 __carry; \
      __asm__ ("adds %0, %0, %5\n" \
	       "adcs %1, %1, %6\n" \
	       "adcs %2, %2, %7\n" \
	       "adcs %3, %3, %8\n" \
	       : "+r" (A0), "+r" (A1), "+r" (A2), "+r" (A3), \
	         "=@cccs" (__carry) \
	       : "r" (B0), "r" (B1), "r" (B2), "r" (B3) \
	       : ); \
      (A4) += (B4) + __carry; \
    } while (0)
#else
#  define ADD_1305_32(A4, A3, A2, A1, A0, B4, B3, B2, B1, B0) do { \
      u32 __carry = (B0); \
      __asm__ ("adds %0, %0, %2\n" \
	       "adcs %1, %1, %3\n" \
	       "rrx %2, %2\n" /* carry to 31th bit */ \
	       : "+r" (A0), "+r" (A1), "+r" (__carry) \
	       : "r" (B1), "r" (0) \
	       : "cc" ); \
      __asm__ ("lsls %0, %0, #1\n" /* carry from 31th bit */ \
	       "adcs %1, %1, %4\n" \
	       "adcs %2, %2, %5\n" \
	       "adc  %3, %3, %6\n" \
	       : "+r" (__carry), "+r" (A2), "+r" (A3), "+r" (A4) \
	       : "r" (B2), "r" (B3), "r" (B4) \
	       : "cc" ); \
    } while (0)
#endif

#endif /* HAVE_COMPATIBLE_GCC_ARM_PLATFORM_AS */

#if defined (__i386__) && defined(HAVE_CPU_ARCH_X86) && __GNUC__ >= 5
/* Note: ADD_1305_32 below does not compile on GCC-4.7 */

/* A += B (i386) */
#define ADD_1305_32(A4, A3, A2, A1, A0, B4, B3, B2, B1, B0) \
      __asm__ ("addl %5, %0\n" \
	       "adcl %6, %1\n" \
	       "adcl %7, %2\n" \
	       "adcl %8, %3\n" \
	       "adcl %9, %4\n" \
	       : "+r" (A0), "+r" (A1), "+r" (A2), "+r" (A3), "+r" (A4) \
	       : "g" (B0), "g" (B1), "g" (B2), "g" (B3), "g" (B4) \
	       : "cc" )

#endif /* __i386__ */

#ifndef UMUL_ADD_32
/* HI:LO += A * B (generic, mpi) */
#  define UMUL_ADD_32(HI, LO, A, B) do { \
    u32 t_lo, t_hi; \
    umul_ppmm(t_hi, t_lo, A, B); \
    add_ssaaaa(HI, LO, HI, LO, t_hi, t_lo); \
  } while (0)
#endif

#ifndef ADD_1305_32
/* A += B (generic, mpi) */
#  define ADD_1305_32(A4, A3, A2, A1, A0, B4, B3, B2, B1, B0) do { \
    u32 carry0, carry1, carry2; \
    add_ssaaaa(carry0, A0, 0, A0, 0, B0); \
    add_ssaaaa(carry1, A1, 0, A1, 0, B1); \
    add_ssaaaa(carry1, A1, carry1, A1, 0, carry0); \
    add_ssaaaa(carry2, A2, 0, A2, 0, B2); \
    add_ssaaaa(carry2, A2, carry2, A2, 0, carry1); \
    add_ssaaaa(A4, A3, A4, A3, B4, B3); \
    add_ssaaaa(A4, A3, A4, A3, 0, carry2); \
  } while (0)
#endif

/* H = H * R mod 2¹³⁰-5 */
#define MUL_MOD_1305_32(H4, H3, H2, H1, H0, R3, R2, R1, R0, \
                        R3_MULT5, R2_MULT5, R1_MULT5) do { \
    u32 x0_lo, x0_hi, x1_lo, x1_hi, x2_lo, x2_hi, x3_lo, x3_hi; \
    u32 t0_lo, t0_hi; \
    \
    /* x = a * r (partial mod 2^130-5) */ \
    umul_ppmm(x0_hi, x0_lo, H0, R0);  /* h0 * r0 */ \
    umul_ppmm(x1_hi, x1_lo, H0, R1);  /* h0 * r1 */ \
    umul_ppmm(x2_hi, x2_lo, H0, R2);  /* h0 * r2 */ \
    umul_ppmm(x3_hi, x3_lo, H0, R3);  /* h0 * r3 */ \
    \
    UMUL_ADD_32(x0_hi, x0_lo, H1, R3_MULT5); /* h1 * r3 mod 2^130-5 */ \
    UMUL_ADD_32(x1_hi, x1_lo, H1, R0);       /* h1 * r0 */ \
    UMUL_ADD_32(x2_hi, x2_lo, H1, R1);       /* h1 * r1 */ \
    UMUL_ADD_32(x3_hi, x3_lo, H1, R2);       /* h1 * r2 */ \
    \
    UMUL_ADD_32(x0_hi, x0_lo, H2, R2_MULT5); /* h2 * r2 mod 2^130-5 */ \
    UMUL_ADD_32(x1_hi, x1_lo, H2, R3_MULT5); /* h2 * r3 mod 2^130-5 */ \
    UMUL_ADD_32(x2_hi, x2_lo, H2, R0);       /* h2 * r0 */ \
    UMUL_ADD_32(x3_hi, x3_lo, H2, R1);       /* h2 * r1 */ \
    \
    UMUL_ADD_32(x0_hi, x0_lo, H3, R1_MULT5); /* h3 * r1 mod 2^130-5 */ \
    H1 = x0_hi; \
    UMUL_ADD_32(x1_hi, x1_lo, H3, R2_MULT5); /* h3 * r2 mod 2^130-5 */ \
    UMUL_ADD_32(x2_hi, x2_lo, H3, R3_MULT5); /* h3 * r3 mod 2^130-5 */ \
    UMUL_ADD_32(x3_hi, x3_lo, H3, R0);       /* h3 * r0 */ \
    \
    t0_lo = H4 * R1_MULT5; /* h4 * r1 mod 2^130-5 */ \
    t0_hi = H4 * R2_MULT5; /* h4 * r2 mod 2^130-5 */ \
    add_ssaaaa(H2, x1_lo, x1_hi, x1_lo, 0, t0_lo); \
    add_ssaaaa(H3, x2_lo, x2_hi, x2_lo, 0, t0_hi); \
    t0_lo = H4 * R3_MULT5; /* h4 * r3 mod 2^130-5 */ \
    t0_hi = H4 * R0;       /* h4 * r0 */ \
    add_ssaaaa(H4, x3_lo, x3_hi, x3_lo, t0_hi, t0_lo); \
    \
    /* carry propagation */ \
    H0 = (H4 >> 2) * 5; /* msb mod 2^130-5 */ \
    H4 = H4 & 3; \
    ADD_1305_32(H4, H3, H2, H1, H0, 0, x3_lo, x2_lo, x1_lo, x0_lo); \
  } while (0)

#ifndef HAVE_ASM_POLY1305_BLOCKS

static unsigned int
poly1305_blocks (poly1305_context_t *ctx, const byte *buf, size_t len,
		 byte high_pad)
{
  POLY1305_STATE *st = &ctx->state;
  u32 r1_mult5, r2_mult5, r3_mult5;
  u32 h0, h1, h2, h3, h4;
  u32 m0, m1, m2, m3, m4;

  m4 = high_pad;

  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];
  h3 = st->h[3];
  h4 = st->h[4];

  r1_mult5 = (st->r[1] >> 2) + st->r[1];
  r2_mult5 = (st->r[2] >> 2) + st->r[2];
  r3_mult5 = (st->r[3] >> 2) + st->r[3];

  while (len >= POLY1305_BLOCKSIZE)
    {
      m0 = buf_get_le32(buf + 0);
      m1 = buf_get_le32(buf + 4);
      m2 = buf_get_le32(buf + 8);
      m3 = buf_get_le32(buf + 12);

      /* a = h + m */
      ADD_1305_32(h4, h3, h2, h1, h0, m4, m3, m2, m1, m0);

      /* h = a * r (partial mod 2^130-5) */
      MUL_MOD_1305_32(h4, h3, h2, h1, h0,
		      st->r[3], st->r[2], st->r[1], st->r[0],
		      r3_mult5, r2_mult5, r1_mult5);

      buf += POLY1305_BLOCKSIZE;
      len -= POLY1305_BLOCKSIZE;
    }

  st->h[0] = h0;
  st->h[1] = h1;
  st->h[2] = h2;
  st->h[3] = h3;
  st->h[4] = h4;

  return 6 * sizeof (void *) + 28 * sizeof (u32);
}

#endif /* !HAVE_ASM_POLY1305_BLOCKS */

static unsigned int poly1305_final (poly1305_context_t *ctx,
				    byte mac[POLY1305_TAGLEN])
{
  POLY1305_STATE *st = &ctx->state;
  unsigned int burn = 0;
  u32 carry, tmp0, tmp1, tmp2, u;
  u32 h4, h3, h2, h1, h0;

  /* process the remaining block */
  if (ctx->leftover)
    {
      ctx->buffer[ctx->leftover++] = 1;
      if (ctx->leftover < POLY1305_BLOCKSIZE)
	{
	  memset (&ctx->buffer[ctx->leftover], 0,
		  POLY1305_BLOCKSIZE - ctx->leftover);
	  ctx->leftover = POLY1305_BLOCKSIZE;
	}
      burn = poly1305_blocks (ctx, ctx->buffer, POLY1305_BLOCKSIZE, 0);
    }

  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];
  h3 = st->h[3];
  h4 = st->h[4];

  /* check if h is more than 2^130-5, by adding 5. */
  add_ssaaaa(carry, tmp0, 0, h0, 0, 5);
  add_ssaaaa(carry, tmp0, 0, carry, 0, h1);
  add_ssaaaa(carry, tmp0, 0, carry, 0, h2);
  add_ssaaaa(carry, tmp0, 0, carry, 0, h3);
  u = (carry + h4) >> 2; /* u == 0 or 1 */

  /* minus 2^130-5 ... (+5) */
  u = (-u) & 5;
  add_ssaaaa(carry, h0, 0, h0, 0, u);
  add_ssaaaa(carry, h1, 0, h1, 0, carry);
  add_ssaaaa(carry, h2, 0, h2, 0, carry);
  add_ssaaaa(carry, h3, 0, h3, 0, carry);

  /* add high part of key + h */
  add_ssaaaa(tmp0, h0, 0, h0, 0, st->k[0]);
  add_ssaaaa(tmp1, h1, 0, h1, 0, st->k[1]);
  add_ssaaaa(tmp1, h1, tmp1, h1, 0, tmp0);
  add_ssaaaa(tmp2, h2, 0, h2, 0, st->k[2]);
  add_ssaaaa(tmp2, h2, tmp2, h2, 0, tmp1);
  add_ssaaaa(carry, h3, 0, h3, 0, st->k[3]);
  h3 += tmp2;

  buf_put_le32(mac + 0, h0);
  buf_put_le32(mac + 4, h1);
  buf_put_le32(mac + 8, h2);
  buf_put_le32(mac + 12, h3);

  /* burn_stack */
  return 4 * sizeof (void *) + 10 * sizeof (u32) + burn;
}

#endif /* USE_MPI_32BIT */


unsigned int
_gcry_poly1305_update_burn (poly1305_context_t *ctx, const byte *m,
			    size_t bytes)
{
  unsigned int burn = 0;
  unsigned int nburn;

  /* handle leftover */
  if (ctx->leftover)
    {
      size_t want = (POLY1305_BLOCKSIZE - ctx->leftover);
      if (want > bytes)
	want = bytes;
      buf_cpy (ctx->buffer + ctx->leftover, m, want);
      bytes -= want;
      m += want;
      ctx->leftover += want;
      if (ctx->leftover < POLY1305_BLOCKSIZE)
	return 0;
      nburn = poly1305_blocks (ctx, ctx->buffer, POLY1305_BLOCKSIZE, 1);
      burn = nburn > burn ? nburn : burn;
      ctx->leftover = 0;
    }

#ifdef POLY1305_USE_PPC_VEC
  /* PPC-P10/little-endian: bulk process multiples of eight blocks */
  if (ctx->use_p10 && bytes >= POLY1305_BLOCKSIZE * 8)
    {
      size_t nblks = bytes / (POLY1305_BLOCKSIZE * 8);
      size_t len = nblks * (POLY1305_BLOCKSIZE * 8);
      POLY1305_STATE *st = &ctx->state;
      nburn = gcry_poly1305_p10le_4blocks ((unsigned char *) st, m, len);
      burn = nburn > burn ? nburn : burn;
      m += len;
      bytes -= len;
    }
#endif /* POLY1305_USE_PPC_VEC */

  /* process full blocks */
  if (bytes >= POLY1305_BLOCKSIZE)
    {
      size_t nblks = bytes / POLY1305_BLOCKSIZE;
      nburn = poly1305_blocks (ctx, m, nblks * POLY1305_BLOCKSIZE, 1);
      burn = nburn > burn ? nburn : burn;
      m += nblks * POLY1305_BLOCKSIZE;
      bytes -= nblks * POLY1305_BLOCKSIZE;
    }

  /* store leftover */
  if (bytes)
    {
      buf_cpy (ctx->buffer + ctx->leftover, m, bytes);
      ctx->leftover += bytes;
    }

  return burn;
}


void
_gcry_poly1305_update (poly1305_context_t *ctx, const byte *m, size_t bytes)
{
  unsigned int burn;

  burn = _gcry_poly1305_update_burn (ctx, m, bytes);

  if (burn)
    _gcry_burn_stack (burn);
}


void
_gcry_poly1305_finish (poly1305_context_t *ctx, byte mac[POLY1305_TAGLEN])
{
  unsigned int burn;

  burn = poly1305_final (ctx, mac);

  _gcry_burn_stack (burn);
}


gcry_err_code_t
_gcry_poly1305_init (poly1305_context_t * ctx, const byte * key,
		     size_t keylen)
{
  static int initialized;
  static const char *selftest_failed;

  if (!initialized)
    {
      initialized = 1;
      selftest_failed = selftest ();
      if (selftest_failed)
	log_error ("Poly1305 selftest failed (%s)\n", selftest_failed);
    }

  if (keylen != POLY1305_KEYLEN)
    return GPG_ERR_INV_KEYLEN;

  if (selftest_failed)
    return GPG_ERR_SELFTEST_FAILED;

  poly1305_init (ctx, key);

  return 0;
}


static void
poly1305_auth (byte mac[POLY1305_TAGLEN], const byte * m, size_t bytes,
	       const byte * key)
{
  poly1305_context_t ctx;

  memset (&ctx, 0, sizeof (ctx));

  _gcry_poly1305_init (&ctx, key, POLY1305_KEYLEN);
  _gcry_poly1305_update (&ctx, m, bytes);
  _gcry_poly1305_finish (&ctx, mac);

  wipememory (&ctx, sizeof (ctx));
}


static const char *
selftest (void)
{
  /* example from nacl */
  static const byte nacl_key[POLY1305_KEYLEN] = {
    0xee, 0xa6, 0xa7, 0x25, 0x1c, 0x1e, 0x72, 0x91,
    0x6d, 0x11, 0xc2, 0xcb, 0x21, 0x4d, 0x3c, 0x25,
    0x25, 0x39, 0x12, 0x1d, 0x8e, 0x23, 0x4e, 0x65,
    0x2d, 0x65, 0x1f, 0xa4, 0xc8, 0xcf, 0xf8, 0x80,
  };

  static const byte nacl_msg[131] = {
    0x8e, 0x99, 0x3b, 0x9f, 0x48, 0x68, 0x12, 0x73,
    0xc2, 0x96, 0x50, 0xba, 0x32, 0xfc, 0x76, 0xce,
    0x48, 0x33, 0x2e, 0xa7, 0x16, 0x4d, 0x96, 0xa4,
    0x47, 0x6f, 0xb8, 0xc5, 0x31, 0xa1, 0x18, 0x6a,
    0xc0, 0xdf, 0xc1, 0x7c, 0x98, 0xdc, 0xe8, 0x7b,
    0x4d, 0xa7, 0xf0, 0x11, 0xec, 0x48, 0xc9, 0x72,
    0x71, 0xd2, 0xc2, 0x0f, 0x9b, 0x92, 0x8f, 0xe2,
    0x27, 0x0d, 0x6f, 0xb8, 0x63, 0xd5, 0x17, 0x38,
    0xb4, 0x8e, 0xee, 0xe3, 0x14, 0xa7, 0xcc, 0x8a,
    0xb9, 0x32, 0x16, 0x45, 0x48, 0xe5, 0x26, 0xae,
    0x90, 0x22, 0x43, 0x68, 0x51, 0x7a, 0xcf, 0xea,
    0xbd, 0x6b, 0xb3, 0x73, 0x2b, 0xc0, 0xe9, 0xda,
    0x99, 0x83, 0x2b, 0x61, 0xca, 0x01, 0xb6, 0xde,
    0x56, 0x24, 0x4a, 0x9e, 0x88, 0xd5, 0xf9, 0xb3,
    0x79, 0x73, 0xf6, 0x22, 0xa4, 0x3d, 0x14, 0xa6,
    0x59, 0x9b, 0x1f, 0x65, 0x4c, 0xb4, 0x5a, 0x74,
    0xe3, 0x55, 0xa5
  };

  static const byte nacl_mac[16] = {
    0xf3, 0xff, 0xc7, 0x70, 0x3f, 0x94, 0x00, 0xe5,
    0x2a, 0x7d, 0xfb, 0x4b, 0x3d, 0x33, 0x05, 0xd9
  };

  /* generates a final value of (2^130 - 2) == 3 */
  static const byte wrap_key[POLY1305_KEYLEN] = {
    0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  };

  static const byte wrap_msg[16] = {
    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
  };

  static const byte wrap_mac[16] = {
    0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  };

  /* mac of the macs of messages of length 0 to 256, where the key and messages
   * have all their values set to the length
   */
  static const byte total_key[POLY1305_KEYLEN] = {
    0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
    0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9,
    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
  };

  static const byte total_mac[16] = {
    0x64, 0xaf, 0xe2, 0xe8, 0xd6, 0xad, 0x7b, 0xbd,
    0xd2, 0x87, 0xf9, 0x7c, 0x44, 0x62, 0x3d, 0x39
  };

  poly1305_context_t ctx;
  poly1305_context_t total_ctx;
  byte all_key[POLY1305_KEYLEN];
  byte all_msg[256];
  byte mac[16];
  size_t i, j;

  memset (&ctx, 0, sizeof (ctx));
  memset (&total_ctx, 0, sizeof (total_ctx));

  memset (mac, 0, sizeof (mac));
  poly1305_auth (mac, nacl_msg, sizeof (nacl_msg), nacl_key);
  if (memcmp (nacl_mac, mac, sizeof (nacl_mac)) != 0)
    return "Poly1305 test 1 failed.";

  /* SSE2/AVX have a 32 byte block size, but also support 64 byte blocks, so
   * make sure everything still works varying between them */
  memset (mac, 0, sizeof (mac));
  _gcry_poly1305_init (&ctx, nacl_key, POLY1305_KEYLEN);
  _gcry_poly1305_update (&ctx, nacl_msg + 0, 32);
  _gcry_poly1305_update (&ctx, nacl_msg + 32, 64);
  _gcry_poly1305_update (&ctx, nacl_msg + 96, 16);
  _gcry_poly1305_update (&ctx, nacl_msg + 112, 8);
  _gcry_poly1305_update (&ctx, nacl_msg + 120, 4);
  _gcry_poly1305_update (&ctx, nacl_msg + 124, 2);
  _gcry_poly1305_update (&ctx, nacl_msg + 126, 1);
  _gcry_poly1305_update (&ctx, nacl_msg + 127, 1);
  _gcry_poly1305_update (&ctx, nacl_msg + 128, 1);
  _gcry_poly1305_update (&ctx, nacl_msg + 129, 1);
  _gcry_poly1305_update (&ctx, nacl_msg + 130, 1);
  _gcry_poly1305_finish (&ctx, mac);
  if (memcmp (nacl_mac, mac, sizeof (nacl_mac)) != 0)
    return "Poly1305 test 2 failed.";

  memset (mac, 0, sizeof (mac));
  poly1305_auth (mac, wrap_msg, sizeof (wrap_msg), wrap_key);
  if (memcmp (wrap_mac, mac, sizeof (nacl_mac)) != 0)
    return "Poly1305 test 3 failed.";

  _gcry_poly1305_init (&total_ctx, total_key, POLY1305_KEYLEN);
  for (i = 0; i < 256; i++)
    {
      /* set key and message to 'i,i,i..' */
      for (j = 0; j < sizeof (all_key); j++)
	all_key[j] = i;
      for (j = 0; j < i; j++)
	all_msg[j] = i;
      poly1305_auth (mac, all_msg, i, all_key);
      _gcry_poly1305_update (&total_ctx, mac, 16);
    }
  _gcry_poly1305_finish (&total_ctx, mac);
  if (memcmp (total_mac, mac, sizeof (total_mac)) != 0)
    return "Poly1305 test 4 failed.";

  return NULL;
}