src/backend/jit/llvm/llvmjit_deform.c - master - githubmirror/postgres

src/backend/jit/llvm/llvmjit_deform.c
770 строк22 KB
David Rowley
Update JIT tuple deforming code for virtual generated columns
19 июн 2026, 06:26
19 июн 2026, 06:26dc51167
/*-------------------------------------------------------------------------
 *
 * llvmjit_deform.c
 *	  Generate code for deforming a heap tuple.
 *
 * This gains performance benefits over unJITed deforming from compile-time
 * knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc
 * can be taken advantage of.
 *
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * IDENTIFICATION
 *	  src/backend/jit/llvm/llvmjit_deform.c
 *
 *-------------------------------------------------------------------------
 */

#include "postgres.h"

#include <llvm-c/Core.h>

#include "access/htup_details.h"
#include "access/tupdesc_details.h"
#include "executor/tuptable.h"
#include "jit/llvmjit.h"
#include "jit/llvmjit_emit.h"


/*
 * Create a function that deforms a tuple of type desc up to natts columns.
 */
LLVMValueRef
slot_compile_deform(LLVMJitContext *context, TupleDesc desc,
					const TupleTableSlotOps *ops, int natts)
{
	char	   *funcname;

	LLVMModuleRef mod;
	LLVMContextRef lc;
	LLVMBuilderRef b;

	LLVMTypeRef deform_sig;
	LLVMValueRef v_deform_fn;

	LLVMBasicBlockRef b_entry;
	LLVMBasicBlockRef b_adjust_unavail_cols;
	LLVMBasicBlockRef b_find_start;

	LLVMBasicBlockRef b_out;
	LLVMBasicBlockRef b_dead;
	LLVMBasicBlockRef *attcheckattnoblocks;
	LLVMBasicBlockRef *attstartblocks;
	LLVMBasicBlockRef *attisnullblocks;
	LLVMBasicBlockRef *attcheckalignblocks;
	LLVMBasicBlockRef *attalignblocks;
	LLVMBasicBlockRef *attstoreblocks;

	LLVMValueRef v_offp;

	LLVMValueRef v_tupdata_base;
	LLVMValueRef v_tts_values;
	LLVMValueRef v_tts_nulls;
	LLVMValueRef v_slotoffp;
	LLVMValueRef v_nvalidp;
	LLVMValueRef v_nvalid;
	LLVMValueRef v_maxatt;

	LLVMValueRef v_slot;

	LLVMValueRef v_tupleheaderp;
	LLVMValueRef v_tuplep;
	LLVMValueRef v_infomask1;
	LLVMValueRef v_infomask2;
	LLVMValueRef v_bits;

	LLVMValueRef v_hoff;

	LLVMValueRef v_hasnulls;

	/* last column (0 indexed) guaranteed to exist */
	int			guaranteed_column_number = -1;

	/* current known alignment */
	int			known_alignment = 0;

	/* if true, known_alignment describes definite offset of column */
	bool		attguaranteedalign = true;

	int			attnum;

	/* virtual tuples never need deforming, so don't generate code */
	if (ops == &TTSOpsVirtual)
		return NULL;

	/* decline to JIT for slot types we don't know to handle */
	if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple &&
		ops != &TTSOpsMinimalTuple)
		return NULL;

	mod = llvm_mutable_module(context);
	lc = LLVMGetModuleContext(mod);

	funcname = llvm_expand_funcname(context, "deform");

	/*
	 * Check which columns have to exist in all tuples, so we don't have to
	 * check the row's natts unnecessarily.
	 */
	for (attnum = 0; attnum < desc->natts; attnum++)
	{
		CompactAttribute *catt = TupleDescCompactAttr(desc, attnum);
		Form_pg_attribute attr = TupleDescAttr(desc, attnum);

		/*
		 * If the column is declared NOT NULL then it must be present in every
		 * tuple, unless there's a "missing" entry that could provide a
		 * non-NULL value for it. That in turn guarantees that the NULL bitmap
		 * - if there are any NULLable columns - is at least long enough to
		 * cover columns up to attnum.  We treat virtual generated columns
		 * similar to atthasmissing columns, as these columns could either not
		 * be represented in the tuple or could have the column represented as
		 * a NULL in the null bitmap.
		 *
		 * Be paranoid and also check !attisdropped, even though the
		 * combination of attisdropped && attnotnull combination shouldn't
		 * exist.
		 */
		if (catt->attnullability == ATTNULLABLE_VALID &&
			!catt->atthasmissing &&
			!catt->attisdropped &&
			attr->attgenerated != ATTRIBUTE_GENERATED_VIRTUAL)
			guaranteed_column_number = attnum;
	}

	/* Create the signature and function */
	{
		LLVMTypeRef param_types[1];

		param_types[0] = l_ptr(StructTupleTableSlot);

		deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc),
									  param_types, lengthof(param_types), 0);
	}
	v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig);
	LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage);
	LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF);
	llvm_copy_attributes(AttributeTemplate, v_deform_fn);

	b_entry =
		LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry");
	b_adjust_unavail_cols =
		LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols");
	b_find_start =
		LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock");
	b_out =
		LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock");
	b_dead =
		LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock");

	b = LLVMCreateBuilderInContext(lc);

	attcheckattnoblocks = palloc_array(LLVMBasicBlockRef, natts);
	attstartblocks = palloc_array(LLVMBasicBlockRef, natts);
	attisnullblocks = palloc_array(LLVMBasicBlockRef, natts);
	attcheckalignblocks = palloc_array(LLVMBasicBlockRef, natts);
	attalignblocks = palloc_array(LLVMBasicBlockRef, natts);
	attstoreblocks = palloc_array(LLVMBasicBlockRef, natts);

	known_alignment = 0;

	LLVMPositionBuilderAtEnd(b, b_entry);

	/* perform allocas first, llvm only converts those to registers */
	v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp");

	v_slot = LLVMGetParam(v_deform_fn, 0);

	v_tts_values =
		l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES,
						  "tts_values");
	v_tts_nulls =
		l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL,
						  "tts_ISNULL");
	v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, "");

	if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple)
	{
		LLVMValueRef v_heapslot;

		v_heapslot =
			LLVMBuildBitCast(b,
							 v_slot,
							 l_ptr(StructHeapTupleTableSlot),
							 "heapslot");
		v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, "");
		v_tupleheaderp =
			l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE,
							  "tupleheader");
	}
	else if (ops == &TTSOpsMinimalTuple)
	{
		LLVMValueRef v_minimalslot;

		v_minimalslot =
			LLVMBuildBitCast(b,
							 v_slot,
							 l_ptr(StructMinimalTupleTableSlot),
							 "minimalslot");
		v_slotoffp = l_struct_gep(b,
								  StructMinimalTupleTableSlot,
								  v_minimalslot,
								  FIELDNO_MINIMALTUPLETABLESLOT_OFF, "");
		v_tupleheaderp =
			l_load_struct_gep(b,
							  StructMinimalTupleTableSlot,
							  v_minimalslot,
							  FIELDNO_MINIMALTUPLETABLESLOT_TUPLE,
							  "tupleheader");
	}
	else
	{
		/* should've returned at the start of the function */
		pg_unreachable();
	}

	v_tuplep =
		l_load_struct_gep(b,
						  StructHeapTupleData,
						  v_tupleheaderp,
						  FIELDNO_HEAPTUPLEDATA_DATA,
						  "tuple");
	v_bits =
		LLVMBuildBitCast(b,
						 l_struct_gep(b,
									  StructHeapTupleHeaderData,
									  v_tuplep,
									  FIELDNO_HEAPTUPLEHEADERDATA_BITS,
									  ""),
						 l_ptr(LLVMInt8TypeInContext(lc)),
						 "t_bits");
	v_infomask1 =
		l_load_struct_gep(b,
						  StructHeapTupleHeaderData,
						  v_tuplep,
						  FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK,
						  "infomask1");
	v_infomask2 =
		l_load_struct_gep(b,
						  StructHeapTupleHeaderData,
						  v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2,
						  "infomask2");

	/* t_infomask & HEAP_HASNULL */
	v_hasnulls =
		LLVMBuildICmp(b, LLVMIntNE,
					  LLVMBuildAnd(b,
								   l_int16_const(lc, HEAP_HASNULL),
								   v_infomask1, ""),
					  l_int16_const(lc, 0),
					  "hasnulls");

	/* t_infomask2 & HEAP_NATTS_MASK */
	v_maxatt = LLVMBuildAnd(b,
							l_int16_const(lc, HEAP_NATTS_MASK),
							v_infomask2,
							"maxatt");

	/*
	 * Need to zext, as getelementptr otherwise treats hoff as a signed 8bit
	 * integer, which'd yield a negative offset for t_hoff > 127.
	 */
	v_hoff =
		LLVMBuildZExt(b,
					  l_load_struct_gep(b,
										StructHeapTupleHeaderData,
										v_tuplep,
										FIELDNO_HEAPTUPLEHEADERDATA_HOFF,
										""),
					  LLVMInt32TypeInContext(lc), "t_hoff");

	v_tupdata_base = l_gep(b,
						   LLVMInt8TypeInContext(lc),
						   LLVMBuildBitCast(b,
											v_tuplep,
											l_ptr(LLVMInt8TypeInContext(lc)),
											""),
						   &v_hoff, 1,
						   "v_tupdata_base");

	/*
	 * Load tuple start offset from slot. Will be reset below in case there's
	 * no existing deformed columns in slot.
	 */
	{
		LLVMValueRef v_off_start;

		v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off");
		v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, "");
		LLVMBuildStore(b, v_off_start, v_offp);
	}

	/* build the basic block for each attribute, need them as jump target */
	for (attnum = 0; attnum < natts; attnum++)
	{
		attcheckattnoblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum);
		attstartblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum);
		attisnullblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum);
		attcheckalignblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum);
		attalignblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum);
		attstoreblocks[attnum] =
			l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum);
	}

	/*
	 * Check if it is guaranteed that all the desired attributes are available
	 * in the tuple (but still possibly NULL), by dint of either the last
	 * to-be-deformed column being NOT NULL, or subsequent ones not accessed
	 * here being NOT NULL.  If that's not guaranteed the tuple headers natt's
	 * has to be checked, and missing attributes potentially have to be
	 * fetched (using slot_getmissingattrs().
	 */
	if ((natts - 1) <= guaranteed_column_number)
	{
		/* just skip through unnecessary blocks */
		LLVMBuildBr(b, b_adjust_unavail_cols);
		LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);
		LLVMBuildBr(b, b_find_start);
	}
	else
	{
		LLVMValueRef v_params[3];
		LLVMValueRef f;

		/* branch if not all columns available */
		LLVMBuildCondBr(b,
						LLVMBuildICmp(b, LLVMIntULT,
									  v_maxatt,
									  l_int16_const(lc, natts),
									  ""),
						b_adjust_unavail_cols,
						b_find_start);

		/* if not, memset tts_isnull of relevant cols to true */
		LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols);

		v_params[0] = v_slot;
		v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), "");
		v_params[2] = l_int32_const(lc, natts);
		f = llvm_pg_func(mod, "slot_getmissingattrs");
		l_call(b,
			   LLVMGetFunctionType(f), f,
			   v_params, lengthof(v_params), "");
		LLVMBuildBr(b, b_find_start);
	}

	LLVMPositionBuilderAtEnd(b, b_find_start);

	v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, "");

	/*
	 * Build switch to go from nvalid to the right startblock.  Callers
	 * currently don't have the knowledge, but it'd be good for performance to
	 * avoid this check when it's known that the slot is empty (e.g. in scan
	 * nodes).
	 */
	if (true)
	{
		LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid,
												b_dead, natts);

		for (attnum = 0; attnum < natts; attnum++)
		{
			LLVMValueRef v_attno = l_int16_const(lc, attnum);

			LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]);
		}
	}
	else
	{
		/* jump from entry block to first block */
		LLVMBuildBr(b, attcheckattnoblocks[0]);
	}

	LLVMPositionBuilderAtEnd(b, b_dead);
	LLVMBuildUnreachable(b);

	/*
	 * Iterate over each attribute that needs to be deformed, build code to
	 * deform it.
	 */
	for (attnum = 0; attnum < natts; attnum++)
	{
		CompactAttribute *att = TupleDescCompactAttr(desc, attnum);
		Form_pg_attribute attr = TupleDescAttr(desc, attnum);

		LLVMValueRef v_incby;
		int			alignto = att->attalignby;
		LLVMValueRef l_attno = l_int16_const(lc, attnum);
		LLVMValueRef v_attdatap;
		LLVMValueRef v_resultp;

		/* build block checking whether we did all the necessary attributes */
		LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]);

		/*
		 * If this is the first attribute, slot->tts_nvalid was 0. Therefore
		 * also reset offset to 0, it may be from a previous execution.
		 */
		if (attnum == 0)
		{
			LLVMBuildStore(b, l_sizet_const(0), v_offp);
		}

		/*
		 * Build check whether column is available (i.e. whether the tuple has
		 * that many columns stored). We can avoid the branch if we know
		 * there's a subsequent NOT NULL column.
		 */
		if (attnum <= guaranteed_column_number)
		{
			LLVMBuildBr(b, attstartblocks[attnum]);
		}
		else
		{
			LLVMValueRef v_islast;

			v_islast = LLVMBuildICmp(b, LLVMIntUGE,
									 l_attno,
									 v_maxatt,
									 "heap_natts");
			LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]);
		}
		LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]);

		/*
		 * Check for nulls if necessary. No need to take missing attributes
		 * into account, because if they're present the heaptuple's natts
		 * would have indicated that a slot_getmissingattrs() is needed. When
		 * present in the tuple, virtual generated columns are always stored
		 * as NULL, so we must always perform NULL checks for these.
		 */
		if (att->attnullability != ATTNULLABLE_VALID ||
			attr->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL)
		{
			LLVMBasicBlockRef b_ifnotnull;
			LLVMBasicBlockRef b_ifnull;
			LLVMBasicBlockRef b_next;
			LLVMValueRef v_attisnull;
			LLVMValueRef v_nullbyteno;
			LLVMValueRef v_nullbytemask;
			LLVMValueRef v_nullbyte;
			LLVMValueRef v_nullbit;

			b_ifnotnull = attcheckalignblocks[attnum];
			b_ifnull = attisnullblocks[attnum];

			if (attnum + 1 == natts)
				b_next = b_out;
			else
				b_next = attcheckattnoblocks[attnum + 1];

			v_nullbyteno = l_int32_const(lc, attnum >> 3);
			v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07));
			v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte");

			v_nullbit = LLVMBuildICmp(b,
									  LLVMIntEQ,
									  LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""),
									  l_int8_const(lc, 0),
									  "attisnull");

			v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, "");

			LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull);

			LLVMPositionBuilderAtEnd(b, b_ifnull);

			/* store null-byte */
			LLVMBuildStore(b,
						   l_int8_const(lc, 1),
						   l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, ""));
			/* store zero datum */
			LLVMBuildStore(b,
						   l_datum_const(0),
						   l_gep(b, TypeDatum, v_tts_values, &l_attno, 1, ""));

			LLVMBuildBr(b, b_next);
			attguaranteedalign = false;
		}
		else
		{
			/* nothing to do */
			LLVMBuildBr(b, attcheckalignblocks[attnum]);
			LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]);
			LLVMBuildBr(b, attcheckalignblocks[attnum]);
		}
		LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);

		/* ------
		 * Even if alignment is required, we can skip doing it if provably
		 * unnecessary:
		 * - first column is guaranteed to be aligned
		 * - columns following a NOT NULL fixed width datum have known
		 *   alignment, can skip alignment computation if that known alignment
		 *   is compatible with current column.
		 * ------
		 */
		if (alignto > 1 &&
			(known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment)))
		{
			/*
			 * When accessing a varlena field, we have to "peek" to see if we
			 * are looking at a pad byte or the first byte of a 1-byte-header
			 * datum.  A zero byte must be either a pad byte, or the first
			 * byte of a correctly aligned 4-byte length word; in either case,
			 * we can align safely.  A non-zero byte must be either a 1-byte
			 * length word, or the first byte of a correctly aligned 4-byte
			 * length word; in either case, we need not align.
			 */
			if (att->attlen == -1)
			{
				LLVMValueRef v_possible_padbyte;
				LLVMValueRef v_ispad;
				LLVMValueRef v_off;

				/* don't know if short varlena or not */
				attguaranteedalign = false;

				v_off = l_load(b, TypeSizeT, v_offp, "");

				v_possible_padbyte =
					l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte");
				v_ispad =
					LLVMBuildICmp(b, LLVMIntEQ,
								  v_possible_padbyte, l_int8_const(lc, 0),
								  "ispadbyte");
				LLVMBuildCondBr(b, v_ispad,
								attalignblocks[attnum],
								attstoreblocks[attnum]);
			}
			else
			{
				LLVMBuildBr(b, attalignblocks[attnum]);
			}

			LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);

			/* translation of alignment code (cf TYPEALIGN()) */
			{
				LLVMValueRef v_off_aligned;
				LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");

				/* ((ALIGNVAL) - 1) */
				LLVMValueRef v_alignval = l_sizet_const(alignto - 1);

				/* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */
				LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, "");

				/* ~((uintptr_t) ((ALIGNVAL) - 1)) */
				LLVMValueRef v_rh = l_sizet_const(~(alignto - 1));

				v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset");

				LLVMBuildStore(b, v_off_aligned, v_offp);
			}

			/*
			 * As alignment either was unnecessary or has been performed, we
			 * now know the current alignment. This is only safe because this
			 * value isn't used for varlena and nullable columns.
			 */
			if (known_alignment >= 0)
			{
				Assert(known_alignment != 0);
				known_alignment = TYPEALIGN(alignto, known_alignment);
			}

			LLVMBuildBr(b, attstoreblocks[attnum]);
			LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);
		}
		else
		{
			LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]);
			LLVMBuildBr(b, attalignblocks[attnum]);
			LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]);
			LLVMBuildBr(b, attstoreblocks[attnum]);
		}
		LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]);

		/*
		 * Store the current offset if known to be constant. That allows LLVM
		 * to generate better code. Without that LLVM can't figure out that
		 * the offset might be constant due to the jumps for previously
		 * decoded columns.
		 */
		if (attguaranteedalign)
		{
			Assert(known_alignment >= 0);
			LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
		}

		/* compute what following columns are aligned to */
		if (att->attlen < 0)
		{
			/* can't guarantee any alignment after variable length field */
			known_alignment = -1;
			attguaranteedalign = false;
		}
		else if (att->attnullability == ATTNULLABLE_VALID &&
				 attguaranteedalign && known_alignment >= 0)
		{
			/*
			 * If the offset to the column was previously known, a NOT NULL &
			 * fixed-width column guarantees that alignment is just the
			 * previous alignment plus column width.
			 */
			Assert(att->attlen > 0);
			known_alignment += att->attlen;
		}
		else if (att->attnullability == ATTNULLABLE_VALID &&
				 attr->attgenerated != ATTRIBUTE_GENERATED_VIRTUAL &&
				 (att->attlen % alignto) == 0)
		{
			/*
			 * After a NOT NULL (and not virtual generated) fixed-width column
			 * with a length that is a multiple of its alignment requirement,
			 * we know the following column is aligned to at least the current
			 * column's alignment.
			 */
			Assert(att->attlen > 0);
			known_alignment = alignto;
			Assert(known_alignment > 0);
			attguaranteedalign = false;
		}
		else
		{
			known_alignment = -1;
			attguaranteedalign = false;
		}


		/* compute address to load data from */
		{
			LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");

			v_attdatap =
				l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, "");
		}

		/* compute address to store value at */
		v_resultp = l_gep(b, TypeDatum, v_tts_values, &l_attno, 1, "");

		/* store null-byte (false) */
		LLVMBuildStore(b, l_int8_const(lc, 0),
					   l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, ""));

		/*
		 * Store datum. For byval: datums copy the value, extend to Datum's
		 * width, and store. For byref types: store pointer to data.
		 */
		if (att->attbyval)
		{
			LLVMValueRef v_tmp_loaddata;
			LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8);
			LLVMTypeRef vartypep = LLVMPointerType(vartype, 0);

			v_tmp_loaddata =
				LLVMBuildPointerCast(b, v_attdatap, vartypep, "");
			v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval");
			v_tmp_loaddata = LLVMBuildSExt(b, v_tmp_loaddata, TypeDatum, "");

			LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
		}
		else
		{
			LLVMValueRef v_tmp_loaddata;

			/* store pointer */
			v_tmp_loaddata =
				LLVMBuildPtrToInt(b,
								  v_attdatap,
								  TypeDatum,
								  "attr_ptr");
			LLVMBuildStore(b, v_tmp_loaddata, v_resultp);
		}

		/* increment data pointer */
		if (att->attlen > 0)
		{
			v_incby = l_sizet_const(att->attlen);
		}
		else if (att->attlen == -1)
		{
			v_incby = l_call(b,
							 llvm_pg_var_func_type("varsize_any"),
							 llvm_pg_func(mod, "varsize_any"),
							 &v_attdatap, 1,
							 "varsize_any");
			l_callsite_ro(v_incby);
			l_callsite_alwaysinline(v_incby);
		}
		else if (att->attlen == -2)
		{
			v_incby = l_call(b,
							 llvm_pg_var_func_type("strlen"),
							 llvm_pg_func(mod, "strlen"),
							 &v_attdatap, 1, "strlen");

			l_callsite_ro(v_incby);

			/* add 1 for NUL byte */
			v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), "");
		}
		else
		{
			Assert(false);
			v_incby = NULL;		/* silence compiler */
		}

		if (attguaranteedalign)
		{
			Assert(known_alignment >= 0);
			LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp);
		}
		else
		{
			LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");

			v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset");
			LLVMBuildStore(b, v_off, v_offp);
		}

		/*
		 * jump to next block, unless last possible column, or all desired
		 * (available) attributes have been fetched.
		 */
		if (attnum + 1 == natts)
		{
			/* jump out */
			LLVMBuildBr(b, b_out);
		}
		else
		{
			LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]);
		}
	}


	/* build block that returns */
	LLVMPositionBuilderAtEnd(b, b_out);

	{
		LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, "");

		LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp);
		v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), "");
		LLVMBuildStore(b, v_off, v_slotoffp);
		LLVMBuildRetVoid(b);
	}

	LLVMDisposeBuilder(b);

	return v_deform_fn;
}