src/include/portability/instr_time.h - master - githubmirror/postgres

src/include/portability/instr_time.h
465 строк12 KB
Tomas Vondra
Shorten pg_attribute_always_inline to pg_always_inline
11 июл 2026, 17:17
11 июл 2026, 17:171c4b1de
/*-------------------------------------------------------------------------
 *
 * instr_time.h
 *	  portable high-precision interval timing
 *
 * This file provides an abstraction layer to hide portability issues in
 * interval timing. On x86 we use the RDTSC/RDTSCP instruction directly in
 * certain cases, or alternatively clock_gettime() on Unix-like systems and
 * QueryPerformanceCounter() on Windows. These macros also give some breathing
 * room to use other high-precision-timing APIs.
 *
 * The basic data type is instr_time, which all callers should treat as an
 * opaque typedef.  instr_time can store either an absolute time (of
 * unspecified reference time) or an interval.  The operations provided
 * for it are:
 *
 * INSTR_TIME_IS_ZERO(t)			is t equal to zero?
 *
 * INSTR_TIME_SET_ZERO(t)			set t to zero (memset is acceptable too)
 *
 * INSTR_TIME_SET_CURRENT_FAST(t)	set t to current time without waiting
 * 									for instructions in out-of-order window
 *
 * INSTR_TIME_SET_CURRENT(t)		set t to current time while waiting for
 * 									instructions in OOO to retire
 *
 *
 * INSTR_TIME_ADD(x, y)				x += y
 *
 * INSTR_TIME_ADD_NANOSEC(t, n)		t += n in nanoseconds (converts to ticks)
 *
 * INSTR_TIME_SUBTRACT(x, y)		x -= y
 *
 * INSTR_TIME_ACCUM_DIFF(x, y, z)	x += (y - z)
 *
 * INSTR_TIME_GT(x, y)				x > y
 *
 * INSTR_TIME_GET_DOUBLE(t)			convert t to double (in seconds)
 *
 * INSTR_TIME_GET_MILLISEC(t)		convert t to double (in milliseconds)
 *
 * INSTR_TIME_GET_MICROSEC(t)		convert t to int64 (in microseconds)
 *
 * INSTR_TIME_GET_NANOSEC(t)		convert t to int64 (in nanoseconds)
 *
 * Note that INSTR_TIME_SUBTRACT and INSTR_TIME_ACCUM_DIFF convert
 * absolute times to intervals.  The INSTR_TIME_GET_xxx operations are
 * only useful on intervals.
 *
 * When summing multiple measurements, it's recommended to leave the
 * running sum in instr_time form (ie, use INSTR_TIME_ADD or
 * INSTR_TIME_ACCUM_DIFF) and convert to a result format only at the end.
 *
 * Beware of multiple evaluations of the macro arguments.
 *
 *
 * Copyright (c) 2001-2026, PostgreSQL Global Development Group
 *
 * src/include/portability/instr_time.h
 *
 *-------------------------------------------------------------------------
 */
#ifndef INSTR_TIME_H
#define INSTR_TIME_H


/*
 * We store interval times as an int64 integer on all platforms, as int64 is
 * cheap to add/subtract, the most common operation for instr_time. The
 * acquisition of time and converting to specific units of time is platform
 * specific.
 *
 * To avoid users of the API relying on the integer representation, we wrap
 * the 64bit integer in a struct.
 */
typedef struct instr_time
{
	int64		ticks;			/* in platforms specific unit */
} instr_time;


/* helpers macros used in platform specific code below */

#define NS_PER_S	INT64CONST(1000000000)
#define NS_PER_MS	INT64CONST(1000000)
#define NS_PER_US	INT64CONST(1000)

/* Shift amount for fixed-point ticks-to-nanoseconds conversion. */
#define TICKS_TO_NS_SHIFT 14

/*
 * PG_INSTR_TICKS_TO_NS controls whether pg_ticks_to_ns/pg_ns_to_ticks needs to
 * check ticks_per_ns_scaled and potentially convert ticks <=> nanoseconds.
 *
 * PG_INSTR_TSC_CLOCK controls whether the TSC clock source is compiled in, and
 * potentially used based on timing_tsc_enabled.
 */
#if defined(__x86_64__)
#define PG_INSTR_TICKS_TO_NS 1
#define PG_INSTR_TSC_CLOCK 1
#elif defined(WIN32)
#define PG_INSTR_TICKS_TO_NS 1
#define PG_INSTR_TSC_CLOCK 0
#else
#define PG_INSTR_TICKS_TO_NS 0
#define PG_INSTR_TSC_CLOCK 0
#endif

/*
 * Variables used to translate ticks to nanoseconds, initialized by
 * pg_initialize_timing and adjusted by pg_set_timing_clock_source calls or
 * changes of the "timing_clock_source" GUC.
 *
 * Note that changing these values after setting an instr_time and before
 * reading/converting it will lead to incorrect results. This is technically
 * possible because the GUC can be changed at runtime, but unlikely, and we
 * allow changing this at runtime to simplify testing of different sources.
 */
extern PGDLLIMPORT uint64 ticks_per_ns_scaled;
extern PGDLLIMPORT uint64 max_ticks_no_overflow;
extern PGDLLIMPORT bool timing_initialized;

typedef enum
{
	TIMING_CLOCK_SOURCE_AUTO,
	TIMING_CLOCK_SOURCE_SYSTEM,
#if PG_INSTR_TSC_CLOCK
	TIMING_CLOCK_SOURCE_TSC
#endif
} TimingClockSourceType;

extern PGDLLIMPORT int timing_clock_source;

/*
 * Initialize timing infrastructure
 *
 * This must be called at least once before using INSTR_TIME_SET_CURRENT*
 * macros.
 *
 * If you want to use the TSC clock source in a client program,
 * pg_set_timing_clock_source() needs to also be called.
 */
extern void pg_initialize_timing(void);

/*
 * Sets the time source to be used. Mainly intended for frontend programs,
 * the backend should set it via the timing_clock_source GUC instead.
 *
 * Returns false if the clock source could not be set, for example when TSC
 * is not available despite being explicitly set.
 */
extern bool pg_set_timing_clock_source(TimingClockSourceType source);

/* Whether to actually use TSC based on availability and GUC settings. */
extern PGDLLIMPORT bool timing_tsc_enabled;

/*
 * TSC frequency in kHz, set during initialization.
 *
 * -1 = not yet initialized, 0 = TSC not usable, >0 = frequency in kHz.
 */
extern PGDLLIMPORT int32 timing_tsc_frequency_khz;

#if PG_INSTR_TSC_CLOCK

extern void pg_initialize_timing_tsc(void);

typedef struct TscClockSourceInfo
{
	int32		frequency_khz;	/* from CPUID or calibration */
	int32		calibrated_frequency_khz;	/* from calibration */
	char		frequency_source[128];	/* describes how frequency was
										 * determined */
} TscClockSourceInfo;

extern const TscClockSourceInfo *pg_timing_tsc_clock_source_info(void);

#endif							/* PG_INSTR_TSC_CLOCK */

/*
 * Returns the current timing clock source effectively in use, resolving
 * TIMING_CLOCK_SOURCE_AUTO to either TIMING_CLOCK_SOURCE_SYSTEM or
 * TIMING_CLOCK_SOURCE_TSC.
 */
static inline TimingClockSourceType
pg_current_timing_clock_source(void)
{
#if PG_INSTR_TSC_CLOCK
	if (timing_tsc_enabled)
		return TIMING_CLOCK_SOURCE_TSC;
#endif
	return TIMING_CLOCK_SOURCE_SYSTEM;
}

#ifndef WIN32

/* On POSIX, use clock_gettime() for system clock source */

#include <time.h>

/*
 * The best clockid to use according to the POSIX spec is CLOCK_MONOTONIC,
 * since that will give reliable interval timing even in the face of changes
 * to the system clock.  However, POSIX doesn't require implementations to
 * provide anything except CLOCK_REALTIME, so fall back to that if we don't
 * find CLOCK_MONOTONIC.
 *
 * Also, some implementations have nonstandard clockids with better properties
 * than CLOCK_MONOTONIC.  In particular, as of macOS 10.12, Apple provides
 * CLOCK_MONOTONIC_RAW which is both faster to read and higher resolution than
 * their version of CLOCK_MONOTONIC.
 *
 * Note this does not get used in case the TSC clock source logic is used,
 * which directly calls architecture specific timing instructions (e.g. RDTSC).
 */
#if defined(__darwin__) && defined(CLOCK_MONOTONIC_RAW)
#define PG_INSTR_SYSTEM_CLOCK	CLOCK_MONOTONIC_RAW
#define PG_INSTR_SYSTEM_CLOCK_NAME	"clock_gettime (CLOCK_MONOTONIC_RAW)"
#elif defined(CLOCK_MONOTONIC)
#define PG_INSTR_SYSTEM_CLOCK	CLOCK_MONOTONIC
#define PG_INSTR_SYSTEM_CLOCK_NAME	"clock_gettime (CLOCK_MONOTONIC)"
#else
#define PG_INSTR_SYSTEM_CLOCK	CLOCK_REALTIME
#define PG_INSTR_SYSTEM_CLOCK_NAME	"clock_gettime (CLOCK_REALTIME)"
#endif

static inline instr_time
pg_get_ticks_system(void)
{
	instr_time	now;
	struct timespec tmp;

	Assert(timing_initialized);

	clock_gettime(PG_INSTR_SYSTEM_CLOCK, &tmp);
	now.ticks = tmp.tv_sec * NS_PER_S + tmp.tv_nsec;

	return now;
}

#else							/* WIN32 */

/* On Windows, use QueryPerformanceCounter() for system clock source */

#define PG_INSTR_SYSTEM_CLOCK_NAME	"QueryPerformanceCounter"
static inline instr_time
pg_get_ticks_system(void)
{
	instr_time	now;
	LARGE_INTEGER tmp;

	Assert(timing_initialized);

	QueryPerformanceCounter(&tmp);
	now.ticks = tmp.QuadPart;

	return now;
}

#endif							/* WIN32 */

static inline int64
pg_ticks_to_ns(int64 ticks)
{
#if PG_INSTR_TICKS_TO_NS
	int64		ns = 0;

	Assert(timing_initialized);

	/*
	 * Avoid doing work if we don't use scaled ticks, e.g. system clock on
	 * Unix (in that case ticks is counted in nanoseconds)
	 */
	if (ticks_per_ns_scaled == 0)
		return ticks;

	/*
	 * Would multiplication overflow? If so perform computation in two parts.
	 */
	if (unlikely(ticks > (int64) max_ticks_no_overflow))
	{
		/*
		 * To avoid overflow, first scale total ticks down by the fixed
		 * factor, and *afterwards* multiply them by the frequency-based scale
		 * factor.
		 *
		 * The remaining ticks can follow the regular formula, since they
		 * won't overflow.
		 */
		int64		count = ticks >> TICKS_TO_NS_SHIFT;

		ns = count * ticks_per_ns_scaled;
		ticks -= (count << TICKS_TO_NS_SHIFT);
	}

	ns += (ticks * ticks_per_ns_scaled) >> TICKS_TO_NS_SHIFT;

	return ns;
#else
	Assert(timing_initialized);

	return ticks;
#endif							/* PG_INSTR_TICKS_TO_NS */
}

static inline int64
pg_ns_to_ticks(int64 ns)
{
#if PG_INSTR_TICKS_TO_NS
	int64		ticks = 0;

	Assert(timing_initialized);

	/*
	 * If ticks_per_ns_scaled is zero, ticks are already in nanoseconds (e.g.
	 * system clock on Unix).
	 */
	if (ticks_per_ns_scaled == 0)
		return ns;

	/*
	 * The reverse of pg_ticks_to_ns to avoid a similar overflow problem.
	 */
	if (unlikely(ns > (INT64_MAX >> TICKS_TO_NS_SHIFT)))
	{
		int64		count = ns / ticks_per_ns_scaled;

		ticks = count << TICKS_TO_NS_SHIFT;
		ns -= count * ticks_per_ns_scaled;
	}

	ticks += (ns << TICKS_TO_NS_SHIFT) / ticks_per_ns_scaled;

	return ticks;
#else
	Assert(timing_initialized);

	return ns;
#endif							/* PG_INSTR_TICKS_TO_NS */
}

#if PG_INSTR_TSC_CLOCK

#define PG_INSTR_TSC_CLOCK_NAME_FAST  "RDTSC"
#define PG_INSTR_TSC_CLOCK_NAME "RDTSCP"

#ifdef _MSC_VER
#include <intrin.h>
#endif							/* defined(_MSC_VER) */

/* Helpers to abstract compiler differences for reading the x86 TSC. */
static inline int64
pg_rdtsc(void)
{
#ifdef _MSC_VER
	return __rdtsc();
#else
	return __builtin_ia32_rdtsc();
#endif							/* defined(_MSC_VER) */
}

static inline int64
pg_rdtscp(void)
{
	uint32		unused;

#ifdef _MSC_VER
	return __rdtscp(&unused);
#else
	return __builtin_ia32_rdtscp(&unused);
#endif							/* defined(_MSC_VER) */
}

/*
 * Marked always_inline due to a shortcoming in gcc's heuristics leading to
 * only inlining the function partially.
 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124795
 */
static pg_always_inline instr_time
pg_get_ticks(void)
{
	if (likely(timing_tsc_enabled))
	{
		instr_time	now;

		now.ticks = pg_rdtscp();
		return now;
	}

	return pg_get_ticks_system();
}

static pg_always_inline instr_time
pg_get_ticks_fast(void)
{
	if (likely(timing_tsc_enabled))
	{
		instr_time	now;

		now.ticks = pg_rdtsc();
		return now;
	}

	return pg_get_ticks_system();
}

#else

static pg_always_inline instr_time
pg_get_ticks(void)
{
	return pg_get_ticks_system();
}

static pg_always_inline instr_time
pg_get_ticks_fast(void)
{
	return pg_get_ticks_system();
}

#endif							/* PG_INSTR_TSC_CLOCK */

/*
 * Common macros
 */

#define INSTR_TIME_IS_ZERO(t)	((t).ticks == 0)

#define INSTR_TIME_SET_ZERO(t)	((t).ticks = 0)

#define INSTR_TIME_SET_CURRENT_FAST(t) \
	((t) = pg_get_ticks_fast())

#define INSTR_TIME_SET_CURRENT(t) \
	((t) = pg_get_ticks())


#define INSTR_TIME_ADD(x,y) \
	((x).ticks += (y).ticks)

#define INSTR_TIME_ADD_NANOSEC(t, n) \
	((t).ticks += pg_ns_to_ticks(n))

#define INSTR_TIME_SUBTRACT(x,y) \
	((x).ticks -= (y).ticks)

#define INSTR_TIME_ACCUM_DIFF(x,y,z) \
	((x).ticks += (y).ticks - (z).ticks)

#define INSTR_TIME_GT(x,y) \
	((x).ticks > (y).ticks)

#define INSTR_TIME_GET_NANOSEC(t) \
	(pg_ticks_to_ns((t).ticks))

#define INSTR_TIME_GET_DOUBLE(t) \
	((double) INSTR_TIME_GET_NANOSEC(t) / NS_PER_S)

#define INSTR_TIME_GET_MILLISEC(t) \
	((double) INSTR_TIME_GET_NANOSEC(t) / NS_PER_MS)

#define INSTR_TIME_GET_MICROSEC(t) \
	(INSTR_TIME_GET_NANOSEC(t) / NS_PER_US)

#endif							/* INSTR_TIME_H */