providers/efa/verbs.c - master - githubmirror/rdma-core

providers/efa/verbs.c
3 555 строк93 KB
Yonatan Nachum
efa: Expose 64-bit request ID capability in query QP DV
04 авг 2026, 11:59
04 авг 2026, 11:59305f2f1
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
/*
 * Copyright 2019-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
 */

#include <assert.h>
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>

#include <ccan/minmax.h>

#include <util/compiler.h>
#include <util/mmio.h>
#include <util/util.h>

#include "efa.h"
#include "efa_io_regs_defs.h"
#include "efadv.h"
#include "verbs.h"
#include "efa_trace.h"

#define EFA_DEV_CAP(ctx, cap) \
	((ctx)->device_caps & EFA_QUERY_DEVICE_CAPS_##cap)

#define EFA_IO_TX_DESC_SIZE_64	(sizeof(struct efa_io_tx_wqe))
#define EFA_IO_TX_DESC_SIZE_128 (sizeof(struct efa_io_tx_wqe_128))

static bool is_buf_cleared(void *buf, size_t len)
{
	int i;

	for (i = 0; i < len; i++) {
		if (((uint8_t *)buf)[i])
			return false;
	}

	return true;
}

#define min3(a, b, c) \
	({ \
		typeof(a) _tmpmin = min(a, b); \
		min(_tmpmin, c); \
	})

#define is_ext_cleared(ptr, inlen) \
	is_buf_cleared((uint8_t *)ptr + sizeof(*ptr), inlen - sizeof(*ptr))

#define is_reserved_cleared(reserved) is_buf_cleared(reserved, sizeof(reserved))

struct efa_wq_init_attr {
	uint64_t db_mmap_key;
	uint32_t db_off;
	int cmd_fd;
	int pgsz;
	uint16_t sub_cq_idx;
	bool need_lock;
	uint16_t gen;
};

int efa_query_port_speed(struct ibv_context *context, uint32_t port,
			 uint64_t *speed)
{
	return ibv_cmd_query_port_speed(context, port, speed);
}

int efa_query_port(struct ibv_context *ibvctx, uint8_t port,
		   struct ibv_port_attr *port_attr)
{
	struct ibv_query_port cmd;

	return ibv_cmd_query_port(ibvctx, port, port_attr, &cmd, sizeof(cmd));
}

int efa_query_device_ex(struct ibv_context *context,
			const struct ibv_query_device_ex_input *input,
			struct ibv_device_attr_ex *attr,
			size_t attr_size)
{
	struct efa_context *ctx = to_efa_context(context);
	struct ibv_device_attr *a = &attr->orig_attr;
	struct efa_query_device_ex_resp resp = {};
	size_t resp_size = (ctx->cmds_supp_udata_mask &
			    EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE) ?
				   sizeof(resp) :
				   sizeof(resp.ibv_resp);
	uint8_t fw_ver[8];
	int err;

	err = ibv_cmd_query_device_any(context, input, attr, attr_size,
				       &resp.ibv_resp, &resp_size);
	if (err) {
		verbs_err(verbs_get_ctx(context), "ibv_cmd_query_device_any failed\n");
		return err;
	}

	a->max_qp_wr = min_t(int, a->max_qp_wr,
			     ctx->max_llq_size / EFA_IO_TX_DESC_SIZE_64);
	memcpy(fw_ver, &resp.ibv_resp.base.fw_ver,
	       sizeof(resp.ibv_resp.base.fw_ver));
	snprintf(a->fw_ver, sizeof(a->fw_ver), "%u.%u.%u.%u",
		 fw_ver[0], fw_ver[1], fw_ver[2], fw_ver[3]);

	return 0;
}

int efa_query_device_ctx(struct efa_context *ctx)
{
	struct efa_query_device_ex_resp resp = {};
	struct ibv_device_attr_ex attr;
	size_t resp_size = sizeof(resp);
	unsigned int qp_table_sz;
	int err;

	if (ctx->cmds_supp_udata_mask & EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE) {
		err = ibv_cmd_query_device_any(&ctx->ibvctx.context, NULL,
					       &attr, sizeof(attr),
					       &resp.ibv_resp, &resp_size);
		if (err) {
			verbs_err(&ctx->ibvctx,
				  "ibv_cmd_query_device_any failed\n");
			return err;
		}

		ctx->device_caps = resp.device_caps;
		ctx->max_sq_wr = resp.max_sq_wr;
		ctx->max_rq_wr = resp.max_rq_wr;
		ctx->max_sq_sge = resp.max_sq_sge;
		ctx->max_rq_sge = resp.max_rq_sge;
		ctx->max_rdma_size = resp.max_rdma_size;
	} else {
		err = ibv_cmd_query_device_any(&ctx->ibvctx.context, NULL,
					       &attr, sizeof(attr.orig_attr),
					       NULL, NULL);
		if (err) {
			verbs_err(&ctx->ibvctx,
				  "ibv_cmd_query_device_any failed\n");
			return err;
		}
	}

	ctx->max_wr_rdma_sge = attr.orig_attr.max_sge_rd;
	qp_table_sz = roundup_pow_of_two(attr.orig_attr.max_qp);
	ctx->qp_table_sz_m1 = qp_table_sz - 1;

	ctx->qp_table = calloc(qp_table_sz, sizeof(*ctx->qp_table));
	if (!ctx->qp_table)
		return ENOMEM;

	ctx->qp_gen_table = calloc(qp_table_sz, sizeof(*ctx->qp_gen_table));
	if (!ctx->qp_gen_table) {
		free(ctx->qp_table);
		return ENOMEM;
	}

	return 0;
}

int efadv_query_device(struct ibv_context *ibvctx,
		       struct efadv_device_attr *attr,
		       uint32_t inlen)
{
	struct efa_context *ctx = to_efa_context(ibvctx);
	uint64_t comp_mask_out = 0;

	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		return EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), inline_buf_size_ex, inlen)) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		return EINVAL;
	}

	memset(attr, 0, inlen);
	attr->max_sq_wr = ctx->max_sq_wr;
	attr->max_rq_wr = ctx->max_rq_wr;
	attr->max_sq_sge = ctx->max_sq_sge;
	attr->max_rq_sge = ctx->max_rq_sge;
	attr->inline_buf_size = ctx->inline_buf_size;
	attr->inline_buf_size_ex = ctx->inline_buf_size_ex;

	if (vext_field_avail(typeof(*attr), device_caps, inlen)) {
		if (EFA_DEV_CAP(ctx, RNR_RETRY))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_RNR_RETRY;

		if (EFA_DEV_CAP(ctx, CQ_WITH_SGID))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID;

		if (EFA_DEV_CAP(ctx, UNSOLICITED_WRITE_RECV))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV;

		if (EFA_DEV_CAP(ctx, CQ_WITH_EXT_MEM))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF;

		if (EFA_DEV_CAP(ctx, COMP_CNTR))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_COMP_CNTR;
	}

	if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) {
		attr->max_rdma_size = ctx->max_rdma_size;

		if (EFA_DEV_CAP(ctx, RDMA_READ))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_RDMA_READ;

		if (EFA_DEV_CAP(ctx, RDMA_WRITE))
			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE;
	}

	attr->comp_mask = comp_mask_out;

	return 0;
}

struct ibv_pd *efa_alloc_pd(struct ibv_context *ibvctx)
{
	struct efa_alloc_pd_resp resp = {};
	struct ibv_alloc_pd cmd;
	struct efa_pd *pd;
	int err;

	pd = calloc(1, sizeof(*pd));
	if (!pd)
		return NULL;

	err = ibv_cmd_alloc_pd(ibvctx, &pd->ibvpd, &cmd, sizeof(cmd),
			       &resp.ibv_resp, sizeof(resp));
	if (err) {
		verbs_err(verbs_get_ctx(ibvctx), "Failed to allocate PD\n");
		goto out;
	}

	atomic_init(&pd->refcount, 0);
	pd->pdn = resp.pdn;

	return &pd->ibvpd;

out:
	free(pd);
	errno = err;
	return NULL;
}

struct ibv_pd *efa_alloc_parent_domain(struct ibv_context *ibvctx,
				       struct ibv_parent_domain_init_attr *attr)
{
	struct efa_parent_domain *parent_domain;
	struct efa_pd *pd;

	if (ibv_check_alloc_parent_domain(attr)) {
		errno = EINVAL;
		return NULL;
	}

	if (!check_comp_mask(attr->comp_mask,
			     IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT)) {
		verbs_err(verbs_get_ctx(ibvctx), "Invalid comp_mask\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	pd = to_efa_pd(attr->pd);

	/* We don't allow nested parent domains */
	if (pd->orig_pd) {
		errno = EINVAL;
		return NULL;
	}

	parent_domain = calloc(1, sizeof(*parent_domain));
	if (!parent_domain) {
		errno = ENOMEM;
		return NULL;
	}

	atomic_init(&parent_domain->refcount, 0);

	if (attr->td) {
		parent_domain->td = to_efa_td(attr->td);
		atomic_fetch_add(&parent_domain->td->refcount, 1);
	}

	parent_domain->pd.orig_pd = pd;
	atomic_fetch_add(&pd->refcount, 1);

	ibv_initialize_parent_domain(&parent_domain->pd.ibvpd, attr->pd);

	if (attr->comp_mask & IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT)
		parent_domain->pd_context = attr->pd_context;

	return &parent_domain->pd.ibvpd;
}

static int efa_dealloc_parent_domain(struct efa_parent_domain *parent_domain)
{
	if (atomic_load(&parent_domain->refcount) > 0)
		return EBUSY;

	atomic_fetch_sub(&parent_domain->pd.orig_pd->refcount, 1);

	if (parent_domain->td)
		atomic_fetch_sub(&parent_domain->td->refcount, 1);

	free(parent_domain);
	return 0;
}

int efa_dealloc_pd(struct ibv_pd *ibvpd)
{
	struct efa_parent_domain *parent_domain;
	struct efa_pd *pd = to_efa_pd(ibvpd);
	int err;

	if (pd->orig_pd) {
		parent_domain = to_efa_parent_domain(ibvpd);
		return efa_dealloc_parent_domain(parent_domain);
	}

	if (atomic_load(&pd->refcount) > 0)
		return EBUSY;

	err = ibv_cmd_dealloc_pd(ibvpd);
	if (err) {
		verbs_err(verbs_get_ctx(ibvpd->context),
			  "Failed to deallocate PD\n");
		return err;
	}
	free(pd);

	return 0;
}

struct ibv_mr *efa_reg_dmabuf_mr(struct ibv_pd *ibvpd, uint64_t offset,
				 size_t length, uint64_t iova, int fd, int acc)
{
	struct efa_mr *mr;
	int err;

	mr = calloc(1, sizeof(*mr));
	if (!mr)
		return NULL;

	err = ibv_cmd_reg_dmabuf_mr(ibvpd, offset, length, iova, fd, acc,
				    &mr->vmr, NULL);
	if (err) {
		free(mr);
		errno = err;
		return NULL;
	}

	return &mr->vmr.ibv_mr;
}

struct ibv_mr *efa_reg_mr(struct ibv_pd *ibvpd, void *sva, size_t len,
			  uint64_t hca_va, int access)
{
	struct ib_uverbs_reg_mr_resp resp;
	struct ibv_reg_mr cmd;
	struct efa_mr *mr;
	int err;

	mr = calloc(1, sizeof(*mr));
	if (!mr)
		return NULL;

	err = ibv_cmd_reg_mr(ibvpd, sva, len, hca_va, access, &mr->vmr,
			     &cmd, sizeof(cmd), &resp, sizeof(resp));
	if (err) {
		verbs_err(verbs_get_ctx(ibvpd->context),
			  "Failed to register MR\n");
		free(mr);
		errno = err;
		return NULL;
	}

	return &mr->vmr.ibv_mr;
}

int efadv_query_mr(struct ibv_mr *ibvmr, struct efadv_mr_attr *attr, uint32_t inlen)
{
	uint16_t rdma_read_ic_id = 0;
	uint16_t rdma_recv_ic_id = 0;
	uint16_t ic_id_validity = 0;
	uint16_t recv_ic_id = 0;
	int err;

	DECLARE_COMMAND_BUFFER(cmd,
			       UVERBS_OBJECT_MR,
			       EFA_IB_METHOD_MR_QUERY,
			       5);

	if (!is_efa_dev(ibvmr->context->device)) {
		verbs_err(verbs_get_ctx(ibvmr->context), "Not an EFA device\n");
		return EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), rdma_recv_ic_id, inlen)) {
		verbs_err(verbs_get_ctx(ibvmr->context), "Compatibility issues\n");
		return EINVAL;
	}

	memset(attr, 0, inlen);
	fill_attr_in_obj(cmd, EFA_IB_ATTR_QUERY_MR_HANDLE, ibvmr->handle);
	fill_attr_out(cmd, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
		      &ic_id_validity, sizeof(ic_id_validity));
	fill_attr_out(cmd, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
		      &recv_ic_id, sizeof(recv_ic_id));
	fill_attr_out(cmd, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
		      &rdma_read_ic_id, sizeof(rdma_read_ic_id));
	fill_attr_out(cmd, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
		      &rdma_recv_ic_id, sizeof(rdma_recv_ic_id));

	err = execute_ioctl(ibvmr->context, cmd);
	if (err) {
		verbs_err(verbs_get_ctx(ibvmr->context), "Failed to query MR\n");
		return err;
	}

	if (ic_id_validity & EFA_QUERY_MR_VALIDITY_RECV_IC_ID) {
		attr->recv_ic_id = recv_ic_id;
		attr->ic_id_validity |= EFADV_MR_ATTR_VALIDITY_RECV_IC_ID;
	}
	if (ic_id_validity & EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID) {
		attr->rdma_read_ic_id = rdma_read_ic_id;
		attr->ic_id_validity |= EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID;
	}
	if (ic_id_validity & EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID) {
		attr->rdma_recv_ic_id = rdma_recv_ic_id;
		attr->ic_id_validity |= EFADV_MR_ATTR_VALIDITY_RDMA_RECV_IC_ID;
	}

	return 0;
}

int efa_dereg_mr(struct verbs_mr *vmr)
{
	struct efa_mr *mr = container_of(vmr, struct efa_mr, vmr);
	int err;

	err = ibv_cmd_dereg_mr(vmr);
	if (err) {
		verbs_err(verbs_get_ctx(vmr->ibv_mr.context),
			  "Failed to deregister MR\n");
		return err;
	}
	free(mr);

	return 0;
}

static uint32_t efa_wq_get_next_wrid_idx_locked(struct efa_wq *wq,
						uint64_t wr_id)
{
	uint32_t wrid_idx;

	/* Get the next wrid to be used from the index pool */
	wrid_idx = wq->wrid_idx_pool[wq->wrid_idx_pool_next];
	wq->wrid[wrid_idx] = wr_id;

	/* Will never overlap, as validate function succeeded */
	wq->wrid_idx_pool_next++;
	assert(wq->wrid_idx_pool_next <= wq->wqe_cnt);

	return wrid_idx;
}

static void efa_wq_put_wrid_idx_locked(struct efa_wq *wq, uint32_t wrid_idx)
{
	wq->wrid_idx_pool_next--;
	wq->wrid_idx_pool[wq->wrid_idx_pool_next] = wrid_idx;
}

static uint32_t efa_wq_get_dev_req_id_locked(struct efa_wq *wq, uint64_t wr_id)
{
	return efa_wq_get_next_wrid_idx_locked(wq, wr_id) | wq->shifted_gen;
}

static void efa_wq_cqe_finalize(struct efa_wq *wq, struct efa_io_cdesc_common *cqe)
{
	if (wq->need_lock)
		pthread_spin_lock(&wq->wqlock);

	wq->wqe_completed++;
	if (!wq->req_id_64_bit)
		efa_wq_put_wrid_idx_locked(wq, cqe->req_id & ~wq->gen_mask);

	if (wq->need_lock)
		pthread_spin_unlock(&wq->wqlock);
}

static uint64_t efa_wq_get_wrid_by_dev_req_id(struct efa_wq *wq, uint16_t dev_req_id)
{
	return wq->wrid[dev_req_id & ~wq->gen_mask];
}

static inline uint64_t efa_cqe_get_dev_req_id_64(struct efa_io_tx_cdesc *tcqe) ALWAYS_INLINE;
static inline uint64_t efa_cqe_get_dev_req_id_64(struct efa_io_tx_cdesc *tcqe)
{
	struct efa_io_req_id_ex *req_id_ex = &tcqe->req_id_ex;

	return (uint64_t)tcqe->common.req_id |
	       (uint64_t)req_id_ex->w[0] << 16 |
	       (uint64_t)req_id_ex->w[1] << 32 |
	       (uint64_t)req_id_ex->w[2] << 48;
}

static inline uint64_t efa_get_sq_comp_wrid(struct efa_wq *wq,
					    struct efa_io_cdesc_common *cqe) ALWAYS_INLINE;
static inline uint64_t efa_get_sq_comp_wrid(struct efa_wq *wq,
					    struct efa_io_cdesc_common *cqe)
{
	if (wq->req_id_64_bit) {
		struct efa_io_tx_cdesc *tcqe =
			container_of(cqe, struct efa_io_tx_cdesc, common);

		return efa_cqe_get_dev_req_id_64(tcqe);
	}

	/* We do not have to take the WQ lock here,
	 * because this wrid index has not been freed yet,
	 * so there is no contention on this index.
	 */
	return efa_wq_get_wrid_by_dev_req_id(wq, cqe->req_id);
}

static inline void efa_set_sq_comp_wrid(struct efa_io_tx_meta_desc *md, struct efa_wq *wq,
					uint64_t wr_id) ALWAYS_INLINE;
static inline void efa_set_sq_comp_wrid(struct efa_io_tx_meta_desc *md, struct efa_wq *wq,
					uint64_t wr_id)
{
	if (wq->req_id_64_bit) {
		md->req_id = (uint16_t)wr_id;
		md->req_id_ex.w[0] = (uint16_t)(wr_id >> 16);
		md->req_id_ex.w[1] = (uint16_t)(wr_id >> 32);
		md->req_id_ex.w[2] = (uint16_t)(wr_id >> 48);
	} else {
		md->req_id = efa_wq_get_dev_req_id_locked(wq, wr_id);
	}
}

static uint32_t efa_sub_cq_get_current_index(struct efa_sub_cq *sub_cq)
{
	return sub_cq->consumed_cnt & sub_cq->qmask;
}

static int efa_cqe_is_pending(struct efa_io_cdesc_common *cqe_common,
			      int phase)
{
	return EFA_GET(&cqe_common->flags, EFA_IO_CDESC_COMMON_PHASE) == phase;
}

static struct efa_io_cdesc_common *
efa_sub_cq_get_cqe(struct efa_sub_cq *sub_cq, int entry)
{
	return (struct efa_io_cdesc_common *)(sub_cq->buf +
					      (entry * sub_cq->cqe_size));
}

static void efa_update_cq_doorbell(struct efa_cq *cq, bool arm)
{
	uint32_t db = 0;

	EFA_SET(&db, EFA_IO_REGS_CQ_DB_CONSUMER_INDEX, cq->cc);
	EFA_SET(&db, EFA_IO_REGS_CQ_DB_CMD_SN, cq->cmd_sn & 0x3);
	EFA_SET(&db, EFA_IO_REGS_CQ_DB_ARM, arm);

	mmio_write32(cq->db, db);
}

void efa_cq_event(struct ibv_cq *ibvcq)
{
	to_efa_cq(ibvcq)->cmd_sn++;
}

int efa_arm_cq(struct ibv_cq *ibvcq, int solicited_only)
{
	if (unlikely(solicited_only))
		return EOPNOTSUPP;

	efa_update_cq_doorbell(to_efa_cq(ibvcq), true);
	return 0;
}

static struct efa_io_cdesc_common *
cq_next_sub_cqe_get(struct efa_sub_cq *sub_cq)
{
	struct efa_io_cdesc_common *cqe;
	uint32_t current_index;

	current_index = efa_sub_cq_get_current_index(sub_cq);
	cqe = efa_sub_cq_get_cqe(sub_cq, current_index);
	if (efa_cqe_is_pending(cqe, sub_cq->phase)) {
		/* Do not read the rest of the completion entry before the
		 * phase bit has been validated.
		 */
		udma_from_device_barrier();
		sub_cq->consumed_cnt++;
		if (!efa_sub_cq_get_current_index(sub_cq))
			sub_cq->phase = 1 - sub_cq->phase;
		return cqe;
	}

	return NULL;
}

static enum ibv_wc_status to_ibv_status(enum efa_io_comp_status status)
{
	switch (status) {
	case EFA_IO_COMP_STATUS_OK:
		return IBV_WC_SUCCESS;
	case EFA_IO_COMP_STATUS_FLUSHED:
		return IBV_WC_WR_FLUSH_ERR;
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR:
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP:
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH:
		return IBV_WC_LOC_QP_OP_ERR;
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY:
		return IBV_WC_LOC_PROT_ERR;
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH:
		return IBV_WC_LOC_LEN_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT:
		return IBV_WC_REM_ABORT_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR:
		return IBV_WC_RNR_RETRY_EXC_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN:
		return IBV_WC_REM_INV_RD_REQ_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS:
		return IBV_WC_BAD_RESP_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_FEATURE_MISMATCH:
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH:
		return IBV_WC_REM_INV_REQ_ERR;
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE:
	case EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE:
		return IBV_WC_RESP_TIMEOUT_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS:
		return IBV_WC_REM_ACCESS_ERR;
	case EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER:
		return IBV_WC_REM_OP_ERR;
	default:
		return IBV_WC_GENERAL_ERR;
	}
}

static enum ibv_wc_opcode efa_wc_read_opcode(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	enum efa_io_send_op_type op_type;
	struct efa_io_cdesc_common *cqe;

	cqe = cq->cur_cqe;
	op_type = EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_OP_TYPE);

	if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) ==
		    EFA_IO_SEND_QUEUE) {
		if (op_type == EFA_IO_RDMA_WRITE)
			return IBV_WC_RDMA_WRITE;

		return IBV_WC_SEND;
	}

	if (op_type == EFA_IO_RDMA_WRITE)
		return IBV_WC_RECV_RDMA_WITH_IMM;

	return IBV_WC_RECV;
}

static uint32_t efa_wc_read_vendor_err(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	return cq->cur_cqe->status;
}

static unsigned int efa_wc_read_wc_flags(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	unsigned int wc_flags = 0;

	if (EFA_GET(&cq->cur_cqe->flags, EFA_IO_CDESC_COMMON_HAS_IMM))
		wc_flags |= IBV_WC_WITH_IMM;

	return wc_flags;
}

static uint32_t efa_wc_read_byte_len(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	struct efa_io_cdesc_common *cqe;
	struct efa_io_rx_cdesc_ex *rcqe;
	uint32_t length;

	cqe = cq->cur_cqe;

	if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) != EFA_IO_RECV_QUEUE)
		return 0;

	rcqe = container_of(cqe, struct efa_io_rx_cdesc_ex, base.common);

	length = rcqe->base.length;
	if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_OP_TYPE) == EFA_IO_RDMA_WRITE)
		length |= ((uint32_t)rcqe->u.rdma_write.length_hi << 16);

	return length;
}

static __be32 efa_wc_read_imm_data(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	struct efa_io_rx_cdesc *rcqe;

	rcqe = container_of(cq->cur_cqe, struct efa_io_rx_cdesc, common);

	return htobe32(rcqe->imm);
}

static uint32_t efa_wc_read_qp_num(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	return cq->cur_cqe->qp_num;
}

static uint32_t efa_wc_read_src_qp(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	struct efa_io_rx_cdesc *rcqe;

	rcqe = container_of(cq->cur_cqe, struct efa_io_rx_cdesc, common);

	return rcqe->src_qp_num;
}

static uint32_t efa_wc_read_slid(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	struct efa_io_rx_cdesc *rcqe;

	rcqe = container_of(cq->cur_cqe, struct efa_io_rx_cdesc, common);

	return rcqe->ah;
}

static uint8_t efa_wc_read_sl(struct ibv_cq_ex *ibvcqx)
{
	return 0;
}

static uint8_t efa_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcqx)
{
	return 0;
}

static int efa_wc_read_sgid(struct efadv_cq *efadv_cq, union ibv_gid *sgid)
{
	struct efa_cq *cq = efadv_cq_to_efa_cq(efadv_cq);
	struct efa_io_rx_cdesc_ex *rcqex;

	rcqex = container_of(cq->cur_cqe, struct efa_io_rx_cdesc_ex,
			     base.common);
	if (rcqex->base.ah != 0xFFFF) {
		/* SGID is only available if AH is unknown. */
		return -ENOENT;
	}
	memcpy(sgid->raw, rcqex->u.src_addr, sizeof(sgid->raw));

	return 0;
}

static bool efa_cqe_is_unsolicited(struct efa_io_cdesc_common *cqe)
{
	return EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_UNSOLICITED);
}

static bool efa_wc_is_unsolicited(struct efadv_cq *efadv_cq)
{
	struct efa_cq *cq = efadv_cq_to_efa_cq(efadv_cq);

	return efa_cqe_is_unsolicited(cq->cur_cqe);
}

static void efa_process_cqe(struct efa_cq *cq, struct ibv_wc *wc,
			    struct efa_qp *qp)
{
	struct efa_io_cdesc_common *cqe = cq->cur_cqe;
	enum efa_io_send_op_type op_type;

	wc->status = to_ibv_status(cqe->status);
	wc->vendor_err = cqe->status;
	wc->wc_flags = 0;
	wc->qp_num = cqe->qp_num;

	op_type = EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_OP_TYPE);
	if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) == EFA_IO_SEND_QUEUE) {
		if (op_type == EFA_IO_RDMA_WRITE)
			wc->opcode = IBV_WC_RDMA_WRITE;
		else
			wc->opcode = IBV_WC_SEND;

		cq->cur_wq = &qp->sq.wq;
		wc->wr_id = efa_get_sq_comp_wrid(cq->cur_wq, cqe);

		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, wc->wr_id,
				wc->status, wc->opcode, wc->qp_num, UINT32_MAX, UINT16_MAX,
				wc->byte_len);
	} else {
		struct efa_io_rx_cdesc_ex *rcqe =
			container_of(cqe, struct efa_io_rx_cdesc_ex, base.common);

		if (efa_cqe_is_unsolicited(cqe)) {
			cq->cur_wq = NULL;
			wc->wr_id = 0;
		} else {
			cq->cur_wq = &qp->rq.wq;
			wc->wr_id = efa_wq_get_wrid_by_dev_req_id(cq->cur_wq, cqe->req_id);
		}

		wc->byte_len = rcqe->base.length;

		if (op_type == EFA_IO_RDMA_WRITE) {
			wc->byte_len |= ((uint32_t)rcqe->u.rdma_write.length_hi << 16);
			wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
		} else {
			wc->opcode = IBV_WC_RECV;
		}

		wc->src_qp = rcqe->base.src_qp_num;
		wc->sl = 0;
		wc->slid = rcqe->base.ah;

		if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_HAS_IMM)) {
			wc->imm_data = htobe32(rcqe->base.imm);
			wc->wc_flags |= IBV_WC_WITH_IMM;
		}

		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, wc->wr_id,
				wc->status, wc->opcode, wc->src_qp, wc->qp_num, wc->slid,
				wc->byte_len);
	}
}

static void efa_process_ex_cqe(struct efa_cq *cq, struct efa_qp *qp)
{
	struct ibv_cq_ex *ibvcqx = &cq->verbs_cq.cq_ex;
	struct efa_io_cdesc_common *cqe = cq->cur_cqe;

	if (EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) == EFA_IO_SEND_QUEUE) {
		cq->cur_wq = &qp->sq.wq;
		ibvcqx->wr_id = efa_get_sq_comp_wrid(cq->cur_wq, cqe);
		ibvcqx->status = to_ibv_status(cqe->status);

		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, ibvcqx->wr_id,
				ibvcqx->status, efa_wc_read_opcode(ibvcqx), cqe->qp_num,
				UINT32_MAX, UINT16_MAX, efa_wc_read_byte_len(ibvcqx));
	} else {
		if (efa_cqe_is_unsolicited(cqe)) {
			cq->cur_wq = NULL;
			ibvcqx->wr_id = 0;
		} else {
			cq->cur_wq = &qp->rq.wq;
			ibvcqx->wr_id = efa_wq_get_wrid_by_dev_req_id(cq->cur_wq, cqe->req_id);
		}

		ibvcqx->status = to_ibv_status(cqe->status);

		rdma_tracepoint(rdma_core_efa, process_completion, cq->dev->name, ibvcqx->wr_id,
				ibvcqx->status, efa_wc_read_opcode(ibvcqx),
				efa_wc_read_src_qp(ibvcqx), cqe->qp_num, efa_wc_read_slid(ibvcqx),
				efa_wc_read_byte_len(ibvcqx));
	}
}

static bool efa_cqe_is_valid_req_id_qp_gen(struct efa_io_cdesc_common *cqe, struct efa_qp *qp)
{
	struct efa_wq *wq;

	wq = EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) == EFA_IO_SEND_QUEUE ?
		&qp->sq.wq : &qp->rq.wq;

	return (cqe->req_id & wq->gen_mask) == wq->shifted_gen;
}

static bool efa_cqe_is_64_bit_comp(struct efa_io_cdesc_common *cqe, struct efa_qp *qp)
{
	bool sq_cqe = EFA_GET(&cqe->flags, EFA_IO_CDESC_COMMON_Q_TYPE) == EFA_IO_SEND_QUEUE;

	return (sq_cqe && qp->sq.wq.req_id_64_bit);
}

static inline int efa_poll_sub_cq(struct efa_cq *cq, struct efa_sub_cq *sub_cq,
				  struct ibv_wc *wc,
				  bool extended) ALWAYS_INLINE;
static inline int efa_poll_sub_cq(struct efa_cq *cq, struct efa_sub_cq *sub_cq,
				  struct ibv_wc *wc, bool extended)
{
	struct efa_context *ctx = to_efa_context(cq->verbs_cq.cq.context);
	struct efa_qp *qp;
	uint32_t qpn;

	cq->cur_cqe = cq_next_sub_cqe_get(sub_cq);
	if (!cq->cur_cqe)
		return ENOENT;

	qpn = cq->cur_cqe->qp_num;
	/* We do not have to take the QP table lock here,
	 * because CQs will be locked while QPs are removed
	 * from the table.
	 */
	qp = ctx->qp_table[qpn & ctx->qp_table_sz_m1];
	if (!qp || qpn != qp->verbs_qp.qp.qp_num ||
	    (!efa_cqe_is_unsolicited(cq->cur_cqe) &&
	     !efa_cqe_is_64_bit_comp(cq->cur_cqe, qp) &&
	     !efa_cqe_is_valid_req_id_qp_gen(cq->cur_cqe, qp))) {
		cq->cur_wq = NULL;
		verbs_err(&ctx->ibvctx, "Invalid QP[%u]\n", qpn);
		return EINVAL;
	}

	if (extended) {
		efa_process_ex_cqe(cq, qp);
	} else {
		efa_process_cqe(cq, wc, qp);
		if (cq->cur_wq)
			efa_wq_cqe_finalize(cq->cur_wq, cq->cur_cqe);
	}

	return 0;
}

static inline int efa_poll_sub_cqs(struct efa_cq *cq, struct ibv_wc *wc,
				   bool extended) ALWAYS_INLINE;
static inline int efa_poll_sub_cqs(struct efa_cq *cq, struct ibv_wc *wc,
				   bool extended)
{
	uint16_t num_sub_cqs = cq->num_sub_cqs;
	struct efa_sub_cq *sub_cq;
	uint16_t sub_cq_idx;
	int err = ENOENT;

	for (sub_cq_idx = 0; sub_cq_idx < num_sub_cqs; sub_cq_idx++) {
		sub_cq = &cq->sub_cq_arr[cq->next_poll_idx++];
		cq->next_poll_idx %= num_sub_cqs;

		err = efa_poll_sub_cq(cq, sub_cq, wc, extended);
		if (err != ENOENT) {
			cq->cc++;
			break;
		}
	}

	return err;
}

int efa_poll_cq(struct ibv_cq *ibvcq, int nwc, struct ibv_wc *wc)
{
	struct efa_cq *cq = to_efa_cq(ibvcq);
	int ret = 0;
	int i;

	pthread_spin_lock(&cq->lock);
	for (i = 0; i < nwc; i++) {
		ret = efa_poll_sub_cqs(cq, &wc[i], false);
		if (ret) {
			if (ret == ENOENT)
				ret = 0;
			break;
		}
	}

	if (i && cq->db)
		efa_update_cq_doorbell(cq, false);
	pthread_spin_unlock(&cq->lock);

	return i ?: -ret;
}

static inline int efa_start_poll_comp_check(struct ibv_cq_ex *ibvcqx,
					    struct ibv_poll_cq_attr *attr) ALWAYS_INLINE;
static inline int efa_start_poll_comp_check(struct ibv_cq_ex *ibvcqx,
					    struct ibv_poll_cq_attr *attr)
{
	if (unlikely(attr->comp_mask)) {
		verbs_err(verbs_get_ctx(ibvcqx->context),
			  "Invalid comp_mask %u\n",
			  attr->comp_mask);
		return EINVAL;
	}

	return 0;
}

static inline void efa_end_poll_common(struct efa_cq *cq) ALWAYS_INLINE;
static inline void efa_end_poll_common(struct efa_cq *cq)
{
	if (cq->cur_cqe) {
		if (cq->cur_wq)
			efa_wq_cqe_finalize(cq->cur_wq, cq->cur_cqe);
		if (cq->db)
			efa_update_cq_doorbell(cq, false);
	}
}

static int efa_start_poll(struct ibv_cq_ex *ibvcqx,
			  struct ibv_poll_cq_attr *attr)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	int ret;

	if (efa_start_poll_comp_check(ibvcqx, attr))
		return EINVAL;

	pthread_spin_lock(&cq->lock);

	ret = efa_poll_sub_cqs(cq, NULL, true);
	if (ret)
		pthread_spin_unlock(&cq->lock);

	return ret;
}

static int efa_next_poll(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	int ret;

	if (cq->cur_wq)
		efa_wq_cqe_finalize(cq->cur_wq, cq->cur_cqe);
	ret = efa_poll_sub_cqs(cq, NULL, true);

	return ret;
}

static void efa_end_poll(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	efa_end_poll_common(cq);
	pthread_spin_unlock(&cq->lock);
}

static int efa_start_poll_single_sub_cq(struct ibv_cq_ex *ibvcqx,
					struct ibv_poll_cq_attr *attr)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	int ret;

	if (efa_start_poll_comp_check(ibvcqx, attr))
		return EINVAL;

	pthread_spin_lock(&cq->lock);
	ret = efa_poll_sub_cq(cq, cq->sub_cq_arr, NULL, true);
	if (ret != ENOENT)
		cq->cc++;

	if (ret)
		pthread_spin_unlock(&cq->lock);

	return ret;
}

static int efa_next_poll_single_sub_cq(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	int ret;

	if (cq->cur_wq)
		efa_wq_cqe_finalize(cq->cur_wq, cq->cur_cqe);

	ret = efa_poll_sub_cq(cq, cq->sub_cq_arr, NULL, true);
	if (ret != ENOENT)
		cq->cc++;

	return ret;
}

static int efa_start_poll_single_thread(struct ibv_cq_ex *ibvcqx,
					struct ibv_poll_cq_attr *attr)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	if (efa_start_poll_comp_check(ibvcqx, attr))
		return EINVAL;

	return efa_poll_sub_cqs(cq, NULL, true);
}

static void efa_end_poll_single_thread(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	efa_end_poll_common(cq);
}

static int efa_start_poll_single_sub_cq_single_thread(struct ibv_cq_ex *ibvcqx,
						      struct ibv_poll_cq_attr *attr)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);
	int ret;

	if (efa_start_poll_comp_check(ibvcqx, attr))
		return EINVAL;

	ret = efa_poll_sub_cq(cq, cq->sub_cq_arr, NULL, true);
	if (ret != ENOENT)
		cq->cc++;

	return ret;
}

enum cq_pfns_attr {
	SINGLE_SUB_CQ_PFNS = BIT(0),
	SINGLE_THREAD_PFNS = BIT(1),
};

#define efa_start_poll_name(single_sub_cq, single_thread) efa_start_poll##single_sub_cq##single_thread
#define efa_next_poll_name(single_sub_cq) efa_next_poll##single_sub_cq
#define efa_end_poll_name(single_thread) efa_end_poll##single_thread

#define POLL_FN_ENTRY(single_sub_cq, single_thread) { \
		.start_poll = efa_start_poll_name(single_sub_cq, single_thread), \
		.next_poll = efa_next_poll_name(single_sub_cq), \
		.end_poll = efa_end_poll_name(single_thread), \
	}

static struct cq_base_ops {
	int (*start_poll)(struct ibv_cq_ex *ibcq, struct ibv_poll_cq_attr *attr);
	int (*next_poll)(struct ibv_cq_ex *ibcq);
	void (*end_poll)(struct ibv_cq_ex *ibcq);
} base_ops[] = {
	[0] = POLL_FN_ENTRY(,),
	[SINGLE_SUB_CQ_PFNS] = POLL_FN_ENTRY(_single_sub_cq,),
	[SINGLE_THREAD_PFNS] = POLL_FN_ENTRY(, _single_thread),
	[SINGLE_SUB_CQ_PFNS | SINGLE_THREAD_PFNS] = POLL_FN_ENTRY(_single_sub_cq, _single_thread)
};

static void efa_cq_fill_pfns(struct efa_cq *cq,
			     struct ibv_cq_init_attr_ex *attr,
			     struct efadv_cq_init_attr *efa_attr)
{
	struct ibv_cq_ex *ibvcqx = &cq->verbs_cq.cq_ex;
	const struct cq_base_ops *cq_ops;
	uint32_t cq_pfns_mask = 0;

	if (cq->num_sub_cqs == 1)
		cq_pfns_mask |= SINGLE_SUB_CQ_PFNS;

	if ((cq->parent_domain && cq->parent_domain->td) ||
	    attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED)
		cq_pfns_mask |= SINGLE_THREAD_PFNS;

	cq_ops = &base_ops[cq_pfns_mask];
	ibvcqx->start_poll = cq_ops->start_poll;
	ibvcqx->next_poll = cq_ops->next_poll;
	ibvcqx->end_poll = cq_ops->end_poll;

	ibvcqx->read_opcode = efa_wc_read_opcode;
	ibvcqx->read_vendor_err = efa_wc_read_vendor_err;
	ibvcqx->read_wc_flags = efa_wc_read_wc_flags;

	if (attr->wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
		ibvcqx->read_byte_len = efa_wc_read_byte_len;
	if (attr->wc_flags & IBV_WC_EX_WITH_IMM)
		ibvcqx->read_imm_data = efa_wc_read_imm_data;
	if (attr->wc_flags & IBV_WC_EX_WITH_QP_NUM)
		ibvcqx->read_qp_num = efa_wc_read_qp_num;
	if (attr->wc_flags & IBV_WC_EX_WITH_SRC_QP)
		ibvcqx->read_src_qp = efa_wc_read_src_qp;
	if (attr->wc_flags & IBV_WC_EX_WITH_SLID)
		ibvcqx->read_slid = efa_wc_read_slid;
	if (attr->wc_flags & IBV_WC_EX_WITH_SL)
		ibvcqx->read_sl = efa_wc_read_sl;
	if (attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
		ibvcqx->read_dlid_path_bits = efa_wc_read_dlid_path_bits;

	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
		cq->dv_cq.wc_read_sgid = efa_wc_read_sgid;
	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED)
		cq->dv_cq.wc_is_unsolicited = efa_wc_is_unsolicited;
}

static void efa_sub_cq_initialize(struct efa_sub_cq *sub_cq, uint8_t *buf,
				  int sub_cq_size, int cqe_size)
{
	sub_cq->consumed_cnt = 0;
	sub_cq->phase = 1;
	sub_cq->buf = buf;
	sub_cq->qmask = sub_cq_size - 1;
	sub_cq->cqe_size = cqe_size;
}

static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
				   struct ibv_cq_init_attr_ex *attr,
				   struct efadv_cq_init_attr *efa_attr)
{
	struct efa_context *ctx = to_efa_context(ibvctx);
	struct verbs_create_cq_prov_attr prov_attr = {};
	struct efa_parent_domain *parent_domain = NULL;
	uint16_t cqe_size = ctx->ex_cqe_size;
	struct efa_create_cq_resp resp = {};
	struct efa_create_cq cmd = {};
	uint32_t cmd_flags = 0;
	uint16_t num_sub_cqs;
	struct efa_cq *cq;
	struct efa_pd *pd;
	int sub_buf_size;
	int sub_cq_size;
	uint8_t *buf;
	int err;
	int i;

#define EFA_CREATE_CQ_SUPP_ATTR_MASK \
	(IBV_CQ_INIT_ATTR_MASK_PD | IBV_CQ_INIT_ATTR_MASK_FLAGS)

	if (!check_comp_mask(attr->comp_mask, EFA_CREATE_CQ_SUPP_ATTR_MASK) ||
	    !check_comp_mask(attr->wc_flags, IBV_WC_STANDARD_FLAGS)) {
		verbs_err(verbs_get_ctx(ibvctx),
			  "Invalid comp_mask or wc_flags\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS &&
	    !check_comp_mask(attr->flags, IBV_CREATE_CQ_ATTR_SINGLE_THREADED)) {
		verbs_err(verbs_get_ctx(ibvctx),
			  "Invalid flags\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (attr->channel &&
	    !EFA_DEV_CAP(ctx, CQ_NOTIFICATIONS)) {
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) {
		pd = to_efa_pd(attr->parent_domain);
		if (!pd->orig_pd) {
			verbs_err(verbs_get_ctx(ibvctx), "Parent domain set but not provided\n");
			errno = EINVAL;
			return NULL;
		}

		parent_domain = to_efa_parent_domain(attr->parent_domain);
	}

	cq = calloc(1, sizeof(*cq) +
		       sizeof(*cq->sub_cq_arr) * ctx->sub_cqs_per_cq);
	if (!cq)
		return NULL;

	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
		cmd.flags |= EFA_CREATE_CQ_WITH_SGID;

	num_sub_cqs = ctx->sub_cqs_per_cq;
	cmd.num_sub_cqs = num_sub_cqs;
	cmd.cq_entry_size = cqe_size;

	if (efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF) {
		prov_attr.buffer.length = efa_attr->ext_mem_dmabuf.length;
		prov_attr.buffer.dmabuf.offset = efa_attr->ext_mem_dmabuf.offset;
		prov_attr.buffer.dmabuf.fd = efa_attr->ext_mem_dmabuf.fd;
		cmd_flags = CREATE_CQ_CMD_FLAGS_WITH_MEM_DMABUF;
	}

	if (attr->channel)
		cmd.flags |= EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL;

	if (EFA_DEV_CAP(ctx, SQ_64_BIT_REQ_ID))
		cmd.flags |= EFA_CREATE_CQ_WITH_SQ_COMP_64_BIT_REQ_ID;

	attr->cqe = roundup_pow_of_two(attr->cqe);
	err = ibv_cmd_create_cq_ex(ibvctx, attr, &prov_attr, &cq->verbs_cq,
				   &cmd.ibv_cmd, sizeof(cmd),
				   &resp.ibv_resp, sizeof(resp), cmd_flags);
	if (err) {
		errno = err;
		goto err_free_cq;
	}

	sub_cq_size = cq->verbs_cq.cq.cqe;
	cq->cqn = resp.cq_idx;
	cq->num_sub_cqs = num_sub_cqs;
	cq->cqe_size = cqe_size;
	cq->dev = ibvctx->device;
	cq->parent_domain = parent_domain;

	if (efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF) {
		cq->buf_size = efa_attr->ext_mem_dmabuf.length;
		cq->buf = efa_attr->ext_mem_dmabuf.buffer;
	} else {
		cq->buf_size = resp.q_mmap_size;
		cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED, ibvctx->cmd_fd,
			       resp.q_mmap_key);
		if (cq->buf == MAP_FAILED)
			goto err_destroy_cq;

		cq->buf_mmaped = true;
	}

	if (cq->buf) {
		buf = cq->buf;
		sub_buf_size = cq->cqe_size * sub_cq_size;
		for (i = 0; i < num_sub_cqs; i++) {
			efa_sub_cq_initialize(&cq->sub_cq_arr[i], buf, sub_cq_size, cq->cqe_size);
			buf += sub_buf_size;
		}
	}

	if (resp.comp_mask & EFA_CREATE_CQ_RESP_DB_OFF) {
		cq->db_mmap_addr = mmap(NULL,
					to_efa_dev(ibvctx->device)->pg_sz, PROT_WRITE,
					MAP_SHARED, ibvctx->cmd_fd, resp.db_mmap_key);
		if (cq->db_mmap_addr == MAP_FAILED)
			goto err_unmap_cq;

		cq->db = (uint32_t *)(cq->db_mmap_addr + resp.db_off);
	}

	efa_cq_fill_pfns(cq, attr, efa_attr);
	pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
	if (cq->parent_domain)
		atomic_fetch_add(&cq->parent_domain->refcount, 1);

	return &cq->verbs_cq.cq_ex;

err_unmap_cq:
	if (cq->buf_mmaped)
		munmap(cq->buf, cq->buf_size);
err_destroy_cq:
	ibv_cmd_destroy_cq(&cq->verbs_cq.cq);
err_free_cq:
	free(cq);
	verbs_err(verbs_get_ctx(ibvctx), "Failed to create CQ\n");
	return NULL;
}

struct ibv_cq *efa_create_cq(struct ibv_context *ibvctx, int ncqe,
			     struct ibv_comp_channel *channel, int vec)
{
	struct efadv_cq_init_attr efa_attr = {};
	struct ibv_cq_init_attr_ex attr_ex = {
		.cqe = ncqe,
		.channel = channel,
		.comp_vector = vec
	};
	struct ibv_cq_ex *ibvcqx;

	ibvcqx = create_cq(ibvctx, &attr_ex, &efa_attr);

	return ibvcqx ? ibv_cq_ex_to_cq(ibvcqx) : NULL;
}

struct ibv_cq_ex *efa_create_cq_ex(struct ibv_context *ibvctx,
				   struct ibv_cq_init_attr_ex *attr_ex)
{
	struct efadv_cq_init_attr efa_attr = {};

	return create_cq(ibvctx, attr_ex, &efa_attr);
}

struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
				  struct ibv_cq_init_attr_ex *attr_ex,
				  struct efadv_cq_init_attr *efa_attr,
				  uint32_t inlen)
{
	struct efadv_cq_init_attr local_efa_attr = {};
	uint64_t supp_wc_flags = 0;
	struct efa_context *ctx;

	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (!vext_field_avail(struct efadv_cq_init_attr, wc_flags, inlen) ||
	    efa_attr->comp_mask ||
	    (inlen > sizeof(*efa_attr) && !is_ext_cleared(efa_attr, inlen))) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		errno = EINVAL;
		return NULL;
	}

	ctx = to_efa_context(ibvctx);
	if (EFA_DEV_CAP(ctx, CQ_WITH_SGID))
		supp_wc_flags |= EFADV_WC_EX_WITH_SGID;
	if (EFA_DEV_CAP(ctx, UNSOLICITED_WRITE_RECV))
		supp_wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED;
	if (!check_comp_mask(efa_attr->wc_flags, supp_wc_flags)) {
		verbs_err(verbs_get_ctx(ibvctx),
			  "Invalid EFA wc_flags[%#lx]\n", efa_attr->wc_flags);
		errno = EOPNOTSUPP;
		return NULL;
	}

	memcpy(&local_efa_attr, efa_attr, min_t(uint32_t, inlen, sizeof(local_efa_attr)));
	return create_cq(ibvctx, attr_ex, &local_efa_attr);
}

int efadv_query_cq(struct ibv_cq *ibvcq, struct efadv_cq_attr *attr, uint32_t inlen)
{
	struct efa_cq *cq = to_efa_cq(ibvcq);

	if (!is_efa_dev(ibvcq->context->device)) {
		verbs_err(verbs_get_ctx(ibvcq->context), "Not an EFA device\n");
		return EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), num_entries, inlen)) {
		verbs_err(verbs_get_ctx(ibvcq->context), "Compatibility issues\n");
		return EINVAL;
	}

	attr->comp_mask = 0;
	attr->buffer = cq->buf;
	attr->entry_size = cq->cqe_size;
	attr->num_entries = ibvcq->cqe;

	if (vext_field_avail(typeof(*attr), doorbell, inlen))
		attr->doorbell = cq->db;

	return 0;
}

struct efadv_cq *efadv_cq_from_ibv_cq_ex(struct ibv_cq_ex *ibvcqx)
{
	struct efa_cq *cq = to_efa_cq_ex(ibvcqx);

	return &cq->dv_cq;
}

int efa_destroy_cq(struct ibv_cq *ibvcq)
{
	struct efa_cq *cq = to_efa_cq(ibvcq);
	int err;

	err = ibv_cmd_destroy_cq(ibvcq);
	if (err) {
		verbs_err(verbs_get_ctx(ibvcq->context),
			  "Failed to destroy CQ[%u]\n", cq->cqn);
		return err;
	}

	munmap(cq->db_mmap_addr, to_efa_dev(cq->dev)->pg_sz);
	if (cq->buf_mmaped)
		munmap(cq->buf, cq->buf_size);

	pthread_spin_destroy(&cq->lock);
	if (cq->parent_domain)
		atomic_fetch_sub(&cq->parent_domain->refcount, 1);

	free(cq);

	return 0;
}

static void efa_fill_buffer_desc_va(struct ib_uverbs_buffer_desc *desc, uint64_t addr,
				    uint64_t length)
{
	desc->type = IB_UVERBS_BUFFER_TYPE_VA;
	desc->addr = addr;
	desc->length = length;
}

static void efa_fill_buffer_desc_dmabuf(struct ib_uverbs_buffer_desc *desc, int32_t fd,
					uint64_t offset, uint64_t length)
{
	desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF;
	desc->fd = fd;
	desc->addr = offset;
	desc->length = length;
}

static void efa_fill_buffer_desc_from_mem_loc(struct ib_uverbs_buffer_desc *desc,
					      struct efadv_memory_location *mem, uint64_t length)
{
	if (mem->type == EFADV_MEMORY_LOCATION_DMABUF)
		efa_fill_buffer_desc_dmabuf(desc, mem->dmabuf.fd, mem->dmabuf.offset, length);
	else
		efa_fill_buffer_desc_va(desc, (uintptr_t)mem->ptr, length);
}

static inline bool efa_comp_cntr_mem_type_supported(uint32_t mem_type)
{
	return mem_type == EFADV_MEMORY_LOCATION_VA || mem_type == EFADV_MEMORY_LOCATION_DMABUF;
}

static struct ibv_comp_cntr *efa_create_comp_cntr_impl(struct ibv_context *ibvctx,
						       struct ibv_comp_cntr_init_attr *attr,
						       struct efadv_comp_cntr_init_attr *efa_attr)
{
	uint32_t supported_efa_flags = EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM |
				       EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM;
	DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR,
				    UVERBS_METHOD_COMP_CNTR_CREATE, 2, NULL);
	struct ib_uverbs_buffer_desc comp_desc = {};
	struct ib_uverbs_buffer_desc err_desc = {};
	struct efa_comp_cntr *cc;
	int err;

	if (attr->comp_mask || attr->flags ||
	    attr->type != IBV_COMP_CNTR_TYPE_WRS ||
	    efa_attr->comp_mask ||
	    !check_comp_mask(efa_attr->flags, supported_efa_flags) ||
	    !efa_comp_cntr_mem_type_supported(efa_attr->comp_cntr_ext_mem.type) ||
	    !efa_comp_cntr_mem_type_supported(efa_attr->err_cntr_ext_mem.type)) {
		verbs_err(verbs_get_ctx(ibvctx), "Unsupported type or flag\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	cc = calloc(1, sizeof(*cc));
	if (!cc) {
		errno = ENOMEM;
		return NULL;
	}

	if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM) {
		efa_fill_buffer_desc_from_mem_loc(&comp_desc, &efa_attr->comp_cntr_ext_mem,
						  sizeof(uint64_t));
		cc->comp_ptr = (uint64_t *)efa_attr->comp_cntr_ext_mem.ptr;
	} else {
		efa_fill_buffer_desc_va(&comp_desc, (uintptr_t)&cc->comp_val, sizeof(uint64_t));
		cc->comp_ptr = &cc->comp_val;
	}

	fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, &comp_desc);

	if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM) {
		efa_fill_buffer_desc_from_mem_loc(&err_desc, &efa_attr->err_cntr_ext_mem,
						  sizeof(uint64_t));
		cc->err_ptr = (uint64_t *)efa_attr->err_cntr_ext_mem.ptr;
	} else {
		efa_fill_buffer_desc_va(&err_desc, (uintptr_t)&cc->err_val, sizeof(uint64_t));
		cc->err_ptr = &cc->err_val;
	}

	fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, &err_desc);

	err = ibv_cmd_create_comp_cntr(ibvctx, &cc->ibv_comp_cntr, cmdb);
	if (err) {
		free(cc);
		errno = err;
		return NULL;
	}

	return &cc->ibv_comp_cntr;
}

struct ibv_comp_cntr *efa_create_comp_cntr(struct ibv_context *ibvctx,
					   struct ibv_comp_cntr_init_attr *attr)
{
	struct efadv_comp_cntr_init_attr efa_attr = {};

	return efa_create_comp_cntr_impl(ibvctx, attr, &efa_attr);
}

struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx,
					     struct ibv_comp_cntr_init_attr *attr,
					     struct efadv_comp_cntr_init_attr *efa_attr,
					     uint32_t inlen)
{
	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (!vext_field_avail(struct efadv_comp_cntr_init_attr, err_cntr_ext_mem, inlen) ||
	    (inlen > sizeof(*efa_attr) && !is_ext_cleared(efa_attr, inlen))) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		errno = EINVAL;
		return NULL;
	}

	return efa_create_comp_cntr_impl(ibvctx, attr, efa_attr);
}

int efa_destroy_comp_cntr(struct ibv_comp_cntr *ibvcc)
{
	struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc);
	int err;

	err = ibv_cmd_destroy_comp_cntr(ibvcc);
	if (err)
		return err;

	free(cc);
	return 0;
}

int efa_set_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value)
{
	return ibv_cmd_set_comp_cntr(ibvcc, value);
}

int efa_set_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value)
{
	return ibv_cmd_set_err_comp_cntr(ibvcc, value);
}

int efa_inc_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount)
{
	return ibv_cmd_inc_comp_cntr(ibvcc, amount);
}

int efa_inc_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount)
{
	return ibv_cmd_inc_err_comp_cntr(ibvcc, amount);
}

int efa_read_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value)
{
	struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc);

	if (!cc->comp_ptr)
		return EOPNOTSUPP;

	*value = *cc->comp_ptr;
	return 0;
}

int efa_read_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value)
{
	struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc);

	if (!cc->err_ptr)
		return EOPNOTSUPP;

	*value = *cc->err_ptr;
	return 0;
}

static void efa_wq_terminate(struct efa_wq *wq, int pgsz)
{
	void *db_aligned;

	if (wq->need_lock)
		pthread_spin_destroy(&wq->wqlock);

	db_aligned = (void *)((uintptr_t)wq->db & ~(pgsz - 1));
	munmap(db_aligned, pgsz);

	if (wq->wrid_idx_pool)
		free(wq->wrid_idx_pool);

	if (wq->wrid)
		free(wq->wrid);
}

static int efa_wq_initialize(struct efa_wq *wq, struct efa_wq_init_attr *attr)
{
	uint16_t wrid_idx_mask;
	uint8_t *db_base;
	int err;
	int i;

	if (!wq->req_id_64_bit) {
		wq->wrid = malloc(wq->wqe_cnt * sizeof(*wq->wrid));
		if (!wq->wrid)
			return ENOMEM;

		wq->wrid_idx_pool = malloc(wq->wqe_cnt * sizeof(uint32_t));
		if (!wq->wrid_idx_pool) {
			err = ENOMEM;
			goto err_free_wrid;
		}

		/* Initialize the wrid free indexes pool. */
		for (i = 0; i < wq->wqe_cnt; i++)
			wq->wrid_idx_pool[i] = i;
	}

	wrid_idx_mask = roundup_pow_of_two(wq->wqe_cnt) - 1;
	wq->gen_mask = ~wrid_idx_mask;
	wq->shifted_gen = attr->gen << __bf_shf(wrid_idx_mask + 1);

	db_base = mmap(NULL, attr->pgsz, PROT_WRITE, MAP_SHARED, attr->cmd_fd,
		       attr->db_mmap_key);
	if (db_base == MAP_FAILED) {
		err = errno;
		goto err_free_wrid_idx_pool;
	}

	wq->db = (uint32_t *)(db_base + attr->db_off);

	wq->need_lock = attr->need_lock;
	if (wq->need_lock)
		pthread_spin_init(&wq->wqlock, PTHREAD_PROCESS_PRIVATE);

	wq->sub_cq_idx = attr->sub_cq_idx;

	return 0;

err_free_wrid_idx_pool:
	if (wq->wrid_idx_pool)
		free(wq->wrid_idx_pool);
err_free_wrid:
	if (wq->wrid)
		free(wq->wrid);
	return err;
}

static bool efa_check_cq_on_same_pd_td(struct ibv_pd *ibvpd, struct ibv_cq *ibvcq)
{
	struct efa_parent_domain *parent_domain;
	struct efa_pd *pd;
	struct efa_cq *cq;

	pd = to_efa_pd(ibvpd);
	cq = to_efa_cq(ibvcq);

	if (pd->orig_pd) {
		parent_domain = to_efa_parent_domain(ibvpd);
		if (parent_domain == cq->parent_domain && parent_domain->td)
			return true;
	}

	return false;
}

static void efa_sq_terminate(struct efa_qp *qp)
{
	struct efa_sq *sq = &qp->sq;

	if (!sq->wq.wqe_cnt)
		return;

	munmap(sq->desc - sq->desc_offset, sq->desc_ring_mmap_size);
	free(sq->local_queue);

	efa_wq_terminate(&sq->wq, qp->page_size);
}

static int efa_sq_initialize(struct efa_qp *qp,
			     const struct ibv_qp_init_attr_ex *attr,
			     struct efa_create_qp_resp *resp)
{
	struct efa_context *ctx = to_efa_context(qp->verbs_qp.qp.context);
	struct efa_wq_init_attr wq_attr;
	struct efa_sq *sq = &qp->sq;
	size_t desc_ring_size;
	bool need_lock;
	int err;

	if (!sq->wq.wqe_cnt)
		return 0;

	need_lock = !efa_check_cq_on_same_pd_td(attr->pd, attr->send_cq);

	wq_attr = (struct efa_wq_init_attr) {
		.db_mmap_key = resp->sq_db_mmap_key,
		.db_off = resp->sq_db_offset,
		.cmd_fd = qp->verbs_qp.qp.context->cmd_fd,
		.pgsz = qp->page_size,
		.sub_cq_idx = resp->send_sub_cq_idx,
		.need_lock = need_lock,
		.gen = qp->gen,
	};

	err = efa_wq_initialize(&qp->sq.wq, &wq_attr);
	if (err) {
		verbs_err(&ctx->ibvctx, "SQ[%u] efa_wq_initialize failed\n",
			  qp->verbs_qp.qp.qp_num);
		return err;
	}

	sq->desc_offset = resp->llq_desc_offset;
	desc_ring_size = sq->wq.wqe_cnt * sq->wqe_size;
	sq->desc_ring_mmap_size = align(desc_ring_size + sq->desc_offset,
					qp->page_size);
	sq->max_inline_data = attr->cap.max_inline_data;

	sq->local_queue = malloc(desc_ring_size);
	if (!sq->local_queue) {
		err = ENOMEM;
		goto err_terminate_wq;
	}

	sq->desc = mmap(NULL, sq->desc_ring_mmap_size, PROT_WRITE,
			MAP_SHARED, qp->verbs_qp.qp.context->cmd_fd,
			resp->llq_desc_mmap_key);
	if (sq->desc == MAP_FAILED) {
		verbs_err(&ctx->ibvctx, "SQ buffer mmap failed\n");
		err = errno;
		goto err_free_local_queue;
	}

	sq->desc += sq->desc_offset;
	sq->max_wr_rdma_sge = min_t(uint16_t, ctx->max_wr_rdma_sge,
				    EFA_IO_TX_DESC_NUM_RDMA_BUFS);
	sq->max_batch_wr = ctx->max_tx_batch ?
		(ctx->max_tx_batch * 64) / sq->wqe_size :
		UINT16_MAX;
	if (ctx->min_sq_wr) {
		/* The device can't accept a doorbell for the whole SQ at once,
		 * set the max batch to at least (SQ size - 1).
		 */
		sq->max_batch_wr = min_t(uint32_t, sq->max_batch_wr,
					 sq->wq.wqe_cnt - 1);
	}

	return 0;

err_free_local_queue:
	free(sq->local_queue);
err_terminate_wq:
	efa_wq_terminate(&sq->wq, qp->page_size);
	return err;
}

static void efa_rq_terminate(struct efa_qp *qp)
{
	struct efa_rq *rq = &qp->rq;

	if (!rq->wq.wqe_cnt)
		return;

	munmap(rq->buf, rq->buf_size);

	efa_wq_terminate(&rq->wq, qp->page_size);
}

static int efa_rq_initialize(struct efa_qp *qp,
			     const struct ibv_qp_init_attr_ex *attr,
			     struct efa_create_qp_resp *resp)
{
	struct efa_wq_init_attr wq_attr;
	struct efa_rq *rq = &qp->rq;
	bool need_lock;
	int err;

	if (!rq->wq.wqe_cnt)
		return 0;

	need_lock = !efa_check_cq_on_same_pd_td(attr->pd, attr->recv_cq);

	wq_attr = (struct efa_wq_init_attr) {
		.db_mmap_key = resp->rq_db_mmap_key,
		.db_off = resp->rq_db_offset,
		.cmd_fd = qp->verbs_qp.qp.context->cmd_fd,
		.pgsz = qp->page_size,
		.sub_cq_idx = resp->recv_sub_cq_idx,
		.need_lock = need_lock,
		.gen = qp->gen,
	};

	err = efa_wq_initialize(&qp->rq.wq, &wq_attr);
	if (err) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "RQ efa_wq_initialize failed\n");
		return err;
	}

	rq->buf_size = resp->rq_mmap_size;
	rq->buf = mmap(NULL, rq->buf_size, PROT_WRITE, MAP_SHARED,
		       qp->verbs_qp.qp.context->cmd_fd, resp->rq_mmap_key);
	if (rq->buf == MAP_FAILED) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "RQ buffer mmap failed\n");
		err = errno;
		goto err_terminate_wq;
	}

	return 0;

err_terminate_wq:
	efa_wq_terminate(&rq->wq, qp->page_size);
	return err;
}

static void efa_qp_init_indices(struct efa_qp *qp)
{
	qp->sq.wq.wqe_posted = 0;
	qp->sq.wq.wqe_completed = 0;
	qp->sq.wq.pc = 0;
	qp->sq.wq.wrid_idx_pool_next = 0;

	qp->rq.wq.wqe_posted = 0;
	qp->rq.wq.wqe_completed = 0;
	qp->rq.wq.pc = 0;
	qp->rq.wq.wrid_idx_pool_next = 0;
}

static int efa_calc_sq_wqe_size(uint32_t max_inline_data, bool inline_write_enabled)
{
	if (max_inline_data > EFA_IO_TX_DESC_INLINE_MAX_SIZE || inline_write_enabled)
		return EFA_IO_TX_DESC_SIZE_128;

	return EFA_IO_TX_DESC_SIZE_64;
}

static int efa_calc_sq_max_depth(struct efa_context *ctx, uint32_t max_inline_data,
				 bool write_with_inline)
{
	int sq_wqe_size = efa_calc_sq_wqe_size(max_inline_data, write_with_inline);

	return rounddown_pow_of_two(ctx->max_llq_size / sq_wqe_size);
}

int efadv_get_max_sq_depth(struct ibv_context *ibvctx, struct efadv_sq_depth_attr *attr,
			   uint32_t inlen)
{
	bool write_with_inline = !!(attr->flags & EFADV_SQ_DEPTH_ATTR_INLINE_WRITE);
	struct efa_context *ctx = to_efa_context(ibvctx);

	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		return -EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), max_inline_data, inlen) || attr->comp_mask) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		return -EINVAL;
	}

	if (attr->max_send_sge > ctx->max_sq_sge) {
		verbs_err(verbs_get_ctx(ibvctx), "Max send SGE %u > %u\n", attr->max_send_sge,
			  ctx->max_sq_sge);
		return -EINVAL;
	}

	if (attr->max_rdma_sge > ctx->max_wr_rdma_sge) {
		verbs_err(verbs_get_ctx(ibvctx), "Max RDMA SGE %u > %u\n", attr->max_rdma_sge,
			  ctx->max_wr_rdma_sge);
		return -EINVAL;
	}

	if (attr->max_inline_data > ctx->inline_buf_size_ex) {
		verbs_err(verbs_get_ctx(ibvctx), "Max inline data %u > %u\n", attr->max_inline_data,
			  ctx->inline_buf_size_ex);
		return -EINVAL;
	}

	return efa_calc_sq_max_depth(ctx, attr->max_inline_data, write_with_inline);
}

static int efa_calc_rq_max_depth(struct efa_context *ctx, uint32_t max_recv_sge)
{
	return ctx->max_rq_wr / max_recv_sge;
}

int efadv_get_max_rq_depth(struct ibv_context *ibvctx, struct efadv_rq_depth_attr *attr,
			   uint32_t inlen)
{
	struct efa_context *ctx = to_efa_context(ibvctx);

	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		return -EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), max_recv_sge, inlen) || attr->comp_mask) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		return -EINVAL;
	}

	if (attr->max_recv_sge > ctx->max_rq_sge) {
		verbs_err(verbs_get_ctx(ibvctx), "Max receive SGE %u > %u\n", attr->max_recv_sge,
			  ctx->max_rq_sge);
		return -EINVAL;
	}

	return efa_calc_rq_max_depth(ctx, attr->max_recv_sge);
}

static void efa_setup_qp(struct efa_context *ctx,
			 struct efa_qp *qp,
			 struct ibv_qp_init_attr_ex *attr,
			 struct efadv_qp_init_attr *efa_attr,
			 size_t page_size)
{
	bool inline_write_enabled = !!(efa_attr->flags & EFADV_QP_FLAGS_INLINE_WRITE);
	struct ibv_qp_cap *cap = &attr->cap;
	uint16_t rq_desc_cnt;

	efa_qp_init_indices(qp);

	qp->sq.wqe_size = efa_calc_sq_wqe_size(cap->max_inline_data, inline_write_enabled);
	qp->sq.wq.wqe_cnt = roundup_pow_of_two(max_t(uint32_t, cap->max_send_wr,
						     ctx->min_sq_wr));
	qp->sq.wq.max_sge = cap->max_send_sge;
	qp->sq.wq.desc_mask = qp->sq.wq.wqe_cnt - 1;
	qp->sq.wq.req_id_64_bit = !!(EFA_DEV_CAP(ctx, SQ_64_BIT_REQ_ID));
	qp->sq.inline_write_enabled = inline_write_enabled;

	qp->rq.wq.max_sge = cap->max_recv_sge;
	rq_desc_cnt = roundup_pow_of_two(cap->max_recv_sge * cap->max_recv_wr);
	qp->rq.wq.desc_mask = rq_desc_cnt - 1;
	qp->rq.wq.wqe_cnt = rq_desc_cnt / qp->rq.wq.max_sge;
	qp->rq.wq.req_id_64_bit = false;

	qp->page_size = page_size;
}

static void efa_lock_cqs(struct ibv_qp *ibvqp)
{
	struct efa_cq *send_cq = to_efa_cq(ibvqp->send_cq);
	struct efa_cq *recv_cq = to_efa_cq(ibvqp->recv_cq);

	if (recv_cq == send_cq) {
		pthread_spin_lock(&recv_cq->lock);
	} else {
		pthread_spin_lock(&recv_cq->lock);
		pthread_spin_lock(&send_cq->lock);
	}
}

static void efa_unlock_cqs(struct ibv_qp *ibvqp)
{
	struct efa_cq *send_cq = to_efa_cq(ibvqp->send_cq);
	struct efa_cq *recv_cq = to_efa_cq(ibvqp->recv_cq);

	if (recv_cq == send_cq) {
		pthread_spin_unlock(&recv_cq->lock);
	} else {
		pthread_spin_unlock(&recv_cq->lock);
		pthread_spin_unlock(&send_cq->lock);
	}
}

static void efa_qp_fill_wr_pfns(struct efa_qp *qp,
				struct ibv_qp_init_attr_ex *attr_ex,
				struct efadv_qp_init_attr *efa_attr,
				uint16_t wqe_size);

static int efa_check_qp_attr(struct efa_context *ctx,
			     struct ibv_qp_init_attr_ex *attr,
			     struct efadv_qp_init_attr *efa_attr)
{
	uint64_t supp_ud_send_ops_mask = IBV_QP_EX_WITH_SEND |
					 IBV_QP_EX_WITH_SEND_WITH_IMM;
	uint64_t supp_srd_send_ops_mask = IBV_QP_EX_WITH_SEND |
					  IBV_QP_EX_WITH_SEND_WITH_IMM;
	uint64_t supp_send_ops_mask;
	uint16_t supp_efa_flags = 0;

	if (EFA_DEV_CAP(ctx, RDMA_READ))
		supp_srd_send_ops_mask |= IBV_QP_EX_WITH_RDMA_READ;
	if (EFA_DEV_CAP(ctx, RDMA_WRITE)) {
		supp_efa_flags |= EFADV_QP_FLAGS_INLINE_WRITE;
		supp_srd_send_ops_mask |= IBV_QP_EX_WITH_RDMA_WRITE |
					  IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
	}
	if (EFA_DEV_CAP(ctx, UNSOLICITED_WRITE_RECV))
		supp_efa_flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;

#define EFA_CREATE_QP_SUPP_ATTR_MASK \
	(IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)

	if (attr->qp_type == IBV_QPT_DRIVER &&
	    efa_attr->driver_qp_type != EFADV_QP_DRIVER_TYPE_SRD) {
		verbs_err(&ctx->ibvctx, "Driver QP type must be SRD\n");
		return EOPNOTSUPP;
	}

	if (!check_comp_mask(efa_attr->flags, supp_efa_flags)) {
		verbs_err(&ctx->ibvctx,
			  "Unsupported EFA flags[%#x] supported[%#x]\n",
			  efa_attr->flags, supp_efa_flags);
		return EOPNOTSUPP;
	}

	if (!check_comp_mask(attr->comp_mask, EFA_CREATE_QP_SUPP_ATTR_MASK)) {
		verbs_err(&ctx->ibvctx,
			  "Unsupported comp_mask[%#x] supported[%#x]\n",
			  attr->comp_mask, EFA_CREATE_QP_SUPP_ATTR_MASK);
		return EOPNOTSUPP;
	}

	if (!(attr->comp_mask & IBV_QP_INIT_ATTR_PD)) {
		verbs_err(&ctx->ibvctx, "Does not support PD in init attr\n");
		return EINVAL;
	}

	if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) {
		switch (attr->qp_type) {
		case IBV_QPT_UD:
			supp_send_ops_mask = supp_ud_send_ops_mask;
			break;
		case IBV_QPT_DRIVER:
			supp_send_ops_mask = supp_srd_send_ops_mask;
			break;
		default:
			verbs_err(&ctx->ibvctx, "Invalid QP type %u\n",
				  attr->qp_type);
			return EOPNOTSUPP;
		}

		if (!check_comp_mask(attr->send_ops_flags,
				     supp_send_ops_mask)) {
			verbs_err(&ctx->ibvctx,
				  "Unsupported send_ops_flags[%" PRIx64 "] supported [%" PRIx64 "]\n",
				  attr->send_ops_flags, supp_send_ops_mask);
			return EOPNOTSUPP;
		}
	}

	if (!check_comp_mask(efa_attr->wr_flags, EFADV_WR_EX_WITH_PROCESSING_HINTS)) {
		verbs_err(&ctx->ibvctx, "Unsupported wr_flags[%" PRIx64 "]\n",
			  efa_attr->wr_flags);
		return EOPNOTSUPP;
	}

	if (!attr->recv_cq || !attr->send_cq) {
		verbs_err(&ctx->ibvctx, "Send/Receive CQ not provided\n");
		return EINVAL;
	}

	if (attr->srq) {
		verbs_err(&ctx->ibvctx, "SRQ is not supported\n");
		return EINVAL;
	}

	return 0;
}

static int efa_check_qp_limits(struct efa_context *ctx,
			       struct ibv_qp_init_attr_ex *attr,
			       struct efadv_qp_init_attr *efa_attr)
{
	bool inline_write_enabled = !!(efa_attr->flags & EFADV_QP_FLAGS_INLINE_WRITE);
	int sq_max_depth, rq_max_depth;

	if (attr->cap.max_send_sge > ctx->max_sq_sge) {
		verbs_err(&ctx->ibvctx,
			  "Max send SGE %u > %u\n", attr->cap.max_send_sge,
			  ctx->max_sq_sge);
		return EINVAL;
	}

	if (attr->cap.max_recv_sge > ctx->max_rq_sge) {
		verbs_err(&ctx->ibvctx,
			  "Max receive SGE %u > %u\n", attr->cap.max_recv_sge,
			  ctx->max_rq_sge);
		return EINVAL;
	}

	sq_max_depth = efa_calc_sq_max_depth(ctx, attr->cap.max_inline_data, inline_write_enabled);
	if (attr->cap.max_send_wr > sq_max_depth) {
		verbs_err(&ctx->ibvctx,
			  "Max Send WR %u > %u\n", attr->cap.max_send_wr, sq_max_depth);
		return EINVAL;
	}

	rq_max_depth = efa_calc_rq_max_depth(ctx, attr->cap.max_recv_sge);
	if (attr->cap.max_recv_wr > rq_max_depth) {
		verbs_err(&ctx->ibvctx,
			  "Requested max SGE %u, max receive WR %u > %u\n", attr->cap.max_recv_sge,
			  attr->cap.max_recv_wr, rq_max_depth);
		return EINVAL;
	}

	if (attr->cap.max_inline_data > ctx->inline_buf_size_ex) {
		verbs_err(&ctx->ibvctx,
			  "Max inline data %u > %u\n", attr->cap.max_inline_data,
			  ctx->inline_buf_size_ex);
		return EINVAL;
	}

	return 0;
}

static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
				struct ibv_qp_init_attr_ex *attr,
				struct efadv_qp_init_attr *efa_attr)
{
	struct efa_context *ctx = to_efa_context(ibvctx);
	struct efa_dev *dev = to_efa_dev(ibvctx->device);
	struct efa_parent_domain *parent_domain;
	struct efa_create_qp_resp resp = {};
	struct efa_create_qp req = {};
	struct ibv_qp *ibvqp;
	struct efa_qp *qp;
	struct efa_pd *pd;
	int err;

	err = efa_check_qp_attr(ctx, attr, efa_attr);
	if (err)
		goto err_out;

	err = efa_check_qp_limits(ctx, attr, efa_attr);
	if (err)
		goto err_out;

	qp = calloc(1, sizeof(*qp));
	if (!qp) {
		err = ENOMEM;
		goto err_out;
	}

	efa_setup_qp(ctx, qp, attr, efa_attr, dev->pg_sz);

	attr->cap.max_send_wr = qp->sq.wq.wqe_cnt;
	attr->cap.max_recv_wr = qp->rq.wq.wqe_cnt;

	req.rq_ring_size = (qp->rq.wq.desc_mask + 1) *
		sizeof(struct efa_io_rx_desc);
	req.sq_ring_size = attr->cap.max_send_wr * qp->sq.wqe_size;
	if (attr->qp_type == IBV_QPT_DRIVER)
		req.driver_qp_type = efa_attr->driver_qp_type;
	if (efa_attr->flags & EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV)
		req.flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV;

	req.sl = efa_attr->sl;

	if (qp->sq.wq.req_id_64_bit)
		req.flags |= EFA_CREATE_QP_WITH_SQ_64_BIT_REQ_ID;

	err = ibv_cmd_create_qp_ex(ibvctx, &qp->verbs_qp,
				   attr, &req.ibv_cmd, sizeof(req),
				   &resp.ibv_resp, sizeof(resp));
	if (err)
		goto err_free_qp;

	ibvqp = &qp->verbs_qp.qp;
	ibvqp->state = IBV_QPS_RESET;
	qp->sq_sig_all = attr->sq_sig_all;
	qp->dev = ibvctx->device;

	pthread_spin_lock(&ctx->qp_table_lock);
	qp->gen = ++ctx->qp_gen_table[ibvqp->qp_num & ctx->qp_table_sz_m1];
	pthread_spin_unlock(&ctx->qp_table_lock);

	err = efa_rq_initialize(qp, attr, &resp);
	if (err)
		goto err_destroy_qp;

	err = efa_sq_initialize(qp, attr, &resp);
	if (err)
		goto err_terminate_rq;

	pthread_spin_lock(&ctx->qp_table_lock);
	ctx->qp_table[ibvqp->qp_num & ctx->qp_table_sz_m1] = qp;
	pthread_spin_unlock(&ctx->qp_table_lock);

	if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) {
		efa_qp_fill_wr_pfns(qp, attr, efa_attr, qp->sq.wqe_size);
		qp->verbs_qp.comp_mask |= VERBS_QP_EX;
	}

	pd = to_efa_pd(attr->pd);
	if (pd->orig_pd) {
		parent_domain = to_efa_parent_domain(attr->pd);
		qp->parent_domain = parent_domain;
		atomic_fetch_add(&parent_domain->refcount, 1);
	}

	return ibvqp;

err_terminate_rq:
	efa_rq_terminate(qp);
err_destroy_qp:
	ibv_cmd_destroy_qp(ibvqp);
err_free_qp:
	free(qp);
err_out:
	errno = err;
	verbs_err(verbs_get_ctx(ibvctx), "Failed to create QP\n");
	return NULL;
}

struct ibv_qp *efa_create_qp(struct ibv_pd *ibvpd,
			     struct ibv_qp_init_attr *attr)
{
	struct ibv_qp_init_attr_ex attr_ex = {};
	struct efadv_qp_init_attr efa_attr = {};
	struct ibv_qp *ibvqp;

	if (attr->qp_type != IBV_QPT_UD) {
		verbs_err(verbs_get_ctx(ibvpd->context),
			  "Unsupported QP type %d\n", attr->qp_type);
		errno = EOPNOTSUPP;
		return NULL;
	}

	memcpy(&attr_ex, attr, sizeof(*attr));
	attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD;
	attr_ex.pd = ibvpd;

	ibvqp = create_qp(ibvpd->context, &attr_ex, &efa_attr);
	if (ibvqp)
		memcpy(attr, &attr_ex, sizeof(*attr));

	return ibvqp;
}

struct ibv_qp *efa_create_qp_ex(struct ibv_context *ibvctx,
				struct ibv_qp_init_attr_ex *attr_ex)
{
	struct efadv_qp_init_attr efa_attr = {};

	if (attr_ex->qp_type != IBV_QPT_UD) {
		verbs_err(verbs_get_ctx(ibvctx), "Unsupported QP type\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	return create_qp(ibvctx, attr_ex, &efa_attr);
}

struct ibv_qp *efadv_create_driver_qp(struct ibv_pd *ibvpd,
				      struct ibv_qp_init_attr *attr,
				      uint32_t driver_qp_type)
{
	struct ibv_qp_init_attr_ex attr_ex = {};
	struct efadv_qp_init_attr efa_attr = {};
	struct ibv_qp *ibvqp;

	if (!is_efa_dev(ibvpd->context->device)) {
		verbs_err(verbs_get_ctx(ibvpd->context), "Not an EFA device\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (attr->qp_type != IBV_QPT_DRIVER) {
		verbs_err(verbs_get_ctx(ibvpd->context),
			  "QP type not IBV_QPT_DRIVER\n");
		errno = EINVAL;
		return NULL;
	}

	memcpy(&attr_ex, attr, sizeof(*attr));
	attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD;
	attr_ex.pd = ibvpd;
	efa_attr.driver_qp_type = driver_qp_type;

	ibvqp = create_qp(ibvpd->context, &attr_ex, &efa_attr);
	if (ibvqp)
		memcpy(attr, &attr_ex, sizeof(*attr));

	return ibvqp;
}

struct ibv_qp *efadv_create_qp_ex(struct ibv_context *ibvctx,
				  struct ibv_qp_init_attr_ex *attr_ex,
				  struct efadv_qp_init_attr *efa_attr,
				  uint32_t inlen)
{
	struct efadv_qp_init_attr local_efa_attr = {};

	if (!is_efa_dev(ibvctx->device)) {
		verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	if (attr_ex->qp_type != IBV_QPT_DRIVER ||
	    !vext_field_avail(struct efadv_qp_init_attr,
			      driver_qp_type, inlen) ||
	    efa_attr->comp_mask ||
	    efa_attr->reserved ||
	    (inlen > sizeof(*efa_attr) && !is_ext_cleared(efa_attr, inlen))) {
		verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n");
		errno = EINVAL;
		return NULL;
	}

	memcpy(&local_efa_attr, efa_attr, min_t(uint32_t, inlen, sizeof(local_efa_attr)));
	return create_qp(ibvctx, attr_ex, &local_efa_attr);
}

int efa_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr,
		  int attr_mask)
{
	struct efa_qp *qp = to_efa_qp(ibvqp);
	struct ibv_modify_qp cmd = {};
	int err;

	err = ibv_cmd_modify_qp(ibvqp, attr, attr_mask, &cmd, sizeof(cmd));
	if (err) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "Failed to modify QP[%u]\n", qp->verbs_qp.qp.qp_num);
		return err;
	}

	if (attr_mask & IBV_QP_STATE) {
		qp->verbs_qp.qp.state = attr->qp_state;
		/* transition to reset */
		if (qp->verbs_qp.qp.state == IBV_QPS_RESET)
			efa_qp_init_indices(qp);
	}

	return 0;
}

int efa_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr,
			    struct ibv_qp_attach_comp_cntr_attr *attr)
{
	return ibv_cmd_qp_attach_comp_cntr(qp, comp_cntr, attr);
}

int efa_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr,
		 int attr_mask, struct ibv_qp_init_attr *init_attr)
{
	struct ibv_query_qp cmd;

	return ibv_cmd_query_qp(ibvqp, attr, attr_mask, init_attr,
				&cmd, sizeof(cmd));
}

int efadv_query_qp_wqs(struct ibv_qp *ibvqp, struct efadv_wq_attr *sq_attr,
		       struct efadv_wq_attr *rq_attr, uint32_t inlen)
{
	struct efa_qp *qp = to_efa_qp(ibvqp);

	if (!is_efa_dev(ibvqp->context->device)) {
		verbs_err(verbs_get_ctx(ibvqp->context), "Not an EFA device\n");
		return EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*sq_attr), max_batch, inlen)) {
		verbs_err(verbs_get_ctx(ibvqp->context), "Compatibility issues\n");
		return EINVAL;
	}

	sq_attr->comp_mask = 0;
	sq_attr->buffer = qp->sq.desc;
	sq_attr->entry_size = qp->sq.wqe_size;
	sq_attr->num_entries = qp->sq.wq.wqe_cnt;
	sq_attr->doorbell = qp->sq.wq.db;
	sq_attr->max_batch = qp->sq.max_batch_wr;
	if (vext_field_avail(typeof(*sq_attr), reserved, inlen)) {
		sq_attr->caps = 0;
		memset(sq_attr->reserved, 0, sizeof(sq_attr->reserved));
		if (qp->sq.wq.req_id_64_bit)
			sq_attr->caps |= EFADV_WQ_CAPS_64_BIT_REQ_ID;
	}

	rq_attr->comp_mask = 0;
	rq_attr->buffer = qp->rq.buf;
	rq_attr->entry_size = sizeof(struct efa_io_rx_desc);
	rq_attr->num_entries = qp->rq.wq.desc_mask + 1;
	rq_attr->doorbell = qp->rq.wq.db;
	rq_attr->max_batch = rq_attr->num_entries;
	if (vext_field_avail(typeof(*rq_attr), reserved, inlen)) {
		memset(rq_attr->reserved, 0, sizeof(rq_attr->reserved));
		rq_attr->caps = 0;
	}

	return 0;
}

int efa_query_qp_data_in_order(struct ibv_qp *ibvqp, enum ibv_wr_opcode op,
			       uint32_t flags)
{
	struct efa_context *ctx = to_efa_context(ibvqp->context);
	int caps = 0;

	if (EFA_DEV_CAP(ctx, DATA_POLLING_128))
		caps |= IBV_QUERY_QP_DATA_IN_ORDER_ALIGNED_128_BYTES;

	return caps;
}

struct efadv_qp *efadv_qp_from_ibv_qp_ex(struct ibv_qp_ex *ibvqpx)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);

	return &qp->dv_qp;
}

int efa_destroy_qp(struct ibv_qp *ibvqp)
{
	struct efa_context *ctx = to_efa_context(ibvqp->context);
	struct efa_qp *qp = to_efa_qp(ibvqp);
	int err;

	err = ibv_cmd_destroy_qp(ibvqp);
	if (err) {
		verbs_err(&ctx->ibvctx, "Failed to destroy QP[%u]\n",
			  ibvqp->qp_num);
		return err;
	}

	if (qp->parent_domain)
		atomic_fetch_sub(&qp->parent_domain->refcount, 1);

	pthread_spin_lock(&ctx->qp_table_lock);
	efa_lock_cqs(ibvqp);

	ctx->qp_table[ibvqp->qp_num & ctx->qp_table_sz_m1] = NULL;

	efa_unlock_cqs(ibvqp);
	pthread_spin_unlock(&ctx->qp_table_lock);

	efa_sq_terminate(qp);
	efa_rq_terminate(qp);

	free(qp);
	return 0;
}

static void efa_set_tx_buf(struct efa_io_tx_buf_desc *tx_buf,
			   uint64_t addr, uint32_t lkey,
			   uint32_t length)
{
	tx_buf->length = length;
	EFA_SET(&tx_buf->lkey, EFA_IO_TX_BUF_DESC_LKEY, lkey);
	tx_buf->buf_addr_lo = addr & 0xffffffff;
	tx_buf->buf_addr_hi = addr >> 32;
}

static void efa_post_send_sgl(struct efa_io_tx_buf_desc *tx_bufs,
			      struct efa_io_tx_meta_desc *md,
			      const struct ibv_sge *sg_list,
			      int num_sge)
{
	const struct ibv_sge *sge;
	size_t i;

	md->length = num_sge;

	for (i = 0; i < num_sge; i++) {
		sge = &sg_list[i];
		efa_set_tx_buf(&tx_bufs[i], sge->addr, sge->lkey, sge->length);
	}
}

static void efa_post_send_inline_data(const struct ibv_send_wr *wr, struct efa_io_tx_meta_desc *md,
				      uint8_t *inline_data)
{
	const struct ibv_sge *sgl = wr->sg_list;
	uint32_t total_length = 0;
	uint32_t length;
	size_t i;

	for (i = 0; i < wr->num_sge; i++) {
		length = sgl[i].length;

		memcpy(inline_data + total_length,
		       (void *)(uintptr_t)sgl[i].addr, length);
		total_length += length;
	}

	EFA_SET(&md->ctrl1, EFA_IO_TX_META_DESC_INLINE_MSG, 1);
	md->length = total_length;
}

static size_t efa_sge_total_bytes(const struct ibv_sge *sg_list, int num_sge)
{
	size_t bytes = 0;
	size_t i;

	for (i = 0; i < num_sge; i++)
		bytes += sg_list[i].length;

	return bytes;
}

static size_t efa_buf_list_total_bytes(const struct ibv_data_buf *buf_list,
				       size_t num_buf)
{
	size_t bytes = 0;
	size_t i;

	for (i = 0; i < num_buf; i++)
		bytes += buf_list[i].length;

	return bytes;
}

static void efa_sq_advance_post_idx(struct efa_sq *sq)
{
	struct efa_wq *wq = &sq->wq;

	wq->wqe_posted++;
	wq->pc++;

	if (!(wq->pc & wq->desc_mask))
		wq->phase++;
}

static inline void efa_rq_ring_doorbell(struct efa_rq *rq, uint16_t pc)
{
	udma_to_device_barrier();
	mmio_write32(rq->wq.db, pc);
}

static inline void efa_sq_ring_doorbell(struct efa_sq *sq, uint16_t pc)
{
	mmio_write32(sq->wq.db, pc);
}

static void efa_set_common_ctrl_flags(struct efa_io_tx_meta_desc *desc,
				      struct efa_sq *sq,
				      enum efa_io_send_op_type op_type)
{
	EFA_SET(&desc->ctrl1, EFA_IO_TX_META_DESC_META_DESC, 1);
	EFA_SET(&desc->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE, op_type);
	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_PHASE, sq->wq.phase);
	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_FIRST, 1);
	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_LAST, 1);
	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_COMP_REQ, 1);
}

#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
static uint32_t efa_wqe_get_data_length(struct efa_sq *sq)
{
	struct efa_io_tx_meta_desc *md = sq->curr_tx_wqe.md;
	enum efa_io_send_op_type op_type;
	uint32_t length = 0;
	size_t i;

	op_type = EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE);
	switch (op_type) {
	case EFA_IO_SEND:
		if (EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_INLINE_MSG))
			return md->length;

		for (i = 0; i < md->length; i++)
			length += sq->curr_tx_wqe.local_mem[i].length;

		return length;
	case EFA_IO_RDMA_READ:
	case EFA_IO_RDMA_WRITE:
		return sq->curr_tx_wqe.remote_mem->length;
	}

	return 0;
}
#endif

static int efa_post_send_validate(struct efa_qp *qp,
				  unsigned int wr_flags)
{
	if (unlikely(qp->verbs_qp.qp.state != IBV_QPS_RTS &&
		     qp->verbs_qp.qp.state != IBV_QPS_SQD)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] is in invalid state\n",
			  qp->verbs_qp.qp.qp_num);
		return EINVAL;
	}

	if (unlikely(!(wr_flags & IBV_SEND_SIGNALED) && !qp->sq_sig_all)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] Non signaled WRs not supported\n",
			  qp->verbs_qp.qp.qp_num);
		return EINVAL;
	}

	if (unlikely(wr_flags & ~(IBV_SEND_SIGNALED | IBV_SEND_INLINE))) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] Unsupported wr_flags[%#x] supported[%#x]\n",
			  qp->verbs_qp.qp.qp_num, wr_flags,
			  ~(IBV_SEND_SIGNALED | IBV_SEND_INLINE));
		return EINVAL;
	}

	if (unlikely(qp->sq.wq.wqe_posted - qp->sq.wq.wqe_completed ==
		     qp->sq.wq.wqe_cnt)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] is full wqe_posted[%u] wqe_completed[%u] wqe_cnt[%u]\n",
			  qp->verbs_qp.qp.qp_num, qp->sq.wq.wqe_posted,
			  qp->sq.wq.wqe_completed, qp->sq.wq.wqe_cnt);
		return ENOMEM;
	}

	return 0;
}

static int efa_post_send_validate_wr(struct efa_qp *qp,
				     const struct ibv_send_wr *wr)
{
	int err;

	err = efa_post_send_validate(qp, wr->send_flags);
	if (unlikely(err))
		return err;

	if (unlikely(wr->opcode != IBV_WR_SEND &&
		     wr->opcode != IBV_WR_SEND_WITH_IMM)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] unsupported opcode %d\n",
			  qp->verbs_qp.qp.qp_num, wr->opcode);
		return EINVAL;
	}

	if (wr->send_flags & IBV_SEND_INLINE) {
		if (unlikely(efa_sge_total_bytes(wr->sg_list, wr->num_sge) >
			     qp->sq.max_inline_data)) {
			verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
				  "SQ[%u] WR total bytes %zu > %zu\n",
				  qp->verbs_qp.qp.qp_num,
				  efa_sge_total_bytes(wr->sg_list,
						      wr->num_sge),
				  qp->sq.max_inline_data);
			return EINVAL;
		}
	} else {
		if (unlikely(wr->num_sge > qp->sq.wq.max_sge)) {
			verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
				  "SQ[%u] WR num_sge %d > %d\n",
				  qp->verbs_qp.qp.qp_num, wr->num_sge,
				  qp->sq.wq.max_sge);
			return EINVAL;
	}
	}

	return 0;
}

int efa_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
		  struct ibv_send_wr **bad)
{
	uint8_t wqe_buf[EFA_IO_TX_DESC_SIZE_128];
	struct efa_qp *qp = to_efa_qp(ibvqp);
	struct efa_io_tx_wqe_128 *tx_wqe_128;
	struct efa_io_tx_meta_desc *md;
	struct efa_io_tx_buf_desc *sgl;
	struct efa_io_tx_wqe *tx_wqe;
	struct efa_sq *sq = &qp->sq;
	struct efa_wq *wq = &sq->wq;
	uint32_t sq_desc_offset;
	uint32_t curbatch = 0;
	uint8_t *inline_data;
	struct efa_ah *ah;
	int i, err = 0;

	switch (sq->wqe_size) {
	case EFA_IO_TX_DESC_SIZE_64:
		tx_wqe = (struct efa_io_tx_wqe *)wqe_buf;
		md = &tx_wqe->meta;
		sgl = tx_wqe->data.sgl;
		inline_data = tx_wqe->data.inline_data;
		break;
	case EFA_IO_TX_DESC_SIZE_128:
		tx_wqe_128 = (struct efa_io_tx_wqe_128 *)wqe_buf;
		md = &tx_wqe_128->meta;
		sgl = tx_wqe_128->data.sgl;
		inline_data = tx_wqe_128->data.inline_data;
		break;

	default:
		return EINVAL;
	}

	if (wq->need_lock)
		mmio_wc_spinlock(&wq->wqlock);
	else
		mmio_wc_start();

	while (wr) {
		err = efa_post_send_validate_wr(qp, wr);
		if (err) {
			*bad = wr;
			goto ring_db;
		}

		for (i = 0; i < sq->wqe_size / sizeof(uint64_t); i++)
			((uint64_t *)wqe_buf)[i] = 0;

		ah = to_efa_ah(wr->wr.ud.ah);

		if (wr->send_flags & IBV_SEND_INLINE) {
			efa_post_send_inline_data(wr, md, inline_data);
		} else {
			efa_post_send_sgl(sgl, md, wr->sg_list, wr->num_sge);
		}

		if (wr->opcode == IBV_WR_SEND_WITH_IMM) {
			md->immediate_data = be32toh(wr->imm_data);
			EFA_SET(&md->ctrl1, EFA_IO_TX_META_DESC_HAS_IMM, 1);
		}

		/* Set rest of the descriptor fields */
		efa_set_common_ctrl_flags(md, sq, EFA_IO_SEND);
		efa_set_sq_comp_wrid(md, wq, wr->wr_id);

		md->dest_qp_num = wr->wr.ud.remote_qpn;
		md->ah = ah->efa_ah;
		md->qkey = wr->wr.ud.remote_qkey;

		/* Copy descriptor */
		sq_desc_offset = (wq->pc & wq->desc_mask) * sq->wqe_size;
		mmio_memcpy_x64(sq->desc + sq_desc_offset, wqe_buf, sq->wqe_size);

		/* advance index and change phase */
		efa_sq_advance_post_idx(sq);
		curbatch++;

		if (curbatch == sq->max_batch_wr) {
			curbatch = 0;
			mmio_flush_writes();
			efa_sq_ring_doorbell(sq, wq->pc);
			mmio_wc_start();
		}
		rdma_tracepoint(rdma_core_efa, post_send, qp->dev->name, wr->wr_id,
				EFA_IO_SEND, ibvqp->qp_num, md->dest_qp_num,
				ah->efa_ah, efa_wqe_get_data_length(sq));
		wr = wr->next;
	}

ring_db:
	if (curbatch) {
		mmio_flush_writes();
		efa_sq_ring_doorbell(sq, wq->pc);
	}

	/*
	 * Not using mmio_wc_spinunlock as the doorbell write should be done
	 * inside the lock.
	 */
	if (wq->need_lock)
		pthread_spin_unlock(&wq->wqlock);

	return err;
}

static void *efa_send_wr_alloc(struct efa_qp *qp, struct ibv_qp_ex *ibvqpx)
{
	struct efa_sq *sq = &qp->sq;
	int err, i;

	if (unlikely(qp->wr_session_err))
		return NULL;

	err = efa_post_send_validate(qp, ibvqpx->wr_flags);
	if (unlikely(err)) {
		qp->wr_session_err = err;
		return NULL;
	}

	sq->curr_tx_wqe.buff = sq->local_queue + sq->num_wqe_pending * sq->wqe_size;
	for (i = 0; i < sq->wqe_size / sizeof(uint64_t); i++)
		((uint64_t *)sq->curr_tx_wqe.buff)[i] = 0;

	return sq->curr_tx_wqe.buff;
}

static void efa_send_wr_init(struct efa_qp *qp, struct ibv_qp_ex *ibvqpx,
			     enum efa_io_send_op_type op_type, uint8_t max_sge,
			     struct efa_io_tx_meta_desc *md,
			     struct efa_io_tx_buf_desc *local_mem,
			     struct efa_io_remote_mem_addr *remote_mem,
			     uint8_t *inline_data)
{
	struct efa_sq *sq = &qp->sq;

	sq->curr_tx_wqe.md = md;
	efa_set_common_ctrl_flags(sq->curr_tx_wqe.md, sq, op_type);
	efa_set_sq_comp_wrid(sq->curr_tx_wqe.md, &sq->wq, ibvqpx->wr_id);

	/* advance index and change phase */
	efa_sq_advance_post_idx(sq);
	sq->num_wqe_pending++;

	sq->curr_tx_wqe.local_mem = local_mem;
	sq->curr_tx_wqe.remote_mem = remote_mem;
	sq->curr_tx_wqe.inline_data = inline_data;
	sq->curr_tx_wqe.max_sge = max_sge;
}

static void efa_send_wr_set_imm_data(struct efa_io_tx_meta_desc *meta_desc, __be32 imm_data)
{
	meta_desc->immediate_data = be32toh(imm_data);
	EFA_SET(&meta_desc->ctrl1, EFA_IO_TX_META_DESC_HAS_IMM, 1);
}

static void efa_send_wr_set_rdma_addr(struct efa_io_remote_mem_addr *remote_mem, uint32_t rkey,
				      uint64_t remote_addr)
{
	remote_mem->rkey = rkey;
	remote_mem->buf_addr_lo = remote_addr & 0xFFFFFFFF;
	remote_mem->buf_addr_hi = remote_addr >> 32;
}

static void efa_send_wr_send_64(struct ibv_qp_ex *ibvqpx)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_init(qp, ibvqpx, EFA_IO_SEND, qp->sq.wq.max_sge, &tx_wqe->meta,
			 tx_wqe->data.sgl, NULL, tx_wqe->data.inline_data);
}

static void efa_send_wr_send_128(struct ibv_qp_ex *ibvqpx)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe_128 *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe_128 *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_init(qp, ibvqpx, EFA_IO_SEND, qp->sq.wq.max_sge, &tx_wqe->meta,
			 tx_wqe->data.sgl, NULL, tx_wqe->data.inline_data);
}

static void efa_send_wr_send_imm_64(struct ibv_qp_ex *ibvqpx, __be32 imm_data)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_init(qp, ibvqpx, EFA_IO_SEND, qp->sq.wq.max_sge, &tx_wqe->meta,
			 tx_wqe->data.sgl, NULL, tx_wqe->data.inline_data);
	efa_send_wr_set_imm_data(qp->sq.curr_tx_wqe.md, imm_data);
}

static void efa_send_wr_send_imm_128(struct ibv_qp_ex *ibvqpx, __be32 imm_data)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe_128 *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe_128 *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_init(qp, ibvqpx, EFA_IO_SEND, qp->sq.wq.max_sge, &tx_wqe->meta,
			 tx_wqe->data.sgl, NULL, tx_wqe->data.inline_data);
	efa_send_wr_set_imm_data(qp->sq.curr_tx_wqe.md, imm_data);
}

static inline void efa_send_wr_rdma_common(struct efa_qp *qp, struct ibv_qp_ex *ibvqpx,
					   uint32_t rkey, uint64_t remote_addr,
					   enum efa_io_send_op_type op_type,
					   struct efa_io_tx_meta_desc *md,
					   struct efa_io_tx_buf_desc *local_mem,
					   struct efa_io_remote_mem_addr *remote_mem,
					   uint8_t *inline_data) ALWAYS_INLINE;
static inline void efa_send_wr_rdma_common(struct efa_qp *qp, struct ibv_qp_ex *ibvqpx,
					   uint32_t rkey, uint64_t remote_addr,
					   enum efa_io_send_op_type op_type,
					   struct efa_io_tx_meta_desc *md,
					   struct efa_io_tx_buf_desc *local_mem,
					   struct efa_io_remote_mem_addr *remote_mem,
					   uint8_t *inline_data)
{
	efa_send_wr_init(qp, ibvqpx, op_type, qp->sq.max_wr_rdma_sge, md,
			 local_mem, remote_mem, inline_data);

	efa_send_wr_set_rdma_addr(remote_mem, rkey, remote_addr);
}

static void efa_send_wr_rdma_read_64(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
				     uint64_t remote_addr)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_READ,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem, NULL);
}

static void efa_send_wr_rdma_read_128(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
				      uint64_t remote_addr)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe_128 *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe_128 *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_READ,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem, NULL);
}

static void efa_send_wr_rdma_write_64(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
				      uint64_t remote_addr)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_WRITE,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem, NULL);
}

static void efa_send_wr_rdma_write_128(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
				       uint64_t remote_addr)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe_128 *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe_128 *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_WRITE,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem,
				qp->sq.inline_write_enabled ? tx_wqe->data.rdma_req.inline_data :
							      NULL);
}

static void efa_send_wr_rdma_write_imm_64(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
					  uint64_t remote_addr, __be32 imm_data)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_WRITE,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem, NULL);
	efa_send_wr_set_imm_data(qp->sq.curr_tx_wqe.md, imm_data);
}

static void efa_send_wr_rdma_write_imm_128(struct ibv_qp_ex *ibvqpx, uint32_t rkey,
					   uint64_t remote_addr, __be32 imm_data)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_wqe_128 *tx_wqe;

	tx_wqe = (struct efa_io_tx_wqe_128 *)efa_send_wr_alloc(qp, ibvqpx);
	if (unlikely(!tx_wqe))
		return;

	efa_send_wr_rdma_common(qp, ibvqpx, rkey, remote_addr, EFA_IO_RDMA_WRITE,
				&tx_wqe->meta, tx_wqe->data.rdma_req.local_mem,
				&tx_wqe->data.rdma_req.remote_mem,
				qp->sq.inline_write_enabled ? tx_wqe->data.rdma_req.inline_data :
							      NULL);
	efa_send_wr_set_imm_data(qp->sq.curr_tx_wqe.md, imm_data);
}

static void efa_send_wr_set_sge(struct ibv_qp_ex *ibvqpx, uint32_t lkey,
				uint64_t addr, uint32_t length)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_meta_desc *md;
	uint8_t op_type;

	if (unlikely(qp->wr_session_err))
		return;

	md = qp->sq.curr_tx_wqe.md;
	md->length = 1;

	op_type = EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE);
	if (op_type == EFA_IO_RDMA_READ || op_type == EFA_IO_RDMA_WRITE)
		qp->sq.curr_tx_wqe.remote_mem->length = length;

	efa_set_tx_buf(qp->sq.curr_tx_wqe.local_mem, addr, lkey, length);
}

static void efa_send_wr_set_sge_list(struct ibv_qp_ex *ibvqpx, size_t num_sge,
				     const struct ibv_sge *sg_list)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_meta_desc *md;
	struct efa_sq *sq = &qp->sq;
	uint8_t op_type;

	if (unlikely(qp->wr_session_err))
		return;

	md = sq->curr_tx_wqe.md;

	op_type = EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE);
	if (unlikely(num_sge > sq->curr_tx_wqe.max_sge)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] op_type[%u] num_sge[%zu] > max_sge[%u]\n",
			  ibvqpx->qp_base.qp_num, op_type, num_sge,
			  sq->curr_tx_wqe.max_sge);
		qp->wr_session_err = EINVAL;
		return;
	}

	if (op_type == EFA_IO_RDMA_READ || op_type == EFA_IO_RDMA_WRITE)
		sq->curr_tx_wqe.remote_mem->length = efa_sge_total_bytes(sg_list, num_sge);

	efa_post_send_sgl(sq->curr_tx_wqe.local_mem, md, sg_list, num_sge);
}

static void efa_send_wr_set_inline_data(struct ibv_qp_ex *ibvqpx, void *addr,
					size_t length)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_meta_desc *md = qp->sq.curr_tx_wqe.md;
	uint8_t op_type;

	if (unlikely(qp->wr_session_err))
		return;

	if (unlikely(length > qp->sq.max_inline_data)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] WR inline length %zu > %zu\n",
			  ibvqpx->qp_base.qp_num, length,
			  qp->sq.max_inline_data);
		qp->wr_session_err = EINVAL;
		return;
	}

	if (unlikely(!qp->sq.curr_tx_wqe.inline_data)) {
		op_type = EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE);
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] inline op_type[%u] isn't supported\n",
			  ibvqpx->qp_base.qp_num, op_type);
		qp->wr_session_err = EINVAL;
		return;
	}

	EFA_SET(&md->ctrl1, EFA_IO_TX_META_DESC_INLINE_MSG, 1);
	memcpy(qp->sq.curr_tx_wqe.inline_data, addr, length);
	md->length = length;
	if (qp->sq.curr_tx_wqe.remote_mem)
		qp->sq.curr_tx_wqe.remote_mem->length = length;
}

static void
efa_send_wr_set_inline_data_list(struct ibv_qp_ex *ibvqpx,
				 size_t num_buf,
				 const struct ibv_data_buf *buf_list)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_io_tx_meta_desc *md = qp->sq.curr_tx_wqe.md;
	uint32_t length, total_length = 0;
	uint8_t op_type;
	size_t i;

	if (unlikely(qp->wr_session_err))
		return;

	if (unlikely(efa_buf_list_total_bytes(buf_list, num_buf) >
		     qp->sq.max_inline_data)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] WR inline length %zu > %zu\n",
			  ibvqpx->qp_base.qp_num,
			  efa_buf_list_total_bytes(buf_list, num_buf),
			  qp->sq.max_inline_data);
		qp->wr_session_err = EINVAL;
		return;
	}

	if (unlikely(!qp->sq.curr_tx_wqe.inline_data)) {
		op_type = EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE);
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "SQ[%u] inline op_type[%u] isn't supported\n",
			  ibvqpx->qp_base.qp_num, op_type);
		qp->wr_session_err = EINVAL;
		return;
	}

	for (i = 0; i < num_buf; i++) {
		length = buf_list[i].length;

		memcpy(qp->sq.curr_tx_wqe.inline_data + total_length,
		       buf_list[i].addr, length);
		total_length += length;
	}

	EFA_SET(&md->ctrl1, EFA_IO_TX_META_DESC_INLINE_MSG, 1);
	md->length = total_length;
	if (qp->sq.curr_tx_wqe.remote_mem)
		qp->sq.curr_tx_wqe.remote_mem->length = total_length;
}

static void efa_send_wr_set_addr(struct ibv_qp_ex *ibvqpx,
				 struct ibv_ah *ibvah,
				 uint32_t remote_qpn, uint32_t remote_qkey)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_ah *ah = to_efa_ah(ibvah);
	struct efa_io_tx_meta_desc *md;

	if (unlikely(qp->wr_session_err))
		return;

	md = qp->sq.curr_tx_wqe.md;

	md->dest_qp_num = remote_qpn;
	md->ah = ah->efa_ah;
	md->qkey = remote_qkey;

	rdma_tracepoint(rdma_core_efa, post_send, qp->dev->name, ibvqpx->wr_id,
			EFA_GET(&md->ctrl1, EFA_IO_TX_META_DESC_OP_TYPE),
			ibvqpx->qp_base.qp_num, remote_qpn, ah->efa_ah,
			efa_wqe_get_data_length(&qp->sq));
}

static void efa_send_wr_set_processing_hints(struct efadv_qp *efadv_qp, uint32_t hints)
{
	struct efa_qp *qp = efadv_qp_to_efa_qp(efadv_qp);
	uint8_t wqe_hints = 0;

	if (unlikely(qp->wr_session_err))
		return;

	if (hints & EFADV_WR_PROCESSING_HINT_BURST_PPS_SENSITIVE)
		wqe_hints |= EFA_IO_PROCESSING_HINT_BURST_PPS_SENSITIVE;

	EFA_SET(&qp->sq.curr_tx_wqe.md->ctrl3, EFA_IO_TX_META_DESC_PROCESSING_HINTS, wqe_hints);
}

static void efa_send_wr_start(struct ibv_qp_ex *ibvqpx)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_sq *sq = &qp->sq;

	if (qp->sq.wq.need_lock)
		mmio_wc_spinlock(&qp->sq.wq.wqlock);
	else
		mmio_wc_start();

	qp->wr_session_err = 0;
	sq->num_wqe_pending = 0;
	sq->phase_rb = qp->sq.wq.phase;
}

static inline void efa_sq_roll_back(struct efa_sq *sq)
{
	struct efa_qp *qp = container_of(sq, struct efa_qp, sq);
	struct efa_wq *wq = &sq->wq;

	verbs_debug(verbs_get_ctx(qp->verbs_qp.qp.context),
		    "SQ[%u] Rollback num_wqe_pending = %u\n",
		    qp->verbs_qp.qp.qp_num, sq->num_wqe_pending);
	wq->wqe_posted -= sq->num_wqe_pending;
	wq->pc -= sq->num_wqe_pending;
	wq->wrid_idx_pool_next -= sq->num_wqe_pending;
	wq->phase = sq->phase_rb;
}

static int efa_send_wr_complete(struct ibv_qp_ex *ibvqpx)
{
	struct efa_qp *qp = to_efa_qp_ex(ibvqpx);
	struct efa_sq *sq = &qp->sq;
	uint32_t max_txbatch = sq->max_batch_wr;
	uint32_t num_wqe_to_copy;
	uint16_t local_idx = 0;
	uint16_t curbatch = 0;
	uint16_t sq_desc_idx;
	uint16_t pc;

	if (unlikely(qp->wr_session_err)) {
		efa_sq_roll_back(sq);
		goto out;
	}

	/*
	 * Copy local queue to device in chunks, handling wraparound and max
	 * doorbell batch.
	 */
	pc = sq->wq.pc - sq->num_wqe_pending;
	sq_desc_idx = pc & sq->wq.desc_mask;

	/* mmio_wc_start() comes from efa_send_wr_start() */
	while (sq->num_wqe_pending) {
		num_wqe_to_copy = min3(sq->num_wqe_pending,
				       sq->wq.wqe_cnt - sq_desc_idx,
				       max_txbatch - curbatch);
		mmio_memcpy_x64(sq->desc + sq_desc_idx * sq->wqe_size,
				sq->local_queue + local_idx * sq->wqe_size,
				num_wqe_to_copy * sq->wqe_size);

		sq->num_wqe_pending -= num_wqe_to_copy;
		local_idx += num_wqe_to_copy;
		curbatch += num_wqe_to_copy;
		pc += num_wqe_to_copy;
		sq_desc_idx = (sq_desc_idx + num_wqe_to_copy) &
			      sq->wq.desc_mask;

		if (curbatch == max_txbatch) {
			mmio_flush_writes();
			efa_sq_ring_doorbell(sq, pc);
			curbatch = 0;
			mmio_wc_start();
		}
	}

	if (curbatch) {
		mmio_flush_writes();
		efa_sq_ring_doorbell(sq, sq->wq.pc);
	}
out:
	/*
	 * Not using mmio_wc_spinunlock as the doorbell write should be done
	 * inside the lock.
	 */
	if (sq->wq.need_lock)
		pthread_spin_unlock(&sq->wq.wqlock);

	return qp->wr_session_err;
}

static void efa_send_wr_abort(struct ibv_qp_ex *ibvqpx)
{
	struct efa_sq *sq = &to_efa_qp_ex(ibvqpx)->sq;

	efa_sq_roll_back(sq);
	if (sq->wq.need_lock)
		pthread_spin_unlock(&sq->wq.wqlock);
}

static void efa_qp_fill_wr_pfns(struct efa_qp *qp,
				struct ibv_qp_init_attr_ex *attr_ex,
				struct efadv_qp_init_attr *efa_attr,
				uint16_t wqe_size)
{
	bool use_64 = wqe_size == EFA_IO_TX_DESC_SIZE_64;
	struct ibv_qp_ex *ibvqpx = &qp->verbs_qp.qp_ex;

	ibvqpx->wr_start = efa_send_wr_start;
	ibvqpx->wr_complete = efa_send_wr_complete;
	ibvqpx->wr_abort = efa_send_wr_abort;

	if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_SEND)
		ibvqpx->wr_send = use_64 ? efa_send_wr_send_64 : efa_send_wr_send_128;

	if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_SEND_WITH_IMM)
		ibvqpx->wr_send_imm = use_64 ? efa_send_wr_send_imm_64 : efa_send_wr_send_imm_128;

	if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_RDMA_READ)
		ibvqpx->wr_rdma_read = use_64 ? efa_send_wr_rdma_read_64 :
						efa_send_wr_rdma_read_128;

	if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_RDMA_WRITE)
		ibvqpx->wr_rdma_write = use_64 ? efa_send_wr_rdma_write_64 :
						 efa_send_wr_rdma_write_128;

	if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
		ibvqpx->wr_rdma_write_imm = use_64 ? efa_send_wr_rdma_write_imm_64 :
						     efa_send_wr_rdma_write_imm_128;

	ibvqpx->wr_set_inline_data = efa_send_wr_set_inline_data;
	ibvqpx->wr_set_inline_data_list = efa_send_wr_set_inline_data_list;
	ibvqpx->wr_set_sge = efa_send_wr_set_sge;
	ibvqpx->wr_set_sge_list = efa_send_wr_set_sge_list;
	ibvqpx->wr_set_ud_addr = efa_send_wr_set_addr;

	if (efa_attr->wr_flags & EFADV_WR_EX_WITH_PROCESSING_HINTS)
		qp->dv_qp.wr_set_processing_hints = efa_send_wr_set_processing_hints;
}

static int efa_post_recv_validate(struct efa_qp *qp, struct ibv_recv_wr *wr)
{
	if (unlikely(qp->verbs_qp.qp.state == IBV_QPS_RESET ||
		     qp->verbs_qp.qp.state == IBV_QPS_ERR)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "RQ[%u] Invalid QP state\n",
			  qp->verbs_qp.qp.qp_num);
		return EINVAL;
	}

	if (unlikely(wr->num_sge > qp->rq.wq.max_sge)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "RQ[%u] WR num_sge %d > %d\n",
			  qp->verbs_qp.qp.qp_num, wr->num_sge,
			  qp->rq.wq.max_sge);
		return EINVAL;
	}

	if (unlikely(qp->rq.wq.wqe_posted - qp->rq.wq.wqe_completed ==
		     qp->rq.wq.wqe_cnt)) {
		verbs_err(verbs_get_ctx(qp->verbs_qp.qp.context),
			  "RQ[%u] is full wqe_posted[%u] wqe_completed[%u] wqe_cnt[%u]\n",
			  qp->verbs_qp.qp.qp_num, qp->rq.wq.wqe_posted,
			  qp->rq.wq.wqe_completed, qp->rq.wq.wqe_cnt);
		return ENOMEM;
	}

	return 0;
}

int efa_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
		  struct ibv_recv_wr **bad)
{
	struct efa_qp *qp = to_efa_qp(ibvqp);
	struct efa_wq *wq = &qp->rq.wq;
	struct efa_io_rx_desc rx_buf;
	uint32_t rq_desc_offset;
	uintptr_t addr;
	int err = 0;
	size_t i;

	if (wq->need_lock)
		pthread_spin_lock(&wq->wqlock);

	while (wr) {
		err = efa_post_recv_validate(qp, wr);
		if (err) {
			*bad = wr;
			goto ring_db;
		}

		memset(&rx_buf, 0, sizeof(rx_buf));

		rx_buf.req_id = efa_wq_get_dev_req_id_locked(wq, wr->wr_id);
		wq->wqe_posted++;

		/* Default init of the rx buffer */
		EFA_SET(&rx_buf.lkey_ctrl, EFA_IO_RX_DESC_FIRST, 1);
		EFA_SET(&rx_buf.lkey_ctrl, EFA_IO_RX_DESC_LAST, 0);

		for (i = 0; i < wr->num_sge; i++) {
			/* Set last indication if need) */
			if (i == wr->num_sge - 1)
				EFA_SET(&rx_buf.lkey_ctrl, EFA_IO_RX_DESC_LAST,
					1);

			addr = wr->sg_list[i].addr;

			/* Set RX buffer desc from SGE */
			rx_buf.length = min_t(uint32_t, wr->sg_list[i].length, UINT16_MAX);
			EFA_SET(&rx_buf.lkey_ctrl, EFA_IO_RX_DESC_LKEY,
				wr->sg_list[i].lkey);
			rx_buf.buf_addr_lo = addr;
			rx_buf.buf_addr_hi = (uint64_t)addr >> 32;

			/* Copy descriptor to RX ring */
			rq_desc_offset = (wq->pc & wq->desc_mask) *
					 sizeof(rx_buf);
			memcpy(qp->rq.buf + rq_desc_offset, &rx_buf, sizeof(rx_buf));

			/* Wrap rx descriptor index */
			wq->pc++;
			if (!(wq->pc & wq->desc_mask))
				wq->phase++;

			/* reset descriptor for next iov */
			memset(&rx_buf, 0, sizeof(rx_buf));
		}
		rdma_tracepoint(rdma_core_efa, post_recv, qp->dev->name, wr->wr_id,
				ibvqp->qp_num, wr->num_sge);
		wr = wr->next;
	}

ring_db:
	efa_rq_ring_doorbell(&qp->rq, wq->pc);

	if (wq->need_lock)
		pthread_spin_unlock(&wq->wqlock);

	return err;
}

int efadv_query_ah(struct ibv_ah *ibvah, struct efadv_ah_attr *attr,
		   uint32_t inlen)
{
	uint64_t comp_mask_out = 0;

	if (!is_efa_dev(ibvah->context->device)) {
		verbs_err(verbs_get_ctx(ibvah->context), "Not an EFA device\n");
		return EOPNOTSUPP;
	}

	if (!vext_field_avail(typeof(*attr), ahn, inlen)) {
		verbs_err(verbs_get_ctx(ibvah->context),
			  "Compatibility issues\n");
		return EINVAL;
	}

	memset(attr, 0, inlen);
	attr->ahn = to_efa_ah(ibvah)->efa_ah;

	attr->comp_mask = comp_mask_out;

	return 0;
}

struct ibv_ah *efa_create_ah(struct ibv_pd *ibvpd, struct ibv_ah_attr *attr)
{
	struct efa_create_ah_resp resp = {};
	struct efa_ah *ah;
	int err;

	ah = calloc(1, sizeof(*ah));
	if (!ah)
		return NULL;

	err = ibv_cmd_create_ah(ibvpd, &ah->ibvah, attr,
				&resp.ibv_resp, sizeof(resp));
	if (err) {
		verbs_err(verbs_get_ctx(ibvpd->context),
			  "Failed to create AH\n");
		free(ah);
		errno = err;
		return NULL;
	}

	ah->efa_ah = resp.efa_address_handle;

	return &ah->ibvah;
}

int efa_destroy_ah(struct ibv_ah *ibvah)
{
	struct efa_ah *ah;
	int err;

	ah = to_efa_ah(ibvah);
	err = ibv_cmd_destroy_ah(ibvah);
	if (err) {
		verbs_err(verbs_get_ctx(ibvah->context),
			  "Failed to destroy AH\n");
		return err;
	}
	free(ah);

	return 0;
}

struct ibv_td *efa_alloc_td(struct ibv_context *ibvctx, struct ibv_td_init_attr *init_attr)
{
	struct efa_td *td;

	if (!check_comp_mask(init_attr->comp_mask, 0)) {
		verbs_err(verbs_get_ctx(ibvctx), "Invalid comp_mask\n");
		errno = EOPNOTSUPP;
		return NULL;
	}

	td = calloc(1, sizeof(*td));
	if (!td) {
		errno = ENOMEM;
		return NULL;
	}

	td->ibvtd.context = ibvctx;
	atomic_init(&td->refcount, 0);

	return &td->ibvtd;
}

int efa_dealloc_td(struct ibv_td *ibvtd)
{
	struct efa_td *td;

	td = to_efa_td(ibvtd);
	if (atomic_load(&td->refcount) > 0)
		return EBUSY;

	free(td);

	return 0;
}