/
githubmirror
/
rdma-core
Обзор
Документация
Войти
/
githubmirror
/
rdma-core
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
pyverbs/cmid.pyx
921 строка
36 KB
Mazal Or Yakuel
pyverbs: Add resolve_addrinfo & query_addrinfo
11 янв 2026, 11:56
11 янв 2026, 11:56
a019dc7
Код
Авторство
О чём код?
from libc.stdint cimport uintptr_t, uint8_t from libc.string cimport memset import weakref import ctypes from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError from pyverbs.qp cimport QPInitAttr, QPAttr, ECE from pyverbs.base import PyverbsRDMAErrno from pyverbs.base cimport close_weakrefs cimport pyverbs.libibverbs_enums as e cimport pyverbs.librdmacm_enums as ce from pyverbs.addr cimport AH, AHAttr from pyverbs.device cimport Context cimport pyverbs.libibverbs as v cimport pyverbs.librdmacm as cm from pyverbs.pd cimport PD from pyverbs.mr cimport MR from pyverbs.cq cimport WC cdef class ConnParam(PyverbsObject): def __init__(self, resources=1, depth=1, flow_control=0, retry=5, rnr_retry=5, srq=0, qp_num=0, data_len=0): """ Initialize a ConnParam object over an underlying rdma_conn_param C object which contains connection parameters. There are a few types of port spaces in RDMACM: RDMA_PS_TCP, RDMA_PS_UDP, RDMA_PS_IB and RDMA_PS_IPOIB. RDMA_PS_TCP resembles RC QP connection, which provides reliable, connection-oriented QP communication. This object applies only to RDMA_PS_TCP port space. :param resources: Max outstanding RDMA read and atomic ops that local side will accept from the remote side. :param depth: Max outstanding RDMA read and atomic ops that local side will have to the remote side. :param flow_control: Specifies if hardware flow control is available. :param retry: Max number of times that a send, RDMA or atomic op from the remote peer should be retried. :param rnr_retry: The maximum number of times that a send operation from the remote peer should be retried on a connection after receiving a receiver not ready (RNR) error. :param srq: Specifies if the QP using shared receive queue, ignored if the QP created by CMID. :param qp_num: Specifies the QP number, ignored if the QP created by CMID. :param data_len: Specifies the private data length. RDMA_PS_TCP connect: 56 accept: 196 RDMA_PS_UDP connect: 180 accept: 136 :return: ConnParam object """ super().__init__() memset(&self.conn_param, 0, sizeof(cm.rdma_conn_param)) self.conn_param.responder_resources = resources self.conn_param.initiator_depth = depth self.conn_param.flow_control = flow_control self.conn_param.retry_count = retry self.conn_param.rnr_retry_count = rnr_retry self.conn_param.srq = srq self.conn_param.qp_num = qp_num self.data = (ctypes.c_char * data_len)() @property def qpn(self): return self.conn_param.qp_num @qpn.setter def qpn(self, val): self.conn_param.qp_num = val @property def private_data(self): data_array = bytearray(self.conn_param.private_data_len) cdef const unsigned char *p = <const unsigned char*>self.conn_param.private_data for i in range(self.conn_param.private_data_len): data_array[i] = p[i] return bytes(data_array) def set_private_data(self, data): if (min(len(self.data), len(data)) == 0): return for i in range(len(self.data)): self.data[i] = 0 for i in range(min(len(self.data), len(data))): self.data[i] = data[i] cdef size_t ptr = ctypes.addressof(self.data) self.conn_param.private_data = <const void*>ptr self.conn_param.private_data_len = min(len(self.data), len(data)) def __str__(self): print_format = '{:<4}: {:<4}\n' return '{}: {}\n'.format('Connection parameters', "") +\ print_format.format('responder resources', self.conn_param.responder_resources) +\ print_format.format('initiator depth', self.conn_param.initiator_depth) +\ print_format.format('flow control', self.conn_param.flow_control) +\ print_format.format('retry count', self.conn_param.retry_count) +\ print_format.format('rnr retry count', self.conn_param.rnr_retry_count) +\ print_format.format('srq', self.conn_param.srq) +\ print_format.format('qp number', self.conn_param.qp_num) cdef class JoinMCAttrEx(PyverbsObject): def __init__(self, AddrInfo addr not None, comp_mask=0, join_flags=0): """ Initialize a JoinMCAttrEx object over an underlying rdma_cm_join_mc_attr_ex C object which contains the extended join multicast attributes. :param addr: Multicast address identifying the group to join. :param comp_mask: Bitwise OR between "rdma_cm_join_mc_attr_mask" enum. :param join_flags: Single flag from "rdma_cm_mc_join_flags" enum. Indicates the type of the join requests. """ super().__init__() self.join_mc_attr_ex.addr = addr.addr_info.ai_src_addr self.join_mc_attr_ex.comp_mask = comp_mask self.join_mc_attr_ex.join_flags = join_flags @property def join_flags(self): return self.join_mc_attr_ex.join_flags @join_flags.setter def join_flags(self, val): self.join_mc_attr_ex.join_flags = val @property def comp_mask(self): return self.join_mc_attr_ex.comp_mask @comp_mask.setter def comp_mask(self, val): self.join_mc_attr_ex.comp_mask = val cdef class UDParam(PyverbsObject): def __init__(self, CMEvent cm_event not None): """ Initialize a UDParam object over an underlying rdma_ud_param C object which contains UD connection parameters. :param cm_event: The creator of UDParam. When the active side gets connection establishment event, the event contains UDParam for the passive CMID details. :return: UDParam object """ super().__init__() memset(&self.ud_param, 0, sizeof(cm.rdma_ud_param)) self.ud_param = (<CMEvent>cm_event).event.param.ud @property def qp_num(self): return self.ud_param.qp_num @property def qkey(self): return self.ud_param.qkey @property def ah_attr(self): ah_attr = AHAttr() ah_attr.init_from_ud_param(self) return ah_attr cdef class AddrInfo(PyverbsObject): def __init__(self, src=None, dst=None, src_service=None, dst_service=None, port_space=0, flags=0): """ Initialize an AddrInfo object over an underlying rdma_addrinfo C object. :param src: Name, dotted-decimal IPv4 or IPv6 hex address to bind to. :param dst: Name, dotted-decimal IPv4 or IPv6 hex address to connect to. :param src_service: The service name or port number of the source address. :param dst_service: The service name or port number of the destination address. :param port_space: RDMA port space used (RDMA_PS_UDP or RDMA_PS_TCP). :param flags: Hint flags which control the operation. :return: An AddrInfo object which contains information needed to establish communication. """ super().__init__() self.addr_info = NULL if src or dst or src_service or dst_service: self.get_addrinfo(src, dst, src_service, dst_service, port_space, flags) def get_addrinfo(self, src=None, dst=None, src_service=None, dst_service=None, port_space=0, flags=0): """ Resolve source and destination addresses. :param src: Name, dotted-decimal IPv4 or IPv6 hex address to bind to. :param dst: Name, dotted-decimal IPv4 or IPv6 hex address to connect to. :param src_service: The service name or port number of the source address. :param dst_service: The service name or port number of the destination address. :param port_space: RDMA port space used (RDMA_PS_UDP, RDMA_PS_TCP or RDMA_PS_IB). :param flags: Hint flags which control the operation. :raises PyverbsRDMAErrno: If fails to get Address Info. :return: None """ cdef char* src_srvc = NULL cdef char* dst_srvc = NULL cdef char* src_addr = NULL cdef char* dst_addr = NULL cdef cm.rdma_addrinfo hints cdef cm.rdma_addrinfo *hints_ptr = NULL cdef cm.rdma_addrinfo *res = NULL if src is not None: if isinstance(src, str): src = src.encode('utf-8') src_addr = <char*>src if dst is not None: if isinstance(dst, str): dst = dst.encode('utf-8') dst_addr = <char*>dst if src_service is not None: if isinstance(src_service, str): src_service = src_service.encode('utf-8') src_srvc = <char*>src_service if dst_service is not None: if isinstance(dst_service, str): dst_service = dst_service.encode('utf-8') dst_srvc = <char*>dst_service hints_ptr = &hints memset(hints_ptr, 0, sizeof(cm.rdma_addrinfo)) hints.ai_port_space = port_space hints.ai_flags = flags # Set hint fields to support GID address if port_space == ce.RDMA_PS_IB: hints.ai_flags |= ce.RAI_FAMILY | ce.RAI_NUMERICHOST hints.ai_family = cm.AF_IB if flags & ce.RAI_PASSIVE: ret = cm.rdma_getaddrinfo(src_addr, src_srvc, hints_ptr, &self.addr_info) else: if src: hints.ai_flags |= ce.RAI_PASSIVE ret = cm.rdma_getaddrinfo(src_addr, src_srvc, hints_ptr, &res) if ret != 0: raise PyverbsRDMAErrno('Failed to get Address Info') hints.ai_src_addr = <cm.sockaddr*>res.ai_src_addr hints.ai_src_len = res.ai_src_len hints.ai_flags &= ~ce.RAI_PASSIVE ret = cm.rdma_getaddrinfo(dst_addr, dst_srvc, hints_ptr, &self.addr_info) if src: cm.rdma_freeaddrinfo(res) if ret != 0: raise PyverbsRDMAErrno('Failed to get Address Info') def __dealloc__(self): self.close() cpdef close(self): if self.addr_info != NULL: if self.logger: self.logger.debug('Closing AddrInfo') cm.rdma_freeaddrinfo(self.addr_info) self.addr_info = NULL cdef class CMEvent(PyverbsObject): def __init__(self, CMEventChannel channel): """ Initialize a CMEvent object over an underlying rdma_cm_event C object :param channel: Event Channel on which this event has been received :return: CMEvent object """ super().__init__() ret = cm.rdma_get_cm_event(channel.event_channel, &self.event) if ret != 0: raise PyverbsRDMAErrno('Failed to create CMEvent') self.logger.debug('Created a CMEvent') def __dealloc__(self): self.close() cpdef close(self): if self.event != NULL: if self.logger: self.logger.debug('Closing CMEvent') self.ack_cm_event() self.event = NULL @property def event_type(self): return self.event.event def ack_cm_event(self): """ Free a communication event. This call frees the event structure and any memory that it references. :return: None """ ret = cm.rdma_ack_cm_event(self.event) if ret != 0: raise PyverbsRDMAErrno('Failed to Acknowledge Event - {}' .format(self.event_str())) self.event = NULL def event_str(self): if self.event == NULL: return '' return (<bytes>cm.rdma_event_str(self.event_type)).decode() @property def private_data(self): data_array = bytearray(self.event.param.conn.private_data_len) cdef const unsigned char *p = <const unsigned char*>self.event.param.conn.private_data for i in range(self.event.param.conn.private_data_len): data_array[i] = p[i] return bytes(data_array) cdef class CMEventChannel(PyverbsObject): def __init__(self): """ Initialize a CMEventChannel object over an underlying rdma_event_channel C object. :return: EventChannel object """ super().__init__() self.event_channel = cm.rdma_create_event_channel() if self.event_channel == NULL: raise PyverbsRDMAErrno('Failed to create CMEventChannel') self.logger.debug('Created a CMEventChannel') def __dealloc__(self): self.close() cpdef close(self): if self.event_channel != NULL: if self.logger: self.logger.debug('Closing CMEventChannel') cm.rdma_destroy_event_channel(self.event_channel) self.event_channel = NULL cdef class CMID(PyverbsCM): def __init__(self, object creator=None, QPInitAttr qp_init_attr=None, PD pd=None, port_space=ce.RDMA_PS_TCP, CMID listen_id=None): """ Initialize a CMID object over an underlying rdma_cm_id C object. This is the main RDMA CM object which provides most of the rdmacm API. Currently only synchronous RDMA_PS_TCP communication supported. Notes: User-specific context, currently not supported. :param creator: For synchronous communication we need AddrInfo object in order to establish connection. We allow creator to be None for inner usage, see get_request method. :param qp_init_attr: Optional initial QP attributes of CMID associated QP. :param pd: Optional parameter, a PD to be associated with this CMID. :param port_space: RDMA port space. :param listen_id: When passive side establishes a connection, it creates a new CMID. listen_id is used to initialize the new CMID. :return: CMID object for synchronous communication. """ cdef v.ibv_qp_init_attr *init cdef v.ibv_pd *in_pd = NULL super().__init__() self.pd = None self.ctx = None self.event_channel = None self.mrs = weakref.WeakSet() if creator is None: return elif isinstance(creator, AddrInfo): init = NULL if qp_init_attr is None else &qp_init_attr.attr if pd is not None: in_pd = pd.pd self.pd = pd ret = cm.rdma_create_ep(&self.id, (<AddrInfo>creator).addr_info, in_pd, init) if ret != 0: raise PyverbsRDMAErrno('Failed to create CM ID') if not (<AddrInfo>creator).addr_info.ai_flags & ce.RAI_PASSIVE: self.ctx = Context(cmid=self) if self.pd is None: self.pd = PD(self) elif isinstance(creator, CMEventChannel): self.event_channel = <CMEventChannel>creator ret = cm.rdma_create_id((<CMEventChannel>creator).event_channel, &self.id, NULL, port_space) if ret != 0: raise PyverbsRDMAErrno('Failed to create CM ID') elif isinstance(creator, CMEvent): if listen_id is None: raise PyverbsUserError('listen ID not provided') self.id = (<CMEvent>creator).event.id self.event_channel = listen_id.event_channel self.ctx = listen_id.ctx self.pd = listen_id.pd else: raise PyverbsRDMAErrno('Cannot create CM ID from {obj}' .format(obj=type(creator))) cdef add_ref(self, obj): if isinstance(obj, MR): self.mrs.add(obj) else: raise PyverbsError('Unrecognized object type') @property def dev_name(self): return str(v.ibv_get_device_name(self.id.verbs.device).decode('utf-8')) @property def port_num(self): return self.id.port_num @property def event_channel(self): return self.event_channel @property def context(self): return self.ctx @property def pd(self): return self.pd @property def qpn(self): if self.id.qp: return self.id.qp.qp_num return None def __dealloc__(self): self.close() cpdef close(self): if self.id != NULL: if self.logger: self.logger.debug('Closing CMID') if self.event_channel is None: cm.rdma_destroy_ep(self.id) else: if self.id.qp != NULL: cm.rdma_destroy_qp(self.id) ret = cm.rdma_destroy_id(self.id) if ret != 0: raise PyverbsRDMAErrno('Failed to close CMID') if self.ctx: (<Context>self.ctx).context = NULL if self.pd: (<PD>self.pd).pd = NULL close_weakrefs([self.mrs]) self.id = NULL def get_request(self): """ Retrieves the next pending connection request event. The call may only be used on listening CMIDs operating synchronously. If the call is successful, a new CMID representing the connection request will be returned to the user. The new CMID will reference event information associated with the request until the user calls reject, accept, or close on the newly created identifier. :return: New CMID representing the connection request. """ to_conn = CMID() with nogil: ret = cm.rdma_get_request(self.id, &to_conn.id) if ret != 0: raise PyverbsRDMAErrno('Failed to get request, no connection established') self.ctx = Context(cmid=to_conn) self.pd = PD(to_conn) return to_conn def bind_addr(self, AddrInfo lai not None): """ Associate a source address with a CMID. If binding to a specific local address, the CMID will also be bound to a local RDMA device. :param lai: Local address information :return: None """ ret = cm.rdma_bind_addr(self.id, lai.addr_info.ai_src_addr) if ret != 0: raise PyverbsRDMAErrno('Failed to Bind ID') # After bind address, cm_id contains ibv_context. # Now we can create Context object. if self.ctx is None: self.ctx = Context(cmid=self) if self.pd is None: self.pd = PD(self) def resolve_addr(self, AddrInfo rai not None, timeout_ms=2000): """ Resolve destination and optional source addresses from IP addresses to an RDMA address. If successful, the specified rdma_cm_id will be bound to a local device. :param rai: Remote address information. :param timeout_ms: Time to wait for resolution to complete [msec] :return: None """ ret = cm.rdma_resolve_addr(self.id, rai.addr_info.ai_src_addr, rai.addr_info.ai_dst_addr, timeout_ms) if ret != 0: raise PyverbsRDMAErrno('Failed to Resolve Address') def join_multicast(self, AddrInfo addr=None, JoinMCAttrEx mc_attr=None, context=0): """ Joins a multicast group and attaches an associated QP to the group. :param addr: Multicast address identifying the group to join. :param mc_attr: JoinMCAttrEx object is requierd to use rdma_join_multicast_ex. This object contains the join flags and the AddrInfo to join. :param context: User-defined context associated with the join request. :return: None """ cdef cm.rdma_cm_join_mc_attr_ex *mc_join_attr = NULL if not addr and not mc_attr: raise PyverbsUserError('Join to multicast must have AddrInfo or JoinMCAttrEx arguments') if not mc_attr: ret = cm.rdma_join_multicast(self.id, addr.addr_info.ai_src_addr, <void*><uintptr_t>context) else: ret = cm.rdma_join_multicast_ex(self.id, &mc_attr.join_mc_attr_ex, <void*><uintptr_t>context) if ret != 0: raise PyverbsRDMAErrno('Failed to Join multicast') def leave_multicast(self, AddrInfo addr not None): """ Leaves a multicast group and detaches an associated QP from the group. :param addr: AddrInfo object, represent the multicast address that identifies the group to leave. :return: None """ ret = cm.rdma_leave_multicast(self.id, addr.addr_info.ai_src_addr) if ret != 0: raise PyverbsRDMAErrno('Failed to leave multicast') def resolve_route(self, timeout_ms=2000): """ Resolve an RDMA route to the destination address in order to establish a connection. The destination must already have been resolved by calling resolve_addr. Thus this function is called on the client side after resolve_addr but before calling connect. :param timeout_ms: Time to wait for resolution to complete :return: None """ ret = cm.rdma_resolve_route(self.id, timeout_ms) if ret != 0: raise PyverbsRDMAErrno('Failed to Resolve Route') # After resolve route, cm_id contains ibv_context. # Now we can create Context object. if self.ctx is None: self.ctx = Context(cmid=self) if self.pd is None: self.pd = PD(self) def listen(self, backlog=0): """ Listen for incoming connection requests or datagram service lookup. The listen is restricted to the locally bound source address. :param backlog: The backlog of incoming connection requests :return: None """ ret = cm.rdma_listen(self.id, backlog) if ret != 0: raise PyverbsRDMAErrno('Listen Failed') def connect(self, ConnParam param=None): """ Initiates an active connection request to a remote destination. :param param: Optional connection parameters :return: None """ cdef cm.rdma_conn_param *conn = ¶m.conn_param if param else NULL ret = cm.rdma_connect(self.id, conn) if ret != 0: raise PyverbsRDMAErrno('Failed to Connect') def disconnect(self): """ Disconnects a connection and transitions any associated QP to error state. :return: None """ ret = cm.rdma_disconnect(self.id) if ret != 0: raise PyverbsRDMAErrno('Failed to Disconnect') def accept(self, ConnParam param=None): """ Is called from the listening side to accept a connection or datagram service lookup request. :param param: Optional connection parameters :return: None """ cdef cm.rdma_conn_param *conn = ¶m.conn_param if param else NULL ret = cm.rdma_accept(self.id, conn) if ret != 0: raise PyverbsRDMAErrno('Failed to Accept Connection') def establish(self): """ Complete an active connection request. If a QP has not been created on the CMID, this method should be called by the active side to complete the connection, after getting connect response event. This will trigger a connection established event on the passive side. This method should not be used on a CMID on which a QP has been created. """ ret = cm.rdma_establish(self.id) if ret != 0: raise PyverbsRDMAErrno('Failed to Complete an active connection request') def set_local_ece(self, ECE ece): """ Set local ECE paraemters to be used for REQ/REP communication. :param ece: ECE object with the requested configuration :return: None """ rc = cm.rdma_set_local_ece(self.id, &ece.ece) if rc != 0: raise PyverbsRDMAErrno('Failed to set local ECE') def get_remote_ece(self): """ Get ECE parameters as were received from the communication peer. :return: ECE object with the ece configuration """ ece = ECE() rc = cm.rdma_get_remote_ece(self.id, &ece.ece) if rc != 0: raise PyverbsRDMAErrno('Failed to get remote ECE') return ece def create_qp(self, QPInitAttr qp_init not None): """ Create a QP, which is associated with CMID. If CMID and qp_init don't hold any CQs, new CQs will be created and associated with CMID. If only qp_init provides CQs, they will not be associated with CMID. If both provide CQs they have to be using the same CQs. :param qp_init: QP init attributes """ ret = cm.rdma_create_qp(self.id, (<PD>self.pd).pd, &qp_init.attr) if ret != 0: raise PyverbsRDMAErrno('Failed to Create QP') def query_qp(self, attr_mask): """ Query QP using ibv_query_qp. :param attr_mask: Which attributes to query (use <enum name> enum) :return: A (QPAttr, QPInitAttr) tuple, containing the relevant QP info """ attr = QPAttr() init_attr = QPInitAttr() rc = v.ibv_query_qp(self.id.qp, &attr.attr, attr_mask, &init_attr.attr) if rc != 0: raise PyverbsRDMAError('Failed to query QP', rc) return attr, init_attr def init_qp_attr(self, qp_state): """ Initialize a QPAttr object used for state transitions of an external QP (a QP which was not created using CMID). When connecting external QPs using CMIDs both sides must call this method before QP state transition to RTR/RTS in order to obtain relevant QP attributes from CMID. :param qp_state: The QP's destination state :return: A (QPAttr, attr_mask) tuple, where attr_mask defines which attributes of QPAttr are valid """ cdef int attr_mask qp_attr = QPAttr() qp_attr.qp_state = qp_state rc = cm.rdma_init_qp_attr(self.id, &qp_attr.attr, &attr_mask) if rc != 0: raise PyverbsRDMAErrno('Failed to get QP attributes') return qp_attr, attr_mask def reg_msgs(self, size): """ Registers a memory region for sending or receiving messages or for RDMA operations. The registered memory may then be posted to an CMID using post_send or post_recv methods. :param size: The total length of the memory to register :return: registered MR """ return MR(self, size, e.IBV_ACCESS_LOCAL_WRITE) def reg_read(self, size=0): """ Registers a memory region for sending or receiving messages or for remote read operations. :param size: The total length of the memory to register :return: registered MR """ return MR(self, size, e.IBV_ACCESS_REMOTE_READ) def reg_write(self, size=0): """ Registers a memory region for sending or receiving messages or for remote write operations. :param size: The total length of the memory to register :return: registered MR """ return MR(self, size, e.IBV_ACCESS_REMOTE_WRITE) def post_recv(self, MR mr not None, length=None): """ Posts a recv_wr via QP associated with CMID. Context param of rdma_post_recv C function currently not supported. :param mr: A valid MR object. :param length: length of buffer to recv (default: mr length). :return: None """ if not length: length = mr.mr.length ret = cm.rdma_post_recv(self.id, NULL, mr.buf, length, mr.mr) if ret != 0: raise PyverbsRDMAErrno('Failed to Post Receive') def post_send(self, MR mr not None, flags=v.IBV_SEND_SIGNALED, length=None): """ Posts a message via QP associated with CMID. Context param of rdma_post_send C function currently not supported. :param mr: A valid MR object which contains message to send. :param flags: flags for send work request. :param length: length of buffer to send (default: mr length). :return: None """ if not length: length = mr.mr.length ret = cm.rdma_post_send(self.id, NULL, mr.buf, length, mr.mr, flags) if ret != 0: raise PyverbsRDMAErrno('Failed to Post Send') def post_read(self, MR mr not None, length, remote_addr, rkey, flags=0): """ Post read WR using the CMIDs internal QP. :param mr: A valid MR object. :param length: length of buffer to send. :param remote_addr: The remote MR address. :param rkey: The remote MR rkey. :param flags: flags for send work request. :return: None """ ret = cm.rdma_post_read(self.id, NULL, mr.buf, length, mr.mr, flags, remote_addr, rkey) if ret != 0: raise PyverbsRDMAErrno('Failed to Post Read') def post_write(self, MR mr not None, length, remote_addr, rkey, flags=0): """ Post write WR using the CMIDs internal QP. :param mr: A valid MR object. :param length: length of buffer to send. :param remote_addr: The remote MR address. :param rkey: The remote MR rkey. :param flags: flags for send work request. :return: None """ ret = cm.rdma_post_write(self.id, NULL, mr.buf, length, mr.mr, flags, remote_addr, rkey) if ret != 0: raise PyverbsRDMAErrno('Failed to Post Write') def post_ud_send(self, MR mr not None, AH ah not None, rqpn=0, flags=v.IBV_SEND_SIGNALED, length=None): """ Posts a message via UD QP associated with CMID to another UD QP. :param mr: A valid MR object which contains message to send. :param ah: The destination AH. :param rqpn: The remote QP number. :param flags: flags for send work request. :param length: length of buffer to send. :return: None """ if not length: length = mr.mr.length ret = cm.rdma_post_ud_send(self.id, NULL, mr.buf, length, mr.mr, flags, ah.ah, rqpn) if ret != 0: raise PyverbsRDMAErrno('Failed to Post Send') def get_recv_comp(self): """ Polls the receive CQ associated with CMID for a work completion. :return: The retrieved WC or None if there is no completions """ cdef v.ibv_wc wc with nogil: ret = cm.rdma_get_recv_comp(self.id, &wc) if ret < 0: raise PyverbsRDMAErrno('Failed to retrieve receive completion') elif ret == 0: return None return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode, vendor_err=wc.vendor_err, byte_len=wc.byte_len, qp_num=wc.qp_num, src_qp=wc.src_qp, imm_data=wc.imm_data, wc_flags=wc.wc_flags, pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl, dlid_path_bits=wc.dlid_path_bits) def get_send_comp(self): """ Polls the send CQ associated with CMID for a work completion. :return: The retrieved WC or None if there is no completions """ cdef v.ibv_wc wc with nogil: ret = cm.rdma_get_send_comp(self.id, &wc) if ret < 0: raise PyverbsRDMAErrno('Failed to retrieve send completion') elif ret == 0: return None return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode, vendor_err=wc.vendor_err, byte_len=wc.byte_len, qp_num=wc.qp_num, src_qp=wc.src_qp, imm_data=wc.imm_data, wc_flags=wc.wc_flags, pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl, dlid_path_bits=wc.dlid_path_bits) def set_option(self, level, optname, optval, optlen): """ Set communication options for a CMID. :param level: The protocol level of the option to set. :param optname: The name of the option to set. :param optval: The option data. :param optlen: The size of the data. """ if optname != ce.RDMA_OPTION_ID_ACK_TIMEOUT: raise PyverbsUserError('Currently only RDMA_OPTION_ID_ACK_TIMEOUT is supported in Pyverbs.') cdef uint8_t value = optval ret = cm.rdma_set_option(self.id, level, optname, <void*>&value, optlen) if ret != 0: raise PyverbsRDMAErrno('Failed to set option') def reject(self, private_data=None): """ Reject a connection or datagram service lookup request. :param private_data: Optional private data to send with the reject message. :param private_data_len: Size (in bytes) of the private data being sent. """ data_len = len(private_data) if private_data else 0 buffer = ctypes.create_string_buffer(data_len) if (private_data): buffer.value = private_data cdef size_t data_ptr = ctypes.addressof(buffer) ret = cm.rdma_reject(self.id, <const void*>data_ptr, data_len) if ret != 0: raise PyverbsRDMAErrno('Failed to Reject Connection') def resolve_addrinfo(self, node=None, service=None, port_space=0, flags=0): """ Resolve source and destination address information using RDMA CM. :param node: Name, dotted-decimal IPv4 or IPv6 hex address to connect to. If None, will use hints.ai_dst_addr if set. :param service: Service name or port number. :param port_space: RDMA port space (RDMA_PS_TCP, RDMA_PS_UDP, or RDMA_PS_IB). :param flags: Optional flags to control resolution: RAI_PASSIVE: Get address for use as server RAI_NUMERICHOST: Node name is numeric RAI_FAMILY: Use specific address family RAI_SA: Use IB SA path record lookup RAI_DNS: Use DNS lookup for IP resolution :return: None """ cdef char* srvc = NULL cdef char* node_addr = NULL cdef cm.rdma_addrinfo hints cdef cm.rdma_addrinfo *hints_ptr = NULL if node is not None: if isinstance(node, str): node = node.encode('utf-8') node_addr = <char*>node if service is not None: if isinstance(service, str): service = service.encode('utf-8') srvc = <char*>service hints_ptr = &hints memset(hints_ptr, 0, sizeof(cm.rdma_addrinfo)) hints.ai_port_space = port_space hints.ai_flags = flags # Set hint fields to support GID address if port_space == ce.RDMA_PS_IB: hints.ai_flags |= ce.RAI_FAMILY | ce.RAI_NUMERICHOST hints.ai_family = cm.AF_IB ret = cm.rdma_resolve_addrinfo(self.id, node_addr, srvc, hints_ptr) if ret != 0: raise PyverbsRDMAErrno('Failed to Resolve AddrInfo') def query_addrinfo(self): """ Query the resolved address info from a CMID. :return: AddrInfo object containing the resolved address information. """ ai = AddrInfo() ret = cm.rdma_query_addrinfo(self.id, &(ai.addr_info)) if ret != 0: raise PyverbsRDMAErrno('Failed to Query AddrInfo') return ai