llvm-project

1
//===--------- device.cpp - Target independent OpenMP target RTL ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// Functionality for managing devices that are handled by RTL plugins.
10
//
11
//===----------------------------------------------------------------------===//
12

13
#include "device.h"
14
#include "OffloadEntry.h"
15
#include "OpenMP/Mapping.h"
16
#include "OpenMP/OMPT/Callback.h"
17
#include "OpenMP/OMPT/Interface.h"
18
#include "PluginManager.h"
19
#include "Shared/APITypes.h"
20
#include "Shared/Debug.h"
21
#include "omptarget.h"
22
#include "private.h"
23
#include "rtl.h"
24

25
#include "Shared/EnvironmentVar.h"
26
#include "llvm/Support/Error.h"
27

28
#include <cassert>
29
#include <climits>
30
#include <cstdint>
31
#include <cstdio>
32
#include <mutex>
33
#include <string>
34
#include <thread>
35

36
#ifdef OMPT_SUPPORT
37
using namespace llvm::omp::target::ompt;
38
#endif
39

40
int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
41
                                            AsyncInfoTy &AsyncInfo) const {
42
  // First, check if the user disabled atomic map transfer/malloc/dealloc.
43
  if (!MappingConfig::get().UseEventsForAtomicTransfers)
44
    return OFFLOAD_SUCCESS;
45

46
  void *Event = getEvent();
47
  bool NeedNewEvent = Event == nullptr;
48
  if (NeedNewEvent && Device.createEvent(&Event) != OFFLOAD_SUCCESS) {
49
    REPORT("Failed to create event\n");
50
    return OFFLOAD_FAIL;
51
  }
52

53
  // We cannot assume the event should not be nullptr because we don't
54
  // know if the target support event. But if a target doesn't,
55
  // recordEvent should always return success.
56
  if (Device.recordEvent(Event, AsyncInfo) != OFFLOAD_SUCCESS) {
57
    REPORT("Failed to set dependence on event " DPxMOD "\n", DPxPTR(Event));
58
    return OFFLOAD_FAIL;
59
  }
60

61
  if (NeedNewEvent)
62
    setEvent(Event);
63

64
  return OFFLOAD_SUCCESS;
65
}
66

67
DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
68
    : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
69
      MappingInfo(*this) {}
70

71
DeviceTy::~DeviceTy() {
72
  if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))
73
    return;
74

75
  ident_t Loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"};
76
  dumpTargetPointerMappings(&Loc, *this);
77
}
78

79
llvm::Error DeviceTy::init() {
80
  int32_t Ret = RTL->init_device(RTLDeviceID);
81
  if (Ret != OFFLOAD_SUCCESS)
82
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
83
                                   "Failed to initialize device %d\n",
84
                                   DeviceID);
85

86
  // Enables recording kernels if set.
87
  BoolEnvar OMPX_RecordKernel("LIBOMPTARGET_RECORD", false);
88
  if (OMPX_RecordKernel) {
89
    // Enables saving the device memory kernel output post execution if set.
90
    BoolEnvar OMPX_ReplaySaveOutput("LIBOMPTARGET_RR_SAVE_OUTPUT", false);
91

92
    uint64_t ReqPtrArgOffset;
93
    RTL->initialize_record_replay(RTLDeviceID, 0, nullptr, true,
94
                                  OMPX_ReplaySaveOutput, ReqPtrArgOffset);
95
  }
96

97
  return llvm::Error::success();
98
}
99

100
// Load binary to device.
101
llvm::Expected<__tgt_device_binary>
102
DeviceTy::loadBinary(__tgt_device_image *Img) {
103
  __tgt_device_binary Binary;
104

105
  if (RTL->load_binary(RTLDeviceID, Img, &Binary) != OFFLOAD_SUCCESS)
106
    return llvm::createStringError(llvm::inconvertibleErrorCode(),
107
                                   "Failed to load binary %p", Img);
108
  return Binary;
109
}
110

111
void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
112
  /// RAII to establish tool anchors before and after data allocation
113
  void *TargetPtr = nullptr;
114
  OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(
115
                    RegionInterface.getCallbacks<ompt_target_data_alloc>(),
116
                    DeviceID, HstPtr, &TargetPtr, Size,
117
                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
118

119
  TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
120
  return TargetPtr;
121
}
122

123
int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
124
  /// RAII to establish tool anchors before and after data deletion
125
  OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(
126
                    RegionInterface.getCallbacks<ompt_target_data_delete>(),
127
                    DeviceID, TgtAllocBegin,
128
                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
129

130
  return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
131
}
132

133
// Submit data to device
134
int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
135
                             AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,
136
                             MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
137
  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
138
    MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true,
139
                              Entry, HDTTMapPtr);
140

141
  /// RAII to establish tool anchors before and after data submit
142
  OMPT_IF_BUILT(
143
      InterfaceRAII TargetDataSubmitRAII(
144
          RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
145
          omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size,
146
          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
147

148
  return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
149
                                AsyncInfo);
150
}
151

152
// Retrieve data from device
153
int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
154
                               int64_t Size, AsyncInfoTy &AsyncInfo,
155
                               HostDataToTargetTy *Entry,
156
                               MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
157
  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
158
    MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false,
159
                              Entry, HDTTMapPtr);
160

161
  /// RAII to establish tool anchors before and after data retrieval
162
  OMPT_IF_BUILT(
163
      InterfaceRAII TargetDataRetrieveRAII(
164
          RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
165
          DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size,
166
          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
167

168
  return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
169
                                  AsyncInfo);
170
}
171

172
// Copy data from current device to destination device directly
173
int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
174
                               int64_t Size, AsyncInfoTy &AsyncInfo) {
175
  /// RAII to establish tool anchors before and after data exchange
176
  /// Note: Despite the fact that this is a data exchange, we use 'from_device'
177
  ///       operation enum (w.r.t. ompt_target_data_op_t) as there is currently
178
  ///       no better alternative. It is still possible to distinguish this
179
  ///       scenario from a real data retrieve by checking if both involved
180
  ///       device numbers are less than omp_get_num_devices().
181
  OMPT_IF_BUILT(
182
      InterfaceRAII TargetDataExchangeRAII(
183
          RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
184
          RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
185
          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
186
  if (!AsyncInfo) {
187
    return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
188
                              Size);
189
  }
190
  return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
191
                                  DstPtr, Size, AsyncInfo);
192
}
193

194
int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) {
195
  DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n",
196
     DPxPTR(HstPtr), Size);
197

198
  if (RTL->data_notify_mapped(RTLDeviceID, HstPtr, Size)) {
199
    REPORT("Notifiying about data mapping failed.\n");
200
    return OFFLOAD_FAIL;
201
  }
202
  return OFFLOAD_SUCCESS;
203
}
204

205
int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {
206
  DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr));
207

208
  if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) {
209
    REPORT("Notifiying about data unmapping failed.\n");
210
    return OFFLOAD_FAIL;
211
  }
212
  return OFFLOAD_SUCCESS;
213
}
214

215
// Run region on device
216
int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
217
                               ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,
218
                               AsyncInfoTy &AsyncInfo) {
219
  return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,
220
                            &KernelArgs, AsyncInfo);
221
}
222

223
// Run region on device
224
bool DeviceTy::printDeviceInfo() {
225
  RTL->print_device_info(RTLDeviceID);
226
  return true;
227
}
228

229
// Whether data can be copied to DstDevice directly
230
bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) {
231
  if (RTL != DstDevice.RTL)
232
    return false;
233

234
  if (RTL->is_data_exchangable(RTLDeviceID, DstDevice.RTLDeviceID))
235
    return true;
236
  return false;
237
}
238

239
int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) {
240
  return RTL->synchronize(RTLDeviceID, AsyncInfo);
241
}
242

243
int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) {
244
  return RTL->query_async(RTLDeviceID, AsyncInfo);
245
}
246

247
int32_t DeviceTy::createEvent(void **Event) {
248
  return RTL->create_event(RTLDeviceID, Event);
249
}
250

251
int32_t DeviceTy::recordEvent(void *Event, AsyncInfoTy &AsyncInfo) {
252
  return RTL->record_event(RTLDeviceID, Event, AsyncInfo);
253
}
254

255
int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) {
256
  return RTL->wait_event(RTLDeviceID, Event, AsyncInfo);
257
}
258

259
int32_t DeviceTy::syncEvent(void *Event) {
260
  return RTL->sync_event(RTLDeviceID, Event);
261
}
262

263
int32_t DeviceTy::destroyEvent(void *Event) {
264
  return RTL->destroy_event(RTLDeviceID, Event);
265
}
266

267
void DeviceTy::dumpOffloadEntries() {
268
  fprintf(stderr, "Device %i offload entries:\n", DeviceID);
269
  for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) {
270
    const char *Kind = "kernel";
271
    if (It.second.isLink())
272
      Kind = "link";
273
    else if (It.second.isGlobal())
274
      Kind = "global var.";
275
    fprintf(stderr, "  %11s: %s\n", Kind, It.second.getNameAsCStr());
276
  }
277
}
278

279
bool DeviceTy::useAutoZeroCopy() {
280
  if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY)
281
    return false;
282
  return RTL->use_auto_zero_copy(RTLDeviceID);
283
}
284