llvm-project

1
//===-- OpenMP/Mapping.cpp - OpenMP/OpenACC pointer mapping impl. ---------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
//===----------------------------------------------------------------------===//
10

11
#include "OpenMP/Mapping.h"
12

13
#include "PluginManager.h"
14
#include "Shared/Debug.h"
15
#include "Shared/Requirements.h"
16
#include "device.h"
17

18
/// Dump a table of all the host-target pointer pairs on failure
19
void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device,
20
                               bool toStdOut) {
21
  MappingInfoTy::HDTTMapAccessorTy HDTTMap =
22
      Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
23
  if (HDTTMap->empty()) {
24
    DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
25
              "OpenMP Host-Device pointer mappings table empty\n");
26
    return;
27
  }
28

29
  SourceInfo Kernel(Loc);
30
  DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
31
            "OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
32
            Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
33
  DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
34
            "%-18s %-18s %s %s %s %s\n", "Host Ptr", "Target Ptr", "Size (B)",
35
            "DynRefCount", "HoldRefCount", "Declaration");
36
  for (const auto &It : *HDTTMap) {
37
    HostDataToTargetTy &HDTT = *It.HDTT;
38
    SourceInfo Info(HDTT.HstPtrName);
39
    DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
40
              DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
41
              DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
42
              HDTT.HstPtrEnd - HDTT.HstPtrBegin,
43
              HDTT.dynRefCountToStr().c_str(), HDTT.holdRefCountToStr().c_str(),
44
              Info.getName(), Info.getFilename(), Info.getLine(),
45
              Info.getColumn());
46
  }
47
}
48

49
int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
50
                                int64_t Size) {
51
  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
52

53
  // Check if entry exists
54
  auto It = HDTTMap->find(HstPtrBegin);
55
  if (It != HDTTMap->end()) {
56
    HostDataToTargetTy &HDTT = *It->HDTT;
57
    std::lock_guard<HostDataToTargetTy> LG(HDTT);
58
    // Mapping already exists
59
    bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
60
                   HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
61
    if (IsValid) {
62
      DP("Attempt to re-associate the same device ptr+offset with the same "
63
         "host ptr, nothing to do\n");
64
      return OFFLOAD_SUCCESS;
65
    }
66
    REPORT("Not allowed to re-associate a different device ptr+offset with "
67
           "the same host ptr\n");
68
    return OFFLOAD_FAIL;
69
  }
70

71
  // Mapping does not exist, allocate it with refCount=INF
72
  const HostDataToTargetTy &NewEntry =
73
      *HDTTMap
74
           ->emplace(new HostDataToTargetTy(
75
               /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
76
               /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
77
               /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
78
               /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
79
               /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
80
               /*UseHoldRefCount=*/false, /*Name=*/nullptr,
81
               /*IsRefCountINF=*/true))
82
           .first->HDTT;
83
  DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
84
     ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
85
     "HoldRefCount=%s\n",
86
     DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
87
     DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
88
     NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
89
  (void)NewEntry;
90

91
  // Notify the plugin about the new mapping.
92
  return Device.notifyDataMapped(HstPtrBegin, Size);
93
}
94

95
int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
96
  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
97

98
  auto It = HDTTMap->find(HstPtrBegin);
99
  if (It == HDTTMap->end()) {
100
    REPORT("Association not found\n");
101
    return OFFLOAD_FAIL;
102
  }
103
  // Mapping exists
104
  HostDataToTargetTy &HDTT = *It->HDTT;
105
  std::lock_guard<HostDataToTargetTy> LG(HDTT);
106

107
  if (HDTT.getHoldRefCount()) {
108
    // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
109
    // "It is an error to call acc_unmap_data if the structured reference
110
    // count for the pointer is not zero."
111
    REPORT("Trying to disassociate a pointer with a non-zero hold reference "
112
           "count\n");
113
    return OFFLOAD_FAIL;
114
  }
115

116
  if (HDTT.isDynRefCountInf()) {
117
    DP("Association found, removing it\n");
118
    void *Event = HDTT.getEvent();
119
    delete &HDTT;
120
    if (Event)
121
      Device.destroyEvent(Event);
122
    HDTTMap->erase(It);
123
    return Device.notifyDataUnmapped(HstPtrBegin);
124
  }
125

126
  REPORT("Trying to disassociate a pointer which was not mapped via "
127
         "omp_target_associate_ptr\n");
128
  return OFFLOAD_FAIL;
129
}
130

131
LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
132
                                          void *HstPtrBegin, int64_t Size,
133
                                          HostDataToTargetTy *OwnedTPR) {
134

135
  uintptr_t HP = (uintptr_t)HstPtrBegin;
136
  LookupResult LR;
137

138
  DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
139
     DPxPTR(HP), Size);
140

141
  if (HDTTMap->empty())
142
    return LR;
143

144
  auto Upper = HDTTMap->upper_bound(HP);
145

146
  if (Size == 0) {
147
    // specification v5.1 Pointer Initialization for Device Data Environments
148
    // upper_bound satisfies
149
    //   std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
150
    if (Upper != HDTTMap->begin()) {
151
      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
152
      // the left side of extended address range is satisified.
153
      // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
154
      // LR.TPR.getEntry()->HstPtrBase
155
      LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
156
                             HP < LR.TPR.getEntry()->HstPtrBase;
157
    }
158

159
    if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
160
      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
161
      // the right side of extended address range is satisified.
162
      // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
163
      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
164
    }
165
  } else {
166
    // check the left bin
167
    if (Upper != HDTTMap->begin()) {
168
      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
169
      // Is it contained?
170
      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
171
                             HP < LR.TPR.getEntry()->HstPtrEnd &&
172
                             (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
173
      // Does it extend beyond the mapped region?
174
      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
175
                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
176
    }
177

178
    // check the right bin
179
    if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
180
        Upper != HDTTMap->end()) {
181
      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
182
      // Does it extend into an already mapped region?
183
      LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
184
                               (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
185
      // Does it extend beyond the mapped region?
186
      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
187
                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
188
    }
189

190
    if (LR.Flags.ExtendsBefore) {
191
      DP("WARNING: Pointer is not mapped but section extends into already "
192
         "mapped data\n");
193
    }
194
    if (LR.Flags.ExtendsAfter) {
195
      DP("WARNING: Pointer is already mapped but section extends beyond mapped "
196
         "region\n");
197
    }
198
  }
199

200
  return LR;
201
}
202

203
TargetPointerResultTy MappingInfoTy::getTargetPointer(
204
    HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
205
    int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
206
    bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
207
    bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
208
    AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
209

210
  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
211
  LR.TPR.Flags.IsPresent = true;
212

213
  // Release the mapping table lock only after the entry is locked by
214
  // attaching it to TPR. Once TPR is destroyed it will release the lock
215
  // on entry. If it is returned the lock will move to the returned object.
216
  // If LR.Entry is already owned/locked we avoid trying to lock it again.
217

218
  // Check if the pointer is contained.
219
  // If a variable is mapped to the device manually by the user - which would
220
  // lead to the IsContained flag to be true - then we must ensure that the
221
  // device address is returned even under unified memory conditions.
222
  if (LR.Flags.IsContained ||
223
      ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
224
    const char *RefCountAction;
225
    if (UpdateRefCount) {
226
      // After this, reference count >= 1. If the reference count was 0 but the
227
      // entry was still there we can reuse the data on the device and avoid a
228
      // new submission.
229
      LR.TPR.getEntry()->incRefCount(HasHoldModifier);
230
      RefCountAction = " (incremented)";
231
    } else {
232
      // It might have been allocated with the parent, but it's still new.
233
      LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
234
      RefCountAction = " (update suppressed)";
235
    }
236
    const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
237
    const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
238
    uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
239
                    ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
240
    INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
241
         "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
242
         ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
243
         (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
244
         Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
245
         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
246
         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
247
    LR.TPR.TargetPointer = (void *)Ptr;
248
  } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
249
    // Explicit extension of mapped data - not allowed.
250
    MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
251
            " (%" PRId64
252
            " bytes), but device allocation maps to host at " DPxMOD
253
            " (%" PRId64 " bytes)",
254
            DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
255
            LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
256
    if (HasPresentModifier)
257
      MESSAGE("device mapping required by 'present' map type modifier does not "
258
              "exist for host address " DPxMOD " (%" PRId64 " bytes)",
259
              DPxPTR(HstPtrBegin), Size);
260
  } else if ((PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
261
              !HasCloseModifier) ||
262
             (PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY)) {
263

264
    // If unified shared memory is active, implicitly mapped variables that are
265
    // not privatized use host address. Any explicitly mapped variables also use
266
    // host address where correctness is not impeded. In all other cases maps
267
    // are respected.
268
    // In addition to the mapping rules above, the close map modifier forces the
269
    // mapping of the variable to the device.
270
    if (Size) {
271
      INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
272
           "Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
273
           "memory\n",
274
           DPxPTR((uintptr_t)HstPtrBegin), Size);
275
      DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
276
         "memory\n",
277
         DPxPTR((uintptr_t)HstPtrBegin), Size);
278
      LR.TPR.Flags.IsPresent = false;
279
      LR.TPR.Flags.IsHostPointer = true;
280
      LR.TPR.TargetPointer = HstPtrBegin;
281
    }
282
  } else if (HasPresentModifier) {
283
    DP("Mapping required by 'present' map type modifier does not exist for "
284
       "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
285
       DPxPTR(HstPtrBegin), Size);
286
    MESSAGE("device mapping required by 'present' map type modifier does not "
287
            "exist for host address " DPxMOD " (%" PRId64 " bytes)",
288
            DPxPTR(HstPtrBegin), Size);
289
  } else if (Size) {
290
    // If it is not contained and Size > 0, we should create a new entry for it.
291
    LR.TPR.Flags.IsNewEntry = true;
292
    uintptr_t TgtAllocBegin =
293
        (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
294
    uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
295
    // Release the mapping table lock only after the entry is locked by
296
    // attaching it to TPR.
297
    LR.TPR.setEntry(HDTTMap
298
                        ->emplace(new HostDataToTargetTy(
299
                            (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
300
                            (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
301
                            TgtPtrBegin, HasHoldModifier, HstPtrName))
302
                        .first->HDTT);
303
    INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
304
         "Creating new map entry with HstPtrBase=" DPxMOD
305
         ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
306
         ", TgtPtrBegin=" DPxMOD
307
         ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
308
         DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
309
         DPxPTR(TgtPtrBegin), Size,
310
         LR.TPR.getEntry()->dynRefCountToStr().c_str(),
311
         LR.TPR.getEntry()->holdRefCountToStr().c_str(),
312
         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
313
    LR.TPR.TargetPointer = (void *)TgtPtrBegin;
314

315
    // Notify the plugin about the new mapping.
316
    if (Device.notifyDataMapped(HstPtrBegin, Size))
317
      return TargetPointerResultTy{};
318
  } else {
319
    // This entry is not present and we did not create a new entry for it.
320
    LR.TPR.Flags.IsPresent = false;
321
  }
322

323
  // All mapping table modifications have been made. If the user requested it we
324
  // give up the lock.
325
  if (ReleaseHDTTMap)
326
    HDTTMap.destroy();
327

328
  // If the target pointer is valid, and we need to transfer data, issue the
329
  // data transfer.
330
  if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
331
      (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
332
    DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
333
       DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
334

335
    int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
336
                                AsyncInfo, LR.TPR.getEntry());
337
    if (Ret != OFFLOAD_SUCCESS) {
338
      REPORT("Copying data to device failed.\n");
339
      // We will also return nullptr if the data movement fails because that
340
      // pointer points to a corrupted memory region so it doesn't make any
341
      // sense to continue to use it.
342
      LR.TPR.TargetPointer = nullptr;
343
    } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
344
               OFFLOAD_SUCCESS)
345
      return TargetPointerResultTy{};
346
  } else {
347
    // If not a host pointer and no present modifier, we need to wait for the
348
    // event if it exists.
349
    // Note: Entry might be nullptr because of zero length array section.
350
    if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
351
        !HasPresentModifier) {
352
      void *Event = LR.TPR.getEntry()->getEvent();
353
      if (Event) {
354
        int Ret = Device.waitEvent(Event, AsyncInfo);
355
        if (Ret != OFFLOAD_SUCCESS) {
356
          // If it fails to wait for the event, we need to return nullptr in
357
          // case of any data race.
358
          REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
359
          return TargetPointerResultTy{};
360
        }
361
      }
362
    }
363
  }
364

365
  return std::move(LR.TPR);
366
}
367

368
TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
369
    void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
370
    bool MustContain, bool ForceDelete, bool FromDataEnd) {
371
  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
372

373
  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
374

375
  LR.TPR.Flags.IsPresent = true;
376

377
  if (LR.Flags.IsContained ||
378
      (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
379
    LR.TPR.Flags.IsLast =
380
        LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
381

382
    if (ForceDelete) {
383
      LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
384
      assert(LR.TPR.Flags.IsLast ==
385
                 LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
386
             "expected correct IsLast prediction for reset");
387
    }
388

389
    // Increment the number of threads that is using the entry on a
390
    // targetDataEnd, tracking the number of possible "deleters". A thread may
391
    // come to own the entry deletion even if it was not the last one querying
392
    // for it. Thus, we must track every query on targetDataEnds to ensure only
393
    // the last thread that holds a reference to an entry actually deletes it.
394
    if (FromDataEnd)
395
      LR.TPR.getEntry()->incDataEndThreadCount();
396

397
    const char *RefCountAction;
398
    if (!UpdateRefCount) {
399
      RefCountAction = " (update suppressed)";
400
    } else if (LR.TPR.Flags.IsLast) {
401
      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
402
      assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
403
             "Expected zero reference count when deletion is scheduled");
404
      if (ForceDelete)
405
        RefCountAction = " (reset, delayed deletion)";
406
      else
407
        RefCountAction = " (decremented, delayed deletion)";
408
    } else {
409
      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
410
      RefCountAction = " (decremented)";
411
    }
412
    const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
413
    const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
414
    uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
415
                   ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
416
    INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
417
         "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
418
         "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
419
         DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
420
         LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
421
         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
422
    LR.TPR.TargetPointer = (void *)TP;
423
  } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY ||
424
             PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY) {
425
    // If the value isn't found in the mapping and unified shared memory
426
    // is on then it means we have stumbled upon a value which we need to
427
    // use directly from the host.
428
    DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
429
       "memory\n",
430
       DPxPTR((uintptr_t)HstPtrBegin), Size);
431
    LR.TPR.Flags.IsPresent = false;
432
    LR.TPR.Flags.IsHostPointer = true;
433
    LR.TPR.TargetPointer = HstPtrBegin;
434
  } else {
435
    // OpenMP Specification v5.2: if a matching list item is not found, the
436
    // pointer retains its original value as per firstprivate semantics.
437
    LR.TPR.Flags.IsPresent = false;
438
    LR.TPR.Flags.IsHostPointer = false;
439
    LR.TPR.TargetPointer = HstPtrBegin;
440
  }
441

442
  return std::move(LR.TPR);
443
}
444

445
// Return the target pointer begin (where the data will be moved).
446
void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
447
                                    void *HstPtrBegin, int64_t Size) {
448
  uintptr_t HP = (uintptr_t)HstPtrBegin;
449
  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
450
  if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
451
    uintptr_t TP =
452
        LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
453
    return (void *)TP;
454
  }
455

456
  return NULL;
457
}
458

459
int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
460
                                 HostDataToTargetTy *Entry, int64_t Size) {
461
  assert(Entry && "Trying to delete a null entry from the HDTT map.");
462
  assert(Entry->getTotalRefCount() == 0 &&
463
         Entry->getDataEndThreadCount() == 0 &&
464
         "Trying to delete entry that is in use or owned by another thread.");
465

466
  INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
467
       "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
468
       ", Size=%" PRId64 ", Name=%s\n",
469
       DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
470
       (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
471
                           : "unknown");
472

473
  if (HDTTMap->erase(Entry) == 0) {
474
    REPORT("Trying to remove a non-existent map entry\n");
475
    return OFFLOAD_FAIL;
476
  }
477

478
  return OFFLOAD_SUCCESS;
479
}
480

481
int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
482
                                         int64_t Size) {
483
  assert(Entry && "Trying to deallocate a null entry.");
484

485
  DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
486
     "starting at " DPxMOD "\n",
487
     DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
488

489
  void *Event = Entry->getEvent();
490
  if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
491
    REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
492
    return OFFLOAD_FAIL;
493
  }
494

495
  int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
496

497
  // Notify the plugin about the unmapped memory.
498
  Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
499

500
  delete Entry;
501

502
  return Ret;
503
}
504

505
static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
506
                              void *DstPtrBegin, int64_t Size,
507
                              HostDataToTargetTy *HT) {
508

509
  INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
510
       "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
511
       ", Size=%" PRId64 ", Name=%s\n",
512
       H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
513
       DPxPTR(H2D ? SrcPtrBegin : DstPtrBegin), H2D ? "Tgt" : "Hst",
514
       DPxPTR(H2D ? DstPtrBegin : SrcPtrBegin), Size,
515
       (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
516
                              : "unknown");
517
}
518

519
void MappingInfoTy::printCopyInfo(
520
    void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
521
    HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
522
  auto HDTTMap =
523
      HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
524
  LookupResult LR;
525
  if (!Entry) {
526
    LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
527
    Entry = LR.TPR.getEntry();
528
  }
529
  printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
530
                    Entry);
531
}
532