framework2

1

2
#include "ofMediaFoundationSoundPlayer.h"
3
#include "ofLog.h"
4

5
#include <condition_variable>
6
#include <propvarutil.h>
7
#include <xaudio2.h>
8
#include <mmreg.h>
9

10
// standard speaker geometry configurations, used with X3DAudioInitialize
11
#if !defined(SPEAKER_MONO)
12
#define SPEAKER_MONO             SPEAKER_FRONT_CENTER
13
#define SPEAKER_STEREO           (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT)
14
#define SPEAKER_2POINT1          (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_LOW_FREQUENCY)
15
#define SPEAKER_SURROUND         (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_BACK_CENTER)
16
#define SPEAKER_QUAD             (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT)
17
#define SPEAKER_4POINT1          (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT)
18
#define SPEAKER_5POINT1          (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT)
19
#define SPEAKER_7POINT1          (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT | SPEAKER_FRONT_LEFT_OF_CENTER | SPEAKER_FRONT_RIGHT_OF_CENTER)
20
#define SPEAKER_5POINT1_SURROUND (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_SIDE_LEFT  | SPEAKER_SIDE_RIGHT)
21
#define SPEAKER_7POINT1_SURROUND (SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT | SPEAKER_SIDE_LEFT  | SPEAKER_SIDE_RIGHT)
22
#endif
23

24
using namespace Microsoft::WRL;
25

26
int ofMediaFoundationSoundPlayer::sNumInstances = 0;
27

28
ComPtr<IXAudio2> ofMediaFoundationSoundPlayer::sXAudio2 = nullptr;
29
std::shared_ptr<IXAudio2MasteringVoice> ofMediaFoundationSoundPlayer::sXAudioMasteringVoice;
30

31

32
int ofMediaFoundationUtils::sNumMFInstances = 0;
33
// media foundation utils
34
//----------------------------------------------
35
bool ofMediaFoundationUtils::InitMediaFoundation() {
36
    if (sNumMFInstances == 0) {
37
        HRESULT hr = MFStartup(MF_VERSION);
38
        // TODO: This is called by ofMediaFoundationPlayer also
39
        // uses CoInitializeEx
40
        // using the coinit in case ofDirectShowPlayer is also being used
41
        hr = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
42
        //hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE);
43
        //HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED | COINIT_DISABLE_OLE1DDE);
44
        ofLogVerbose("ofMediaFoundationUtils :: CoInitializeEx : init ok ") << SUCCEEDED(hr);
45
    }
46
    sNumMFInstances++;
47
    return sNumMFInstances > 0;
48
}
49

50
//----------------------------------------------
51
bool ofMediaFoundationUtils::CloseMediaFoundation() {
52
    sNumMFInstances--;
53
    HRESULT hr = S_OK;
54
    if (sNumMFInstances <= 0) {
55
        CoUninitialize();
56
        ofLogVerbose("ofMediaFoundationUtils") << " calling MFShutdown.";
57
        // Shut down Media Foundation.
58
        hr = MFShutdown();
59
    }
60
    if (sNumMFInstances < 0) {
61
        sNumMFInstances = 0;
62
    }
63
    return (hr == S_OK);
64
}
65

66
//----------------------------------------------
67
int ofMediaFoundationUtils::GetNumInstances() {
68
    return sNumMFInstances;
69
}
70

71
//----------------------------------------------
72
void ofMediaFoundationUtils::CallAsyncBlocking(std::function<void()> aCallBack) {
73
    std::mutex lock;
74
    std::condition_variable wait;
75
    std::atomic_bool isDone(false);
76

77
    HRESULT hr = S_OK;
78

79
    ComPtr<AsyncCallback> pCB(
80
        new AsyncCallback(
81
            [&] {
82
        aCallBack();
83
        isDone.store(true);
84
        wait.notify_one();
85
    }
86
    ));
87

88
    hr = MFPutWorkItem(MFASYNC_CALLBACK_QUEUE_MULTITHREADED, pCB.Get(), NULL);
89
    if (hr == S_OK) {
90
        std::unique_lock lk{ lock };
91
        wait.wait(lk, [&] { return isDone.load(); });
92
    } else {
93
        aCallBack();
94
        if (pCB) {
95
            pCB->Release();
96
            pCB = nullptr;
97
        }
98
    }
99
}
100

101
//----------------------------------------------
102
void ofMediaFoundationSoundPlayer::SetMasterVolume(float apct) {
103
    sInitAudioSystems();
104
    if (sXAudioMasteringVoice) {
105
        sXAudioMasteringVoice->SetVolume(std::clamp(apct, 0.f, 1.0f));
106
    }
107
}
108

109

110
//----------------------------------------------
111
bool ofMediaFoundationSoundPlayer::sInitXAudio2() {
112

113
    if (sXAudio2 == nullptr) {
114
        UINT32 flags = 0;
115
#if defined(TARGET_WINVS)
116
        HRESULT hr = XAudio2Create(sXAudio2.GetAddressOf(), 0U);
117
#else
118
        HRESULT hr = XAudio2Create(sXAudio2.GetAddressOf(), 0U, XAUDIO2_DEFAULT_PROCESSOR );
119
#endif
120
        if (sXAudio2) {
121
            sXAudio2->StartEngine();
122
        }
123
        if (hr == S_OK) {
124
            IXAudio2MasteringVoice* pMVoice = sXAudioMasteringVoice.get();
125
            hr = sXAudio2->CreateMasteringVoice(&pMVoice);
126
            if (hr != S_OK) {
127
                ofLogError("ofMediaFoundationSoundPlayer :: sInitXAudio2") << " error creating master voice.";
128
                sCloseXAudio2();
129
                return false;
130
            }
131
        }
132
    }
133
    return true;
134
}
135

136
//----------------------------------------------
137
bool ofMediaFoundationSoundPlayer::sCloseXAudio2() {
138

139
    if (sXAudioMasteringVoice) {
140
        sXAudioMasteringVoice->DestroyVoice();
141
        sXAudioMasteringVoice.reset();
142
    }
143

144
    if (sXAudio2 != nullptr) {
145
       sXAudio2->StopEngine();
146
       sXAudio2.Reset();
147
    }
148

149
    sXAudio2 = nullptr;
150
    return true;
151
}
152

153

154

155
//----------------------------------------------
156
bool ofMediaFoundationSoundPlayer::sInitAudioSystems() {
157
    ofMediaFoundationUtils::InitMediaFoundation();
158
    if (sNumInstances == 0) {
159
        sInitXAudio2();
160
    }
161
    sNumInstances++;
162
    return sNumInstances > 0;
163
}
164

165
//----------------------------------------------
166
void ofMediaFoundationSoundPlayer::sCloseAudioSystems() {
167
    sNumInstances--;
168
    if (sNumInstances <= 0) {
169
        ofLogVerbose("ofMediaFoundationSoundPlayer") << " closing XAudio2.";
170
        sCloseXAudio2(); 
171
    }
172
    ofMediaFoundationUtils::CloseMediaFoundation();
173
    if (sNumInstances < 0) {
174
        sNumInstances = 0;
175
    }
176
}
177

178
ofMediaFoundationSoundPlayer::ofMediaFoundationSoundPlayer() {
179
    InitializeCriticalSectionEx(&m_critSec, 0, 0);
180
    sInitAudioSystems();
181
}
182
ofMediaFoundationSoundPlayer::~ofMediaFoundationSoundPlayer() {
183
    unload();
184
    sCloseAudioSystems();
185
    DeleteCriticalSection(&m_critSec);
186
}
187

188
//--------------------
189
bool ofMediaFoundationSoundPlayer::load(const of::filesystem::path& fileName, bool stream) {
190
    unload();
191
    
192
    std::string fileStr = fileName.string();
193
    bool bStream = false;
194
    bStream = bStream || ofIsStringInString(fileStr, "http://");
195
    bStream = bStream || ofIsStringInString(fileStr, "https://");
196
    bStream = bStream || ofIsStringInString(fileStr, "rtsp://");
197
    bStream = bStream || ofIsStringInString(fileStr, "rtmp://");
198

199
    of::filesystem::path absPath{ fileStr };
200

201
    if (!bStream) {
202
        if (ofFile::doesFileExist(absPath)) {
203
            absPath = ofFilePath::getAbsolutePath(absPath, true);
204
        } else {
205
            ofLogError("ofMediaFoundationSoundPlayer") << " file does not exist! " << absPath;
206
            return false;
207
        }
208
    }
209

210
    mBStreaming = (bStream || stream);
211

212
    ComPtr<IMFAttributes> attributes;
213
    HRESULT hr = MFCreateAttributes(&attributes, mBStreaming ? 2 : 1);
214
    //hr = attributes->SetUINT32(MF_LOW_LATENCY, TRUE);
215
    if (mBStreaming) {
216
        #if defined MF_LOW_LATENCY
217
        hr = attributes->SetUINT32(MF_LOW_LATENCY, TRUE);
218
        #endif
219
        mSrcReaderCallback = std::make_shared< SourceReaderCallback>();
220
        mSrcReaderCallback->setCB(this);
221
        hr = attributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, mSrcReaderCallback.get());
222
    }
223

224

225
    LPCWSTR path = absPath.c_str();
226
    
227

228
    hr = MFCreateSourceReaderFromURL(
229
        path,
230
        attributes.Get(),
231
        mSrcReader.GetAddressOf());
232

233
    if (hr != S_OK) {
234
        ofLogError("ofMediaFoundationSoundPlayer::load") << " unable to load from: " << absPath;
235
        unload();
236
        return false;
237
    }
238

239
    ofLogVerbose("ofMediaFoundationSoundPlayer::load") << " created the source reader " << absPath;
240
    // Select only the audio stream
241
    hr = mSrcReader->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, false);
242
    if (hr == S_OK) {
243
        hr = mSrcReader->SetStreamSelection(MF_SOURCE_READER_FIRST_AUDIO_STREAM, true);
244
    }
245

246
    if (hr != S_OK) {
247
        unload();
248
        return false;
249
    }
250

251

252
    IMFMediaType* nativeType;
253
    WAVEFORMATEX* nativeFormat;
254
    UINT32 formatSize;
255
    hr = mSrcReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_AUDIO_STREAM, 0, &nativeType);
256

257
    auto nativeTypePtr = std::unique_ptr<IMFMediaType, MyComDeleterFunctor>(nativeType);
258
    // get a wave format 
259
    hr = MFCreateWaveFormatExFromMFMediaType(nativeType, &nativeFormat, &formatSize);
260
    
261
    mNumChannels = nativeFormat->nChannels;
262
    mSampleRate = nativeFormat->nSamplesPerSec;
263

264
    CoTaskMemFree(nativeFormat);
265
    
266

267
    ComPtr<IMFMediaType> mediaType;
268
    hr = MFCreateMediaType(mediaType.GetAddressOf());
269
    if (hr != S_OK) {
270
        return false;
271
    }
272

273
    hr = mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio);
274
    if (hr != S_OK) {
275
        return false;
276
    }
277

278
    hr = mediaType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM);
279
    if (FAILED(hr)) {
280
        return false;
281
    }
282

283
    hr = mSrcReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_AUDIO_STREAM, 0, mediaType.Get() );
284
    if (hr != S_OK) {
285
        return false;
286
    }
287

288
    ComPtr<IMFMediaType> outputMediaType;
289
    hr = mSrcReader->GetCurrentMediaType(MF_SOURCE_READER_FIRST_AUDIO_STREAM, outputMediaType.GetAddressOf());
290
    if (hr != S_OK) {
291
        return false;
292
    }
293

294

295
    UINT32 waveFormatSize = 0;
296
    WAVEFORMATEX* waveFormat = nullptr;
297
    hr = MFCreateWaveFormatExFromMFMediaType(outputMediaType.Get(), &waveFormat, &waveFormatSize);
298
    if (hr != S_OK) {
299
        return false;
300
    }
301

302
    memcpy_s(&mWaveFormatEx, sizeof(mWaveFormatEx), waveFormat, waveFormatSize);
303
    ofLogVerbose("ofMediaFoundationSoundPlayer::load") << "waveFormatEx num channels: " << mWaveFormatEx.nChannels << std::endl;
304

305
    if (!mBStreaming) {
306
        _readToBuffer(mSrcReader.Get());
307
    }
308

309
    CoTaskMemFree(waveFormat);
310

311

312

313
    if (hr != S_OK) {
314
        unload();
315
        return false;
316
    }
317

318
    // get seconds:
319
    if (mBStreaming) {
320
        PROPVARIANT durationProp;
321
        hr = mSrcReader->GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &durationProp);
322
        if (hr == S_OK) {
323
            mDurationSeconds = (double)durationProp.uhVal.QuadPart / 10000000.0;
324
            mDurationMS = (double)durationProp.uhVal.QuadPart / 10000.0;
325
            ofLogVerbose("ofMediaFoundationSoundPlayer::load") << "Reader duration seconds: " << (double)durationProp.uhVal.QuadPart / 10000000.0 << " millis: " << mDurationMS << std::endl;
326
        }
327
    }
328

329
    PROPVARIANT seekProp;
330
    hr = mSrcReader->GetPresentationAttribute(MF_SOURCE_READER_MEDIASOURCE, MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS, &seekProp);
331
    if (hr == S_OK) {
332
        ULONG flags = seekProp.ulVal;
333
        mBCanSeek = ((flags & MFMEDIASOURCE_CAN_SEEK) == MFMEDIASOURCE_CAN_SEEK);
334
        ofLogVerbose("ofMediaFoundationSoundPlayer::load") << "mBCanSeek: " << mBCanSeek << std::endl;
335
    }
336

337
    ofLogVerbose("ofMediaFoundationSoundPlayer::load") << "made it all the way to the end.";
338
    
339
    if (!mBStreaming) {
340
        mSrcReader.Reset();
341
        mSrcReader = nullptr;
342
    }
343

344
    {
345

346
        // create stream context for listening to voice 
347
        mVoiceContext = std::make_shared<StreamingVoiceContext>();
348
        // Create the source voice
349
        IXAudio2SourceVoice* pSourceVoice = nullptr;
350
        // setting max freq ratio to 3, though it may need to be higher to play at a 
351
        // faster pitch
352
        if (mBStreaming) {
353
            hr = sXAudio2->CreateSourceVoice(
354
                &pSourceVoice,
355
                &mWaveFormatEx,
356
                0U, 3.0f,
357
                mVoiceContext.get()
358
            );
359
        } else {
360
            hr = sXAudio2->CreateSourceVoice(&pSourceVoice, &mWaveFormatEx);
361
        }
362
        if (hr != S_OK) {
363
            ofLogError("ofMediaFoundationSoundPlayer :: load") << "error creating voice. hr: " << hr;
364
            unload();
365
            return false;
366
        }
367

368
        mVoice.reset(pSourceVoice);
369
    }
370

371
    if (hr != S_OK) {
372
        unload();
373
        return false;
374
    }
375

376
    mBLoaded = true;
377

378
    return mBLoaded;
379
};
380

381
//--------------------
382
void ofMediaFoundationSoundPlayer::unload() {
383
    removeUpdateListener();
384

385
    EnterCriticalSection(&m_critSec);
386

387
    if (mSrcReader) {
388
        {
389
            std::unique_lock<std::mutex> lk(mSrcReaderMutex);
390
            mBRequestNewReaderSample = false;
391
        }
392

393
        ofMediaFoundationUtils::CallAsyncBlocking(
394
            [&] {
395
                //mSrcReader->Flush(MF_SOURCE_READER_FIRST_AUDIO_STREAM);
396
                mSrcReader.Reset();
397
            }
398
        );
399
    }
400
    mSrcReader = nullptr;
401

402
    if (mSrcReaderCallback) {
403
        ofMediaFoundationUtils::CallAsyncBlocking(
404
            [&] { mSrcReaderCallback.reset(); }
405
        );
406
    }
407

408
    if (mVoice) {
409
        mVoice.reset();
410
    }
411

412
    if (mVoiceContext) {
413
        mVoiceContext.reset();
414
    }
415

416

417
    mBStreaming = false;
418

419
    _clearExtraVoices();
420

421
    mStreamBuffers.clear();
422
    mBuffer.clear();
423
	mPosPct = 0.f;
424
	mBIsPlaying = false;
425
	mBLoaded = false;
426
    mBCanSeek = false;
427
    mDurationSeconds = 0.f;
428
    mDurationMS = 0;
429

430
    mTotalNumFrames = 0;
431
    mNumSamplesAlreadyPlayed = 0;
432
    mBRequestNewReaderSample = false;
433
    
434
    LeaveCriticalSection(&m_critSec);
435
};
436

437
//--------------------
438
void ofMediaFoundationSoundPlayer::update(ofEventArgs& args) {
439
    if (mVoice) {
440
        if (mBStreaming) {
441
            if (isPlaying()) {
442
                XAUDIO2_VOICE_STATE xstate;
443
                mVoice->GetState(&xstate);
444
                mNumSamplesAlreadyPlayed += xstate.SamplesPlayed - mNumSamplesStored;
445
                double seconds = (double)mNumSamplesAlreadyPlayed / (double)mSampleRate;
446
                mPosPct = seconds / mDurationSeconds;
447
                mNumSamplesStored = xstate.SamplesPlayed;
448
                //std::cout << "SamplesPlayed: " << SamplesPlayed << " - " << SamplesPlayed / mSampleRate << " -- " << seconds << " / " << mDurationSeconds << " pos: " << mPosPct << std::endl;
449

450
                {
451
                    bool bRequestStop = false;
452
                    {
453
                        std::unique_lock<std::mutex> lk(mSrcReaderMutex);
454
                        if (mBEndOfStream) {
455
                            if (!xstate.BuffersQueued) {
456
                                if (mBLoop) {
457
                                    mBEndOfStream = false;
458
                                    if (mVoice) {
459
                                        mNumSamplesAlreadyPlayed = 0;
460
                                        // Restart loop
461
                                        PROPVARIANT var = {};
462
                                        var.vt = VT_I8;
463
                                        HRESULT hr = mSrcReader->SetCurrentPosition(GUID_NULL, var);
464
                                        hr = PropVariantClear(&var);
465
                                    }
466
                                    mBRequestNewReaderSample = true;
467
                                } else {
468
                                    bRequestStop = true;
469
                                    // we need to request stop outside of the scope of the lock
470
                                    // since stop() also locks to set vars 
471
                                }
472
                            }
473
                        }
474
                    }
475
                    if (bRequestStop) {
476
                        stop();
477
                    }
478
                }
479

480
                if (mSrcReader) {
481
                    std::unique_lock<std::mutex> lk(mSrcReaderMutex);
482
                    if (mBRequestNewReaderSample) {
483
                        mBRequestNewReaderSample = false;
484
                        HRESULT hr = mSrcReader->ReadSample(MF_SOURCE_READER_FIRST_AUDIO_STREAM, 0, nullptr, nullptr, nullptr, nullptr);
485
                    }
486
                }
487
            }
488
        } else {
489
            if (isPlaying()) {
490
                XAUDIO2_VOICE_STATE xstate;
491
                mVoice->GetState(&xstate);
492
                mNumSamplesAlreadyPlayed += xstate.SamplesPlayed - mNumSamplesStored;
493
                double seconds = (double)mNumSamplesAlreadyPlayed / (double)mSampleRate;
494
                mPosPct = seconds / mDurationSeconds;
495
                mNumSamplesStored = xstate.SamplesPlayed;
496

497
                if (!xstate.BuffersQueued && mExtraVoices.size() < 1) {
498
                    // we have reached the end //
499
                    if (mBLoop) {
500
                        // set isPlaying to false, so that it will create a new instance 
501
                        // for mVoice and not an Extra Voice
502
                        mPosPct = 0.0f;
503
                        mBIsPlaying = false;
504
                        play();
505
                    } else {
506
                        stop();
507
                    }
508
                }
509
            }
510
        }
511
    }
512

513
    for (auto it = mExtraVoices.begin(); it != mExtraVoices.end(); ) {
514
        XAUDIO2_VOICE_STATE xstate;
515
        it->second->GetState(&xstate, XAUDIO2_VOICE_NOSAMPLESPLAYED);
516
        if (!xstate.BuffersQueued) {
517
            std::ignore = it->second->Stop(0);
518
            it->second->DestroyVoice();
519
            it = mExtraVoices.erase(it);
520
        } else {
521
            ++it;
522
        }
523
    }
524
}
525

526
//--------------------
527
void ofMediaFoundationSoundPlayer::play() {
528
    if (!mVoice) {
529
        ofLogError("ofMediaFoundationSoundPlayer :: play") << "error creating sXAudio2.";
530
        return;
531
    }
532

533
    // don't want a ton of loops going on here 
534
    if (mBLoop) {
535
        stop();
536
    }
537
    if (!mBMultiPlay) {
538
        stop();
539
    }
540

541
    if (mBStreaming) {
542
        // just in case, multiplay is not supported for streams 
543
        _clearExtraVoices();
544
    }
545

546
    if (mBMultiPlay && isPlaying()) {
547
        unsigned int voiceKey = 0;
548
        // get next available key //
549
        if (mExtraVoices.size() > 0) {
550
            bool bOkGotAGoodOne = false;
551
            int numAttempts = 0;
552
            while (!bOkGotAGoodOne && (numAttempts < 1024)) {
553
                bool bOneIsAlreadyKey = false;
554
                for (auto& it : mExtraVoices) {
555
                    if (it.first == voiceKey) {
556
                        bOneIsAlreadyKey = true;
557
                        break;
558
                    }
559
                }
560
                if (!bOneIsAlreadyKey) {
561
                    bOkGotAGoodOne = true;
562
                } else {
563
                    voiceKey++;
564
                    numAttempts++;
565
                }
566
            }
567

568
            if (!bOkGotAGoodOne) {
569
                ofLogError("ofMediaFoundationSoundPlayer::play") << " error finding key for multi play";
570
                return;
571
            }
572
        }
573

574
        IXAudio2SourceVoice* pSourceVoice;
575
        HRESULT hr = sXAudio2->CreateSourceVoice(&pSourceVoice, &mWaveFormatEx);
576
        if (hr != S_OK) {
577
            ofLogError("ofMediaFoundationSoundPlayer::play") << " error creating extra multi play sounds.";
578
            return;
579
        }
580

581
        // ok, now lets add some extra players //
582
        // Submit the wave sample data using an XAUDIO2_BUFFER structure
583
        XAUDIO2_BUFFER buffer = {};
584
        buffer.pAudioData = mBuffer.data();
585
        // tell the source voice not to expect any data after this buffer
586
        buffer.Flags = XAUDIO2_END_OF_STREAM;  
587
        buffer.AudioBytes = mBuffer.size();
588

589
        hr = pSourceVoice->SubmitSourceBuffer(&buffer);
590
        pSourceVoice->SetVolume(mVolume);
591
        pSourceVoice->SetFrequencyRatio(mSpeed);
592
        _setPan(pSourceVoice, mPan);
593
        mExtraVoices.emplace_back(std::make_pair(voiceKey, pSourceVoice));
594
        //mExtraVoices.push_back(uptr);
595
        hr = pSourceVoice->Start(0);
596
    } else {
597

598
        if (mBStreaming && mSrcReader) {
599
            mBRequestNewReaderSample = true;
600
        } else {
601
            XAUDIO2_BUFFER buffer = {};
602
            buffer.pAudioData = mBuffer.data();
603
            // tell the source voice not to expect any data after this buffer
604
            buffer.Flags = XAUDIO2_END_OF_STREAM;
605
            buffer.AudioBytes = mBuffer.size();
606

607
            mVoice->SubmitSourceBuffer(&buffer);
608
           
609
        }
610

611
        mVoice->SetVolume(mVolume);
612
        mVoice->SetFrequencyRatio(mSpeed);
613
        _setPan(mVoice.get(), mPan);
614

615
        mVoice->Start(0);
616
        mNumSamplesAlreadyPlayed = 0;
617
    }
618
    addUpdateListener();
619

620
    mBIsPlaying = true;
621
};
622

623
//--------------------
624
void ofMediaFoundationSoundPlayer::stop() {
625
    _clearExtraVoices();
626

627
    if (mBStreaming && mSrcReader) {
628
        ofMediaFoundationUtils::CallAsyncBlocking(
629
            [&] { mSrcReader->Flush(MF_SOURCE_READER_FIRST_AUDIO_STREAM); }
630
        );
631
    }
632

633
    if (mVoice) {
634
        mVoice->Stop();
635
        mVoice->FlushSourceBuffers();
636
    }
637
    {
638
        std::unique_lock<std::mutex> lk(mSrcReaderMutex);
639
        mBEndOfStream = false;
640
        mBRequestNewReaderSample = false;
641
    }
642
    mPosPct = 0.0f;
643
    mBIsPlaying = false;
644
    mNumSamplesAlreadyPlayed = 0;
645
};
646

647
//--------------------
648
void ofMediaFoundationSoundPlayer::setVolume(float vol) {
649
    mVolume = vol;
650
    if (mVoice) {
651
        mVoice->SetVolume(mVolume);
652
    }
653
    for (auto& it : mExtraVoices) {
654
        if (it.second) {
655
            it.second->SetVolume(mVolume);
656
        }
657
    }
658
};
659

660
// https://learn.microsoft.com/en-us/windows/win32/xaudio2/how-to--pan-a-sound
661
//--------------------
662
void ofMediaFoundationSoundPlayer::setPan(float apan) {
663
    if (!sXAudioMasteringVoice) {
664
        return;
665
    }
666

667
    apan = std::clamp(apan, -1.f, 1.f);
668

669
    if (mPan == apan) {
670
        return;
671
    }
672

673
    _setPan(mVoice.get(), apan);
674

675
    mPan = apan;
676
};
677

678
//--------------------
679
void ofMediaFoundationSoundPlayer::setSpeed(float spd) {
680
    if (mVoice) {
681
        mVoice->SetFrequencyRatio(spd);
682
    }
683
    mSpeed = spd;
684
};
685

686
//--------------------
687
void ofMediaFoundationSoundPlayer::setPaused(bool bP) {
688
    if (bP) {
689
        if (mVoice) {
690
            mVoice->Stop();
691
        }
692
        for (auto& it : mExtraVoices) {
693
            if (it.second) {
694
                it.second->Stop();
695
            }
696
        }
697
    } else {
698
        if (mVoice) {
699
            mVoice->Start();
700
        }
701

702
        for (auto& it : mExtraVoices) {
703
            if (it.second) {
704
                it.second->Start();
705
            }
706
        }
707
    }
708
    mBIsPlaying = !bP;
709
};
710

711
//--------------------
712
void ofMediaFoundationSoundPlayer::setLoop(bool bLp) {
713
    if (bLp) {
714
        // we don't want a lot of looping iterations 
715
        _clearExtraVoices();
716
    }
717
    mBLoop = bLp;
718
};
719

720
//--------------------
721
void ofMediaFoundationSoundPlayer::setMultiPlay(bool bMp) {
722
    if (mBStreaming) {
723
        ofLogWarning("ofMediaFoundationSoundPlayer::setMultiPlay") << "multiplay not supported for streams.";
724
        mBMultiPlay = false;
725
        return;
726
    }
727
    if (!mBMultiPlay) {
728
        _clearExtraVoices();
729
    }
730
    mBMultiPlay = bMp;
731
};
732

733
//--------------------
734
void ofMediaFoundationSoundPlayer::setPosition(float pct) {
735
    if (isPlaying()) {
736
        double dpct = static_cast<double>(pct);
737
        dpct = std::clamp(dpct, 0.0, 1.0);
738

739
        if (mBStreaming && !mBCanSeek) {
740
            ofLogWarning("ofMediaFoundationSoundPlayer::setPosition") << " unable to seek.";
741
            return;
742
        }
743
        
744
        // ok we need to kill buffers and resubmit a buffer
745
        if (mVoice) {
746
            std::ignore = mVoice->Stop();
747
            std::ignore = mVoice->FlushSourceBuffers();
748

749
            if (mBStreaming) {
750
                if (mSrcReader && mBCanSeek) {
751
                    std::unique_lock<std::mutex> lk(mSrcReaderMutex);
752
                    double tseconds = dpct * (double)mDurationSeconds;
753
                    PROPVARIANT seekVar;
754
                    HRESULT hr = InitPropVariantFromInt64((LONGLONG)(tseconds * 10000000.0), &seekVar);
755
                    if (hr == S_OK) {
756
                        hr = mSrcReader->SetCurrentPosition(GUID_NULL, seekVar);
757
                        hr = PropVariantClear(&seekVar);
758
                    }
759
                    uint32_t desiredSample = (dpct * mDurationSeconds) * (double)mSampleRate;
760
                    mNumSamplesAlreadyPlayed = desiredSample;
761

762
                    mBRequestNewReaderSample = true;
763
                }
764
                mVoice->Start();
765
            } else {
766
                XAUDIO2_BUFFER buffer = {};
767
                buffer.pAudioData = mBuffer.data();
768
                // tell the source voice not to expect any data after this buffer
769
                buffer.Flags = XAUDIO2_END_OF_STREAM;
770
                buffer.AudioBytes = mBuffer.size();
771
                uint32_t desiredSample = dpct * (double)mTotalNumFrames;
772
                mNumSamplesAlreadyPlayed = desiredSample;
773
                // First sample in this buffer to be played.
774
                buffer.PlayBegin = desiredSample;
775
                mVoice->SubmitSourceBuffer(&buffer);
776
                mVoice->Start();
777
            }
778
        }
779
    }
780
};
781

782
//--------------------
783
void ofMediaFoundationSoundPlayer::setPositionMS(int ms) {
784
    setPosition((double)ms / (double)mDurationMS);
785
};
786

787
//--------------------
788
float ofMediaFoundationSoundPlayer::getPosition() const { 
789
	return mPosPct;
790
};
791

792
//--------------------
793
int ofMediaFoundationSoundPlayer::getPositionMS() const {
794
	return (double)getPosition() * (double)mDurationMS;
795
};
796

797
//--------------------
798
bool ofMediaFoundationSoundPlayer::isPlaying() const {
799
	return mBIsPlaying;
800
};
801

802
//--------------------
803
float ofMediaFoundationSoundPlayer::getSpeed() const {
804
	return mSpeed;
805
};
806

807
//--------------------
808
float ofMediaFoundationSoundPlayer::getPan() const {
809
	return mPan;
810
};
811

812
//--------------------
813
bool ofMediaFoundationSoundPlayer::isLoaded() const {
814
	return mBLoaded;
815
};
816

817
//--------------------
818
float ofMediaFoundationSoundPlayer::getVolume() const {
819
	return mVolume;
820
};
821

822
//--------------------
823
void ofMediaFoundationSoundPlayer::OnSourceReaderEvent(HRESULT hrStatus, DWORD dwStreamIndex,
824
    DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample* pSample) {
825
    HRESULT hr = S_OK;
826
    if (dwStreamFlags & MF_SOURCE_READERF_ENDOFSTREAM) {
827
        ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "End of stream";
828
        // set to the buffer and complete //
829
        std::unique_lock<std::mutex> lk(mSrcReaderMutex);
830
        mBEndOfStream = true;
831
        return;
832
    }
833
    if (dwStreamFlags & MF_SOURCE_READERF_NEWSTREAM) {
834
        ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "New stream";
835
    }
836
    if (dwStreamFlags & MF_SOURCE_READERF_NATIVEMEDIATYPECHANGED) {
837
        ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "Native type changed";
838
    }
839
    if (dwStreamFlags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED) {
840
        ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "Current type changed";
841
    }
842

843
    if (!pSample) {
844
        ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "Null audio sample.";
845
        std::unique_lock<std::mutex> lk(mSrcReaderMutex);
846
        mBRequestNewReaderSample = true;
847
    } else {
848
        EnterCriticalSection(&m_critSec);
849
        ComPtr<IMFMediaBuffer> mediaBuffer;
850
        hr = pSample->ConvertToContiguousBuffer(mediaBuffer.GetAddressOf());
851
        if (hr != S_OK) {
852
            ofLogError("ofMediaFoundationSoundPlayer::OnSourceReaderEvent : ") << "error converting to contiguous buffer";
853
            LeaveCriticalSection(&m_critSec);
854
            std::unique_lock<std::mutex> lk(mSrcReaderMutex);
855
            mBRequestNewReaderSample = true;
856
            return;
857
        }
858

859
        BYTE* audioData = nullptr;
860
        DWORD sampleBufferLength = 0;
861

862
        hr = mediaBuffer->Lock(&audioData, nullptr, &sampleBufferLength);
863
        if (hr != S_OK) {
864
            LeaveCriticalSection(&m_critSec);
865
            std::unique_lock<std::mutex> lk(mSrcReaderMutex);
866
            mBRequestNewReaderSample = true;
867
            return;
868
        }
869

870
        {
871
            if (mStreamBuffers.size() < MAX_BUFFER_COUNT) {
872
                mStreamBuffers.resize(MAX_BUFFER_COUNT);
873
            }
874
            if (mStreamBuffers[currentStreamBuffer].size() < sampleBufferLength) {
875
                mStreamBuffers[currentStreamBuffer].assign(sampleBufferLength,0);
876
            }
877

878
            //memcpy(buffers[currentStreamBuffer].data(), audioData, sampleBufferLength * sizeof(BYTE) );
879
            memcpy_s(mStreamBuffers[currentStreamBuffer].data(), sampleBufferLength, audioData, sampleBufferLength);
880
        }
881

882
        hr = mediaBuffer->Unlock();
883
        
884
        if (mVoice && mVoiceContext) {
885
            XAUDIO2_VOICE_STATE state;
886

887
            while(mVoice) {
888
                mVoice->GetState(&state);
889
                if (state.BuffersQueued < MAX_BUFFER_COUNT-1) {
890
                    break;
891
                }
892
                //WaitForSingleObject(mVoiceContext->hBufferEndEvent, INFINITE);
893
                WaitForSingleObject(mVoiceContext->hBufferEndEvent, 50);
894
            }
895

896
            if (mVoice) {
897
                XAUDIO2_BUFFER buf = {};
898
                buf.AudioBytes = sampleBufferLength;
899
                buf.pAudioData = mStreamBuffers[currentStreamBuffer].data();
900
                mVoice->SubmitSourceBuffer(&buf);
901

902
                currentStreamBuffer++;
903
                currentStreamBuffer %= MAX_BUFFER_COUNT;
904
            }
905
        }
906

907
        LeaveCriticalSection(&m_critSec);
908
        {
909
            std::unique_lock<std::mutex> lk(mSrcReaderMutex);
910
            mBRequestNewReaderSample = true;
911
        }
912
        
913
    }
914
}
915

916
//--------------------
917
void ofMediaFoundationSoundPlayer::addUpdateListener() {
918
    if (!mBAddedUpdateEvent) {
919
        ofAddListener(ofEvents().update, this, &ofMediaFoundationSoundPlayer::update);
920
    }
921
    mBAddedUpdateEvent = true;
922
}
923

924
//--------------------
925
void ofMediaFoundationSoundPlayer::removeUpdateListener() {
926
    if (mBAddedUpdateEvent) {
927
        ofRemoveListener(ofEvents().update, this, &ofMediaFoundationSoundPlayer::update);
928
    }
929
    mBAddedUpdateEvent = false;
930
}
931

932
//--------------------
933
void ofMediaFoundationSoundPlayer::_clearExtraVoices() {
934
    for (auto& it : mExtraVoices) {
935
        if (it.second) {
936
            std::ignore = it.second->Stop();
937
            std::ignore = it.second->FlushSourceBuffers();
938
            it.second->DestroyVoice();
939
        }
940
    }
941
    mExtraVoices.clear();
942
}
943

944
//--------------------
945
void ofMediaFoundationSoundPlayer::_setPan(IXAudio2SourceVoice* avoice, float apan) {
946
    if (!sXAudioMasteringVoice) {
947
        return;
948
    }
949

950
    DWORD dwChannelMask;
951
    sXAudioMasteringVoice->GetChannelMask(&dwChannelMask);
952

953
    float outputMatrix[8];
954
    for (int i = 0; i < 8; i++) outputMatrix[i] = 0;
955

956
    // pan of -1.0 indicates all left speaker, 
957
// 1.0 is all right speaker, 0.0 is split between left and right
958
    float left = 0.5f - apan / 2;
959
    float right = 0.5f + apan / 2;
960

961
    switch (dwChannelMask) {
962
    case SPEAKER_MONO:
963
        outputMatrix[0] = 1.0;
964
        break;
965
    case SPEAKER_STEREO:
966
    case SPEAKER_2POINT1:
967
    case SPEAKER_SURROUND:
968
        outputMatrix[0] = left;
969
        outputMatrix[1] = right;
970
        break;
971
    case SPEAKER_QUAD:
972
        outputMatrix[0] = outputMatrix[2] = left;
973
        outputMatrix[1] = outputMatrix[3] = right;
974
        break;
975
    case SPEAKER_4POINT1:
976
        outputMatrix[0] = outputMatrix[3] = left;
977
        outputMatrix[1] = outputMatrix[4] = right;
978
        break;
979
    case SPEAKER_5POINT1:
980
    case SPEAKER_7POINT1:
981
    case SPEAKER_5POINT1_SURROUND:
982
        outputMatrix[0] = outputMatrix[4] = left;
983
        outputMatrix[1] = outputMatrix[5] = right;
984
        break;
985
    case SPEAKER_7POINT1_SURROUND:
986
        outputMatrix[0] = outputMatrix[4] = outputMatrix[6] = left;
987
        outputMatrix[1] = outputMatrix[5] = outputMatrix[7] = right;
988
        break;
989
    }
990

991
    // Assuming pVoice sends to pMasteringVoice
992

993
    // TODO: Cache this 
994
    XAUDIO2_VOICE_DETAILS MasterVoiceDetails;
995
    sXAudioMasteringVoice->GetVoiceDetails(&MasterVoiceDetails);
996

997
    XAUDIO2_VOICE_DETAILS VoiceDetails;
998
    avoice->GetVoiceDetails(&VoiceDetails);
999
    avoice->SetOutputMatrix(NULL, VoiceDetails.InputChannels, MasterVoiceDetails.InputChannels, outputMatrix);
1000
}
1001

1002
//--------------------
1003
bool ofMediaFoundationSoundPlayer::_readToBuffer(IMFSourceReader* areader) {
1004
    bool bKeepOnReadin = true;
1005

1006
    ofLogVerbose("ofMediaFoundationSoundPlayer::_readToBuffer") << "num channels: " << mNumChannels << " sample rate: " << mSampleRate << std::endl;
1007

1008
    unsigned int totalFrames = 0;
1009

1010
    uint64_t bytes64 = uint64_t(mBytesPerSample);
1011
    uint64_t numChannels64 = uint64_t(mNumChannels);
1012

1013
    while (bKeepOnReadin) {
1014
        // figure out a better way to process this //
1015
        ComPtr<IMFSample> audioSample;
1016
        DWORD streamIndex, flags = 0;
1017
        LONGLONG llAudioTimeStamp;
1018

1019
        HRESULT hr = areader->ReadSample(
1020
            MF_SOURCE_READER_FIRST_AUDIO_STREAM,
1021
            0,                              // Flags.
1022
            &streamIndex,                   // Receives the actual stream index. 
1023
            &flags,                         // Receives status flags.
1024
            &llAudioTimeStamp,              // Receives the time stamp.
1025
            &audioSample                    // Receives the sample or NULL.
1026
        );
1027

1028
        if (flags & MF_SOURCE_READERF_ENDOFSTREAM) {
1029
            ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "End of stream";
1030
            // set to the buffer and complete //
1031
            bKeepOnReadin = false;
1032
            break;
1033
        }
1034

1035
        if (!audioSample) {
1036
            ofLogVerbose("ofMediaFoundationSoundPlayer::update") << "Null audio sample.";
1037
            //audioSample->Release();
1038
        } else {
1039
            ComPtr<IMFMediaBuffer> mediaBuffer;
1040
            hr = audioSample->ConvertToContiguousBuffer(mediaBuffer.GetAddressOf());
1041
            if (hr != S_OK) {
1042
                continue;
1043
            }
1044

1045
            BYTE* audioData = nullptr;
1046
            DWORD sampleBufferLength = 0;
1047

1048
            hr = mediaBuffer->Lock(&audioData, nullptr, &sampleBufferLength);
1049
            if (hr != S_OK) {
1050
                continue;
1051
            }
1052
            
1053
            size_t numFramesRead = uint64_t(sampleBufferLength) / (bytes64 * numChannels64);
1054
            ofLogVerbose("ofMediaFoundationSoundPlayer::_readToBuffer") << "sampleBufferLength : " << sampleBufferLength << " num frames: " << numFramesRead << std::endl;
1055
            totalFrames += numFramesRead;
1056
            std::vector<BYTE> tempBuffer;
1057
            tempBuffer.resize(sampleBufferLength, 0);
1058
            memcpy_s(tempBuffer.data(), sampleBufferLength, audioData, sampleBufferLength);
1059
            // add into the main buffer? 
1060
            mBuffer.insert(mBuffer.end(), tempBuffer.begin(), tempBuffer.end());
1061

1062
            hr = mediaBuffer->Unlock();
1063
            if (hr != S_OK) {
1064
                continue;
1065
            }
1066
        }
1067
    }
1068
    mTotalNumFrames = (mBuffer.size() / uint64_t(mNumChannels)) / uint64_t(mBytesPerSample);
1069
    mDurationSeconds = (double)mTotalNumFrames / (double)mSampleRate;
1070
    mDurationMS = mTotalNumFrames * uint64_t(1000) / uint64_t(mSampleRate);
1071
    auto durMillis = mTotalNumFrames * uint64_t(1000) / uint64_t(mSampleRate);
1072
    double durSeconds = (double)durMillis / 1000.0;
1073
    ofLogVerbose("ofMediaFoundationSoundPlayer::_readToBuffer") << "Total frames read: " << (totalFrames) << " mTotalNumFrames: " << mTotalNumFrames << " dur millis: " << durMillis << " dur seconds: " << durSeconds << std::endl;
1074
    return mBuffer.size() > 0;
1075
}