framework2

1
#include "ofGstUtils.h"
2
#ifndef TARGET_ANDROID
3
#include <gst/app/gstappsink.h>
4
#include <gst/video/video.h>
5

6
#if GST_VERSION_MAJOR>0
7
#include <gst/video/gstvideometa.h>
8
#endif
9

10
#ifdef OF_USE_GST_GL
11
#define GST_USE_UNSTABLE_API
12
#include <gst/gl/gl.h>
13
#endif
14

15
#include <glib-object.h>
16
#include <glib.h>
17
#include <algorithm>
18
#include "ofVideoPlayer.h"
19

20
#ifdef OF_USE_GST_GL
21
#ifndef TARGET_OPENGLES
22
#include <gst/gl/x11/gstgldisplay_x11.h>
23
#endif
24
#include <gst/gl/egl/gstgldisplay_egl.h>
25
#endif
26
#ifdef TARGET_WIN32
27
#include <winbase.h>	// to use SetEnvironmentVariableA
28
#endif
29

30
using std::shared_ptr;
31
using std::string;
32

33
ofGstUtils::ofGstMainLoopThread * ofGstUtils::mainLoop = nullptr;
34

35
void ofGstUtils::startGstMainLoop(){
36
	static bool initialized = false;
37
	if(!initialized){
38
		mainLoop = new ofGstMainLoopThread;
39
		mainLoop->start();
40
		initialized=true;
41
	}
42
}
43

44
GMainLoop * ofGstUtils::getGstMainLoop(){
45
	return mainLoop->getMainLoop();
46
}
47

48
void ofGstUtils::quitGstMainLoop(){
49
	if(mainLoop){
50
		mainLoop->quit();
51
		delete mainLoop;
52
		mainLoop = nullptr;
53
	}
54
}
55

56

57

58
//-------------------------------------------------
59
//----------------------------------------- gstUtils
60
//-------------------------------------------------
61

62
static bool plugin_registered = false;
63
static bool gst_inited = false;
64

65

66
// called when the appsink notifies us that there is a new buffer ready for
67
// processing
68

69
static GstFlowReturn on_new_buffer_from_source (GstAppSink * elt, void * data){
70
#if GST_VERSION_MAJOR==0
71
	shared_ptr<GstBuffer> buffer(gst_app_sink_pull_buffer (GST_APP_SINK (elt)),&gst_buffer_unref);
72
#else
73
	shared_ptr<GstSample> buffer(gst_app_sink_pull_sample (GST_APP_SINK (elt)),&gst_sample_unref);
74
#endif
75
	return ((ofGstUtils*)data)->buffer_cb(buffer);
76
}
77

78
static GstFlowReturn on_new_preroll_from_source (GstAppSink * elt, void * data){
79
#if GST_VERSION_MAJOR==0
80
	shared_ptr<GstBuffer> buffer(gst_app_sink_pull_preroll(GST_APP_SINK (elt)),&gst_buffer_unref);
81
#else
82
	shared_ptr<GstSample> buffer(gst_app_sink_pull_preroll(GST_APP_SINK (elt)),&gst_sample_unref);
83
#endif
84
	return ((ofGstUtils*)data)->preroll_cb(buffer);
85
}
86

87
static void on_eos_from_source (GstAppSink * elt, void * data){
88
	((ofGstUtils*)data)->eos_cb();
89
}
90

91
static gboolean appsink_plugin_init (GstPlugin * plugin)
92
{
93
  gst_element_register (plugin, "appsink", GST_RANK_NONE, GST_TYPE_APP_SINK);
94

95
  return TRUE;
96
}
97

98

99

100
ofGstUtils::ofGstUtils() {
101
	bLoaded 					= false;
102
	speed 						= 1;
103
	bPaused						= false;
104
	bIsMovieDone				= false;
105
	bPlaying					= false;
106
	loopMode					= OF_LOOP_NONE;
107
	bFrameByFrame 				= false;
108

109
	gstPipeline					= NULL;
110
	gstSink						= NULL;
111

112
	durationNanos				= 0;
113

114
	isAppSink					= false;
115
	isStream					= false;
116

117
	appsink						= NULL;
118
	closing 					= false;
119

120
	busWatchID					= 0;
121

122
#if GLIB_MINOR_VERSION<32
123
	if(!g_thread_supported()){
124
		g_thread_init(NULL);
125
	}
126
#endif
127

128
	if(!gst_inited){
129
#ifdef TARGET_WIN32
130
		string gst_path;
131
		if (sizeof(int) == 32){
132
			auto env_path = g_getenv("GSTREAMER_1_0_ROOT_X86");
133
			if (env_path == NULL) {
134
				env_path = (ofGetTargetPlatform() == OF_TARGET_MINGW) ? g_getenv("GSTREAMER_1_0_ROOT_MINGW_X86") : g_getenv("GSTREAMER_1_0_ROOT_MSVC_X86");
135
			}
136
			gst_path = env_path;
137
		}
138
		else {
139
			auto env_path = g_getenv("GSTREAMER_1_0_ROOT_X86_64");
140
			if (env_path == NULL) {
141
				env_path = (ofGetTargetPlatform() == OF_TARGET_MINGW) ? g_getenv("GSTREAMER_1_0_ROOT_MINGW_X86_64") : g_getenv("GSTREAMER_1_0_ROOT_MSVC_X86_64");
142
			}
143
			gst_path = env_path;
144
		}
145
		//putenv(("GST_PLUGIN_PATH_1_0=" + ofFilePath::join(gst_path, "lib\\gstreamer-1.0") + ";.").c_str());
146
		// to make it compatible with gcc and C++11 standard
147
		SetEnvironmentVariableA("GST_PLUGIN_PATH_1_0", ofFilePath::join(gst_path, "lib\\gstreamer-1.0").c_str());
148
#endif
149
		gst_init (NULL, NULL);
150
		gst_inited=true;
151
		ofLogVerbose("ofGstUtils") << "gstreamer inited";
152
	}
153
	if(!plugin_registered){
154
		gst_plugin_register_static(GST_VERSION_MAJOR, GST_VERSION_MINOR,
155
					"appsink", (char*)"Element application sink",
156
					appsink_plugin_init, "0.1", "LGPL", "ofVideoPlayer", "openFrameworks",
157
					"http://openframeworks.cc/");
158
		plugin_registered=true;
159
	}
160

161
}
162

163
ofGstUtils::~ofGstUtils() {
164
	close();
165
}
166

167
#if GST_VERSION_MAJOR==0
168
GstFlowReturn ofGstUtils::preroll_cb(shared_ptr<GstBuffer> buffer){
169
#else
170
GstFlowReturn ofGstUtils::preroll_cb(shared_ptr<GstSample> buffer){
171
#endif
172
	bIsMovieDone = false;
173
	if(appsink) return appsink->on_preroll(buffer);
174
	else return GST_FLOW_OK;
175
}
176

177

178
#if GST_VERSION_MAJOR==0
179
GstFlowReturn ofGstUtils::buffer_cb(shared_ptr<GstBuffer> buffer){
180
#else
181
GstFlowReturn ofGstUtils::buffer_cb(shared_ptr<GstSample> buffer){
182
#endif
183
	bIsMovieDone = false;
184
	if(appsink) return appsink->on_buffer(buffer);
185
	else return GST_FLOW_OK;
186
}
187

188
void ofGstUtils::eos_cb(){
189
	bIsMovieDone = true;
190
	if(appsink && !isAppSink) appsink->on_eos();
191
	if(closing){
192
		std::unique_lock<std::mutex> lck(eosMutex);
193
		eosCondition.notify_all();
194
	}
195
}
196

197
bool ofGstUtils::setPipelineWithSink(string pipeline, string sinkname, bool isStream){
198
	ofGstUtils::startGstMainLoop();
199

200
	GError * error = NULL;
201
	gstPipeline = gst_parse_launch (pipeline.c_str(), &error);
202
	g_object_ref_sink(gstPipeline);
203
	ofLogNotice("ofGstUtils") << "setPipelineWithSink(): gstreamer pipeline: " << pipeline;
204

205
	if(error!=NULL){
206
		ofLogError("ofGstUtils") << "setPipelineWithSink(): couldn't create pipeline: " << error->message;
207
		return false;
208
	}
209

210
	if(sinkname!=""){
211
		gstSink = gst_bin_get_by_name(GST_BIN(gstPipeline),sinkname.c_str());
212

213
		if(!gstSink){
214
			ofLogError("ofGstUtils") << "setPipelineWithSink(): couldn't get sink from string pipeline";
215
		}
216
	}
217

218
	return setPipelineWithSink(gstPipeline,gstSink,isStream);
219
}
220

221
bool ofGstUtils::setPipelineWithSink(GstElement * pipeline, GstElement * sink, bool isStream_){
222
	ofGstUtils::startGstMainLoop();
223

224
	gstPipeline = pipeline;
225
	gstSink = sink;
226
	isStream = isStream_;
227

228
	if(gstSink){
229
		gst_base_sink_set_sync(GST_BASE_SINK(gstSink), true);
230
	}
231

232
	if(gstSink && string(gst_plugin_feature_get_name( GST_PLUGIN_FEATURE(gst_element_get_factory(gstSink))))=="appsink"){
233
		isAppSink = true;
234
	}else{
235
		isAppSink = false;
236
	}
237

238
	return true;
239
}
240

241
void ofGstUtils::setFrameByFrame(bool _bFrameByFrame){
242
	bFrameByFrame = _bFrameByFrame;
243
	if(gstSink){
244
		g_object_set (G_OBJECT (gstSink), "sync", !bFrameByFrame, (void*)NULL);
245
	}
246
}
247

248
bool ofGstUtils::isFrameByFrame() const{
249
	return bFrameByFrame;
250
}
251

252
bool ofGstUtils::startPipeline(){
253

254
	bPaused 			= true;
255
	speed 				= 1.0f;
256

257

258
	GstBus * bus = gst_pipeline_get_bus (GST_PIPELINE(gstPipeline));
259

260
	if(bus){
261
		busWatchID = gst_bus_add_watch (bus, (GstBusFunc) busFunction, this);
262
	}
263

264
	gst_object_unref(bus);
265

266
	// pause the pipeline
267
	//GstState targetState;
268
	GstState state;
269
	auto ret = gst_element_set_state(GST_ELEMENT(gstPipeline), GST_STATE_PAUSED);
270
    switch (ret) {
271
		case GST_STATE_CHANGE_FAILURE:
272
			ofLogError("ofGstUtils") << "startPipeline(): unable to pause pipeline";
273
			return false;
274
			break;
275
		case GST_STATE_CHANGE_NO_PREROLL:
276
			ofLogVerbose() << "Pipeline is live and does not need PREROLL waiting PLAY";
277
			gst_element_set_state(GST_ELEMENT(gstPipeline), GST_STATE_PLAYING);
278
			break;
279
		case GST_STATE_CHANGE_ASYNC:
280
			ofLogVerbose() << "Pipeline is PREROLLING";
281
			//targetState = GST_STATE_PAUSED;
282
			if(!isStream && gst_element_get_state(gstPipeline,&state,NULL,5*GST_SECOND)!=GST_STATE_CHANGE_SUCCESS){
283
				ofLogError("ofGstUtils") << "startPipeline(): unable to pause pipeline after 5s";
284
				return false;
285
			}else{
286
				ofLogVerbose() << "Pipeline is PREROLLED";
287
			}
288
			break;
289
		case GST_STATE_CHANGE_SUCCESS:
290
			ofLogVerbose() << "Pipeline is PREROLLED";
291
			break;
292
    }
293
    
294
    if(isAppSink){
295
		ofLogVerbose("ofGstUtils") << "startPipeline(): attaching callbacks";
296

297
        bool bSignals = false;
298
        
299
#if GST_VERSION_MAJOR==0
300

301
        GstAppSinkCallbacks gstCallbacks;
302
        gstCallbacks.eos = &on_eos_from_source;
303
        gstCallbacks.new_preroll = &on_new_preroll_from_source;
304
        gstCallbacks.new_buffer = &on_new_buffer_from_source;
305
        gst_app_sink_set_callbacks(GST_APP_SINK(gstSink), &gstCallbacks, this, NULL);
306
        
307
#elseif GST_VERSION_MINOR <= 18
308

309
        GstAppSinkCallbacks gstCallbacks;
310
        gstCallbacks.eos = &on_eos_from_source;
311
        gstCallbacks.new_preroll = &on_new_preroll_from_source;
312
        gstCallbacks.new_sample = &on_new_buffer_from_source;
313
        gst_app_sink_set_callbacks(GST_APP_SINK(gstSink), &gstCallbacks, this, NULL);
314
        
315
#else
316

317
        //GStreamer 1.20 onwards crashes with the callback approach above.
318
        //Using signals makes it work! This might be a GStreamer bug - but we'll do this for now
319
        g_signal_connect(GST_APP_SINK(gstSink), "eos", G_CALLBACK(on_eos_from_source), this);
320
        g_signal_connect(GST_APP_SINK(gstSink), "new-sample", G_CALLBACK(on_new_buffer_from_source), this);
321
        g_signal_connect(GST_APP_SINK(gstSink), "new-preroll", G_CALLBACK(on_new_preroll_from_source), this);
322
        
323
        bSignals = true;
324
#endif
325

326
        // set the appsink to not emit signals, we are using callbacks instead
327
        // and frameByFrame to get buffers by polling instead of callback
328
        g_object_set (G_OBJECT (gstSink), "emit-signals", bSignals, "sync", !bFrameByFrame, (void*)NULL);
329

330
        gst_app_sink_set_max_buffers(GST_APP_SINK(gstSink),3);
331
        gst_app_sink_set_drop (GST_APP_SINK(gstSink),true);
332
	}
333
 
334
	// wait for paused state to query the duration
335
	if(!isStream){
336
		bPlaying = true;
337
		bLoaded = true;
338
	}
339

340
	return true;
341
}
342

343
void ofGstUtils::play(){
344
	setPaused(false);
345

346
	//this is if we set the speed first but it only can be set when we are playing.
347
	if(!isStream) setSpeed(speed);
348
}
349

350
void ofGstUtils::setPaused(bool _bPause){
351
	bPaused = _bPause;
352
	//timeLastIdle = ofGetElapsedTimeMillis();
353
	if(bLoaded){
354
		if(bPlaying){
355
			if(bPaused){
356
				gst_element_set_state (gstPipeline, GST_STATE_PAUSED);
357
			}else{
358
				gst_element_set_state (gstPipeline, GST_STATE_PLAYING);
359
			}
360
		}else{
361
			GstState state = GST_STATE_PAUSED;
362
			gst_element_set_state (gstPipeline, state);
363
			gst_element_get_state(gstPipeline,&state,NULL,2*GST_SECOND);
364
			if(!bPaused){
365
				gst_element_set_state (gstPipeline, GST_STATE_PLAYING);
366
			}
367
		}
368
		bPlaying = !bPaused;
369
	}
370
}
371

372
void ofGstUtils::stop(){
373
	if(!bLoaded) return;
374
	GstState state;
375

376
	if(!bPaused){
377
		state = GST_STATE_PAUSED;
378
		gst_element_set_state (gstPipeline, state);
379
		gst_element_get_state(gstPipeline,&state,NULL,2*GST_SECOND);
380
	}
381

382
	state = GST_STATE_READY;
383
	gst_element_set_state (gstPipeline, state);
384
	gst_element_get_state(gstPipeline,&state,NULL,2*GST_SECOND);
385
	bPlaying = false;
386
	bPaused = false;
387
}
388

389
float ofGstUtils::getPosition() const{
390
	if(gstPipeline){
391
		gint64 pos=0;
392
#if GST_VERSION_MAJOR==0
393
		GstFormat format=GST_FORMAT_TIME;
394
		if(!gst_element_query_position(GST_ELEMENT(gstPipeline),&format,&pos)){
395
			ofLogVerbose("ofGstUtils") << "getPosition(): couldn't query position";
396
			return -1;
397
		}
398
#else
399
		if(!gst_element_query_position(GST_ELEMENT(gstPipeline),GST_FORMAT_TIME,&pos)){
400
			ofLogVerbose("ofGstUtils") << "getPosition(): couldn't query position";
401
			return -1;
402
		}
403
#endif
404
		return (float)pos/(float)durationNanos;
405
	}else{
406
		return -1;
407
	}
408
}
409

410
int64_t ofGstUtils::getPositionNanos() const{
411
	if(gstPipeline){
412
		gint64 pos=0;
413
#if GST_VERSION_MAJOR==0
414
		GstFormat format=GST_FORMAT_TIME;
415
		if(!gst_element_query_position(GST_ELEMENT(gstPipeline),&format,&pos)){
416
			ofLogVerbose("ofGstUtils") << "getPosition(): couldn't query position";
417
			return -1;
418
		}
419
#else
420
		if(!gst_element_query_position(GST_ELEMENT(gstPipeline),GST_FORMAT_TIME,&pos)){
421
			ofLogVerbose("ofGstUtils") << "getPosition(): couldn't query position";
422
			return -1;
423
		}
424
#endif
425
		return pos;
426
	}else{
427
		return -1;
428
	}
429
}
430

431

432
float ofGstUtils::getSpeed() const{
433
	return speed;
434
}
435

436
float ofGstUtils::getDuration() const{
437
	return (float)getDurationNanos()/(float)GST_SECOND;
438
}
439

440
int64_t ofGstUtils::getDurationNanos() const{
441
	GstFormat format = GST_FORMAT_TIME;
442

443
#if GST_VERSION_MAJOR==0
444
	if(!gst_element_query_duration(getPipeline(),&format,&durationNanos))
445
		ofLogWarning("ofGstUtils") << "getDurationNanos(): couldn't query time duration";
446
#else
447
	if(!gst_element_query_duration(getPipeline(),format,&durationNanos))
448
		ofLogWarning("ofGstUtils") << "getDurationNanos(): couldn't query time duration";
449
#endif
450
	return durationNanos;
451

452
}
453

454
bool  ofGstUtils::getIsMovieDone() const{
455
	if(isAppSink){
456
		return gst_app_sink_is_eos(GST_APP_SINK(gstSink));
457
	}else{
458
		return bIsMovieDone;
459
	}
460
}
461

462
void ofGstUtils::setPosition(float pct){
463
	//pct = CLAMP(pct, 0,1);// check between 0 and 1;
464
	GstFormat format = GST_FORMAT_TIME;
465
	GstSeekFlags flags = (GstSeekFlags) (GST_SEEK_FLAG_ACCURATE | GST_SEEK_FLAG_FLUSH);
466
	if(speed > 1 || speed < -1){
467
		flags = (GstSeekFlags)(flags | GST_SEEK_FLAG_SKIP);
468
	}
469
	gint64 pos = (guint64)((double)pct*(double)durationNanos);
470

471
	/*if(bPaused){
472
		seek_lock();
473
		gst_element_set_state (gstPipeline, GST_STATE_PLAYING);
474
		posChangingPaused=true;
475
		seek_unlock();
476
	}*/
477
	if(speed>0){
478
		if(!gst_element_seek(GST_ELEMENT(gstPipeline),speed, 	format,
479
				flags,
480
				GST_SEEK_TYPE_SET,
481
				pos,
482
				GST_SEEK_TYPE_SET,
483
				-1)) {
484
		ofLogWarning("ofGstUtils") << "setPosition(): unable to seek";
485
		}
486
	}else{
487
		if(!gst_element_seek(GST_ELEMENT(gstPipeline),speed, 	format,
488
				flags,
489
				GST_SEEK_TYPE_SET,
490
				0,
491
				GST_SEEK_TYPE_SET,
492
				pos)) {
493
		ofLogWarning("ofGstUtils") << "setPosition(): unable to seek";
494
		}
495
	}
496
}
497

498
void ofGstUtils::setVolume(float volume){
499
	gdouble gvolume = volume;
500
	g_object_set(G_OBJECT(gstPipeline), "volume", gvolume, (void*)NULL);
501
}
502

503
void ofGstUtils::setLoopState(ofLoopType state){
504
	loopMode = state;
505
}
506

507
void ofGstUtils::setSpeed(float _speed){
508
	if(_speed == speed) return;
509

510
	GstFormat format = GST_FORMAT_TIME;
511
	GstSeekFlags flags = (GstSeekFlags) (GST_SEEK_FLAG_ACCURATE | GST_SEEK_FLAG_FLUSH);
512

513
    bool needPos = true;
514
    #if GST_VERSION_MAJOR >= 1 && GST_VERSION_MINOR >= 18
515
        //if requested speed is the same direction as we are already going we can do a fast rate change
516
        if( speed * _speed > 0 ){
517
            flags = GST_SEEK_FLAG_INSTANT_RATE_CHANGE;
518
            needPos = false;
519
        }
520
    #endif
521
    
522
	if(_speed > 1 || _speed < -1){
523
		flags = (GstSeekFlags)(flags | GST_SEEK_FLAG_SKIP);
524
	}
525

526
	if(_speed==0){
527
		gst_element_set_state (gstPipeline, GST_STATE_PAUSED);
528
		return;
529
	}
530
  
531

532
	gint64 pos = 0;
533
    GstSeekType seekType = GST_SEEK_TYPE_NONE;
534
    if( needPos ){
535
        seekType = GST_SEEK_TYPE_SET;
536

537
        pos = getPositionNanos();
538
        if( pos == -1 ){
539
            ofLogError("ofGstUtils") << "setSpeed(): couldn't query position";
540
            return;
541
        }
542

543
    }
544

545
	speed = _speed;
546

547
	if(!bPaused)
548
		gst_element_set_state (gstPipeline, GST_STATE_PLAYING);
549

550
	if(speed>0){
551
		if(!gst_element_seek(GST_ELEMENT(gstSink), speed, 	format,
552
				flags,
553
				seekType,
554
				pos,
555
				seekType,
556
				-1)) {
557
			ofLogWarning("ofGstUtils") << "setSpeed(): unable to change speed";
558
		}
559
	}else{
560
		if(!gst_element_seek(GST_ELEMENT(gstSink), speed, 	format,
561
				flags,
562
				seekType,
563
				0,
564
				seekType,
565
				pos)) {
566
			ofLogWarning("ofGstUtils") << "setSpeed(): unable to change speed";
567
		}
568
	}
569

570
	ofLogVerbose("ofGstUtils") << "setSpeed(): speed changed to " << speed;
571

572
}
573

574
void ofGstUtils::close(){
575
	if(bPlaying){
576
		if(!bIsMovieDone && !bPaused && !isStream){
577
			std::unique_lock<std::mutex> lck(eosMutex);
578
			closing = true;
579
			gst_element_send_event(gstPipeline,gst_event_new_eos());
580
			if(eosCondition.wait_for(lck,std::chrono::milliseconds(5000))==std::cv_status::timeout){
581
				ofLogWarning("ofGstUtils") << "didn't received EOS in 5s, closing pipeline anyway";
582
			}
583
			closing = false;
584
		}
585
	}
586
	stop();
587

588
	if(bLoaded){
589
		gst_element_set_state(GST_ELEMENT(gstPipeline), GST_STATE_NULL);
590
		gst_element_get_state(gstPipeline,NULL,NULL,2*GST_SECOND);
591

592
		if(busWatchID!=0) g_source_remove(busWatchID);
593

594
		gst_object_unref(gstPipeline);
595
		gstPipeline = NULL;
596
		gstSink = NULL;
597
	}
598

599
	bLoaded = false;
600
}
601

602
/*static string getName(GstState state){
603
	switch(state){
604
	case   GST_STATE_VOID_PENDING:
605
		return "void pending";
606
	case   GST_STATE_NULL:
607
		return "null";
608
	case   GST_STATE_READY:
609
		return "ready";
610
	case   GST_STATE_PAUSED:
611
		return "paused";
612
	case   GST_STATE_PLAYING:
613
		return "playing";
614
	default:
615
		return "";
616
	}
617
}*/
618

619
bool ofGstUtils::busFunction(GstBus * bus, GstMessage * message, ofGstUtils * gstUtils){
620
	return gstUtils->gstHandleMessage(bus,message);
621
}
622

623
bool ofGstUtils::gstHandleMessage(GstBus * bus, GstMessage * msg){
624
	if(appsink && appsink->on_message(msg)) return true;
625

626
		/*ofLogVerbose("ofGstUtils") << "gstHandleMessage(): got " << GST_MESSAGE_TYPE_NAME(msg)
627
			<< " message from " << GST_MESSAGE_SRC_NAME(msg);*/
628

629
	switch (GST_MESSAGE_TYPE (msg)) {
630

631
		case GST_MESSAGE_BUFFERING:
632
			gint pctBuffered;
633
			gst_message_parse_buffering(msg,&pctBuffered);
634
			ofLogVerbose("ofGstUtils") << "gstHandleMessage(): buffering " << pctBuffered;
635
			if(pctBuffered<100){
636
				gst_element_set_state (gstPipeline, GST_STATE_PAUSED);
637
			}else if(!bPaused){
638
				gst_element_set_state (gstPipeline, GST_STATE_PLAYING);
639
			}
640
		break;
641

642
#if GST_VERSION_MAJOR==0
643
		case GST_MESSAGE_DURATION:{
644
			GstFormat format=GST_FORMAT_TIME;
645
			gst_element_query_duration(gstPipeline,&format,&durationNanos);
646
		}break;
647
#else
648
		case GST_MESSAGE_DURATION_CHANGED:
649
			gst_element_query_duration(gstPipeline,GST_FORMAT_TIME,&durationNanos);
650
			break;
651

652
#endif
653

654
		case GST_MESSAGE_STATE_CHANGED:{
655
			GstState oldstate, newstate, pendstate;
656
			gst_message_parse_state_changed(msg, &oldstate, &newstate, &pendstate);
657
			if(isStream && newstate==GST_STATE_PAUSED && !bPlaying ){
658
				bLoaded = true;
659
				bPlaying = true;
660
				if(!bPaused){
661
					//ofLogVerbose("ofGstUtils") << "gstHandleMessage(): setting stream pipeline to play";
662
					play();
663
				}
664
			}
665

666
			/*ofLogVerbose("ofGstUtils") << "gstHandleMessage(): " << GST_MESSAGE_SRC_NAME(msg) << " state changed from "
667
					<< getName(oldstate) << " to " << getName(newstate) << " (" + getName(pendstate) << ")";*/
668
		}break;
669

670
		case GST_MESSAGE_ASYNC_DONE:
671
			ofLogVerbose("ofGstUtils") << "gstHandleMessage(): async done";
672
		break;
673

674
		case GST_MESSAGE_ERROR: {
675
			GError *err;
676
			gchar *debug;
677
			gst_message_parse_error(msg, &err, &debug);
678
			gchar * name = gst_element_get_name(GST_MESSAGE_SRC (msg));
679

680
			ofLogError("ofGstUtils") << "gstHandleMessage(): embedded video playback halted for plugin, module "
681
				<< name << "  reported: " << err->message;
682

683
			g_free(name);
684
			g_error_free(err);
685
			g_free(debug);
686

687
			gst_element_set_state(GST_ELEMENT(gstPipeline), GST_STATE_NULL);
688

689
		}break;
690

691
		case GST_MESSAGE_EOS:{
692
			ofLogVerbose("ofGstUtils") << "gstHandleMessage(): end of the stream";
693
			bool isClosing = closing;
694
			eos_cb();
695

696
			if(isClosing){
697
				busWatchID = 0;
698
				return false;
699
			}
700

701
			switch(loopMode){
702

703
				case OF_LOOP_NORMAL:{
704
					GstFormat format = GST_FORMAT_TIME;
705
					GstSeekFlags flags = (GstSeekFlags) (GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT);
706
					if(speed>0){
707
						if(!gst_element_seek(GST_ELEMENT(gstPipeline),
708
											speed,
709
											format,
710
											flags,
711
											GST_SEEK_TYPE_SET,
712
											0,
713
											GST_SEEK_TYPE_SET,
714
											-1)) {
715
							ofLogWarning("ofGstUtils") << "gstHandleMessage(): unable to seek";
716
						}
717
					}else if(speed<0){
718
						if(!gst_element_seek(GST_ELEMENT(gstPipeline),speed, 	format,
719
								flags,
720
								GST_SEEK_TYPE_SET,
721
								0,
722
								GST_SEEK_TYPE_SET,
723
								durationNanos-1000000)) {
724
							ofLogWarning("ofGstUtils") << "gstHandleMessage(): unable to seek";
725
						}
726
					}
727
				}break;
728

729
				case OF_LOOP_PALINDROME:{
730
					GstFormat format = GST_FORMAT_TIME;
731
					GstSeekFlags flags = (GstSeekFlags) (GST_SEEK_FLAG_FLUSH |GST_SEEK_FLAG_KEY_UNIT);
732
					gint64 pos;
733
					#if GST_VERSION_MAJOR==0
734
						gst_element_query_position(GST_ELEMENT(gstPipeline),&format,&pos);
735
					#else
736
						gst_element_query_position(GST_ELEMENT(gstPipeline),format,&pos);
737
					#endif
738
					float loopSpeed;
739
					if(pos>0)
740
						loopSpeed=-speed;
741
					else
742
						loopSpeed=speed;
743
					if(!gst_element_seek(GST_ELEMENT(gstPipeline),
744
										loopSpeed,
745
										GST_FORMAT_UNDEFINED,
746
										flags,
747
										GST_SEEK_TYPE_NONE,
748
										0,
749
										GST_SEEK_TYPE_NONE,
750
										0)) {
751
						ofLogWarning("ofGstUtils") << "gstHandleMessage(): unable to seek";
752
					}
753
				}break;
754

755
				default:
756
				break;
757
			}
758

759
		}break;
760
		case GST_MESSAGE_LATENCY:
761
			gst_bin_recalculate_latency (GST_BIN (getPipeline()));
762
			break;
763
		case GST_MESSAGE_REQUEST_STATE:	{
764
			GstState state;
765
			gchar *name = gst_object_get_path_string (GST_MESSAGE_SRC (msg));
766

767
			gst_message_parse_request_state (msg, &state);
768
			gst_element_set_state (getPipeline(), state);
769

770
			g_free (name);
771
			break;
772
		}
773
		
774
#if GST_VERSION_MAJOR==1 && GST_VERSION_MINOR>=2
775
		case GST_MESSAGE_HAVE_CONTEXT:{
776
			GstContext *context;
777
			const gchar *context_type;
778
			gchar *context_str;
779

780
			gst_message_parse_have_context (msg, &context);
781

782
			context_type = gst_context_get_context_type (context);
783
			context_str = gst_structure_to_string (gst_context_get_structure (context));
784
			ofLogNotice("ofGstUtils","Got context from element '%s': %s=%s\n",
785
				GST_ELEMENT_NAME (GST_MESSAGE_SRC (msg)), context_type,
786
				context_str);
787
			g_free (context_str);
788
			gst_context_unref (context);
789
			break;
790
		}
791
#endif
792
		default:
793
			ofLogVerbose("ofGstUtils") << "gstHandleMessage(): unhandled message from " << GST_MESSAGE_SRC_NAME(msg);
794
		break;
795
	}
796

797
	return true;
798
}
799

800
GstElement 	* ofGstUtils::getPipeline() const{
801
	return gstPipeline;
802
}
803

804
GstElement 	* ofGstUtils::getSink() const{
805
	return gstSink;
806
}
807

808
GstElement 	* ofGstUtils::getGstElementByName(const string & name) const{
809
	return gst_bin_get_by_name(GST_BIN(gstPipeline),name.c_str());
810
}
811

812
void ofGstUtils::setSinkListener(ofGstAppSink * appsink_){
813
	appsink = appsink_;
814
}
815

816
uint64_t ofGstUtils::getMinLatencyNanos() const{
817
	GstClockTime minlat=0, maxlat=0;
818
	GstQuery * q = gst_query_new_latency();
819
	if (gst_element_query (gstPipeline, q)) {
820
		 gboolean live;
821
		 gst_query_parse_latency (q, &live, &minlat, &maxlat);
822
	}
823
	gst_query_unref (q);
824
	return minlat;
825
}
826

827
uint64_t ofGstUtils::getMaxLatencyNanos() const{
828
	GstClockTime minlat=0, maxlat=0;
829
	GstQuery * q = gst_query_new_latency();
830
	if (gst_element_query (gstPipeline, q)) {
831
		 gboolean live;
832
		 gst_query_parse_latency (q, &live, &minlat, &maxlat);
833
	}
834
	gst_query_unref (q);
835
	return maxlat;
836
}
837

838

839

840
//-------------------------------------------------
841
//----------------------------------------- videoUtils
842
//-------------------------------------------------
843

844

845

846
ofGstVideoUtils::ofGstVideoUtils(){
847
	bIsFrameNew					= false;
848
	bHavePixelsChanged			= false;
849
	bBackPixelsChanged			= false;
850
#if GST_VERSION_MAJOR==1
851
	GstMapInfo initMapinfo		= {0,};
852
	mapinfo 					= initMapinfo;
853
#endif
854
	internalPixelFormat			= OF_PIXELS_RGB;
855
#ifdef OF_USE_GST_GL
856
	glDisplay = NULL;
857
	glContext = NULL;
858
#endif
859
	copyPixels = false;
860
}
861

862
ofGstVideoUtils::~ofGstVideoUtils(){
863
	close();
864
}
865

866
void ofGstVideoUtils::close(){
867
	ofGstUtils::close();
868
	std::unique_lock<std::mutex> lock(mutex);
869
	pixels.clear();
870
	backPixels.clear();
871
	eventPixels.clear();
872
	bIsFrameNew					= false;
873
	bHavePixelsChanged			= false;
874
	bBackPixelsChanged			= false;
875
	frontBuffer.reset();
876
	backBuffer.reset();
877
	
878
#if GST_VERSION_MAJOR==1
879
	while(!bufferQueue.empty()) bufferQueue.pop();
880
#endif
881
}
882

883
bool ofGstVideoUtils::isInitialized() const{
884
	return isLoaded();
885
}
886

887
bool ofGstVideoUtils::isFrameNew() const{
888
	return bIsFrameNew;
889
}
890

891
ofPixels& ofGstVideoUtils::getPixels(){
892
	return pixels;
893
}
894

895
const ofPixels & ofGstVideoUtils::getPixels() const{
896
	return pixels;
897
}
898

899
ofTexture * ofGstVideoUtils::getTexture(){
900
#ifdef OF_USE_GST_GL
901
	if(frontTexture.isAllocated()){
902
		return &frontTexture;
903
	}else{
904
		return NULL;
905
	}
906
#else
907
	return NULL;
908
#endif
909
}
910

911
void ofGstVideoUtils::update(){
912
	if (isLoaded()){
913
		if(!isFrameByFrame()){
914
			std::unique_lock<std::mutex> lock(mutex);
915
			bHavePixelsChanged = bBackPixelsChanged;
916
			if (bHavePixelsChanged){
917
				bBackPixelsChanged=false;
918
				std::swap(pixels,backPixels);
919
				#ifdef OF_USE_GST_GL
920
				if(backTexture.isAllocated()){
921
					frontTexture.getTextureData() = backTexture.getTextureData();
922
					frontTexture.setTextureMinMagFilter(GL_LINEAR,GL_LINEAR);
923
					frontTexture.setTextureWrap(GL_CLAMP_TO_EDGE,GL_CLAMP_TO_EDGE);
924
				}
925
				#endif
926
				if(!copyPixels){
927
					frontBuffer = backBuffer;
928
				}
929
			}
930
		}else{
931
#if GST_VERSION_MAJOR==0
932
			ofLogError() << "frame by frame doesn't work any more in 0.10";
933
#else
934
			GstBuffer * buffer;
935
			GstSample * sample;
936

937
			//get the buffer from appsink
938
			if(isPaused()){
939
				sample = gst_app_sink_pull_preroll (GST_APP_SINK (getSink()));
940
			}else{
941
				sample = gst_app_sink_pull_sample (GST_APP_SINK (getSink()));
942
			}
943
			buffer = gst_sample_get_buffer(sample);
944

945
			if(buffer){
946
				if(pixels.isAllocated()){
947
					gst_buffer_map (buffer, &mapinfo, GST_MAP_READ);
948
					//TODO: stride = mapinfo.size / height;
949
					pixels.setFromExternalPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getNumChannels());
950
					backBuffer = shared_ptr<GstSample>(sample,gst_sample_unref);
951
					bHavePixelsChanged=true;
952
					gst_buffer_unmap(buffer,&mapinfo);
953
				}
954
			}
955
#endif
956
		}
957
	}else{
958
		ofLogWarning("ofGstVideoUtils") << "update(): ofGstVideoUtils not loaded";
959
	}
960
	bIsFrameNew = bHavePixelsChanged;
961
	bHavePixelsChanged = false;
962
}
963

964
float ofGstVideoUtils::getHeight() const{
965
	return pixels.getHeight();
966
}
967

968
float ofGstVideoUtils::getWidth() const{
969
	return pixels.getWidth();
970
}
971

972

973
#if GST_VERSION_MAJOR>0
974
string	ofGstVideoUtils::getGstFormatName(ofPixelFormat format){
975
	switch(format){
976
	case OF_PIXELS_GRAY:
977
		return "GRAY8";
978
	case OF_PIXELS_RGBA:
979
		return "RGBA";
980
	case OF_PIXELS_BGRA:
981
		return "BGRA";
982
	case OF_PIXELS_RGB565:
983
		return "RGB16";
984
	case OF_PIXELS_NV12:
985
		return "NV12";
986
	case OF_PIXELS_NV21:
987
		return "NV21";
988
	case OF_PIXELS_YV12:
989
		return "YV12";
990
	case OF_PIXELS_I420:
991
		return "I420";
992
	case OF_PIXELS_YUY2:
993
		return "YUY2";
994
	case OF_PIXELS_RGB:
995
		return "RGB";
996
	case OF_PIXELS_BGR:
997
		return "BGR";
998
	default:
999
		return "UNKNOWN";
1000
	}
1001
}
1002

1003
GstVideoFormat	ofGstVideoUtils::getGstFormat(ofPixelFormat format){
1004
	switch(format){
1005
	case OF_PIXELS_GRAY:
1006
		return GST_VIDEO_FORMAT_GRAY8;
1007

1008
	case OF_PIXELS_RGB:
1009
		return GST_VIDEO_FORMAT_RGB;
1010

1011
	case OF_PIXELS_BGR:
1012
		return GST_VIDEO_FORMAT_BGR;
1013

1014
	case OF_PIXELS_RGBA:
1015
		return GST_VIDEO_FORMAT_RGBA;
1016

1017
	case OF_PIXELS_BGRA:
1018
		return GST_VIDEO_FORMAT_BGRA;
1019

1020
	case OF_PIXELS_RGB565:
1021
		return GST_VIDEO_FORMAT_RGB16;
1022

1023
	case OF_PIXELS_NV12:
1024
		return GST_VIDEO_FORMAT_NV12;
1025

1026
	case OF_PIXELS_NV21:
1027
		return GST_VIDEO_FORMAT_NV21;
1028

1029
	case OF_PIXELS_YV12:
1030
		return GST_VIDEO_FORMAT_YV12;
1031

1032
	case OF_PIXELS_I420:
1033
		return GST_VIDEO_FORMAT_I420;
1034

1035
	case OF_PIXELS_YUY2:
1036
		return GST_VIDEO_FORMAT_YUY2;
1037

1038
	case OF_PIXELS_UNKNOWN:
1039
	case OF_PIXELS_GRAY_ALPHA:
1040
	case OF_PIXELS_Y:
1041
	case OF_PIXELS_U:
1042
	case OF_PIXELS_V:
1043
	case OF_PIXELS_UV:
1044
	case OF_PIXELS_VU:
1045
	default:
1046
		return GST_VIDEO_FORMAT_UNKNOWN;
1047
	}
1048
}
1049

1050
ofPixelFormat ofGstVideoUtils::getOFFormat(GstVideoFormat format){
1051
	switch(format){
1052
	case GST_VIDEO_FORMAT_GRAY8:
1053
		return OF_PIXELS_GRAY;
1054

1055
	case GST_VIDEO_FORMAT_RGB:
1056
		return OF_PIXELS_RGB;
1057

1058
	case GST_VIDEO_FORMAT_BGR:
1059
		return OF_PIXELS_BGR;
1060

1061
	case GST_VIDEO_FORMAT_RGBA:
1062
		return OF_PIXELS_RGBA;
1063

1064
	case GST_VIDEO_FORMAT_BGRA:
1065
		return OF_PIXELS_BGRA;
1066

1067
	case GST_VIDEO_FORMAT_RGB16:
1068
		return OF_PIXELS_RGB565;
1069

1070
	case GST_VIDEO_FORMAT_NV12:
1071
		return OF_PIXELS_NV12;
1072

1073
	case GST_VIDEO_FORMAT_NV21:
1074
		return OF_PIXELS_NV21;
1075

1076
	case GST_VIDEO_FORMAT_YV12:
1077
		return OF_PIXELS_YV12;
1078

1079
	case GST_VIDEO_FORMAT_I420:
1080
		return OF_PIXELS_I420;
1081

1082
	case GST_VIDEO_FORMAT_YUY2:
1083
		return OF_PIXELS_YUY2;
1084

1085
	default:
1086
		ofLogError() << "non supported format " << format;
1087
		return OF_PIXELS_UNKNOWN;
1088
	}
1089
}
1090
#endif
1091

1092
/*
1093
gboolean ofGstVideoUtils::sync_bus_call (GstBus * bus, GstMessage * msg, gpointer data)
1094
{
1095
	switch (GST_MESSAGE_TYPE (msg)) {
1096
		case GST_MESSAGE_NEED_CONTEXT:
1097
		{
1098
			ofGstVideoPlayer * player = (ofGstVideoPlayer*)data;
1099
			const gchar *context_type;
1100

1101
			gst_message_parse_context_type (msg, &context_type);
1102
			ofLogNotice("ofGstVideoPlayer","got need context %s\n", context_type);
1103

1104
			if (g_strcmp0 (context_type, GST_GL_DISPLAY_CONTEXT_TYPE) == 0) {
1105
				GstContext *display_context =
1106
				gst_context_new (GST_GL_DISPLAY_CONTEXT_TYPE, TRUE);
1107
				gst_context_set_gl_display (display_context, player->glDisplay);
1108
				//GstStructure *s = gst_context_writable_structure (display_context);
1109
				//gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, player->glContext,	NULL);
1110
				gst_element_set_context (GST_ELEMENT (msg->src), display_context);
1111
				return TRUE;
1112
			} else if (g_strcmp0 (context_type, "gst.gl.app_context") == 0) {
1113
				GstContext *app_context = gst_context_new ("gst.gl.app_context", TRUE);
1114
				GstStructure *s = gst_context_writable_structure (app_context);
1115
				gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, player->glContext,	NULL);
1116
				gst_element_set_context (GST_ELEMENT (msg->src), app_context);
1117
				return TRUE;
1118
			}
1119
			break;
1120
		}
1121
		default:
1122
		break;
1123
	}
1124
	return FALSE;
1125
}*/
1126

1127
void ofGstVideoUtils::setCopyPixels(bool copy){
1128
	copyPixels = copy;
1129
}
1130

1131
bool ofGstVideoUtils::setPipeline(string pipeline, ofPixelFormat pixelFormat, bool isStream, int w, int h){
1132
	internalPixelFormat = pixelFormat;
1133
#ifndef OF_USE_GST_GL
1134
	string caps;
1135
#if GST_VERSION_MAJOR==0
1136
	switch(pixelFormat){
1137
	case OF_PIXELS_MONO:
1138
		caps="video/x-raw-gray, depth=8, bpp=8";
1139
		break;
1140
	case OF_PIXELS_RGBA:
1141
		caps="video/x-raw-rgb, depth=24, bpp=32, endianness=4321, red_mask=0xff0000, green_mask=0x00ff00, blue_mask=0x0000ff, alpha_mask=0x000000ff";
1142
		break;
1143
	case OF_PIXELS_BGRA:
1144
		caps="video/x-raw-rgb, depth=24, bpp=32, endianness=4321, red_mask=0x0000ff, green_mask=0x00ff00, blue_mask=0xff0000, alpha_mask=0x000000ff";
1145
		break;
1146
	case OF_PIXELS_RGB:
1147
	default:
1148
		caps="video/x-raw-rgb, depth=24, bpp=24, endianness=4321, red_mask=0xff0000, green_mask=0x00ff00, blue_mask=0x0000ff, alpha_mask=0x000000ff";
1149
		break;
1150
	}
1151
#else
1152
	if(pixelFormat!=OF_PIXELS_NATIVE){
1153
		caps="video/x-raw, format="+getGstFormatName(pixelFormat);
1154
	}else{
1155
		caps = "video/x-raw,format={RGBA,BGRA,RGB,BGR,RGB16,GRAY8,YV12,I420,NV12,NV21,YUY2}";
1156
	}
1157
#endif
1158

1159
	if(w!=-1 && h!=-1){
1160
		caps+=", width=" + ofToString(w) + ", height=" + ofToString(h);
1161
	}
1162

1163
	string pipeline_string =
1164
		pipeline + " ! appsink name=ofappsink enable-last-sample=0 caps=\"" + caps + "\"";
1165

1166
	if((w==-1 || h==-1) || pixelFormat==OF_PIXELS_NATIVE || allocate(w,h,pixelFormat)){
1167
		return setPipelineWithSink(pipeline_string,"ofappsink",isStream);
1168
	}else{
1169
		return false;
1170
	}
1171
#else
1172
	string pipeline_string =
1173
		pipeline + " ! glcolorscale name=gl_filter ! appsink name=ofappsink enable-last-sample=0 caps=\"video/x-raw,format=RGBA\"";
1174

1175
	bool ret;
1176
	if((w==-1 || h==-1) || pixelFormat==OF_PIXELS_NATIVE || allocate(w,h,pixelFormat)){
1177
		ret = setPipelineWithSink(pipeline_string,"ofappsink",isStream);
1178
	}else{
1179
		ret = false;
1180
	}
1181

1182
	auto glfilter = gst_bin_get_by_name(GST_BIN(getPipeline()),"gl_filter");
1183

1184
#if defined(TARGET_LINUX) && !defined(TARGET_OPENGLES)
1185
	glXMakeCurrent (ofGetX11Display(), None, 0);
1186
	glDisplay = (GstGLDisplay *)gst_gl_display_x11_new_with_display(ofGetX11Display());
1187
	glContext = gst_gl_context_new_wrapped (glDisplay, (guintptr) ofGetGLXContext(),
1188
	    		  GST_GL_PLATFORM_GLX, GST_GL_API_OPENGL);
1189

1190
	g_object_set (G_OBJECT (glfilter), "other-context", glContext, NULL);
1191
	// FIXME: this seems to be the way to add the context in 1.4.5
1192
	//
1193
	// GstBus * bus = gst_pipeline_get_bus (GST_PIPELINE(gstPipeline));
1194
	// gst_bus_enable_sync_message_emission (bus);
1195
	// g_signal_connect (bus, "sync-message", G_CALLBACK (sync_bus_call), this);
1196
	// gst_object_unref(bus);
1197

1198
	glXMakeCurrent (ofGetX11Display(), ofGetX11Window(), ofGetGLXContext());
1199
#elif defined(TARGET_OPENGLES)
1200
	cout << "current display " << ofGetEGLDisplay() << endl;
1201
	eglMakeCurrent (eglGetDisplay(EGL_DEFAULT_DISPLAY), 0,0, 0);
1202
	glDisplay = (GstGLDisplay *)gst_gl_display_egl_new_with_egl_display(eglGetDisplay(EGL_DEFAULT_DISPLAY));
1203
	glContext = gst_gl_context_new_wrapped (glDisplay, (guintptr) ofGetEGLContext(),
1204
	    		  GST_GL_PLATFORM_EGL, GST_GL_API_GLES2);
1205

1206
	g_object_set (G_OBJECT (glfilter), "other-context", glContext, NULL);
1207
	// FIXME: this seems to be the way to add the context in 1.4.5
1208
	//
1209
	// GstBus * bus = gst_pipeline_get_bus (GST_PIPELINE(gstPipeline));
1210
	// gst_bus_enable_sync_message_emission (bus);
1211
	// g_signal_connect (bus, "sync-message", G_CALLBACK (sync_bus_call), this);
1212
	// gst_object_unref(bus);
1213

1214
	eglMakeCurrent (ofGetEGLDisplay(), ofGetEGLSurface(), ofGetEGLSurface(), ofGetEGLContext());
1215

1216
#endif
1217

1218
	return ret;
1219
#endif
1220
}
1221

1222
bool ofGstVideoUtils::setPixelFormat(ofPixelFormat pixelFormat){
1223
	internalPixelFormat = pixelFormat;
1224
	return true;
1225
}
1226

1227
ofPixelFormat ofGstVideoUtils::getPixelFormat() const{
1228
	return internalPixelFormat;
1229
}
1230

1231
bool ofGstVideoUtils::allocate(int w, int h, ofPixelFormat pixelFormat){
1232
	std::unique_lock<std::mutex> lock(mutex);
1233
#if GST_VERSION_MAJOR>0
1234
	if(pixelFormat!=internalPixelFormat){
1235
		ofLogNotice("ofGstVideoUtils") << "allocating with " << w << "x" << h << " " << getGstFormatName(pixelFormat);
1236
	}
1237
#endif
1238
	pixels.allocate(w,h,pixelFormat);
1239
	backPixels.allocate(w,h,pixelFormat);
1240
	pixels.set(0);
1241
	backPixels.set(0);
1242

1243
	bHavePixelsChanged = false;
1244
	bBackPixelsChanged = true;
1245

1246
	internalPixelFormat = pixelFormat;
1247
	return pixels.isAllocated();
1248
}
1249

1250
void ofGstVideoUtils::reallocateOnNextFrame(){
1251
	std::unique_lock<std::mutex> lock(mutex);
1252
	pixels.clear();
1253
	backPixels.clear();
1254
	bIsFrameNew					= false;
1255
	bHavePixelsChanged			= false;
1256
	bBackPixelsChanged			= false;
1257
	frontBuffer.reset();
1258
	backBuffer.reset();
1259
#if GST_VERSION_MAJOR==1
1260
	while(!bufferQueue.empty()) bufferQueue.pop();
1261
#endif
1262
}
1263

1264
#if GST_VERSION_MAJOR==0
1265
GstFlowReturn ofGstVideoUtils::process_buffer(shared_ptr<GstBuffer> _buffer){
1266
	guint size = GST_BUFFER_SIZE (_buffer.get());
1267
	int stride = 0;
1268
	if(pixels.isAllocated() && pixels.getTotalBytes()!=(int)size){
1269
        stride = gst_video_format_get_row_stride( GST_VIDEO_FORMAT_RGB,0, pixels.getWidth());
1270
        if(stride == (pixels.getWidth() * pixels.getHeight() *  pixels.getBytesPerPixel())) {
1271
            ofLogError("ofGstVideoUtils") << "buffer_cb(): error on new buffer, buffer size: " << size << "!= init size: " << pixels.getTotalBytes();
1272
            return GST_FLOW_ERROR;
1273
        }
1274
	}
1275
	mutex.lock();
1276
	if(pixels.isAllocated()){
1277
		backBuffer = _buffer;
1278
        if(stride > 0) {
1279
            backPixels.setFromAlignedPixels(GST_BUFFER_DATA (backBuffer.get()),pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat(),stride);
1280
        }
1281
        else {
1282
            backPixels.setFromExternalPixels(GST_BUFFER_DATA (backBuffer.get()),pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat());
1283
            eventPixels.setFromExternalPixels(GST_BUFFER_DATA (backBuffer.get()),pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat());
1284
        }
1285
		bBackPixelsChanged=true;
1286
		mutex.unlock();
1287
        if(stride == 0) {
1288
        	ofNotifyEvent(prerollEvent,eventPixels);
1289
        }
1290
	}else{
1291
		if(isStream && appsink){
1292
			appsink->on_stream_prepared();
1293
		}else{
1294
			ofLogError("ofGstVideoUtils") << "preroll_cb(): received a preroll without allocation";
1295
		}
1296
		mutex.unlock();
1297
	}
1298
	return GST_FLOW_OK;
1299
}
1300
#else
1301

1302
static GstVideoInfo getVideoInfo(GstSample * sample){
1303
    GstCaps *caps = gst_sample_get_caps(sample);
1304
    GstVideoInfo vinfo;
1305
    if(caps){
1306
		gst_video_info_from_caps (&vinfo, caps);
1307
    }else{
1308
    	ofLogError() << "couldn't get sample caps";
1309
    }
1310
    return vinfo;
1311
}
1312

1313
GstFlowReturn ofGstVideoUtils::process_sample(shared_ptr<GstSample> sample){
1314
	GstBuffer * _buffer = gst_sample_get_buffer(sample.get());
1315

1316
#ifdef OF_USE_GST_GL
1317
	if (gst_buffer_map (_buffer, &mapinfo, (GstMapFlags)(GST_MAP_READ | GST_MAP_GL))){
1318
		if (gst_is_gl_memory (mapinfo.memory)) {
1319
			bufferQueue.push(sample);
1320
			gst_buffer_unmap(_buffer, &mapinfo);
1321
			bool newTexture=false;
1322
			std::unique_lock<std::mutex> lock(mutex);
1323
			while(bufferQueue.size()>2){
1324
				backBuffer = bufferQueue.front();
1325
				bufferQueue.pop();
1326
				newTexture = true;
1327
			}
1328
			if(newTexture){
1329
				GstBuffer * _buffer = gst_sample_get_buffer(backBuffer.get());
1330
				gst_buffer_map (_buffer, &mapinfo, (GstMapFlags)(GST_MAP_READ | GST_MAP_GL));
1331
				auto texId = *(guint*)mapinfo.data;
1332
				backTexture.setUseExternalTextureID(texId);
1333
				ofTextureData & texData = backTexture.getTextureData();
1334
				texData.bAllocated = true;
1335
				texData.bFlipTexture = false;
1336
				texData.glInternalFormat = GL_RGBA;
1337
				texData.height = getHeight();
1338
				texData.width = getWidth();
1339
				texData.magFilter = GL_LINEAR;
1340
				texData.minFilter = GL_LINEAR;
1341
				texData.tex_h = getHeight();
1342
				texData.tex_w = getWidth();
1343
				texData.tex_u = 1;
1344
				texData.tex_t = 1;
1345
				texData.textureID = texId;
1346
				texData.textureTarget = GL_TEXTURE_2D;
1347
				texData.wrapModeHorizontal = GL_CLAMP_TO_EDGE;
1348
				texData.wrapModeVertical = GL_CLAMP_TO_EDGE;
1349
				bBackPixelsChanged=true;
1350
				gst_buffer_unmap(_buffer,&mapinfo);
1351
			}
1352
			return GST_FLOW_OK;
1353
		}
1354
	}
1355
#endif
1356

1357
	// video frame has normal texture
1358
	gst_buffer_map (_buffer, &mapinfo, GST_MAP_READ);
1359
	guint size = mapinfo.size;
1360

1361
	size_t stride = 0;
1362
	if(pixels.isAllocated() && (pixels.getTotalBytes() != size_t(size))){
1363
		GstVideoInfo v_info = getVideoInfo(sample.get());
1364
		stride = v_info.stride[0];
1365

1366
		if(stride == (pixels.getWidth() * pixels.getBytesPerPixel())) {
1367
			ofLogError("ofGstVideoUtils") << "buffer_cb(): error on new buffer, buffer size: " << size << "!= init size: " << pixels.getTotalBytes();
1368
			return GST_FLOW_ERROR;
1369
		}
1370
	}
1371
	mutex.lock();
1372
	if(!copyPixels){
1373
		backBuffer = sample;
1374
	}
1375

1376
	if(pixels.isAllocated()){
1377
		if(stride > 0) {
1378
			if(pixels.getPixelFormat() == OF_PIXELS_I420){
1379
				GstVideoInfo v_info = getVideoInfo(sample.get());
1380
				std::vector<size_t> strides{size_t(v_info.stride[0]),size_t(v_info.stride[1]),size_t(v_info.stride[2])};
1381
				backPixels.setFromAlignedPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat(),strides);
1382
			} else {
1383
				backPixels.setFromAlignedPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat(),stride);
1384
			}
1385
		} else if(!copyPixels){
1386
			backPixels.setFromExternalPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat());
1387
			eventPixels.setFromExternalPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat());
1388
		}else{
1389
			backPixels.setFromPixels(mapinfo.data,pixels.getWidth(),pixels.getHeight(),pixels.getPixelFormat());
1390
		}
1391

1392
		bBackPixelsChanged=true;
1393
		mutex.unlock();
1394
		if(stride == 0) {
1395
			ofNotifyEvent(prerollEvent,eventPixels);
1396
		}
1397
	}else{
1398
		mutex.unlock();
1399
		if(appsink){
1400
			appsink->on_stream_prepared();
1401
		}else{
1402
			GstVideoInfo v_info = getVideoInfo(sample.get());
1403
			allocate(v_info.width,v_info.height,getOFFormat(v_info.finfo->format));
1404
		}
1405
	}
1406
	gst_buffer_unmap(_buffer, &mapinfo);
1407
	return GST_FLOW_OK;
1408
}
1409
#endif
1410

1411
#if GST_VERSION_MAJOR==0
1412
GstFlowReturn ofGstVideoUtils::preroll_cb(shared_ptr<GstBuffer> buffer){
1413
	GstFlowReturn ret = process_buffer(buffer);
1414
	if(ret==GST_FLOW_OK){
1415
		return ofGstUtils::preroll_cb(buffer);
1416
	}else{
1417
		return ret;
1418
	}
1419
}
1420
#else
1421
GstFlowReturn ofGstVideoUtils::preroll_cb(shared_ptr<GstSample> sample){
1422
	GstFlowReturn ret = process_sample(sample);
1423
	if(ret==GST_FLOW_OK){
1424
		return ofGstUtils::preroll_cb(sample);
1425
	}else{
1426
		return ret;
1427
	}
1428
}
1429
#endif
1430

1431
#if GST_VERSION_MAJOR==0
1432
GstFlowReturn ofGstVideoUtils::buffer_cb(shared_ptr<GstBuffer> buffer){
1433
	GstFlowReturn ret = process_buffer(buffer);
1434
	if(ret==GST_FLOW_OK){
1435
		return ofGstUtils::buffer_cb(buffer);
1436
	}else{
1437
		return ret;
1438
	}
1439
}
1440
#else
1441
GstFlowReturn ofGstVideoUtils::buffer_cb(shared_ptr<GstSample> sample){
1442
	GstFlowReturn ret = process_sample(sample);
1443
	if(ret==GST_FLOW_OK){
1444
		return ofGstUtils::buffer_cb(sample);
1445
	}else{
1446
		return ret;
1447
	}
1448
}
1449
#endif
1450

1451
void ofGstVideoUtils::eos_cb(){
1452
	ofGstUtils::eos_cb();
1453
	ofEventArgs args;
1454
	ofNotifyEvent(eosEvent,args);
1455
}
1456

1457
#endif
1458