embox

1
/*
2
 * @date Jan 18, 2018
3
 * @author Anton Bondarev
4
 */
5

6
#include <inttypes.h>
7
#include <linux/wait.h>
8
#include <stdio.h>
9
#include <unistd.h>
10

11
#include <etnaviv_xml/cmdstream.xml.h>
12
#include <etnaviv_xml/common.xml.h>
13
#include <etnaviv_xml/state.xml.h>
14
#include <etnaviv_xml/state_hi.xml.h>
15
#include <kernel/panic.h>
16
#include <util/log.h>
17

18
#include "etnaviv_cmdbuf.h"
19
#include "etnaviv_compat.h"
20
#include "etnaviv_drm.h"
21
#include "etnaviv_drv.h"
22
#include "etnaviv_gem.h"
23
#include "etnaviv_gpu.h"
24

25
int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, uint32_t param,
26
    uint64_t *value) {
27
	switch (param) {
28
	case ETNAVIV_PARAM_GPU_MODEL:
29
		*value = gpu->identity.model;
30
		break;
31

32
	case ETNAVIV_PARAM_GPU_REVISION:
33
		*value = gpu->identity.revision;
34
		break;
35

36
	case ETNAVIV_PARAM_GPU_FEATURES_0:
37
		*value = gpu->identity.features;
38
		break;
39

40
	case ETNAVIV_PARAM_GPU_FEATURES_1:
41
		*value = gpu->identity.minor_features0;
42
		break;
43

44
	case ETNAVIV_PARAM_GPU_FEATURES_2:
45
		*value = gpu->identity.minor_features1;
46
		break;
47

48
	case ETNAVIV_PARAM_GPU_FEATURES_3:
49
		*value = gpu->identity.minor_features2;
50
		break;
51

52
	case ETNAVIV_PARAM_GPU_FEATURES_4:
53
		*value = gpu->identity.minor_features3;
54
		break;
55

56
	case ETNAVIV_PARAM_GPU_FEATURES_5:
57
		*value = gpu->identity.minor_features4;
58
		break;
59

60
	case ETNAVIV_PARAM_GPU_FEATURES_6:
61
		*value = gpu->identity.minor_features5;
62
		break;
63

64
	case ETNAVIV_PARAM_GPU_STREAM_COUNT:
65
		*value = gpu->identity.stream_count;
66
		break;
67

68
	case ETNAVIV_PARAM_GPU_REGISTER_MAX:
69
		*value = gpu->identity.register_max;
70
		break;
71

72
	case ETNAVIV_PARAM_GPU_THREAD_COUNT:
73
		*value = gpu->identity.thread_count;
74
		break;
75

76
	case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE:
77
		*value = gpu->identity.vertex_cache_size;
78
		break;
79

80
	case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT:
81
		*value = gpu->identity.shader_core_count;
82
		break;
83

84
	case ETNAVIV_PARAM_GPU_PIXEL_PIPES:
85
		*value = gpu->identity.pixel_pipes;
86
		break;
87

88
	case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE:
89
		*value = gpu->identity.vertex_output_buffer_size;
90
		break;
91

92
	case ETNAVIV_PARAM_GPU_BUFFER_SIZE:
93
		*value = gpu->identity.buffer_size;
94
		break;
95

96
	case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT:
97
		*value = gpu->identity.instruction_count;
98
		break;
99

100
	case ETNAVIV_PARAM_GPU_NUM_CONSTANTS:
101
		*value = gpu->identity.num_constants;
102
		break;
103

104
	case ETNAVIV_PARAM_GPU_NUM_VARYINGS:
105
		*value = gpu->identity.varyings_count;
106
		break;
107

108
	default:
109
		log_debug("invalid param: %u", param);
110
		return -EINVAL;
111
	}
112

113
	return 0;
114
}
115

116
#define etnaviv_is_model_rev(gpu, mod, rev)   \
117
	((gpu)->identity.model == chipModel_##mod \
118
	    && (gpu)->identity.revision == rev)
119
#define etnaviv_field(val, field) (((val)&field##__MASK) >> field##__SHIFT)
120

121
static void etnaviv_hw_specs(struct etnaviv_gpu *gpu) {
122
	if (gpu->identity.minor_features0
123
	    & chipMinorFeatures0_MORE_MINOR_FEATURES) {
124
		uint32_t specs[4];
125
		unsigned int streams;
126

127
		specs[0] = gpu_read(gpu, VIVS_HI_CHIP_SPECS);
128
		specs[1] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_2);
129
		specs[2] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_3);
130
		specs[3] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_4);
131

132
		gpu->identity.stream_count = etnaviv_field(specs[0],
133
		    VIVS_HI_CHIP_SPECS_STREAM_COUNT);
134
		gpu->identity.register_max = etnaviv_field(specs[0],
135
		    VIVS_HI_CHIP_SPECS_REGISTER_MAX);
136
		gpu->identity.thread_count = etnaviv_field(specs[0],
137
		    VIVS_HI_CHIP_SPECS_THREAD_COUNT);
138
		gpu->identity.vertex_cache_size = etnaviv_field(specs[0],
139
		    VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE);
140
		gpu->identity.shader_core_count = etnaviv_field(specs[0],
141
		    VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT);
142
		gpu->identity.pixel_pipes = etnaviv_field(specs[0],
143
		    VIVS_HI_CHIP_SPECS_PIXEL_PIPES);
144
		gpu->identity.vertex_output_buffer_size = etnaviv_field(specs[0],
145
		    VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE);
146

147
		gpu->identity.buffer_size = etnaviv_field(specs[1],
148
		    VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE);
149
		gpu->identity.instruction_count = etnaviv_field(specs[1],
150
		    VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT);
151
		gpu->identity.num_constants = etnaviv_field(specs[1],
152
		    VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS);
153

154
		gpu->identity.varyings_count = etnaviv_field(specs[2],
155
		    VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT);
156

157
		/* This overrides the value from older register if non-zero */
158
		streams = etnaviv_field(specs[3], VIVS_HI_CHIP_SPECS_4_STREAM_COUNT);
159
		if (streams)
160
			gpu->identity.stream_count = streams;
161
	}
162

163
	/* Fill in the stream count if not specified */
164
	if (gpu->identity.stream_count == 0) {
165
		if (gpu->identity.model >= 0x1000)
166
			gpu->identity.stream_count = 4;
167
		else
168
			gpu->identity.stream_count = 1;
169
	}
170

171
	/* Convert the register max value */
172
	if (gpu->identity.register_max)
173
		gpu->identity.register_max = 1 << gpu->identity.register_max;
174
	else if (gpu->identity.model == chipModel_GC400)
175
		gpu->identity.register_max = 32;
176
	else
177
		gpu->identity.register_max = 64;
178

179
	/* Convert thread count */
180
	if (gpu->identity.thread_count)
181
		gpu->identity.thread_count = 1 << gpu->identity.thread_count;
182
	else if (gpu->identity.model == chipModel_GC400)
183
		gpu->identity.thread_count = 64;
184
	else if (gpu->identity.model == chipModel_GC500
185
	         || gpu->identity.model == chipModel_GC530)
186
		gpu->identity.thread_count = 128;
187
	else
188
		gpu->identity.thread_count = 256;
189

190
	if (gpu->identity.vertex_cache_size == 0)
191
		gpu->identity.vertex_cache_size = 8;
192

193
	if (gpu->identity.shader_core_count == 0) {
194
		if (gpu->identity.model >= 0x1000)
195
			gpu->identity.shader_core_count = 2;
196
		else
197
			gpu->identity.shader_core_count = 1;
198
	}
199

200
	if (gpu->identity.pixel_pipes == 0)
201
		gpu->identity.pixel_pipes = 1;
202

203
	/* Convert virtex buffer size */
204
	if (gpu->identity.vertex_output_buffer_size) {
205
		gpu->identity.vertex_output_buffer_size =
206
		    1 << gpu->identity.vertex_output_buffer_size;
207
	}
208
	else if (gpu->identity.model == chipModel_GC400) {
209
		if (gpu->identity.revision < 0x4000)
210
			gpu->identity.vertex_output_buffer_size = 512;
211
		else if (gpu->identity.revision < 0x4200)
212
			gpu->identity.vertex_output_buffer_size = 256;
213
		else
214
			gpu->identity.vertex_output_buffer_size = 128;
215
	}
216
	else {
217
		gpu->identity.vertex_output_buffer_size = 512;
218
	}
219

220
	switch (gpu->identity.instruction_count) {
221
	case 0:
222
		if (etnaviv_is_model_rev(gpu, GC2000, 0x5108)
223
		    || gpu->identity.model == chipModel_GC880)
224
			gpu->identity.instruction_count = 512;
225
		else
226
			gpu->identity.instruction_count = 256;
227
		break;
228

229
	case 1:
230
		gpu->identity.instruction_count = 1024;
231
		break;
232

233
	case 2:
234
		gpu->identity.instruction_count = 2048;
235
		break;
236

237
	default:
238
		gpu->identity.instruction_count = 256;
239
		break;
240
	}
241

242
	if (gpu->identity.num_constants == 0)
243
		gpu->identity.num_constants = 168;
244

245
	if (gpu->identity.varyings_count == 0) {
246
		if (gpu->identity.minor_features1 & chipMinorFeatures1_HALTI0)
247
			gpu->identity.varyings_count = 12;
248
		else
249
			gpu->identity.varyings_count = 8;
250
	}
251

252
	/*
253
	 * For some cores, two varyings are consumed for position, so the
254
	 * maximum varying count needs to be reduced by one.
255
	 */
256
	if (etnaviv_is_model_rev(gpu, GC5000, 0x5434)
257
	    || etnaviv_is_model_rev(gpu, GC4000, 0x5222)
258
	    || etnaviv_is_model_rev(gpu, GC4000, 0x5245)
259
	    || etnaviv_is_model_rev(gpu, GC4000, 0x5208)
260
	    || etnaviv_is_model_rev(gpu, GC3000, 0x5435)
261
	    || etnaviv_is_model_rev(gpu, GC2200, 0x5244)
262
	    || etnaviv_is_model_rev(gpu, GC2100, 0x5108)
263
	    || etnaviv_is_model_rev(gpu, GC2000, 0x5108)
264
	    || etnaviv_is_model_rev(gpu, GC1500, 0x5246)
265
	    || etnaviv_is_model_rev(gpu, GC880, 0x5107)
266
	    || etnaviv_is_model_rev(gpu, GC880, 0x5106))
267
		gpu->identity.varyings_count -= 1;
268
}
269

270
static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) {
271
	uint32_t chipIdentity;
272

273
	chipIdentity = gpu_read(gpu, VIVS_HI_CHIP_IDENTITY);
274

275
	/* Special case for older graphic cores. */
276
	if (etnaviv_field(chipIdentity, VIVS_HI_CHIP_IDENTITY_FAMILY) == 0x01) {
277
		gpu->identity.model = chipModel_GC500;
278
		gpu->identity.revision = etnaviv_field(chipIdentity,
279
		    VIVS_HI_CHIP_IDENTITY_REVISION);
280
	}
281
	else {
282
		gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL);
283
		gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV);
284

285
		/*
286
		 * !!!! HACK ALERT !!!!
287
		 * Because people change device IDs without letting software
288
		 * know about it - here is the hack to make it all look the
289
		 * same.  Only for GC400 family.
290
		 */
291
		if ((gpu->identity.model & 0xff00) == 0x0400
292
		    && gpu->identity.model != chipModel_GC420) {
293
			gpu->identity.model = gpu->identity.model & 0x0400;
294
		}
295

296
		/* Another special case */
297
		if (etnaviv_is_model_rev(gpu, GC300, 0x2201)) {
298
			uint32_t chipDate = gpu_read(gpu, VIVS_HI_CHIP_DATE);
299
			uint32_t chipTime = gpu_read(gpu, VIVS_HI_CHIP_TIME);
300

301
			if (chipDate == 0x20080814 && chipTime == 0x12051100) {
302
				/*
303
				 * This IP has an ECO; put the correct
304
				 * revision in it.
305
				 */
306
				gpu->identity.revision = 0x1051;
307
			}
308
		}
309

310
		/*
311
		 * NXP likes to call the GPU on the i.MX6QP GC2000+, but in
312
		 * reality it's just a re-branded GC3000. We can identify this
313
		 * core by the upper half of the revision register being all 1.
314
		 * Fix model/rev here, so all other places can refer to this
315
		 * core by its real identity.
316
		 */
317
		if (etnaviv_is_model_rev(gpu, GC2000, 0xffff5450)) {
318
			gpu->identity.model = chipModel_GC3000;
319
			gpu->identity.revision &= 0xffff;
320
		}
321
	}
322

323
	log_info("model: GC%" PRIu32 ", revision: %" PRIu32, gpu->identity.model,
324
	    gpu->identity.revision);
325

326
	gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE);
327

328
	/* Disable fast clear on GC700. */
329
	if (gpu->identity.model == chipModel_GC700)
330
		gpu->identity.features &= ~chipFeatures_FAST_CLEAR;
331

332
	if ((gpu->identity.model == chipModel_GC500 && gpu->identity.revision < 2)
333
	    || (gpu->identity.model == chipModel_GC300
334
	        && gpu->identity.revision < 0x2000)) {
335
		/*
336
		 * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these
337
		 * registers.
338
		 */
339
		gpu->identity.minor_features0 = 0;
340
		gpu->identity.minor_features1 = 0;
341
		gpu->identity.minor_features2 = 0;
342
		gpu->identity.minor_features3 = 0;
343
		gpu->identity.minor_features4 = 0;
344
		gpu->identity.minor_features5 = 0;
345
	}
346
	else
347
		gpu->identity.minor_features0 = gpu_read(gpu,
348
		    VIVS_HI_CHIP_MINOR_FEATURE_0);
349

350
	if (gpu->identity.minor_features0
351
	    & chipMinorFeatures0_MORE_MINOR_FEATURES) {
352
		gpu->identity.minor_features1 = gpu_read(gpu,
353
		    VIVS_HI_CHIP_MINOR_FEATURE_1);
354
		gpu->identity.minor_features2 = gpu_read(gpu,
355
		    VIVS_HI_CHIP_MINOR_FEATURE_2);
356
		gpu->identity.minor_features3 = gpu_read(gpu,
357
		    VIVS_HI_CHIP_MINOR_FEATURE_3);
358
		gpu->identity.minor_features4 = gpu_read(gpu,
359
		    VIVS_HI_CHIP_MINOR_FEATURE_4);
360
		gpu->identity.minor_features5 = gpu_read(gpu,
361
		    VIVS_HI_CHIP_MINOR_FEATURE_5);
362
	}
363

364
	/* GC600 idle register reports zero bits where modules aren't present */
365
	if (gpu->identity.model == chipModel_GC600) {
366
		gpu->idle_mask = VIVS_HI_IDLE_STATE_TX | VIVS_HI_IDLE_STATE_RA
367
		                 | VIVS_HI_IDLE_STATE_SE | VIVS_HI_IDLE_STATE_PA
368
		                 | VIVS_HI_IDLE_STATE_SH | VIVS_HI_IDLE_STATE_PE
369
		                 | VIVS_HI_IDLE_STATE_DE | VIVS_HI_IDLE_STATE_FE;
370
	}
371
	else {
372
		gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP;
373
	}
374

375
	etnaviv_hw_specs(gpu);
376
}
377

378
static void etnaviv_gpu_load_clock(struct etnaviv_gpu *gpu, uint32_t clock) {
379
	gpu_write(gpu, VIVS_HI_CLOCK_CONTROL,
380
	    clock | VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD);
381
	gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock);
382
}
383

384
static void etnaviv_gpu_update_clock(struct etnaviv_gpu *gpu) {
385
	unsigned int fscale = 1 << (6 - 1); //gpu->freq_scale);
386
	uint32_t clock = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS
387
	                 | VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(fscale);
388

389
	etnaviv_gpu_load_clock(gpu, clock);
390
}
391

392
static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) {
393
	uint32_t control, idle;
394
	bool failed = true;
395

396
	/* TODO
397
	 *
398
	 * - clock gating
399
	 * - puls eater
400
	 * - what about VG?
401
	 */
402

403
	/* We hope that the GPU resets in under one second */
404
	while (1) {
405
		/* enable clock */
406
		etnaviv_gpu_update_clock(gpu);
407

408
		control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL);
409

410
		/* Wait for stable clock.  Vivante's code waited for 1ms */
411
		usleep(1500);
412

413
		/* isolate the GPU. */
414
		control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU;
415
		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
416

417
		/* set soft reset. */
418
		control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET;
419
		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
420

421
		/* wait for reset. */
422
		usleep(1500);
423

424
		/* reset soft reset bit. */
425
		control &= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET;
426
		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
427

428
		/* reset GPU isolation. */
429
		control &= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU;
430
		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
431

432
		/* read idle register. */
433
		idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
434

435
		/* try reseting again if FE it not idle */
436
		if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) {
437
			log_debug("FE is not idle");
438
			continue;
439
		}
440

441
		/* read reset register. */
442
		control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL);
443

444
		/* is the GPU idle? */
445
		if (((control & VIVS_HI_CLOCK_CONTROL_IDLE_3D) == 0)
446
		    || ((control & VIVS_HI_CLOCK_CONTROL_IDLE_2D) == 0)) {
447
			log_debug("GPU is not idle");
448
			continue;
449
		}
450

451
		failed = false;
452
		break;
453
	}
454

455
	if (failed) {
456
		idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
457
		control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL);
458

459
		log_error("GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle",
460
		    idle & VIVS_HI_IDLE_STATE_FE ? "" : "not ",
461
		    control & VIVS_HI_CLOCK_CONTROL_IDLE_3D ? "" : "not ",
462
		    control & VIVS_HI_CLOCK_CONTROL_IDLE_2D ? "" : "not ");
463

464
		return -EBUSY;
465
	}
466

467
	/* We rely on the GPU running, so program the clock */
468
	etnaviv_gpu_update_clock(gpu);
469

470
	return 0;
471
}
472

473
static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) {
474
	uint32_t pmc, ppc;
475

476
	/* enable clock gating */
477
	ppc = gpu_read(gpu, VIVS_PM_POWER_CONTROLS);
478
	ppc |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING;
479

480
	/* Disable stall module clock gating for 4.3.0.1 and 4.3.0.2 revs */
481
	if (gpu->identity.revision == 0x4301 || gpu->identity.revision == 0x4302)
482
		ppc |= VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING;
483

484
	gpu_write(gpu, VIVS_PM_POWER_CONTROLS, ppc);
485

486
	pmc = gpu_read(gpu, VIVS_PM_MODULE_CONTROLS);
487

488
	/* Disable PA clock gating for GC400+ except for GC420 */
489
	if (gpu->identity.model >= chipModel_GC400
490
	    && gpu->identity.model != chipModel_GC420)
491
		pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PA;
492

493
	/*
494
	 * Disable PE clock gating on revs < 5.0.0.0 when HZ is
495
	 * present without a bug fix.
496
	 */
497
	if (gpu->identity.revision < 0x5000
498
	    && gpu->identity.minor_features0 & chipMinorFeatures0_HZ
499
	    && !(gpu->identity.minor_features1
500
	         & chipMinorFeatures1_DISABLE_PE_GATING))
501
		pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE;
502

503
	if (gpu->identity.revision < 0x5422)
504
		pmc |= BIT(15); /* Unknown bit */
505

506
	pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ;
507
	pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ;
508

509
	gpu_write(gpu, VIVS_PM_MODULE_CONTROLS, pmc);
510
}
511

512
void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, uint32_t address,
513
    uint16_t prefetch) {
514
	gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS, address);
515
	gpu_write(gpu, VIVS_FE_COMMAND_CONTROL,
516
	    VIVS_FE_COMMAND_CONTROL_ENABLE
517
	        | VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch));
518
}
519

520
static void etnaviv_gpu_setup_pulse_eater(struct etnaviv_gpu *gpu) {
521
	/*
522
	 * Base value for VIVS_PM_PULSE_EATER register on models where it
523
	 * cannot be read, extracted from vivante kernel driver.
524
	 */
525
	uint32_t pulse_eater = 0x01590880;
526

527
	if (etnaviv_is_model_rev(gpu, GC4000, 0x5208)
528
	    || etnaviv_is_model_rev(gpu, GC4000, 0x5222)) {
529
		pulse_eater |= BIT(23);
530
	}
531

532
	if (etnaviv_is_model_rev(gpu, GC1000, 0x5039)
533
	    || etnaviv_is_model_rev(gpu, GC1000, 0x5040)) {
534
		pulse_eater &= ~BIT(16);
535
		pulse_eater |= BIT(17);
536
	}
537
	if ((gpu->identity.revision > 0x5420)
538
	    && (gpu->identity.features & chipFeatures_PIPE_3D)) {
539
		/* Performance fix: disable internal DFS */
540
		pulse_eater = gpu_read(gpu, VIVS_PM_PULSE_EATER);
541
		pulse_eater |= BIT(18);
542
	}
543

544
	gpu_write(gpu, VIVS_PM_PULSE_EATER, pulse_eater);
545
}
546

547
static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) {
548
	uint16_t prefetch;
549

550
	/* enable module-level clock gating */
551
	etnaviv_gpu_enable_mlcg(gpu);
552

553
	/*
554
	 * Update GPU AXI cache atttribute to "cacheable, no allocate".
555
	 * This is necessary to prevent the iMX6 SoC locking up.
556
	 */
557
	gpu_write(gpu, VIVS_HI_AXI_CONFIG,
558
	    VIVS_HI_AXI_CONFIG_AWCACHE(2) | VIVS_HI_AXI_CONFIG_ARCACHE(2));
559

560
	/* GC2000 rev 5108 needs a special bus config */
561
	if (etnaviv_is_model_rev(gpu, GC2000, 0x5108)) {
562
		uint32_t bus_config = gpu_read(gpu, VIVS_MC_BUS_CONFIG);
563
		bus_config &= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK
564
		                | VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK);
565
		bus_config |= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1)
566
		              | VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0);
567
		gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config);
568
	}
569

570
	/* setup the pulse eater */
571
	etnaviv_gpu_setup_pulse_eater(gpu);
572

573
	/* setup the MMU */
574
	etnaviv_iommu_restore(gpu);
575

576
	/* Start command processor */
577
	gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U);
578

579
	prefetch = etnaviv_buffer_init(gpu);
580

581
	etnaviv_gpu_start_fe(gpu, etnaviv_cmdbuf_get_va(gpu->buffer), prefetch);
582
}
583

584
int etnaviv_gpu_init(struct etnaviv_gpu *gpu) {
585
	etnaviv_hw_identify(gpu);
586

587
	if (gpu->identity.model == 0) {
588
		log_error("Unknown GPU model");
589
		return -ENXIO;
590
	}
591

592
	/* Exclude VG cores with FE2.0 */
593
	if (gpu->identity.features & chipFeatures_PIPE_VG
594
	    && gpu->identity.features & chipFeatures_FE20) {
595
		panic("Wrong GPU register values, try to restart");
596
		return -ENXIO;
597
	}
598

599
	/*
600
	 * Set the GPU linear window to be at the end of the DMA window, where
601
	 * the CMA area is likely to reside. This ensures that we are able to
602
	 * map the command buffers while having the linear window overlap as
603
	 * much RAM as possible, so we can optimize mappings for other buffers.
604
	 *
605
	 * For 3D cores only do this if MC2.0 is present, as with MC1.0 it leads
606
	 * to different views of the memory on the individual engines.
607
	 */
608
	if (!(gpu->identity.features & chipFeatures_PIPE_3D)
609
	    || (gpu->identity.minor_features0 & chipMinorFeatures0_MC20)) {
610
		uint32_t dma_mask = (uint32_t)dma_get_required_mask(gpu);
611
		if (dma_mask < PHYS_OFFSET + SZ_2G)
612
			gpu->memory_base = PHYS_OFFSET;
613
		else
614
			gpu->memory_base = dma_mask - SZ_2G + 1;
615
	}
616
	else if (PHYS_OFFSET >= SZ_2G) {
617
		log_info("Need to move linear window on MC1.0, disabling TS");
618
		gpu->memory_base = PHYS_OFFSET;
619
		gpu->identity.features &= ~chipFeatures_FAST_CLEAR;
620
	}
621

622
	log_debug("memory base %p\n", (void *)gpu->memory_base);
623

624
	if (etnaviv_hw_reset(gpu)) {
625
		log_error("GPU reset failed");
626
		return -1;
627
	}
628

629
	if (etnaviv_iommu_init(gpu)) {
630
		log_error("Failed to instantiate GPU IOMMU");
631
		return -1;
632
	}
633

634
	gpu->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(gpu);
635
	if (IS_ERR(gpu->cmdbuf_suballoc)) {
636
		log_error("Failed to create cmdbuf suballocator\n");
637
		return -1;
638
	}
639
	/* Create buffer: */
640
	gpu->buffer = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc, 4096, 0);
641
	if (!gpu->buffer) {
642
		log_error("could not create command buffer");
643
		return -ENOMEM;
644
	}
645

646
	if (gpu->mmu.version == ETNAVIV_IOMMU_V1
647
	    && etnaviv_cmdbuf_get_va(gpu->buffer) > 0x80000000) {
648
		log_debug("buffer %p va %p\n", gpu->buffer,
649
		    etnaviv_cmdbuf_get_va(gpu->buffer));
650
		log_error("command buffer outside valid memory window");
651
		return -EINVAL;
652
	}
653

654
	etnaviv_gpu_hw_init(gpu);
655

656
	return 0;
657
}
658

659
struct dma_debug {
660
	uint32_t address[2];
661
	uint32_t state[2];
662
};
663

664
static void verify_dma(struct etnaviv_gpu *gpu, struct dma_debug *debug) {
665
	uint32_t i;
666

667
	debug->address[0] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
668
	debug->state[0] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE);
669

670
	for (i = 0; i < 500; i++) {
671
		debug->address[1] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
672
		debug->state[1] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE);
673

674
		if (debug->address[0] != debug->address[1])
675
			break;
676

677
		if (debug->state[0] != debug->state[1])
678
			break;
679
	}
680
}
681

682
#define seq_puts(a, ...)   printf(__VA_ARGS__)
683
#define seq_printf(a, ...) printf(__VA_ARGS__)
684
struct seq_file;
685
int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, char *s) {
686
	struct dma_debug debug;
687
	uint32_t dma_lo, dma_hi, axi, idle;
688
	int ret;
689
	int cmdState, cmdDmaState, cmdFetState, dmaReqState, calState, veReqState;
690

691
	static const char *_cmdState[] = {"PAR_IDLE_ST", "PAR_DEC_ST",
692
	    "PAR_ADR0_ST", "PAR_LOAD0_ST", "PAR_ADR1_ST", "PAR_LOAD1_ST",
693
	    "PAR_3DADR_ST", "PAR_3DCMD_ST", "PAR_3DCNTL_ST", "PAR_3DIDXCNTL_ST",
694
	    "PAR_INITREQDMA_ST", "PAR_DRAWIDX_ST", "PAR_DRAW_ST", "PAR_2DRECT0_ST",
695
	    "PAR_2DRECT1_ST", "PAR_2DDATA0_ST", "PAR_2DDATA1_ST", "PAR_WAITFIFO_ST",
696
	    "PAR_WAIT_ST", "PAR_LINK_ST", "PAR_END_ST", "PAR_STALL_ST"};
697

698
	static const char *_cmdDmaState[] = {"CMD_IDLE_ST", "CMD_START_ST",
699
	    "CMD_REQ_ST", "CMD_END_ST"};
700

701
	static const char *_cmdFetState[] = {"FET_IDLE_ST", "FET_RAMVALID_ST",
702
	    "FET_VALID_ST"};
703

704
	static const char *_reqDmaState[] = {"REQ_IDLE_ST", "REQ_WAITIDX_ST",
705
	    "REQ_CAL_ST"};
706

707
	static const char *_calState[] = {"CAL_IDLE_ST", "CAL_LDADR_ST",
708
	    "CAL_IDXCALC_ST"};
709

710
	static const char *_veReqState[] = {"VER_IDLE_ST", "VER_CKCACHE_ST",
711
	    "VER_MISS_ST"};
712

713
	seq_printf(m, "%s Status:\n", s);
714

715
	dma_lo = gpu_read(gpu, VIVS_FE_DMA_LOW);
716
	dma_hi = gpu_read(gpu, VIVS_FE_DMA_HIGH);
717
	axi = gpu_read(gpu, VIVS_HI_AXI_STATUS);
718
	idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
719

720
	verify_dma(gpu, &debug);
721

722
	seq_puts(m, "\tfeatures\n");
723
	seq_printf(m, "\t minor_features0: 0x%08" PRIu32 "\n",
724
	    gpu->identity.minor_features0);
725
	seq_printf(m, "\t minor_features1: 0x%08" PRIu32 "\n",
726
	    gpu->identity.minor_features1);
727
	seq_printf(m, "\t minor_features2: 0x%08" PRIu32 "\n",
728
	    gpu->identity.minor_features2);
729
	seq_printf(m, "\t minor_features3: 0x%08" PRIu32 "\n",
730
	    gpu->identity.minor_features3);
731
	seq_printf(m, "\t minor_features4: 0x%08" PRIu32 "\n",
732
	    gpu->identity.minor_features4);
733
	seq_printf(m, "\t minor_features5: 0x%08" PRIu32 "\n",
734
	    gpu->identity.minor_features5);
735

736
	seq_puts(m, "\tspecs\n");
737
	seq_printf(m, "\t stream_count:  %" PRIu32 "\n",
738
	    gpu->identity.stream_count);
739
	seq_printf(m, "\t register_max: %" PRIu32 "\n", gpu->identity.register_max);
740
	seq_printf(m, "\t thread_count: %" PRIu32 "\n", gpu->identity.thread_count);
741
	seq_printf(m, "\t vertex_cache_size: %" PRIu32 "\n",
742
	    gpu->identity.vertex_cache_size);
743
	seq_printf(m, "\t shader_core_count: %" PRIu32 "\n",
744
	    gpu->identity.shader_core_count);
745
	seq_printf(m, "\t pixel_pipes: %" PRIu32 "\n", gpu->identity.pixel_pipes);
746
	seq_printf(m, "\t vertex_output_buffer_size: %" PRIu32 "\n",
747
	    gpu->identity.vertex_output_buffer_size);
748
	seq_printf(m, "\t buffer_size: %" PRIu32 "\n", gpu->identity.buffer_size);
749
	seq_printf(m, "\t instruction_count: %" PRIu32 "\n",
750
	    gpu->identity.instruction_count);
751
	seq_printf(m, "\t num_constants: %" PRIu32 "\n",
752
	    gpu->identity.num_constants);
753
	seq_printf(m, "\t varyings_count: %" PRIu8 "\n",
754
	    gpu->identity.varyings_count);
755

756
	seq_printf(m, "\taxi: 0x%08" PRIu32 "\n", axi);
757
	seq_printf(m, "\tidle: 0x%08" PRIu32 "\n", idle);
758
	idle |= ~gpu->idle_mask & ~VIVS_HI_IDLE_STATE_AXI_LP;
759
	if ((idle & VIVS_HI_IDLE_STATE_FE) == 0)
760
		seq_puts(m, "\t FE is not idle\n");
761
	if ((idle & VIVS_HI_IDLE_STATE_DE) == 0)
762
		seq_puts(m, "\t DE is not idle\n");
763
	if ((idle & VIVS_HI_IDLE_STATE_PE) == 0)
764
		seq_puts(m, "\t PE is not idle\n");
765
	if ((idle & VIVS_HI_IDLE_STATE_SH) == 0)
766
		seq_puts(m, "\t SH is not idle\n");
767
	if ((idle & VIVS_HI_IDLE_STATE_PA) == 0)
768
		seq_puts(m, "\t PA is not idle\n");
769
	if ((idle & VIVS_HI_IDLE_STATE_SE) == 0)
770
		seq_puts(m, "\t SE is not idle\n");
771
	if ((idle & VIVS_HI_IDLE_STATE_RA) == 0)
772
		seq_puts(m, "\t RA is not idle\n");
773
	if ((idle & VIVS_HI_IDLE_STATE_TX) == 0)
774
		seq_puts(m, "\t TX is not idle\n");
775
	if ((idle & VIVS_HI_IDLE_STATE_VG) == 0)
776
		seq_puts(m, "\t VG is not idle\n");
777
	if ((idle & VIVS_HI_IDLE_STATE_IM) == 0)
778
		seq_puts(m, "\t IM is not idle\n");
779
	if ((idle & VIVS_HI_IDLE_STATE_FP) == 0)
780
		seq_puts(m, "\t FP is not idle\n");
781
	if ((idle & VIVS_HI_IDLE_STATE_TS) == 0)
782
		seq_puts(m, "\t TS is not idle\n");
783
	if (idle & VIVS_HI_IDLE_STATE_AXI_LP)
784
		seq_puts(m, "\t AXI low power mode\n");
785

786
	if (gpu->identity.features & chipFeatures_DEBUG_MODE) {
787
		uint32_t read0 = gpu_read(gpu, VIVS_MC_DEBUG_READ0);
788
		uint32_t read1 = gpu_read(gpu, VIVS_MC_DEBUG_READ1);
789
		uint32_t write = gpu_read(gpu, VIVS_MC_DEBUG_WRITE);
790

791
		seq_puts(m, "\tMC\n");
792
		seq_printf(m, "\t read0: 0x%08" PRIu32 "\n", read0);
793
		seq_printf(m, "\t read1: 0x%08" PRIu32 "\n", read1);
794
		seq_printf(m, "\t write: 0x%08" PRIu32 "\n", write);
795
	}
796

797
	seq_puts(m, "\tDMA ");
798

799
	if (debug.address[0] == debug.address[1]
800
	    && debug.state[0] == debug.state[1]) {
801
		seq_puts(m, "seems to be stuck\n");
802
	}
803
	else if (debug.address[0] == debug.address[1]) {
804
		seq_puts(m, "address is constant\n");
805
	}
806
	else {
807
		seq_puts(m, "is running\n");
808
	}
809

810
	cmdState = debug.state[1] & 0x1F;
811
	cmdDmaState = (debug.state[1] >> 8) & 0x03;
812
	cmdFetState = (debug.state[1] >> 10) & 0x03;
813
	dmaReqState = (debug.state[1] >> 12) & 0x03;
814
	calState = (debug.state[1] >> 14) & 0x03;
815
	veReqState = (debug.state[1] >> 16) & 0x03;
816

817
	seq_printf(m, "\t address 0: 0x%08" PRIu32 "\n", debug.address[0]);
818
	seq_printf(m, "\t address 1: 0x%08" PRIu32 "\n", debug.address[1]);
819
	seq_printf(m, "\t state 0: 0x%08" PRIu32 "\n", debug.state[0]);
820
	seq_printf(m, "\t state 1: 0x%08" PRIu32 "\n", debug.state[1]);
821
	seq_printf(m, "\t    command state       = %d (%s)\n", cmdState,
822
	    _cmdState[cmdState]);
823
	seq_printf(m, "\t    command DMA state   = %d (%s)\n", cmdDmaState,
824
	    _cmdDmaState[cmdDmaState]);
825
	seq_printf(m, "\t    command fetch state = %d (%s)\n", cmdFetState,
826
	    _cmdFetState[cmdFetState]);
827
	seq_printf(m, "\t    DMA request state   = %d (%s)\n", dmaReqState,
828
	    _reqDmaState[dmaReqState]);
829
	seq_printf(m, "\t    cal state           = %d (%s)\n", calState,
830
	    _calState[calState]);
831
	seq_printf(m, "\t    VE request state    = %d (%s)\n", veReqState,
832
	    _veReqState[veReqState]);
833
	seq_printf(m,
834
	    "\t last fetch 64 bit word: 0x%08" PRIu32 " 0x%08" PRIu32 "\n", dma_lo,
835
	    dma_hi);
836

837
	ret = 0;
838

839
	return ret;
840
}
841

842
int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu,
843
    uint32_t fence, struct timespec *timeout) {
844
	int ret;
845

846
	if (fence_after(fence, gpu->next_fence)) {
847
		log_error("waiting on invalid fence: %u (of %u)\n", fence,
848
		    gpu->next_fence);
849
		return -EINVAL;
850
	}
851

852
	if (!timeout) {
853
		/* No timeout was requested: just test for completion */
854
		ret = fence_completed(gpu, fence) ? 0 : -EBUSY;
855
	}
856
	else {
857
		unsigned long remaining = etnaviv_timeout_to_jiffies(timeout);
858

859
		ret = wait_event_interruptible_timeout(gpu->fence_event,
860
		    fence_completed(gpu, fence), remaining);
861
		if (ret == 0) {
862
			log_debug("timeout waiting for fence: %u (retired: %u completed: "
863
			          "%u)",
864
			    fence, gpu->retired_fence, gpu->completed_fence);
865
			ret = -ETIMEDOUT;
866
		}
867
	}
868

869
	return ret;
870
}
871
/* add bo's to gpu's ring, and kick gpu: */
872
extern void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,
873
    struct etnaviv_cmdbuf *buf, uint32_t off, uint32_t len);
874

875
int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
876
    struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf) {
877
	unsigned int event = 0;
878
	/*
879
	 * TODO
880
	 * - flush
881
	 * - data endian
882
	 * - prefetch
883
	 */
884
	if (gpu->lastctx != cmdbuf->ctx) {
885
		gpu->mmu.need_flush = true;
886
		gpu->switch_context = true;
887
		gpu->lastctx = cmdbuf->ctx;
888
	}
889

890
	while (gpu->busy) {}; /* Wait for interrupt for previous command buffer */
891
	gpu->busy = 1;
892

893
	etnaviv_buffer_queue(gpu, event, cmdbuf);
894
	cmdbuf->nr_bos = submit->nr_bos;
895
	etnaviv_buffer_dump(gpu, cmdbuf, 0, cmdbuf->user_size);
896
	etnaviv_buffer_dump(gpu, gpu->buffer, 0, gpu->buffer->user_size);
897

898
	return 0;
899
}
900

901
int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms) {
902
	timeout_ms *= 0xfff;
903
	do {
904
		uint32_t idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
905

906
		if ((idle & gpu->idle_mask) == gpu->idle_mask)
907
			return 0;
908

909
		timeout_ms--;
910

911
		if (timeout_ms == 0) {
912
			log_warning("timed out waiting for idle: idle=0x%x", idle);
913
			return -ETIMEDOUT;
914
		}
915
	} while (1);
916
}
917