qemu

1
/*
2
 * Tiny Code Generator for QEMU
3
 *
4
 * Copyright (c) 2018 Linaro, Inc.
5
 *
6
 * This library is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU Lesser General Public
8
 * License as published by the Free Software Foundation; either
9
 * version 2.1 of the License, or (at your option) any later version.
10
 *
11
 * This library is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
 * Lesser General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18
 */
19

20
#include "qemu/osdep.h"
21
#include "tcg/tcg.h"
22
#include "tcg/tcg-temp-internal.h"
23
#include "tcg/tcg-op-common.h"
24
#include "tcg/tcg-mo.h"
25
#include "tcg-internal.h"
26

27
/*
28
 * Vector optional opcode tracking.
29
 * Except for the basic logical operations (and, or, xor), and
30
 * data movement (mov, ld, st, dupi), many vector opcodes are
31
 * optional and may not be supported on the host.  Thank Intel
32
 * for the irregularity in their instruction set.
33
 *
34
 * The gvec expanders allow custom vector operations to be composed,
35
 * generally via the .fniv callback in the GVecGen* structures.  At
36
 * the same time, in deciding whether to use this hook we need to
37
 * know if the host supports the required operations.  This is
38
 * presented as an array of opcodes, terminated by 0.  Each opcode
39
 * is assumed to be expanded with the given VECE.
40
 *
41
 * For debugging, we want to validate this array.  Therefore, when
42
 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
43
 * will validate that their opcode is present in the list.
44
 */
45
static void tcg_assert_listed_vecop(TCGOpcode op)
46
{
47
#ifdef CONFIG_DEBUG_TCG
48
    const TCGOpcode *p = tcg_ctx->vecop_list;
49
    if (p) {
50
        for (; *p; ++p) {
51
            if (*p == op) {
52
                return;
53
            }
54
        }
55
        g_assert_not_reached();
56
    }
57
#endif
58
}
59

60
bool tcg_can_emit_vecop_list(const TCGOpcode *list,
61
                             TCGType type, unsigned vece)
62
{
63
    if (list == NULL) {
64
        return true;
65
    }
66

67
    for (; *list; ++list) {
68
        TCGOpcode opc = *list;
69

70
#ifdef CONFIG_DEBUG_TCG
71
        switch (opc) {
72
        case INDEX_op_and_vec:
73
        case INDEX_op_or_vec:
74
        case INDEX_op_xor_vec:
75
        case INDEX_op_mov_vec:
76
        case INDEX_op_dup_vec:
77
        case INDEX_op_dup2_vec:
78
        case INDEX_op_ld_vec:
79
        case INDEX_op_st_vec:
80
        case INDEX_op_bitsel_vec:
81
            /* These opcodes are mandatory and should not be listed.  */
82
            g_assert_not_reached();
83
        case INDEX_op_not_vec:
84
            /* These opcodes have generic expansions using the above.  */
85
            g_assert_not_reached();
86
        default:
87
            break;
88
        }
89
#endif
90

91
        if (tcg_can_emit_vec_op(opc, type, vece)) {
92
            continue;
93
        }
94

95
        /*
96
         * The opcode list is created by front ends based on what they
97
         * actually invoke.  We must mirror the logic in the routines
98
         * below for generic expansions using other opcodes.
99
         */
100
        switch (opc) {
101
        case INDEX_op_neg_vec:
102
            if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
103
                continue;
104
            }
105
            break;
106
        case INDEX_op_abs_vec:
107
            if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
108
                && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
109
                    || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
110
                    || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
111
                continue;
112
            }
113
            break;
114
        case INDEX_op_usadd_vec:
115
            if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
116
                tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
117
                continue;
118
            }
119
            break;
120
        case INDEX_op_ussub_vec:
121
            if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
122
                tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
123
                continue;
124
            }
125
            break;
126
        case INDEX_op_cmpsel_vec:
127
        case INDEX_op_smin_vec:
128
        case INDEX_op_smax_vec:
129
        case INDEX_op_umin_vec:
130
        case INDEX_op_umax_vec:
131
            if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
132
                continue;
133
            }
134
            break;
135
        default:
136
            break;
137
        }
138
        return false;
139
    }
140
    return true;
141
}
142

143
void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
144
{
145
    TCGOp *op = tcg_emit_op(opc, 2);
146
    TCGOP_VECL(op) = type - TCG_TYPE_V64;
147
    TCGOP_VECE(op) = vece;
148
    op->args[0] = r;
149
    op->args[1] = a;
150
}
151

152
void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
153
               TCGArg r, TCGArg a, TCGArg b)
154
{
155
    TCGOp *op = tcg_emit_op(opc, 3);
156
    TCGOP_VECL(op) = type - TCG_TYPE_V64;
157
    TCGOP_VECE(op) = vece;
158
    op->args[0] = r;
159
    op->args[1] = a;
160
    op->args[2] = b;
161
}
162

163
void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
164
               TCGArg r, TCGArg a, TCGArg b, TCGArg c)
165
{
166
    TCGOp *op = tcg_emit_op(opc, 4);
167
    TCGOP_VECL(op) = type - TCG_TYPE_V64;
168
    TCGOP_VECE(op) = vece;
169
    op->args[0] = r;
170
    op->args[1] = a;
171
    op->args[2] = b;
172
    op->args[3] = c;
173
}
174

175
static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
176
                      TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
177
{
178
    TCGOp *op = tcg_emit_op(opc, 6);
179
    TCGOP_VECL(op) = type - TCG_TYPE_V64;
180
    TCGOP_VECE(op) = vece;
181
    op->args[0] = r;
182
    op->args[1] = a;
183
    op->args[2] = b;
184
    op->args[3] = c;
185
    op->args[4] = d;
186
    op->args[5] = e;
187
}
188

189
static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
190
{
191
    TCGTemp *rt = tcgv_vec_temp(r);
192
    TCGTemp *at = tcgv_vec_temp(a);
193
    TCGType type = rt->base_type;
194

195
    /* Must enough inputs for the output.  */
196
    tcg_debug_assert(at->base_type >= type);
197
    vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
198
}
199

200
static void vec_gen_op3(TCGOpcode opc, unsigned vece,
201
                        TCGv_vec r, TCGv_vec a, TCGv_vec b)
202
{
203
    TCGTemp *rt = tcgv_vec_temp(r);
204
    TCGTemp *at = tcgv_vec_temp(a);
205
    TCGTemp *bt = tcgv_vec_temp(b);
206
    TCGType type = rt->base_type;
207

208
    /* Must enough inputs for the output.  */
209
    tcg_debug_assert(at->base_type >= type);
210
    tcg_debug_assert(bt->base_type >= type);
211
    vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
212
}
213

214
void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
215
{
216
    if (r != a) {
217
        vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
218
    }
219
}
220

221
void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
222
{
223
    TCGTemp *rt = tcgv_vec_temp(r);
224
    tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
225
}
226

227
void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
228
{
229
    TCGArg ri = tcgv_vec_arg(r);
230
    TCGTemp *rt = arg_temp(ri);
231
    TCGType type = rt->base_type;
232

233
    if (TCG_TARGET_REG_BITS == 64) {
234
        TCGArg ai = tcgv_i64_arg(a);
235
        vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
236
    } else if (vece == MO_64) {
237
        TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
238
        TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
239
        vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
240
    } else {
241
        TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
242
        vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
243
    }
244
}
245

246
void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
247
{
248
    TCGArg ri = tcgv_vec_arg(r);
249
    TCGArg ai = tcgv_i32_arg(a);
250
    TCGTemp *rt = arg_temp(ri);
251
    TCGType type = rt->base_type;
252

253
    vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
254
}
255

256
void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
257
                         tcg_target_long ofs)
258
{
259
    TCGArg ri = tcgv_vec_arg(r);
260
    TCGArg bi = tcgv_ptr_arg(b);
261
    TCGTemp *rt = arg_temp(ri);
262
    TCGType type = rt->base_type;
263

264
    vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
265
}
266

267
static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
268
{
269
    TCGArg ri = tcgv_vec_arg(r);
270
    TCGArg bi = tcgv_ptr_arg(b);
271
    TCGTemp *rt = arg_temp(ri);
272
    TCGType type = rt->base_type;
273

274
    vec_gen_3(opc, type, 0, ri, bi, o);
275
}
276

277
void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
278
{
279
    vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
280
}
281

282
void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
283
{
284
    vec_gen_ldst(INDEX_op_st_vec, r, b, o);
285
}
286

287
void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
288
{
289
    TCGArg ri = tcgv_vec_arg(r);
290
    TCGArg bi = tcgv_ptr_arg(b);
291
    TCGTemp *rt = arg_temp(ri);
292
    TCGType type = rt->base_type;
293

294
    tcg_debug_assert(low_type >= TCG_TYPE_V64);
295
    tcg_debug_assert(low_type <= type);
296
    vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
297
}
298

299
void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
300
{
301
    vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
302
}
303

304
void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
305
{
306
    vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
307
}
308

309
void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
310
{
311
    vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
312
}
313

314
void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
315
{
316
    if (TCG_TARGET_HAS_andc_vec) {
317
        vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
318
    } else {
319
        TCGv_vec t = tcg_temp_new_vec_matching(r);
320
        tcg_gen_not_vec(0, t, b);
321
        tcg_gen_and_vec(0, r, a, t);
322
        tcg_temp_free_vec(t);
323
    }
324
}
325

326
void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
327
{
328
    if (TCG_TARGET_HAS_orc_vec) {
329
        vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
330
    } else {
331
        TCGv_vec t = tcg_temp_new_vec_matching(r);
332
        tcg_gen_not_vec(0, t, b);
333
        tcg_gen_or_vec(0, r, a, t);
334
        tcg_temp_free_vec(t);
335
    }
336
}
337

338
void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
339
{
340
    if (TCG_TARGET_HAS_nand_vec) {
341
        vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b);
342
    } else {
343
        tcg_gen_and_vec(0, r, a, b);
344
        tcg_gen_not_vec(0, r, r);
345
    }
346
}
347

348
void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349
{
350
    if (TCG_TARGET_HAS_nor_vec) {
351
        vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b);
352
    } else {
353
        tcg_gen_or_vec(0, r, a, b);
354
        tcg_gen_not_vec(0, r, r);
355
    }
356
}
357

358
void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
359
{
360
    if (TCG_TARGET_HAS_eqv_vec) {
361
        vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b);
362
    } else {
363
        tcg_gen_xor_vec(0, r, a, b);
364
        tcg_gen_not_vec(0, r, r);
365
    }
366
}
367

368
static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
369
{
370
    TCGTemp *rt = tcgv_vec_temp(r);
371
    TCGTemp *at = tcgv_vec_temp(a);
372
    TCGArg ri = temp_arg(rt);
373
    TCGArg ai = temp_arg(at);
374
    TCGType type = rt->base_type;
375
    int can;
376

377
    tcg_debug_assert(at->base_type >= type);
378
    tcg_assert_listed_vecop(opc);
379
    can = tcg_can_emit_vec_op(opc, type, vece);
380
    if (can > 0) {
381
        vec_gen_2(opc, type, vece, ri, ai);
382
    } else if (can < 0) {
383
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
384
        tcg_expand_vec_op(opc, type, vece, ri, ai);
385
        tcg_swap_vecop_list(hold_list);
386
    } else {
387
        return false;
388
    }
389
    return true;
390
}
391

392
void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
393
{
394
    if (TCG_TARGET_HAS_not_vec) {
395
        vec_gen_op2(INDEX_op_not_vec, 0, r, a);
396
    } else {
397
        tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1));
398
    }
399
}
400

401
void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
402
{
403
    const TCGOpcode *hold_list;
404

405
    tcg_assert_listed_vecop(INDEX_op_neg_vec);
406
    hold_list = tcg_swap_vecop_list(NULL);
407

408
    if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
409
        tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a);
410
    }
411
    tcg_swap_vecop_list(hold_list);
412
}
413

414
void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
415
{
416
    const TCGOpcode *hold_list;
417

418
    tcg_assert_listed_vecop(INDEX_op_abs_vec);
419
    hold_list = tcg_swap_vecop_list(NULL);
420

421
    if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
422
        TCGType type = tcgv_vec_temp(r)->base_type;
423
        TCGv_vec t = tcg_temp_new_vec(type);
424

425
        tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
426
        if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
427
            tcg_gen_neg_vec(vece, t, a);
428
            tcg_gen_smax_vec(vece, r, a, t);
429
        } else {
430
            if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
431
                tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
432
            } else {
433
                tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
434
                                tcg_constant_vec(type, vece, 0));
435
            }
436
            tcg_gen_xor_vec(vece, r, a, t);
437
            tcg_gen_sub_vec(vece, r, r, t);
438
        }
439

440
        tcg_temp_free_vec(t);
441
    }
442
    tcg_swap_vecop_list(hold_list);
443
}
444

445
static void do_shifti(TCGOpcode opc, unsigned vece,
446
                      TCGv_vec r, TCGv_vec a, int64_t i)
447
{
448
    TCGTemp *rt = tcgv_vec_temp(r);
449
    TCGTemp *at = tcgv_vec_temp(a);
450
    TCGArg ri = temp_arg(rt);
451
    TCGArg ai = temp_arg(at);
452
    TCGType type = rt->base_type;
453
    int can;
454

455
    tcg_debug_assert(at->base_type == type);
456
    tcg_debug_assert(i >= 0 && i < (8 << vece));
457
    tcg_assert_listed_vecop(opc);
458

459
    if (i == 0) {
460
        tcg_gen_mov_vec(r, a);
461
        return;
462
    }
463

464
    can = tcg_can_emit_vec_op(opc, type, vece);
465
    if (can > 0) {
466
        vec_gen_3(opc, type, vece, ri, ai, i);
467
    } else {
468
        /* We leave the choice of expansion via scalar or vector shift
469
           to the target.  Often, but not always, dupi can feed a vector
470
           shift easier than a scalar.  */
471
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
472
        tcg_debug_assert(can < 0);
473
        tcg_expand_vec_op(opc, type, vece, ri, ai, i);
474
        tcg_swap_vecop_list(hold_list);
475
    }
476
}
477

478
void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
479
{
480
    do_shifti(INDEX_op_shli_vec, vece, r, a, i);
481
}
482

483
void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
484
{
485
    do_shifti(INDEX_op_shri_vec, vece, r, a, i);
486
}
487

488
void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
489
{
490
    do_shifti(INDEX_op_sari_vec, vece, r, a, i);
491
}
492

493
void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
494
{
495
    do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
496
}
497

498
void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
499
{
500
    int bits = 8 << vece;
501
    tcg_debug_assert(i >= 0 && i < bits);
502
    do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
503
}
504

505
void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
506
                     TCGv_vec r, TCGv_vec a, TCGv_vec b)
507
{
508
    TCGTemp *rt = tcgv_vec_temp(r);
509
    TCGTemp *at = tcgv_vec_temp(a);
510
    TCGTemp *bt = tcgv_vec_temp(b);
511
    TCGTemp *tt = NULL;
512
    TCGArg ri = temp_arg(rt);
513
    TCGArg ai = temp_arg(at);
514
    TCGArg bi = temp_arg(bt);
515
    TCGArg ti;
516
    TCGType type = rt->base_type;
517
    int can;
518

519
    tcg_debug_assert(at->base_type >= type);
520
    tcg_debug_assert(bt->base_type >= type);
521
    tcg_assert_listed_vecop(INDEX_op_cmp_vec);
522
    can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
523

524
    if (!TCG_TARGET_HAS_tst_vec && is_tst_cond(cond)) {
525
        tt = tcg_temp_new_internal(type, TEMP_EBB);
526
        ti = temp_arg(tt);
527
        vec_gen_3(INDEX_op_and_vec, type, 0, ti, ai, bi);
528
        at = tt;
529
        ai = ti;
530
        bt = tcg_constant_internal(type, 0);
531
        bi = temp_arg(bt);
532
        cond = tcg_tst_eqne_cond(cond);
533
    }
534

535
    if (can > 0) {
536
        vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
537
    } else {
538
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
539
        tcg_debug_assert(can < 0);
540
        tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
541
        tcg_swap_vecop_list(hold_list);
542
    }
543

544
    if (tt) {
545
        tcg_temp_free_internal(tt);
546
    }
547
}
548

549
static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
550
                   TCGv_vec b, TCGOpcode opc)
551
{
552
    TCGTemp *rt = tcgv_vec_temp(r);
553
    TCGTemp *at = tcgv_vec_temp(a);
554
    TCGTemp *bt = tcgv_vec_temp(b);
555
    TCGArg ri = temp_arg(rt);
556
    TCGArg ai = temp_arg(at);
557
    TCGArg bi = temp_arg(bt);
558
    TCGType type = rt->base_type;
559
    int can;
560

561
    tcg_debug_assert(at->base_type >= type);
562
    tcg_debug_assert(bt->base_type >= type);
563
    tcg_assert_listed_vecop(opc);
564
    can = tcg_can_emit_vec_op(opc, type, vece);
565
    if (can > 0) {
566
        vec_gen_3(opc, type, vece, ri, ai, bi);
567
    } else if (can < 0) {
568
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
569
        tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
570
        tcg_swap_vecop_list(hold_list);
571
    } else {
572
        return false;
573
    }
574
    return true;
575
}
576

577
static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
578
                          TCGv_vec b, TCGOpcode opc)
579
{
580
    bool ok = do_op3(vece, r, a, b, opc);
581
    tcg_debug_assert(ok);
582
}
583

584
void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
585
{
586
    do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
587
}
588

589
void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
590
{
591
    do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
592
}
593

594
void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
595
{
596
    do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
597
}
598

599
void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
600
{
601
    do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
602
}
603

604
void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
605
{
606
    if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
607
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
608
        TCGv_vec t = tcg_temp_new_vec_matching(r);
609

610
        /* usadd(a, b) = min(a, ~b) + b */
611
        tcg_gen_not_vec(vece, t, b);
612
        tcg_gen_umin_vec(vece, t, t, a);
613
        tcg_gen_add_vec(vece, r, t, b);
614

615
        tcg_temp_free_vec(t);
616
        tcg_swap_vecop_list(hold_list);
617
    }
618
}
619

620
void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
621
{
622
    do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
623
}
624

625
void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
626
{
627
    if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
628
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
629
        TCGv_vec t = tcg_temp_new_vec_matching(r);
630

631
        /* ussub(a, b) = max(a, b) - b */
632
        tcg_gen_umax_vec(vece, t, a, b);
633
        tcg_gen_sub_vec(vece, r, t, b);
634

635
        tcg_temp_free_vec(t);
636
        tcg_swap_vecop_list(hold_list);
637
    }
638
}
639

640
static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
641
                      TCGv_vec b, TCGOpcode opc, TCGCond cond)
642
{
643
    if (!do_op3(vece, r, a, b, opc)) {
644
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
645
        tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
646
        tcg_swap_vecop_list(hold_list);
647
    }
648
}
649

650
void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
651
{
652
    do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
653
}
654

655
void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
656
{
657
    do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
658
}
659

660
void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
661
{
662
    do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
663
}
664

665
void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
666
{
667
    do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
668
}
669

670
void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
671
{
672
    do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
673
}
674

675
void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
676
{
677
    do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
678
}
679

680
void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
681
{
682
    do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
683
}
684

685
void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
686
{
687
    do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
688
}
689

690
void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
691
{
692
    do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
693
}
694

695
static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
696
                      TCGv_i32 s, TCGOpcode opc)
697
{
698
    TCGTemp *rt = tcgv_vec_temp(r);
699
    TCGTemp *at = tcgv_vec_temp(a);
700
    TCGTemp *st = tcgv_i32_temp(s);
701
    TCGArg ri = temp_arg(rt);
702
    TCGArg ai = temp_arg(at);
703
    TCGArg si = temp_arg(st);
704
    TCGType type = rt->base_type;
705
    int can;
706

707
    tcg_debug_assert(at->base_type >= type);
708
    tcg_assert_listed_vecop(opc);
709
    can = tcg_can_emit_vec_op(opc, type, vece);
710
    if (can > 0) {
711
        vec_gen_3(opc, type, vece, ri, ai, si);
712
    } else if (can < 0) {
713
        const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
714
        tcg_expand_vec_op(opc, type, vece, ri, ai, si);
715
        tcg_swap_vecop_list(hold_list);
716
    } else {
717
        g_assert_not_reached();
718
    }
719
}
720

721
void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
722
{
723
    do_shifts(vece, r, a, b, INDEX_op_shls_vec);
724
}
725

726
void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
727
{
728
    do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
729
}
730

731
void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
732
{
733
    do_shifts(vece, r, a, b, INDEX_op_sars_vec);
734
}
735

736
void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
737
{
738
    do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
739
}
740

741
void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
742
                        TCGv_vec b, TCGv_vec c)
743
{
744
    TCGTemp *rt = tcgv_vec_temp(r);
745
    TCGTemp *at = tcgv_vec_temp(a);
746
    TCGTemp *bt = tcgv_vec_temp(b);
747
    TCGTemp *ct = tcgv_vec_temp(c);
748
    TCGType type = rt->base_type;
749

750
    tcg_debug_assert(at->base_type >= type);
751
    tcg_debug_assert(bt->base_type >= type);
752
    tcg_debug_assert(ct->base_type >= type);
753

754
    if (TCG_TARGET_HAS_bitsel_vec) {
755
        vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
756
                  temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
757
    } else {
758
        TCGv_vec t = tcg_temp_new_vec(type);
759
        tcg_gen_and_vec(MO_8, t, a, b);
760
        tcg_gen_andc_vec(MO_8, r, c, a);
761
        tcg_gen_or_vec(MO_8, r, r, t);
762
        tcg_temp_free_vec(t);
763
    }
764
}
765

766
void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
767
                        TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
768
{
769
    TCGTemp *rt = tcgv_vec_temp(r);
770
    TCGTemp *at = tcgv_vec_temp(a);
771
    TCGTemp *bt = tcgv_vec_temp(b);
772
    TCGTemp *ct = tcgv_vec_temp(c);
773
    TCGTemp *dt = tcgv_vec_temp(d);
774
    TCGArg ri = temp_arg(rt);
775
    TCGArg ai = temp_arg(at);
776
    TCGArg bi = temp_arg(bt);
777
    TCGArg ci = temp_arg(ct);
778
    TCGArg di = temp_arg(dt);
779
    TCGType type = rt->base_type;
780
    const TCGOpcode *hold_list;
781
    int can;
782

783
    tcg_debug_assert(at->base_type >= type);
784
    tcg_debug_assert(bt->base_type >= type);
785
    tcg_debug_assert(ct->base_type >= type);
786
    tcg_debug_assert(dt->base_type >= type);
787

788
    tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
789
    hold_list = tcg_swap_vecop_list(NULL);
790
    can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
791

792
    if (can > 0) {
793
        vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
794
    } else if (can < 0) {
795
        tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
796
                          ri, ai, bi, ci, di, cond);
797
    } else {
798
        TCGv_vec t = tcg_temp_new_vec(type);
799
        tcg_gen_cmp_vec(cond, vece, t, a, b);
800
        tcg_gen_bitsel_vec(vece, r, t, c, d);
801
        tcg_temp_free_vec(t);
802
    }
803
    tcg_swap_vecop_list(hold_list);
804
}
805