qemu

1
/*
2
 * Tiny Code Generator for QEMU
3
 *
4
 * Copyright (c) 2008 Fabrice Bellard
5
 *
6
 * Permission is hereby granted, free of charge, to any person obtaining a copy
7
 * of this software and associated documentation files (the "Software"), to deal
8
 * in the Software without restriction, including without limitation the rights
9
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
 * copies of the Software, and to permit persons to whom the Software is
11
 * furnished to do so, subject to the following conditions:
12
 *
13
 * The above copyright notice and this permission notice shall be included in
14
 * all copies or substantial portions of the Software.
15
 *
16
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
 * THE SOFTWARE.
23
 */
24

25
#include "qemu/osdep.h"
26
#include "tcg/tcg.h"
27
#include "tcg/tcg-temp-internal.h"
28
#include "tcg/tcg-op-common.h"
29
#include "exec/translation-block.h"
30
#include "exec/plugin-gen.h"
31
#include "tcg-internal.h"
32

33

34
/*
35
 * Encourage the compiler to tail-call to a function, rather than inlining.
36
 * Minimizes code size across 99 bottles of beer on the wall.
37
 */
38
#define NI  __attribute__((noinline))
39

40
void NI tcg_gen_op1(TCGOpcode opc, TCGArg a1)
41
{
42
    TCGOp *op = tcg_emit_op(opc, 1);
43
    op->args[0] = a1;
44
}
45

46
void NI tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
47
{
48
    TCGOp *op = tcg_emit_op(opc, 2);
49
    op->args[0] = a1;
50
    op->args[1] = a2;
51
}
52

53
void NI tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
54
{
55
    TCGOp *op = tcg_emit_op(opc, 3);
56
    op->args[0] = a1;
57
    op->args[1] = a2;
58
    op->args[2] = a3;
59
}
60

61
void NI tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
62
{
63
    TCGOp *op = tcg_emit_op(opc, 4);
64
    op->args[0] = a1;
65
    op->args[1] = a2;
66
    op->args[2] = a3;
67
    op->args[3] = a4;
68
}
69

70
void NI tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
71
                     TCGArg a4, TCGArg a5)
72
{
73
    TCGOp *op = tcg_emit_op(opc, 5);
74
    op->args[0] = a1;
75
    op->args[1] = a2;
76
    op->args[2] = a3;
77
    op->args[3] = a4;
78
    op->args[4] = a5;
79
}
80

81
void NI tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
82
                     TCGArg a4, TCGArg a5, TCGArg a6)
83
{
84
    TCGOp *op = tcg_emit_op(opc, 6);
85
    op->args[0] = a1;
86
    op->args[1] = a2;
87
    op->args[2] = a3;
88
    op->args[3] = a4;
89
    op->args[4] = a5;
90
    op->args[5] = a6;
91
}
92

93
/*
94
 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line
95
 * assertion check.  Force tail calls to avoid too much code expansion.
96
 */
97
#ifdef CONFIG_DEBUG_TCG
98
# define DNI NI
99
#else
100
# define DNI
101
#endif
102

103
static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 a1)
104
{
105
    tcg_gen_op1(opc, tcgv_i32_arg(a1));
106
}
107

108
static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 a1)
109
{
110
    tcg_gen_op1(opc, tcgv_i64_arg(a1));
111
}
112

113
static void DNI tcg_gen_op1i(TCGOpcode opc, TCGArg a1)
114
{
115
    tcg_gen_op1(opc, a1);
116
}
117

118
static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2)
119
{
120
    tcg_gen_op2(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2));
121
}
122

123
static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2)
124
{
125
    tcg_gen_op2(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2));
126
}
127

128
static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1,
129
                                TCGv_i32 a2, TCGv_i32 a3)
130
{
131
    tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), tcgv_i32_arg(a3));
132
}
133

134
static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1,
135
                                TCGv_i64 a2, TCGv_i64 a3)
136
{
137
    tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), tcgv_i64_arg(a3));
138
}
139

140
static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1,
141
                                 TCGv_i32 a2, TCGArg a3)
142
{
143
    tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3);
144
}
145

146
static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1,
147
                                 TCGv_i64 a2, TCGArg a3)
148
{
149
    tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3);
150
}
151

152
static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val,
153
                                    TCGv_ptr base, TCGArg offset)
154
{
155
    tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_ptr_arg(base), offset);
156
}
157

158
static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
159
                                    TCGv_ptr base, TCGArg offset)
160
{
161
    tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_ptr_arg(base), offset);
162
}
163

164
static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
165
                                TCGv_i32 a3, TCGv_i32 a4)
166
{
167
    tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
168
                tcgv_i32_arg(a3), tcgv_i32_arg(a4));
169
}
170

171
static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
172
                                TCGv_i64 a3, TCGv_i64 a4)
173
{
174
    tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
175
                tcgv_i64_arg(a3), tcgv_i64_arg(a4));
176
}
177

178
static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
179
                                 TCGv_i32 a3, TCGArg a4)
180
{
181
    tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
182
                tcgv_i32_arg(a3), a4);
183
}
184

185
static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
186
                                 TCGv_i64 a3, TCGArg a4)
187
{
188
    tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
189
                tcgv_i64_arg(a3), a4);
190
}
191

192
static void DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
193
                                  TCGArg a3, TCGArg a4)
194
{
195
    tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4);
196
}
197

198
static void DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
199
                                  TCGArg a3, TCGArg a4)
200
{
201
    tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4);
202
}
203

204
static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
205
                                TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5)
206
{
207
    tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
208
                tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5));
209
}
210

211
static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
212
                                TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5)
213
{
214
    tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
215
                tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5));
216
}
217

218
static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
219
                                  TCGv_i32 a3, TCGArg a4, TCGArg a5)
220
{
221
    tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
222
                tcgv_i32_arg(a3), a4, a5);
223
}
224

225
static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
226
                                  TCGv_i64 a3, TCGArg a4, TCGArg a5)
227
{
228
    tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
229
                tcgv_i64_arg(a3), a4, a5);
230
}
231

232
static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
233
                                TCGv_i32 a3, TCGv_i32 a4,
234
                                TCGv_i32 a5, TCGv_i32 a6)
235
{
236
    tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
237
                tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5),
238
                tcgv_i32_arg(a6));
239
}
240

241
static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
242
                                TCGv_i64 a3, TCGv_i64 a4,
243
                                TCGv_i64 a5, TCGv_i64 a6)
244
{
245
    tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
246
                tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5),
247
                tcgv_i64_arg(a6));
248
}
249

250
static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
251
                                 TCGv_i32 a3, TCGv_i32 a4,
252
                                 TCGv_i32 a5, TCGArg a6)
253
{
254
    tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
255
                tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6);
256
}
257

258
static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
259
                                 TCGv_i64 a3, TCGv_i64 a4,
260
                                 TCGv_i64 a5, TCGArg a6)
261
{
262
    tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
263
                tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6);
264
}
265

266
static void DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
267
                                  TCGv_i32 a3, TCGv_i32 a4,
268
                                  TCGArg a5, TCGArg a6)
269
{
270
    tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
271
                tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6);
272
}
273

274
/* Generic ops.  */
275

276
void gen_set_label(TCGLabel *l)
277
{
278
    l->present = 1;
279
    tcg_gen_op1(INDEX_op_set_label, label_arg(l));
280
}
281

282
static void add_last_as_label_use(TCGLabel *l)
283
{
284
    TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse));
285

286
    u->op = tcg_last_op();
287
    QSIMPLEQ_INSERT_TAIL(&l->branches, u, next);
288
}
289

290
void tcg_gen_br(TCGLabel *l)
291
{
292
    tcg_gen_op1(INDEX_op_br, label_arg(l));
293
    add_last_as_label_use(l);
294
}
295

296
void tcg_gen_mb(TCGBar mb_type)
297
{
298
#ifdef CONFIG_USER_ONLY
299
    bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL;
300
#else
301
    /*
302
     * It is tempting to elide the barrier in a uniprocessor context.
303
     * However, even with a single cpu we have i/o threads running in
304
     * parallel, and lack of memory order can result in e.g. virtio
305
     * queue entries being read incorrectly.
306
     */
307
    bool parallel = true;
308
#endif
309

310
    if (parallel) {
311
        tcg_gen_op1(INDEX_op_mb, mb_type);
312
    }
313
}
314

315
void tcg_gen_plugin_cb(unsigned from)
316
{
317
    tcg_gen_op1(INDEX_op_plugin_cb, from);
318
}
319

320
void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo)
321
{
322
    tcg_gen_op2(INDEX_op_plugin_mem_cb, tcgv_i64_arg(addr), meminfo);
323
}
324

325
/* 32 bit ops */
326

327
void tcg_gen_discard_i32(TCGv_i32 arg)
328
{
329
    tcg_gen_op1_i32(INDEX_op_discard, arg);
330
}
331

332
void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg)
333
{
334
    if (ret != arg) {
335
        tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg);
336
    }
337
}
338

339
void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
340
{
341
    tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
342
}
343

344
void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
345
{
346
    tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2);
347
}
348

349
void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
350
{
351
    /* some cases can be optimized here */
352
    if (arg2 == 0) {
353
        tcg_gen_mov_i32(ret, arg1);
354
    } else {
355
        tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
356
    }
357
}
358

359
void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
360
{
361
    tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2);
362
}
363

364
void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
365
{
366
    if (arg1 == 0) {
367
        tcg_gen_neg_i32(ret, arg2);
368
    } else {
369
        tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
370
    }
371
}
372

373
void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
374
{
375
    tcg_gen_addi_i32(ret, arg1, -arg2);
376
}
377

378
void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg)
379
{
380
    tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg);
381
}
382

383
void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
384
{
385
    tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2);
386
}
387

388
void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
389
{
390
    /* Some cases can be optimized here.  */
391
    switch (arg2) {
392
    case 0:
393
        tcg_gen_movi_i32(ret, 0);
394
        return;
395
    case -1:
396
        tcg_gen_mov_i32(ret, arg1);
397
        return;
398
    case 0xff:
399
        /* Don't recurse with tcg_gen_ext8u_i32.  */
400
        if (TCG_TARGET_HAS_ext8u_i32) {
401
            tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
402
            return;
403
        }
404
        break;
405
    case 0xffff:
406
        if (TCG_TARGET_HAS_ext16u_i32) {
407
            tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
408
            return;
409
        }
410
        break;
411
    }
412

413
    tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
414
}
415

416
void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
417
{
418
    tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2);
419
}
420

421
void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
422
{
423
    /* Some cases can be optimized here.  */
424
    if (arg2 == -1) {
425
        tcg_gen_movi_i32(ret, -1);
426
    } else if (arg2 == 0) {
427
        tcg_gen_mov_i32(ret, arg1);
428
    } else {
429
        tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
430
    }
431
}
432

433
void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
434
{
435
    tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2);
436
}
437

438
void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
439
{
440
    /* Some cases can be optimized here.  */
441
    if (arg2 == 0) {
442
        tcg_gen_mov_i32(ret, arg1);
443
    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
444
        /* Don't recurse with tcg_gen_not_i32.  */
445
        tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
446
    } else {
447
        tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
448
    }
449
}
450

451
void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg)
452
{
453
    if (TCG_TARGET_HAS_not_i32) {
454
        tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg);
455
    } else {
456
        tcg_gen_xori_i32(ret, arg, -1);
457
    }
458
}
459

460
void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
461
{
462
    tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2);
463
}
464

465
void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
466
{
467
    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
468
    if (arg2 == 0) {
469
        tcg_gen_mov_i32(ret, arg1);
470
    } else {
471
        tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
472
    }
473
}
474

475
void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
476
{
477
    tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2);
478
}
479

480
void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
481
{
482
    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
483
    if (arg2 == 0) {
484
        tcg_gen_mov_i32(ret, arg1);
485
    } else {
486
        tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
487
    }
488
}
489

490
void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
491
{
492
    tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2);
493
}
494

495
void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
496
{
497
    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
498
    if (arg2 == 0) {
499
        tcg_gen_mov_i32(ret, arg1);
500
    } else {
501
        tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
502
    }
503
}
504

505
void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
506
{
507
    if (cond == TCG_COND_ALWAYS) {
508
        tcg_gen_br(l);
509
    } else if (cond != TCG_COND_NEVER) {
510
        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
511
        add_last_as_label_use(l);
512
    }
513
}
514

515
void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
516
{
517
    if (cond == TCG_COND_ALWAYS) {
518
        tcg_gen_br(l);
519
    } else if (cond != TCG_COND_NEVER) {
520
        tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
521
    }
522
}
523

524
void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
525
                         TCGv_i32 arg1, TCGv_i32 arg2)
526
{
527
    if (cond == TCG_COND_ALWAYS) {
528
        tcg_gen_movi_i32(ret, 1);
529
    } else if (cond == TCG_COND_NEVER) {
530
        tcg_gen_movi_i32(ret, 0);
531
    } else {
532
        tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
533
    }
534
}
535

536
void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
537
                          TCGv_i32 arg1, int32_t arg2)
538
{
539
    tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
540
}
541

542
void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret,
543
                            TCGv_i32 arg1, TCGv_i32 arg2)
544
{
545
    if (cond == TCG_COND_ALWAYS) {
546
        tcg_gen_movi_i32(ret, -1);
547
    } else if (cond == TCG_COND_NEVER) {
548
        tcg_gen_movi_i32(ret, 0);
549
    } else if (TCG_TARGET_HAS_negsetcond_i32) {
550
        tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond);
551
    } else {
552
        tcg_gen_setcond_i32(cond, ret, arg1, arg2);
553
        tcg_gen_neg_i32(ret, ret);
554
    }
555
}
556

557
void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret,
558
                             TCGv_i32 arg1, int32_t arg2)
559
{
560
    tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
561
}
562

563
void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
564
{
565
    tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2);
566
}
567

568
void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
569
{
570
    if (arg2 == 0) {
571
        tcg_gen_movi_i32(ret, 0);
572
    } else if (is_power_of_2(arg2)) {
573
        tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
574
    } else {
575
        tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
576
    }
577
}
578

579
void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
580
{
581
    if (TCG_TARGET_HAS_div_i32) {
582
        tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
583
    } else if (TCG_TARGET_HAS_div2_i32) {
584
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
585
        tcg_gen_sari_i32(t0, arg1, 31);
586
        tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
587
        tcg_temp_free_i32(t0);
588
    } else {
589
        gen_helper_div_i32(ret, arg1, arg2);
590
    }
591
}
592

593
void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
594
{
595
    if (TCG_TARGET_HAS_rem_i32) {
596
        tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
597
    } else if (TCG_TARGET_HAS_div_i32) {
598
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
599
        tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
600
        tcg_gen_mul_i32(t0, t0, arg2);
601
        tcg_gen_sub_i32(ret, arg1, t0);
602
        tcg_temp_free_i32(t0);
603
    } else if (TCG_TARGET_HAS_div2_i32) {
604
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
605
        tcg_gen_sari_i32(t0, arg1, 31);
606
        tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
607
        tcg_temp_free_i32(t0);
608
    } else {
609
        gen_helper_rem_i32(ret, arg1, arg2);
610
    }
611
}
612

613
void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
614
{
615
    if (TCG_TARGET_HAS_div_i32) {
616
        tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
617
    } else if (TCG_TARGET_HAS_div2_i32) {
618
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
619
        TCGv_i32 zero = tcg_constant_i32(0);
620
        tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, zero, arg2);
621
        tcg_temp_free_i32(t0);
622
    } else {
623
        gen_helper_divu_i32(ret, arg1, arg2);
624
    }
625
}
626

627
void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
628
{
629
    if (TCG_TARGET_HAS_rem_i32) {
630
        tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
631
    } else if (TCG_TARGET_HAS_div_i32) {
632
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
633
        tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
634
        tcg_gen_mul_i32(t0, t0, arg2);
635
        tcg_gen_sub_i32(ret, arg1, t0);
636
        tcg_temp_free_i32(t0);
637
    } else if (TCG_TARGET_HAS_div2_i32) {
638
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
639
        TCGv_i32 zero = tcg_constant_i32(0);
640
        tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, zero, arg2);
641
        tcg_temp_free_i32(t0);
642
    } else {
643
        gen_helper_remu_i32(ret, arg1, arg2);
644
    }
645
}
646

647
void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
648
{
649
    if (TCG_TARGET_HAS_andc_i32) {
650
        tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
651
    } else {
652
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
653
        tcg_gen_not_i32(t0, arg2);
654
        tcg_gen_and_i32(ret, arg1, t0);
655
        tcg_temp_free_i32(t0);
656
    }
657
}
658

659
void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
660
{
661
    if (TCG_TARGET_HAS_eqv_i32) {
662
        tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
663
    } else {
664
        tcg_gen_xor_i32(ret, arg1, arg2);
665
        tcg_gen_not_i32(ret, ret);
666
    }
667
}
668

669
void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
670
{
671
    if (TCG_TARGET_HAS_nand_i32) {
672
        tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
673
    } else {
674
        tcg_gen_and_i32(ret, arg1, arg2);
675
        tcg_gen_not_i32(ret, ret);
676
    }
677
}
678

679
void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
680
{
681
    if (TCG_TARGET_HAS_nor_i32) {
682
        tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
683
    } else {
684
        tcg_gen_or_i32(ret, arg1, arg2);
685
        tcg_gen_not_i32(ret, ret);
686
    }
687
}
688

689
void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
690
{
691
    if (TCG_TARGET_HAS_orc_i32) {
692
        tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
693
    } else {
694
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
695
        tcg_gen_not_i32(t0, arg2);
696
        tcg_gen_or_i32(ret, arg1, t0);
697
        tcg_temp_free_i32(t0);
698
    }
699
}
700

701
void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
702
{
703
    if (TCG_TARGET_HAS_clz_i32) {
704
        tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
705
    } else if (TCG_TARGET_HAS_clz_i64) {
706
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
707
        TCGv_i64 t2 = tcg_temp_ebb_new_i64();
708
        tcg_gen_extu_i32_i64(t1, arg1);
709
        tcg_gen_extu_i32_i64(t2, arg2);
710
        tcg_gen_addi_i64(t2, t2, 32);
711
        tcg_gen_clz_i64(t1, t1, t2);
712
        tcg_gen_extrl_i64_i32(ret, t1);
713
        tcg_temp_free_i64(t1);
714
        tcg_temp_free_i64(t2);
715
        tcg_gen_subi_i32(ret, ret, 32);
716
    } else {
717
        gen_helper_clz_i32(ret, arg1, arg2);
718
    }
719
}
720

721
void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
722
{
723
    tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
724
}
725

726
void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
727
{
728
    if (TCG_TARGET_HAS_ctz_i32) {
729
        tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
730
    } else if (TCG_TARGET_HAS_ctz_i64) {
731
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
732
        TCGv_i64 t2 = tcg_temp_ebb_new_i64();
733
        tcg_gen_extu_i32_i64(t1, arg1);
734
        tcg_gen_extu_i32_i64(t2, arg2);
735
        tcg_gen_ctz_i64(t1, t1, t2);
736
        tcg_gen_extrl_i64_i32(ret, t1);
737
        tcg_temp_free_i64(t1);
738
        tcg_temp_free_i64(t2);
739
    } else if (TCG_TARGET_HAS_ctpop_i32
740
               || TCG_TARGET_HAS_ctpop_i64
741
               || TCG_TARGET_HAS_clz_i32
742
               || TCG_TARGET_HAS_clz_i64) {
743
        TCGv_i32 z, t = tcg_temp_ebb_new_i32();
744

745
        if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
746
            tcg_gen_subi_i32(t, arg1, 1);
747
            tcg_gen_andc_i32(t, t, arg1);
748
            tcg_gen_ctpop_i32(t, t);
749
        } else {
750
            /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
751
            tcg_gen_neg_i32(t, arg1);
752
            tcg_gen_and_i32(t, t, arg1);
753
            tcg_gen_clzi_i32(t, t, 32);
754
            tcg_gen_xori_i32(t, t, 31);
755
        }
756
        z = tcg_constant_i32(0);
757
        tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
758
        tcg_temp_free_i32(t);
759
    } else {
760
        gen_helper_ctz_i32(ret, arg1, arg2);
761
    }
762
}
763

764
void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
765
{
766
    if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
767
        /* This equivalence has the advantage of not requiring a fixup.  */
768
        TCGv_i32 t = tcg_temp_ebb_new_i32();
769
        tcg_gen_subi_i32(t, arg1, 1);
770
        tcg_gen_andc_i32(t, t, arg1);
771
        tcg_gen_ctpop_i32(ret, t);
772
        tcg_temp_free_i32(t);
773
    } else {
774
        tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
775
    }
776
}
777

778
void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
779
{
780
    if (TCG_TARGET_HAS_clz_i32) {
781
        TCGv_i32 t = tcg_temp_ebb_new_i32();
782
        tcg_gen_sari_i32(t, arg, 31);
783
        tcg_gen_xor_i32(t, t, arg);
784
        tcg_gen_clzi_i32(t, t, 32);
785
        tcg_gen_subi_i32(ret, t, 1);
786
        tcg_temp_free_i32(t);
787
    } else {
788
        gen_helper_clrsb_i32(ret, arg);
789
    }
790
}
791

792
void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
793
{
794
    if (TCG_TARGET_HAS_ctpop_i32) {
795
        tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
796
    } else if (TCG_TARGET_HAS_ctpop_i64) {
797
        TCGv_i64 t = tcg_temp_ebb_new_i64();
798
        tcg_gen_extu_i32_i64(t, arg1);
799
        tcg_gen_ctpop_i64(t, t);
800
        tcg_gen_extrl_i64_i32(ret, t);
801
        tcg_temp_free_i64(t);
802
    } else {
803
        gen_helper_ctpop_i32(ret, arg1);
804
    }
805
}
806

807
void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
808
{
809
    if (TCG_TARGET_HAS_rot_i32) {
810
        tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
811
    } else {
812
        TCGv_i32 t0, t1;
813

814
        t0 = tcg_temp_ebb_new_i32();
815
        t1 = tcg_temp_ebb_new_i32();
816
        tcg_gen_shl_i32(t0, arg1, arg2);
817
        tcg_gen_subfi_i32(t1, 32, arg2);
818
        tcg_gen_shr_i32(t1, arg1, t1);
819
        tcg_gen_or_i32(ret, t0, t1);
820
        tcg_temp_free_i32(t0);
821
        tcg_temp_free_i32(t1);
822
    }
823
}
824

825
void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
826
{
827
    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
828
    /* some cases can be optimized here */
829
    if (arg2 == 0) {
830
        tcg_gen_mov_i32(ret, arg1);
831
    } else if (TCG_TARGET_HAS_rot_i32) {
832
        tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
833
    } else {
834
        TCGv_i32 t0, t1;
835
        t0 = tcg_temp_ebb_new_i32();
836
        t1 = tcg_temp_ebb_new_i32();
837
        tcg_gen_shli_i32(t0, arg1, arg2);
838
        tcg_gen_shri_i32(t1, arg1, 32 - arg2);
839
        tcg_gen_or_i32(ret, t0, t1);
840
        tcg_temp_free_i32(t0);
841
        tcg_temp_free_i32(t1);
842
    }
843
}
844

845
void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
846
{
847
    if (TCG_TARGET_HAS_rot_i32) {
848
        tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
849
    } else {
850
        TCGv_i32 t0, t1;
851

852
        t0 = tcg_temp_ebb_new_i32();
853
        t1 = tcg_temp_ebb_new_i32();
854
        tcg_gen_shr_i32(t0, arg1, arg2);
855
        tcg_gen_subfi_i32(t1, 32, arg2);
856
        tcg_gen_shl_i32(t1, arg1, t1);
857
        tcg_gen_or_i32(ret, t0, t1);
858
        tcg_temp_free_i32(t0);
859
        tcg_temp_free_i32(t1);
860
    }
861
}
862

863
void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
864
{
865
    tcg_debug_assert(arg2 >= 0 && arg2 < 32);
866
    /* some cases can be optimized here */
867
    if (arg2 == 0) {
868
        tcg_gen_mov_i32(ret, arg1);
869
    } else {
870
        tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
871
    }
872
}
873

874
void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
875
                         unsigned int ofs, unsigned int len)
876
{
877
    uint32_t mask;
878
    TCGv_i32 t1;
879

880
    tcg_debug_assert(ofs < 32);
881
    tcg_debug_assert(len > 0);
882
    tcg_debug_assert(len <= 32);
883
    tcg_debug_assert(ofs + len <= 32);
884

885
    if (len == 32) {
886
        tcg_gen_mov_i32(ret, arg2);
887
        return;
888
    }
889
    if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
890
        tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
891
        return;
892
    }
893

894
    t1 = tcg_temp_ebb_new_i32();
895

896
    if (TCG_TARGET_HAS_extract2_i32) {
897
        if (ofs + len == 32) {
898
            tcg_gen_shli_i32(t1, arg1, len);
899
            tcg_gen_extract2_i32(ret, t1, arg2, len);
900
            goto done;
901
        }
902
        if (ofs == 0) {
903
            tcg_gen_extract2_i32(ret, arg1, arg2, len);
904
            tcg_gen_rotli_i32(ret, ret, len);
905
            goto done;
906
        }
907
    }
908

909
    mask = (1u << len) - 1;
910
    if (ofs + len < 32) {
911
        tcg_gen_andi_i32(t1, arg2, mask);
912
        tcg_gen_shli_i32(t1, t1, ofs);
913
    } else {
914
        tcg_gen_shli_i32(t1, arg2, ofs);
915
    }
916
    tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
917
    tcg_gen_or_i32(ret, ret, t1);
918
 done:
919
    tcg_temp_free_i32(t1);
920
}
921

922
void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
923
                           unsigned int ofs, unsigned int len)
924
{
925
    tcg_debug_assert(ofs < 32);
926
    tcg_debug_assert(len > 0);
927
    tcg_debug_assert(len <= 32);
928
    tcg_debug_assert(ofs + len <= 32);
929

930
    if (ofs + len == 32) {
931
        tcg_gen_shli_i32(ret, arg, ofs);
932
    } else if (ofs == 0) {
933
        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
934
    } else if (TCG_TARGET_HAS_deposit_i32
935
               && TCG_TARGET_deposit_i32_valid(ofs, len)) {
936
        TCGv_i32 zero = tcg_constant_i32(0);
937
        tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
938
    } else {
939
        /* To help two-operand hosts we prefer to zero-extend first,
940
           which allows ARG to stay live.  */
941
        switch (len) {
942
        case 16:
943
            if (TCG_TARGET_HAS_ext16u_i32) {
944
                tcg_gen_ext16u_i32(ret, arg);
945
                tcg_gen_shli_i32(ret, ret, ofs);
946
                return;
947
            }
948
            break;
949
        case 8:
950
            if (TCG_TARGET_HAS_ext8u_i32) {
951
                tcg_gen_ext8u_i32(ret, arg);
952
                tcg_gen_shli_i32(ret, ret, ofs);
953
                return;
954
            }
955
            break;
956
        }
957
        /* Otherwise prefer zero-extension over AND for code size.  */
958
        switch (ofs + len) {
959
        case 16:
960
            if (TCG_TARGET_HAS_ext16u_i32) {
961
                tcg_gen_shli_i32(ret, arg, ofs);
962
                tcg_gen_ext16u_i32(ret, ret);
963
                return;
964
            }
965
            break;
966
        case 8:
967
            if (TCG_TARGET_HAS_ext8u_i32) {
968
                tcg_gen_shli_i32(ret, arg, ofs);
969
                tcg_gen_ext8u_i32(ret, ret);
970
                return;
971
            }
972
            break;
973
        }
974
        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
975
        tcg_gen_shli_i32(ret, ret, ofs);
976
    }
977
}
978

979
void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
980
                         unsigned int ofs, unsigned int len)
981
{
982
    tcg_debug_assert(ofs < 32);
983
    tcg_debug_assert(len > 0);
984
    tcg_debug_assert(len <= 32);
985
    tcg_debug_assert(ofs + len <= 32);
986

987
    /* Canonicalize certain special cases, even if extract is supported.  */
988
    if (ofs + len == 32) {
989
        tcg_gen_shri_i32(ret, arg, 32 - len);
990
        return;
991
    }
992
    if (ofs == 0) {
993
        tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
994
        return;
995
    }
996

997
    if (TCG_TARGET_HAS_extract_i32
998
        && TCG_TARGET_extract_i32_valid(ofs, len)) {
999
        tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
1000
        return;
1001
    }
1002

1003
    /* Assume that zero-extension, if available, is cheaper than a shift.  */
1004
    switch (ofs + len) {
1005
    case 16:
1006
        if (TCG_TARGET_HAS_ext16u_i32) {
1007
            tcg_gen_ext16u_i32(ret, arg);
1008
            tcg_gen_shri_i32(ret, ret, ofs);
1009
            return;
1010
        }
1011
        break;
1012
    case 8:
1013
        if (TCG_TARGET_HAS_ext8u_i32) {
1014
            tcg_gen_ext8u_i32(ret, arg);
1015
            tcg_gen_shri_i32(ret, ret, ofs);
1016
            return;
1017
        }
1018
        break;
1019
    }
1020

1021
    /* ??? Ideally we'd know what values are available for immediate AND.
1022
       Assume that 8 bits are available, plus the special case of 16,
1023
       so that we get ext8u, ext16u.  */
1024
    switch (len) {
1025
    case 1 ... 8: case 16:
1026
        tcg_gen_shri_i32(ret, arg, ofs);
1027
        tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
1028
        break;
1029
    default:
1030
        tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
1031
        tcg_gen_shri_i32(ret, ret, 32 - len);
1032
        break;
1033
    }
1034
}
1035

1036
void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
1037
                          unsigned int ofs, unsigned int len)
1038
{
1039
    tcg_debug_assert(ofs < 32);
1040
    tcg_debug_assert(len > 0);
1041
    tcg_debug_assert(len <= 32);
1042
    tcg_debug_assert(ofs + len <= 32);
1043

1044
    /* Canonicalize certain special cases, even if extract is supported.  */
1045
    if (ofs + len == 32) {
1046
        tcg_gen_sari_i32(ret, arg, 32 - len);
1047
        return;
1048
    }
1049
    if (ofs == 0) {
1050
        switch (len) {
1051
        case 16:
1052
            tcg_gen_ext16s_i32(ret, arg);
1053
            return;
1054
        case 8:
1055
            tcg_gen_ext8s_i32(ret, arg);
1056
            return;
1057
        }
1058
    }
1059

1060
    if (TCG_TARGET_HAS_sextract_i32
1061
        && TCG_TARGET_extract_i32_valid(ofs, len)) {
1062
        tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
1063
        return;
1064
    }
1065

1066
    /* Assume that sign-extension, if available, is cheaper than a shift.  */
1067
    switch (ofs + len) {
1068
    case 16:
1069
        if (TCG_TARGET_HAS_ext16s_i32) {
1070
            tcg_gen_ext16s_i32(ret, arg);
1071
            tcg_gen_sari_i32(ret, ret, ofs);
1072
            return;
1073
        }
1074
        break;
1075
    case 8:
1076
        if (TCG_TARGET_HAS_ext8s_i32) {
1077
            tcg_gen_ext8s_i32(ret, arg);
1078
            tcg_gen_sari_i32(ret, ret, ofs);
1079
            return;
1080
        }
1081
        break;
1082
    }
1083
    switch (len) {
1084
    case 16:
1085
        if (TCG_TARGET_HAS_ext16s_i32) {
1086
            tcg_gen_shri_i32(ret, arg, ofs);
1087
            tcg_gen_ext16s_i32(ret, ret);
1088
            return;
1089
        }
1090
        break;
1091
    case 8:
1092
        if (TCG_TARGET_HAS_ext8s_i32) {
1093
            tcg_gen_shri_i32(ret, arg, ofs);
1094
            tcg_gen_ext8s_i32(ret, ret);
1095
            return;
1096
        }
1097
        break;
1098
    }
1099

1100
    tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
1101
    tcg_gen_sari_i32(ret, ret, 32 - len);
1102
}
1103

1104
/*
1105
 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
1106
 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
1107
 */
1108
void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
1109
                          unsigned int ofs)
1110
{
1111
    tcg_debug_assert(ofs <= 32);
1112
    if (ofs == 0) {
1113
        tcg_gen_mov_i32(ret, al);
1114
    } else if (ofs == 32) {
1115
        tcg_gen_mov_i32(ret, ah);
1116
    } else if (al == ah) {
1117
        tcg_gen_rotri_i32(ret, al, ofs);
1118
    } else if (TCG_TARGET_HAS_extract2_i32) {
1119
        tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
1120
    } else {
1121
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1122
        tcg_gen_shri_i32(t0, al, ofs);
1123
        tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
1124
        tcg_temp_free_i32(t0);
1125
    }
1126
}
1127

1128
void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
1129
                         TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
1130
{
1131
    if (cond == TCG_COND_ALWAYS) {
1132
        tcg_gen_mov_i32(ret, v1);
1133
    } else if (cond == TCG_COND_NEVER) {
1134
        tcg_gen_mov_i32(ret, v2);
1135
    } else {
1136
        tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
1137
    }
1138
}
1139

1140
void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
1141
                      TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
1142
{
1143
    if (TCG_TARGET_HAS_add2_i32) {
1144
        tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
1145
    } else {
1146
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1147
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1148
        tcg_gen_concat_i32_i64(t0, al, ah);
1149
        tcg_gen_concat_i32_i64(t1, bl, bh);
1150
        tcg_gen_add_i64(t0, t0, t1);
1151
        tcg_gen_extr_i64_i32(rl, rh, t0);
1152
        tcg_temp_free_i64(t0);
1153
        tcg_temp_free_i64(t1);
1154
    }
1155
}
1156

1157
void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
1158
                      TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
1159
{
1160
    if (TCG_TARGET_HAS_sub2_i32) {
1161
        tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
1162
    } else {
1163
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1164
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1165
        tcg_gen_concat_i32_i64(t0, al, ah);
1166
        tcg_gen_concat_i32_i64(t1, bl, bh);
1167
        tcg_gen_sub_i64(t0, t0, t1);
1168
        tcg_gen_extr_i64_i32(rl, rh, t0);
1169
        tcg_temp_free_i64(t0);
1170
        tcg_temp_free_i64(t1);
1171
    }
1172
}
1173

1174
void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1175
{
1176
    if (TCG_TARGET_HAS_mulu2_i32) {
1177
        tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
1178
    } else if (TCG_TARGET_HAS_muluh_i32) {
1179
        TCGv_i32 t = tcg_temp_ebb_new_i32();
1180
        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
1181
        tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
1182
        tcg_gen_mov_i32(rl, t);
1183
        tcg_temp_free_i32(t);
1184
    } else if (TCG_TARGET_REG_BITS == 64) {
1185
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1186
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1187
        tcg_gen_extu_i32_i64(t0, arg1);
1188
        tcg_gen_extu_i32_i64(t1, arg2);
1189
        tcg_gen_mul_i64(t0, t0, t1);
1190
        tcg_gen_extr_i64_i32(rl, rh, t0);
1191
        tcg_temp_free_i64(t0);
1192
        tcg_temp_free_i64(t1);
1193
    } else {
1194
        qemu_build_not_reached();
1195
    }
1196
}
1197

1198
void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1199
{
1200
    if (TCG_TARGET_HAS_muls2_i32) {
1201
        tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
1202
    } else if (TCG_TARGET_HAS_mulsh_i32) {
1203
        TCGv_i32 t = tcg_temp_ebb_new_i32();
1204
        tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
1205
        tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
1206
        tcg_gen_mov_i32(rl, t);
1207
        tcg_temp_free_i32(t);
1208
    } else if (TCG_TARGET_REG_BITS == 32) {
1209
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1210
        TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1211
        TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1212
        TCGv_i32 t3 = tcg_temp_ebb_new_i32();
1213
        tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
1214
        /* Adjust for negative inputs.  */
1215
        tcg_gen_sari_i32(t2, arg1, 31);
1216
        tcg_gen_sari_i32(t3, arg2, 31);
1217
        tcg_gen_and_i32(t2, t2, arg2);
1218
        tcg_gen_and_i32(t3, t3, arg1);
1219
        tcg_gen_sub_i32(rh, t1, t2);
1220
        tcg_gen_sub_i32(rh, rh, t3);
1221
        tcg_gen_mov_i32(rl, t0);
1222
        tcg_temp_free_i32(t0);
1223
        tcg_temp_free_i32(t1);
1224
        tcg_temp_free_i32(t2);
1225
        tcg_temp_free_i32(t3);
1226
    } else {
1227
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1228
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1229
        tcg_gen_ext_i32_i64(t0, arg1);
1230
        tcg_gen_ext_i32_i64(t1, arg2);
1231
        tcg_gen_mul_i64(t0, t0, t1);
1232
        tcg_gen_extr_i64_i32(rl, rh, t0);
1233
        tcg_temp_free_i64(t0);
1234
        tcg_temp_free_i64(t1);
1235
    }
1236
}
1237

1238
void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1239
{
1240
    if (TCG_TARGET_REG_BITS == 32) {
1241
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1242
        TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1243
        TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1244
        tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
1245
        /* Adjust for negative input for the signed arg1.  */
1246
        tcg_gen_sari_i32(t2, arg1, 31);
1247
        tcg_gen_and_i32(t2, t2, arg2);
1248
        tcg_gen_sub_i32(rh, t1, t2);
1249
        tcg_gen_mov_i32(rl, t0);
1250
        tcg_temp_free_i32(t0);
1251
        tcg_temp_free_i32(t1);
1252
        tcg_temp_free_i32(t2);
1253
    } else {
1254
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1255
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1256
        tcg_gen_ext_i32_i64(t0, arg1);
1257
        tcg_gen_extu_i32_i64(t1, arg2);
1258
        tcg_gen_mul_i64(t0, t0, t1);
1259
        tcg_gen_extr_i64_i32(rl, rh, t0);
1260
        tcg_temp_free_i64(t0);
1261
        tcg_temp_free_i64(t1);
1262
    }
1263
}
1264

1265
void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
1266
{
1267
    if (TCG_TARGET_HAS_ext8s_i32) {
1268
        tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
1269
    } else {
1270
        tcg_gen_shli_i32(ret, arg, 24);
1271
        tcg_gen_sari_i32(ret, ret, 24);
1272
    }
1273
}
1274

1275
void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
1276
{
1277
    if (TCG_TARGET_HAS_ext16s_i32) {
1278
        tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
1279
    } else {
1280
        tcg_gen_shli_i32(ret, arg, 16);
1281
        tcg_gen_sari_i32(ret, ret, 16);
1282
    }
1283
}
1284

1285
void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
1286
{
1287
    if (TCG_TARGET_HAS_ext8u_i32) {
1288
        tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
1289
    } else {
1290
        tcg_gen_andi_i32(ret, arg, 0xffu);
1291
    }
1292
}
1293

1294
void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1295
{
1296
    if (TCG_TARGET_HAS_ext16u_i32) {
1297
        tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1298
    } else {
1299
        tcg_gen_andi_i32(ret, arg, 0xffffu);
1300
    }
1301
}
1302

1303
/*
1304
 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value.
1305
 *
1306
 * Byte pattern: xxab -> yyba
1307
 *
1308
 * With TCG_BSWAP_IZ, x == zero, else undefined.
1309
 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
1310
 */
1311
void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1312
{
1313
    /* Only one extension flag may be present. */
1314
    tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1315

1316
    if (TCG_TARGET_HAS_bswap16_i32) {
1317
        tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1318
    } else {
1319
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1320
        TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1321

1322
                                            /* arg = ..ab (IZ) xxab (!IZ) */
1323
        tcg_gen_shri_i32(t0, arg, 8);       /*  t0 = ...a (IZ) .xxa (!IZ) */
1324
        if (!(flags & TCG_BSWAP_IZ)) {
1325
            tcg_gen_ext8u_i32(t0, t0);      /*  t0 = ...a */
1326
        }
1327

1328
        if (flags & TCG_BSWAP_OS) {
1329
            tcg_gen_shli_i32(t1, arg, 24);  /*  t1 = b... */
1330
            tcg_gen_sari_i32(t1, t1, 16);   /*  t1 = ssb. */
1331
        } else if (flags & TCG_BSWAP_OZ) {
1332
            tcg_gen_ext8u_i32(t1, arg);     /*  t1 = ...b */
1333
            tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = ..b. */
1334
        } else {
1335
            tcg_gen_shli_i32(t1, arg, 8);   /*  t1 = xab. */
1336
        }
1337

1338
        tcg_gen_or_i32(ret, t0, t1);        /* ret = ..ba (OZ) */
1339
                                            /*     = ssba (OS) */
1340
                                            /*     = xaba (no flag) */
1341
        tcg_temp_free_i32(t0);
1342
        tcg_temp_free_i32(t1);
1343
    }
1344
}
1345

1346
/*
1347
 * bswap32_i32: 32-bit byte swap on a 32-bit value.
1348
 *
1349
 * Byte pattern: abcd -> dcba
1350
 */
1351
void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1352
{
1353
    if (TCG_TARGET_HAS_bswap32_i32) {
1354
        tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1355
    } else {
1356
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1357
        TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1358
        TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1359

1360
                                        /* arg = abcd */
1361
        tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
1362
        tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
1363
        tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
1364
        tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
1365
        tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
1366

1367
        tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
1368
        tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
1369
        tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
1370

1371
        tcg_temp_free_i32(t0);
1372
        tcg_temp_free_i32(t1);
1373
    }
1374
}
1375

1376
/*
1377
 * hswap_i32: Swap 16-bit halfwords within a 32-bit value.
1378
 *
1379
 * Byte pattern: abcd -> cdab
1380
 */
1381
void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1382
{
1383
    /* Swapping 2 16-bit elements is a rotate. */
1384
    tcg_gen_rotli_i32(ret, arg, 16);
1385
}
1386

1387
void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1388
{
1389
    tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1390
}
1391

1392
void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1393
{
1394
    tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1395
}
1396

1397
void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1398
{
1399
    tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1400
}
1401

1402
void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1403
{
1404
    tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1405
}
1406

1407
void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1408
{
1409
    TCGv_i32 t = tcg_temp_ebb_new_i32();
1410

1411
    tcg_gen_sari_i32(t, a, 31);
1412
    tcg_gen_xor_i32(ret, a, t);
1413
    tcg_gen_sub_i32(ret, ret, t);
1414
    tcg_temp_free_i32(t);
1415
}
1416

1417
void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1418
{
1419
    tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset);
1420
}
1421

1422
void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1423
{
1424
    tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset);
1425
}
1426

1427
void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1428
{
1429
    tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset);
1430
}
1431

1432
void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1433
{
1434
    tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset);
1435
}
1436

1437
void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1438
{
1439
    tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset);
1440
}
1441

1442
void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1443
{
1444
    tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset);
1445
}
1446

1447
void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1448
{
1449
    tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset);
1450
}
1451

1452
void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1453
{
1454
    tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset);
1455
}
1456

1457

1458
/* 64-bit ops */
1459

1460
void tcg_gen_discard_i64(TCGv_i64 arg)
1461
{
1462
    if (TCG_TARGET_REG_BITS == 64) {
1463
        tcg_gen_op1_i64(INDEX_op_discard, arg);
1464
    } else {
1465
        tcg_gen_discard_i32(TCGV_LOW(arg));
1466
        tcg_gen_discard_i32(TCGV_HIGH(arg));
1467
    }
1468
}
1469

1470
void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1471
{
1472
    if (ret == arg) {
1473
        return;
1474
    }
1475
    if (TCG_TARGET_REG_BITS == 64) {
1476
        tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg);
1477
    } else {
1478
        TCGTemp *ts = tcgv_i64_temp(arg);
1479

1480
        /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1481
        if (ts->kind == TEMP_CONST) {
1482
            tcg_gen_movi_i64(ret, ts->val);
1483
        } else {
1484
            tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1485
            tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1486
        }
1487
    }
1488
}
1489

1490
void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1491
{
1492
    if (TCG_TARGET_REG_BITS == 64) {
1493
        tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1494
    } else {
1495
        tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1496
        tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1497
    }
1498
}
1499

1500
void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1501
{
1502
    if (TCG_TARGET_REG_BITS == 64) {
1503
        tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset);
1504
    } else {
1505
        tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1506
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1507
    }
1508
}
1509

1510
void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1511
{
1512
    if (TCG_TARGET_REG_BITS == 64) {
1513
        tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset);
1514
    } else {
1515
        tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1516
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1517
    }
1518
}
1519

1520
void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1521
{
1522
    if (TCG_TARGET_REG_BITS == 64) {
1523
        tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset);
1524
    } else {
1525
        tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1526
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1527
    }
1528
}
1529

1530
void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1531
{
1532
    if (TCG_TARGET_REG_BITS == 64) {
1533
        tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset);
1534
    } else {
1535
        tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1536
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1537
    }
1538
}
1539

1540
void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1541
{
1542
    if (TCG_TARGET_REG_BITS == 64) {
1543
        tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset);
1544
    } else {
1545
        tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1546
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1547
    }
1548
}
1549

1550
void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1551
{
1552
    if (TCG_TARGET_REG_BITS == 64) {
1553
        tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset);
1554
    } else {
1555
        tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1556
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1557
    }
1558
}
1559

1560
void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1561
{
1562
    /*
1563
     * For 32-bit host, since arg2 and ret have different types,
1564
     * they cannot be the same temporary -- no chance of overlap.
1565
     */
1566
    if (TCG_TARGET_REG_BITS == 64) {
1567
        tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset);
1568
    } else if (HOST_BIG_ENDIAN) {
1569
        tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1570
        tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1571
    } else {
1572
        tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1573
        tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1574
    }
1575
}
1576

1577
void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1578
{
1579
    if (TCG_TARGET_REG_BITS == 64) {
1580
        tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset);
1581
    } else {
1582
        tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset);
1583
    }
1584
}
1585

1586
void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1587
{
1588
    if (TCG_TARGET_REG_BITS == 64) {
1589
        tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset);
1590
    } else {
1591
        tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset);
1592
    }
1593
}
1594

1595
void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1596
{
1597
    if (TCG_TARGET_REG_BITS == 64) {
1598
        tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset);
1599
    } else {
1600
        tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1601
    }
1602
}
1603

1604
void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1605
{
1606
    if (TCG_TARGET_REG_BITS == 64) {
1607
        tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset);
1608
    } else if (HOST_BIG_ENDIAN) {
1609
        tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1610
        tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1611
    } else {
1612
        tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1613
        tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1614
    }
1615
}
1616

1617
void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1618
{
1619
    if (TCG_TARGET_REG_BITS == 64) {
1620
        tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2);
1621
    } else {
1622
        tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1623
                         TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1624
    }
1625
}
1626

1627
void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1628
{
1629
    if (TCG_TARGET_REG_BITS == 64) {
1630
        tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2);
1631
    } else {
1632
        tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1),
1633
                         TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2));
1634
    }
1635
}
1636

1637
void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1638
{
1639
    if (TCG_TARGET_REG_BITS == 64) {
1640
        tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2);
1641
    } else {
1642
        tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1643
        tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1644
    }
1645
}
1646

1647
void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1648
{
1649
    if (TCG_TARGET_REG_BITS == 64) {
1650
        tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2);
1651
    } else {
1652
        tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1653
        tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1654
    }
1655
}
1656

1657
void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1658
{
1659
    if (TCG_TARGET_REG_BITS == 64) {
1660
        tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2);
1661
    } else {
1662
        tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1663
        tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1664
    }
1665
}
1666

1667
void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1668
{
1669
    if (TCG_TARGET_REG_BITS == 64) {
1670
        tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2);
1671
    } else {
1672
        gen_helper_shl_i64(ret, arg1, arg2);
1673
    }
1674
}
1675

1676
void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1677
{
1678
    if (TCG_TARGET_REG_BITS == 64) {
1679
        tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2);
1680
    } else {
1681
        gen_helper_shr_i64(ret, arg1, arg2);
1682
    }
1683
}
1684

1685
void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1686
{
1687
    if (TCG_TARGET_REG_BITS == 64) {
1688
        tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2);
1689
    } else {
1690
        gen_helper_sar_i64(ret, arg1, arg2);
1691
    }
1692
}
1693

1694
void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1695
{
1696
    TCGv_i64 t0;
1697
    TCGv_i32 t1;
1698

1699
    if (TCG_TARGET_REG_BITS == 64) {
1700
        tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2);
1701
        return;
1702
    }
1703

1704

1705
    t0 = tcg_temp_ebb_new_i64();
1706
    t1 = tcg_temp_ebb_new_i32();
1707

1708
    tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1709
                      TCGV_LOW(arg1), TCGV_LOW(arg2));
1710

1711
    tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1712
    tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1713
    tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1714
    tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1715

1716
    tcg_gen_mov_i64(ret, t0);
1717
    tcg_temp_free_i64(t0);
1718
    tcg_temp_free_i32(t1);
1719
}
1720

1721
void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1722
{
1723
    /* some cases can be optimized here */
1724
    if (arg2 == 0) {
1725
        tcg_gen_mov_i64(ret, arg1);
1726
    } else if (TCG_TARGET_REG_BITS == 64) {
1727
        tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1728
    } else {
1729
        tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1730
                         TCGV_LOW(arg1), TCGV_HIGH(arg1),
1731
                         tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1732
    }
1733
}
1734

1735
void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1736
{
1737
    if (arg1 == 0) {
1738
        tcg_gen_neg_i64(ret, arg2);
1739
    } else if (TCG_TARGET_REG_BITS == 64) {
1740
        tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1741
    } else {
1742
        tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1743
                         tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1744
                         TCGV_LOW(arg2), TCGV_HIGH(arg2));
1745
    }
1746
}
1747

1748
void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1749
{
1750
    tcg_gen_addi_i64(ret, arg1, -arg2);
1751
}
1752

1753
void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg)
1754
{
1755
    if (TCG_TARGET_REG_BITS == 64) {
1756
        tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg);
1757
    } else {
1758
        TCGv_i32 zero = tcg_constant_i32(0);
1759
        tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1760
                         zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg));
1761
    }
1762
}
1763

1764
void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1765
{
1766
    if (TCG_TARGET_REG_BITS == 32) {
1767
        tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1768
        tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1769
        return;
1770
    }
1771

1772
    /* Some cases can be optimized here.  */
1773
    switch (arg2) {
1774
    case 0:
1775
        tcg_gen_movi_i64(ret, 0);
1776
        return;
1777
    case -1:
1778
        tcg_gen_mov_i64(ret, arg1);
1779
        return;
1780
    case 0xff:
1781
        /* Don't recurse with tcg_gen_ext8u_i64.  */
1782
        if (TCG_TARGET_HAS_ext8u_i64) {
1783
            tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1784
            return;
1785
        }
1786
        break;
1787
    case 0xffff:
1788
        if (TCG_TARGET_HAS_ext16u_i64) {
1789
            tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1790
            return;
1791
        }
1792
        break;
1793
    case 0xffffffffu:
1794
        if (TCG_TARGET_HAS_ext32u_i64) {
1795
            tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1796
            return;
1797
        }
1798
        break;
1799
    }
1800

1801
    tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1802
}
1803

1804
void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1805
{
1806
    if (TCG_TARGET_REG_BITS == 32) {
1807
        tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1808
        tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1809
        return;
1810
    }
1811
    /* Some cases can be optimized here.  */
1812
    if (arg2 == -1) {
1813
        tcg_gen_movi_i64(ret, -1);
1814
    } else if (arg2 == 0) {
1815
        tcg_gen_mov_i64(ret, arg1);
1816
    } else {
1817
        tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1818
    }
1819
}
1820

1821
void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1822
{
1823
    if (TCG_TARGET_REG_BITS == 32) {
1824
        tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1825
        tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1826
        return;
1827
    }
1828
    /* Some cases can be optimized here.  */
1829
    if (arg2 == 0) {
1830
        tcg_gen_mov_i64(ret, arg1);
1831
    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1832
        /* Don't recurse with tcg_gen_not_i64.  */
1833
        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1834
    } else {
1835
        tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1836
    }
1837
}
1838

1839
static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1840
                                      unsigned c, bool right, bool arith)
1841
{
1842
    tcg_debug_assert(c < 64);
1843
    if (c == 0) {
1844
        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1845
        tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1846
    } else if (c >= 32) {
1847
        c -= 32;
1848
        if (right) {
1849
            if (arith) {
1850
                tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1851
                tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1852
            } else {
1853
                tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1854
                tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1855
            }
1856
        } else {
1857
            tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1858
            tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1859
        }
1860
    } else if (right) {
1861
        if (TCG_TARGET_HAS_extract2_i32) {
1862
            tcg_gen_extract2_i32(TCGV_LOW(ret),
1863
                                 TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1864
        } else {
1865
            tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1866
            tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1867
                                TCGV_HIGH(arg1), 32 - c, c);
1868
        }
1869
        if (arith) {
1870
            tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1871
        } else {
1872
            tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1873
        }
1874
    } else {
1875
        if (TCG_TARGET_HAS_extract2_i32) {
1876
            tcg_gen_extract2_i32(TCGV_HIGH(ret),
1877
                                 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1878
        } else {
1879
            TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1880
            tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1881
            tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1882
                                TCGV_HIGH(arg1), c, 32 - c);
1883
            tcg_temp_free_i32(t0);
1884
        }
1885
        tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1886
    }
1887
}
1888

1889
void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1890
{
1891
    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1892
    if (TCG_TARGET_REG_BITS == 32) {
1893
        tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1894
    } else if (arg2 == 0) {
1895
        tcg_gen_mov_i64(ret, arg1);
1896
    } else {
1897
        tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1898
    }
1899
}
1900

1901
void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1902
{
1903
    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1904
    if (TCG_TARGET_REG_BITS == 32) {
1905
        tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1906
    } else if (arg2 == 0) {
1907
        tcg_gen_mov_i64(ret, arg1);
1908
    } else {
1909
        tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1910
    }
1911
}
1912

1913
void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1914
{
1915
    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1916
    if (TCG_TARGET_REG_BITS == 32) {
1917
        tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1918
    } else if (arg2 == 0) {
1919
        tcg_gen_mov_i64(ret, arg1);
1920
    } else {
1921
        tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1922
    }
1923
}
1924

1925
void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1926
{
1927
    if (cond == TCG_COND_ALWAYS) {
1928
        tcg_gen_br(l);
1929
    } else if (cond != TCG_COND_NEVER) {
1930
        if (TCG_TARGET_REG_BITS == 32) {
1931
            tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1932
                              TCGV_HIGH(arg1), TCGV_LOW(arg2),
1933
                              TCGV_HIGH(arg2), cond, label_arg(l));
1934
        } else {
1935
            tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1936
                              label_arg(l));
1937
        }
1938
        add_last_as_label_use(l);
1939
    }
1940
}
1941

1942
void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1943
{
1944
    if (TCG_TARGET_REG_BITS == 64) {
1945
        tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1946
    } else if (cond == TCG_COND_ALWAYS) {
1947
        tcg_gen_br(l);
1948
    } else if (cond != TCG_COND_NEVER) {
1949
        tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1950
                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1951
                          tcg_constant_i32(arg2),
1952
                          tcg_constant_i32(arg2 >> 32),
1953
                          cond, label_arg(l));
1954
        add_last_as_label_use(l);
1955
    }
1956
}
1957

1958
void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1959
                         TCGv_i64 arg1, TCGv_i64 arg2)
1960
{
1961
    if (cond == TCG_COND_ALWAYS) {
1962
        tcg_gen_movi_i64(ret, 1);
1963
    } else if (cond == TCG_COND_NEVER) {
1964
        tcg_gen_movi_i64(ret, 0);
1965
    } else {
1966
        if (TCG_TARGET_REG_BITS == 32) {
1967
            tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1968
                             TCGV_LOW(arg1), TCGV_HIGH(arg1),
1969
                             TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1970
            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1971
        } else {
1972
            tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1973
        }
1974
    }
1975
}
1976

1977
void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1978
                          TCGv_i64 arg1, int64_t arg2)
1979
{
1980
    if (TCG_TARGET_REG_BITS == 64) {
1981
        tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1982
    } else if (cond == TCG_COND_ALWAYS) {
1983
        tcg_gen_movi_i64(ret, 1);
1984
    } else if (cond == TCG_COND_NEVER) {
1985
        tcg_gen_movi_i64(ret, 0);
1986
    } else {
1987
        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1988
                         TCGV_LOW(arg1), TCGV_HIGH(arg1),
1989
                         tcg_constant_i32(arg2),
1990
                         tcg_constant_i32(arg2 >> 32), cond);
1991
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1992
    }
1993
}
1994

1995
void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret,
1996
                             TCGv_i64 arg1, int64_t arg2)
1997
{
1998
    tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1999
}
2000

2001
void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret,
2002
                            TCGv_i64 arg1, TCGv_i64 arg2)
2003
{
2004
    if (cond == TCG_COND_ALWAYS) {
2005
        tcg_gen_movi_i64(ret, -1);
2006
    } else if (cond == TCG_COND_NEVER) {
2007
        tcg_gen_movi_i64(ret, 0);
2008
    } else if (TCG_TARGET_HAS_negsetcond_i64) {
2009
        tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond);
2010
    } else if (TCG_TARGET_REG_BITS == 32) {
2011
        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
2012
                         TCGV_LOW(arg1), TCGV_HIGH(arg1),
2013
                         TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
2014
        tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret));
2015
        tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret));
2016
    } else {
2017
        tcg_gen_setcond_i64(cond, ret, arg1, arg2);
2018
        tcg_gen_neg_i64(ret, ret);
2019
    }
2020
}
2021

2022
void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2023
{
2024
    if (arg2 == 0) {
2025
        tcg_gen_movi_i64(ret, 0);
2026
    } else if (is_power_of_2(arg2)) {
2027
        tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
2028
    } else {
2029
        tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2));
2030
    }
2031
}
2032

2033
void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2034
{
2035
    if (TCG_TARGET_HAS_div_i64) {
2036
        tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
2037
    } else if (TCG_TARGET_HAS_div2_i64) {
2038
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2039
        tcg_gen_sari_i64(t0, arg1, 63);
2040
        tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
2041
        tcg_temp_free_i64(t0);
2042
    } else {
2043
        gen_helper_div_i64(ret, arg1, arg2);
2044
    }
2045
}
2046

2047
void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2048
{
2049
    if (TCG_TARGET_HAS_rem_i64) {
2050
        tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
2051
    } else if (TCG_TARGET_HAS_div_i64) {
2052
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2053
        tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
2054
        tcg_gen_mul_i64(t0, t0, arg2);
2055
        tcg_gen_sub_i64(ret, arg1, t0);
2056
        tcg_temp_free_i64(t0);
2057
    } else if (TCG_TARGET_HAS_div2_i64) {
2058
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2059
        tcg_gen_sari_i64(t0, arg1, 63);
2060
        tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
2061
        tcg_temp_free_i64(t0);
2062
    } else {
2063
        gen_helper_rem_i64(ret, arg1, arg2);
2064
    }
2065
}
2066

2067
void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2068
{
2069
    if (TCG_TARGET_HAS_div_i64) {
2070
        tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
2071
    } else if (TCG_TARGET_HAS_div2_i64) {
2072
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2073
        TCGv_i64 zero = tcg_constant_i64(0);
2074
        tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, zero, arg2);
2075
        tcg_temp_free_i64(t0);
2076
    } else {
2077
        gen_helper_divu_i64(ret, arg1, arg2);
2078
    }
2079
}
2080

2081
void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2082
{
2083
    if (TCG_TARGET_HAS_rem_i64) {
2084
        tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
2085
    } else if (TCG_TARGET_HAS_div_i64) {
2086
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2087
        tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
2088
        tcg_gen_mul_i64(t0, t0, arg2);
2089
        tcg_gen_sub_i64(ret, arg1, t0);
2090
        tcg_temp_free_i64(t0);
2091
    } else if (TCG_TARGET_HAS_div2_i64) {
2092
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2093
        TCGv_i64 zero = tcg_constant_i64(0);
2094
        tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, zero, arg2);
2095
        tcg_temp_free_i64(t0);
2096
    } else {
2097
        gen_helper_remu_i64(ret, arg1, arg2);
2098
    }
2099
}
2100

2101
void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
2102
{
2103
    if (TCG_TARGET_REG_BITS == 32) {
2104
        tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2105
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2106
    } else if (TCG_TARGET_HAS_ext8s_i64) {
2107
        tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
2108
    } else {
2109
        tcg_gen_shli_i64(ret, arg, 56);
2110
        tcg_gen_sari_i64(ret, ret, 56);
2111
    }
2112
}
2113

2114
void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
2115
{
2116
    if (TCG_TARGET_REG_BITS == 32) {
2117
        tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2118
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2119
    } else if (TCG_TARGET_HAS_ext16s_i64) {
2120
        tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
2121
    } else {
2122
        tcg_gen_shli_i64(ret, arg, 48);
2123
        tcg_gen_sari_i64(ret, ret, 48);
2124
    }
2125
}
2126

2127
void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
2128
{
2129
    if (TCG_TARGET_REG_BITS == 32) {
2130
        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2131
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2132
    } else if (TCG_TARGET_HAS_ext32s_i64) {
2133
        tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
2134
    } else {
2135
        tcg_gen_shli_i64(ret, arg, 32);
2136
        tcg_gen_sari_i64(ret, ret, 32);
2137
    }
2138
}
2139

2140
void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
2141
{
2142
    if (TCG_TARGET_REG_BITS == 32) {
2143
        tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2144
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2145
    } else if (TCG_TARGET_HAS_ext8u_i64) {
2146
        tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
2147
    } else {
2148
        tcg_gen_andi_i64(ret, arg, 0xffu);
2149
    }
2150
}
2151

2152
void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
2153
{
2154
    if (TCG_TARGET_REG_BITS == 32) {
2155
        tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2156
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2157
    } else if (TCG_TARGET_HAS_ext16u_i64) {
2158
        tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
2159
    } else {
2160
        tcg_gen_andi_i64(ret, arg, 0xffffu);
2161
    }
2162
}
2163

2164
void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
2165
{
2166
    if (TCG_TARGET_REG_BITS == 32) {
2167
        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2168
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2169
    } else if (TCG_TARGET_HAS_ext32u_i64) {
2170
        tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
2171
    } else {
2172
        tcg_gen_andi_i64(ret, arg, 0xffffffffu);
2173
    }
2174
}
2175

2176
/*
2177
 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value.
2178
 *
2179
 * Byte pattern: xxxxxxxxab -> yyyyyyyyba
2180
 *
2181
 * With TCG_BSWAP_IZ, x == zero, else undefined.
2182
 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
2183
 */
2184
void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
2185
{
2186
    /* Only one extension flag may be present. */
2187
    tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
2188

2189
    if (TCG_TARGET_REG_BITS == 32) {
2190
        tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
2191
        if (flags & TCG_BSWAP_OS) {
2192
            tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2193
        } else {
2194
            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2195
        }
2196
    } else if (TCG_TARGET_HAS_bswap16_i64) {
2197
        tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
2198
    } else {
2199
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2200
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2201

2202
                                            /* arg = ......ab or xxxxxxab */
2203
        tcg_gen_shri_i64(t0, arg, 8);       /*  t0 = .......a or .xxxxxxa */
2204
        if (!(flags & TCG_BSWAP_IZ)) {
2205
            tcg_gen_ext8u_i64(t0, t0);      /*  t0 = .......a */
2206
        }
2207

2208
        if (flags & TCG_BSWAP_OS) {
2209
            tcg_gen_shli_i64(t1, arg, 56);  /*  t1 = b....... */
2210
            tcg_gen_sari_i64(t1, t1, 48);   /*  t1 = ssssssb. */
2211
        } else if (flags & TCG_BSWAP_OZ) {
2212
            tcg_gen_ext8u_i64(t1, arg);     /*  t1 = .......b */
2213
            tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = ......b. */
2214
        } else {
2215
            tcg_gen_shli_i64(t1, arg, 8);   /*  t1 = xxxxxab. */
2216
        }
2217

2218
        tcg_gen_or_i64(ret, t0, t1);        /* ret = ......ba (OZ) */
2219
                                            /*       ssssssba (OS) */
2220
                                            /*       xxxxxaba (no flag) */
2221
        tcg_temp_free_i64(t0);
2222
        tcg_temp_free_i64(t1);
2223
    }
2224
}
2225

2226
/*
2227
 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value.
2228
 *
2229
 * Byte pattern: xxxxabcd -> yyyydcba
2230
 *
2231
 * With TCG_BSWAP_IZ, x == zero, else undefined.
2232
 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
2233
 */
2234
void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
2235
{
2236
    /* Only one extension flag may be present. */
2237
    tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
2238

2239
    if (TCG_TARGET_REG_BITS == 32) {
2240
        tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2241
        if (flags & TCG_BSWAP_OS) {
2242
            tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2243
        } else {
2244
            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2245
        }
2246
    } else if (TCG_TARGET_HAS_bswap32_i64) {
2247
        tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
2248
    } else {
2249
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2250
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2251
        TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
2252

2253
                                            /* arg = xxxxabcd */
2254
        tcg_gen_shri_i64(t0, arg, 8);       /*  t0 = .xxxxabc */
2255
        tcg_gen_and_i64(t1, arg, t2);       /*  t1 = .....b.d */
2256
        tcg_gen_and_i64(t0, t0, t2);        /*  t0 = .....a.c */
2257
        tcg_gen_shli_i64(t1, t1, 8);        /*  t1 = ....b.d. */
2258
        tcg_gen_or_i64(ret, t0, t1);        /* ret = ....badc */
2259

2260
        tcg_gen_shli_i64(t1, ret, 48);      /*  t1 = dc...... */
2261
        tcg_gen_shri_i64(t0, ret, 16);      /*  t0 = ......ba */
2262
        if (flags & TCG_BSWAP_OS) {
2263
            tcg_gen_sari_i64(t1, t1, 32);   /*  t1 = ssssdc.. */
2264
        } else {
2265
            tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
2266
        }
2267
        tcg_gen_or_i64(ret, t0, t1);        /* ret = ssssdcba (OS) */
2268
                                            /*       ....dcba (else) */
2269

2270
        tcg_temp_free_i64(t0);
2271
        tcg_temp_free_i64(t1);
2272
    }
2273
}
2274

2275
/*
2276
 * bswap64_i64: 64-bit byte swap on a 64-bit value.
2277
 *
2278
 * Byte pattern: abcdefgh -> hgfedcba
2279
 */
2280
void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
2281
{
2282
    if (TCG_TARGET_REG_BITS == 32) {
2283
        TCGv_i32 t0, t1;
2284
        t0 = tcg_temp_ebb_new_i32();
2285
        t1 = tcg_temp_ebb_new_i32();
2286

2287
        tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
2288
        tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
2289
        tcg_gen_mov_i32(TCGV_LOW(ret), t1);
2290
        tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
2291
        tcg_temp_free_i32(t0);
2292
        tcg_temp_free_i32(t1);
2293
    } else if (TCG_TARGET_HAS_bswap64_i64) {
2294
        tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
2295
    } else {
2296
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2297
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2298
        TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2299

2300
                                        /* arg = abcdefgh */
2301
        tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
2302
        tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
2303
        tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
2304
        tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
2305
        tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
2306
        tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
2307

2308
        tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
2309
        tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
2310
        tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
2311
        tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
2312
        tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
2313
        tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
2314

2315
        tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
2316
        tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
2317
        tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
2318

2319
        tcg_temp_free_i64(t0);
2320
        tcg_temp_free_i64(t1);
2321
        tcg_temp_free_i64(t2);
2322
    }
2323
}
2324

2325
/*
2326
 * hswap_i64: Swap 16-bit halfwords within a 64-bit value.
2327
 * See also include/qemu/bitops.h, hswap64.
2328
 *
2329
 * Byte pattern: abcdefgh -> ghefcdab
2330
 */
2331
void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
2332
{
2333
    uint64_t m = 0x0000ffff0000ffffull;
2334
    TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2335
    TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2336

2337
                                        /* arg = abcdefgh */
2338
    tcg_gen_rotli_i64(t1, arg, 32);     /*  t1 = efghabcd */
2339
    tcg_gen_andi_i64(t0, t1, m);        /*  t0 = ..gh..cd */
2340
    tcg_gen_shli_i64(t0, t0, 16);       /*  t0 = gh..cd.. */
2341
    tcg_gen_shri_i64(t1, t1, 16);       /*  t1 = ..efghab */
2342
    tcg_gen_andi_i64(t1, t1, m);        /*  t1 = ..ef..ab */
2343
    tcg_gen_or_i64(ret, t0, t1);        /* ret = ghefcdab */
2344

2345
    tcg_temp_free_i64(t0);
2346
    tcg_temp_free_i64(t1);
2347
}
2348

2349
/*
2350
 * wswap_i64: Swap 32-bit words within a 64-bit value.
2351
 *
2352
 * Byte pattern: abcdefgh -> efghabcd
2353
 */
2354
void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
2355
{
2356
    /* Swapping 2 32-bit elements is a rotate. */
2357
    tcg_gen_rotli_i64(ret, arg, 32);
2358
}
2359

2360
void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
2361
{
2362
    if (TCG_TARGET_REG_BITS == 32) {
2363
        tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2364
        tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
2365
    } else if (TCG_TARGET_HAS_not_i64) {
2366
        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
2367
    } else {
2368
        tcg_gen_xori_i64(ret, arg, -1);
2369
    }
2370
}
2371

2372
void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2373
{
2374
    if (TCG_TARGET_REG_BITS == 32) {
2375
        tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
2376
        tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
2377
    } else if (TCG_TARGET_HAS_andc_i64) {
2378
        tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
2379
    } else {
2380
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2381
        tcg_gen_not_i64(t0, arg2);
2382
        tcg_gen_and_i64(ret, arg1, t0);
2383
        tcg_temp_free_i64(t0);
2384
    }
2385
}
2386

2387
void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2388
{
2389
    if (TCG_TARGET_REG_BITS == 32) {
2390
        tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
2391
        tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
2392
    } else if (TCG_TARGET_HAS_eqv_i64) {
2393
        tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
2394
    } else {
2395
        tcg_gen_xor_i64(ret, arg1, arg2);
2396
        tcg_gen_not_i64(ret, ret);
2397
    }
2398
}
2399

2400
void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2401
{
2402
    if (TCG_TARGET_REG_BITS == 32) {
2403
        tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
2404
        tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
2405
    } else if (TCG_TARGET_HAS_nand_i64) {
2406
        tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
2407
    } else {
2408
        tcg_gen_and_i64(ret, arg1, arg2);
2409
        tcg_gen_not_i64(ret, ret);
2410
    }
2411
}
2412

2413
void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2414
{
2415
    if (TCG_TARGET_REG_BITS == 32) {
2416
        tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
2417
        tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
2418
    } else if (TCG_TARGET_HAS_nor_i64) {
2419
        tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
2420
    } else {
2421
        tcg_gen_or_i64(ret, arg1, arg2);
2422
        tcg_gen_not_i64(ret, ret);
2423
    }
2424
}
2425

2426
void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2427
{
2428
    if (TCG_TARGET_REG_BITS == 32) {
2429
        tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
2430
        tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
2431
    } else if (TCG_TARGET_HAS_orc_i64) {
2432
        tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
2433
    } else {
2434
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2435
        tcg_gen_not_i64(t0, arg2);
2436
        tcg_gen_or_i64(ret, arg1, t0);
2437
        tcg_temp_free_i64(t0);
2438
    }
2439
}
2440

2441
void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2442
{
2443
    if (TCG_TARGET_HAS_clz_i64) {
2444
        tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
2445
    } else {
2446
        gen_helper_clz_i64(ret, arg1, arg2);
2447
    }
2448
}
2449

2450
void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
2451
{
2452
    if (TCG_TARGET_REG_BITS == 32
2453
        && TCG_TARGET_HAS_clz_i32
2454
        && arg2 <= 0xffffffffu) {
2455
        TCGv_i32 t = tcg_temp_ebb_new_i32();
2456
        tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
2457
        tcg_gen_addi_i32(t, t, 32);
2458
        tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
2459
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2460
        tcg_temp_free_i32(t);
2461
    } else {
2462
        tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2));
2463
    }
2464
}
2465

2466
void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2467
{
2468
    if (TCG_TARGET_HAS_ctz_i64) {
2469
        tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
2470
    } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
2471
        TCGv_i64 z, t = tcg_temp_ebb_new_i64();
2472

2473
        if (TCG_TARGET_HAS_ctpop_i64) {
2474
            tcg_gen_subi_i64(t, arg1, 1);
2475
            tcg_gen_andc_i64(t, t, arg1);
2476
            tcg_gen_ctpop_i64(t, t);
2477
        } else {
2478
            /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
2479
            tcg_gen_neg_i64(t, arg1);
2480
            tcg_gen_and_i64(t, t, arg1);
2481
            tcg_gen_clzi_i64(t, t, 64);
2482
            tcg_gen_xori_i64(t, t, 63);
2483
        }
2484
        z = tcg_constant_i64(0);
2485
        tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
2486
        tcg_temp_free_i64(t);
2487
        tcg_temp_free_i64(z);
2488
    } else {
2489
        gen_helper_ctz_i64(ret, arg1, arg2);
2490
    }
2491
}
2492

2493
void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
2494
{
2495
    if (TCG_TARGET_REG_BITS == 32
2496
        && TCG_TARGET_HAS_ctz_i32
2497
        && arg2 <= 0xffffffffu) {
2498
        TCGv_i32 t32 = tcg_temp_ebb_new_i32();
2499
        tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
2500
        tcg_gen_addi_i32(t32, t32, 32);
2501
        tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
2502
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2503
        tcg_temp_free_i32(t32);
2504
    } else if (!TCG_TARGET_HAS_ctz_i64
2505
               && TCG_TARGET_HAS_ctpop_i64
2506
               && arg2 == 64) {
2507
        /* This equivalence has the advantage of not requiring a fixup.  */
2508
        TCGv_i64 t = tcg_temp_ebb_new_i64();
2509
        tcg_gen_subi_i64(t, arg1, 1);
2510
        tcg_gen_andc_i64(t, t, arg1);
2511
        tcg_gen_ctpop_i64(ret, t);
2512
        tcg_temp_free_i64(t);
2513
    } else {
2514
        tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2));
2515
    }
2516
}
2517

2518
void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
2519
{
2520
    if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
2521
        TCGv_i64 t = tcg_temp_ebb_new_i64();
2522
        tcg_gen_sari_i64(t, arg, 63);
2523
        tcg_gen_xor_i64(t, t, arg);
2524
        tcg_gen_clzi_i64(t, t, 64);
2525
        tcg_gen_subi_i64(ret, t, 1);
2526
        tcg_temp_free_i64(t);
2527
    } else {
2528
        gen_helper_clrsb_i64(ret, arg);
2529
    }
2530
}
2531

2532
void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2533
{
2534
    if (TCG_TARGET_HAS_ctpop_i64) {
2535
        tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
2536
    } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
2537
        tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2538
        tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2539
        tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
2540
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2541
    } else {
2542
        gen_helper_ctpop_i64(ret, arg1);
2543
    }
2544
}
2545

2546
void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2547
{
2548
    if (TCG_TARGET_HAS_rot_i64) {
2549
        tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
2550
    } else {
2551
        TCGv_i64 t0, t1;
2552
        t0 = tcg_temp_ebb_new_i64();
2553
        t1 = tcg_temp_ebb_new_i64();
2554
        tcg_gen_shl_i64(t0, arg1, arg2);
2555
        tcg_gen_subfi_i64(t1, 64, arg2);
2556
        tcg_gen_shr_i64(t1, arg1, t1);
2557
        tcg_gen_or_i64(ret, t0, t1);
2558
        tcg_temp_free_i64(t0);
2559
        tcg_temp_free_i64(t1);
2560
    }
2561
}
2562

2563
void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2564
{
2565
    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2566
    /* some cases can be optimized here */
2567
    if (arg2 == 0) {
2568
        tcg_gen_mov_i64(ret, arg1);
2569
    } else if (TCG_TARGET_HAS_rot_i64) {
2570
        tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2571
    } else {
2572
        TCGv_i64 t0, t1;
2573
        t0 = tcg_temp_ebb_new_i64();
2574
        t1 = tcg_temp_ebb_new_i64();
2575
        tcg_gen_shli_i64(t0, arg1, arg2);
2576
        tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2577
        tcg_gen_or_i64(ret, t0, t1);
2578
        tcg_temp_free_i64(t0);
2579
        tcg_temp_free_i64(t1);
2580
    }
2581
}
2582

2583
void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2584
{
2585
    if (TCG_TARGET_HAS_rot_i64) {
2586
        tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2587
    } else {
2588
        TCGv_i64 t0, t1;
2589
        t0 = tcg_temp_ebb_new_i64();
2590
        t1 = tcg_temp_ebb_new_i64();
2591
        tcg_gen_shr_i64(t0, arg1, arg2);
2592
        tcg_gen_subfi_i64(t1, 64, arg2);
2593
        tcg_gen_shl_i64(t1, arg1, t1);
2594
        tcg_gen_or_i64(ret, t0, t1);
2595
        tcg_temp_free_i64(t0);
2596
        tcg_temp_free_i64(t1);
2597
    }
2598
}
2599

2600
void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2601
{
2602
    tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2603
    /* some cases can be optimized here */
2604
    if (arg2 == 0) {
2605
        tcg_gen_mov_i64(ret, arg1);
2606
    } else {
2607
        tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2608
    }
2609
}
2610

2611
void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2612
                         unsigned int ofs, unsigned int len)
2613
{
2614
    uint64_t mask;
2615
    TCGv_i64 t1;
2616

2617
    tcg_debug_assert(ofs < 64);
2618
    tcg_debug_assert(len > 0);
2619
    tcg_debug_assert(len <= 64);
2620
    tcg_debug_assert(ofs + len <= 64);
2621

2622
    if (len == 64) {
2623
        tcg_gen_mov_i64(ret, arg2);
2624
        return;
2625
    }
2626
    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2627
        tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2628
        return;
2629
    }
2630

2631
    if (TCG_TARGET_REG_BITS == 32) {
2632
        if (ofs >= 32) {
2633
            tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2634
                                TCGV_LOW(arg2), ofs - 32, len);
2635
            tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2636
            return;
2637
        }
2638
        if (ofs + len <= 32) {
2639
            tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2640
                                TCGV_LOW(arg2), ofs, len);
2641
            tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2642
            return;
2643
        }
2644
    }
2645

2646
    t1 = tcg_temp_ebb_new_i64();
2647

2648
    if (TCG_TARGET_HAS_extract2_i64) {
2649
        if (ofs + len == 64) {
2650
            tcg_gen_shli_i64(t1, arg1, len);
2651
            tcg_gen_extract2_i64(ret, t1, arg2, len);
2652
            goto done;
2653
        }
2654
        if (ofs == 0) {
2655
            tcg_gen_extract2_i64(ret, arg1, arg2, len);
2656
            tcg_gen_rotli_i64(ret, ret, len);
2657
            goto done;
2658
        }
2659
    }
2660

2661
    mask = (1ull << len) - 1;
2662
    if (ofs + len < 64) {
2663
        tcg_gen_andi_i64(t1, arg2, mask);
2664
        tcg_gen_shli_i64(t1, t1, ofs);
2665
    } else {
2666
        tcg_gen_shli_i64(t1, arg2, ofs);
2667
    }
2668
    tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2669
    tcg_gen_or_i64(ret, ret, t1);
2670
 done:
2671
    tcg_temp_free_i64(t1);
2672
}
2673

2674
void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2675
                           unsigned int ofs, unsigned int len)
2676
{
2677
    tcg_debug_assert(ofs < 64);
2678
    tcg_debug_assert(len > 0);
2679
    tcg_debug_assert(len <= 64);
2680
    tcg_debug_assert(ofs + len <= 64);
2681

2682
    if (ofs + len == 64) {
2683
        tcg_gen_shli_i64(ret, arg, ofs);
2684
    } else if (ofs == 0) {
2685
        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2686
    } else if (TCG_TARGET_HAS_deposit_i64
2687
               && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2688
        TCGv_i64 zero = tcg_constant_i64(0);
2689
        tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2690
    } else {
2691
        if (TCG_TARGET_REG_BITS == 32) {
2692
            if (ofs >= 32) {
2693
                tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2694
                                      ofs - 32, len);
2695
                tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2696
                return;
2697
            }
2698
            if (ofs + len <= 32) {
2699
                tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2700
                tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2701
                return;
2702
            }
2703
        }
2704
        /* To help two-operand hosts we prefer to zero-extend first,
2705
           which allows ARG to stay live.  */
2706
        switch (len) {
2707
        case 32:
2708
            if (TCG_TARGET_HAS_ext32u_i64) {
2709
                tcg_gen_ext32u_i64(ret, arg);
2710
                tcg_gen_shli_i64(ret, ret, ofs);
2711
                return;
2712
            }
2713
            break;
2714
        case 16:
2715
            if (TCG_TARGET_HAS_ext16u_i64) {
2716
                tcg_gen_ext16u_i64(ret, arg);
2717
                tcg_gen_shli_i64(ret, ret, ofs);
2718
                return;
2719
            }
2720
            break;
2721
        case 8:
2722
            if (TCG_TARGET_HAS_ext8u_i64) {
2723
                tcg_gen_ext8u_i64(ret, arg);
2724
                tcg_gen_shli_i64(ret, ret, ofs);
2725
                return;
2726
            }
2727
            break;
2728
        }
2729
        /* Otherwise prefer zero-extension over AND for code size.  */
2730
        switch (ofs + len) {
2731
        case 32:
2732
            if (TCG_TARGET_HAS_ext32u_i64) {
2733
                tcg_gen_shli_i64(ret, arg, ofs);
2734
                tcg_gen_ext32u_i64(ret, ret);
2735
                return;
2736
            }
2737
            break;
2738
        case 16:
2739
            if (TCG_TARGET_HAS_ext16u_i64) {
2740
                tcg_gen_shli_i64(ret, arg, ofs);
2741
                tcg_gen_ext16u_i64(ret, ret);
2742
                return;
2743
            }
2744
            break;
2745
        case 8:
2746
            if (TCG_TARGET_HAS_ext8u_i64) {
2747
                tcg_gen_shli_i64(ret, arg, ofs);
2748
                tcg_gen_ext8u_i64(ret, ret);
2749
                return;
2750
            }
2751
            break;
2752
        }
2753
        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2754
        tcg_gen_shli_i64(ret, ret, ofs);
2755
    }
2756
}
2757

2758
void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2759
                         unsigned int ofs, unsigned int len)
2760
{
2761
    tcg_debug_assert(ofs < 64);
2762
    tcg_debug_assert(len > 0);
2763
    tcg_debug_assert(len <= 64);
2764
    tcg_debug_assert(ofs + len <= 64);
2765

2766
    /* Canonicalize certain special cases, even if extract is supported.  */
2767
    if (ofs + len == 64) {
2768
        tcg_gen_shri_i64(ret, arg, 64 - len);
2769
        return;
2770
    }
2771
    if (ofs == 0) {
2772
        tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2773
        return;
2774
    }
2775

2776
    if (TCG_TARGET_REG_BITS == 32) {
2777
        /* Look for a 32-bit extract within one of the two words.  */
2778
        if (ofs >= 32) {
2779
            tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2780
            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2781
            return;
2782
        }
2783
        if (ofs + len <= 32) {
2784
            tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2785
            tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2786
            return;
2787
        }
2788
        /* The field is split across two words.  One double-word
2789
           shift is better than two double-word shifts.  */
2790
        goto do_shift_and;
2791
    }
2792

2793
    if (TCG_TARGET_HAS_extract_i64
2794
        && TCG_TARGET_extract_i64_valid(ofs, len)) {
2795
        tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2796
        return;
2797
    }
2798

2799
    /* Assume that zero-extension, if available, is cheaper than a shift.  */
2800
    switch (ofs + len) {
2801
    case 32:
2802
        if (TCG_TARGET_HAS_ext32u_i64) {
2803
            tcg_gen_ext32u_i64(ret, arg);
2804
            tcg_gen_shri_i64(ret, ret, ofs);
2805
            return;
2806
        }
2807
        break;
2808
    case 16:
2809
        if (TCG_TARGET_HAS_ext16u_i64) {
2810
            tcg_gen_ext16u_i64(ret, arg);
2811
            tcg_gen_shri_i64(ret, ret, ofs);
2812
            return;
2813
        }
2814
        break;
2815
    case 8:
2816
        if (TCG_TARGET_HAS_ext8u_i64) {
2817
            tcg_gen_ext8u_i64(ret, arg);
2818
            tcg_gen_shri_i64(ret, ret, ofs);
2819
            return;
2820
        }
2821
        break;
2822
    }
2823

2824
    /* ??? Ideally we'd know what values are available for immediate AND.
2825
       Assume that 8 bits are available, plus the special cases of 16 and 32,
2826
       so that we get ext8u, ext16u, and ext32u.  */
2827
    switch (len) {
2828
    case 1 ... 8: case 16: case 32:
2829
    do_shift_and:
2830
        tcg_gen_shri_i64(ret, arg, ofs);
2831
        tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2832
        break;
2833
    default:
2834
        tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2835
        tcg_gen_shri_i64(ret, ret, 64 - len);
2836
        break;
2837
    }
2838
}
2839

2840
void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2841
                          unsigned int ofs, unsigned int len)
2842
{
2843
    tcg_debug_assert(ofs < 64);
2844
    tcg_debug_assert(len > 0);
2845
    tcg_debug_assert(len <= 64);
2846
    tcg_debug_assert(ofs + len <= 64);
2847

2848
    /* Canonicalize certain special cases, even if sextract is supported.  */
2849
    if (ofs + len == 64) {
2850
        tcg_gen_sari_i64(ret, arg, 64 - len);
2851
        return;
2852
    }
2853
    if (ofs == 0) {
2854
        switch (len) {
2855
        case 32:
2856
            tcg_gen_ext32s_i64(ret, arg);
2857
            return;
2858
        case 16:
2859
            tcg_gen_ext16s_i64(ret, arg);
2860
            return;
2861
        case 8:
2862
            tcg_gen_ext8s_i64(ret, arg);
2863
            return;
2864
        }
2865
    }
2866

2867
    if (TCG_TARGET_REG_BITS == 32) {
2868
        /* Look for a 32-bit extract within one of the two words.  */
2869
        if (ofs >= 32) {
2870
            tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2871
        } else if (ofs + len <= 32) {
2872
            tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2873
        } else if (ofs == 0) {
2874
            tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2875
            tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2876
            return;
2877
        } else if (len > 32) {
2878
            TCGv_i32 t = tcg_temp_ebb_new_i32();
2879
            /* Extract the bits for the high word normally.  */
2880
            tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2881
            /* Shift the field down for the low part.  */
2882
            tcg_gen_shri_i64(ret, arg, ofs);
2883
            /* Overwrite the shift into the high part.  */
2884
            tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2885
            tcg_temp_free_i32(t);
2886
            return;
2887
        } else {
2888
            /* Shift the field down for the low part, such that the
2889
               field sits at the MSB.  */
2890
            tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2891
            /* Shift the field down from the MSB, sign extending.  */
2892
            tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2893
        }
2894
        /* Sign-extend the field from 32 bits.  */
2895
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2896
        return;
2897
    }
2898

2899
    if (TCG_TARGET_HAS_sextract_i64
2900
        && TCG_TARGET_extract_i64_valid(ofs, len)) {
2901
        tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2902
        return;
2903
    }
2904

2905
    /* Assume that sign-extension, if available, is cheaper than a shift.  */
2906
    switch (ofs + len) {
2907
    case 32:
2908
        if (TCG_TARGET_HAS_ext32s_i64) {
2909
            tcg_gen_ext32s_i64(ret, arg);
2910
            tcg_gen_sari_i64(ret, ret, ofs);
2911
            return;
2912
        }
2913
        break;
2914
    case 16:
2915
        if (TCG_TARGET_HAS_ext16s_i64) {
2916
            tcg_gen_ext16s_i64(ret, arg);
2917
            tcg_gen_sari_i64(ret, ret, ofs);
2918
            return;
2919
        }
2920
        break;
2921
    case 8:
2922
        if (TCG_TARGET_HAS_ext8s_i64) {
2923
            tcg_gen_ext8s_i64(ret, arg);
2924
            tcg_gen_sari_i64(ret, ret, ofs);
2925
            return;
2926
        }
2927
        break;
2928
    }
2929
    switch (len) {
2930
    case 32:
2931
        if (TCG_TARGET_HAS_ext32s_i64) {
2932
            tcg_gen_shri_i64(ret, arg, ofs);
2933
            tcg_gen_ext32s_i64(ret, ret);
2934
            return;
2935
        }
2936
        break;
2937
    case 16:
2938
        if (TCG_TARGET_HAS_ext16s_i64) {
2939
            tcg_gen_shri_i64(ret, arg, ofs);
2940
            tcg_gen_ext16s_i64(ret, ret);
2941
            return;
2942
        }
2943
        break;
2944
    case 8:
2945
        if (TCG_TARGET_HAS_ext8s_i64) {
2946
            tcg_gen_shri_i64(ret, arg, ofs);
2947
            tcg_gen_ext8s_i64(ret, ret);
2948
            return;
2949
        }
2950
        break;
2951
    }
2952
    tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2953
    tcg_gen_sari_i64(ret, ret, 64 - len);
2954
}
2955

2956
/*
2957
 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2958
 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2959
 */
2960
void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2961
                          unsigned int ofs)
2962
{
2963
    tcg_debug_assert(ofs <= 64);
2964
    if (ofs == 0) {
2965
        tcg_gen_mov_i64(ret, al);
2966
    } else if (ofs == 64) {
2967
        tcg_gen_mov_i64(ret, ah);
2968
    } else if (al == ah) {
2969
        tcg_gen_rotri_i64(ret, al, ofs);
2970
    } else if (TCG_TARGET_HAS_extract2_i64) {
2971
        tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2972
    } else {
2973
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2974
        tcg_gen_shri_i64(t0, al, ofs);
2975
        tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2976
        tcg_temp_free_i64(t0);
2977
    }
2978
}
2979

2980
void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2981
                         TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2982
{
2983
    if (cond == TCG_COND_ALWAYS) {
2984
        tcg_gen_mov_i64(ret, v1);
2985
    } else if (cond == TCG_COND_NEVER) {
2986
        tcg_gen_mov_i64(ret, v2);
2987
    } else if (TCG_TARGET_REG_BITS == 64) {
2988
        tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2989
    } else {
2990
        TCGv_i32 t0 = tcg_temp_ebb_new_i32();
2991
        TCGv_i32 zero = tcg_constant_i32(0);
2992

2993
        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2994
                         TCGV_LOW(c1), TCGV_HIGH(c1),
2995
                         TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2996

2997
        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero,
2998
                            TCGV_LOW(v1), TCGV_LOW(v2));
2999
        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero,
3000
                            TCGV_HIGH(v1), TCGV_HIGH(v2));
3001

3002
        tcg_temp_free_i32(t0);
3003
    }
3004
}
3005

3006
void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
3007
                      TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
3008
{
3009
    if (TCG_TARGET_HAS_add2_i64) {
3010
        tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
3011
    } else {
3012
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3013
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3014
        tcg_gen_add_i64(t0, al, bl);
3015
        tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
3016
        tcg_gen_add_i64(rh, ah, bh);
3017
        tcg_gen_add_i64(rh, rh, t1);
3018
        tcg_gen_mov_i64(rl, t0);
3019
        tcg_temp_free_i64(t0);
3020
        tcg_temp_free_i64(t1);
3021
    }
3022
}
3023

3024
void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
3025
                      TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
3026
{
3027
    if (TCG_TARGET_HAS_sub2_i64) {
3028
        tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
3029
    } else {
3030
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3031
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3032
        tcg_gen_sub_i64(t0, al, bl);
3033
        tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
3034
        tcg_gen_sub_i64(rh, ah, bh);
3035
        tcg_gen_sub_i64(rh, rh, t1);
3036
        tcg_gen_mov_i64(rl, t0);
3037
        tcg_temp_free_i64(t0);
3038
        tcg_temp_free_i64(t1);
3039
    }
3040
}
3041

3042
void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
3043
{
3044
    if (TCG_TARGET_HAS_mulu2_i64) {
3045
        tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
3046
    } else if (TCG_TARGET_HAS_muluh_i64) {
3047
        TCGv_i64 t = tcg_temp_ebb_new_i64();
3048
        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
3049
        tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
3050
        tcg_gen_mov_i64(rl, t);
3051
        tcg_temp_free_i64(t);
3052
    } else {
3053
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3054
        tcg_gen_mul_i64(t0, arg1, arg2);
3055
        gen_helper_muluh_i64(rh, arg1, arg2);
3056
        tcg_gen_mov_i64(rl, t0);
3057
        tcg_temp_free_i64(t0);
3058
    }
3059
}
3060

3061
void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
3062
{
3063
    if (TCG_TARGET_HAS_muls2_i64) {
3064
        tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
3065
    } else if (TCG_TARGET_HAS_mulsh_i64) {
3066
        TCGv_i64 t = tcg_temp_ebb_new_i64();
3067
        tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
3068
        tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
3069
        tcg_gen_mov_i64(rl, t);
3070
        tcg_temp_free_i64(t);
3071
    } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
3072
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3073
        TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3074
        TCGv_i64 t2 = tcg_temp_ebb_new_i64();
3075
        TCGv_i64 t3 = tcg_temp_ebb_new_i64();
3076
        tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
3077
        /* Adjust for negative inputs.  */
3078
        tcg_gen_sari_i64(t2, arg1, 63);
3079
        tcg_gen_sari_i64(t3, arg2, 63);
3080
        tcg_gen_and_i64(t2, t2, arg2);
3081
        tcg_gen_and_i64(t3, t3, arg1);
3082
        tcg_gen_sub_i64(rh, t1, t2);
3083
        tcg_gen_sub_i64(rh, rh, t3);
3084
        tcg_gen_mov_i64(rl, t0);
3085
        tcg_temp_free_i64(t0);
3086
        tcg_temp_free_i64(t1);
3087
        tcg_temp_free_i64(t2);
3088
        tcg_temp_free_i64(t3);
3089
    } else {
3090
        TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3091
        tcg_gen_mul_i64(t0, arg1, arg2);
3092
        gen_helper_mulsh_i64(rh, arg1, arg2);
3093
        tcg_gen_mov_i64(rl, t0);
3094
        tcg_temp_free_i64(t0);
3095
    }
3096
}
3097

3098
void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
3099
{
3100
    TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3101
    TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3102
    TCGv_i64 t2 = tcg_temp_ebb_new_i64();
3103
    tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
3104
    /* Adjust for negative input for the signed arg1.  */
3105
    tcg_gen_sari_i64(t2, arg1, 63);
3106
    tcg_gen_and_i64(t2, t2, arg2);
3107
    tcg_gen_sub_i64(rh, t1, t2);
3108
    tcg_gen_mov_i64(rl, t0);
3109
    tcg_temp_free_i64(t0);
3110
    tcg_temp_free_i64(t1);
3111
    tcg_temp_free_i64(t2);
3112
}
3113

3114
void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
3115
{
3116
    tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
3117
}
3118

3119
void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
3120
{
3121
    tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
3122
}
3123

3124
void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
3125
{
3126
    tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
3127
}
3128

3129
void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
3130
{
3131
    tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
3132
}
3133

3134
void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
3135
{
3136
    TCGv_i64 t = tcg_temp_ebb_new_i64();
3137

3138
    tcg_gen_sari_i64(t, a, 63);
3139
    tcg_gen_xor_i64(ret, a, t);
3140
    tcg_gen_sub_i64(ret, ret, t);
3141
    tcg_temp_free_i64(t);
3142
}
3143

3144
/* Size changing operations.  */
3145

3146
void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
3147
{
3148
    if (TCG_TARGET_REG_BITS == 32) {
3149
        tcg_gen_mov_i32(ret, TCGV_LOW(arg));
3150
    } else if (TCG_TARGET_HAS_extr_i64_i32) {
3151
        tcg_gen_op2(INDEX_op_extrl_i64_i32,
3152
                    tcgv_i32_arg(ret), tcgv_i64_arg(arg));
3153
    } else {
3154
        tcg_gen_mov_i32(ret, (TCGv_i32)arg);
3155
    }
3156
}
3157

3158
void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
3159
{
3160
    if (TCG_TARGET_REG_BITS == 32) {
3161
        tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
3162
    } else if (TCG_TARGET_HAS_extr_i64_i32) {
3163
        tcg_gen_op2(INDEX_op_extrh_i64_i32,
3164
                    tcgv_i32_arg(ret), tcgv_i64_arg(arg));
3165
    } else {
3166
        TCGv_i64 t = tcg_temp_ebb_new_i64();
3167
        tcg_gen_shri_i64(t, arg, 32);
3168
        tcg_gen_mov_i32(ret, (TCGv_i32)t);
3169
        tcg_temp_free_i64(t);
3170
    }
3171
}
3172

3173
void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
3174
{
3175
    if (TCG_TARGET_REG_BITS == 32) {
3176
        tcg_gen_mov_i32(TCGV_LOW(ret), arg);
3177
        tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
3178
    } else {
3179
        tcg_gen_op2(INDEX_op_extu_i32_i64,
3180
                    tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3181
    }
3182
}
3183

3184
void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
3185
{
3186
    if (TCG_TARGET_REG_BITS == 32) {
3187
        tcg_gen_mov_i32(TCGV_LOW(ret), arg);
3188
        tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
3189
    } else {
3190
        tcg_gen_op2(INDEX_op_ext_i32_i64,
3191
                    tcgv_i64_arg(ret), tcgv_i32_arg(arg));
3192
    }
3193
}
3194

3195
void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
3196
{
3197
    TCGv_i64 tmp;
3198

3199
    if (TCG_TARGET_REG_BITS == 32) {
3200
        tcg_gen_mov_i32(TCGV_LOW(dest), low);
3201
        tcg_gen_mov_i32(TCGV_HIGH(dest), high);
3202
        return;
3203
    }
3204

3205
    tmp = tcg_temp_ebb_new_i64();
3206
    /* These extensions are only needed for type correctness.
3207
       We may be able to do better given target specific information.  */
3208
    tcg_gen_extu_i32_i64(tmp, high);
3209
    tcg_gen_extu_i32_i64(dest, low);
3210
    /* If deposit is available, use it.  Otherwise use the extra
3211
       knowledge that we have of the zero-extensions above.  */
3212
    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
3213
        tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
3214
    } else {
3215
        tcg_gen_shli_i64(tmp, tmp, 32);
3216
        tcg_gen_or_i64(dest, dest, tmp);
3217
    }
3218
    tcg_temp_free_i64(tmp);
3219
}
3220

3221
void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
3222
{
3223
    if (TCG_TARGET_REG_BITS == 32) {
3224
        tcg_gen_mov_i32(lo, TCGV_LOW(arg));
3225
        tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
3226
    } else {
3227
        tcg_gen_extrl_i64_i32(lo, arg);
3228
        tcg_gen_extrh_i64_i32(hi, arg);
3229
    }
3230
}
3231

3232
void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
3233
{
3234
    tcg_gen_ext32u_i64(lo, arg);
3235
    tcg_gen_shri_i64(hi, arg, 32);
3236
}
3237

3238
void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi)
3239
{
3240
    tcg_gen_deposit_i64(ret, lo, hi, 32, 32);
3241
}
3242

3243
void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg)
3244
{
3245
    tcg_gen_mov_i64(lo, TCGV128_LOW(arg));
3246
    tcg_gen_mov_i64(hi, TCGV128_HIGH(arg));
3247
}
3248

3249
void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi)
3250
{
3251
    tcg_gen_mov_i64(TCGV128_LOW(ret), lo);
3252
    tcg_gen_mov_i64(TCGV128_HIGH(ret), hi);
3253
}
3254

3255
void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src)
3256
{
3257
    if (dst != src) {
3258
        tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src));
3259
        tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src));
3260
    }
3261
}
3262

3263
void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset)
3264
{
3265
    if (HOST_BIG_ENDIAN) {
3266
        tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset);
3267
        tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8);
3268
    } else {
3269
        tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset);
3270
        tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8);
3271
    }
3272
}
3273

3274
void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset)
3275
{
3276
    if (HOST_BIG_ENDIAN) {
3277
        tcg_gen_st_i64(TCGV128_HIGH(val), base, offset);
3278
        tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8);
3279
    } else {
3280
        tcg_gen_st_i64(TCGV128_LOW(val), base, offset);
3281
        tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8);
3282
    }
3283
}
3284

3285
/* QEMU specific operations.  */
3286

3287
void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
3288
{
3289
    /*
3290
     * Let the jit code return the read-only version of the
3291
     * TranslationBlock, so that we minimize the pc-relative
3292
     * distance of the address of the exit_tb code to TB.
3293
     * This will improve utilization of pc-relative address loads.
3294
     *
3295
     * TODO: Move this to translator_loop, so that all const
3296
     * TranslationBlock pointers refer to read-only memory.
3297
     * This requires coordination with targets that do not use
3298
     * the translator_loop.
3299
     */
3300
    uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
3301

3302
    if (tb == NULL) {
3303
        tcg_debug_assert(idx == 0);
3304
    } else if (idx <= TB_EXIT_IDXMAX) {
3305
#ifdef CONFIG_DEBUG_TCG
3306
        /* This is an exit following a goto_tb.  Verify that we have
3307
           seen this numbered exit before, via tcg_gen_goto_tb.  */
3308
        tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
3309
#endif
3310
    } else {
3311
        /* This is an exit via the exitreq label.  */
3312
        tcg_debug_assert(idx == TB_EXIT_REQUESTED);
3313
    }
3314

3315
    tcg_gen_op1i(INDEX_op_exit_tb, val);
3316
}
3317

3318
void tcg_gen_goto_tb(unsigned idx)
3319
{
3320
    /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
3321
    tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
3322
    /* We only support two chained exits.  */
3323
    tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
3324
#ifdef CONFIG_DEBUG_TCG
3325
    /* Verify that we haven't seen this numbered exit before.  */
3326
    tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
3327
    tcg_ctx->goto_tb_issue_mask |= 1 << idx;
3328
#endif
3329
    plugin_gen_disable_mem_helpers();
3330
    tcg_gen_op1i(INDEX_op_goto_tb, idx);
3331
}
3332

3333
void tcg_gen_lookup_and_goto_ptr(void)
3334
{
3335
    TCGv_ptr ptr;
3336

3337
    if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
3338
        tcg_gen_exit_tb(NULL, 0);
3339
        return;
3340
    }
3341

3342
    plugin_gen_disable_mem_helpers();
3343
    ptr = tcg_temp_ebb_new_ptr();
3344
    gen_helper_lookup_tb_ptr(ptr, tcg_env);
3345
    tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
3346
    tcg_temp_free_ptr(ptr);
3347
}
3348