podman

1
// +build go1.16,!go1.17
2

3
/*
4
 * Copyright 2021 ByteDance Inc.
5
 *
6
 * Licensed under the Apache License, Version 2.0 (the "License");
7
 * you may not use this file except in compliance with the License.
8
 * You may obtain a copy of the License at
9
 *
10
 *     http://www.apache.org/licenses/LICENSE-2.0
11
 *
12
 * Unless required by applicable law or agreed to in writing, software
13
 * distributed under the License is distributed on an "AS IS" BASIS,
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
 * See the License for the specific language governing permissions and
16
 * limitations under the License.
17
 */
18

19
package encoder
20

21
import (
22
    `fmt`
23
    `reflect`
24
    `strconv`
25
    `unsafe`
26

27
    `github.com/bytedance/sonic/internal/cpu`
28
    `github.com/bytedance/sonic/internal/jit`
29
    `github.com/bytedance/sonic/internal/native/types`
30
    `github.com/twitchyliquid64/golang-asm/obj`
31
    `github.com/twitchyliquid64/golang-asm/obj/x86`
32

33
    `github.com/bytedance/sonic/internal/native`
34
    `github.com/bytedance/sonic/internal/rt`
35
)
36

37
/** Register Allocations
38
 *
39
 *  State Registers:
40
 *
41
 *      %rbx : stack base
42
 *      %rdi : result pointer
43
 *      %rsi : result length
44
 *      %rdx : result capacity
45
 *      %r12 : sp->p
46
 *      %r13 : sp->q
47
 *      %r14 : sp->x
48
 *      %r15 : sp->f
49
 *
50
 *  Error Registers:
51
 *
52
 *      %r10 : error type register
53
 *      %r11 : error pointer register
54
 */
55

56
/** Function Prototype & Stack Map
57
 *
58
 *  func (buf *[]byte, p unsafe.Pointer, sb *_Stack, fv uint64) (err error)
59
 *
60
 *  buf    :   (FP)
61
 *  p      :  8(FP)
62
 *  sb     : 16(FP)
63
 *  fv     : 24(FP)
64
 *  err.vt : 32(FP)
65
 *  err.vp : 40(FP)
66
 */
67

68
const (
69
    _S_cond = iota
70
    _S_init
71
)
72

73
const (
74
    _FP_args   = 48     // 48 bytes for passing arguments to this function
75
    _FP_fargs  = 64     // 64 bytes for passing arguments to other Go functions
76
    _FP_saves  = 64     // 64 bytes for saving the registers before CALL instructions
77
    _FP_locals = 24     // 24 bytes for local variables
78
)
79

80
const (
81
    _FP_offs = _FP_fargs + _FP_saves + _FP_locals
82
    _FP_size = _FP_offs + 8     // 8 bytes for the parent frame pointer
83
    _FP_base = _FP_size + 8     // 8 bytes for the return address
84
)
85

86
const (
87
    _FM_exp32 = 0x7f800000
88
    _FM_exp64 = 0x7ff0000000000000
89
)
90

91
const (
92
    _IM_null   = 0x6c6c756e           // 'null'
93
    _IM_true   = 0x65757274           // 'true'
94
    _IM_fals   = 0x736c6166           // 'fals' ('false' without the 'e')
95
    _IM_open   = 0x00225c22           // '"\"∅'
96
    _IM_array  = 0x5d5b               // '[]'
97
    _IM_object = 0x7d7b               // '{}'
98
    _IM_mulv   = -0x5555555555555555
99
)
100

101
const (
102
    _LB_more_space        = "_more_space"
103
    _LB_more_space_return = "_more_space_return_"
104
)
105

106
const (
107
    _LB_error                 = "_error"
108
    _LB_error_too_deep        = "_error_too_deep"
109
    _LB_error_invalid_number  = "_error_invalid_number"
110
    _LB_error_nan_or_infinite = "_error_nan_or_infinite"
111
    _LB_panic = "_panic"
112
)
113

114
var (
115
    _AX = jit.Reg("AX")
116
    _CX = jit.Reg("CX")
117
    _DX = jit.Reg("DX")
118
    _DI = jit.Reg("DI")
119
    _SI = jit.Reg("SI")
120
    _BP = jit.Reg("BP")
121
    _SP = jit.Reg("SP")
122
    _R8 = jit.Reg("R8")
123
)
124

125
var (
126
    _X0 = jit.Reg("X0")
127
    _Y0 = jit.Reg("Y0")
128
)
129

130
var (
131
    _ST = jit.Reg("BX")
132
    _RP = jit.Reg("DI")
133
    _RL = jit.Reg("SI")
134
    _RC = jit.Reg("DX")
135
)
136

137
var (
138
    _LR  = jit.Reg("R9")
139
    _R10 = jit.Reg("R10")   // used for gcWriterBarrier
140
    _ET  = jit.Reg("R10")
141
    _EP  = jit.Reg("R11")
142
)
143

144
var (
145
    _SP_p = jit.Reg("R12")
146
    _SP_q = jit.Reg("R13")
147
    _SP_x = jit.Reg("R14")
148
    _SP_f = jit.Reg("R15")
149
)
150

151
var (
152
    _ARG_rb = jit.Ptr(_SP, _FP_base)
153
    _ARG_vp = jit.Ptr(_SP, _FP_base + 8)
154
    _ARG_sb = jit.Ptr(_SP, _FP_base + 16)
155
    _ARG_fv = jit.Ptr(_SP, _FP_base + 24)
156
)
157

158
var (
159
    _RET_et = jit.Ptr(_SP, _FP_base + 32)
160
    _RET_ep = jit.Ptr(_SP, _FP_base + 40)
161
)
162

163
var (
164
    _VAR_sp = jit.Ptr(_SP, _FP_fargs + _FP_saves)
165
    _VAR_dn = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8)
166
    _VAR_vp = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16)
167
)
168

169
var (
170
    _REG_ffi = []obj.Addr{_RP, _RL, _RC}
171
    _REG_enc = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RL}
172
    _REG_jsr = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _LR}
173
    _REG_all = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC}
174
)
175

176
type _Assembler struct {
177
    jit.BaseAssembler
178
    p _Program
179
    x int
180
    name string
181
}
182

183
func newAssembler(p _Program) *_Assembler {
184
    return new(_Assembler).Init(p)
185
}
186

187
/** Assembler Interface **/
188
func (self *_Assembler) Load() _Encoder {
189
    return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs))
190
}
191

192
func (self *_Assembler) Init(p _Program) *_Assembler {
193
    self.p = p
194
    self.BaseAssembler.Init(self.compile)
195
    return self
196
}
197

198
func (self *_Assembler) compile() {
199
    self.prologue()
200
    self.instrs()
201
    self.epilogue()
202
    self.builtins()
203
}
204

205
/** Assembler Stages **/
206

207
var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
208
    _OP_null           : (*_Assembler)._asm_OP_null,
209
    _OP_empty_arr      : (*_Assembler)._asm_OP_empty_arr,
210
    _OP_empty_obj      : (*_Assembler)._asm_OP_empty_obj, 
211
    _OP_bool           : (*_Assembler)._asm_OP_bool,
212
    _OP_i8             : (*_Assembler)._asm_OP_i8,
213
    _OP_i16            : (*_Assembler)._asm_OP_i16,
214
    _OP_i32            : (*_Assembler)._asm_OP_i32,
215
    _OP_i64            : (*_Assembler)._asm_OP_i64,
216
    _OP_u8             : (*_Assembler)._asm_OP_u8,
217
    _OP_u16            : (*_Assembler)._asm_OP_u16,
218
    _OP_u32            : (*_Assembler)._asm_OP_u32,
219
    _OP_u64            : (*_Assembler)._asm_OP_u64,
220
    _OP_f32            : (*_Assembler)._asm_OP_f32,
221
    _OP_f64            : (*_Assembler)._asm_OP_f64,
222
    _OP_str            : (*_Assembler)._asm_OP_str,
223
    _OP_bin            : (*_Assembler)._asm_OP_bin,
224
    _OP_quote          : (*_Assembler)._asm_OP_quote,
225
    _OP_number         : (*_Assembler)._asm_OP_number,
226
    _OP_eface          : (*_Assembler)._asm_OP_eface,
227
    _OP_iface          : (*_Assembler)._asm_OP_iface,
228
    _OP_byte           : (*_Assembler)._asm_OP_byte,
229
    _OP_text           : (*_Assembler)._asm_OP_text,
230
    _OP_deref          : (*_Assembler)._asm_OP_deref,
231
    _OP_index          : (*_Assembler)._asm_OP_index,
232
    _OP_load           : (*_Assembler)._asm_OP_load,
233
    _OP_save           : (*_Assembler)._asm_OP_save,
234
    _OP_drop           : (*_Assembler)._asm_OP_drop,
235
    _OP_drop_2         : (*_Assembler)._asm_OP_drop_2,
236
    _OP_recurse        : (*_Assembler)._asm_OP_recurse,
237
    _OP_is_nil         : (*_Assembler)._asm_OP_is_nil,
238
    _OP_is_nil_p1      : (*_Assembler)._asm_OP_is_nil_p1,
239
    _OP_is_zero_1      : (*_Assembler)._asm_OP_is_zero_1,
240
    _OP_is_zero_2      : (*_Assembler)._asm_OP_is_zero_2,
241
    _OP_is_zero_4      : (*_Assembler)._asm_OP_is_zero_4,
242
    _OP_is_zero_8      : (*_Assembler)._asm_OP_is_zero_8,
243
    _OP_is_zero_map    : (*_Assembler)._asm_OP_is_zero_map,
244
    _OP_goto           : (*_Assembler)._asm_OP_goto,
245
    _OP_map_iter       : (*_Assembler)._asm_OP_map_iter,
246
    _OP_map_stop       : (*_Assembler)._asm_OP_map_stop,
247
    _OP_map_check_key  : (*_Assembler)._asm_OP_map_check_key,
248
    _OP_map_write_key  : (*_Assembler)._asm_OP_map_write_key,
249
    _OP_map_value_next : (*_Assembler)._asm_OP_map_value_next,
250
    _OP_slice_len      : (*_Assembler)._asm_OP_slice_len,
251
    _OP_slice_next     : (*_Assembler)._asm_OP_slice_next,
252
    _OP_marshal        : (*_Assembler)._asm_OP_marshal,
253
    _OP_marshal_p      : (*_Assembler)._asm_OP_marshal_p,
254
    _OP_marshal_text   : (*_Assembler)._asm_OP_marshal_text,
255
    _OP_marshal_text_p : (*_Assembler)._asm_OP_marshal_text_p,
256
    _OP_cond_set       : (*_Assembler)._asm_OP_cond_set,
257
    _OP_cond_testc     : (*_Assembler)._asm_OP_cond_testc,
258
}
259

260
func (self *_Assembler) instr(v *_Instr) {
261
    if fn := _OpFuncTab[v.op()]; fn != nil {
262
        fn(self, v)
263
    } else {
264
        panic(fmt.Sprintf("invalid opcode: %d", v.op()))
265
    }
266
}
267

268
func (self *_Assembler) instrs() {
269
    for i, v := range self.p {
270
        self.Mark(i)
271
        self.instr(&v)
272
        self.debug_instr(i, &v)
273
    }
274
}
275

276
func (self *_Assembler) builtins() {
277
    self.more_space()
278
    self.error_too_deep()
279
    self.error_invalid_number()
280
    self.error_nan_or_infinite()
281
    self.go_panic()
282
}
283

284
func (self *_Assembler) epilogue() {
285
    self.Mark(len(self.p))
286
    self.Emit("XORL", _ET, _ET)
287
    self.Emit("XORL", _EP, _EP)
288
    self.Link(_LB_error)
289
    self.Emit("MOVQ", _ARG_rb, _AX)                 // MOVQ rb<>+0(FP), AX
290
    self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8))         // MOVQ RL, 8(AX)
291
    self.Emit("MOVQ", _ET, _RET_et)                 // MOVQ ET, et<>+24(FP)
292
    self.Emit("MOVQ", _EP, _RET_ep)                 // MOVQ EP, ep<>+32(FP)
293
    self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP)  // MOVQ _FP_offs(SP), BP
294
    self.Emit("ADDQ", jit.Imm(_FP_size), _SP)       // ADDQ $_FP_size, SP
295
    self.Emit("RET")                                // RET
296
}
297

298
func (self *_Assembler) prologue() {
299
    self.Emit("SUBQ", jit.Imm(_FP_size), _SP)       // SUBQ $_FP_size, SP
300
    self.Emit("MOVQ", _BP, jit.Ptr(_SP, _FP_offs))  // MOVQ BP, _FP_offs(SP)
301
    self.Emit("LEAQ", jit.Ptr(_SP, _FP_offs), _BP)  // LEAQ _FP_offs(SP), BP
302
    self.load_buffer()                              // LOAD {buf}
303
    self.Emit("MOVQ", _ARG_vp, _SP_p)               // MOVQ vp<>+8(FP), SP.p
304
    self.Emit("MOVQ", _ARG_sb, _ST)                 // MOVQ sb<>+16(FP), ST
305
    self.Emit("XORL", _SP_x, _SP_x)                 // XORL SP.x, SP.x
306
    self.Emit("XORL", _SP_f, _SP_f)                 // XORL SP.f, SP.f
307
    self.Emit("XORL", _SP_q, _SP_q)                 // XORL SP.q, SP.q
308
}
309

310
/** Assembler Inline Functions **/
311

312
func (self *_Assembler) xsave(reg ...obj.Addr) {
313
    for i, v := range reg {
314
        if i > _FP_saves / 8 - 1 {
315
            panic("too many registers to save")
316
        } else {
317
            self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + int64(i) * 8))
318
        }
319
    }
320
}
321

322
func (self *_Assembler) xload(reg ...obj.Addr) {
323
    for i, v := range reg {
324
        if i > _FP_saves / 8 - 1 {
325
            panic("too many registers to load")
326
        } else {
327
            self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + int64(i) * 8), v)
328
        }
329
    }
330
}
331

332
func (self *_Assembler) rbuf_di() {
333
    if _RP.Reg != x86.REG_DI {
334
        panic("register allocation messed up: RP != DI")
335
    } else {
336
        self.Emit("ADDQ", _RL, _RP)
337
    }
338
}
339

340
func (self *_Assembler) store_int(nd int, fn obj.Addr, ins string) {
341
    self.check_size(nd)
342
    self.save_c()                           // SAVE   $C_regs
343
    self.rbuf_di()                          // MOVQ   RP, DI
344
    self.Emit(ins, jit.Ptr(_SP_p, 0), _SI)  // $ins   (SP.p), SI
345
    self.call_c(fn)                         // CALL_C $fn
346
    self.Emit("ADDQ", _AX, _RL)             // ADDQ   AX, RL
347
}
348

349
func (self *_Assembler) store_str(s string) {
350
    i := 0
351
    m := rt.Str2Mem(s)
352

353
    /* 8-byte stores */
354
    for i <= len(m) - 8 {
355
        self.Emit("MOVQ", jit.Imm(rt.Get64(m[i:])), _AX)        // MOVQ $s[i:], AX
356
        self.Emit("MOVQ", _AX, jit.Sib(_RP, _RL, 1, int64(i)))  // MOVQ AX, i(RP)(RL)
357
        i += 8
358
    }
359

360
    /* 4-byte stores */
361
    if i <= len(m) - 4 {
362
        self.Emit("MOVL", jit.Imm(int64(rt.Get32(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i)))  // MOVL $s[i:], i(RP)(RL)
363
        i += 4
364
    }
365

366
    /* 2-byte stores */
367
    if i <= len(m) - 2 {
368
        self.Emit("MOVW", jit.Imm(int64(rt.Get16(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i)))  // MOVW $s[i:], i(RP)(RL)
369
        i += 2
370
    }
371

372
    /* last byte */
373
    if i < len(m) {
374
        self.Emit("MOVB", jit.Imm(int64(m[i])), jit.Sib(_RP, _RL, 1, int64(i)))     // MOVB $s[i:], i(RP)(RL)
375
    }
376
}
377

378
func (self *_Assembler) check_size(n int) {
379
    self.check_size_rl(jit.Ptr(_RL, int64(n)))
380
}
381

382
func (self *_Assembler) check_size_r(r obj.Addr, d int) {
383
    self.check_size_rl(jit.Sib(_RL, r, 1, int64(d)))
384
}
385

386
func (self *_Assembler) check_size_rl(v obj.Addr) {
387
    idx := self.x
388
    key := _LB_more_space_return + strconv.Itoa(idx)
389

390
    /* the following code relies on LR == R9 to work */
391
    if _LR.Reg != x86.REG_R9 {
392
        panic("register allocation messed up: LR != R9")
393
    }
394

395
    /* check for buffer capacity */
396
    self.x++
397
    self.Emit("LEAQ", v, _AX)       // LEAQ $v, AX
398
    self.Emit("CMPQ", _AX, _RC)     // CMPQ AX, RC
399
    self.Sjmp("JBE" , key)          // JBE  _more_space_return_{n}
400
    self.slice_grow_ax(key)         // GROW $key
401
    self.Link(key)                  // _more_space_return_{n}:
402
}
403

404
func (self *_Assembler) slice_grow_ax(ret string) {
405
    self.Byte(0x4c, 0x8d, 0x0d)         // LEAQ ?(PC), R9
406
    self.Sref(ret, 4)                   // .... &ret
407
    self.Sjmp("JMP" , _LB_more_space)   // JMP  _more_space
408
}
409

410
/** State Stack Helpers **/
411

412
const (
413
    _StateSize  = int64(unsafe.Sizeof(_State{}))
414
    _StackLimit = _MaxStack * _StateSize
415
)
416

417
func (self *_Assembler) save_state() {
418
    self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX)             // MOVQ (ST), CX
419
    self.Emit("LEAQ", jit.Ptr(_CX, _StateSize), _R8)    // LEAQ _StateSize(CX), R8
420
    self.Emit("CMPQ", _R8, jit.Imm(_StackLimit))        // CMPQ R8, $_StackLimit
421
    self.Sjmp("JAE" , _LB_error_too_deep)               // JA   _error_too_deep
422
    self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _CX, 1, 8))   // MOVQ SP.x, 8(ST)(CX)
423
    self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _CX, 1, 16))  // MOVQ SP.f, 16(ST)(CX)
424
    self.WritePtr(0, _SP_p, jit.Sib(_ST, _CX, 1, 24)) // MOVQ SP.p, 24(ST)(CX)
425
    self.WritePtr(1, _SP_q, jit.Sib(_ST, _CX, 1, 32)) // MOVQ SP.q, 32(ST)(CX)
426
    self.Emit("MOVQ", _R8, jit.Ptr(_ST, 0))             // MOVQ R8, (ST)
427
}
428

429
func (self *_Assembler) drop_state(decr int64) {
430
    self.Emit("MOVQ" , jit.Ptr(_ST, 0), _AX)                // MOVQ  (ST), AX
431
    self.Emit("SUBQ" , jit.Imm(decr), _AX)                  // SUBQ  $decr, AX
432
    self.Emit("MOVQ" , _AX, jit.Ptr(_ST, 0))                // MOVQ  AX, (ST)
433
    self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 8), _SP_x)      // MOVQ  8(ST)(AX), SP.x
434
    self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 16), _SP_f)     // MOVQ  16(ST)(AX), SP.f
435
    self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 24), _SP_p)     // MOVQ  24(ST)(AX), SP.p
436
    self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 32), _SP_q)     // MOVQ  32(ST)(AX), SP.q
437
    self.Emit("PXOR" , _X0, _X0)                            // PXOR  X0, X0
438
    self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 8))        // MOVOU X0, 8(ST)(AX)
439
    self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 24))       // MOVOU X0, 24(ST)(AX)
440
}
441

442
/** Buffer Helpers **/
443

444
func (self *_Assembler) add_char(ch byte) {
445
    self.Emit("MOVB", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0))  // MOVB $ch, (RP)(RL)
446
    self.Emit("ADDQ", jit.Imm(1), _RL)                              // ADDQ $1, RL
447
}
448

449
func (self *_Assembler) add_long(ch uint32, n int64) {
450
    self.Emit("MOVL", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0))  // MOVL $ch, (RP)(RL)
451
    self.Emit("ADDQ", jit.Imm(n), _RL)                              // ADDQ $n, RL
452
}
453

454
func (self *_Assembler) add_text(ss string) {
455
    self.store_str(ss)                                  // TEXT $ss
456
    self.Emit("ADDQ", jit.Imm(int64(len(ss))), _RL)     // ADDQ ${len(ss)}, RL
457
}
458

459
func (self *_Assembler) prep_buffer() {
460
    self.Emit("MOVQ", _ARG_rb, _AX)             // MOVQ rb<>+0(FP), AX
461
    self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8))     // MOVQ RL, 8(AX)
462
    self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0))     // MOVQ AX, (SP)
463
}
464

465
func (self *_Assembler) prep_buffer_c() {
466
    self.Emit("MOVQ", _ARG_rb, _DI)             // MOVQ rb<>+0(FP), DI
467
    self.Emit("MOVQ", _RL, jit.Ptr(_DI, 8))     // MOVQ RL, 8(DI)
468
}
469

470
func (self *_Assembler) save_buffer() {
471
    self.Emit("MOVQ", _ARG_rb, _CX)             // MOVQ rb<>+0(FP), CX
472
    self.Emit("MOVQ", _RP, jit.Ptr(_CX,  0))    // MOVQ RP, (CX)
473
    self.Emit("MOVQ", _RL, jit.Ptr(_CX,  8))    // MOVQ RL, 8(CX)
474
    self.Emit("MOVQ", _RC, jit.Ptr(_CX, 16))    // MOVQ RC, 16(CX)
475
}
476

477
func (self *_Assembler) load_buffer() {
478
    self.Emit("MOVQ", _ARG_rb, _AX)             // MOVQ rb<>+0(FP), AX
479
    self.Emit("MOVQ", jit.Ptr(_AX,  0), _RP)    // MOVQ (AX), RP
480
    self.Emit("MOVQ", jit.Ptr(_AX,  8), _RL)    // MOVQ 8(AX), RL
481
    self.Emit("MOVQ", jit.Ptr(_AX, 16), _RC)    // MOVQ 16(AX), RC
482
}
483

484
/** Function Interface Helpers **/
485

486
func (self *_Assembler) call(pc obj.Addr) {
487
    self.Emit("MOVQ", pc, _AX)  // MOVQ $pc, AX
488
    self.Rjmp("CALL", _AX)      // CALL AX
489
}
490

491
func (self *_Assembler) save_c() {
492
    self.xsave(_REG_ffi...)     // SAVE $REG_ffi
493
}
494

495
func (self *_Assembler) call_c(pc obj.Addr) {
496
    self.call(pc)               // CALL $pc
497
    self.xload(_REG_ffi...)     // LOAD $REG_ffi
498
}
499

500
func (self *_Assembler) call_go(pc obj.Addr) {
501
    self.xsave(_REG_all...)     // SAVE $REG_all
502
    self.call(pc)               // CALL $pc
503
    self.xload(_REG_all...)     // LOAD $REG_all
504
}
505

506
func (self *_Assembler) call_encoder(pc obj.Addr) {
507
    self.xsave(_REG_enc...)     // SAVE $REG_enc
508
    self.call(pc)               // CALL $pc
509
    self.xload(_REG_enc...)     // LOAD $REG_enc
510
    self.load_buffer()          // LOAD {buf}
511
}
512

513
func (self *_Assembler) call_marshaler(fn obj.Addr, it *rt.GoType, vt reflect.Type) {
514
    switch vt.Kind() {
515
        case reflect.Interface       : self.call_marshaler_i(fn, it)
516
        case reflect.Ptr, reflect.Map: self.call_marshaler_v(fn, it, vt, true)
517
        // struct/array of 1 direct iface type can be direct
518
        default                      : self.call_marshaler_v(fn, it, vt, !rt.UnpackType(vt).Indirect())
519
    }
520
}
521

522
func (self *_Assembler) call_marshaler_i(fn obj.Addr, it *rt.GoType) {
523
    self.Emit("MOVQ" , jit.Gtype(it), _AX)                          // MOVQ    $it, AX
524
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0))                        // MOVQ    AX, (SP)
525
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX)                      // MOVQ    (SP.p), AX
526
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _CX)                      // MOVQ    8(SP.p), CX
527
    self.Emit("TESTQ", _AX, _AX)                                    // TESTQ   AX, AX
528
    self.Sjmp("JZ"   , "_null_{n}")                                 // JZ      _null_{n}
529
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8))                        // MOVQ    AX, 8(SP)
530
    self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 16))                       // MOVQ    CX, 16(SP)
531
    self.call_go(_F_assertI2I)                                      // CALL_GO assertI2I
532
    self.prep_buffer()                                              // MOVE    {buf}, (SP)
533
    self.Emit("MOVOU", jit.Ptr(_SP, 24), _X0)                       // MOVOU   24(SP), X0
534
    self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8))                        // MOVOU   X0, 8(SP)
535
    self.Emit("MOVQ", _ARG_fv,  _CX)                                // MOVQ   ARG.fv, CX
536
    self.Emit("MOVQ", _CX,  jit.Ptr(_SP, 24))                       // MOVQ   CX, 24(SP)
537
    self.call_encoder(fn)                                           // CALL    $fn
538
    self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET)                       // MOVQ    32(SP), ET
539
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP)                       // MOVQ    40(SP), EP
540
    self.Emit("TESTQ", _ET, _ET)                                    // TESTQ   ET, ET
541
    self.Sjmp("JNZ"  , _LB_error)                                   // JNZ     _error
542
    self.Sjmp("JMP"  , "_done_{n}")                                 // JMP     _done_{n}
543
    self.Link("_null_{n}")                                          // _null_{n}:
544
    self.check_size(4)                                              // SIZE    $4
545
    self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0))   // MOVL    $'null', (RP)(RL*1)
546
    self.Emit("ADDQ", jit.Imm(4), _RL)                              // ADDQ    $4, RL
547
    self.Link("_done_{n}")                                          // _done_{n}:
548
}
549

550
func (self *_Assembler) call_marshaler_v(fn obj.Addr, it *rt.GoType, vt reflect.Type, deref bool) {
551
    self.prep_buffer()                          // MOVE {buf}, (SP)
552
    self.Emit("MOVQ", jit.Itab(it, vt), _AX)    // MOVQ $(itab(it, vt)), AX
553
    self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8))     // MOVQ AX, 8(SP)
554

555
    /* dereference the pointer if needed */
556
    if !deref {
557
        self.Emit("MOVQ", _SP_p, jit.Ptr(_SP, 16))  // MOVQ SP.p, 16(SP)
558
    } else {
559
        self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX)   // MOVQ (SP.p), AX
560
        self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16))    // MOVQ AX, 16(SP)
561
    }
562

563
    /* call the encoder, and perform error checks */
564
    self.Emit("MOVQ", _ARG_fv,  _CX)            // MOVQ   ARG.fv, CX
565
    self.Emit("MOVQ", _CX,  jit.Ptr(_SP, 24))   // MOVQ   CX, 24(SP)
566
    self.call_encoder(fn)                       // CALL  $fn
567
    self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET)   // MOVQ  32(SP), ET
568
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP)   // MOVQ  40(SP), EP
569
    self.Emit("TESTQ", _ET, _ET)                // TESTQ ET, ET
570
    self.Sjmp("JNZ"  , _LB_error)               // JNZ   _error
571
}
572

573
/** Builtin: _more_space **/
574

575
var (
576
    _T_byte      = jit.Type(byteType)
577
    _F_growslice = jit.Func(growslice)
578
)
579

580
func (self *_Assembler) more_space() {
581
    self.Link(_LB_more_space)
582
    self.Emit("MOVQ", _T_byte, _AX)                 // MOVQ $_T_byte, _AX
583
    self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0))         // MOVQ _AX, (SP)
584
    self.Emit("MOVQ", _RP, jit.Ptr(_SP, 8))         // MOVQ RP, 8(SP)
585
    self.Emit("MOVQ", _RL, jit.Ptr(_SP, 16))        // MOVQ RL, 16(SP)
586
    self.Emit("MOVQ", _RC, jit.Ptr(_SP, 24))        // MOVQ RC, 24(SP)
587
    self.Emit("MOVQ", _AX, jit.Ptr(_SP, 32))        // MOVQ AX, 32(SP)
588
    self.xsave(_REG_jsr...)                         // SAVE $REG_jsr
589
    self.call(_F_growslice)                         // CALL $pc
590
    self.xload(_REG_jsr...)                         // LOAD $REG_jsr
591
    self.Emit("MOVQ", jit.Ptr(_SP, 40), _RP)        // MOVQ 40(SP), RP
592
    self.Emit("MOVQ", jit.Ptr(_SP, 48), _RL)        // MOVQ 48(SP), RL
593
    self.Emit("MOVQ", jit.Ptr(_SP, 56), _RC)        // MOVQ 56(SP), RC
594
    self.save_buffer()                              // SAVE {buf}
595
    self.Rjmp("JMP" , _LR)                          // JMP  LR
596
}
597

598
/** Builtin Errors **/
599

600
var (
601
    _V_ERR_too_deep               = jit.Imm(int64(uintptr(unsafe.Pointer(_ERR_too_deep))))
602
    _V_ERR_nan_or_infinite        = jit.Imm(int64(uintptr(unsafe.Pointer(_ERR_nan_or_infinite))))
603
    _I_json_UnsupportedValueError = jit.Itab(rt.UnpackType(errorType), jsonUnsupportedValueType)
604
)
605

606
func (self *_Assembler) error_too_deep() {
607
    self.Link(_LB_error_too_deep)
608
    self.Emit("MOVQ", _V_ERR_too_deep, _EP)                 // MOVQ $_V_ERR_too_deep, EP
609
    self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET)   // MOVQ $_I_json_UnsupportedValuError, ET
610
    self.Sjmp("JMP" , _LB_error)                            // JMP  _error
611
}
612

613
func (self *_Assembler) error_invalid_number() {
614
    self.Link(_LB_error_invalid_number)
615
    self.call_go(_F_error_number)               // CALL_GO error_number
616
    self.Emit("MOVQ", jit.Ptr(_SP, 16), _ET)    // MOVQ    16(SP), ET
617
    self.Emit("MOVQ", jit.Ptr(_SP, 24), _EP)    // MOVQ    24(SP), EP
618
    self.Sjmp("JMP" , _LB_error)                // JMP     _error
619
}
620

621
func (self *_Assembler) error_nan_or_infinite()  {
622
    self.Link(_LB_error_nan_or_infinite)
623
    self.Emit("MOVQ", _V_ERR_nan_or_infinite, _EP)          // MOVQ $_V_ERR_nan_or_infinite, EP
624
    self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET)   // MOVQ $_I_json_UnsupportedValuError, ET
625
    self.Sjmp("JMP" , _LB_error)                            // JMP  _error
626
}
627

628
/** String Encoding Routine **/
629

630
var (
631
    _F_quote = jit.Imm(int64(native.S_quote))
632
    _F_panic = jit.Func(goPanic)
633
)
634

635
func (self *_Assembler) go_panic() {
636
    self.Link(_LB_panic)
637
    self.Emit("MOVQ", _SP_p, jit.Ptr(_SP, 8))
638
    self.call_go(_F_panic)
639
}
640

641
func (self *_Assembler) encode_string(doubleQuote bool) {
642
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _AX)  // MOVQ  8(SP.p), AX
643
    self.Emit("TESTQ", _AX, _AX)                // TESTQ AX, AX
644
    self.Sjmp("JZ"   , "_str_empty_{n}")        // JZ    _str_empty_{n}
645
    self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0))
646
    self.Sjmp("JNE"   , "_str_next_{n}") 
647
    self.Emit("MOVQ", jit.Imm(int64(panicNilPointerOfNonEmptyString)), jit.Ptr(_SP, 0))
648
    self.Sjmp("JMP", _LB_panic)
649
    self.Link("_str_next_{n}")
650

651
    /* openning quote, check for double quote */
652
    if !doubleQuote {
653
        self.check_size_r(_AX, 2)   // SIZE $2
654
        self.add_char('"')          // CHAR $'"'
655
    } else {
656
        self.check_size_r(_AX, 6)   // SIZE $6
657
        self.add_long(_IM_open, 3)  // TEXT $`"\"`
658
    }
659

660
    /* quoting loop */
661
    self.Emit("XORL", _AX, _AX)         // XORL AX, AX
662
    self.Emit("MOVQ", _AX, _VAR_sp)     // MOVQ AX, sp
663
    self.Link("_str_loop_{n}")          // _str_loop_{n}:
664
    self.save_c()                       // SAVE $REG_ffi
665

666
    /* load the output buffer first, and then input buffer,
667
     * because the parameter registers collide with RP / RL / RC */
668
    self.Emit("MOVQ", _RC, _CX)                         // MOVQ RC, CX
669
    self.Emit("SUBQ", _RL, _CX)                         // SUBQ RL, CX
670
    self.Emit("MOVQ", _CX, _VAR_dn)                     // MOVQ CX, dn
671
    self.Emit("LEAQ", jit.Sib(_RP, _RL, 1, 0), _DX)     // LEAQ (RP)(RL), DX
672
    self.Emit("LEAQ", _VAR_dn, _CX)                     // LEAQ dn, CX
673
    self.Emit("MOVQ", _VAR_sp, _AX)                     // MOVQ sp, AX
674
    self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _DI)           // MOVQ (SP.p), DI
675
    self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _SI)           // MOVQ 8(SP.p), SI
676
    self.Emit("ADDQ", _AX, _DI)                         // ADDQ AX, DI
677
    self.Emit("SUBQ", _AX, _SI)                         // SUBQ AX, SI
678

679
    /* set the flags based on `doubleQuote` */
680
    if !doubleQuote {
681
        self.Emit("XORL", _R8, _R8)                                 // XORL R8, R8
682
    } else {
683
        self.Emit("MOVL", jit.Imm(types.F_DOUBLE_UNQUOTE), _R8)     // MOVL ${types.F_DOUBLE_UNQUOTE}, R8
684
    }
685

686
    /* call the native quoter */
687
    self.call_c(_F_quote)                   // CALL  quote
688
    self.Emit("ADDQ" , _VAR_dn, _RL)        // ADDQ  dn, RL
689
    self.Emit("TESTQ", _AX, _AX)            // TESTQ AX, AX
690
    self.Sjmp("JS"   , "_str_space_{n}")    // JS    _str_space_{n}
691

692
    /* close the string, check for double quote */
693
    if !doubleQuote {
694
        self.check_size(1)                  // SIZE $1
695
        self.add_char('"')                  // CHAR $'"'
696
        self.Sjmp("JMP", "_str_end_{n}")    // JMP  _str_end_{n}
697
    } else {
698
        self.check_size(3)                  // SIZE $3
699
        self.add_text("\\\"\"")             // TEXT $'\""'
700
        self.Sjmp("JMP", "_str_end_{n}")    // JMP  _str_end_{n}
701
    }
702

703
    /* not enough space to contain the quoted string */
704
    self.Link("_str_space_{n}")                         // _str_space_{n}:
705
    self.Emit("NOTQ", _AX)                              // NOTQ AX
706
    self.Emit("ADDQ", _AX, _VAR_sp)                     // ADDQ AX, sp
707
    self.Emit("LEAQ", jit.Sib(_RC, _RC, 1, 0), _AX)     // LEAQ (RC)(RC), AX
708
    self.slice_grow_ax("_str_loop_{n}")                 // GROW _str_loop_{n}
709

710
    /* empty string, check for double quote */
711
    if !doubleQuote {
712
        self.Link("_str_empty_{n}")     // _str_empty_{n}:
713
        self.check_size(2)              // SIZE $2
714
        self.add_text("\"\"")           // TEXT $'""'
715
        self.Link("_str_end_{n}")       // _str_end_{n}:
716
    } else {
717
        self.Link("_str_empty_{n}")     // _str_empty_{n}:
718
        self.check_size(6)              // SIZE $6
719
        self.add_text("\"\\\"\\\"\"")   // TEXT $'"\"\""'
720
        self.Link("_str_end_{n}")       // _str_end_{n}:
721
    }
722
}
723

724
/** OpCode Assembler Functions **/
725

726
var (
727
    _T_json_Marshaler         = rt.UnpackType(jsonMarshalerType)
728
    _T_encoding_TextMarshaler = rt.UnpackType(encodingTextMarshalerType)
729
)
730

731
var (
732
    _F_f64toa    = jit.Imm(int64(native.S_f64toa))
733
    _F_f32toa    = jit.Imm(int64(native.S_f32toa))
734
    _F_i64toa    = jit.Imm(int64(native.S_i64toa))
735
    _F_u64toa    = jit.Imm(int64(native.S_u64toa))
736
    _F_b64encode = jit.Imm(int64(_subr__b64encode))
737
)
738

739
var (
740
    _F_memmove       = jit.Func(memmove)
741
    _F_error_number  = jit.Func(error_number)
742
    _F_isValidNumber = jit.Func(isValidNumber)
743
)
744

745
var (
746
    _F_iteratorStop  = jit.Func(iteratorStop)
747
    _F_iteratorNext  = jit.Func(iteratorNext)
748
    _F_iteratorStart = jit.Func(iteratorStart)
749
)
750

751
var (
752
    _F_encodeTypedPointer  obj.Addr
753
    _F_encodeJsonMarshaler obj.Addr
754
    _F_encodeTextMarshaler obj.Addr
755
)
756

757
const (
758
    _MODE_AVX2 = 1 << 2
759
)
760

761
func init() {
762
    _F_encodeTypedPointer  = jit.Func(encodeTypedPointer)
763
    _F_encodeJsonMarshaler = jit.Func(encodeJsonMarshaler)
764
    _F_encodeTextMarshaler = jit.Func(encodeTextMarshaler)
765
}
766

767
func (self *_Assembler) _asm_OP_null(_ *_Instr) {
768
    self.check_size(4)
769
    self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0))  // MOVL $'null', (RP)(RL*1)
770
    self.Emit("ADDQ", jit.Imm(4), _RL)                             // ADDQ $4, RL
771
}
772

773
func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
774
    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
775
    self.Sjmp("JC", "_empty_arr_{n}")
776
    self._asm_OP_null(nil)
777
    self.Sjmp("JMP", "_empty_arr_end_{n}")
778
    self.Link("_empty_arr_{n}")
779
    self.check_size(2)
780
    self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) 
781
    self.Emit("ADDQ", jit.Imm(2), _RL)    
782
    self.Link("_empty_arr_end_{n}")                  
783
}
784

785
func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
786
    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
787
    self.Sjmp("JC", "_empty_obj_{n}")
788
    self._asm_OP_null(nil)
789
    self.Sjmp("JMP", "_empty_obj_end_{n}")
790
    self.Link("_empty_obj_{n}")
791
    self.check_size(2)
792
    self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))  
793
    self.Emit("ADDQ", jit.Imm(2), _RL) 
794
    self.Link("_empty_obj_end_{n}")                                             
795
}
796

797
func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
798
    self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))                // CMPB (SP.p), $0
799
    self.Sjmp("JE"  , "_false_{n}")                                 // JE   _false_{n}
800
    self.check_size(4)                                              // SIZE $4
801
    self.Emit("MOVL", jit.Imm(_IM_true), jit.Sib(_RP, _RL, 1, 0))   // MOVL $'true', (RP)(RL*1)
802
    self.Emit("ADDQ", jit.Imm(4), _RL)                              // ADDQ $4, RL
803
    self.Sjmp("JMP" , "_end_{n}")                                   // JMP  _end_{n}
804
    self.Link("_false_{n}")                                         // _false_{n}:
805
    self.check_size(5)                                              // SIZE $5
806
    self.Emit("MOVL", jit.Imm(_IM_fals), jit.Sib(_RP, _RL, 1, 0))   // MOVL $'fals', (RP)(RL*1)
807
    self.Emit("MOVB", jit.Imm('e'), jit.Sib(_RP, _RL, 1, 4))        // MOVB $'e', 4(RP)(RL*1)
808
    self.Emit("ADDQ", jit.Imm(5), _RL)                              // ADDQ $5, RL
809
    self.Link("_end_{n}")                                           // _end_{n}:
810
}
811

812
func (self *_Assembler) _asm_OP_i8(_ *_Instr) {
813
    self.store_int(4, _F_i64toa, "MOVBQSX")
814
}
815

816
func (self *_Assembler) _asm_OP_i16(_ *_Instr) {
817
    self.store_int(6, _F_i64toa, "MOVWQSX")
818
}
819

820
func (self *_Assembler) _asm_OP_i32(_ *_Instr) {
821
    self.store_int(17, _F_i64toa, "MOVLQSX")
822
}
823

824
func (self *_Assembler) _asm_OP_i64(_ *_Instr) {
825
    self.store_int(21, _F_i64toa, "MOVQ")
826
}
827

828
func (self *_Assembler) _asm_OP_u8(_ *_Instr) {
829
    self.store_int(3, _F_u64toa, "MOVBQZX")
830
}
831

832
func (self *_Assembler) _asm_OP_u16(_ *_Instr) {
833
    self.store_int(5, _F_u64toa, "MOVWQZX")
834
}
835

836
func (self *_Assembler) _asm_OP_u32(_ *_Instr) {
837
    self.store_int(16, _F_u64toa, "MOVLQZX")
838
}
839

840
func (self *_Assembler) _asm_OP_u64(_ *_Instr) {
841
    self.store_int(20, _F_u64toa, "MOVQ")
842
}
843

844
func (self *_Assembler) _asm_OP_f32(_ *_Instr) {
845
    self.check_size(32)
846
    self.Emit("MOVL"    , jit.Ptr(_SP_p, 0), _AX)       // MOVL     (SP.p), AX
847
    self.Emit("ANDL"    , jit.Imm(_FM_exp32), _AX)      // ANDL     $_FM_exp32, AX
848
    self.Emit("XORL"    , jit.Imm(_FM_exp32), _AX)      // XORL     $_FM_exp32, AX
849
    self.Sjmp("JZ"      , _LB_error_nan_or_infinite)    // JZ       _error_nan_or_infinite
850
    self.save_c()                                       // SAVE     $C_regs
851
    self.rbuf_di()                                      // MOVQ     RP, DI
852
    self.Emit("MOVSS"   , jit.Ptr(_SP_p, 0), _X0)       // MOVSS    (SP.p), X0
853
    self.call_c(_F_f32toa)                              // CALL_C   f64toa
854
    self.Emit("ADDQ"    , _AX, _RL)                     // ADDQ     AX, RL
855
}
856

857
func (self *_Assembler) _asm_OP_f64(_ *_Instr) {
858
    self.check_size(32)
859
    self.Emit("MOVQ"  , jit.Ptr(_SP_p, 0), _AX)     // MOVQ   (SP.p), AX
860
    self.Emit("MOVQ"  , jit.Imm(_FM_exp64), _CX)    // MOVQ   $_FM_exp64, CX
861
    self.Emit("ANDQ"  , _CX, _AX)                   // ANDQ   CX, AX
862
    self.Emit("XORQ"  , _CX, _AX)                   // XORQ   CX, AX
863
    self.Sjmp("JZ"    , _LB_error_nan_or_infinite)  // JZ     _error_nan_or_infinite
864
    self.save_c()                                   // SAVE   $C_regs
865
    self.rbuf_di()                                  // MOVQ   RP, DI
866
    self.Emit("MOVSD" , jit.Ptr(_SP_p, 0), _X0)     // MOVSD  (SP.p), X0
867
    self.call_c(_F_f64toa)                          // CALL_C f64toa
868
    self.Emit("ADDQ"  , _AX, _RL)                   // ADDQ   AX, RL
869
}
870

871
func (self *_Assembler) _asm_OP_str(_ *_Instr) {
872
    self.encode_string(false)
873
}
874

875
func (self *_Assembler) _asm_OP_bin(_ *_Instr) {
876
    self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _AX)           // MOVQ 8(SP.p), AX
877
    self.Emit("ADDQ", jit.Imm(2), _AX)                  // ADDQ $2, AX
878
    self.Emit("MOVQ", jit.Imm(_IM_mulv), _CX)           // MOVQ $_MF_mulv, CX
879
    self.Emit("MOVQ", _DX, _R8)                         // MOVQ DX, R8
880
    self.From("MULQ", _CX)                              // MULQ CX
881
    self.Emit("LEAQ", jit.Sib(_DX, _DX, 1, 1), _AX)     // LEAQ 1(DX)(DX), AX
882
    self.Emit("ORQ" , jit.Imm(2), _AX)                  // ORQ  $2, AX
883
    self.Emit("MOVQ", _R8, _DX)                         // MOVQ R8, DX
884
    self.check_size_r(_AX, 0)                           // SIZE AX
885
    self.add_char('"')                                  // CHAR $'"'
886
    self.save_c()                                       // SAVE $REG_ffi
887
    self.prep_buffer_c()                                // MOVE {buf}, DI
888
    self.Emit("MOVQ", _SP_p, _SI)                       // MOVQ SP.p, SI
889

890
    /* check for AVX2 support */
891
    if !cpu.HasAVX2 {
892
        self.Emit("XORL", _DX, _DX)                     // XORL DX, DX
893
    } else {
894
        self.Emit("MOVL", jit.Imm(_MODE_AVX2), _DX)     // MOVL $_MODE_AVX2, DX
895
    }
896

897
    /* call the encoder */
898
    self.call_c(_F_b64encode)   // CALL b64encode
899
    self.load_buffer()          // LOAD {buf}
900
    self.add_char('"')          // CHAR $'"'
901
}
902

903
func (self *_Assembler) _asm_OP_quote(_ *_Instr) {
904
    self.encode_string(true)
905
}
906

907
func (self *_Assembler) _asm_OP_number(_ *_Instr) {
908
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _CX)          // MOVQ    (SP.p), CX
909
    self.Emit("TESTQ", _CX, _CX)                        // TESTQ   CX, CX
910
    self.Sjmp("JZ"   , "_empty_{n}")                    // JZ      _empty_{n}
911
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX)          // MOVQ    (SP.p), AX
912
    self.Emit("TESTQ", _AX, _AX)                        // TESTQ   AX, AX
913
    self.Sjmp("JNZ"   , "_number_next_{n}") 
914
    self.Emit("MOVQ", jit.Imm(int64(panicNilPointerOfNonEmptyString)), jit.Ptr(_SP, 0))
915
    self.Sjmp("JMP", _LB_panic)
916
    self.Link("_number_next_{n}")
917
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0))            // MOVQ    AX, (SP)
918
    self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8))            // MOVQ    CX, 8(SP)
919
    self.call_go(_F_isValidNumber)                      // CALL_GO isValidNumber
920
    self.Emit("CMPB" , jit.Ptr(_SP, 16), jit.Imm(0))    // CMPB    16(SP), $0
921
    self.Sjmp("JE"   , _LB_error_invalid_number)        // JE      _error_invalid_number
922
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _AX)          // MOVQ    8(SP.p), AX
923
    self.check_size_r(_AX, 0)                           // SIZE    AX
924
    self.Emit("LEAQ" , jit.Sib(_RP, _RL, 1, 0), _AX)    // LEAQ    (RP)(RL), AX
925
    self.Emit("ADDQ" , jit.Ptr(_SP_p, 8), _RL)          // ADDQ    8(SP.p), RL
926
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0))            // MOVQ    AX, (SP)
927
    self.Emit("MOVOU", jit.Ptr(_SP_p, 0), _X0)          // MOVOU   (SP.p), X0
928
    self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8))            // MOVOU   X0, 8(SP)
929
    self.call_go(_F_memmove)                            // CALL_GO memmove
930
    self.Sjmp("JMP"  , "_done_{n}")                     // JMP     _done_{n}
931
    self.Link("_empty_{n}")                             // _empty_{n}:
932
    self.check_size(1)                                  // SIZE    $1
933
    self.add_char('0')                                  // CHAR    $'0'
934
    self.Link("_done_{n}")                              // _done_{n}:
935
}
936

937
func (self *_Assembler) _asm_OP_eface(_ *_Instr) {
938
    self.prep_buffer()                          // MOVE  {buf}, (SP)s
939
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX)  // MOVQ  (SP.p), AX
940
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8))    // MOVQ  AX, 8(SP)
941
    self.Emit("LEAQ" , jit.Ptr(_SP_p, 8), _AX)  // LEAQ  8(SP.p), AX
942
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16))   // MOVQ  AX, 16(SP)
943
    self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24))   // MOVQ  ST, 24(SP)
944
    self.Emit("MOVQ" , _ARG_fv, _AX)            // MOVQ  fv, AX
945
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32))   // MOVQ  AX, 32(SP)
946
    self.call_encoder(_F_encodeTypedPointer)    // CALL  encodeTypedPointer
947
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET)   // MOVQ  40(SP), ET
948
    self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP)   // MOVQ  48(SP), EP
949
    self.Emit("TESTQ", _ET, _ET)                // TESTQ ET, ET
950
    self.Sjmp("JNZ"  , _LB_error)               // JNZ   _error
951
}
952

953
func (self *_Assembler) _asm_OP_iface(_ *_Instr) {
954
    self.prep_buffer()                          // MOVE  {buf}, (SP)
955
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX)  // MOVQ  (SP.p), AX
956
    self.Emit("MOVQ" , jit.Ptr(_AX, 8), _AX)    // MOVQ  8(AX), AX
957
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8))    // MOVQ  AX, 8(SP)
958
    self.Emit("LEAQ" , jit.Ptr(_SP_p, 8), _AX)  // LEAQ  8(SP.p), AX
959
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16))   // MOVQ  AX, 16(SP)
960
    self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24))   // MOVQ  ST, 24(SP)
961
    self.Emit("MOVQ" , _ARG_fv, _AX)            // MOVQ  fv, AX
962
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32))   // MOVQ  AX, 32(SP)
963
    self.call_encoder(_F_encodeTypedPointer)    // CALL  encodeTypedPointer
964
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET)   // MOVQ  40(SP), ET
965
    self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP)   // MOVQ  48(SP), EP
966
    self.Emit("TESTQ", _ET, _ET)                // TESTQ ET, ET
967
    self.Sjmp("JNZ"  , _LB_error)               // JNZ   _error
968
}
969

970
func (self *_Assembler) _asm_OP_byte(p *_Instr) {
971
    self.check_size(1)
972
    self.Emit("MOVB", jit.Imm(p.i64()), jit.Sib(_RP, _RL, 1, 0))    // MOVL p.vi(), (RP)(RL*1)
973
    self.Emit("ADDQ", jit.Imm(1), _RL)                              // ADDQ $1, RL
974
}
975

976
func (self *_Assembler) _asm_OP_text(p *_Instr) {
977
    self.check_size(len(p.vs()))    // SIZE ${len(p.vs())}
978
    self.add_text(p.vs())           // TEXT ${p.vs()}
979
}
980

981
func (self *_Assembler) _asm_OP_deref(_ *_Instr) {
982
    self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _SP_p)     // MOVQ (SP.p), SP.p
983
}
984

985
func (self *_Assembler) _asm_OP_index(p *_Instr) {
986
    self.Emit("MOVQ", jit.Imm(p.i64()), _AX)    // MOVQ $p.vi(), AX
987
    self.Emit("ADDQ", _AX, _SP_p)               // ADDQ AX, SP.p
988
}
989

990
func (self *_Assembler) _asm_OP_load(_ *_Instr) {
991
    self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX)                 // MOVQ (ST), AX
992
    self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -24), _SP_x)     // MOVQ -24(ST)(AX), SP.x
993
    self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -8), _SP_p)      // MOVQ -8(ST)(AX), SP.p
994
    self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _SP_q)       // MOVQ (ST)(AX), SP.q
995
}
996

997
func (self *_Assembler) _asm_OP_save(_ *_Instr) {
998
    self.save_state()
999
}
1000

1001
func (self *_Assembler) _asm_OP_drop(_ *_Instr) {
1002
    self.drop_state(_StateSize)
1003
}
1004

1005
func (self *_Assembler) _asm_OP_drop_2(_ *_Instr) {
1006
    self.drop_state(_StateSize * 2)                     // DROP  $(_StateSize * 2)
1007
    self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 56))   // MOVOU X0, 56(ST)(AX)
1008
}
1009

1010
func (self *_Assembler) _asm_OP_recurse(p *_Instr) {
1011
    self.prep_buffer()                          // MOVE {buf}, (SP)
1012
    vt, pv := p.vp()
1013
    self.Emit("MOVQ", jit.Type(vt), _AX)    // MOVQ $(type(p.vt())), AX
1014
    self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8))     // MOVQ AX, 8(SP)
1015

1016
    /* check for indirection */
1017
    if !rt.UnpackType(vt).Indirect() {
1018
        self.Emit("MOVQ", _SP_p, _AX)               // MOVQ SP.p, AX
1019
    } else {
1020
        self.Emit("MOVQ", _SP_p, _VAR_vp)  // MOVQ SP.p, 48(SP)
1021
        self.Emit("LEAQ", _VAR_vp, _AX)    // LEAQ 48(SP), AX
1022
    }
1023

1024
    /* call the encoder */
1025
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16))   // MOVQ  AX, 16(SP)
1026
    self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24))   // MOVQ  ST, 24(SP)
1027
    self.Emit("MOVQ" , _ARG_fv, _AX)            // MOVQ  fv, AX
1028
    if pv {
1029
        self.Emit("BTCQ", jit.Imm(bitPointerValue), _AX)  // BTCQ $1, AX
1030
    }
1031
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32))   // MOVQ  AX, 32(SP)
1032
    self.call_encoder(_F_encodeTypedPointer)    // CALL  encodeTypedPointer
1033
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET)   // MOVQ  40(SP), ET
1034
    self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP)   // MOVQ  48(SP), EP
1035
    self.Emit("TESTQ", _ET, _ET)                // TESTQ ET, ET
1036
    self.Sjmp("JNZ"  , _LB_error)               // JNZ   _error
1037
}
1038

1039
func (self *_Assembler) _asm_OP_is_nil(p *_Instr) {
1040
    self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0))    // CMPQ (SP.p), $0
1041
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1042
}
1043

1044
func (self *_Assembler) _asm_OP_is_nil_p1(p *_Instr) {
1045
    self.Emit("CMPQ", jit.Ptr(_SP_p, 8), jit.Imm(0))    // CMPQ 8(SP.p), $0
1046
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1047
}
1048

1049
func (self *_Assembler) _asm_OP_is_zero_1(p *_Instr) {
1050
    self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))    // CMPB (SP.p), $0
1051
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1052
}
1053

1054
func (self *_Assembler) _asm_OP_is_zero_2(p *_Instr) {
1055
    self.Emit("CMPW", jit.Ptr(_SP_p, 0), jit.Imm(0))    // CMPW (SP.p), $0
1056
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1057
}
1058

1059
func (self *_Assembler) _asm_OP_is_zero_4(p *_Instr) {
1060
    self.Emit("CMPL", jit.Ptr(_SP_p, 0), jit.Imm(0))    // CMPL (SP.p), $0
1061
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1062
}
1063

1064
func (self *_Assembler) _asm_OP_is_zero_8(p *_Instr) {
1065
    self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0))    // CMPQ (SP.p), $0
1066
    self.Xjmp("JE"  , p.vi())                           // JE   p.vi()
1067
}
1068

1069
func (self *_Assembler) _asm_OP_is_zero_map(p *_Instr) {
1070
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX)          // MOVQ  (SP.p), AX
1071
    self.Emit("TESTQ", _AX, _AX)                        // TESTQ AX, AX
1072
    self.Xjmp("JZ"   , p.vi())                          // JZ    p.vi()
1073
    self.Emit("CMPQ" , jit.Ptr(_AX, 0), jit.Imm(0))     // CMPQ  (AX), $0
1074
    self.Xjmp("JE"   , p.vi())                          // JE    p.vi()
1075
}
1076

1077
func (self *_Assembler) _asm_OP_goto(p *_Instr) {
1078
    self.Xjmp("JMP", p.vi())
1079
}
1080

1081
func (self *_Assembler) _asm_OP_map_iter(p *_Instr) {
1082
    self.Emit("MOVQ" , jit.Type(p.vt()), _AX)       // MOVQ    $p.vt(), AX
1083
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _CX)      // MOVQ    (SP.p), CX
1084
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0))        // MOVQ    AX, (SP)
1085
    self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8))        // MOVQ    CX, 8(SP)
1086
    self.Emit("MOVQ" , _ARG_fv, _AX)                // MOVQ    fv, AX
1087
    self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16))       // MOVQ    AX, 16(SP)
1088
    self.call_go(_F_iteratorStart)                  // CALL_GO iteratorStart
1089
    self.Emit("MOVQ" , jit.Ptr(_SP, 24), _SP_q)     // MOVQ    24(SP), SP.q
1090
    self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET)       // MOVQ    32(SP), ET
1091
    self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP)       // MOVQ    40(SP), EP
1092
    self.Emit("TESTQ", _ET, _ET)                    // TESTQ   ET, ET
1093
    self.Sjmp("JNZ"  , _LB_error)                   // JNZ     _error
1094
}
1095

1096
func (self *_Assembler) _asm_OP_map_stop(_ *_Instr) {
1097
    self.Emit("MOVQ", _SP_q, jit.Ptr(_SP, 0))   // MOVQ    SP.q, 0(SP)
1098
    self.call_go(_F_iteratorStop)               // CALL_GO iteratorStop
1099
    self.Emit("XORL", _SP_q, _SP_q)             // XORL    SP.q, SP.q
1100
}
1101

1102
func (self *_Assembler) _asm_OP_map_check_key(p *_Instr) {
1103
    self.Emit("MOVQ" , jit.Ptr(_SP_q, 0), _SP_p)    // MOVQ    (SP.q), SP.p
1104
    self.Emit("TESTQ", _SP_p, _SP_p)                // TESTQ   SP.p, SP.p
1105
    self.Xjmp("JZ"   , p.vi())                      // JNZ     p.vi()
1106
}
1107

1108
func (self *_Assembler) _asm_OP_map_write_key(p *_Instr) {
1109
    self.Emit("BTQ", jit.Imm(bitSortMapKeys), _ARG_fv)      // BTQ ${SortMapKeys}, fv
1110
    self.Sjmp("JNC", "_unordered_key_{n}")                  // JNC _unordered_key_{n}
1111
    self.encode_string(false)                               // STR $false
1112
    self.Xjmp("JMP", p.vi())                                // JMP ${p.vi()}
1113
    self.Link("_unordered_key_{n}")                         // _unordered_key_{n}:
1114
}
1115

1116
func (self *_Assembler) _asm_OP_map_value_next(_ *_Instr) {
1117
    self.Emit("MOVQ", jit.Ptr(_SP_q, 8), _SP_p)     // MOVQ    8(SP.q), SP.p
1118
    self.Emit("MOVQ", _SP_q, jit.Ptr(_SP, 0))       // MOVQ    SP.q, (SP)
1119
    self.call_go(_F_iteratorNext)                   // CALL_GO iteratorNext
1120
}
1121

1122
func (self *_Assembler) _asm_OP_slice_len(_ *_Instr) {
1123
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _SP_x)        // MOVQ  8(SP.p), SP.x
1124
    self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _SP_p)        // MOVQ  (SP.p), SP.p
1125
    self.Emit("ORQ"  , jit.Imm(1 << _S_init), _SP_f)    // ORQ   $(1<<_S_init), SP.f
1126
}
1127

1128
func (self *_Assembler) _asm_OP_slice_next(p *_Instr) {
1129
    self.Emit("TESTQ"  , _SP_x, _SP_x)                          // TESTQ   SP.x, SP.x
1130
    self.Xjmp("JZ"     , p.vi())                                // JZ      p.vi()
1131
    self.Emit("SUBQ"   , jit.Imm(1), _SP_x)                     // SUBQ    $1, SP.x
1132
    self.Emit("BTRQ"   , jit.Imm(_S_init), _SP_f)               // BTRQ    $_S_init, SP.f
1133
    self.Emit("LEAQ"   , jit.Ptr(_SP_p, int64(p.vlen())), _AX)  // LEAQ    $(p.vlen())(SP.p), AX
1134
    self.Emit("CMOVQCC", _AX, _SP_p)                            // CMOVQNC AX, SP.p
1135
}
1136

1137
func (self *_Assembler) _asm_OP_marshal(p *_Instr) {
1138
    self.call_marshaler(_F_encodeJsonMarshaler, _T_json_Marshaler, p.vt())
1139
}
1140

1141
func (self *_Assembler) _asm_OP_marshal_p(p *_Instr) {
1142
    if p.vk() != reflect.Ptr {
1143
        panic("marshal_p: invalid type")
1144
    } else {
1145
        self.call_marshaler_v(_F_encodeJsonMarshaler, _T_json_Marshaler, p.vt(), false)
1146
    }
1147
}
1148

1149
func (self *_Assembler) _asm_OP_marshal_text(p *_Instr) {
1150
    self.call_marshaler(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.vt())
1151
}
1152

1153
func (self *_Assembler) _asm_OP_marshal_text_p(p *_Instr) {
1154
    if p.vk() != reflect.Ptr {
1155
        panic("marshal_text_p: invalid type")
1156
    } else {
1157
        self.call_marshaler_v(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.vt(), false)
1158
    }
1159
}
1160

1161
func (self *_Assembler) _asm_OP_cond_set(_ *_Instr) {
1162
    self.Emit("ORQ", jit.Imm(1 << _S_cond), _SP_f)  // ORQ $(1<<_S_cond), SP.f
1163
}
1164

1165
func (self *_Assembler) _asm_OP_cond_testc(p *_Instr) {
1166
    self.Emit("BTRQ", jit.Imm(_S_cond), _SP_f)      // BTRQ $_S_cond, SP.f
1167
    self.Xjmp("JC"  , p.vi())
1168
}
1169

1170
func (self *_Assembler) print_gc(i int, p1 *_Instr, p2 *_Instr) {
1171
    self.Emit("MOVQ", jit.Imm(int64(p2.op())),  jit.Ptr(_SP, 16))// MOVQ $(p2.op()), 16(SP)
1172
    self.Emit("MOVQ", jit.Imm(int64(p1.op())),  jit.Ptr(_SP, 8)) // MOVQ $(p1.op()), 8(SP)
1173
    self.Emit("MOVQ", jit.Imm(int64(i)),  jit.Ptr(_SP, 0))       // MOVQ $(i), (SP)
1174
    self.call_go(_F_println)
1175
}
1176