/
githubmirror
/
FFmpeg
Обзор
Документация
Войти
/
githubmirror
/
FFmpeg
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
libavutil/aarch64/tx_float_neon.S
1 288 строк
49 KB
Martin Storsjö
aarch64: Add PAC sign/validation of the link register
20 мар 2026, 14:16
20 мар 2026, 14:16
f72f692
Код
Авторство
О чём код?
/* * Copyright (c) Lynne * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavutil/aarch64/asm.S" /* Open `doc/transforms.md` to see the code upon which the transforms here were * based upon. * * File conventions: * GPRs: x0-x3 - arguments, untouched * x4 - Lookup table base pointer * x5-x6 - macro ld1 temps/function scratch * x7-x9 - FFT table state * x10-x17 - lookup table/macro scratch * w19-w20 - current/target length when needed * x21-x22 - len*2, len*6 * * Vectors: v0-v7 - coefficients * v8-v15 - coefficients when needed, otherwise untouched * v16-v30 - used as needed * v31 - -1.0, +1.0, -1.0, +1.0. Never touched after loading. * * Stack: backup for v8-v15 and x19-x22 when needed, and transform lengths */ #define M_SQRT1_2 0.707106781186547524401 #define COS16_1 0.92387950420379638671875 #define COS16_3 0.3826834261417388916015625 /* We only ever load this once at the start, and then live with losing an * entire register as we need to lug this all the time everywhere. * Clearly should be integrated into an fsadd and fmlsa, but "muh RISC!". */ const subadd, align=4 .float -1.0, 1.0, -1.0, 1.0 endconst .macro LOAD_SUBADD movrel x5, subadd ld1 { v31.4s }, [x5] .endm .macro SETUP_LUT no_lut=0 .if \no_lut == 0 ldr x4, [x0, #8] .endif .endm .macro LOAD_INPUT dst1, dst2, dst3, dst4, src, no_lut=0, discont=0 .if \no_lut == 1 .if \discont == 1 ldp q\dst1\(), q\dst2\(), [\src\()] ldp q\dst3\(), q\dst4\(), [\src\(), #32] add \src\(), \src\(), #64 .else ld1 { v\dst1\().4s, v\dst2\().4s, v\dst3\().4s, v\dst4\().4s }, [\src], #64 .endif .else ldp w10, w11, [x4, #0 ] ldp w12, w13, [x4, #8 ] ldp w14, w15, [x4, #16] ldp w16, w17, [x4, #24] add x4, x4, #32 ldr d\dst1, [\src, x10, lsl #3] add x11, \src, x11, lsl #3 ldr d\dst2, [\src, x12, lsl #3] add x13, \src, x13, lsl #3 ldr d\dst3, [\src, x14, lsl #3] add x15, \src, x15, lsl #3 ldr d\dst4, [\src, x16, lsl #3] add x17, \src, x17, lsl #3 ld1 { v\dst1\().d }[1], [x11] ld1 { v\dst2\().d }[1], [x13] ld1 { v\dst3\().d }[1], [x15] ld1 { v\dst4\().d }[1], [x17] .endif .endm .macro FFT4 e0, o0, standalone fadd v16.4s, \e0\().4s, \o0\().4s // r1..4 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1..4 rev64 v18.4s, \e0\().4s zip2 \o0\().2d, v16.2d, \e0\().2d zip1 v17.2d, v16.2d, \e0\().2d mov \o0\().d[1], v18.d[1] fadd \e0\().4s, v17.4s, \o0\().4s // a1,2 b1,4 fsub v16.4s, v17.4s, \o0\().4s // a3,4 b3,2 mov \o0\().16b, v16.16b // Swap once again... mov \o0\().s[3], \e0\().s[3] mov \e0\().s[3], v16.s[3] .if \standalone == 0 uzp2 \o0\().2d, \e0\().2d, \o0\().2d uzp1 \e0\().2d, \e0\().2d, v16.2d .endif .endm // Identical to FFT4, but does 2 transforms in parallel, with no deinterleaving .macro FFT4_X2 e0, o0, e1, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 fadd \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \t2\().4s, \e1\().4s, \o1\().4s // r1234 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1234 fsub \e1\().4s, \e1\().4s, \o1\().4s // t1234 rev64 \t4\().4s, \e0\().4s rev64 \t5\().4s, \e1\().4s zip2 \o0\().2d, \t0\().2d, \t4\().2d // t3,4 r3,4 zip2 \o1\().2d, \t2\().2d, \t5\().2d // t3,4 r3,4 fneg \t3\().4s, \t4\().4s fneg \t4\().4s, \t5\().4s mov \o0\().s[3], \t3\().s[3] mov \o1\().s[3], \t4\().s[3] zip1 \t1\().2d, \t0\().2d, \e0\().2d // t1,2 r1,2 zip1 \t3\().2d, \t2\().2d, \e1\().2d // t1,2 r1,2 fsub \t4\().4s, \t1\().4s, \o0\().4s // a34 b32 fadd \t5\().4s, \t1\().4s, \o0\().4s // a12 b14 fsub \t2\().4s, \t3\().4s, \o1\().4s // a34 b32 fadd \t3\().4s, \t3\().4s, \o1\().4s // a12 b14 zip1 \e0\().2d, \t5\().2d, \t4\().2d // a1234 zip2 \o0\().2d, \t5\().2d, \t4\().2d // b1432 zip1 \e1\().2d, \t3\().2d, \t2\().2d // a1234 zip2 \o1\().2d, \t3\().2d, \t2\().2d // b1432 .endm const tab_8pt, align=4 .float M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2 endconst .macro FFT8 e0, e1, o0, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 movrel x5, tab_8pt fsub \t1\().4s, \e1\().4s, \o1\().4s // j1234 fadd \o1\().4s, \e1\().4s, \o1\().4s // k1234 fsub \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \o0\().4s, \e0\().4s, \o0\().4s // q1234 ld1 { \t5\().4s }, [x5] ext \t4\().16b, \o1\().16b, \o1\().16b, #12 rev64 \t4\().4s, \t4\().4s ext \t2\().16b, \o0\().16b, \t4\().16b, #8 // o0[0,1], o1[3,2] mov \o0\().d[1], \t4\().d[1] // o0[3, 4]; o1[1, 4] fsub \e1\().4s, \o0\().4s, \t2\().4s // s34, g43 fadd \t2\().4s, \o0\().4s, \t2\().4s // s12, g12 rev64 \t6\().4s, v31.4s // 1, -1, 1, -1 dup \o0\().2d, \t0\().d[0] // r1212 dup \o1\().2d, \t0\().d[1] // r3434 rev64 \t4\().4s, \e1\().4s // xxg34 rev64 \o1\().4s, \o1\().4s // r4343 ext \t6\().16b, v31.16b, \t6\().16b, #8 // -1, 1, 1, -1 zip1 \t3\().2d, \t2\().2d, \e1\().2d // s1234 zip2 \t2\().2d, \t2\().2d, \t4\().2d // g1234 fadd \e0\().4s, \t3\().4s, \t2\().4s // out_e1 fsub \e1\().4s, \t3\().4s, \t2\().4s // out_e2 fmul \t1\().4s, \t1\().4s, \t5\().4s // j * +--+M_SQRT1_2 fmls \o0\().4s, \o1\().4s, \t6\().4s // z1234 rev64 \t4\().4s, \t1\().4s // j2143 fmla \t1\().4s, \t4\().4s, v31.4s // l2143 rev64 \t4\().4s, \t1\().4s // l1234 ext \t4\().16b, \t4\().16b, \t4\().16b, #8 // l3412 fmla \t4\().4s, \t1\().4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, \t4\().4s // out_o2 fsub \o0\().4s, \o0\().4s, \t4\().4s // out_o1 .endm // Identical as FFT8, but does 2 transforms in parallel .macro FFT8_X2 e0, e1, o0, o1, e2, e3, o2, o3 movrel x5, tab_8pt fadd v19.4s, \e3\().4s, \o3\().4s // k1234 fadd v17.4s, \e1\().4s, \o1\().4s // k1234 fadd v18.4s, \e2\().4s, \o2\().4s // q1234 fadd v16.4s, \e0\().4s, \o0\().4s // q1234 ld1 { v23.4s }, [x5] ext v22.16b, v19.16b, v19.16b, #12 ext v21.16b, v17.16b, v17.16b, #12 rev64 v22.4s, v22.4s rev64 v21.4s, v21.4s ext v19.16b, v18.16b, v22.16b, #8 ext v17.16b, v16.16b, v21.16b, #8 mov v18.d[1], v22.d[1] mov v21.d[0], v16.d[0] fadd v22.4s, v18.4s, v19.4s // s12, g12 fsub v19.4s, v18.4s, v19.4s // s34, g43 fsub v18.4s, v21.4s, v17.4s // s34, g43 fadd v16.4s, v21.4s, v17.4s // s12, g12 fsub \e0\().4s, \e0\().4s, \o0\().4s // r1234 fsub v20.4s, \e1\().4s, \o1\().4s // j1234 fsub \e2\().4s, \e2\().4s, \o2\().4s // r1234 fsub v21.4s, \e3\().4s, \o3\().4s // j1234 rev64 v24.4s, v31.4s // 1, -1, 1, -1 zip1 v17.2d, v16.2d, v18.2d // s1234 zip1 \e1\().2d, v22.2d, v19.2d // s1234 rev64 v18.4s, v18.4s // xxg34 rev64 v19.4s, v19.4s // xxg34 zip2 v16.2d, v16.2d, v18.2d // g1234 zip2 \e3\().2d, v22.2d, v19.2d // g1234 dup \o0\().2d, \e0\().d[0] // r1212 dup \o1\().2d, \e0\().d[1] // r3434 dup \o2\().2d, \e2\().d[0] // r1212 dup \o3\().2d, \e2\().d[1] // r3434 fadd \e2\().4s, \e1\().4s, \e3\().4s // out_e1 fsub \e3\().4s, \e1\().4s, \e3\().4s // out_e2 fadd \e0\().4s, v17.4s, v16.4s // out_e1 fsub \e1\().4s, v17.4s, v16.4s // out_e2 ext v24.16b, v31.16b, v24.16b, #8 // -1, 1, 1, -1 rev64 \o1\().4s, \o1\().4s // r4343 rev64 \o3\().4s, \o3\().4s // r4343 fmul v19.4s, v20.4s, v23.4s // j * +--+M_SQRT1_2 fmul v21.4s, v21.4s, v23.4s // j * +--+M_SQRT1_2 rev64 v20.4s, v19.4s // j2143 rev64 v18.4s, v21.4s // j2143 fmls \o0\().4s, \o1\().4s, v24.4s // z1234 fmls \o2\().4s, \o3\().4s, v24.4s // z1234 fmla v19.4s, v20.4s, v31.4s // l2143 fmla v21.4s, v18.4s, v31.4s // l2143 rev64 v20.4s, v19.4s // l1234 rev64 v18.4s, v21.4s // l1234 ext v20.16b, v20.16b, v20.16b, #8 // l3412 ext v18.16b, v18.16b, v18.16b, #8 // l3412 fmla v20.4s, v19.4s, v31.4s // t1234 fmla v18.4s, v21.4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, v20.4s // out_o2 fadd \o3\().4s, \o2\().4s, v18.4s // out_o2 fsub \o0\().4s, \o0\().4s, v20.4s // out_o1 fsub \o2\().4s, \o2\().4s, v18.4s // out_o1 .endm const tab_16pt, align=4 .float -COS16_1, COS16_1, -COS16_3, COS16_3 // Could be +-+- too .float COS16_3, COS16_3, COS16_1, COS16_1 .float 1.0, 1.0, M_SQRT1_2, M_SQRT1_2 endconst // 16-point FFT // t3, t4, t5, t6 must be sequential .macro FFT16 e0, e1, e2, e3, o0, o1, o2, o3, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 FFT8 \e0, \e1, \e2, \e3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 FFT4_X2 \o0, \o1, \o2, \o3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 movrel x5, tab_16pt rev64 \t0\().4s, \o0\().4s // z[ 8, 9].imre rev64 \t1\().4s, \o2\().4s // z[10,11].imre ins \t0\().d[0], xzr ins \t1\().d[0], xzr ld1 { \t4\().4s, \t5\().4s, \t6\().4s }, [x5] // TODO: We could derive \t4\() or \t5\() from either, but it seems cheaper to load fmla \o2\().4s, \t1\().4s, v31.4s // s[4567] fmls \o0\().4s, \t0\().4s, v31.4s // s[0123] fmul \t2\().4s, \o1\().4s, \t4\().4s fmul \t3\().4s, \o3\().4s, \t4\().4s rev64 \o3\().4s, \o3\().4s rev64 \o1\().4s, \o1\().4s fmla \t3\().4s, \o3\().4s, \t5\().4s // s[12, 13, 14, 15] fmls \t2\().4s, \o1\().4s, \t5\().4s // s[ 8, 9, 10, 11] fmul \t1\().4s, \o2\().4s, \t6\().4s // s[4567] * mult fmul \t0\().4s, \o0\().4s, \t6\().4s // s[0123] * mult mov \o1\().16b, \t3\().16b mov \o2\().16b, \t1\().16b fsub \t3\().4s, \t3\().4s, \t2\().4s // y34, u34 fsub \t1\().4s, \t1\().4s, \t0\().4s // w34, x34 fadd \t2\().4s, \t2\().4s, \o1\().4s // y56, u56 rev64 \t3\().4s, \t3\().4s fadd \t0\().4s, \t0\().4s, \o2\().4s // w56, x56 rev64 \t1\().4s, \t1\().4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fadd \o3\().4s, \e3\().4s, \t3\().4s fsub \o2\().4s, \e3\().4s, \t3\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o0\().4s, \e2\().4s, \t2\().4s fsub \e2\().4s, \e0\().4s, \t0\().4s fadd \e0\().4s, \e0\().4s, \t0\().4s fsub \e3\().4s, \e1\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t1\().4s .endm function ff_tx_fft2_float_neon, export=1 ld2r { v0.2d, v1.2d }, [x2] fneg v2.2s, v1.2s mov v2.d[1], v1.d[0] fsub v2.4s, v0.4s, v2.4s st1 { v2.4s }, [x1] ret endfunc .macro FFT4_FN name, inv function ff_tx_fft4_\name\()_float_neon, export=1 ld1 {v0.4s, v1.4s}, [x2] .if \inv == 1 mov v2.d[0], v0.d[1] mov v0.d[1], v1.d[1] mov v1.d[1], v2.d[0] .endif FFT4 v0, v1, 1 st1 { v0.4s, v1.4s }, [x1] ret endfunc .endm FFT4_FN fwd, 0 FFT4_FN inv, 1 .macro FFT8_FN name, no_perm function ff_tx_fft8_\name\()_neon, export=1 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_SUBADD FFT8 v0, v1, v2, v3 zip1 v16.2d, v0.2d, v2.2d zip2 v17.2d, v0.2d, v2.2d zip1 v18.2d, v1.2d, v3.2d zip2 v19.2d, v1.2d, v3.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1] ret endfunc .endm FFT8_FN float, 0 FFT8_FN ns_float, 1 .macro FFT16_FN name, no_perm function ff_tx_fft16_\name\()_neon, export=1 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_SUBADD FFT16 v0, v1, v2, v3, v4, v5, v6, v7 zip1 v20.2d, v0.2d, v4.2d zip2 v21.2d, v0.2d, v4.2d zip1 v22.2d, v1.2d, v6.2d zip2 v23.2d, v1.2d, v6.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v2.2d, v5.2d zip2 v25.2d, v2.2d, v5.2d zip1 v26.2d, v3.2d, v7.2d zip2 v27.2d, v3.2d, v7.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1] ret endfunc .endm FFT16_FN float, 0 FFT16_FN ns_float, 1 .macro SETUP_SR_RECOMB len, re, im, dec ldr w5, =(\len - 4*7) movrel \re, X(ff_tx_tab_\len\()_float) add \im, \re, x5 mov \dec, #-32 .if \len > 32 mov x21, #2*\len add x22, x21, x21, lsl #1 .endif .endm .macro SR_COMBINE e0, e1, e2, e3, e4, e5, e6, e7, \ o0, o1, o2, o3, o4, o5, o6, o7, \ re, im, dec, swap_im, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, \ t6=v22, t7=v23, t8=v24, t9=v25, ta=v26, tb=v27 ld1 { \t8\().4s, \t9\().4s }, [\im], \dec ld1 { \t0\().4s, \t1\().4s }, [\re], #32 .if \swap_im == 1 ext \t2\().16b, \t9\().16b, \t9\().16b, #8 ext \t3\().16b, \t8\().16b, \t8\().16b, #8 .else ext \t2\().16b, \t8\().16b, \t8\().16b, #8 ext \t3\().16b, \t9\().16b, \t9\().16b, #8 .endif trn1 \t4\().4s, \t0\().4s, \t0\().4s // cos0022 trn2 \t0\().4s, \t0\().4s, \t0\().4s // cos4466 trn1 \t5\().4s, \t1\().4s, \t1\().4s // cos1133 trn2 \t1\().4s, \t1\().4s, \t1\().4s // cos5577 rev64 \t6\().4s, \o0\().4s // E m2[0,1].imre rev64 \t7\().4s, \o2\().4s // O m2[0,1].imre rev64 \t8\().4s, \o4\().4s // E m2[2,3].imre rev64 \t9\().4s, \o6\().4s // O m2[2,3].imre fmul \t6\().4s, \t6\().4s, \t4\().4s // E m2[0,1].imre*t1[0,2] fmul \t7\().4s, \t7\().4s, \t0\().4s // O m2[0,1].imre*t1[0,2] fmul \t8\().4s, \t8\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t9\().4s, \t9\().4s, \t0\().4s // O m2[2,3].imre*t1[0,2] rev64 \ta\().4s, \o1\().4s // E m3[0,1].imre rev64 \tb\().4s, \o3\().4s // O m3[0,1].imre rev64 \t4\().4s, \o5\().4s // E m3[2,3].imre rev64 \t0\().4s, \o7\().4s // O m3[2,3].imre fmul \ta\().4s, \ta\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmul \tb\().4s, \tb\().4s, \t1\().4s // O m3[0,1].imre*t1[4,6] fmul \t4\().4s, \t4\().4s, \t5\().4s // E m3[2,3].imre*t1[4,6] fmul \t0\().4s, \t0\().4s, \t1\().4s // O m3[2,3].imre*t1[4,6] trn1 \t5\().4s, \t3\().4s, \t3\().4s // wim2200 trn2 \t3\().4s, \t3\().4s, \t3\().4s // wim3311 trn1 \t1\().4s, \t2\().4s, \t2\().4s // wim6644 trn2 \t2\().4s, \t2\().4s, \t2\().4s // wim7755 fmul \t5\().4s, \t5\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmla \t7\().4s, \o2\().4s, \t5\().4s // O w0123 fmls \t9\().4s, \o6\().4s, \t5\().4s // O j0123 fmla \t6\().4s, \o0\().4s, \t3\().4s // E w0123 fmls \t8\().4s, \o4\().4s, \t3\().4s // E j0123 fmla \ta\().4s, \o1\().4s, \t2\().4s // E w4567 fmla \tb\().4s, \o3\().4s, \t1\().4s // O w4567 fmls \t4\().4s, \o5\().4s, \t2\().4s // E j4567 fmls \t0\().4s, \o7\().4s, \t1\().4s // O j4567 fsub \t2\().4s, \t7\().4s, \t9\().4s fsub \t1\().4s, \t8\().4s, \t6\().4s fsub \t3\().4s, \t4\().4s, \ta\().4s fsub \t5\().4s, \t0\().4s, \tb\().4s fadd \t6\().4s, \t8\().4s, \t6\().4s fadd \t7\().4s, \t9\().4s, \t7\().4s fadd \t8\().4s, \t4\().4s, \ta\().4s fadd \t9\().4s, \t0\().4s, \tb\().4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t5\().4s, \t5\().4s, v31.4s rev64 \t6\().4s, \t6\().4s rev64 \t8\().4s, \t8\().4s rev64 \t7\().4s, \t7\().4s rev64 \t9\().4s, \t9\().4s fsub \o0\().4s, \e0\().4s, \t6\().4s fsub \o1\().4s, \e1\().4s, \t8\().4s fsub \o2\().4s, \e2\().4s, \t1\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s fsub \o4\().4s, \e4\().4s, \t7\().4s fsub \o5\().4s, \e6\().4s, \t9\().4s fadd \o6\().4s, \e5\().4s, \t2\().4s fsub \o7\().4s, \e7\().4s, \t5\().4s fadd \e0\().4s, \e0\().4s, \t6\().4s fadd \e1\().4s, \e1\().4s, \t8\().4s fadd \e2\().4s, \e2\().4s, \t1\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s fadd \e4\().4s, \e4\().4s, \t7\().4s fsub \e5\().4s, \e5\().4s, \t2\().4s // swapped fadd \e6\().4s, \e6\().4s, \t9\().4s // swapped fadd \e7\().4s, \e7\().4s, \t5\().4s .endm .macro SR_COMBINE_HALF e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, t3, t4, t5, part .if \part == 0 trn1 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn1 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn2 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .endif .if \part == 0 trn2 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn2 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn1 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t5\().4s, \t5\().4s, v31.4s fmul \c3\().4s, \c3\().4s, v31.4s rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \t3\().4s, \o3\().4s // E m3[2,3].imre fmul \o0\().4s, \o0\().4s, \c3\().4s // E m2[0,1].imre*t1[0,2] fmul \o1\().4s, \o1\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmla \o0\().4s, \t0\().4s, \t4\().4s // E w0123 fmla \o1\().4s, \t2\().4s, \c1\().4s // E w4567 fmul \t1\().4s, \t1\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t3\().4s, \t3\().4s, \c1\().4s // E m3[2,3].imre*t1[4,6] fmls \t1\().4s, \o2\().4s, \c3\().4s // E j0123 fmls \t3\().4s, \o3\().4s, \t5\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \t1\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \t3\().4s, \o1\().4s fsub \t3\().4s, \t3\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s rev64 \t1\().4s, \t1\().4s rev64 \t2\().4s, \t2\().4s .if \part == 0 fsub \o0\().4s, \e0\().4s, \t1\().4s fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s .else fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \o2\().4s, \e1\().4s, \t2\().4s fsub \o1\().4s, \e2\().4s, \t0\().4s fadd \o3\().4s, \e3\().4s, \t3\().4s .endif .if \part == 0 fadd \e0\().4s, \e0\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s .else fadd \e0\().4s, \e0\().4s, \t1\().4s fsub \e1\().4s, \e1\().4s, \t2\().4s // swapped fadd \e2\().4s, \e2\().4s, \t0\().4s // swapped fsub \e3\().4s, \e3\().4s, \t3\().4s .endif .endm /* Same as SR_COMBINE_HALF, but heroically tries to use 3 temporary registers * without touching the tables. */ .macro SR_COMBINE_LITE e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, part rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre .if \part == 0 trn2 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t2\().4s, \t2\().4s, v31.4s fmul \o2\().4s, \o2\().4s, \t2\().4s fmul \o0\().4s, \o0\().4s, \t2\().4s // E m2[0,1].imre*t1[0,2] .if \part == 0 trn1 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .else trn2 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .endif fmul \t1\().4s, \t1\().4s, \t2\().4s // E m2[2,3].imre*t1[0,2] fmla \o0\().4s, \t0\().4s, \t2\().4s // E w0123 fsub \t1\().4s, \t1\().4s, \o2\().4s // E j0123 rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \o2\().4s, \o3\().4s // E m3[2,3].imre .if \part == 0 trn2 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .else trn1 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .endif fmul \t0\().4s, \t0\().4s, v31.4s fmul \o1\().4s, \o1\().4s, \t0\().4s // E m3[0,1].imre*t1[4,6] fmul \o3\().4s, \o3\().4s, \t0\().4s .if \part == 0 trn1 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .endif fmul \o2\().4s, \o2\().4s, \t0\().4s // E m3[2,3].imre*t1[4,6] fmla \o1\().4s, \t2\().4s, \t0\().4s // E w4567 fsub \o2\().4s, \o2\().4s, \o3\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \o0\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \o2\().4s, \o1\().4s fsub \t1\().4s, \o2\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s rev64 \t2\().4s, \t2\().4s rev64 \o0\().4s, \o0\().4s .if \part == 0 fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t1\().4s .else fadd \o2\().4s, \e1\().4s, \t0\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o3\().4s, \e3\().4s, \t1\().4s .endif .if \part == 0 fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t1\().4s .else fsub \e1\().4s, \e1\().4s, \t0\().4s // swapped fadd \e2\().4s, \e2\().4s, \t2\().4s // swapped fsub \e3\().4s, \e3\().4s, \t1\().4s .endif mov \t1\().16b, \o0\().16b fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \e0\().4s, \e0\().4s, \t1\().4s .endm .macro SR_COMBINE_4 len, part, off add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 ldp q0, q1, [x1, #((0 + \part)*32 + \off)] ldp q4, q5, [x1, #((2 + \part)*32 + \off)] ldp q2, q3, [x10, #((0 + \part)*32 + \off)] ldp q6, q7, [x10, #((2 + \part)*32 + \off)] ldp q8, q9, [x11, #((0 + \part)*32 + \off)] ldp q10, q11, [x11, #((2 + \part)*32 + \off)] ldp q12, q13, [x12, #((0 + \part)*32 + \off)] ldp q14, q15, [x12, #((2 + \part)*32 + \off)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q0, q1, [x1, #((0 + \part)*32 + \off)] stp q4, q5, [x1, #((2 + \part)*32 + \off)] stp q2, q3, [x10, #((0 + \part)*32 + \off)] stp q6, q7, [x10, #((2 + \part)*32 + \off)] stp q8, q9, [x11, #((0 + \part)*32 + \off)] stp q12, q13, [x11, #((2 + \part)*32 + \off)] stp q10, q11, [x12, #((0 + \part)*32 + \off)] stp q14, q15, [x12, #((2 + \part)*32 + \off)] .endm .macro SR_COMBINE_FULL len, off=0 add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 SR_COMBINE_4 \len, 0, \off SR_COMBINE_4 \len, 1, \off SR_COMBINE_4 \len, 4, \off SR_COMBINE_4 \len, 5, \off .endm .macro SR_COMBINE_D2 part, off add x10, x1, #((\part)*32 + \off) add x11, x14, #((\part)*32 + \off) add x12, x15, #((\part)*32 + \off) add x13, x16, #((\part)*32 + \off) ldp q0, q1, [x10] ldp q4, q5, [x10, #(2*32)] ldp q2, q3, [x11] ldp q6, q7, [x11, #(2*32)] ldp q8, q9, [x12] ldp q10, q11, [x12, #(2*32)] ldp q12, q13, [x13] ldp q14, q15, [x13, #(2*32)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d zip1 v20.2d, v2.2d, v6.2d zip2 v21.2d, v2.2d, v6.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d ldp q0, q1, [x10, #(1*32)] ldp q4, q5, [x10, #(3*32)] ldp q2, q3, [x11, #(1*32)] ldp q6, q7, [x11, #(3*32)] st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10], #64 st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x11], #64 zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d ldp q8, q9, [x12, #(1*32)] ldp q10, q11, [x12, #(3*32)] ldp q12, q13, [x13, #(1*32)] ldp q14, q15, [x13, #(3*32)] st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12], #64 st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13], #64 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10] zip1 v16.2d, v2.2d, v6.2d zip2 v17.2d, v2.2d, v6.2d zip1 v18.2d, v3.2d, v7.2d zip2 v19.2d, v3.2d, v7.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x11] zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12] zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13] .endm .macro SR_COMBINE_DINT off=0 add x14, x1, x21 add x15, x1, x21, lsl #1 add x16, x1, x22 SR_COMBINE_D2 0, \off SR_COMBINE_D2 4, \off .endm .macro FFT32_FN name, no_perm function ff_tx_fft32_\name\()_neon, export=1 stp d14, d15, [sp, #-16*4]! stp d8, d9, [sp, #16*3] stp d10, d11, [sp, #16*2] stp d12, d13, [sp, #16] LOAD_SUBADD SETUP_SR_RECOMB 32, x7, x8, x9 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v5, v6, v7 SR_COMBINE v0, v1, v2, v3, v4, v5, v6, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v6.2d zip2 v19.2d, v1.2d, v6.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1], #64 zip1 v20.2d, v2.2d, v5.2d zip2 v21.2d, v2.2d, v5.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v8.2d, v12.2d zip2 v25.2d, v8.2d, v12.2d zip1 v26.2d, v9.2d, v13.2d zip2 v27.2d, v9.2d, v13.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1], #64 zip1 v28.2d, v10.2d, v14.2d zip2 v29.2d, v10.2d, v14.2d zip1 v30.2d, v11.2d, v15.2d zip2 v31.2d, v11.2d, v15.2d st1 { v28.4s, v29.4s, v30.4s, v31.4s }, [x1] ldp d12, d13, [sp, #16] ldp d10, d11, [sp, #16*2] ldp d8, d9, [sp, #16*3] ldp d14, d15, [sp], #16*4 ret endfunc .endm FFT32_FN float, 0 FFT32_FN ns_float, 1 .macro cmp_imm reg, imm .if \imm >= 4096 cmp \reg, #((\imm)/4096), lsl #12 .else cmp \reg, #(\imm) .endif .endm .macro SR_TRANSFORM_DEF len, next=0 \len: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #(\len/4) mov x5, #((\len*4) - (\len/1)) add x1, x1, x5 bl 32b mov x5, #((\len*2) - (\len/2)) add x1, x1, x5 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER ldr w5, =(\len*6 + \len/2) sub x1, x1, x5 SETUP_SR_RECOMB \len, x7, x8, x9 .if \next\() != 0 cmp_imm w19, \len b.eq 0f mov w5, #(\len/128) \len\()5: SR_COMBINE_FULL \len add x1, x1, 8*32 subs w5, w5, 1 b.gt \len\()5b cmp_imm w20, \len b.gt \next\()f ret .endif .endm .macro FFT_SPLIT_RADIX_FN name, no_perm function ff_tx_fft_sr_\name\()_neon, export=1 stp x21, x22, [sp, #-16*6]! stp d8, d9, [sp, #16*5] stp d10, d11, [sp, #16*4] stp d12, d13, [sp, #16*3] stp d14, d15, [sp, #16*2] stp x19, x20, [sp, #16] ldr w19, [x0, #0] // global target mov w20, w19 // local length LOAD_SUBADD SETUP_LUT \no_perm 32: SETUP_SR_RECOMB 32, x7, x8, x9 LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 6, 5, 7, x2, \no_perm, 1 LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v6, v5, v7 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q2, q3, [x1, #32*1] stp q6, q7, [x1, #32*3] stp q10, q11, [x1, #32*5] stp q14, q15, [x1, #32*7] cmp w20, #32 b.gt 64f stp q0, q1, [x1, #32*0] stp q4, q5, [x1, #32*2] stp q8, q9, [x1, #32*4] stp q12, q13, [x1, #32*6] ret 64: SETUP_SR_RECOMB 64, x7, x8, x9 LOAD_INPUT 2, 3, 10, 11, x2, \no_perm, 1 LOAD_INPUT 6, 14, 7, 15, x2, \no_perm, 1 FFT16 v2, v3, v10, v11, v6, v14, v7, v15 LOAD_INPUT 16, 17, 18, 19, x2, \no_perm LOAD_INPUT 20, 22, 21, 23, x2, \no_perm, 1 FFT16 v16, v17, v18, v19, v20, v22, v21, v23, \ v24, v25, v26, v27, v28, v29, v30 ld1 { v26.4s, v27.4s }, [x8], x9 ldp q24, q25, [x7], #32 ext v26.16b, v26.16b, v26.16b, #8 ext v27.16b, v27.16b, v27.16b, #8 cmp w19, #64 b.eq 2f // custom deinterleave // TODO: investigate doing the 2 combines like in deinterleave // TODO: experiment with spilling to gprs and converting to HALF or full SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 stp q0, q1, [x1, #32* 0] stp q8, q9, [x1, #32* 4] stp q2, q3, [x1, #32* 8] stp q16, q17, [x1, #32*12] SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v0, v1, v8, 1 stp q4, q20, [x1, #32* 2] stp q12, q21, [x1, #32* 6] stp q6, q5, [x1, #32*10] stp q7, q13, [x1, #32*14] ldp q2, q3, [x1, #32*1] ldp q6, q7, [x1, #32*3] ldp q12, q13, [x1, #32*5] ldp q16, q17, [x1, #32*7] SR_COMBINE v2, v3, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v8, v0, v1, v4, v5 stp q2, q3, [x1, #32* 1] stp q6, q7, [x1, #32* 3] stp q12, q13, [x1, #32* 5] stp q16, q17, [x1, #32* 7] stp q10, q11, [x1, #32* 9] stp q18, q19, [x1, #32*11] stp q14, q15, [x1, #32*13] stp q22, q23, [x1, #32*15] cmp w20, #64 b.gt 128f ret 128: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #32 add x1, x1, #16*32 bl 32b add x1, x1, #8*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #24*32 SETUP_SR_RECOMB 128, x7, x8, x9 cmp w19, #128 b.eq 0f SR_COMBINE_FULL 128 cmp w20, #128 b.gt 256f ret 256: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #64 add x1, x1, #32*32 bl 32b add x1, x1, #16*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #48*32 SETUP_SR_RECOMB 256, x7, x8, x9 cmp w19, #256 b.eq 0f SR_COMBINE_FULL 256 SR_COMBINE_FULL 256, 8*32 cmp w20, #256 b.gt 512f ret 512: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #128 add x1, x1, #64*32 bl 32b add x1, x1, #32*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #96*32 SETUP_SR_RECOMB 512, x7, x8, x9 cmp w19, #512 b.eq 0f mov x5, 4 5125: SR_COMBINE_FULL 512 add x1, x1, 8*32 subs w5, w5, 1 b.gt 5125b cmp w20, #512 b.gt 1024f ret 1024: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #256 add x1, x1, #96*32 bl 32b add x1, x1, #64*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER mov x5, #192*32 sub x1, x1, x5 SETUP_SR_RECOMB 1024, x7, x8, x9 cmp w19, #1024 b.eq 0f mov w5, 8 10245: SR_COMBINE_FULL 1024 add x1, x1, 8*32 subs w5, w5, 1 b.gt 10245b cmp w20, #1024 b.gt 2048f ret SR_TRANSFORM_DEF 2048, 4096 SR_TRANSFORM_DEF 4096, 8192 SR_TRANSFORM_DEF 8192, 16384 SR_TRANSFORM_DEF 16384, 32768 SR_TRANSFORM_DEF 32768, 65536 SR_TRANSFORM_DEF 65536, 131072 SR_TRANSFORM_DEF 131072 0: // general deinterleave loop SR_COMBINE_DINT add x1, x1, #32*8 subs w19, w19, #32*4 b.gt 0b ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret 2: // special case for 64 point deinterleave mov x10, v23.d[0] mov x11, v23.d[1] SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v23, v24, v26, 1 zip1 v23.2d, v0.2d, v4.2d zip2 v24.2d, v0.2d, v4.2d zip1 v25.2d, v1.2d, v20.2d zip2 v26.2d, v1.2d, v20.2d zip1 v27.2d, v8.2d, v12.2d zip2 v28.2d, v8.2d, v12.2d zip1 v29.2d, v9.2d, v21.2d zip2 v30.2d, v9.2d, v21.2d mov v20.16b, v5.16b mov v21.16b, v7.16b mov x12, x1 add x13, x1, #32* 4 add x14, x1, #32* 8 add x15, x1, #32*12 zip1 v4.2d, v2.2d, v6.2d zip2 v5.2d, v2.2d, v6.2d zip1 v6.2d, v3.2d, v20.2d zip2 v7.2d, v3.2d, v20.2d zip1 v0.2d, v16.2d, v21.2d zip2 v1.2d, v16.2d, v21.2d zip1 v2.2d, v17.2d, v13.2d zip2 v3.2d, v17.2d, v13.2d // stp is faster by a little on A53, but this is faster on M1s (theory) ldp q8, q9, [x1, #32*1] ldp q12, q13, [x1, #32*5] st1 { v23.4s, v24.4s, v25.4s, v26.4s }, [x12], #64 // 32* 0...1 st1 { v27.4s, v28.4s, v29.4s, v30.4s }, [x13], #64 // 32* 4...5 st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x14], #64 // 32* 8...9 st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x15], #64 // 32*12..13 mov v23.d[0], x10 mov v23.d[1], x11 ldp q6, q7, [x1, #32*3] ldp q16, q17, [x1, #32*7] SR_COMBINE v8, v9, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v4, v0, v1, v5, v20 zip1 v0.2d, v8.2d, v6.2d zip2 v1.2d, v8.2d, v6.2d zip1 v2.2d, v9.2d, v7.2d zip2 v3.2d, v9.2d, v7.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x12] zip1 v4.2d, v12.2d, v16.2d zip2 v5.2d, v12.2d, v16.2d zip1 v6.2d, v13.2d, v17.2d zip2 v7.2d, v13.2d, v17.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x13] zip1 v0.2d, v10.2d, v18.2d zip2 v1.2d, v10.2d, v18.2d zip1 v2.2d, v11.2d, v19.2d zip2 v3.2d, v11.2d, v19.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x14] zip1 v4.2d, v14.2d, v22.2d zip2 v5.2d, v14.2d, v22.2d zip1 v6.2d, v15.2d, v23.2d zip2 v7.2d, v15.2d, v23.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x15] ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret endfunc .endm FFT_SPLIT_RADIX_FN float, 0 FFT_SPLIT_RADIX_FN ns_float, 1