cubefs
394 строки · 10.5 Кб
1//+build !noasm2//+build !appengine3//+build !gccgo4// Copyright 2015, Klaus Post, see LICENSE for details.6// Based on http://www.snia.org/sites/default/files2/SDC2013/presentations/NewThinking/EthanMiller_Screaming_Fast_Galois_Field%20Arithmetic_SIMD%20Instructions.pdf8// and http://jerasure.org/jerasure/gf-complete/tree/master9// func galMulSSSE3Xor(low, high, in, out []byte)11TEXT ·galMulSSSE3Xor(SB), 7, $012MOVQ low+0(FP), SI // SI: &low13MOVQ high+24(FP), DX // DX: &high14MOVOU (SI), X6 // X6 low15MOVOU (DX), X7 // X7: high16MOVQ $15, BX // BX: low mask17MOVQ BX, X818PXOR X5, X519MOVQ in+48(FP), SI // R11: &in20MOVQ in_len+56(FP), R9 // R9: len(in)21MOVQ out+72(FP), DX // DX: &out22PSHUFB X5, X8 // X8: lomask (unpacked)23SHRQ $4, R9 // len(in) / 1624MOVQ SI, AX25MOVQ DX, BX26ANDQ $15, AX27ANDQ $15, BX28CMPQ R9, $029JEQ done_xor30ORQ AX, BX31CMPQ BX, $032JNZ loopback_xor33loopback_xor_aligned:35MOVOA (SI), X0 // in[x]36MOVOA (DX), X4 // out[x]37MOVOA X0, X1 // in[x]38MOVOA X6, X2 // low copy39MOVOA X7, X3 // high copy40PSRLQ $4, X1 // X1: high input41PAND X8, X0 // X0: low input42PAND X8, X1 // X0: high input43PSHUFB X0, X2 // X2: mul low part44PSHUFB X1, X3 // X3: mul high part45PXOR X2, X3 // X3: Result46PXOR X4, X3 // X3: Result xor existing out47MOVOA X3, (DX) // Store48ADDQ $16, SI // in+=1649ADDQ $16, DX // out+=1650SUBQ $1, R951JNZ loopback_xor_aligned52JMP done_xor53loopback_xor:55MOVOU (SI), X0 // in[x]56MOVOU (DX), X4 // out[x]57MOVOU X0, X1 // in[x]58MOVOU X6, X2 // low copy59MOVOU X7, X3 // high copy60PSRLQ $4, X1 // X1: high input61PAND X8, X0 // X0: low input62PAND X8, X1 // X0: high input63PSHUFB X0, X2 // X2: mul low part64PSHUFB X1, X3 // X3: mul high part65PXOR X2, X3 // X3: Result66PXOR X4, X3 // X3: Result xor existing out67MOVOU X3, (DX) // Store68ADDQ $16, SI // in+=1669ADDQ $16, DX // out+=1670SUBQ $1, R971JNZ loopback_xor72done_xor:74RET75// func galMulSSSE3(low, high, in, out []byte)77TEXT ·galMulSSSE3(SB), 7, $078MOVQ low+0(FP), SI // SI: &low79MOVQ high+24(FP), DX // DX: &high80MOVOU (SI), X6 // X6 low81MOVOU (DX), X7 // X7: high82MOVQ $15, BX // BX: low mask83MOVQ BX, X884PXOR X5, X585MOVQ in+48(FP), SI // R11: &in86MOVQ in_len+56(FP), R9 // R9: len(in)87MOVQ out+72(FP), DX // DX: &out88PSHUFB X5, X8 // X8: lomask (unpacked)89MOVQ SI, AX90MOVQ DX, BX91SHRQ $4, R9 // len(in) / 1692ANDQ $15, AX93ANDQ $15, BX94CMPQ R9, $095JEQ done96ORQ AX, BX97CMPQ BX, $098JNZ loopback99loopback_aligned:101MOVOA (SI), X0 // in[x]102MOVOA X0, X1 // in[x]103MOVOA X6, X2 // low copy104MOVOA X7, X3 // high copy105PSRLQ $4, X1 // X1: high input106PAND X8, X0 // X0: low input107PAND X8, X1 // X0: high input108PSHUFB X0, X2 // X2: mul low part109PSHUFB X1, X3 // X3: mul high part110PXOR X2, X3 // X3: Result111MOVOA X3, (DX) // Store112ADDQ $16, SI // in+=16113ADDQ $16, DX // out+=16114SUBQ $1, R9115JNZ loopback_aligned116JMP done117loopback:119MOVOU (SI), X0 // in[x]120MOVOU X0, X1 // in[x]121MOVOA X6, X2 // low copy122MOVOA X7, X3 // high copy123PSRLQ $4, X1 // X1: high input124PAND X8, X0 // X0: low input125PAND X8, X1 // X0: high input126PSHUFB X0, X2 // X2: mul low part127PSHUFB X1, X3 // X3: mul high part128PXOR X2, X3 // X3: Result129MOVOU X3, (DX) // Store130ADDQ $16, SI // in+=16131ADDQ $16, DX // out+=16132SUBQ $1, R9133JNZ loopback134done:136RET137// func galMulAVX2Xor(low, high, in, out []byte)139TEXT ·galMulAVX2Xor(SB), 7, $0140MOVQ low+0(FP), SI // SI: &low141MOVQ high+24(FP), DX // DX: &high142MOVQ $15, BX // BX: low mask143MOVQ BX, X5144MOVOU (SI), X6 // X6: low145MOVOU (DX), X7 // X7: high146MOVQ in_len+56(FP), R9 // R9: len(in)147VINSERTI128 $1, X6, Y6, Y6 // low149VINSERTI128 $1, X7, Y7, Y7 // high150VPBROADCASTB X5, Y8 // Y8: lomask (unpacked)151SHRQ $5, R9 // len(in) / 32153MOVQ out+72(FP), DX // DX: &out154MOVQ in+48(FP), SI // SI: &in155TESTQ R9, R9156JZ done_xor_avx2157loopback_xor_avx2:159VMOVDQU (SI), Y0160VMOVDQU (DX), Y4161VPSRLQ $4, Y0, Y1 // Y1: high input162VPAND Y8, Y0, Y0 // Y0: low input163VPAND Y8, Y1, Y1 // Y1: high input164VPSHUFB Y0, Y6, Y2 // Y2: mul low part165VPSHUFB Y1, Y7, Y3 // Y3: mul high part166VPXOR Y3, Y2, Y3 // Y3: Result167VPXOR Y4, Y3, Y4 // Y4: Result168VMOVDQU Y4, (DX)169ADDQ $32, SI // in+=32171ADDQ $32, DX // out+=32172SUBQ $1, R9173JNZ loopback_xor_avx2174done_xor_avx2:176VZEROUPPER177RET178// func galMulAVX2(low, high, in, out []byte)180TEXT ·galMulAVX2(SB), 7, $0181MOVQ low+0(FP), SI // SI: &low182MOVQ high+24(FP), DX // DX: &high183MOVQ $15, BX // BX: low mask184MOVQ BX, X5185MOVOU (SI), X6 // X6: low186MOVOU (DX), X7 // X7: high187MOVQ in_len+56(FP), R9 // R9: len(in)188VINSERTI128 $1, X6, Y6, Y6 // low190VINSERTI128 $1, X7, Y7, Y7 // high191VPBROADCASTB X5, Y8 // Y8: lomask (unpacked)192SHRQ $5, R9 // len(in) / 32194MOVQ out+72(FP), DX // DX: &out195MOVQ in+48(FP), SI // SI: &in196TESTQ R9, R9197JZ done_avx2198loopback_avx2:200VMOVDQU (SI), Y0201VPSRLQ $4, Y0, Y1 // Y1: high input202VPAND Y8, Y0, Y0 // Y0: low input203VPAND Y8, Y1, Y1 // Y1: high input204VPSHUFB Y0, Y6, Y2 // Y2: mul low part205VPSHUFB Y1, Y7, Y3 // Y3: mul high part206VPXOR Y3, Y2, Y4 // Y4: Result207VMOVDQU Y4, (DX)208ADDQ $32, SI // in+=32210ADDQ $32, DX // out+=32211SUBQ $1, R9212JNZ loopback_avx2213done_avx2:215VZEROUPPER216RET217// func sSE2XorSlice(in, out []byte)219TEXT ·sSE2XorSlice(SB), 7, $0220MOVQ in+0(FP), SI // SI: &in221MOVQ in_len+8(FP), R9 // R9: len(in)222MOVQ out+24(FP), DX // DX: &out223SHRQ $4, R9 // len(in) / 16224CMPQ R9, $0225JEQ done_xor_sse2226loopback_xor_sse2:228MOVOU (SI), X0 // in[x]229MOVOU (DX), X1 // out[x]230PXOR X0, X1231MOVOU X1, (DX)232ADDQ $16, SI // in+=16233ADDQ $16, DX // out+=16234SUBQ $1, R9235JNZ loopback_xor_sse2236done_xor_sse2:238RET239// func galMulAVX2Xor_64(low, high, in, out []byte)241TEXT ·galMulAVX2Xor_64(SB), 7, $0242MOVQ low+0(FP), SI // SI: &low243MOVQ high+24(FP), DX // DX: &high244MOVQ $15, BX // BX: low mask245MOVQ BX, X5246MOVQ in_len+56(FP), R9 // R9: len(in)247VBROADCASTI128 (SI), Y6 // low table249VBROADCASTI128 (DX), Y7 // high high table250VPBROADCASTB X5, Y8 // Y8: lomask (unpacked)251SHRQ $6, R9 // len(in) / 64253MOVQ out+72(FP), DX // DX: &out254MOVQ in+48(FP), SI // SI: &in255TESTQ R9, R9256JZ done_xor_avx2_64257loopback_xor_avx2_64:259VMOVDQU (SI), Y0260VMOVDQU 32(SI), Y10261VMOVDQU (DX), Y4262VMOVDQU 32(DX), Y14263VPSRLQ $4, Y0, Y1 // Y1: high input264VPSRLQ $4, Y10, Y11 // Y11: high input 2265VPAND Y8, Y0, Y0 // Y0: low input266VPAND Y8, Y10, Y10 // Y10: low input 2267VPAND Y8, Y1, Y1 // Y11: high input268VPAND Y8, Y11, Y11 // Y11: high input 2269VPSHUFB Y0, Y6, Y2 // Y2: mul low part270VPSHUFB Y10, Y6, Y12 // Y12: mul low part 2271VPSHUFB Y1, Y7, Y3 // Y3: mul high part272VPSHUFB Y11, Y7, Y13 // Y13: mul high part 2273VPXOR Y3, Y2, Y3 // Y3: Result274VPXOR Y13, Y12, Y13 // Y13: Result 2275VPXOR Y4, Y3, Y4 // Y4: Result276VPXOR Y14, Y13, Y14 // Y4: Result 2277VMOVDQU Y4, (DX)278VMOVDQU Y14, 32(DX)279ADDQ $64, SI // in+=64281ADDQ $64, DX // out+=64282SUBQ $1, R9283JNZ loopback_xor_avx2_64284done_xor_avx2_64:286VZEROUPPER287RET288// func galMulAVX2_64(low, high, in, out []byte)290TEXT ·galMulAVX2_64(SB), 7, $0291MOVQ low+0(FP), SI // SI: &low292MOVQ high+24(FP), DX // DX: &high293MOVQ $15, BX // BX: low mask294MOVQ BX, X5295MOVQ in_len+56(FP), R9 // R9: len(in)296VBROADCASTI128 (SI), Y6 // low table297VBROADCASTI128 (DX), Y7 // high high table298VPBROADCASTB X5, Y8 // Y8: lomask (unpacked)299SHRQ $6, R9 // len(in) / 64301MOVQ out+72(FP), DX // DX: &out302MOVQ in+48(FP), SI // SI: &in303TESTQ R9, R9304JZ done_avx2_64305loopback_avx2_64:307VMOVDQU (SI), Y0308VMOVDQU 32(SI), Y10309VPSRLQ $4, Y0, Y1 // Y1: high input310VPSRLQ $4, Y10, Y11 // Y11: high input 2311VPAND Y8, Y0, Y0 // Y0: low input312VPAND Y8, Y10, Y10 // Y10: low input313VPAND Y8, Y1, Y1 // Y1: high input314VPAND Y8, Y11, Y11 // Y11: high input 2315VPSHUFB Y0, Y6, Y2 // Y2: mul low part316VPSHUFB Y10, Y6, Y12 // Y12: mul low part 2317VPSHUFB Y1, Y7, Y3 // Y3: mul high part318VPSHUFB Y11, Y7, Y13 // Y13: mul high part 2319VPXOR Y3, Y2, Y4 // Y4: Result320VPXOR Y13, Y12, Y14 // Y14: Result 2321VMOVDQU Y4, (DX)322VMOVDQU Y14, 32(DX)323ADDQ $64, SI // in+=64325ADDQ $64, DX // out+=64326SUBQ $1, R9327JNZ loopback_avx2_64328done_avx2_64:330VZEROUPPER331RET332// func sSE2XorSlice_64(in, out []byte)334TEXT ·sSE2XorSlice_64(SB), 7, $0335MOVQ in+0(FP), SI // SI: &in336MOVQ in_len+8(FP), R9 // R9: len(in)337MOVQ out+24(FP), DX // DX: &out338SHRQ $6, R9 // len(in) / 64339CMPQ R9, $0340JEQ done_xor_sse2_64341loopback_xor_sse2_64:343MOVOU (SI), X0 // in[x]344MOVOU 16(SI), X2 // in[x]345MOVOU 32(SI), X4 // in[x]346MOVOU 48(SI), X6 // in[x]347MOVOU (DX), X1 // out[x]348MOVOU 16(DX), X3 // out[x]349MOVOU 32(DX), X5 // out[x]350MOVOU 48(DX), X7 // out[x]351PXOR X0, X1352PXOR X2, X3353PXOR X4, X5354PXOR X6, X7355MOVOU X1, (DX)356MOVOU X3, 16(DX)357MOVOU X5, 32(DX)358MOVOU X7, 48(DX)359ADDQ $64, SI // in+=64360ADDQ $64, DX // out+=64361SUBQ $1, R9362JNZ loopback_xor_sse2_64363done_xor_sse2_64:365RET366// func avx2XorSlice_64(in, out []byte)368TEXT ·avx2XorSlice_64(SB), 7, $0369MOVQ in+0(FP), SI // SI: &in370MOVQ in_len+8(FP), R9 // R9: len(in)371MOVQ out+24(FP), DX // DX: &out372SHRQ $6, R9 // len(in) / 64373CMPQ R9, $0374JEQ done_xor_avx2_64375loopback_xor_avx2_64:377VMOVDQU (SI), Y0378VMOVDQU 32(SI), Y2379VMOVDQU (DX), Y1380VMOVDQU 32(DX), Y3381VPXOR Y0, Y1, Y1382VPXOR Y2, Y3, Y3383VMOVDQU Y1, (DX)384VMOVDQU Y3, 32(DX)385ADDQ $64, SI // in+=64387ADDQ $64, DX // out+=64388SUBQ $1, R9389JNZ loopback_xor_avx2_64390VZEROUPPER391done_xor_avx2_64:393RET395