jdk

1
/*
2
 * Copyright (c) 2016, Linaro Ltd. All rights reserved.
3
 * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
4
 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
5
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6
 *
7
 * This code is free software; you can redistribute it and/or modify it
8
 * under the terms of the GNU General Public License version 2 only, as
9
 * published by the Free Software Foundation.
10
 *
11
 * This code is distributed in the hope that it will be useful, but WITHOUT
12
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14
 * version 2 for more details (a copy is included in the LICENSE file that
15
 * accompanied this code).
16
 *
17
 * You should have received a copy of the GNU General Public License version
18
 * 2 along with this work; if not, write to the Free Software Foundation,
19
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20
 *
21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22
 * or visit www.oracle.com if you need additional information or have any
23
 * questions.
24
 *
25
 */
26

27
#include "defs.S.inc"
28

29
s       .req    x0
30
d       .req    x1
31
count   .req    x2
32
t0      .req    x3
33
t1      .req    x4
34
t2      .req    x5
35
t3      .req    x6
36
t4      .req    x7
37
t5      .req    x8
38
t6      .req    x9
39
t7      .req    x10
40

41
        .align  6
42
DECLARE_FUNC(_Copy_disjoint_words):
43
        // Ensure 2 word aligned
44
        tbz     s, #3, fwd_copy_aligned
45
        ldr     t0, [s], #8
46
        str     t0, [d], #8
47
        sub     count, count, #1
48

49
fwd_copy_aligned:
50
        // Bias s & d so we only pre index on the last copy
51
        sub     s, s, #16
52
        sub     d, d, #16
53

54
        ldp     t0, t1, [s, #16]
55
        ldp     t2, t3, [s, #32]
56
        ldp     t4, t5, [s, #48]
57
        ldp     t6, t7, [s, #64]!
58

59
        subs    count, count, #16
60
        blo     fwd_copy_drain
61

62
fwd_copy_again:
63
        prfm    pldl1keep, [s, #256]
64
        stp     t0, t1, [d, #16]
65
        ldp     t0, t1, [s, #16]
66
        stp     t2, t3, [d, #32]
67
        ldp     t2, t3, [s, #32]
68
        stp     t4, t5, [d, #48]
69
        ldp     t4, t5, [s, #48]
70
        stp     t6, t7, [d, #64]!
71
        ldp     t6, t7, [s, #64]!
72
        subs    count, count, #8
73
        bhs     fwd_copy_again
74

75
fwd_copy_drain:
76
        stp     t0, t1, [d, #16]
77
        stp     t2, t3, [d, #32]
78
        stp     t4, t5, [d, #48]
79
        stp     t6, t7, [d, #64]!
80

81
        // count is now -8..-1 for 0..7 words to copy
82
        adr     t0, 0f
83
        add     t0, t0, count, lsl #5
84
        br      t0
85

86
        .align  5
87
        ret                             // -8 == 0 words
88
        .align  5
89
        ldr     t0, [s, #16]            // -7 == 1 word
90
        str     t0, [d, #16]
91
        ret
92
        .align  5
93
        ldp     t0, t1, [s, #16]        // -6 = 2 words
94
        stp     t0, t1, [d, #16]
95
        ret
96
        .align  5
97
        ldp     t0, t1, [s, #16]        // -5 = 3 words
98
        ldr     t2, [s, #32]
99
        stp     t0, t1, [d, #16]
100
        str     t2, [d, #32]
101
        ret
102
        .align  5
103
        ldp     t0, t1, [s, #16]        // -4 = 4 words
104
        ldp     t2, t3, [s, #32]
105
        stp     t0, t1, [d, #16]
106
        stp     t2, t3, [d, #32]
107
        ret
108
        .align  5
109
        ldp     t0, t1, [s, #16]        // -3 = 5 words
110
        ldp     t2, t3, [s, #32]
111
        ldr     t4, [s, #48]
112
        stp     t0, t1, [d, #16]
113
        stp     t2, t3, [d, #32]
114
        str     t4, [d, #48]
115
        ret
116
        .align  5
117
        ldp     t0, t1, [s, #16]        // -2 = 6 words
118
        ldp     t2, t3, [s, #32]
119
        ldp     t4, t5, [s, #48]
120
        stp     t0, t1, [d, #16]
121
        stp     t2, t3, [d, #32]
122
        stp     t4, t5, [d, #48]
123
        ret
124
        .align  5
125
        ldp     t0, t1, [s, #16]        // -1 = 7 words
126
        ldp     t2, t3, [s, #32]
127
        ldp     t4, t5, [s, #48]
128
        ldr     t6, [s, #64]
129
        stp     t0, t1, [d, #16]
130
        stp     t2, t3, [d, #32]
131
        stp     t4, t5, [d, #48]
132
        str     t6, [d, #64]
133
        // Is always aligned here, code for 7 words is one instruction
134
        // too large so it just falls through.
135
        .align  5
136
0:
137
        ret
138

139
        .align  6
140
DECLARE_FUNC(_Copy_conjoint_words):
141
        sub     t0, d, s
142
        cmp     t0, count, lsl #3
143
        bhs     SYMBOL(_Copy_disjoint_words)
144

145
        add     s, s, count, lsl #3
146
        add     d, d, count, lsl #3
147

148
        // Ensure 2 word aligned
149
        tbz     s, #3, bwd_copy_aligned
150
        ldr     t0, [s, #-8]!
151
        str     t0, [d, #-8]!
152
        sub     count, count, #1
153

154
bwd_copy_aligned:
155
        ldp     t0, t1, [s, #-16]
156
        ldp     t2, t3, [s, #-32]
157
        ldp     t4, t5, [s, #-48]
158
        ldp     t6, t7, [s, #-64]!
159

160
        subs    count, count, #16
161
        blo     bwd_copy_drain
162

163
bwd_copy_again:
164
        prfum   pldl1keep, [s, #-256]
165
        stp     t0, t1, [d, #-16]
166
        ldp     t0, t1, [s, #-16]
167
        stp     t2, t3, [d, #-32]
168
        ldp     t2, t3, [s, #-32]
169
        stp     t4, t5, [d, #-48]
170
        ldp     t4, t5, [s, #-48]
171
        stp     t6, t7, [d, #-64]!
172
        ldp     t6, t7, [s, #-64]!
173
        subs    count, count, #8
174
        bhs     bwd_copy_again
175

176
bwd_copy_drain:
177
        stp     t0, t1, [d, #-16]
178
        stp     t2, t3, [d, #-32]
179
        stp     t4, t5, [d, #-48]
180
        stp     t6, t7, [d, #-64]!
181

182
        // count is now -8..-1 for 0..7 words to copy
183
        adr     t0, 0f
184
        add     t0, t0, count, lsl #5
185
        br      t0
186

187
        .align  5
188
        ret                             // -8 == 0 words
189
        .align  5
190
        ldr     t0, [s, #-8]            // -7 == 1 word
191
        str     t0, [d, #-8]
192
        ret
193
        .align  5
194
        ldp     t0, t1, [s, #-16]       // -6 = 2 words
195
        stp     t0, t1, [d, #-16]
196
        ret
197
        .align  5
198
        ldp     t0, t1, [s, #-16]       // -5 = 3 words
199
        ldr     t2, [s, #-24]
200
        stp     t0, t1, [d, #-16]
201
        str     t2, [d, #-24]
202
        ret
203
        .align  5
204
        ldp     t0, t1, [s, #-16]       // -4 = 4 words
205
        ldp     t2, t3, [s, #-32]
206
        stp     t0, t1, [d, #-16]
207
        stp     t2, t3, [d, #-32]
208
        ret
209
        .align  5
210
        ldp     t0, t1, [s, #-16]       // -3 = 5 words
211
        ldp     t2, t3, [s, #-32]
212
        ldr     t4, [s, #-40]
213
        stp     t0, t1, [d, #-16]
214
        stp     t2, t3, [d, #-32]
215
        str     t4, [d, #-40]
216
        ret
217
        .align  5
218
        ldp     t0, t1, [s, #-16]       // -2 = 6 words
219
        ldp     t2, t3, [s, #-32]
220
        ldp     t4, t5, [s, #-48]
221
        stp     t0, t1, [d, #-16]
222
        stp     t2, t3, [d, #-32]
223
        stp     t4, t5, [d, #-48]
224
        ret
225
        .align  5
226
        ldp     t0, t1, [s, #-16]       // -1 = 7 words
227
        ldp     t2, t3, [s, #-32]
228
        ldp     t4, t5, [s, #-48]
229
        ldr     t6, [s, #-56]
230
        stp     t0, t1, [d, #-16]
231
        stp     t2, t3, [d, #-32]
232
        stp     t4, t5, [d, #-48]
233
        str     t6, [d, #-56]
234
        // Is always aligned here, code for 7 words is one instruction
235
        // too large so it just falls through.
236
        .align  5
237
0:
238
        ret
239