Text file
src/runtime/asm_arm64.s
1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "tls_arm64.h"
8 #include "funcdata.h"
9 #include "textflag.h"
10
11 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
12 // SP = stack; R0 = argc; R1 = argv
13
14 SUB $32, RSP
15 MOVW R0, 8(RSP) // argc
16 MOVD R1, 16(RSP) // argv
17
18 #ifdef TLS_darwin
19 // Initialize TLS.
20 MOVD ZR, g // clear g, make sure it's not junk.
21 SUB $32, RSP
22 MRS_TPIDR_R0
23 AND $~7, R0
24 MOVD R0, 16(RSP) // arg2: TLS base
25 MOVD $runtime·tls_g(SB), R2
26 MOVD R2, 8(RSP) // arg1: &tlsg
27 BL ·tlsinit(SB)
28 ADD $32, RSP
29 #endif
30
31 // create istack out of the given (operating system) stack.
32 // _cgo_init may update stackguard.
33 MOVD $runtime·g0(SB), g
34 MOVD RSP, R7
35 MOVD $(-64*1024)(R7), R0
36 MOVD R0, g_stackguard0(g)
37 MOVD R0, g_stackguard1(g)
38 MOVD R0, (g_stack+stack_lo)(g)
39 MOVD R7, (g_stack+stack_hi)(g)
40
41 // if there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CBZ R12, nocgo
44
45 #ifdef GOOS_android
46 MRS_TPIDR_R0 // load TLS base pointer
47 MOVD R0, R3 // arg 3: TLS base pointer
48 MOVD $runtime·tls_g(SB), R2 // arg 2: &tls_g
49 #else
50 MOVD $0, R2 // arg 2: not used when using platform's TLS
51 #endif
52 MOVD $setg_gcc<>(SB), R1 // arg 1: setg
53 MOVD g, R0 // arg 0: G
54 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved.
55 BL (R12)
56 ADD $16, RSP
57
58 nocgo:
59 BL runtime·save_g(SB)
60 // update stackguard after _cgo_init
61 MOVD (g_stack+stack_lo)(g), R0
62 ADD $const__StackGuard, R0
63 MOVD R0, g_stackguard0(g)
64 MOVD R0, g_stackguard1(g)
65
66 // set the per-goroutine and per-mach "registers"
67 MOVD $runtime·m0(SB), R0
68
69 // save m->g0 = g0
70 MOVD g, m_g0(R0)
71 // save m0 to g0->m
72 MOVD R0, g_m(g)
73
74 BL runtime·check(SB)
75
76 #ifdef GOOS_windows
77 BL runtime·wintls(SB)
78 #endif
79
80 MOVW 8(RSP), R0 // copy argc
81 MOVW R0, -8(RSP)
82 MOVD 16(RSP), R0 // copy argv
83 MOVD R0, 0(RSP)
84 BL runtime·args(SB)
85 BL runtime·osinit(SB)
86 BL runtime·schedinit(SB)
87
88 // create a new goroutine to start program
89 MOVD $runtime·mainPC(SB), R0 // entry
90 SUB $16, RSP
91 MOVD R0, 8(RSP) // arg
92 MOVD $0, 0(RSP) // dummy LR
93 BL runtime·newproc(SB)
94 ADD $16, RSP
95
96 // start this M
97 BL runtime·mstart(SB)
98
99 // Prevent dead-code elimination of debugCallV2, which is
100 // intended to be called by debuggers.
101 MOVD $runtime·debugCallV2<ABIInternal>(SB), R0
102
103 MOVD $0, R0
104 MOVD R0, (R0) // boom
105 UNDEF
106
107 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
108 GLOBL runtime·mainPC(SB),RODATA,$8
109
110 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
111 BRK
112 RET
113
114 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
115 RET
116
117 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
118 BL runtime·mstart0(SB)
119 RET // not reached
120
121 /*
122 * go-routine
123 */
124
125 // void gogo(Gobuf*)
126 // restore state from Gobuf; longjmp
127 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
128 MOVD buf+0(FP), R5
129 MOVD gobuf_g(R5), R6
130 MOVD 0(R6), R4 // make sure g != nil
131 B gogo<>(SB)
132
133 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
134 MOVD R6, g
135 BL runtime·save_g(SB)
136
137 MOVD gobuf_sp(R5), R0
138 MOVD R0, RSP
139 MOVD gobuf_bp(R5), R29
140 MOVD gobuf_lr(R5), LR
141 MOVD gobuf_ret(R5), R0
142 MOVD gobuf_ctxt(R5), R26
143 MOVD $0, gobuf_sp(R5)
144 MOVD $0, gobuf_bp(R5)
145 MOVD $0, gobuf_ret(R5)
146 MOVD $0, gobuf_lr(R5)
147 MOVD $0, gobuf_ctxt(R5)
148 CMP ZR, ZR // set condition codes for == test, needed by stack split
149 MOVD gobuf_pc(R5), R6
150 B (R6)
151
152 // void mcall(fn func(*g))
153 // Switch to m->g0's stack, call fn(g).
154 // Fn must never return. It should gogo(&g->sched)
155 // to keep running g.
156 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
157 #ifdef GOEXPERIMENT_regabiargs
158 MOVD R0, R26 // context
159 #else
160 MOVD fn+0(FP), R26 // context
161 #endif
162
163 // Save caller state in g->sched
164 MOVD RSP, R0
165 MOVD R0, (g_sched+gobuf_sp)(g)
166 MOVD R29, (g_sched+gobuf_bp)(g)
167 MOVD LR, (g_sched+gobuf_pc)(g)
168 MOVD $0, (g_sched+gobuf_lr)(g)
169
170 // Switch to m->g0 & its stack, call fn.
171 MOVD g, R3
172 MOVD g_m(g), R8
173 MOVD m_g0(R8), g
174 BL runtime·save_g(SB)
175 CMP g, R3
176 BNE 2(PC)
177 B runtime·badmcall(SB)
178
179 MOVD (g_sched+gobuf_sp)(g), R0
180 MOVD R0, RSP // sp = m->g0->sched.sp
181 MOVD (g_sched+gobuf_bp)(g), R29
182 #ifdef GOEXPERIMENT_regabiargs
183 MOVD R3, R0 // arg = g
184 #else
185 MOVD R3, -8(RSP) // arg = g
186 #endif
187 MOVD $0, -16(RSP) // dummy LR
188 SUB $16, RSP
189 MOVD 0(R26), R4 // code pointer
190 BL (R4)
191 B runtime·badmcall2(SB)
192
193 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
194 // of the G stack. We need to distinguish the routine that
195 // lives at the bottom of the G stack from the one that lives
196 // at the top of the system stack because the one at the top of
197 // the system stack terminates the stack walk (see topofstack()).
198 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
199 UNDEF
200 BL (LR) // make sure this function is not leaf
201 RET
202
203 // func systemstack(fn func())
204 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
205 MOVD fn+0(FP), R3 // R3 = fn
206 MOVD R3, R26 // context
207 MOVD g_m(g), R4 // R4 = m
208
209 MOVD m_gsignal(R4), R5 // R5 = gsignal
210 CMP g, R5
211 BEQ noswitch
212
213 MOVD m_g0(R4), R5 // R5 = g0
214 CMP g, R5
215 BEQ noswitch
216
217 MOVD m_curg(R4), R6
218 CMP g, R6
219 BEQ switch
220
221 // Bad: g is not gsignal, not g0, not curg. What is it?
222 // Hide call from linker nosplit analysis.
223 MOVD $runtime·badsystemstack(SB), R3
224 BL (R3)
225 B runtime·abort(SB)
226
227 switch:
228 // save our state in g->sched. Pretend to
229 // be systemstack_switch if the G stack is scanned.
230 BL gosave_systemstack_switch<>(SB)
231
232 // switch to g0
233 MOVD R5, g
234 BL runtime·save_g(SB)
235 MOVD (g_sched+gobuf_sp)(g), R3
236 MOVD R3, RSP
237 MOVD (g_sched+gobuf_bp)(g), R29
238
239 // call target function
240 MOVD 0(R26), R3 // code pointer
241 BL (R3)
242
243 // switch back to g
244 MOVD g_m(g), R3
245 MOVD m_curg(R3), g
246 BL runtime·save_g(SB)
247 MOVD (g_sched+gobuf_sp)(g), R0
248 MOVD R0, RSP
249 MOVD (g_sched+gobuf_bp)(g), R29
250 MOVD $0, (g_sched+gobuf_sp)(g)
251 MOVD $0, (g_sched+gobuf_bp)(g)
252 RET
253
254 noswitch:
255 // already on m stack, just call directly
256 // Using a tail call here cleans up tracebacks since we won't stop
257 // at an intermediate systemstack.
258 MOVD 0(R26), R3 // code pointer
259 MOVD.P 16(RSP), R30 // restore LR
260 SUB $8, RSP, R29 // restore FP
261 B (R3)
262
263 /*
264 * support for morestack
265 */
266
267 // Called during function prolog when more stack is needed.
268 // Caller has already loaded:
269 // R3 prolog's LR (R30)
270 //
271 // The traceback routines see morestack on a g0 as being
272 // the top of a stack (for example, morestack calling newstack
273 // calling the scheduler calling newm calling gc), so we must
274 // record an argument size. For that purpose, it has no arguments.
275 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
276 // Cannot grow scheduler stack (m->g0).
277 MOVD g_m(g), R8
278 MOVD m_g0(R8), R4
279 CMP g, R4
280 BNE 3(PC)
281 BL runtime·badmorestackg0(SB)
282 B runtime·abort(SB)
283
284 // Cannot grow signal stack (m->gsignal).
285 MOVD m_gsignal(R8), R4
286 CMP g, R4
287 BNE 3(PC)
288 BL runtime·badmorestackgsignal(SB)
289 B runtime·abort(SB)
290
291 // Called from f.
292 // Set g->sched to context in f
293 MOVD RSP, R0
294 MOVD R0, (g_sched+gobuf_sp)(g)
295 MOVD R29, (g_sched+gobuf_bp)(g)
296 MOVD LR, (g_sched+gobuf_pc)(g)
297 MOVD R3, (g_sched+gobuf_lr)(g)
298 MOVD R26, (g_sched+gobuf_ctxt)(g)
299
300 // Called from f.
301 // Set m->morebuf to f's callers.
302 MOVD R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
303 MOVD RSP, R0
304 MOVD R0, (m_morebuf+gobuf_sp)(R8) // f's caller's RSP
305 MOVD g, (m_morebuf+gobuf_g)(R8)
306
307 // Call newstack on m->g0's stack.
308 MOVD m_g0(R8), g
309 BL runtime·save_g(SB)
310 MOVD (g_sched+gobuf_sp)(g), R0
311 MOVD R0, RSP
312 MOVD (g_sched+gobuf_bp)(g), R29
313 MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned)
314 BL runtime·newstack(SB)
315
316 // Not reached, but make sure the return PC from the call to newstack
317 // is still in this function, and not the beginning of the next.
318 UNDEF
319
320 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
321 MOVW $0, R26
322 B runtime·morestack(SB)
323
324 #ifdef GOEXPERIMENT_regabireflect
325 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
326 TEXT ·spillArgs(SB),NOSPLIT,$0-0
327 MOVD R0, (0*8)(R20)
328 MOVD R1, (1*8)(R20)
329 MOVD R2, (2*8)(R20)
330 MOVD R3, (3*8)(R20)
331 MOVD R4, (4*8)(R20)
332 MOVD R5, (5*8)(R20)
333 MOVD R6, (6*8)(R20)
334 MOVD R7, (7*8)(R20)
335 MOVD R8, (8*8)(R20)
336 MOVD R9, (9*8)(R20)
337 MOVD R10, (10*8)(R20)
338 MOVD R11, (11*8)(R20)
339 MOVD R12, (12*8)(R20)
340 MOVD R13, (13*8)(R20)
341 MOVD R14, (14*8)(R20)
342 MOVD R15, (15*8)(R20)
343 FMOVD F0, (16*8)(R20)
344 FMOVD F1, (17*8)(R20)
345 FMOVD F2, (18*8)(R20)
346 FMOVD F3, (19*8)(R20)
347 FMOVD F4, (20*8)(R20)
348 FMOVD F5, (21*8)(R20)
349 FMOVD F6, (22*8)(R20)
350 FMOVD F7, (23*8)(R20)
351 FMOVD F8, (24*8)(R20)
352 FMOVD F9, (25*8)(R20)
353 FMOVD F10, (26*8)(R20)
354 FMOVD F11, (27*8)(R20)
355 FMOVD F12, (28*8)(R20)
356 FMOVD F13, (29*8)(R20)
357 FMOVD F14, (30*8)(R20)
358 FMOVD F15, (31*8)(R20)
359 RET
360
361 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
362 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
363 MOVD (0*8)(R20), R0
364 MOVD (1*8)(R20), R1
365 MOVD (2*8)(R20), R2
366 MOVD (3*8)(R20), R3
367 MOVD (4*8)(R20), R4
368 MOVD (5*8)(R20), R5
369 MOVD (6*8)(R20), R6
370 MOVD (7*8)(R20), R7
371 MOVD (8*8)(R20), R8
372 MOVD (9*8)(R20), R9
373 MOVD (10*8)(R20), R10
374 MOVD (11*8)(R20), R11
375 MOVD (12*8)(R20), R12
376 MOVD (13*8)(R20), R13
377 MOVD (14*8)(R20), R14
378 MOVD (15*8)(R20), R15
379 FMOVD (16*8)(R20), F0
380 FMOVD (17*8)(R20), F1
381 FMOVD (18*8)(R20), F2
382 FMOVD (19*8)(R20), F3
383 FMOVD (20*8)(R20), F4
384 FMOVD (21*8)(R20), F5
385 FMOVD (22*8)(R20), F6
386 FMOVD (23*8)(R20), F7
387 FMOVD (24*8)(R20), F8
388 FMOVD (25*8)(R20), F9
389 FMOVD (26*8)(R20), F10
390 FMOVD (27*8)(R20), F11
391 FMOVD (28*8)(R20), F12
392 FMOVD (29*8)(R20), F13
393 FMOVD (30*8)(R20), F14
394 FMOVD (31*8)(R20), F15
395 RET
396 #else
397 TEXT ·spillArgs(SB),NOSPLIT,$0-0
398 RET
399
400 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
401 RET
402 #endif
403
404 // reflectcall: call a function with the given argument list
405 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
406 // we don't have variable-sized frames, so we use a small number
407 // of constant-sized-frame functions to encode a few bits of size in the pc.
408 // Caution: ugly multiline assembly macros in your future!
409
410 #define DISPATCH(NAME,MAXSIZE) \
411 MOVD $MAXSIZE, R27; \
412 CMP R27, R16; \
413 BGT 3(PC); \
414 MOVD $NAME(SB), R27; \
415 B (R27)
416 // Note: can't just "B NAME(SB)" - bad inlining results.
417
418 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
419 MOVWU frameSize+32(FP), R16
420 DISPATCH(runtime·call16, 16)
421 DISPATCH(runtime·call32, 32)
422 DISPATCH(runtime·call64, 64)
423 DISPATCH(runtime·call128, 128)
424 DISPATCH(runtime·call256, 256)
425 DISPATCH(runtime·call512, 512)
426 DISPATCH(runtime·call1024, 1024)
427 DISPATCH(runtime·call2048, 2048)
428 DISPATCH(runtime·call4096, 4096)
429 DISPATCH(runtime·call8192, 8192)
430 DISPATCH(runtime·call16384, 16384)
431 DISPATCH(runtime·call32768, 32768)
432 DISPATCH(runtime·call65536, 65536)
433 DISPATCH(runtime·call131072, 131072)
434 DISPATCH(runtime·call262144, 262144)
435 DISPATCH(runtime·call524288, 524288)
436 DISPATCH(runtime·call1048576, 1048576)
437 DISPATCH(runtime·call2097152, 2097152)
438 DISPATCH(runtime·call4194304, 4194304)
439 DISPATCH(runtime·call8388608, 8388608)
440 DISPATCH(runtime·call16777216, 16777216)
441 DISPATCH(runtime·call33554432, 33554432)
442 DISPATCH(runtime·call67108864, 67108864)
443 DISPATCH(runtime·call134217728, 134217728)
444 DISPATCH(runtime·call268435456, 268435456)
445 DISPATCH(runtime·call536870912, 536870912)
446 DISPATCH(runtime·call1073741824, 1073741824)
447 MOVD $runtime·badreflectcall(SB), R0
448 B (R0)
449
450 #define CALLFN(NAME,MAXSIZE) \
451 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
452 NO_LOCAL_POINTERS; \
453 /* copy arguments to stack */ \
454 MOVD stackArgs+16(FP), R3; \
455 MOVWU stackArgsSize+24(FP), R4; \
456 ADD $8, RSP, R5; \
457 BIC $0xf, R4, R6; \
458 CBZ R6, 6(PC); \
459 /* if R6=(argsize&~15) != 0 */ \
460 ADD R6, R5, R6; \
461 /* copy 16 bytes a time */ \
462 LDP.P 16(R3), (R7, R8); \
463 STP.P (R7, R8), 16(R5); \
464 CMP R5, R6; \
465 BNE -3(PC); \
466 AND $0xf, R4, R6; \
467 CBZ R6, 6(PC); \
468 /* if R6=(argsize&15) != 0 */ \
469 ADD R6, R5, R6; \
470 /* copy 1 byte a time for the rest */ \
471 MOVBU.P 1(R3), R7; \
472 MOVBU.P R7, 1(R5); \
473 CMP R5, R6; \
474 BNE -3(PC); \
475 /* set up argument registers */ \
476 MOVD regArgs+40(FP), R20; \
477 CALL ·unspillArgs(SB); \
478 /* call function */ \
479 MOVD f+8(FP), R26; \
480 MOVD (R26), R20; \
481 PCDATA $PCDATA_StackMapIndex, $0; \
482 BL (R20); \
483 /* copy return values back */ \
484 MOVD regArgs+40(FP), R20; \
485 CALL ·spillArgs(SB); \
486 MOVD stackArgsType+0(FP), R7; \
487 MOVD stackArgs+16(FP), R3; \
488 MOVWU stackArgsSize+24(FP), R4; \
489 MOVWU stackRetOffset+28(FP), R6; \
490 ADD $8, RSP, R5; \
491 ADD R6, R5; \
492 ADD R6, R3; \
493 SUB R6, R4; \
494 BL callRet<>(SB); \
495 RET
496
497 // callRet copies return values back at the end of call*. This is a
498 // separate function so it can allocate stack space for the arguments
499 // to reflectcallmove. It does not follow the Go ABI; it expects its
500 // arguments in registers.
501 TEXT callRet<>(SB), NOSPLIT, $48-0
502 NO_LOCAL_POINTERS
503 MOVD R7, 8(RSP)
504 MOVD R3, 16(RSP)
505 MOVD R5, 24(RSP)
506 MOVD R4, 32(RSP)
507 MOVD R20, 40(RSP)
508 BL runtime·reflectcallmove(SB)
509 RET
510
511 CALLFN(·call16, 16)
512 CALLFN(·call32, 32)
513 CALLFN(·call64, 64)
514 CALLFN(·call128, 128)
515 CALLFN(·call256, 256)
516 CALLFN(·call512, 512)
517 CALLFN(·call1024, 1024)
518 CALLFN(·call2048, 2048)
519 CALLFN(·call4096, 4096)
520 CALLFN(·call8192, 8192)
521 CALLFN(·call16384, 16384)
522 CALLFN(·call32768, 32768)
523 CALLFN(·call65536, 65536)
524 CALLFN(·call131072, 131072)
525 CALLFN(·call262144, 262144)
526 CALLFN(·call524288, 524288)
527 CALLFN(·call1048576, 1048576)
528 CALLFN(·call2097152, 2097152)
529 CALLFN(·call4194304, 4194304)
530 CALLFN(·call8388608, 8388608)
531 CALLFN(·call16777216, 16777216)
532 CALLFN(·call33554432, 33554432)
533 CALLFN(·call67108864, 67108864)
534 CALLFN(·call134217728, 134217728)
535 CALLFN(·call268435456, 268435456)
536 CALLFN(·call536870912, 536870912)
537 CALLFN(·call1073741824, 1073741824)
538
539 // func memhash32(p unsafe.Pointer, h uintptr) uintptr
540 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
541 MOVB runtime·useAeshash(SB), R10
542 CBZ R10, noaes
543 #ifndef GOEXPERIMENT_regabiargs
544 MOVD p+0(FP), R0
545 MOVD h+8(FP), R1
546 MOVD $ret+16(FP), R2
547 #endif
548 MOVD $runtime·aeskeysched+0(SB), R3
549
550 VEOR V0.B16, V0.B16, V0.B16
551 VLD1 (R3), [V2.B16]
552 VLD1 (R0), V0.S[1]
553 VMOV R1, V0.S[0]
554
555 AESE V2.B16, V0.B16
556 AESMC V0.B16, V0.B16
557 AESE V2.B16, V0.B16
558 AESMC V0.B16, V0.B16
559 AESE V2.B16, V0.B16
560
561 #ifdef GOEXPERIMENT_regabiargs
562 VMOV V0.D[0], R0
563 #else
564 VST1 [V0.D1], (R2)
565 #endif
566 RET
567 noaes:
568 B runtime·memhash32Fallback<ABIInternal>(SB)
569
570 // func memhash64(p unsafe.Pointer, h uintptr) uintptr
571 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
572 MOVB runtime·useAeshash(SB), R10
573 CBZ R10, noaes
574 #ifndef GOEXPERIMENT_regabiargs
575 MOVD p+0(FP), R0
576 MOVD h+8(FP), R1
577 MOVD $ret+16(FP), R2
578 #endif
579 MOVD $runtime·aeskeysched+0(SB), R3
580
581 VEOR V0.B16, V0.B16, V0.B16
582 VLD1 (R3), [V2.B16]
583 VLD1 (R0), V0.D[1]
584 VMOV R1, V0.D[0]
585
586 AESE V2.B16, V0.B16
587 AESMC V0.B16, V0.B16
588 AESE V2.B16, V0.B16
589 AESMC V0.B16, V0.B16
590 AESE V2.B16, V0.B16
591
592 #ifdef GOEXPERIMENT_regabiargs
593 VMOV V0.D[0], R0
594 #else
595 VST1 [V0.D1], (R2)
596 #endif
597 RET
598 noaes:
599 B runtime·memhash64Fallback<ABIInternal>(SB)
600
601 // func memhash(p unsafe.Pointer, h, size uintptr) uintptr
602 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
603 MOVB runtime·useAeshash(SB), R10
604 CBZ R10, noaes
605 #ifndef GOEXPERIMENT_regabiargs
606 MOVD p+0(FP), R0
607 MOVD h+8(FP), R1
608 MOVD s+16(FP), R2
609 MOVD $ret+24(FP), R8
610 #endif
611 B aeshashbody<>(SB)
612 noaes:
613 B runtime·memhashFallback<ABIInternal>(SB)
614
615 // func strhash(p unsafe.Pointer, h uintptr) uintptr
616 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
617 MOVB runtime·useAeshash(SB), R10
618 CBZ R10, noaes
619 #ifdef GOEXPERIMENT_regabiargs
620 LDP (R0), (R0, R2) // string data / length
621 #else
622 MOVD p+0(FP), R10 // string pointer
623 LDP (R10), (R0, R2) // string data / length
624 MOVD h+8(FP), R1
625 MOVD $ret+16(FP), R8 // return adddress
626 #endif
627 B aeshashbody<>(SB)
628 noaes:
629 B runtime·strhashFallback<ABIInternal>(SB)
630
631 // R0: data
632 // R1: seed data
633 // R2: length
634 #ifdef GOEXPERIMENT_regabiargs
635 // At return, R0 = return value
636 #else
637 // R8: address to put return value
638 #endif
639 TEXT aeshashbody<>(SB),NOSPLIT|NOFRAME,$0
640 VEOR V30.B16, V30.B16, V30.B16
641 VMOV R1, V30.D[0]
642 VMOV R2, V30.D[1] // load length into seed
643
644 MOVD $runtime·aeskeysched+0(SB), R4
645 VLD1.P 16(R4), [V0.B16]
646 AESE V30.B16, V0.B16
647 AESMC V0.B16, V0.B16
648 CMP $16, R2
649 BLO aes0to15
650 BEQ aes16
651 CMP $32, R2
652 BLS aes17to32
653 CMP $64, R2
654 BLS aes33to64
655 CMP $128, R2
656 BLS aes65to128
657 B aes129plus
658
659 aes0to15:
660 CBZ R2, aes0
661 VEOR V2.B16, V2.B16, V2.B16
662 TBZ $3, R2, less_than_8
663 VLD1.P 8(R0), V2.D[0]
664
665 less_than_8:
666 TBZ $2, R2, less_than_4
667 VLD1.P 4(R0), V2.S[2]
668
669 less_than_4:
670 TBZ $1, R2, less_than_2
671 VLD1.P 2(R0), V2.H[6]
672
673 less_than_2:
674 TBZ $0, R2, done
675 VLD1 (R0), V2.B[14]
676 done:
677 AESE V0.B16, V2.B16
678 AESMC V2.B16, V2.B16
679 AESE V0.B16, V2.B16
680 AESMC V2.B16, V2.B16
681 AESE V0.B16, V2.B16
682
683 #ifdef GOEXPERIMENT_regabiargs
684 VMOV V2.D[0], R0
685 #else
686 VST1 [V2.D1], (R8)
687 #endif
688 RET
689
690 aes0:
691 #ifdef GOEXPERIMENT_regabiargs
692 VMOV V0.D[0], R0
693 #else
694 VST1 [V0.D1], (R8)
695 #endif
696 RET
697
698 aes16:
699 VLD1 (R0), [V2.B16]
700 B done
701
702 aes17to32:
703 // make second seed
704 VLD1 (R4), [V1.B16]
705 AESE V30.B16, V1.B16
706 AESMC V1.B16, V1.B16
707 SUB $16, R2, R10
708 VLD1.P (R0)(R10), [V2.B16]
709 VLD1 (R0), [V3.B16]
710
711 AESE V0.B16, V2.B16
712 AESMC V2.B16, V2.B16
713 AESE V1.B16, V3.B16
714 AESMC V3.B16, V3.B16
715
716 AESE V0.B16, V2.B16
717 AESMC V2.B16, V2.B16
718 AESE V1.B16, V3.B16
719 AESMC V3.B16, V3.B16
720
721 AESE V0.B16, V2.B16
722 AESE V1.B16, V3.B16
723
724 VEOR V3.B16, V2.B16, V2.B16
725 #ifdef GOEXPERIMENT_regabiargs
726 VMOV V2.D[0], R0
727 #else
728 VST1 [V2.D1], (R8)
729 #endif
730 RET
731
732 aes33to64:
733 VLD1 (R4), [V1.B16, V2.B16, V3.B16]
734 AESE V30.B16, V1.B16
735 AESMC V1.B16, V1.B16
736 AESE V30.B16, V2.B16
737 AESMC V2.B16, V2.B16
738 AESE V30.B16, V3.B16
739 AESMC V3.B16, V3.B16
740 SUB $32, R2, R10
741
742 VLD1.P (R0)(R10), [V4.B16, V5.B16]
743 VLD1 (R0), [V6.B16, V7.B16]
744
745 AESE V0.B16, V4.B16
746 AESMC V4.B16, V4.B16
747 AESE V1.B16, V5.B16
748 AESMC V5.B16, V5.B16
749 AESE V2.B16, V6.B16
750 AESMC V6.B16, V6.B16
751 AESE V3.B16, V7.B16
752 AESMC V7.B16, V7.B16
753
754 AESE V0.B16, V4.B16
755 AESMC V4.B16, V4.B16
756 AESE V1.B16, V5.B16
757 AESMC V5.B16, V5.B16
758 AESE V2.B16, V6.B16
759 AESMC V6.B16, V6.B16
760 AESE V3.B16, V7.B16
761 AESMC V7.B16, V7.B16
762
763 AESE V0.B16, V4.B16
764 AESE V1.B16, V5.B16
765 AESE V2.B16, V6.B16
766 AESE V3.B16, V7.B16
767
768 VEOR V6.B16, V4.B16, V4.B16
769 VEOR V7.B16, V5.B16, V5.B16
770 VEOR V5.B16, V4.B16, V4.B16
771
772 #ifdef GOEXPERIMENT_regabiargs
773 VMOV V4.D[0], R0
774 #else
775 VST1 [V4.D1], (R8)
776 #endif
777 RET
778
779 aes65to128:
780 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
781 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
782 AESE V30.B16, V1.B16
783 AESMC V1.B16, V1.B16
784 AESE V30.B16, V2.B16
785 AESMC V2.B16, V2.B16
786 AESE V30.B16, V3.B16
787 AESMC V3.B16, V3.B16
788 AESE V30.B16, V4.B16
789 AESMC V4.B16, V4.B16
790 AESE V30.B16, V5.B16
791 AESMC V5.B16, V5.B16
792 AESE V30.B16, V6.B16
793 AESMC V6.B16, V6.B16
794 AESE V30.B16, V7.B16
795 AESMC V7.B16, V7.B16
796
797 SUB $64, R2, R10
798 VLD1.P (R0)(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
799 VLD1 (R0), [V12.B16, V13.B16, V14.B16, V15.B16]
800 AESE V0.B16, V8.B16
801 AESMC V8.B16, V8.B16
802 AESE V1.B16, V9.B16
803 AESMC V9.B16, V9.B16
804 AESE V2.B16, V10.B16
805 AESMC V10.B16, V10.B16
806 AESE V3.B16, V11.B16
807 AESMC V11.B16, V11.B16
808 AESE V4.B16, V12.B16
809 AESMC V12.B16, V12.B16
810 AESE V5.B16, V13.B16
811 AESMC V13.B16, V13.B16
812 AESE V6.B16, V14.B16
813 AESMC V14.B16, V14.B16
814 AESE V7.B16, V15.B16
815 AESMC V15.B16, V15.B16
816
817 AESE V0.B16, V8.B16
818 AESMC V8.B16, V8.B16
819 AESE V1.B16, V9.B16
820 AESMC V9.B16, V9.B16
821 AESE V2.B16, V10.B16
822 AESMC V10.B16, V10.B16
823 AESE V3.B16, V11.B16
824 AESMC V11.B16, V11.B16
825 AESE V4.B16, V12.B16
826 AESMC V12.B16, V12.B16
827 AESE V5.B16, V13.B16
828 AESMC V13.B16, V13.B16
829 AESE V6.B16, V14.B16
830 AESMC V14.B16, V14.B16
831 AESE V7.B16, V15.B16
832 AESMC V15.B16, V15.B16
833
834 AESE V0.B16, V8.B16
835 AESE V1.B16, V9.B16
836 AESE V2.B16, V10.B16
837 AESE V3.B16, V11.B16
838 AESE V4.B16, V12.B16
839 AESE V5.B16, V13.B16
840 AESE V6.B16, V14.B16
841 AESE V7.B16, V15.B16
842
843 VEOR V12.B16, V8.B16, V8.B16
844 VEOR V13.B16, V9.B16, V9.B16
845 VEOR V14.B16, V10.B16, V10.B16
846 VEOR V15.B16, V11.B16, V11.B16
847 VEOR V10.B16, V8.B16, V8.B16
848 VEOR V11.B16, V9.B16, V9.B16
849 VEOR V9.B16, V8.B16, V8.B16
850
851 #ifdef GOEXPERIMENT_regabiargs
852 VMOV V8.D[0], R0
853 #else
854 VST1 [V8.D1], (R8)
855 #endif
856 RET
857
858 aes129plus:
859 PRFM (R0), PLDL1KEEP
860 VLD1.P 64(R4), [V1.B16, V2.B16, V3.B16, V4.B16]
861 VLD1 (R4), [V5.B16, V6.B16, V7.B16]
862 AESE V30.B16, V1.B16
863 AESMC V1.B16, V1.B16
864 AESE V30.B16, V2.B16
865 AESMC V2.B16, V2.B16
866 AESE V30.B16, V3.B16
867 AESMC V3.B16, V3.B16
868 AESE V30.B16, V4.B16
869 AESMC V4.B16, V4.B16
870 AESE V30.B16, V5.B16
871 AESMC V5.B16, V5.B16
872 AESE V30.B16, V6.B16
873 AESMC V6.B16, V6.B16
874 AESE V30.B16, V7.B16
875 AESMC V7.B16, V7.B16
876 ADD R0, R2, R10
877 SUB $128, R10, R10
878 VLD1.P 64(R10), [V8.B16, V9.B16, V10.B16, V11.B16]
879 VLD1 (R10), [V12.B16, V13.B16, V14.B16, V15.B16]
880 SUB $1, R2, R2
881 LSR $7, R2, R2
882
883 aesloop:
884 AESE V8.B16, V0.B16
885 AESMC V0.B16, V0.B16
886 AESE V9.B16, V1.B16
887 AESMC V1.B16, V1.B16
888 AESE V10.B16, V2.B16
889 AESMC V2.B16, V2.B16
890 AESE V11.B16, V3.B16
891 AESMC V3.B16, V3.B16
892 AESE V12.B16, V4.B16
893 AESMC V4.B16, V4.B16
894 AESE V13.B16, V5.B16
895 AESMC V5.B16, V5.B16
896 AESE V14.B16, V6.B16
897 AESMC V6.B16, V6.B16
898 AESE V15.B16, V7.B16
899 AESMC V7.B16, V7.B16
900
901 VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16]
902 AESE V8.B16, V0.B16
903 AESMC V0.B16, V0.B16
904 AESE V9.B16, V1.B16
905 AESMC V1.B16, V1.B16
906 AESE V10.B16, V2.B16
907 AESMC V2.B16, V2.B16
908 AESE V11.B16, V3.B16
909 AESMC V3.B16, V3.B16
910
911 VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16]
912 AESE V12.B16, V4.B16
913 AESMC V4.B16, V4.B16
914 AESE V13.B16, V5.B16
915 AESMC V5.B16, V5.B16
916 AESE V14.B16, V6.B16
917 AESMC V6.B16, V6.B16
918 AESE V15.B16, V7.B16
919 AESMC V7.B16, V7.B16
920 SUB $1, R2, R2
921 CBNZ R2, aesloop
922
923 AESE V8.B16, V0.B16
924 AESMC V0.B16, V0.B16
925 AESE V9.B16, V1.B16
926 AESMC V1.B16, V1.B16
927 AESE V10.B16, V2.B16
928 AESMC V2.B16, V2.B16
929 AESE V11.B16, V3.B16
930 AESMC V3.B16, V3.B16
931 AESE V12.B16, V4.B16
932 AESMC V4.B16, V4.B16
933 AESE V13.B16, V5.B16
934 AESMC V5.B16, V5.B16
935 AESE V14.B16, V6.B16
936 AESMC V6.B16, V6.B16
937 AESE V15.B16, V7.B16
938 AESMC V7.B16, V7.B16
939
940 AESE V8.B16, V0.B16
941 AESMC V0.B16, V0.B16
942 AESE V9.B16, V1.B16
943 AESMC V1.B16, V1.B16
944 AESE V10.B16, V2.B16
945 AESMC V2.B16, V2.B16
946 AESE V11.B16, V3.B16
947 AESMC V3.B16, V3.B16
948 AESE V12.B16, V4.B16
949 AESMC V4.B16, V4.B16
950 AESE V13.B16, V5.B16
951 AESMC V5.B16, V5.B16
952 AESE V14.B16, V6.B16
953 AESMC V6.B16, V6.B16
954 AESE V15.B16, V7.B16
955 AESMC V7.B16, V7.B16
956
957 AESE V8.B16, V0.B16
958 AESE V9.B16, V1.B16
959 AESE V10.B16, V2.B16
960 AESE V11.B16, V3.B16
961 AESE V12.B16, V4.B16
962 AESE V13.B16, V5.B16
963 AESE V14.B16, V6.B16
964 AESE V15.B16, V7.B16
965
966 VEOR V0.B16, V1.B16, V0.B16
967 VEOR V2.B16, V3.B16, V2.B16
968 VEOR V4.B16, V5.B16, V4.B16
969 VEOR V6.B16, V7.B16, V6.B16
970 VEOR V0.B16, V2.B16, V0.B16
971 VEOR V4.B16, V6.B16, V4.B16
972 VEOR V4.B16, V0.B16, V0.B16
973
974 #ifdef GOEXPERIMENT_regabiargs
975 VMOV V0.D[0], R0
976 #else
977 VST1 [V0.D1], (R8)
978 #endif
979 RET
980
981 TEXT runtime·procyield(SB),NOSPLIT,$0-0
982 MOVWU cycles+0(FP), R0
983 again:
984 YIELD
985 SUBW $1, R0
986 CBNZ R0, again
987 RET
988
989 // Save state of caller into g->sched,
990 // but using fake PC from systemstack_switch.
991 // Must only be called from functions with no locals ($0)
992 // or else unwinding from systemstack_switch is incorrect.
993 // Smashes R0.
994 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
995 MOVD $runtime·systemstack_switch(SB), R0
996 ADD $8, R0 // get past prologue
997 MOVD R0, (g_sched+gobuf_pc)(g)
998 MOVD RSP, R0
999 MOVD R0, (g_sched+gobuf_sp)(g)
1000 MOVD R29, (g_sched+gobuf_bp)(g)
1001 MOVD $0, (g_sched+gobuf_lr)(g)
1002 MOVD $0, (g_sched+gobuf_ret)(g)
1003 // Assert ctxt is zero. See func save.
1004 MOVD (g_sched+gobuf_ctxt)(g), R0
1005 CBZ R0, 2(PC)
1006 CALL runtime·abort(SB)
1007 RET
1008
1009 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
1010 // Call fn(arg) aligned appropriately for the gcc ABI.
1011 // Called on a system stack, and there may be no g yet (during needm).
1012 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
1013 MOVD fn+0(FP), R1
1014 MOVD arg+8(FP), R0
1015 SUB $16, RSP // skip over saved frame pointer below RSP
1016 BL (R1)
1017 ADD $16, RSP // skip over saved frame pointer below RSP
1018 RET
1019
1020 // func asmcgocall(fn, arg unsafe.Pointer) int32
1021 // Call fn(arg) on the scheduler stack,
1022 // aligned appropriately for the gcc ABI.
1023 // See cgocall.go for more details.
1024 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
1025 MOVD fn+0(FP), R1
1026 MOVD arg+8(FP), R0
1027
1028 MOVD RSP, R2 // save original stack pointer
1029 CBZ g, nosave
1030 MOVD g, R4
1031
1032 // Figure out if we need to switch to m->g0 stack.
1033 // We get called to create new OS threads too, and those
1034 // come in on the m->g0 stack already. Or we might already
1035 // be on the m->gsignal stack.
1036 MOVD g_m(g), R8
1037 MOVD m_gsignal(R8), R3
1038 CMP R3, g
1039 BEQ nosave
1040 MOVD m_g0(R8), R3
1041 CMP R3, g
1042 BEQ nosave
1043
1044 // Switch to system stack.
1045 MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0
1046 BL gosave_systemstack_switch<>(SB)
1047 MOVD R3, g
1048 BL runtime·save_g(SB)
1049 MOVD (g_sched+gobuf_sp)(g), R0
1050 MOVD R0, RSP
1051 MOVD (g_sched+gobuf_bp)(g), R29
1052 MOVD R9, R0
1053
1054 // Now on a scheduling stack (a pthread-created stack).
1055 // Save room for two of our pointers /*, plus 32 bytes of callee
1056 // save area that lives on the caller stack. */
1057 MOVD RSP, R13
1058 SUB $16, R13
1059 MOVD R13, RSP
1060 MOVD R4, 0(RSP) // save old g on stack
1061 MOVD (g_stack+stack_hi)(R4), R4
1062 SUB R2, R4
1063 MOVD R4, 8(RSP) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
1064 BL (R1)
1065 MOVD R0, R9
1066
1067 // Restore g, stack pointer. R0 is errno, so don't touch it
1068 MOVD 0(RSP), g
1069 BL runtime·save_g(SB)
1070 MOVD (g_stack+stack_hi)(g), R5
1071 MOVD 8(RSP), R6
1072 SUB R6, R5
1073 MOVD R9, R0
1074 MOVD R5, RSP
1075
1076 MOVW R0, ret+16(FP)
1077 RET
1078
1079 nosave:
1080 // Running on a system stack, perhaps even without a g.
1081 // Having no g can happen during thread creation or thread teardown
1082 // (see needm/dropm on Solaris, for example).
1083 // This code is like the above sequence but without saving/restoring g
1084 // and without worrying about the stack moving out from under us
1085 // (because we're on a system stack, not a goroutine stack).
1086 // The above code could be used directly if already on a system stack,
1087 // but then the only path through this code would be a rare case on Solaris.
1088 // Using this code for all "already on system stack" calls exercises it more,
1089 // which should help keep it correct.
1090 MOVD RSP, R13
1091 SUB $16, R13
1092 MOVD R13, RSP
1093 MOVD $0, R4
1094 MOVD R4, 0(RSP) // Where above code stores g, in case someone looks during debugging.
1095 MOVD R2, 8(RSP) // Save original stack pointer.
1096 BL (R1)
1097 // Restore stack pointer.
1098 MOVD 8(RSP), R2
1099 MOVD R2, RSP
1100 MOVD R0, ret+16(FP)
1101 RET
1102
1103 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
1104 // See cgocall.go for more details.
1105 TEXT ·cgocallback(SB),NOSPLIT,$24-24
1106 NO_LOCAL_POINTERS
1107
1108 // Load g from thread-local storage.
1109 BL runtime·load_g(SB)
1110
1111 // If g is nil, Go did not create the current thread.
1112 // Call needm to obtain one for temporary use.
1113 // In this case, we're running on the thread stack, so there's
1114 // lots of space, but the linker doesn't know. Hide the call from
1115 // the linker analysis by using an indirect call.
1116 CBZ g, needm
1117
1118 MOVD g_m(g), R8
1119 MOVD R8, savedm-8(SP)
1120 B havem
1121
1122 needm:
1123 MOVD g, savedm-8(SP) // g is zero, so is m.
1124 MOVD $runtime·needm(SB), R0
1125 BL (R0)
1126
1127 // Set m->g0->sched.sp = SP, so that if a panic happens
1128 // during the function we are about to execute, it will
1129 // have a valid SP to run on the g0 stack.
1130 // The next few lines (after the havem label)
1131 // will save this SP onto the stack and then write
1132 // the same SP back to m->sched.sp. That seems redundant,
1133 // but if an unrecovered panic happens, unwindm will
1134 // restore the g->sched.sp from the stack location
1135 // and then systemstack will try to use it. If we don't set it here,
1136 // that restored SP will be uninitialized (typically 0) and
1137 // will not be usable.
1138 MOVD g_m(g), R8
1139 MOVD m_g0(R8), R3
1140 MOVD RSP, R0
1141 MOVD R0, (g_sched+gobuf_sp)(R3)
1142 MOVD R29, (g_sched+gobuf_bp)(R3)
1143
1144 havem:
1145 // Now there's a valid m, and we're running on its m->g0.
1146 // Save current m->g0->sched.sp on stack and then set it to SP.
1147 // Save current sp in m->g0->sched.sp in preparation for
1148 // switch back to m->curg stack.
1149 // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
1150 // Beware that the frame size is actually 32+16.
1151 MOVD m_g0(R8), R3
1152 MOVD (g_sched+gobuf_sp)(R3), R4
1153 MOVD R4, savedsp-16(SP)
1154 MOVD RSP, R0
1155 MOVD R0, (g_sched+gobuf_sp)(R3)
1156
1157 // Switch to m->curg stack and call runtime.cgocallbackg.
1158 // Because we are taking over the execution of m->curg
1159 // but *not* resuming what had been running, we need to
1160 // save that information (m->curg->sched) so we can restore it.
1161 // We can restore m->curg->sched.sp easily, because calling
1162 // runtime.cgocallbackg leaves SP unchanged upon return.
1163 // To save m->curg->sched.pc, we push it onto the curg stack and
1164 // open a frame the same size as cgocallback's g0 frame.
1165 // Once we switch to the curg stack, the pushed PC will appear
1166 // to be the return PC of cgocallback, so that the traceback
1167 // will seamlessly trace back into the earlier calls.
1168 MOVD m_curg(R8), g
1169 BL runtime·save_g(SB)
1170 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
1171 MOVD (g_sched+gobuf_pc)(g), R5
1172 MOVD R5, -48(R4)
1173 MOVD (g_sched+gobuf_bp)(g), R5
1174 MOVD R5, -56(R4)
1175 // Gather our arguments into registers.
1176 MOVD fn+0(FP), R1
1177 MOVD frame+8(FP), R2
1178 MOVD ctxt+16(FP), R3
1179 MOVD $-48(R4), R0 // maintain 16-byte SP alignment
1180 MOVD R0, RSP // switch stack
1181 MOVD R1, 8(RSP)
1182 MOVD R2, 16(RSP)
1183 MOVD R3, 24(RSP)
1184 MOVD $runtime·cgocallbackg(SB), R0
1185 CALL (R0) // indirect call to bypass nosplit check. We're on a different stack now.
1186
1187 // Restore g->sched (== m->curg->sched) from saved values.
1188 MOVD 0(RSP), R5
1189 MOVD R5, (g_sched+gobuf_pc)(g)
1190 MOVD RSP, R4
1191 ADD $48, R4, R4
1192 MOVD R4, (g_sched+gobuf_sp)(g)
1193
1194 // Switch back to m->g0's stack and restore m->g0->sched.sp.
1195 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
1196 // so we do not have to restore it.)
1197 MOVD g_m(g), R8
1198 MOVD m_g0(R8), g
1199 BL runtime·save_g(SB)
1200 MOVD (g_sched+gobuf_sp)(g), R0
1201 MOVD R0, RSP
1202 MOVD savedsp-16(SP), R4
1203 MOVD R4, (g_sched+gobuf_sp)(g)
1204
1205 // If the m on entry was nil, we called needm above to borrow an m
1206 // for the duration of the call. Since the call is over, return it with dropm.
1207 MOVD savedm-8(SP), R6
1208 CBNZ R6, droppedm
1209 MOVD $runtime·dropm(SB), R0
1210 BL (R0)
1211 droppedm:
1212
1213 // Done!
1214 RET
1215
1216 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1217 // Must obey the gcc calling convention.
1218 TEXT _cgo_topofstack(SB),NOSPLIT,$24
1219 // g (R28) and REGTMP (R27) might be clobbered by load_g. They
1220 // are callee-save in the gcc calling convention, so save them.
1221 MOVD R27, savedR27-8(SP)
1222 MOVD g, saveG-16(SP)
1223
1224 BL runtime·load_g(SB)
1225 MOVD g_m(g), R0
1226 MOVD m_curg(R0), R0
1227 MOVD (g_stack+stack_hi)(R0), R0
1228
1229 MOVD saveG-16(SP), g
1230 MOVD savedR28-8(SP), R27
1231 RET
1232
1233 // void setg(G*); set g. for use by needm.
1234 TEXT runtime·setg(SB), NOSPLIT, $0-8
1235 MOVD gg+0(FP), g
1236 // This only happens if iscgo, so jump straight to save_g
1237 BL runtime·save_g(SB)
1238 RET
1239
1240 // void setg_gcc(G*); set g called from gcc
1241 TEXT setg_gcc<>(SB),NOSPLIT,$8
1242 MOVD R0, g
1243 MOVD R27, savedR27-8(SP)
1244 BL runtime·save_g(SB)
1245 MOVD savedR27-8(SP), R27
1246 RET
1247
1248 TEXT runtime·emptyfunc(SB),0,$0-0
1249 RET
1250
1251 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
1252 MOVD ZR, R0
1253 MOVD (R0), R0
1254 UNDEF
1255
1256 TEXT runtime·return0(SB), NOSPLIT, $0
1257 MOVW $0, R0
1258 RET
1259
1260 // The top-most function running on a goroutine
1261 // returns to goexit+PCQuantum.
1262 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1263 MOVD R0, R0 // NOP
1264 BL runtime·goexit1(SB) // does not return
1265
1266 // This is called from .init_array and follows the platform, not Go, ABI.
1267 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
1268 SUB $0x10, RSP
1269 MOVD R27, 8(RSP) // The access to global variables below implicitly uses R27, which is callee-save
1270 MOVD runtime·lastmoduledatap(SB), R1
1271 MOVD R0, moduledata_next(R1)
1272 MOVD R0, runtime·lastmoduledatap(SB)
1273 MOVD 8(RSP), R27
1274 ADD $0x10, RSP
1275 RET
1276
1277 TEXT ·checkASM(SB),NOSPLIT,$0-1
1278 MOVW $1, R3
1279 MOVB R3, ret+0(FP)
1280 RET
1281
1282 // gcWriteBarrier performs a heap pointer write and informs the GC.
1283 //
1284 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
1285 // - R2 is the destination of the write
1286 // - R3 is the value being written at R2
1287 // It clobbers condition codes.
1288 // It does not clobber any general-purpose registers,
1289 // but may clobber others (e.g., floating point registers)
1290 // The act of CALLing gcWriteBarrier will clobber R30 (LR).
1291 //
1292 // Defined as ABIInternal since the compiler generates ABIInternal
1293 // calls to it directly and it does not use the stack-based Go ABI.
1294 TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$200
1295 // Save the registers clobbered by the fast path.
1296 MOVD R0, 184(RSP)
1297 MOVD R1, 192(RSP)
1298 MOVD g_m(g), R0
1299 MOVD m_p(R0), R0
1300 MOVD (p_wbBuf+wbBuf_next)(R0), R1
1301 // Increment wbBuf.next position.
1302 ADD $16, R1
1303 MOVD R1, (p_wbBuf+wbBuf_next)(R0)
1304 MOVD (p_wbBuf+wbBuf_end)(R0), R0
1305 CMP R1, R0
1306 // Record the write.
1307 MOVD R3, -16(R1) // Record value
1308 MOVD (R2), R0 // TODO: This turns bad writes into bad reads.
1309 MOVD R0, -8(R1) // Record *slot
1310 // Is the buffer full? (flags set in CMP above)
1311 BEQ flush
1312 ret:
1313 MOVD 184(RSP), R0
1314 MOVD 192(RSP), R1
1315 // Do the write.
1316 MOVD R3, (R2)
1317 RET
1318
1319 flush:
1320 // Save all general purpose registers since these could be
1321 // clobbered by wbBufFlush and were not saved by the caller.
1322 MOVD R2, 8(RSP) // Also first argument to wbBufFlush
1323 MOVD R3, 16(RSP) // Also second argument to wbBufFlush
1324 // R0 already saved
1325 // R1 already saved
1326 MOVD R4, 24(RSP)
1327 MOVD R5, 32(RSP)
1328 MOVD R6, 40(RSP)
1329 MOVD R7, 48(RSP)
1330 MOVD R8, 56(RSP)
1331 MOVD R9, 64(RSP)
1332 MOVD R10, 72(RSP)
1333 MOVD R11, 80(RSP)
1334 MOVD R12, 88(RSP)
1335 MOVD R13, 96(RSP)
1336 MOVD R14, 104(RSP)
1337 MOVD R15, 112(RSP)
1338 // R16, R17 may be clobbered by linker trampoline
1339 // R18 is unused.
1340 MOVD R19, 120(RSP)
1341 MOVD R20, 128(RSP)
1342 MOVD R21, 136(RSP)
1343 MOVD R22, 144(RSP)
1344 MOVD R23, 152(RSP)
1345 MOVD R24, 160(RSP)
1346 MOVD R25, 168(RSP)
1347 MOVD R26, 176(RSP)
1348 // R27 is temp register.
1349 // R28 is g.
1350 // R29 is frame pointer (unused).
1351 // R30 is LR, which was saved by the prologue.
1352 // R31 is SP.
1353
1354 // This takes arguments R2 and R3.
1355 CALL runtime·wbBufFlush(SB)
1356
1357 MOVD 8(RSP), R2
1358 MOVD 16(RSP), R3
1359 MOVD 24(RSP), R4
1360 MOVD 32(RSP), R5
1361 MOVD 40(RSP), R6
1362 MOVD 48(RSP), R7
1363 MOVD 56(RSP), R8
1364 MOVD 64(RSP), R9
1365 MOVD 72(RSP), R10
1366 MOVD 80(RSP), R11
1367 MOVD 88(RSP), R12
1368 MOVD 96(RSP), R13
1369 MOVD 104(RSP), R14
1370 MOVD 112(RSP), R15
1371 MOVD 120(RSP), R19
1372 MOVD 128(RSP), R20
1373 MOVD 136(RSP), R21
1374 MOVD 144(RSP), R22
1375 MOVD 152(RSP), R23
1376 MOVD 160(RSP), R24
1377 MOVD 168(RSP), R25
1378 MOVD 176(RSP), R26
1379 JMP ret
1380
1381 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1382 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1383
1384 // debugCallV2 is the entry point for debugger-injected function
1385 // calls on running goroutines. It informs the runtime that a
1386 // debug call has been injected and creates a call frame for the
1387 // debugger to fill in.
1388 //
1389 // To inject a function call, a debugger should:
1390 // 1. Check that the goroutine is in state _Grunning and that
1391 // there are at least 288 bytes free on the stack.
1392 // 2. Set SP as SP-16.
1393 // 3. Store the current LR in (SP) (using the SP after step 2).
1394 // 4. Store the current PC in the LR register.
1395 // 5. Write the desired argument frame size at SP-16
1396 // 6. Save all machine registers (including flags and fpsimd reigsters)
1397 // so they can be restored later by the debugger.
1398 // 7. Set the PC to debugCallV2 and resume execution.
1399 //
1400 // If the goroutine is in state _Grunnable, then it's not generally
1401 // safe to inject a call because it may return out via other runtime
1402 // operations. Instead, the debugger should unwind the stack to find
1403 // the return to non-runtime code, add a temporary breakpoint there,
1404 // and inject the call once that breakpoint is hit.
1405 //
1406 // If the goroutine is in any other state, it's not safe to inject a call.
1407 //
1408 // This function communicates back to the debugger by setting R20 and
1409 // invoking BRK to raise a breakpoint signal. See the comments in the
1410 // implementation for the protocol the debugger is expected to
1411 // follow. InjectDebugCall in the runtime tests demonstrates this protocol.
1412 //
1413 // The debugger must ensure that any pointers passed to the function
1414 // obey escape analysis requirements. Specifically, it must not pass
1415 // a stack pointer to an escaping argument. debugCallV2 cannot check
1416 // this invariant.
1417 //
1418 // This is ABIInternal because Go code injects its PC directly into new
1419 // goroutine stacks.
1420 TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
1421 STP (R29, R30), -280(RSP)
1422 SUB $272, RSP, RSP
1423 SUB $8, RSP, R29
1424 // Save all registers that may contain pointers so they can be
1425 // conservatively scanned.
1426 //
1427 // We can't do anything that might clobber any of these
1428 // registers before this.
1429 STP (R27, g), (30*8)(RSP)
1430 STP (R25, R26), (28*8)(RSP)
1431 STP (R23, R24), (26*8)(RSP)
1432 STP (R21, R22), (24*8)(RSP)
1433 STP (R19, R20), (22*8)(RSP)
1434 STP (R16, R17), (20*8)(RSP)
1435 STP (R14, R15), (18*8)(RSP)
1436 STP (R12, R13), (16*8)(RSP)
1437 STP (R10, R11), (14*8)(RSP)
1438 STP (R8, R9), (12*8)(RSP)
1439 STP (R6, R7), (10*8)(RSP)
1440 STP (R4, R5), (8*8)(RSP)
1441 STP (R2, R3), (6*8)(RSP)
1442 STP (R0, R1), (4*8)(RSP)
1443
1444 // Perform a safe-point check.
1445 MOVD R30, 8(RSP) // Caller's PC
1446 CALL runtime·debugCallCheck(SB)
1447 MOVD 16(RSP), R0
1448 CBZ R0, good
1449
1450 // The safety check failed. Put the reason string at the top
1451 // of the stack.
1452 MOVD R0, 8(RSP)
1453 MOVD 24(RSP), R0
1454 MOVD R0, 16(RSP)
1455
1456 // Set R20 to 8 and invoke BRK. The debugger should get the
1457 // reason a call can't be injected from SP+8 and resume execution.
1458 MOVD $8, R20
1459 BRK
1460 JMP restore
1461
1462 good:
1463 // Registers are saved and it's safe to make a call.
1464 // Open up a call frame, moving the stack if necessary.
1465 //
1466 // Once the frame is allocated, this will set R20 to 0 and
1467 // invoke BRK. The debugger should write the argument
1468 // frame for the call at SP+8, set up argument registers,
1469 // set the lr as the signal PC + 4, set the PC to the function
1470 // to call, set R26 to point to the closure (if a closure call),
1471 // and resume execution.
1472 //
1473 // If the function returns, this will set R20 to 1 and invoke
1474 // BRK. The debugger can then inspect any return value saved
1475 // on the stack at SP+8 and in registers and resume execution again.
1476 //
1477 // If the function panics, this will set R20 to 2 and invoke BRK.
1478 // The interface{} value of the panic will be at SP+8. The debugger
1479 // can inspect the panic value and resume execution again.
1480 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1481 CMP $MAXSIZE, R0; \
1482 BGT 5(PC); \
1483 MOVD $NAME(SB), R0; \
1484 MOVD R0, 8(RSP); \
1485 CALL runtime·debugCallWrap(SB); \
1486 JMP restore
1487
1488 MOVD 256(RSP), R0 // the argument frame size
1489 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1490 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1491 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1492 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1493 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1494 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1495 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1496 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1497 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1498 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1499 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1500 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1501 // The frame size is too large. Report the error.
1502 MOVD $debugCallFrameTooLarge<>(SB), R0
1503 MOVD R0, 8(RSP)
1504 MOVD $20, R0
1505 MOVD R0, 16(RSP) // length of debugCallFrameTooLarge string
1506 MOVD $8, R20
1507 BRK
1508 JMP restore
1509
1510 restore:
1511 // Calls and failures resume here.
1512 //
1513 // Set R20 to 16 and invoke BRK. The debugger should restore
1514 // all registers except for PC and RSP and resume execution.
1515 MOVD $16, R20
1516 BRK
1517 // We must not modify flags after this point.
1518
1519 // Restore pointer-containing registers, which may have been
1520 // modified from the debugger's copy by stack copying.
1521 LDP (30*8)(RSP), (R27, g)
1522 LDP (28*8)(RSP), (R25, R26)
1523 LDP (26*8)(RSP), (R23, R24)
1524 LDP (24*8)(RSP), (R21, R22)
1525 LDP (22*8)(RSP), (R19, R20)
1526 LDP (20*8)(RSP), (R16, R17)
1527 LDP (18*8)(RSP), (R14, R15)
1528 LDP (16*8)(RSP), (R12, R13)
1529 LDP (14*8)(RSP), (R10, R11)
1530 LDP (12*8)(RSP), (R8, R9)
1531 LDP (10*8)(RSP), (R6, R7)
1532 LDP (8*8)(RSP), (R4, R5)
1533 LDP (6*8)(RSP), (R2, R3)
1534 LDP (4*8)(RSP), (R0, R1)
1535
1536 LDP -8(RSP), (R29, R27)
1537 ADD $288, RSP, RSP // Add 16 more bytes, see saveSigContext
1538 MOVD -16(RSP), R30 // restore old lr
1539 JMP (R27)
1540
1541 // runtime.debugCallCheck assumes that functions defined with the
1542 // DEBUG_CALL_FN macro are safe points to inject calls.
1543 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1544 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1545 NO_LOCAL_POINTERS; \
1546 MOVD $0, R20; \
1547 BRK; \
1548 MOVD $1, R20; \
1549 BRK; \
1550 RET
1551 DEBUG_CALL_FN(debugCall32<>, 32)
1552 DEBUG_CALL_FN(debugCall64<>, 64)
1553 DEBUG_CALL_FN(debugCall128<>, 128)
1554 DEBUG_CALL_FN(debugCall256<>, 256)
1555 DEBUG_CALL_FN(debugCall512<>, 512)
1556 DEBUG_CALL_FN(debugCall1024<>, 1024)
1557 DEBUG_CALL_FN(debugCall2048<>, 2048)
1558 DEBUG_CALL_FN(debugCall4096<>, 4096)
1559 DEBUG_CALL_FN(debugCall8192<>, 8192)
1560 DEBUG_CALL_FN(debugCall16384<>, 16384)
1561 DEBUG_CALL_FN(debugCall32768<>, 32768)
1562 DEBUG_CALL_FN(debugCall65536<>, 65536)
1563
1564 // func debugCallPanicked(val interface{})
1565 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
1566 // Copy the panic value to the top of stack at SP+8.
1567 MOVD val_type+0(FP), R0
1568 MOVD R0, 8(RSP)
1569 MOVD val_data+8(FP), R0
1570 MOVD R0, 16(RSP)
1571 MOVD $2, R20
1572 BRK
1573 RET
1574
1575 // Note: these functions use a special calling convention to save generated code space.
1576 // Arguments are passed in registers, but the space for those arguments are allocated
1577 // in the caller's stack frame. These stubs write the args into that stack space and
1578 // then tail call to the corresponding runtime handler.
1579 // The tail call makes these stubs disappear in backtraces.
1580 //
1581 // Defined as ABIInternal since the compiler generates ABIInternal
1582 // calls to it directly and it does not use the stack-based Go ABI.
1583 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1584 #ifndef GOEXPERIMENT_regabiargs
1585 MOVD R0, x+0(FP)
1586 MOVD R1, y+8(FP)
1587 #endif
1588 JMP runtime·goPanicIndex<ABIInternal>(SB)
1589 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1590 #ifndef GOEXPERIMENT_regabiargs
1591 MOVD R0, x+0(FP)
1592 MOVD R1, y+8(FP)
1593 #endif
1594 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1595 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1596 #ifdef GOEXPERIMENT_regabiargs
1597 MOVD R1, R0
1598 MOVD R2, R1
1599 #else
1600 MOVD R1, x+0(FP)
1601 MOVD R2, y+8(FP)
1602 #endif
1603 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1604 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1605 #ifdef GOEXPERIMENT_regabiargs
1606 MOVD R1, R0
1607 MOVD R2, R1
1608 #else
1609 MOVD R1, x+0(FP)
1610 MOVD R2, y+8(FP)
1611 #endif
1612 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1613 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1614 #ifdef GOEXPERIMENT_regabiargs
1615 MOVD R1, R0
1616 MOVD R2, R1
1617 #else
1618 MOVD R1, x+0(FP)
1619 MOVD R2, y+8(FP)
1620 #endif
1621 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1622 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1623 #ifdef GOEXPERIMENT_regabiargs
1624 MOVD R1, R0
1625 MOVD R2, R1
1626 #else
1627 MOVD R1, x+0(FP)
1628 MOVD R2, y+8(FP)
1629 #endif
1630 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1631 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1632 #ifndef GOEXPERIMENT_regabiargs
1633 MOVD R0, x+0(FP)
1634 MOVD R1, y+8(FP)
1635 #endif
1636 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1637 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1638 #ifndef GOEXPERIMENT_regabiargs
1639 MOVD R0, x+0(FP)
1640 MOVD R1, y+8(FP)
1641 #endif
1642 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1643 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1644 #ifdef GOEXPERIMENT_regabiargs
1645 MOVD R2, R0
1646 MOVD R3, R1
1647 #else
1648 MOVD R2, x+0(FP)
1649 MOVD R3, y+8(FP)
1650 #endif
1651 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1652 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1653 #ifdef GOEXPERIMENT_regabiargs
1654 MOVD R2, R0
1655 MOVD R3, R1
1656 #else
1657 MOVD R2, x+0(FP)
1658 MOVD R3, y+8(FP)
1659 #endif
1660 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1661 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1662 #ifdef GOEXPERIMENT_regabiargs
1663 MOVD R2, R0
1664 MOVD R3, R1
1665 #else
1666 MOVD R2, x+0(FP)
1667 MOVD R3, y+8(FP)
1668 #endif
1669 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1670 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1671 #ifdef GOEXPERIMENT_regabiargs
1672 MOVD R2, R0
1673 MOVD R3, R1
1674 #else
1675 MOVD R2, x+0(FP)
1676 MOVD R3, y+8(FP)
1677 #endif
1678 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1679 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1680 #ifdef GOEXPERIMENT_regabiargs
1681 MOVD R1, R0
1682 MOVD R2, R1
1683 #else
1684 MOVD R1, x+0(FP)
1685 MOVD R2, y+8(FP)
1686 #endif
1687 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1688 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1689 #ifdef GOEXPERIMENT_regabiargs
1690 MOVD R1, R0
1691 MOVD R2, R1
1692 #else
1693 MOVD R1, x+0(FP)
1694 MOVD R2, y+8(FP)
1695 #endif
1696 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1697 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1698 #ifndef GOEXPERIMENT_regabiargs
1699 MOVD R0, x+0(FP)
1700 MOVD R1, y+8(FP)
1701 #endif
1702 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1703 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1704 #ifndef GOEXPERIMENT_regabiargs
1705 MOVD R0, x+0(FP)
1706 MOVD R1, y+8(FP)
1707 #endif
1708 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1709 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1710 #ifdef GOEXPERIMENT_regabiargs
1711 MOVD R2, R0
1712 MOVD R3, R1
1713 #else
1714 MOVD R2, x+0(FP)
1715 MOVD R3, y+8(FP)
1716 #endif
1717 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1718
View as plain text