Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12
13 #ifdef GOOS_aix
14 #define cgoCalleeStackSize 48
15 #else
16 #define cgoCalleeStackSize 32
17 #endif
18
19 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // if there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R0, R12
44 BEQ nocgo
45 #ifdef GOARCH_ppc64
46 // ppc64 use elf ABI v1. we must get the real entry address from
47 // first slot of the function descriptor before call.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50 #endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62 #ifndef GOOS_aix
63 MOVD 24(R1), R2
64 #endif
65 XOR R0, R0 // fix R0
66
67 nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const__StackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 BL runtime·newproc(SB)
97 ADD $(8+FIXED_FRAME), R1
98
99 // start this M
100 BL runtime·mstart(SB)
101
102 MOVD R0, 0(R0)
103 RET
104
105 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
106 GLOBL runtime·mainPC(SB),RODATA,$8
107
108 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
109 MOVD R0, 0(R0) // TODO: TD
110 RET
111
112 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
113 RET
114
115 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
116 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
117 // crosscall_ppc64 and crosscall2 need to reginit, but can't
118 // get at the 'runtime.reginit' symbol.
119 BR runtime·reginit(SB)
120
121 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
122 // set R0 to zero, it's expected by the toolchain
123 XOR R0, R0
124 RET
125
126 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
127 BL runtime·mstart0(SB)
128 RET // not reached
129
130 /*
131 * go-routine
132 */
133
134 // void gogo(Gobuf*)
135 // restore state from Gobuf; longjmp
136 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
137 MOVD buf+0(FP), R5
138 MOVD gobuf_g(R5), R6
139 MOVD 0(R6), R4 // make sure g != nil
140 BR gogo<>(SB)
141
142 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
143 MOVD R6, g
144 BL runtime·save_g(SB)
145
146 MOVD gobuf_sp(R5), R1
147 MOVD gobuf_lr(R5), R31
148 #ifndef GOOS_aix
149 MOVD 24(R1), R2 // restore R2
150 #endif
151 MOVD R31, LR
152 MOVD gobuf_ret(R5), R3
153 MOVD gobuf_ctxt(R5), R11
154 MOVD R0, gobuf_sp(R5)
155 MOVD R0, gobuf_ret(R5)
156 MOVD R0, gobuf_lr(R5)
157 MOVD R0, gobuf_ctxt(R5)
158 CMP R0, R0 // set condition codes for == test, needed by stack split
159 MOVD gobuf_pc(R5), R12
160 MOVD R12, CTR
161 BR (CTR)
162
163 // void mcall(fn func(*g))
164 // Switch to m->g0's stack, call fn(g).
165 // Fn must never return. It should gogo(&g->sched)
166 // to keep running g.
167 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
168 // Save caller state in g->sched
169 // R11 should be safe across save_g??
170 #ifdef GOEXPERIMENT_regabiargs
171 MOVD R3, R11
172 #else
173 MOVD fn+0(FP), R11
174 #endif
175 MOVD R1, (g_sched+gobuf_sp)(g)
176 MOVD LR, R31
177 MOVD R31, (g_sched+gobuf_pc)(g)
178 MOVD R0, (g_sched+gobuf_lr)(g)
179
180 // Switch to m->g0 & its stack, call fn.
181 MOVD g, R3
182 MOVD g_m(g), R8
183 MOVD m_g0(R8), g
184 BL runtime·save_g(SB)
185 CMP g, R3
186 BNE 2(PC)
187 BR runtime·badmcall(SB)
188 MOVD 0(R11), R12 // code pointer
189 MOVD R12, CTR
190 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
191 // Don't need to do anything special for regabiargs here
192 // R3 is g; stack is set anyway
193 MOVDU R3, -8(R1)
194 MOVDU R0, -8(R1)
195 MOVDU R0, -8(R1)
196 MOVDU R0, -8(R1)
197 MOVDU R0, -8(R1)
198 BL (CTR)
199 MOVD 24(R1), R2
200 BR runtime·badmcall2(SB)
201
202 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
203 // of the G stack. We need to distinguish the routine that
204 // lives at the bottom of the G stack from the one that lives
205 // at the top of the system stack because the one at the top of
206 // the system stack terminates the stack walk (see topofstack()).
207 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
208 // We have several undefs here so that 16 bytes past
209 // $runtime·systemstack_switch lies within them whether or not the
210 // instructions that derive r2 from r12 are there.
211 UNDEF
212 UNDEF
213 UNDEF
214 BL (LR) // make sure this function is not leaf
215 RET
216
217 // func systemstack(fn func())
218 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
219 MOVD fn+0(FP), R3 // R3 = fn
220 MOVD R3, R11 // context
221 MOVD g_m(g), R4 // R4 = m
222
223 MOVD m_gsignal(R4), R5 // R5 = gsignal
224 CMP g, R5
225 BEQ noswitch
226
227 MOVD m_g0(R4), R5 // R5 = g0
228 CMP g, R5
229 BEQ noswitch
230
231 MOVD m_curg(R4), R6
232 CMP g, R6
233 BEQ switch
234
235 // Bad: g is not gsignal, not g0, not curg. What is it?
236 // Hide call from linker nosplit analysis.
237 MOVD $runtime·badsystemstack(SB), R12
238 MOVD R12, CTR
239 BL (CTR)
240 BL runtime·abort(SB)
241
242 switch:
243 // save our state in g->sched. Pretend to
244 // be systemstack_switch if the G stack is scanned.
245 BL gosave_systemstack_switch<>(SB)
246
247 // switch to g0
248 MOVD R5, g
249 BL runtime·save_g(SB)
250 MOVD (g_sched+gobuf_sp)(g), R1
251
252 // call target function
253 MOVD 0(R11), R12 // code pointer
254 MOVD R12, CTR
255 BL (CTR)
256
257 // restore TOC pointer. It seems unlikely that we will use systemstack
258 // to call a function defined in another module, but the results of
259 // doing so would be so confusing that it's worth doing this.
260 MOVD g_m(g), R3
261 MOVD m_curg(R3), g
262 MOVD (g_sched+gobuf_sp)(g), R3
263 #ifndef GOOS_aix
264 MOVD 24(R3), R2
265 #endif
266 // switch back to g
267 MOVD g_m(g), R3
268 MOVD m_curg(R3), g
269 BL runtime·save_g(SB)
270 MOVD (g_sched+gobuf_sp)(g), R1
271 MOVD R0, (g_sched+gobuf_sp)(g)
272 RET
273
274 noswitch:
275 // already on m stack, just call directly
276 // On other arches we do a tail call here, but it appears to be
277 // impossible to tail call a function pointer in shared mode on
278 // ppc64 because the caller is responsible for restoring the TOC.
279 MOVD 0(R11), R12 // code pointer
280 MOVD R12, CTR
281 BL (CTR)
282 #ifndef GOOS_aix
283 MOVD 24(R1), R2
284 #endif
285 RET
286
287 /*
288 * support for morestack
289 */
290
291 // Called during function prolog when more stack is needed.
292 // Caller has already loaded:
293 // R3: framesize, R4: argsize, R5: LR
294 //
295 // The traceback routines see morestack on a g0 as being
296 // the top of a stack (for example, morestack calling newstack
297 // calling the scheduler calling newm calling gc), so we must
298 // record an argument size. For that purpose, it has no arguments.
299 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
300 // Cannot grow scheduler stack (m->g0).
301 MOVD g_m(g), R7
302 MOVD m_g0(R7), R8
303 CMP g, R8
304 BNE 3(PC)
305 BL runtime·badmorestackg0(SB)
306 BL runtime·abort(SB)
307
308 // Cannot grow signal stack (m->gsignal).
309 MOVD m_gsignal(R7), R8
310 CMP g, R8
311 BNE 3(PC)
312 BL runtime·badmorestackgsignal(SB)
313 BL runtime·abort(SB)
314
315 // Called from f.
316 // Set g->sched to context in f.
317 MOVD R1, (g_sched+gobuf_sp)(g)
318 MOVD LR, R8
319 MOVD R8, (g_sched+gobuf_pc)(g)
320 MOVD R5, (g_sched+gobuf_lr)(g)
321 MOVD R11, (g_sched+gobuf_ctxt)(g)
322
323 // Called from f.
324 // Set m->morebuf to f's caller.
325 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
326 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
327 MOVD g, (m_morebuf+gobuf_g)(R7)
328
329 // Call newstack on m->g0's stack.
330 MOVD m_g0(R7), g
331 BL runtime·save_g(SB)
332 MOVD (g_sched+gobuf_sp)(g), R1
333 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
334 BL runtime·newstack(SB)
335
336 // Not reached, but make sure the return PC from the call to newstack
337 // is still in this function, and not the beginning of the next.
338 UNDEF
339
340 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
341 MOVD R0, R11
342 BR runtime·morestack(SB)
343
344 // reflectcall: call a function with the given argument list
345 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
346 // we don't have variable-sized frames, so we use a small number
347 // of constant-sized-frame functions to encode a few bits of size in the pc.
348 // Caution: ugly multiline assembly macros in your future!
349
350 #define DISPATCH(NAME,MAXSIZE) \
351 MOVD $MAXSIZE, R31; \
352 CMP R3, R31; \
353 BGT 4(PC); \
354 MOVD $NAME(SB), R12; \
355 MOVD R12, CTR; \
356 BR (CTR)
357 // Note: can't just "BR NAME(SB)" - bad inlining results.
358
359 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
360 MOVWZ frameSize+32(FP), R3
361 DISPATCH(runtime·call16, 16)
362 DISPATCH(runtime·call32, 32)
363 DISPATCH(runtime·call64, 64)
364 DISPATCH(runtime·call128, 128)
365 DISPATCH(runtime·call256, 256)
366 DISPATCH(runtime·call512, 512)
367 DISPATCH(runtime·call1024, 1024)
368 DISPATCH(runtime·call2048, 2048)
369 DISPATCH(runtime·call4096, 4096)
370 DISPATCH(runtime·call8192, 8192)
371 DISPATCH(runtime·call16384, 16384)
372 DISPATCH(runtime·call32768, 32768)
373 DISPATCH(runtime·call65536, 65536)
374 DISPATCH(runtime·call131072, 131072)
375 DISPATCH(runtime·call262144, 262144)
376 DISPATCH(runtime·call524288, 524288)
377 DISPATCH(runtime·call1048576, 1048576)
378 DISPATCH(runtime·call2097152, 2097152)
379 DISPATCH(runtime·call4194304, 4194304)
380 DISPATCH(runtime·call8388608, 8388608)
381 DISPATCH(runtime·call16777216, 16777216)
382 DISPATCH(runtime·call33554432, 33554432)
383 DISPATCH(runtime·call67108864, 67108864)
384 DISPATCH(runtime·call134217728, 134217728)
385 DISPATCH(runtime·call268435456, 268435456)
386 DISPATCH(runtime·call536870912, 536870912)
387 DISPATCH(runtime·call1073741824, 1073741824)
388 MOVD $runtime·badreflectcall(SB), R12
389 MOVD R12, CTR
390 BR (CTR)
391
392 #define CALLFN(NAME,MAXSIZE) \
393 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
394 NO_LOCAL_POINTERS; \
395 /* copy arguments to stack */ \
396 MOVD stackArgs+16(FP), R3; \
397 MOVWZ stackArgsSize+24(FP), R4; \
398 MOVD R1, R5; \
399 CMP R4, $8; \
400 BLT tailsetup; \
401 /* copy 8 at a time if possible */ \
402 ADD $(FIXED_FRAME-8), R5; \
403 SUB $8, R3; \
404 top: \
405 MOVDU 8(R3), R7; \
406 MOVDU R7, 8(R5); \
407 SUB $8, R4; \
408 CMP R4, $8; \
409 BGE top; \
410 /* handle remaining bytes */ \
411 CMP $0, R4; \
412 BEQ callfn; \
413 ADD $7, R3; \
414 ADD $7, R5; \
415 BR tail; \
416 tailsetup: \
417 CMP $0, R4; \
418 BEQ callfn; \
419 ADD $(FIXED_FRAME-1), R5; \
420 SUB $1, R3; \
421 tail: \
422 MOVBU 1(R3), R6; \
423 MOVBU R6, 1(R5); \
424 SUB $1, R4; \
425 CMP $0, R4; \
426 BGT tail; \
427 callfn: \
428 /* call function */ \
429 MOVD f+8(FP), R11; \
430 #ifdef GOOS_aix \
431 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
432 /* So it manually triggers it */ \
433 CMP R0, R11 \
434 BNE 2(PC) \
435 MOVD R0, 0(R0) \
436 #endif \
437 MOVD regArgs+40(FP), R20; \
438 BL runtime·unspillArgs(SB); \
439 MOVD (R11), R12; \
440 MOVD R12, CTR; \
441 PCDATA $PCDATA_StackMapIndex, $0; \
442 BL (CTR); \
443 #ifndef GOOS_aix \
444 MOVD 24(R1), R2; \
445 #endif \
446 /* copy return values back */ \
447 MOVD regArgs+40(FP), R20; \
448 BL runtime·spillArgs(SB); \
449 MOVD stackArgsType+0(FP), R7; \
450 MOVD stackArgs+16(FP), R3; \
451 MOVWZ stackArgsSize+24(FP), R4; \
452 MOVWZ stackRetOffset+28(FP), R6; \
453 ADD $FIXED_FRAME, R1, R5; \
454 ADD R6, R5; \
455 ADD R6, R3; \
456 SUB R6, R4; \
457 BL callRet<>(SB); \
458 RET
459
460 // callRet copies return values back at the end of call*. This is a
461 // separate function so it can allocate stack space for the arguments
462 // to reflectcallmove. It does not follow the Go ABI; it expects its
463 // arguments in registers.
464 TEXT callRet<>(SB), NOSPLIT, $40-0
465 NO_LOCAL_POINTERS
466 MOVD R7, FIXED_FRAME+0(R1)
467 MOVD R3, FIXED_FRAME+8(R1)
468 MOVD R5, FIXED_FRAME+16(R1)
469 MOVD R4, FIXED_FRAME+24(R1)
470 MOVD R20, FIXED_FRAME+32(R1)
471 BL runtime·reflectcallmove(SB)
472 RET
473
474 CALLFN(·call16, 16)
475 CALLFN(·call32, 32)
476 CALLFN(·call64, 64)
477 CALLFN(·call128, 128)
478 CALLFN(·call256, 256)
479 CALLFN(·call512, 512)
480 CALLFN(·call1024, 1024)
481 CALLFN(·call2048, 2048)
482 CALLFN(·call4096, 4096)
483 CALLFN(·call8192, 8192)
484 CALLFN(·call16384, 16384)
485 CALLFN(·call32768, 32768)
486 CALLFN(·call65536, 65536)
487 CALLFN(·call131072, 131072)
488 CALLFN(·call262144, 262144)
489 CALLFN(·call524288, 524288)
490 CALLFN(·call1048576, 1048576)
491 CALLFN(·call2097152, 2097152)
492 CALLFN(·call4194304, 4194304)
493 CALLFN(·call8388608, 8388608)
494 CALLFN(·call16777216, 16777216)
495 CALLFN(·call33554432, 33554432)
496 CALLFN(·call67108864, 67108864)
497 CALLFN(·call134217728, 134217728)
498 CALLFN(·call268435456, 268435456)
499 CALLFN(·call536870912, 536870912)
500 CALLFN(·call1073741824, 1073741824)
501
502 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
503 MOVW cycles+0(FP), R7
504 // POWER does not have a pause/yield instruction equivalent.
505 // Instead, we can lower the program priority by setting the
506 // Program Priority Register prior to the wait loop and set it
507 // back to default afterwards. On Linux, the default priority is
508 // medium-low. For details, see page 837 of the ISA 3.0.
509 OR R1, R1, R1 // Set PPR priority to low
510 again:
511 SUB $1, R7
512 CMP $0, R7
513 BNE again
514 OR R6, R6, R6 // Set PPR priority back to medium-low
515 RET
516
517 // Save state of caller into g->sched,
518 // but using fake PC from systemstack_switch.
519 // Must only be called from functions with no locals ($0)
520 // or else unwinding from systemstack_switch is incorrect.
521 // Smashes R31.
522 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
523 MOVD $runtime·systemstack_switch(SB), R31
524 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
525 MOVD R31, (g_sched+gobuf_pc)(g)
526 MOVD R1, (g_sched+gobuf_sp)(g)
527 MOVD R0, (g_sched+gobuf_lr)(g)
528 MOVD R0, (g_sched+gobuf_ret)(g)
529 // Assert ctxt is zero. See func save.
530 MOVD (g_sched+gobuf_ctxt)(g), R31
531 CMP R0, R31
532 BEQ 2(PC)
533 BL runtime·abort(SB)
534 RET
535
536 #ifdef GOOS_aix
537 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
538 #else
539 #define asmcgocallSaveOffset cgoCalleeStackSize
540 #endif
541
542 // func asmcgocall(fn, arg unsafe.Pointer) int32
543 // Call fn(arg) on the scheduler stack,
544 // aligned appropriately for the gcc ABI.
545 // See cgocall.go for more details.
546 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
547 MOVD fn+0(FP), R3
548 MOVD arg+8(FP), R4
549
550 MOVD R1, R7 // save original stack pointer
551 MOVD g, R5
552
553 // Figure out if we need to switch to m->g0 stack.
554 // We get called to create new OS threads too, and those
555 // come in on the m->g0 stack already. Or we might already
556 // be on the m->gsignal stack.
557 MOVD g_m(g), R8
558 MOVD m_gsignal(R8), R6
559 CMP R6, g
560 BEQ g0
561 MOVD m_g0(R8), R6
562 CMP R6, g
563 BEQ g0
564 BL gosave_systemstack_switch<>(SB)
565 MOVD R6, g
566 BL runtime·save_g(SB)
567 MOVD (g_sched+gobuf_sp)(g), R1
568
569 // Now on a scheduling stack (a pthread-created stack).
570 g0:
571 #ifdef GOOS_aix
572 // Create a fake LR to improve backtrace.
573 MOVD $runtime·asmcgocall(SB), R6
574 MOVD R6, 16(R1)
575 // AIX also save one argument on the stack.
576 SUB $8, R1
577 #endif
578 // Save room for two of our pointers, plus the callee
579 // save area that lives on the caller stack.
580 SUB $(asmcgocallSaveOffset+16), R1
581 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
582 MOVD R5, (asmcgocallSaveOffset+8)(R1)// save old g on stack
583 MOVD (g_stack+stack_hi)(R5), R5
584 SUB R7, R5
585 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
586 #ifdef GOOS_aix
587 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
588 #else
589 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
590 #endif
591 // This is a "global call", so put the global entry point in r12
592 MOVD R3, R12
593
594 #ifdef GOARCH_ppc64
595 // ppc64 use elf ABI v1. we must get the real entry address from
596 // first slot of the function descriptor before call.
597 // Same for AIX.
598 MOVD 8(R12), R2
599 MOVD (R12), R12
600 #endif
601 MOVD R12, CTR
602 MOVD R4, R3 // arg in r3
603 BL (CTR)
604 // C code can clobber R0, so set it back to 0. F27-F31 are
605 // callee save, so we don't need to recover those.
606 XOR R0, R0
607 // Restore g, stack pointer, toc pointer.
608 // R3 is errno, so don't touch it
609 MOVD (asmcgocallSaveOffset+8)(R1), g
610 MOVD (g_stack+stack_hi)(g), R5
611 MOVD asmcgocallSaveOffset(R1), R6
612 SUB R6, R5
613 #ifndef GOOS_aix
614 MOVD 24(R5), R2
615 #endif
616 MOVD R5, R1
617 BL runtime·save_g(SB)
618
619 MOVW R3, ret+16(FP)
620 RET
621
622 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
623 // See cgocall.go for more details.
624 TEXT ·cgocallback(SB),NOSPLIT,$24-24
625 NO_LOCAL_POINTERS
626
627 // Load m and g from thread-local storage.
628 MOVBZ runtime·iscgo(SB), R3
629 CMP R3, $0
630 BEQ nocgo
631 BL runtime·load_g(SB)
632 nocgo:
633
634 // If g is nil, Go did not create the current thread.
635 // Call needm to obtain one for temporary use.
636 // In this case, we're running on the thread stack, so there's
637 // lots of space, but the linker doesn't know. Hide the call from
638 // the linker analysis by using an indirect call.
639 CMP g, $0
640 BEQ needm
641
642 MOVD g_m(g), R8
643 MOVD R8, savedm-8(SP)
644 BR havem
645
646 needm:
647 MOVD g, savedm-8(SP) // g is zero, so is m.
648 MOVD $runtime·needm(SB), R12
649 MOVD R12, CTR
650 BL (CTR)
651
652 // Set m->sched.sp = SP, so that if a panic happens
653 // during the function we are about to execute, it will
654 // have a valid SP to run on the g0 stack.
655 // The next few lines (after the havem label)
656 // will save this SP onto the stack and then write
657 // the same SP back to m->sched.sp. That seems redundant,
658 // but if an unrecovered panic happens, unwindm will
659 // restore the g->sched.sp from the stack location
660 // and then systemstack will try to use it. If we don't set it here,
661 // that restored SP will be uninitialized (typically 0) and
662 // will not be usable.
663 MOVD g_m(g), R8
664 MOVD m_g0(R8), R3
665 MOVD R1, (g_sched+gobuf_sp)(R3)
666
667 havem:
668 // Now there's a valid m, and we're running on its m->g0.
669 // Save current m->g0->sched.sp on stack and then set it to SP.
670 // Save current sp in m->g0->sched.sp in preparation for
671 // switch back to m->curg stack.
672 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
673 MOVD m_g0(R8), R3
674 MOVD (g_sched+gobuf_sp)(R3), R4
675 MOVD R4, savedsp-24(SP) // must match frame size
676 MOVD R1, (g_sched+gobuf_sp)(R3)
677
678 // Switch to m->curg stack and call runtime.cgocallbackg.
679 // Because we are taking over the execution of m->curg
680 // but *not* resuming what had been running, we need to
681 // save that information (m->curg->sched) so we can restore it.
682 // We can restore m->curg->sched.sp easily, because calling
683 // runtime.cgocallbackg leaves SP unchanged upon return.
684 // To save m->curg->sched.pc, we push it onto the curg stack and
685 // open a frame the same size as cgocallback's g0 frame.
686 // Once we switch to the curg stack, the pushed PC will appear
687 // to be the return PC of cgocallback, so that the traceback
688 // will seamlessly trace back into the earlier calls.
689 MOVD m_curg(R8), g
690 BL runtime·save_g(SB)
691 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
692 MOVD (g_sched+gobuf_pc)(g), R5
693 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
694 // Gather our arguments into registers.
695 MOVD fn+0(FP), R5
696 MOVD frame+8(FP), R6
697 MOVD ctxt+16(FP), R7
698 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
699 MOVD R5, FIXED_FRAME+0(R1)
700 MOVD R6, FIXED_FRAME+8(R1)
701 MOVD R7, FIXED_FRAME+16(R1)
702
703 MOVD $runtime·cgocallbackg(SB), R12
704 MOVD R12, CTR
705 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
706
707 // Restore g->sched (== m->curg->sched) from saved values.
708 MOVD 0(R1), R5
709 MOVD R5, (g_sched+gobuf_pc)(g)
710 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
711 MOVD R4, (g_sched+gobuf_sp)(g)
712
713 // Switch back to m->g0's stack and restore m->g0->sched.sp.
714 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
715 // so we do not have to restore it.)
716 MOVD g_m(g), R8
717 MOVD m_g0(R8), g
718 BL runtime·save_g(SB)
719 MOVD (g_sched+gobuf_sp)(g), R1
720 MOVD savedsp-24(SP), R4 // must match frame size
721 MOVD R4, (g_sched+gobuf_sp)(g)
722
723 // If the m on entry was nil, we called needm above to borrow an m
724 // for the duration of the call. Since the call is over, return it with dropm.
725 MOVD savedm-8(SP), R6
726 CMP R6, $0
727 BNE droppedm
728 MOVD $runtime·dropm(SB), R12
729 MOVD R12, CTR
730 BL (CTR)
731 droppedm:
732
733 // Done!
734 RET
735
736 // void setg(G*); set g. for use by needm.
737 TEXT runtime·setg(SB), NOSPLIT, $0-8
738 MOVD gg+0(FP), g
739 // This only happens if iscgo, so jump straight to save_g
740 BL runtime·save_g(SB)
741 RET
742
743 #ifdef GOARCH_ppc64
744 #ifdef GOOS_aix
745 DATA setg_gcc<>+0(SB)/8, $_setg_gcc<>(SB)
746 DATA setg_gcc<>+8(SB)/8, $TOC(SB)
747 DATA setg_gcc<>+16(SB)/8, $0
748 GLOBL setg_gcc<>(SB), NOPTR, $24
749 #else
750 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
751 DWORD $_setg_gcc<>(SB)
752 DWORD $0
753 DWORD $0
754 #endif
755 #endif
756
757 // void setg_gcc(G*); set g in C TLS.
758 // Must obey the gcc calling convention.
759 #ifdef GOARCH_ppc64le
760 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
761 #else
762 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
763 #endif
764 // The standard prologue clobbers R31, which is callee-save in
765 // the C ABI, so we have to use $-8-0 and save LR ourselves.
766 MOVD LR, R4
767 // Also save g and R31, since they're callee-save in C ABI
768 MOVD R31, R5
769 MOVD g, R6
770
771 MOVD R3, g
772 BL runtime·save_g(SB)
773
774 MOVD R6, g
775 MOVD R5, R31
776 MOVD R4, LR
777 RET
778
779 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
780 MOVW (R0), R0
781 UNDEF
782
783 #define TBR 268
784
785 // int64 runtime·cputicks(void)
786 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
787 MOVD SPR(TBR), R3
788 MOVD R3, ret+0(FP)
789 RET
790
791 #ifdef GOEXPERIMENT_regabireflect
792 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
793 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
794 MOVD R3, 0(R20)
795 MOVD R4, 8(R20)
796 MOVD R5, 16(R20)
797 MOVD R6, 24(R20)
798 MOVD R7, 32(R20)
799 MOVD R8, 40(R20)
800 MOVD R9, 48(R20)
801 MOVD R10, 56(R20)
802 MOVD R14, 64(R20)
803 MOVD R15, 72(R20)
804 MOVD R16, 80(R20)
805 MOVD R17, 88(R20)
806 FMOVD F1, 96(R20)
807 FMOVD F2, 104(R20)
808 FMOVD F3, 112(R20)
809 FMOVD F4, 120(R20)
810 FMOVD F5, 128(R20)
811 FMOVD F6, 136(R20)
812 FMOVD F7, 144(R20)
813 FMOVD F8, 152(R20)
814 FMOVD F9, 160(R20)
815 FMOVD F10, 168(R20)
816 FMOVD F11, 176(R20)
817 FMOVD F12, 184(R20)
818 RET
819
820 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
821 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
822 MOVD 0(R20), R3
823 MOVD 8(R20), R4
824 MOVD 16(R20), R5
825 MOVD 24(R20), R6
826 MOVD 32(R20), R7
827 MOVD 40(R20), R8
828 MOVD 48(R20), R9
829 MOVD 56(R20), R10
830 MOVD 64(R20), R14
831 MOVD 72(R20), R15
832 MOVD 80(R20), R16
833 MOVD 88(R20), R17
834 FMOVD 96(R20), F1
835 FMOVD 104(R20), F2
836 FMOVD 112(R20), F3
837 FMOVD 120(R20), F4
838 FMOVD 128(R20), F5
839 FMOVD 136(R20), F6
840 FMOVD 144(R20), F7
841 FMOVD 152(R20), F8
842 FMOVD 160(R20), F9
843 FMOVD 168(R20), F10
844 FMOVD 176(R20), F11
845 FMOVD 184(R20), F12
846 RET
847 #else
848
849 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
850 RET
851
852 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
853 RET
854 #endif
855
856 // AES hashing not implemented for ppc64
857 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
858 JMP runtime·memhashFallback<ABIInternal>(SB)
859 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
860 JMP runtime·strhashFallback<ABIInternal>(SB)
861 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
862 JMP runtime·memhash32Fallback<ABIInternal>(SB)
863 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
864 JMP runtime·memhash64Fallback<ABIInternal>(SB)
865
866 TEXT runtime·return0(SB), NOSPLIT, $0
867 MOVW $0, R3
868 RET
869
870 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
871 // Must obey the gcc calling convention.
872 #ifdef GOOS_aix
873 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
874 // be a longcall in order to prevent trampolines from ld.
875 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
876 #else
877 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
878 #endif
879 // g (R30) and R31 are callee-save in the C ABI, so save them
880 MOVD g, R4
881 MOVD R31, R5
882 MOVD LR, R6
883
884 BL runtime·load_g(SB) // clobbers g (R30), R31
885 MOVD g_m(g), R3
886 MOVD m_curg(R3), R3
887 MOVD (g_stack+stack_hi)(R3), R3
888
889 MOVD R4, g
890 MOVD R5, R31
891 MOVD R6, LR
892 RET
893
894 // The top-most function running on a goroutine
895 // returns to goexit+PCQuantum.
896 //
897 // When dynamically linking Go, it can be returned to from a function
898 // implemented in a different module and so needs to reload the TOC pointer
899 // from the stack (although this function declares that it does not set up x-a
900 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
901 // pointer in the correct place).
902 // goexit+_PCQuantum is halfway through the usual global entry point prologue
903 // that derives r2 from r12 which is a bit silly, but not harmful.
904 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
905 MOVD 24(R1), R2
906 BL runtime·goexit1(SB) // does not return
907 // traceback from goexit1 must hit code range of goexit
908 MOVD R0, R0 // NOP
909
910 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
911 // module containing runtime) to the frame that goexit will execute in when
912 // the goroutine exits. It's implemented in assembly mainly because that's the
913 // easiest way to get access to R2.
914 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
915 MOVD sp+0(FP), R3
916 MOVD R2, 24(R3)
917 RET
918
919 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
920 ADD $-8, R1
921 MOVD R31, 0(R1)
922 MOVD runtime·lastmoduledatap(SB), R4
923 MOVD R3, moduledata_next(R4)
924 MOVD R3, runtime·lastmoduledatap(SB)
925 MOVD 0(R1), R31
926 ADD $8, R1
927 RET
928
929 TEXT ·checkASM(SB),NOSPLIT,$0-1
930 MOVW $1, R3
931 MOVB R3, ret+0(FP)
932 RET
933
934 // gcWriteBarrier performs a heap pointer write and informs the GC.
935 //
936 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
937 // - R20 is the destination of the write
938 // - R21 is the value being written at R20.
939 // It clobbers condition codes.
940 // It does not clobber R0 through R17 (except special registers),
941 // but may clobber any other register, *including* R31.
942 TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$112
943 // The standard prologue clobbers R31.
944 // We use R18 and R19 as scratch registers.
945 MOVD g_m(g), R18
946 MOVD m_p(R18), R18
947 MOVD (p_wbBuf+wbBuf_next)(R18), R19
948 // Increment wbBuf.next position.
949 ADD $16, R19
950 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
951 MOVD (p_wbBuf+wbBuf_end)(R18), R18
952 CMP R18, R19
953 // Record the write.
954 MOVD R21, -16(R19) // Record value
955 MOVD (R20), R18 // TODO: This turns bad writes into bad reads.
956 MOVD R18, -8(R19) // Record *slot
957 // Is the buffer full? (flags set in CMP above)
958 BEQ flush
959 ret:
960 // Do the write.
961 MOVD R21, (R20)
962 RET
963
964 flush:
965 // Save registers R0 through R15 since these were not saved by the caller.
966 // We don't save all registers on ppc64 because it takes too much space.
967 MOVD R20, (FIXED_FRAME+0)(R1) // Also first argument to wbBufFlush
968 MOVD R21, (FIXED_FRAME+8)(R1) // Also second argument to wbBufFlush
969 // R0 is always 0, so no need to spill.
970 // R1 is SP.
971 // R2 is SB.
972 MOVD R3, (FIXED_FRAME+16)(R1)
973 MOVD R4, (FIXED_FRAME+24)(R1)
974 MOVD R5, (FIXED_FRAME+32)(R1)
975 MOVD R6, (FIXED_FRAME+40)(R1)
976 MOVD R7, (FIXED_FRAME+48)(R1)
977 MOVD R8, (FIXED_FRAME+56)(R1)
978 MOVD R9, (FIXED_FRAME+64)(R1)
979 MOVD R10, (FIXED_FRAME+72)(R1)
980 // R11, R12 may be clobbered by external-linker-inserted trampoline
981 // R13 is REGTLS
982 MOVD R14, (FIXED_FRAME+80)(R1)
983 MOVD R15, (FIXED_FRAME+88)(R1)
984 MOVD R16, (FIXED_FRAME+96)(R1)
985 MOVD R17, (FIXED_FRAME+104)(R1)
986
987 // This takes arguments R20 and R21.
988 CALL runtime·wbBufFlush(SB)
989
990 MOVD (FIXED_FRAME+0)(R1), R20
991 MOVD (FIXED_FRAME+8)(R1), R21
992 MOVD (FIXED_FRAME+16)(R1), R3
993 MOVD (FIXED_FRAME+24)(R1), R4
994 MOVD (FIXED_FRAME+32)(R1), R5
995 MOVD (FIXED_FRAME+40)(R1), R6
996 MOVD (FIXED_FRAME+48)(R1), R7
997 MOVD (FIXED_FRAME+56)(R1), R8
998 MOVD (FIXED_FRAME+64)(R1), R9
999 MOVD (FIXED_FRAME+72)(R1), R10
1000 MOVD (FIXED_FRAME+80)(R1), R14
1001 MOVD (FIXED_FRAME+88)(R1), R15
1002 MOVD (FIXED_FRAME+96)(R1), R16
1003 MOVD (FIXED_FRAME+104)(R1), R17
1004 JMP ret
1005
1006 // Note: these functions use a special calling convention to save generated code space.
1007 // Arguments are passed in registers, but the space for those arguments are allocated
1008 // in the caller's stack frame. These stubs write the args into that stack space and
1009 // then tail call to the corresponding runtime handler.
1010 // The tail call makes these stubs disappear in backtraces.
1011 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1012 #ifndef GOEXPERIMENT_regabiargs
1013 MOVD R3, x+0(FP)
1014 MOVD R4, y+8(FP)
1015 #endif
1016 JMP runtime·goPanicIndex<ABIInternal>(SB)
1017 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1018 #ifndef GOEXPERIMENT_regabiargs
1019 MOVD R3, x+0(FP)
1020 MOVD R4, y+8(FP)
1021 #endif
1022 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1023 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1024 #ifdef GOEXPERIMENT_regabiargs
1025 MOVD R4, R3
1026 MOVD R5, R4
1027 #else
1028 MOVD R4, x+0(FP)
1029 MOVD R5, y+8(FP)
1030 #endif
1031 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1032 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1033 #ifdef GOEXPERIMENT_regabiargs
1034 MOVD R4, R3
1035 MOVD R5, R4
1036 #else
1037 MOVD R4, x+0(FP)
1038 MOVD R5, y+8(FP)
1039 #endif
1040 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1041 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1042 #ifdef GOEXPERIMENT_regabiargs
1043 MOVD R4, R3
1044 MOVD R5, R4
1045 #else
1046 MOVD R4, x+0(FP)
1047 MOVD R5, y+8(FP)
1048 #endif
1049 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1050 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1051 #ifdef GOEXPERIMENT_regabiargs
1052 MOVD R4, R3
1053 MOVD R5, R4
1054 #else
1055 MOVD R4, x+0(FP)
1056 MOVD R5, y+8(FP)
1057 #endif
1058 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1059 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1060 #ifndef GOEXPERIMENT_regabiargs
1061 MOVD R3, x+0(FP)
1062 MOVD R4, y+8(FP)
1063 #endif
1064 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1065 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1066 #ifndef GOEXPERIMENT_regabiargs
1067 MOVD R3, x+0(FP)
1068 MOVD R4, y+8(FP)
1069 #endif
1070 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1071 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1072 #ifdef GOEXPERIMENT_regabiargs
1073 MOVD R5, R3
1074 MOVD R6, R4
1075 #else
1076 MOVD R5, x+0(FP)
1077 MOVD R6, y+8(FP)
1078 #endif
1079 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1080 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1081 #ifdef GOEXPERIMENT_regabiargs
1082 MOVD R5, R3
1083 MOVD R6, R4
1084 #else
1085 MOVD R5, x+0(FP)
1086 MOVD R6, y+8(FP)
1087 #endif
1088 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1089 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1090 #ifdef GOEXPERIMENT_regabiargs
1091 MOVD R5, R3
1092 MOVD R6, R4
1093 #else
1094 MOVD R5, x+0(FP)
1095 MOVD R6, y+8(FP)
1096 #endif
1097 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1098 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1099 #ifdef GOEXPERIMENT_regabiargs
1100 MOVD R5, R3
1101 MOVD R6, R4
1102 #else
1103 MOVD R5, x+0(FP)
1104 MOVD R6, y+8(FP)
1105 #endif
1106 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1107 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1108 #ifdef GOEXPERIMENT_regabiargs
1109 MOVD R4, R3
1110 MOVD R5, R4
1111 #else
1112 MOVD R4, x+0(FP)
1113 MOVD R5, y+8(FP)
1114 #endif
1115 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1116 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1117 #ifdef GOEXPERIMENT_regabiargs
1118 MOVD R4, R3
1119 MOVD R5, R4
1120 #else
1121 MOVD R4, x+0(FP)
1122 MOVD R5, y+8(FP)
1123 #endif
1124 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1125 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1126 #ifndef GOEXPERIMENT_regabiargs
1127 MOVD R3, x+0(FP)
1128 MOVD R4, y+8(FP)
1129 #endif
1130 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1131 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1132 #ifndef GOEXPERIMENT_regabiargs
1133 MOVD R3, x+0(FP)
1134 MOVD R4, y+8(FP)
1135 #endif
1136 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1137 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1138 #ifdef GOEXPERIMENT_regabiargs
1139 MOVD R5, R3
1140 MOVD R6, R4
1141 #else
1142 MOVD R5, x+0(FP)
1143 MOVD R6, y+8(FP)
1144 #endif
1145 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1146
View as plain text