Source file src/cmd/compile/internal/ssa/expand_calls.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/abi"
     9  	"cmd/compile/internal/base"
    10  	"cmd/compile/internal/ir"
    11  	"cmd/compile/internal/types"
    12  	"cmd/internal/src"
    13  	"fmt"
    14  	"sort"
    15  )
    16  
    17  type selKey struct {
    18  	from          *Value // what is selected from
    19  	offsetOrIndex int64  // whatever is appropriate for the selector
    20  	size          int64
    21  	typ           *types.Type
    22  }
    23  
    24  type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1.
    25  
    26  func isBlockMultiValueExit(b *Block) bool {
    27  	return (b.Kind == BlockRet || b.Kind == BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == OpMakeResult
    28  }
    29  
    30  func badVal(s string, v *Value) error {
    31  	return fmt.Errorf("%s %s", s, v.LongString())
    32  }
    33  
    34  // removeTrivialWrapperTypes unwraps layers of
    35  // struct { singleField SomeType } and [1]SomeType
    36  // until a non-wrapper type is reached.  This is useful
    37  // for working with assignments to/from interface data
    38  // fields (either second operand to OpIMake or OpIData)
    39  // where the wrapping or type conversion can be elided
    40  // because of type conversions/assertions in source code
    41  // that do not appear in SSA.
    42  func removeTrivialWrapperTypes(t *types.Type) *types.Type {
    43  	for {
    44  		if t.IsStruct() && t.NumFields() == 1 {
    45  			t = t.Field(0).Type
    46  			continue
    47  		}
    48  		if t.IsArray() && t.NumElem() == 1 {
    49  			t = t.Elem()
    50  			continue
    51  		}
    52  		break
    53  	}
    54  	return t
    55  }
    56  
    57  // A registerCursor tracks which register is used for an Arg or regValues, or a piece of such.
    58  type registerCursor struct {
    59  	// TODO(register args) convert this to a generalized target cursor.
    60  	storeDest *Value // if there are no register targets, then this is the base of the store.
    61  	regsLen   int    // the number of registers available for this Arg/result (which is all in registers or not at all)
    62  	nextSlice Abi1RO // the next register/register-slice offset
    63  	config    *abi.ABIConfig
    64  	regValues *[]*Value // values assigned to registers accumulate here
    65  }
    66  
    67  func (rc *registerCursor) String() string {
    68  	dest := "<none>"
    69  	if rc.storeDest != nil {
    70  		dest = rc.storeDest.String()
    71  	}
    72  	regs := "<none>"
    73  	if rc.regValues != nil {
    74  		regs = ""
    75  		for i, x := range *rc.regValues {
    76  			if i > 0 {
    77  				regs = regs + "; "
    78  			}
    79  			regs = regs + x.LongString()
    80  		}
    81  	}
    82  	// not printing the config because that has not been useful
    83  	return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, rc.regsLen, rc.nextSlice, regs)
    84  }
    85  
    86  // next effectively post-increments the register cursor; the receiver is advanced,
    87  // the old value is returned.
    88  func (c *registerCursor) next(t *types.Type) registerCursor {
    89  	rc := *c
    90  	if int(c.nextSlice) < c.regsLen {
    91  		w := c.config.NumParamRegs(t)
    92  		c.nextSlice += Abi1RO(w)
    93  	}
    94  	return rc
    95  }
    96  
    97  // plus returns a register cursor offset from the original, without modifying the original.
    98  func (c *registerCursor) plus(regWidth Abi1RO) registerCursor {
    99  	rc := *c
   100  	rc.nextSlice += regWidth
   101  	return rc
   102  }
   103  
   104  const (
   105  	// Register offsets for fields of built-in aggregate types; the ones not listed are zero.
   106  	RO_complex_imag = 1
   107  	RO_string_len   = 1
   108  	RO_slice_len    = 1
   109  	RO_slice_cap    = 2
   110  	RO_iface_data   = 1
   111  )
   112  
   113  func (x *expandState) regWidth(t *types.Type) Abi1RO {
   114  	return Abi1RO(x.abi1.NumParamRegs(t))
   115  }
   116  
   117  // regOffset returns the register offset of the i'th element of type t
   118  func (x *expandState) regOffset(t *types.Type, i int) Abi1RO {
   119  	// TODO maybe cache this in a map if profiling recommends.
   120  	if i == 0 {
   121  		return 0
   122  	}
   123  	if t.IsArray() {
   124  		return Abi1RO(i) * x.regWidth(t.Elem())
   125  	}
   126  	if t.IsStruct() {
   127  		k := Abi1RO(0)
   128  		for j := 0; j < i; j++ {
   129  			k += x.regWidth(t.FieldType(j))
   130  		}
   131  		return k
   132  	}
   133  	panic("Haven't implemented this case yet, do I need to?")
   134  }
   135  
   136  // at returns the register cursor for component i of t, where the first
   137  // component is numbered 0.
   138  func (c *registerCursor) at(t *types.Type, i int) registerCursor {
   139  	rc := *c
   140  	if i == 0 || c.regsLen == 0 {
   141  		return rc
   142  	}
   143  	if t.IsArray() {
   144  		w := c.config.NumParamRegs(t.Elem())
   145  		rc.nextSlice += Abi1RO(i * w)
   146  		return rc
   147  	}
   148  	if t.IsStruct() {
   149  		for j := 0; j < i; j++ {
   150  			rc.next(t.FieldType(j))
   151  		}
   152  		return rc
   153  	}
   154  	panic("Haven't implemented this case yet, do I need to?")
   155  }
   156  
   157  func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*Value, storeDest *Value) {
   158  	c.regsLen = len(regs)
   159  	c.nextSlice = 0
   160  	if len(regs) == 0 {
   161  		c.storeDest = storeDest // only save this if there are no registers, will explode if misused.
   162  		return
   163  	}
   164  	c.config = info.Config()
   165  	c.regValues = result
   166  }
   167  
   168  func (c *registerCursor) addArg(v *Value) {
   169  	*c.regValues = append(*c.regValues, v)
   170  }
   171  
   172  func (c *registerCursor) hasRegs() bool {
   173  	return c.regsLen > 0
   174  }
   175  
   176  type expandState struct {
   177  	f                  *Func
   178  	abi1               *abi.ABIConfig
   179  	debug              int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both)
   180  	canSSAType         func(*types.Type) bool
   181  	regSize            int64
   182  	sp                 *Value
   183  	typs               *Types
   184  	ptrSize            int64
   185  	hiOffset           int64
   186  	lowOffset          int64
   187  	hiRo               Abi1RO
   188  	loRo               Abi1RO
   189  	namedSelects       map[*Value][]namedVal
   190  	sdom               SparseTree
   191  	commonSelectors    map[selKey]*Value // used to de-dupe selectors
   192  	commonArgs         map[selKey]*Value // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg
   193  	memForCall         map[ID]*Value     // For a call, need to know the unique selector that gets the mem.
   194  	transformedSelects map[ID]bool       // OpSelectN after rewriting, either created or renumbered.
   195  	indentLevel        int               // Indentation for debugging recursion
   196  }
   197  
   198  // intPairTypes returns the pair of 32-bit int types needed to encode a 64-bit integer type on a target
   199  // that has no 64-bit integer registers.
   200  func (x *expandState) intPairTypes(et types.Kind) (tHi, tLo *types.Type) {
   201  	tHi = x.typs.UInt32
   202  	if et == types.TINT64 {
   203  		tHi = x.typs.Int32
   204  	}
   205  	tLo = x.typs.UInt32
   206  	return
   207  }
   208  
   209  // isAlreadyExpandedAggregateType returns whether a type is an SSA-able "aggregate" (multiple register) type
   210  // that was expanded in an earlier phase (currently, expand_calls is intended to run after decomposeBuiltin,
   211  // so this is all aggregate types -- small struct and array, complex, interface, string, slice, and 64-bit
   212  // integer on 32-bit).
   213  func (x *expandState) isAlreadyExpandedAggregateType(t *types.Type) bool {
   214  	if !x.canSSAType(t) {
   215  		return false
   216  	}
   217  	return t.IsStruct() || t.IsArray() || t.IsComplex() || t.IsInterface() || t.IsString() || t.IsSlice() ||
   218  		(t.Size() > x.regSize && (t.IsInteger() || (x.f.Config.SoftFloat && t.IsFloat())))
   219  }
   220  
   221  // offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP
   222  // TODO should also optimize offsets from SB?
   223  func (x *expandState) offsetFrom(b *Block, from *Value, offset int64, pt *types.Type) *Value {
   224  	ft := from.Type
   225  	if offset == 0 {
   226  		if ft == pt {
   227  			return from
   228  		}
   229  		// This captures common, (apparently) safe cases.  The unsafe cases involve ft == uintptr
   230  		if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() {
   231  			return from
   232  		}
   233  	}
   234  	// Simplify, canonicalize
   235  	for from.Op == OpOffPtr {
   236  		offset += from.AuxInt
   237  		from = from.Args[0]
   238  	}
   239  	if from == x.sp {
   240  		return x.f.ConstOffPtrSP(pt, offset, x.sp)
   241  	}
   242  	return b.NewValue1I(from.Pos.WithNotStmt(), OpOffPtr, pt, offset, from)
   243  }
   244  
   245  // splitSlots splits one "field" (specified by sfx, offset, and ty) out of the LocalSlots in ls and returns the new LocalSlots this generates.
   246  func (x *expandState) splitSlots(ls []*LocalSlot, sfx string, offset int64, ty *types.Type) []*LocalSlot {
   247  	var locs []*LocalSlot
   248  	for i := range ls {
   249  		locs = append(locs, x.f.SplitSlot(ls[i], sfx, offset, ty))
   250  	}
   251  	return locs
   252  }
   253  
   254  // prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg.
   255  func (x *expandState) prAssignForArg(v *Value) *abi.ABIParamAssignment {
   256  	if v.Op != OpArg {
   257  		panic(badVal("Wanted OpArg, instead saw", v))
   258  	}
   259  	return ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name))
   260  }
   261  
   262  // ParamAssignmentForArgName returns the ABIParamAssignment for f's arg with matching name.
   263  func ParamAssignmentForArgName(f *Func, name *ir.Name) *abi.ABIParamAssignment {
   264  	abiInfo := f.OwnAux.abiInfo
   265  	ip := abiInfo.InParams()
   266  	for i, a := range ip {
   267  		if a.Name == name {
   268  			return &ip[i]
   269  		}
   270  	}
   271  	panic(fmt.Errorf("Did not match param %v in prInfo %+v", name, abiInfo.InParams()))
   272  }
   273  
   274  // indent increments (or decrements) the indentation.
   275  func (x *expandState) indent(n int) {
   276  	x.indentLevel += n
   277  }
   278  
   279  // Printf does an indented fmt.Printf on te format and args.
   280  func (x *expandState) Printf(format string, a ...interface{}) (n int, err error) {
   281  	if x.indentLevel > 0 {
   282  		fmt.Printf("%[1]*s", x.indentLevel, "")
   283  	}
   284  	return fmt.Printf(format, a...)
   285  }
   286  
   287  // Calls that need lowering have some number of inputs, including a memory input,
   288  // and produce a tuple of (value1, value2, ..., mem) where valueK may or may not be SSA-able.
   289  
   290  // With the current ABI those inputs need to be converted into stores to memory,
   291  // rethreading the call's memory input to the first, and the new call now receiving the last.
   292  
   293  // With the current ABI, the outputs need to be converted to loads, which will all use the call's
   294  // memory output as their input.
   295  
   296  // rewriteSelect recursively walks from leaf selector to a root (OpSelectN, OpLoad, OpArg)
   297  // through a chain of Struct/Array/builtin Select operations.  If the chain of selectors does not
   298  // end in an expected root, it does nothing (this can happen depending on compiler phase ordering).
   299  // The "leaf" provides the type, the root supplies the container, and the leaf-to-root path
   300  // accumulates the offset.
   301  // It emits the code necessary to implement the leaf select operation that leads to the root.
   302  //
   303  // TODO when registers really arrive, must also decompose anything split across two registers or registers and memory.
   304  func (x *expandState) rewriteSelect(leaf *Value, selector *Value, offset int64, regOffset Abi1RO) []*LocalSlot {
   305  	if x.debug > 1 {
   306  		x.indent(3)
   307  		defer x.indent(-3)
   308  		x.Printf("rewriteSelect(%s; %s; memOff=%d; regOff=%d)\n", leaf.LongString(), selector.LongString(), offset, regOffset)
   309  	}
   310  	var locs []*LocalSlot
   311  	leafType := leaf.Type
   312  	if len(selector.Args) > 0 {
   313  		w := selector.Args[0]
   314  		if w.Op == OpCopy {
   315  			for w.Op == OpCopy {
   316  				w = w.Args[0]
   317  			}
   318  			selector.SetArg(0, w)
   319  		}
   320  	}
   321  	switch selector.Op {
   322  	case OpArgIntReg, OpArgFloatReg:
   323  		if leafType == selector.Type { // OpIData leads us here, sometimes.
   324  			leaf.copyOf(selector)
   325  		} else {
   326  			x.f.Fatalf("Unexpected %s type, selector=%s, leaf=%s\n", selector.Op.String(), selector.LongString(), leaf.LongString())
   327  		}
   328  		if x.debug > 1 {
   329  			x.Printf("---%s, break\n", selector.Op.String())
   330  		}
   331  	case OpArg:
   332  		if !x.isAlreadyExpandedAggregateType(selector.Type) {
   333  			if leafType == selector.Type { // OpIData leads us here, sometimes.
   334  				x.newArgToMemOrRegs(selector, leaf, offset, regOffset, leafType, leaf.Pos)
   335  			} else {
   336  				x.f.Fatalf("Unexpected OpArg type, selector=%s, leaf=%s\n", selector.LongString(), leaf.LongString())
   337  			}
   338  			if x.debug > 1 {
   339  				x.Printf("---OpArg, break\n")
   340  			}
   341  			break
   342  		}
   343  		switch leaf.Op {
   344  		case OpIData, OpStructSelect, OpArraySelect:
   345  			leafType = removeTrivialWrapperTypes(leaf.Type)
   346  		}
   347  		x.newArgToMemOrRegs(selector, leaf, offset, regOffset, leafType, leaf.Pos)
   348  
   349  		for _, s := range x.namedSelects[selector] {
   350  			locs = append(locs, x.f.Names[s.locIndex])
   351  		}
   352  
   353  	case OpLoad: // We end up here because of IData of immediate structures.
   354  		// Failure case:
   355  		// (note the failure case is very rare; w/o this case, make.bash and run.bash both pass, as well as
   356  		// the hard cases of building {syscall,math,math/cmplx,math/bits,go/constant} on ppc64le and mips-softfloat).
   357  		//
   358  		// GOSSAFUNC='(*dumper).dump' go build -gcflags=-l -tags=math_big_pure_go cmd/compile/internal/gc
   359  		// cmd/compile/internal/gc/dump.go:136:14: internal compiler error: '(*dumper).dump': not lowered: v827, StructSelect PTR PTR
   360  		// b2: ← b1
   361  		// v20 (+142) = StaticLECall <interface {},mem> {AuxCall{reflect.Value.Interface([reflect.Value,0])[interface {},24]}} [40] v8 v1
   362  		// v21 (142) = SelectN <mem> [1] v20
   363  		// v22 (142) = SelectN <interface {}> [0] v20
   364  		// b15: ← b8
   365  		// v71 (+143) = IData <Nodes> v22 (v[Nodes])
   366  		// v73 (+146) = StaticLECall <[]*Node,mem> {AuxCall{"".Nodes.Slice([Nodes,0])[[]*Node,8]}} [32] v71 v21
   367  		//
   368  		// translates (w/o the "case OpLoad:" above) to:
   369  		//
   370  		// b2: ← b1
   371  		// v20 (+142) = StaticCall <mem> {AuxCall{reflect.Value.Interface([reflect.Value,0])[interface {},24]}} [40] v715
   372  		// v23 (142) = Load <*uintptr> v19 v20
   373  		// v823 (142) = IsNonNil <bool> v23
   374  		// v67 (+143) = Load <*[]*Node> v880 v20
   375  		// b15: ← b8
   376  		// v827 (146) = StructSelect <*[]*Node> [0] v67
   377  		// v846 (146) = Store <mem> {*[]*Node} v769 v827 v20
   378  		// v73 (+146) = StaticCall <mem> {AuxCall{"".Nodes.Slice([Nodes,0])[[]*Node,8]}} [32] v846
   379  		// i.e., the struct select is generated and remains in because it is not applied to an actual structure.
   380  		// The OpLoad was created to load the single field of the IData
   381  		// This case removes that StructSelect.
   382  		if leafType != selector.Type {
   383  			if x.f.Config.SoftFloat && selector.Type.IsFloat() {
   384  				if x.debug > 1 {
   385  					x.Printf("---OpLoad, break\n")
   386  				}
   387  				break // softfloat pass will take care of that
   388  			}
   389  			x.f.Fatalf("Unexpected Load as selector, leaf=%s, selector=%s\n", leaf.LongString(), selector.LongString())
   390  		}
   391  		leaf.copyOf(selector)
   392  		for _, s := range x.namedSelects[selector] {
   393  			locs = append(locs, x.f.Names[s.locIndex])
   394  		}
   395  
   396  	case OpSelectN:
   397  		// TODO(register args) result case
   398  		// if applied to Op-mumble-call, the Aux tells us which result, regOffset specifies offset within result.  If a register, should rewrite to OpSelectN for new call.
   399  		// TODO these may be duplicated. Should memoize. Intermediate selectors will go dead, no worries there.
   400  		call := selector.Args[0]
   401  		call0 := call
   402  		aux := call.Aux.(*AuxCall)
   403  		which := selector.AuxInt
   404  		if x.transformedSelects[selector.ID] {
   405  			// This is a minor hack.  Either this select has had its operand adjusted (mem) or
   406  			// it is some other intermediate node that was rewritten to reference a register (not a generic arg).
   407  			// This can occur with chains of selection/indexing from single field/element aggregates.
   408  			leaf.copyOf(selector)
   409  			break
   410  		}
   411  		if which == aux.NResults() { // mem is after the results.
   412  			// rewrite v as a Copy of call -- the replacement call will produce a mem.
   413  			if leaf != selector {
   414  				panic(fmt.Errorf("Unexpected selector of memory, selector=%s, call=%s, leaf=%s", selector.LongString(), call.LongString(), leaf.LongString()))
   415  			}
   416  			if aux.abiInfo == nil {
   417  				panic(badVal("aux.abiInfo nil for call", call))
   418  			}
   419  			if existing := x.memForCall[call.ID]; existing == nil {
   420  				selector.AuxInt = int64(aux.abiInfo.OutRegistersUsed())
   421  				x.memForCall[call.ID] = selector
   422  				x.transformedSelects[selector.ID] = true // operand adjusted
   423  			} else {
   424  				selector.copyOf(existing)
   425  			}
   426  
   427  		} else {
   428  			leafType := removeTrivialWrapperTypes(leaf.Type)
   429  			if x.canSSAType(leafType) {
   430  				pt := types.NewPtr(leafType)
   431  				// Any selection right out of the arg area/registers has to be same Block as call, use call as mem input.
   432  				// Create a "mem" for any loads that need to occur.
   433  				if mem := x.memForCall[call.ID]; mem != nil {
   434  					if mem.Block != call.Block {
   435  						panic(fmt.Errorf("selector and call need to be in same block, selector=%s; call=%s", selector.LongString(), call.LongString()))
   436  					}
   437  					call = mem
   438  				} else {
   439  					mem = call.Block.NewValue1I(call.Pos.WithNotStmt(), OpSelectN, types.TypeMem, int64(aux.abiInfo.OutRegistersUsed()), call)
   440  					x.transformedSelects[mem.ID] = true // select uses post-expansion indexing
   441  					x.memForCall[call.ID] = mem
   442  					call = mem
   443  				}
   444  				outParam := aux.abiInfo.OutParam(int(which))
   445  				if len(outParam.Registers) > 0 {
   446  					firstReg := uint32(0)
   447  					for i := 0; i < int(which); i++ {
   448  						firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers))
   449  					}
   450  					reg := int64(regOffset + Abi1RO(firstReg))
   451  					if leaf.Block == call.Block {
   452  						leaf.reset(OpSelectN)
   453  						leaf.SetArgs1(call0)
   454  						leaf.Type = leafType
   455  						leaf.AuxInt = reg
   456  						x.transformedSelects[leaf.ID] = true // leaf, rewritten to use post-expansion indexing.
   457  					} else {
   458  						w := call.Block.NewValue1I(leaf.Pos, OpSelectN, leafType, reg, call0)
   459  						x.transformedSelects[w.ID] = true // select, using post-expansion indexing.
   460  						leaf.copyOf(w)
   461  					}
   462  				} else {
   463  					off := x.offsetFrom(x.f.Entry, x.sp, offset+aux.OffsetOfResult(which), pt)
   464  					if leaf.Block == call.Block {
   465  						leaf.reset(OpLoad)
   466  						leaf.SetArgs2(off, call)
   467  						leaf.Type = leafType
   468  					} else {
   469  						w := call.Block.NewValue2(leaf.Pos, OpLoad, leafType, off, call)
   470  						leaf.copyOf(w)
   471  						if x.debug > 1 {
   472  							x.Printf("---new %s\n", w.LongString())
   473  						}
   474  					}
   475  				}
   476  				for _, s := range x.namedSelects[selector] {
   477  					locs = append(locs, x.f.Names[s.locIndex])
   478  				}
   479  			} else {
   480  				x.f.Fatalf("Should not have non-SSA-able OpSelectN, selector=%s", selector.LongString())
   481  			}
   482  		}
   483  
   484  	case OpStructSelect:
   485  		w := selector.Args[0]
   486  		var ls []*LocalSlot
   487  		if w.Type.Kind() != types.TSTRUCT { // IData artifact
   488  			ls = x.rewriteSelect(leaf, w, offset, regOffset)
   489  		} else {
   490  			fldi := int(selector.AuxInt)
   491  			ls = x.rewriteSelect(leaf, w, offset+w.Type.FieldOff(fldi), regOffset+x.regOffset(w.Type, fldi))
   492  			if w.Op != OpIData {
   493  				for _, l := range ls {
   494  					locs = append(locs, x.f.SplitStruct(l, int(selector.AuxInt)))
   495  				}
   496  			}
   497  		}
   498  
   499  	case OpArraySelect:
   500  		w := selector.Args[0]
   501  		index := selector.AuxInt
   502  		x.rewriteSelect(leaf, w, offset+selector.Type.Size()*index, regOffset+x.regOffset(w.Type, int(index)))
   503  
   504  	case OpInt64Hi:
   505  		w := selector.Args[0]
   506  		ls := x.rewriteSelect(leaf, w, offset+x.hiOffset, regOffset+x.hiRo)
   507  		locs = x.splitSlots(ls, ".hi", x.hiOffset, leafType)
   508  
   509  	case OpInt64Lo:
   510  		w := selector.Args[0]
   511  		ls := x.rewriteSelect(leaf, w, offset+x.lowOffset, regOffset+x.loRo)
   512  		locs = x.splitSlots(ls, ".lo", x.lowOffset, leafType)
   513  
   514  	case OpStringPtr:
   515  		ls := x.rewriteSelect(leaf, selector.Args[0], offset, regOffset)
   516  		locs = x.splitSlots(ls, ".ptr", 0, x.typs.BytePtr)
   517  
   518  	case OpSlicePtr, OpSlicePtrUnchecked:
   519  		w := selector.Args[0]
   520  		ls := x.rewriteSelect(leaf, w, offset, regOffset)
   521  		locs = x.splitSlots(ls, ".ptr", 0, types.NewPtr(w.Type.Elem()))
   522  
   523  	case OpITab:
   524  		w := selector.Args[0]
   525  		ls := x.rewriteSelect(leaf, w, offset, regOffset)
   526  		sfx := ".itab"
   527  		if w.Type.IsEmptyInterface() {
   528  			sfx = ".type"
   529  		}
   530  		locs = x.splitSlots(ls, sfx, 0, x.typs.Uintptr)
   531  
   532  	case OpComplexReal:
   533  		ls := x.rewriteSelect(leaf, selector.Args[0], offset, regOffset)
   534  		locs = x.splitSlots(ls, ".real", 0, selector.Type)
   535  
   536  	case OpComplexImag:
   537  		ls := x.rewriteSelect(leaf, selector.Args[0], offset+selector.Type.Size(), regOffset+RO_complex_imag) // result is FloatNN, width of result is offset of imaginary part.
   538  		locs = x.splitSlots(ls, ".imag", selector.Type.Size(), selector.Type)
   539  
   540  	case OpStringLen, OpSliceLen:
   541  		ls := x.rewriteSelect(leaf, selector.Args[0], offset+x.ptrSize, regOffset+RO_slice_len)
   542  		locs = x.splitSlots(ls, ".len", x.ptrSize, leafType)
   543  
   544  	case OpIData:
   545  		ls := x.rewriteSelect(leaf, selector.Args[0], offset+x.ptrSize, regOffset+RO_iface_data)
   546  		locs = x.splitSlots(ls, ".data", x.ptrSize, leafType)
   547  
   548  	case OpSliceCap:
   549  		ls := x.rewriteSelect(leaf, selector.Args[0], offset+2*x.ptrSize, regOffset+RO_slice_cap)
   550  		locs = x.splitSlots(ls, ".cap", 2*x.ptrSize, leafType)
   551  
   552  	case OpCopy: // If it's an intermediate result, recurse
   553  		locs = x.rewriteSelect(leaf, selector.Args[0], offset, regOffset)
   554  		for _, s := range x.namedSelects[selector] {
   555  			// this copy may have had its own name, preserve that, too.
   556  			locs = append(locs, x.f.Names[s.locIndex])
   557  		}
   558  
   559  	default:
   560  		// Ignore dead ends. These can occur if this phase is run before decompose builtin (which is not intended, but allowed).
   561  	}
   562  
   563  	return locs
   564  }
   565  
   566  func (x *expandState) rewriteDereference(b *Block, base, a, mem *Value, offset, size int64, typ *types.Type, pos src.XPos) *Value {
   567  	source := a.Args[0]
   568  	dst := x.offsetFrom(b, base, offset, source.Type)
   569  	if a.Uses == 1 && a.Block == b {
   570  		a.reset(OpMove)
   571  		a.Pos = pos
   572  		a.Type = types.TypeMem
   573  		a.Aux = typ
   574  		a.AuxInt = size
   575  		a.SetArgs3(dst, source, mem)
   576  		mem = a
   577  	} else {
   578  		mem = b.NewValue3A(pos, OpMove, types.TypeMem, typ, dst, source, mem)
   579  		mem.AuxInt = size
   580  	}
   581  	return mem
   582  }
   583  
   584  var indexNames [1]string = [1]string{"[0]"}
   585  
   586  // pathTo returns the selection path to the leaf type at offset within container.
   587  // e.g. len(thing.field[0]) => ".field[0].len"
   588  // this is for purposes of generating names ultimately fed to a debugger.
   589  func (x *expandState) pathTo(container, leaf *types.Type, offset int64) string {
   590  	if container == leaf || offset == 0 && container.Size() == leaf.Size() {
   591  		return ""
   592  	}
   593  	path := ""
   594  outer:
   595  	for {
   596  		switch container.Kind() {
   597  		case types.TARRAY:
   598  			container = container.Elem()
   599  			if container.Size() == 0 {
   600  				return path
   601  			}
   602  			i := offset / container.Size()
   603  			offset = offset % container.Size()
   604  			// If a future compiler/ABI supports larger SSA/Arg-able arrays, expand indexNames.
   605  			path = path + indexNames[i]
   606  			continue
   607  		case types.TSTRUCT:
   608  			for i := 0; i < container.NumFields(); i++ {
   609  				fld := container.Field(i)
   610  				if fld.Offset+fld.Type.Size() > offset {
   611  					offset -= fld.Offset
   612  					path += "." + fld.Sym.Name
   613  					container = fld.Type
   614  					continue outer
   615  				}
   616  			}
   617  			return path
   618  		case types.TINT64, types.TUINT64:
   619  			if container.Size() == x.regSize {
   620  				return path
   621  			}
   622  			if offset == x.hiOffset {
   623  				return path + ".hi"
   624  			}
   625  			return path + ".lo"
   626  		case types.TINTER:
   627  			if offset != 0 {
   628  				return path + ".data"
   629  			}
   630  			if container.IsEmptyInterface() {
   631  				return path + ".type"
   632  			}
   633  			return path + ".itab"
   634  
   635  		case types.TSLICE:
   636  			if offset == 2*x.regSize {
   637  				return path + ".cap"
   638  			}
   639  			fallthrough
   640  		case types.TSTRING:
   641  			if offset == 0 {
   642  				return path + ".ptr"
   643  			}
   644  			return path + ".len"
   645  		case types.TCOMPLEX64, types.TCOMPLEX128:
   646  			if offset == 0 {
   647  				return path + ".real"
   648  			}
   649  			return path + ".imag"
   650  		}
   651  		return path
   652  	}
   653  }
   654  
   655  // decomposeArg is a helper for storeArgOrLoad.
   656  // It decomposes a Load or an Arg into smaller parts and returns the new mem.
   657  // If the type does not match one of the expected aggregate types, it returns nil instead.
   658  // Parameters:
   659  //  pos           -- the location of any generated code.
   660  //  b             -- the block into which any generated code should normally be placed
   661  //  source        -- the value, possibly an aggregate, to be stored.
   662  //  mem           -- the mem flowing into this decomposition (loads depend on it, stores updated it)
   663  //  t             -- the type of the value to be stored
   664  //  storeOffset   -- if the value is stored in memory, it is stored at base (see storeRc) + storeOffset
   665  //  loadRegOffset -- regarding source as a value in registers, the register offset in ABI1.  Meaningful only if source is OpArg.
   666  //  storeRc       -- storeRC; if the value is stored in registers, this specifies the registers.
   667  //                   StoreRc also identifies whether the target is registers or memory, and has the base for the store operation.
   668  func (x *expandState) decomposeArg(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   669  
   670  	pa := x.prAssignForArg(source)
   671  	var locs []*LocalSlot
   672  	for _, s := range x.namedSelects[source] {
   673  		locs = append(locs, x.f.Names[s.locIndex])
   674  	}
   675  
   676  	if len(pa.Registers) > 0 {
   677  		// Handle the in-registers case directly
   678  		rts, offs := pa.RegisterTypesAndOffsets()
   679  		last := loadRegOffset + x.regWidth(t)
   680  		if offs[loadRegOffset] != 0 {
   681  			// Document the problem before panicking.
   682  			for i := 0; i < len(rts); i++ {
   683  				rt := rts[i]
   684  				off := offs[i]
   685  				fmt.Printf("rt=%s, off=%d, rt.Width=%d, rt.Align=%d\n", rt.String(), off, rt.Size(), uint8(rt.Alignment()))
   686  			}
   687  			panic(fmt.Errorf("offset %d of requested register %d should be zero, source=%s", offs[loadRegOffset], loadRegOffset, source.LongString()))
   688  		}
   689  
   690  		if x.debug > 1 {
   691  			x.Printf("decompose arg %s has %d locs\n", source.LongString(), len(locs))
   692  		}
   693  
   694  		for i := loadRegOffset; i < last; i++ {
   695  			rt := rts[i]
   696  			off := offs[i]
   697  			w := x.commonArgs[selKey{source, off, rt.Size(), rt}]
   698  			if w == nil {
   699  				w = x.newArgToMemOrRegs(source, w, off, i, rt, pos)
   700  				suffix := x.pathTo(source.Type, rt, off)
   701  				if suffix != "" {
   702  					x.splitSlotsIntoNames(locs, suffix, off, rt, w)
   703  				}
   704  			}
   705  			if t.IsPtrShaped() {
   706  				// Preserve the original store type. This ensures pointer type
   707  				// properties aren't discarded (e.g, notinheap).
   708  				if rt.Size() != t.Size() || len(pa.Registers) != 1 || i != loadRegOffset {
   709  					b.Func.Fatalf("incompatible store type %v and %v, i=%d", t, rt, i)
   710  				}
   711  				rt = t
   712  			}
   713  			mem = x.storeArgOrLoad(pos, b, w, mem, rt, storeOffset+off, i, storeRc.next(rt))
   714  		}
   715  		return mem
   716  	}
   717  
   718  	u := source.Type
   719  	switch u.Kind() {
   720  	case types.TARRAY:
   721  		elem := u.Elem()
   722  		elemRO := x.regWidth(elem)
   723  		for i := int64(0); i < u.NumElem(); i++ {
   724  			elemOff := i * elem.Size()
   725  			mem = storeOneArg(x, pos, b, locs, indexNames[i], source, mem, elem, elemOff, storeOffset+elemOff, loadRegOffset, storeRc.next(elem))
   726  			loadRegOffset += elemRO
   727  			pos = pos.WithNotStmt()
   728  		}
   729  		return mem
   730  	case types.TSTRUCT:
   731  		for i := 0; i < u.NumFields(); i++ {
   732  			fld := u.Field(i)
   733  			mem = storeOneArg(x, pos, b, locs, "."+fld.Sym.Name, source, mem, fld.Type, fld.Offset, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type))
   734  			loadRegOffset += x.regWidth(fld.Type)
   735  			pos = pos.WithNotStmt()
   736  		}
   737  		return mem
   738  	case types.TINT64, types.TUINT64:
   739  		if t.Size() == x.regSize {
   740  			break
   741  		}
   742  		tHi, tLo := x.intPairTypes(t.Kind())
   743  		mem = storeOneArg(x, pos, b, locs, ".hi", source, mem, tHi, x.hiOffset, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo))
   744  		pos = pos.WithNotStmt()
   745  		return storeOneArg(x, pos, b, locs, ".lo", source, mem, tLo, x.lowOffset, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.loRo))
   746  	case types.TINTER:
   747  		sfx := ".itab"
   748  		if u.IsEmptyInterface() {
   749  			sfx = ".type"
   750  		}
   751  		return storeTwoArg(x, pos, b, locs, sfx, ".idata", source, mem, x.typs.Uintptr, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc)
   752  	case types.TSTRING:
   753  		return storeTwoArg(x, pos, b, locs, ".ptr", ".len", source, mem, x.typs.BytePtr, x.typs.Int, 0, storeOffset, loadRegOffset, storeRc)
   754  	case types.TCOMPLEX64:
   755  		return storeTwoArg(x, pos, b, locs, ".real", ".imag", source, mem, x.typs.Float32, x.typs.Float32, 0, storeOffset, loadRegOffset, storeRc)
   756  	case types.TCOMPLEX128:
   757  		return storeTwoArg(x, pos, b, locs, ".real", ".imag", source, mem, x.typs.Float64, x.typs.Float64, 0, storeOffset, loadRegOffset, storeRc)
   758  	case types.TSLICE:
   759  		mem = storeOneArg(x, pos, b, locs, ".ptr", source, mem, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr))
   760  		return storeTwoArg(x, pos, b, locs, ".len", ".cap", source, mem, x.typs.Int, x.typs.Int, x.ptrSize, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc)
   761  	}
   762  	return nil
   763  }
   764  
   765  func (x *expandState) splitSlotsIntoNames(locs []*LocalSlot, suffix string, off int64, rt *types.Type, w *Value) {
   766  	wlocs := x.splitSlots(locs, suffix, off, rt)
   767  	for _, l := range wlocs {
   768  		old, ok := x.f.NamedValues[*l]
   769  		x.f.NamedValues[*l] = append(old, w)
   770  		if !ok {
   771  			x.f.Names = append(x.f.Names, l)
   772  		}
   773  	}
   774  }
   775  
   776  // decomposeLoad is a helper for storeArgOrLoad.
   777  // It decomposes a Load  into smaller parts and returns the new mem.
   778  // If the type does not match one of the expected aggregate types, it returns nil instead.
   779  // Parameters:
   780  //  pos           -- the location of any generated code.
   781  //  b             -- the block into which any generated code should normally be placed
   782  //  source        -- the value, possibly an aggregate, to be stored.
   783  //  mem           -- the mem flowing into this decomposition (loads depend on it, stores updated it)
   784  //  t             -- the type of the value to be stored
   785  //  storeOffset   -- if the value is stored in memory, it is stored at base (see storeRc) + offset
   786  //  loadRegOffset -- regarding source as a value in registers, the register offset in ABI1.  Meaningful only if source is OpArg.
   787  //  storeRc       -- storeRC; if the value is stored in registers, this specifies the registers.
   788  //                   StoreRc also identifies whether the target is registers or memory, and has the base for the store operation.
   789  //
   790  // TODO -- this needs cleanup; it just works for SSA-able aggregates, and won't fully generalize to register-args aggregates.
   791  func (x *expandState) decomposeLoad(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   792  	u := source.Type
   793  	switch u.Kind() {
   794  	case types.TARRAY:
   795  		elem := u.Elem()
   796  		elemRO := x.regWidth(elem)
   797  		for i := int64(0); i < u.NumElem(); i++ {
   798  			elemOff := i * elem.Size()
   799  			mem = storeOneLoad(x, pos, b, source, mem, elem, elemOff, storeOffset+elemOff, loadRegOffset, storeRc.next(elem))
   800  			loadRegOffset += elemRO
   801  			pos = pos.WithNotStmt()
   802  		}
   803  		return mem
   804  	case types.TSTRUCT:
   805  		for i := 0; i < u.NumFields(); i++ {
   806  			fld := u.Field(i)
   807  			mem = storeOneLoad(x, pos, b, source, mem, fld.Type, fld.Offset, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type))
   808  			loadRegOffset += x.regWidth(fld.Type)
   809  			pos = pos.WithNotStmt()
   810  		}
   811  		return mem
   812  	case types.TINT64, types.TUINT64:
   813  		if t.Size() == x.regSize {
   814  			break
   815  		}
   816  		tHi, tLo := x.intPairTypes(t.Kind())
   817  		mem = storeOneLoad(x, pos, b, source, mem, tHi, x.hiOffset, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo))
   818  		pos = pos.WithNotStmt()
   819  		return storeOneLoad(x, pos, b, source, mem, tLo, x.lowOffset, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.loRo))
   820  	case types.TINTER:
   821  		return storeTwoLoad(x, pos, b, source, mem, x.typs.Uintptr, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc)
   822  	case types.TSTRING:
   823  		return storeTwoLoad(x, pos, b, source, mem, x.typs.BytePtr, x.typs.Int, 0, storeOffset, loadRegOffset, storeRc)
   824  	case types.TCOMPLEX64:
   825  		return storeTwoLoad(x, pos, b, source, mem, x.typs.Float32, x.typs.Float32, 0, storeOffset, loadRegOffset, storeRc)
   826  	case types.TCOMPLEX128:
   827  		return storeTwoLoad(x, pos, b, source, mem, x.typs.Float64, x.typs.Float64, 0, storeOffset, loadRegOffset, storeRc)
   828  	case types.TSLICE:
   829  		mem = storeOneLoad(x, pos, b, source, mem, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr))
   830  		return storeTwoLoad(x, pos, b, source, mem, x.typs.Int, x.typs.Int, x.ptrSize, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc)
   831  	}
   832  	return nil
   833  }
   834  
   835  // storeOneArg creates a decomposed (one step) arg that is then stored.
   836  // pos and b locate the store instruction, source is the "base" of the value input,
   837  // mem is the input mem, t is the type in question, and offArg and offStore are the offsets from the respective bases.
   838  func storeOneArg(x *expandState, pos src.XPos, b *Block, locs []*LocalSlot, suffix string, source, mem *Value, t *types.Type, argOffset, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   839  	if x.debug > 1 {
   840  		x.indent(3)
   841  		defer x.indent(-3)
   842  		x.Printf("storeOneArg(%s;  %s;  %s; aO=%d; sO=%d; lrO=%d; %s)\n", source.LongString(), mem.String(), t.String(), argOffset, storeOffset, loadRegOffset, storeRc.String())
   843  	}
   844  
   845  	w := x.commonArgs[selKey{source, argOffset, t.Size(), t}]
   846  	if w == nil {
   847  		w = x.newArgToMemOrRegs(source, w, argOffset, loadRegOffset, t, pos)
   848  		x.splitSlotsIntoNames(locs, suffix, argOffset, t, w)
   849  	}
   850  	return x.storeArgOrLoad(pos, b, w, mem, t, storeOffset, loadRegOffset, storeRc)
   851  }
   852  
   853  // storeOneLoad creates a decomposed (one step) load that is then stored.
   854  func storeOneLoad(x *expandState, pos src.XPos, b *Block, source, mem *Value, t *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   855  	from := x.offsetFrom(source.Block, source.Args[0], offArg, types.NewPtr(t))
   856  	w := source.Block.NewValue2(source.Pos, OpLoad, t, from, mem)
   857  	return x.storeArgOrLoad(pos, b, w, mem, t, offStore, loadRegOffset, storeRc)
   858  }
   859  
   860  func storeTwoArg(x *expandState, pos src.XPos, b *Block, locs []*LocalSlot, suffix1 string, suffix2 string, source, mem *Value, t1, t2 *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   861  	mem = storeOneArg(x, pos, b, locs, suffix1, source, mem, t1, offArg, offStore, loadRegOffset, storeRc.next(t1))
   862  	pos = pos.WithNotStmt()
   863  	t1Size := t1.Size()
   864  	return storeOneArg(x, pos, b, locs, suffix2, source, mem, t2, offArg+t1Size, offStore+t1Size, loadRegOffset+1, storeRc)
   865  }
   866  
   867  // storeTwoLoad creates a pair of decomposed (one step) loads that are then stored.
   868  // the elements of the pair must not require any additional alignment.
   869  func storeTwoLoad(x *expandState, pos src.XPos, b *Block, source, mem *Value, t1, t2 *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   870  	mem = storeOneLoad(x, pos, b, source, mem, t1, offArg, offStore, loadRegOffset, storeRc.next(t1))
   871  	pos = pos.WithNotStmt()
   872  	t1Size := t1.Size()
   873  	return storeOneLoad(x, pos, b, source, mem, t2, offArg+t1Size, offStore+t1Size, loadRegOffset+1, storeRc)
   874  }
   875  
   876  // storeArgOrLoad converts stores of SSA-able potentially aggregatable arguments (passed to a call) into a series of primitive-typed
   877  // stores of non-aggregate types.  It recursively walks up a chain of selectors until it reaches a Load or an Arg.
   878  // If it does not reach a Load or an Arg, nothing happens; this allows a little freedom in phase ordering.
   879  func (x *expandState) storeArgOrLoad(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value {
   880  	if x.debug > 1 {
   881  		x.indent(3)
   882  		defer x.indent(-3)
   883  		x.Printf("storeArgOrLoad(%s;  %s;  %s; %d; %s)\n", source.LongString(), mem.String(), t.String(), storeOffset, storeRc.String())
   884  	}
   885  
   886  	// Start with Opcodes that can be disassembled
   887  	switch source.Op {
   888  	case OpCopy:
   889  		return x.storeArgOrLoad(pos, b, source.Args[0], mem, t, storeOffset, loadRegOffset, storeRc)
   890  
   891  	case OpLoad, OpDereference:
   892  		ret := x.decomposeLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc)
   893  		if ret != nil {
   894  			return ret
   895  		}
   896  
   897  	case OpArg:
   898  		ret := x.decomposeArg(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc)
   899  		if ret != nil {
   900  			return ret
   901  		}
   902  
   903  	case OpArrayMake0, OpStructMake0:
   904  		// TODO(register args) is this correct for registers?
   905  		return mem
   906  
   907  	case OpStructMake1, OpStructMake2, OpStructMake3, OpStructMake4:
   908  		for i := 0; i < t.NumFields(); i++ {
   909  			fld := t.Field(i)
   910  			mem = x.storeArgOrLoad(pos, b, source.Args[i], mem, fld.Type, storeOffset+fld.Offset, 0, storeRc.next(fld.Type))
   911  			pos = pos.WithNotStmt()
   912  		}
   913  		return mem
   914  
   915  	case OpArrayMake1:
   916  		return x.storeArgOrLoad(pos, b, source.Args[0], mem, t.Elem(), storeOffset, 0, storeRc.at(t, 0))
   917  
   918  	case OpInt64Make:
   919  		tHi, tLo := x.intPairTypes(t.Kind())
   920  		mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, tHi, storeOffset+x.hiOffset, 0, storeRc.next(tHi))
   921  		pos = pos.WithNotStmt()
   922  		return x.storeArgOrLoad(pos, b, source.Args[1], mem, tLo, storeOffset+x.lowOffset, 0, storeRc)
   923  
   924  	case OpComplexMake:
   925  		tPart := x.typs.Float32
   926  		wPart := t.Size() / 2
   927  		if wPart == 8 {
   928  			tPart = x.typs.Float64
   929  		}
   930  		mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, tPart, storeOffset, 0, storeRc.next(tPart))
   931  		pos = pos.WithNotStmt()
   932  		return x.storeArgOrLoad(pos, b, source.Args[1], mem, tPart, storeOffset+wPart, 0, storeRc)
   933  
   934  	case OpIMake:
   935  		mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.Uintptr, storeOffset, 0, storeRc.next(x.typs.Uintptr))
   936  		pos = pos.WithNotStmt()
   937  		return x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.BytePtr, storeOffset+x.ptrSize, 0, storeRc)
   938  
   939  	case OpStringMake:
   940  		mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.BytePtr, storeOffset, 0, storeRc.next(x.typs.BytePtr))
   941  		pos = pos.WithNotStmt()
   942  		return x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.Int, storeOffset+x.ptrSize, 0, storeRc)
   943  
   944  	case OpSliceMake:
   945  		mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.BytePtr, storeOffset, 0, storeRc.next(x.typs.BytePtr))
   946  		pos = pos.WithNotStmt()
   947  		mem = x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.Int, storeOffset+x.ptrSize, 0, storeRc.next(x.typs.Int))
   948  		return x.storeArgOrLoad(pos, b, source.Args[2], mem, x.typs.Int, storeOffset+2*x.ptrSize, 0, storeRc)
   949  	}
   950  
   951  	// For nodes that cannot be taken apart -- OpSelectN, other structure selectors.
   952  	switch t.Kind() {
   953  	case types.TARRAY:
   954  		elt := t.Elem()
   955  		if source.Type != t && t.NumElem() == 1 && elt.Size() == t.Size() && t.Size() == x.regSize {
   956  			t = removeTrivialWrapperTypes(t)
   957  			// it could be a leaf type, but the "leaf" could be complex64 (for example)
   958  			return x.storeArgOrLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc)
   959  		}
   960  		eltRO := x.regWidth(elt)
   961  		source.Type = t
   962  		for i := int64(0); i < t.NumElem(); i++ {
   963  			sel := source.Block.NewValue1I(pos, OpArraySelect, elt, i, source)
   964  			mem = x.storeArgOrLoad(pos, b, sel, mem, elt, storeOffset+i*elt.Size(), loadRegOffset, storeRc.at(t, 0))
   965  			loadRegOffset += eltRO
   966  			pos = pos.WithNotStmt()
   967  		}
   968  		return mem
   969  
   970  	case types.TSTRUCT:
   971  		if source.Type != t && t.NumFields() == 1 && t.Field(0).Type.Size() == t.Size() && t.Size() == x.regSize {
   972  			// This peculiar test deals with accesses to immediate interface data.
   973  			// It works okay because everything is the same size.
   974  			// Example code that triggers this can be found in go/constant/value.go, function ToComplex
   975  			// v119 (+881) = IData <intVal> v6
   976  			// v121 (+882) = StaticLECall <floatVal,mem> {AuxCall{"".itof([intVal,0])[floatVal,8]}} [16] v119 v1
   977  			// This corresponds to the generic rewrite rule "(StructSelect [0] (IData x)) => (IData x)"
   978  			// Guard against "struct{struct{*foo}}"
   979  			// Other rewriting phases create minor glitches when they transform IData, for instance the
   980  			// interface-typed Arg "x" of ToFloat in go/constant/value.go
   981  			//   v6 (858) = Arg <Value> {x} (x[Value], x[Value])
   982  			// is rewritten by decomposeArgs into
   983  			//   v141 (858) = Arg <uintptr> {x}
   984  			//   v139 (858) = Arg <*uint8> {x} [8]
   985  			// because of a type case clause on line 862 of go/constant/value.go
   986  			//  	case intVal:
   987  			//		   return itof(x)
   988  			// v139 is later stored as an intVal == struct{val *big.Int} which naively requires the fields of
   989  			// of a *uint8, which does not succeed.
   990  			t = removeTrivialWrapperTypes(t)
   991  			// it could be a leaf type, but the "leaf" could be complex64 (for example)
   992  			return x.storeArgOrLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc)
   993  		}
   994  
   995  		source.Type = t
   996  		for i := 0; i < t.NumFields(); i++ {
   997  			fld := t.Field(i)
   998  			sel := source.Block.NewValue1I(pos, OpStructSelect, fld.Type, int64(i), source)
   999  			mem = x.storeArgOrLoad(pos, b, sel, mem, fld.Type, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type))
  1000  			loadRegOffset += x.regWidth(fld.Type)
  1001  			pos = pos.WithNotStmt()
  1002  		}
  1003  		return mem
  1004  
  1005  	case types.TINT64, types.TUINT64:
  1006  		if t.Size() == x.regSize {
  1007  			break
  1008  		}
  1009  		tHi, tLo := x.intPairTypes(t.Kind())
  1010  		sel := source.Block.NewValue1(pos, OpInt64Hi, tHi, source)
  1011  		mem = x.storeArgOrLoad(pos, b, sel, mem, tHi, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo))
  1012  		pos = pos.WithNotStmt()
  1013  		sel = source.Block.NewValue1(pos, OpInt64Lo, tLo, source)
  1014  		return x.storeArgOrLoad(pos, b, sel, mem, tLo, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.hiRo))
  1015  
  1016  	case types.TINTER:
  1017  		sel := source.Block.NewValue1(pos, OpITab, x.typs.BytePtr, source)
  1018  		mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr))
  1019  		pos = pos.WithNotStmt()
  1020  		sel = source.Block.NewValue1(pos, OpIData, x.typs.BytePtr, source)
  1021  		return x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset+x.ptrSize, loadRegOffset+RO_iface_data, storeRc)
  1022  
  1023  	case types.TSTRING:
  1024  		sel := source.Block.NewValue1(pos, OpStringPtr, x.typs.BytePtr, source)
  1025  		mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr))
  1026  		pos = pos.WithNotStmt()
  1027  		sel = source.Block.NewValue1(pos, OpStringLen, x.typs.Int, source)
  1028  		return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+x.ptrSize, loadRegOffset+RO_string_len, storeRc)
  1029  
  1030  	case types.TSLICE:
  1031  		et := types.NewPtr(t.Elem())
  1032  		sel := source.Block.NewValue1(pos, OpSlicePtr, et, source)
  1033  		mem = x.storeArgOrLoad(pos, b, sel, mem, et, storeOffset, loadRegOffset, storeRc.next(et))
  1034  		pos = pos.WithNotStmt()
  1035  		sel = source.Block.NewValue1(pos, OpSliceLen, x.typs.Int, source)
  1036  		mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc.next(x.typs.Int))
  1037  		sel = source.Block.NewValue1(pos, OpSliceCap, x.typs.Int, source)
  1038  		return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+2*x.ptrSize, loadRegOffset+RO_slice_cap, storeRc)
  1039  
  1040  	case types.TCOMPLEX64:
  1041  		sel := source.Block.NewValue1(pos, OpComplexReal, x.typs.Float32, source)
  1042  		mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float32, storeOffset, loadRegOffset, storeRc.next(x.typs.Float32))
  1043  		pos = pos.WithNotStmt()
  1044  		sel = source.Block.NewValue1(pos, OpComplexImag, x.typs.Float32, source)
  1045  		return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float32, storeOffset+4, loadRegOffset+RO_complex_imag, storeRc)
  1046  
  1047  	case types.TCOMPLEX128:
  1048  		sel := source.Block.NewValue1(pos, OpComplexReal, x.typs.Float64, source)
  1049  		mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float64, storeOffset, loadRegOffset, storeRc.next(x.typs.Float64))
  1050  		pos = pos.WithNotStmt()
  1051  		sel = source.Block.NewValue1(pos, OpComplexImag, x.typs.Float64, source)
  1052  		return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float64, storeOffset+8, loadRegOffset+RO_complex_imag, storeRc)
  1053  	}
  1054  
  1055  	s := mem
  1056  	if source.Op == OpDereference {
  1057  		source.Op = OpLoad // For purposes of parameter passing expansion, a Dereference is a Load.
  1058  	}
  1059  	if storeRc.hasRegs() {
  1060  		storeRc.addArg(source)
  1061  	} else {
  1062  		dst := x.offsetFrom(b, storeRc.storeDest, storeOffset, types.NewPtr(t))
  1063  		s = b.NewValue3A(pos, OpStore, types.TypeMem, t, dst, source, mem)
  1064  	}
  1065  	if x.debug > 1 {
  1066  		x.Printf("-->storeArg returns %s, storeRc=%s\n", s.LongString(), storeRc.String())
  1067  	}
  1068  	return s
  1069  }
  1070  
  1071  // rewriteArgs replaces all the call-parameter Args to a call with their register translation (if any).
  1072  // Preceding parameters (code pointers, closure pointer) are preserved, and the memory input is modified
  1073  // to account for any parameter stores required.
  1074  // Any of the old Args that have their use count fall to zero are marked OpInvalid.
  1075  func (x *expandState) rewriteArgs(v *Value, firstArg int) {
  1076  	if x.debug > 1 {
  1077  		x.indent(3)
  1078  		defer x.indent(-3)
  1079  		x.Printf("rewriteArgs(%s; %d)\n", v.LongString(), firstArg)
  1080  	}
  1081  	// Thread the stores on the memory arg
  1082  	aux := v.Aux.(*AuxCall)
  1083  	m0 := v.MemoryArg()
  1084  	mem := m0
  1085  	newArgs := []*Value{}
  1086  	oldArgs := []*Value{}
  1087  	sp := x.sp
  1088  	if v.Op == OpTailLECall {
  1089  		// For tail call, we unwind the frame before the call so we'll use the caller's
  1090  		// SP.
  1091  		sp = x.f.Entry.NewValue0(src.NoXPos, OpGetCallerSP, x.typs.Uintptr)
  1092  	}
  1093  	for i, a := range v.Args[firstArg : len(v.Args)-1] { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
  1094  		oldArgs = append(oldArgs, a)
  1095  		auxI := int64(i)
  1096  		aRegs := aux.RegsOfArg(auxI)
  1097  		aType := aux.TypeOfArg(auxI)
  1098  		if len(aRegs) == 0 && a.Op == OpDereference {
  1099  			aOffset := aux.OffsetOfArg(auxI)
  1100  			if a.MemoryArg() != m0 {
  1101  				x.f.Fatalf("Op...LECall and OpDereference have mismatched mem, %s and %s", v.LongString(), a.LongString())
  1102  			}
  1103  			if v.Op == OpTailLECall {
  1104  				// It's common for a tail call passing the same arguments (e.g. method wrapper),
  1105  				// so this would be a self copy. Detect this and optimize it out.
  1106  				a0 := a.Args[0]
  1107  				if a0.Op == OpLocalAddr {
  1108  					n := a0.Aux.(*ir.Name)
  1109  					if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.FixedFrameSize() == aOffset {
  1110  						continue
  1111  					}
  1112  				}
  1113  			}
  1114  			// "Dereference" of addressed (probably not-SSA-eligible) value becomes Move
  1115  			// TODO(register args) this will be more complicated with registers in the picture.
  1116  			mem = x.rewriteDereference(v.Block, sp, a, mem, aOffset, aux.SizeOfArg(auxI), aType, a.Pos)
  1117  		} else {
  1118  			var rc registerCursor
  1119  			var result *[]*Value
  1120  			var aOffset int64
  1121  			if len(aRegs) > 0 {
  1122  				result = &newArgs
  1123  			} else {
  1124  				aOffset = aux.OffsetOfArg(auxI)
  1125  			}
  1126  			if v.Op == OpTailLECall && a.Op == OpArg && a.AuxInt == 0 {
  1127  				// It's common for a tail call passing the same arguments (e.g. method wrapper),
  1128  				// so this would be a self copy. Detect this and optimize it out.
  1129  				n := a.Aux.(*ir.Name)
  1130  				if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.FixedFrameSize() == aOffset {
  1131  					continue
  1132  				}
  1133  			}
  1134  			if x.debug > 1 {
  1135  				x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
  1136  			}
  1137  			rc.init(aRegs, aux.abiInfo, result, sp)
  1138  			mem = x.storeArgOrLoad(a.Pos, v.Block, a, mem, aType, aOffset, 0, rc)
  1139  		}
  1140  	}
  1141  	var preArgStore [2]*Value
  1142  	preArgs := append(preArgStore[:0], v.Args[0:firstArg]...)
  1143  	v.resetArgs()
  1144  	v.AddArgs(preArgs...)
  1145  	v.AddArgs(newArgs...)
  1146  	v.AddArg(mem)
  1147  	for _, a := range oldArgs {
  1148  		if a.Uses == 0 {
  1149  			x.invalidateRecursively(a)
  1150  		}
  1151  	}
  1152  
  1153  	return
  1154  }
  1155  
  1156  func (x *expandState) invalidateRecursively(a *Value) {
  1157  	var s string
  1158  	if x.debug > 0 {
  1159  		plus := " "
  1160  		if a.Pos.IsStmt() == src.PosIsStmt {
  1161  			plus = " +"
  1162  		}
  1163  		s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString()
  1164  		if x.debug > 1 {
  1165  			x.Printf("...marking %v unused\n", s)
  1166  		}
  1167  	}
  1168  	lost := a.invalidateRecursively()
  1169  	if x.debug&1 != 0 && lost { // For odd values of x.debug, do this.
  1170  		x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s)
  1171  	}
  1172  }
  1173  
  1174  // expandCalls converts LE (Late Expansion) calls that act like they receive value args into a lower-level form
  1175  // that is more oriented to a platform's ABI.  The SelectN operations that extract results are rewritten into
  1176  // more appropriate forms, and any StructMake or ArrayMake inputs are decomposed until non-struct values are
  1177  // reached.  On the callee side, OpArg nodes are not decomposed until this phase is run.
  1178  // TODO results should not be lowered until this phase.
  1179  func expandCalls(f *Func) {
  1180  	// Calls that need lowering have some number of inputs, including a memory input,
  1181  	// and produce a tuple of (value1, value2, ..., mem) where valueK may or may not be SSA-able.
  1182  
  1183  	// With the current ABI those inputs need to be converted into stores to memory,
  1184  	// rethreading the call's memory input to the first, and the new call now receiving the last.
  1185  
  1186  	// With the current ABI, the outputs need to be converted to loads, which will all use the call's
  1187  	// memory output as their input.
  1188  	sp, _ := f.spSb()
  1189  	x := &expandState{
  1190  		f:                  f,
  1191  		abi1:               f.ABI1,
  1192  		debug:              f.pass.debug,
  1193  		canSSAType:         f.fe.CanSSA,
  1194  		regSize:            f.Config.RegSize,
  1195  		sp:                 sp,
  1196  		typs:               &f.Config.Types,
  1197  		ptrSize:            f.Config.PtrSize,
  1198  		namedSelects:       make(map[*Value][]namedVal),
  1199  		sdom:               f.Sdom(),
  1200  		commonArgs:         make(map[selKey]*Value),
  1201  		memForCall:         make(map[ID]*Value),
  1202  		transformedSelects: make(map[ID]bool),
  1203  	}
  1204  
  1205  	// For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness.
  1206  	if f.Config.BigEndian {
  1207  		x.lowOffset, x.hiOffset = 4, 0
  1208  		x.loRo, x.hiRo = 1, 0
  1209  	} else {
  1210  		x.lowOffset, x.hiOffset = 0, 4
  1211  		x.loRo, x.hiRo = 0, 1
  1212  	}
  1213  
  1214  	if x.debug > 1 {
  1215  		x.Printf("\nexpandsCalls(%s)\n", f.Name)
  1216  	}
  1217  
  1218  	for i, name := range f.Names {
  1219  		t := name.Type
  1220  		if x.isAlreadyExpandedAggregateType(t) {
  1221  			for j, v := range f.NamedValues[*name] {
  1222  				if v.Op == OpSelectN || v.Op == OpArg && x.isAlreadyExpandedAggregateType(v.Type) {
  1223  					ns := x.namedSelects[v]
  1224  					x.namedSelects[v] = append(ns, namedVal{locIndex: i, valIndex: j})
  1225  				}
  1226  			}
  1227  		}
  1228  	}
  1229  
  1230  	// TODO if too slow, whole program iteration can be replaced w/ slices of appropriate values, accumulated in first loop here.
  1231  
  1232  	// Step 0: rewrite the calls to convert args to calls into stores/register movement.
  1233  	for _, b := range f.Blocks {
  1234  		for _, v := range b.Values {
  1235  			firstArg := 0
  1236  			switch v.Op {
  1237  			case OpStaticLECall, OpTailLECall:
  1238  			case OpInterLECall:
  1239  				firstArg = 1
  1240  			case OpClosureLECall:
  1241  				firstArg = 2
  1242  			default:
  1243  				continue
  1244  			}
  1245  			x.rewriteArgs(v, firstArg)
  1246  		}
  1247  		if isBlockMultiValueExit(b) {
  1248  			x.indent(3)
  1249  			// Very similar to code in rewriteArgs, but results instead of args.
  1250  			v := b.Controls[0]
  1251  			m0 := v.MemoryArg()
  1252  			mem := m0
  1253  			aux := f.OwnAux
  1254  			allResults := []*Value{}
  1255  			if x.debug > 1 {
  1256  				x.Printf("multiValueExit rewriting %s\n", v.LongString())
  1257  			}
  1258  			var oldArgs []*Value
  1259  			for j, a := range v.Args[:len(v.Args)-1] {
  1260  				oldArgs = append(oldArgs, a)
  1261  				i := int64(j)
  1262  				auxType := aux.TypeOfResult(i)
  1263  				auxBase := b.NewValue2A(v.Pos, OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem)
  1264  				auxOffset := int64(0)
  1265  				auxSize := aux.SizeOfResult(i)
  1266  				aRegs := aux.RegsOfResult(int64(j))
  1267  				if len(aRegs) == 0 && a.Op == OpDereference {
  1268  					// Avoid a self-move, and if one is detected try to remove the already-inserted VarDef for the assignment that won't happen.
  1269  					if dAddr, dMem := a.Args[0], a.Args[1]; dAddr.Op == OpLocalAddr && dAddr.Args[0].Op == OpSP &&
  1270  						dAddr.Args[1] == dMem && dAddr.Aux == aux.NameOfResult(i) {
  1271  						if dMem.Op == OpVarDef && dMem.Aux == dAddr.Aux {
  1272  							dMem.copyOf(dMem.MemoryArg()) // elide the VarDef
  1273  						}
  1274  						continue
  1275  					}
  1276  					mem = x.rewriteDereference(v.Block, auxBase, a, mem, auxOffset, auxSize, auxType, a.Pos)
  1277  				} else {
  1278  					if a.Op == OpLoad && a.Args[0].Op == OpLocalAddr {
  1279  						addr := a.Args[0] // This is a self-move. // TODO(register args) do what here for registers?
  1280  						if addr.MemoryArg() == a.MemoryArg() && addr.Aux == aux.NameOfResult(i) {
  1281  							continue
  1282  						}
  1283  					}
  1284  					var rc registerCursor
  1285  					var result *[]*Value
  1286  					if len(aRegs) > 0 {
  1287  						result = &allResults
  1288  					}
  1289  					rc.init(aRegs, aux.abiInfo, result, auxBase)
  1290  					mem = x.storeArgOrLoad(v.Pos, b, a, mem, aux.TypeOfResult(i), auxOffset, 0, rc)
  1291  				}
  1292  			}
  1293  			v.resetArgs()
  1294  			v.AddArgs(allResults...)
  1295  			v.AddArg(mem)
  1296  			v.Type = types.NewResults(append(abi.RegisterTypes(aux.abiInfo.OutParams()), types.TypeMem))
  1297  			b.SetControl(v)
  1298  			for _, a := range oldArgs {
  1299  				if a.Uses == 0 {
  1300  					if x.debug > 1 {
  1301  						x.Printf("...marking %v unused\n", a.LongString())
  1302  					}
  1303  					x.invalidateRecursively(a)
  1304  				}
  1305  			}
  1306  			if x.debug > 1 {
  1307  				x.Printf("...multiValueExit new result %s\n", v.LongString())
  1308  			}
  1309  			x.indent(-3)
  1310  		}
  1311  	}
  1312  
  1313  	// Step 1: any stores of aggregates remaining are believed to be sourced from call results or args.
  1314  	// Decompose those stores into a series of smaller stores, adding selection ops as necessary.
  1315  	for _, b := range f.Blocks {
  1316  		for _, v := range b.Values {
  1317  			if v.Op == OpStore {
  1318  				t := v.Aux.(*types.Type)
  1319  				source := v.Args[1]
  1320  				tSrc := source.Type
  1321  				iAEATt := x.isAlreadyExpandedAggregateType(t)
  1322  
  1323  				if !iAEATt {
  1324  					// guarding against store immediate struct into interface data field -- store type is *uint8
  1325  					// TODO can this happen recursively?
  1326  					iAEATt = x.isAlreadyExpandedAggregateType(tSrc)
  1327  					if iAEATt {
  1328  						t = tSrc
  1329  					}
  1330  				}
  1331  				dst, mem := v.Args[0], v.Args[2]
  1332  				mem = x.storeArgOrLoad(v.Pos, b, source, mem, t, 0, 0, registerCursor{storeDest: dst})
  1333  				v.copyOf(mem)
  1334  			}
  1335  		}
  1336  	}
  1337  
  1338  	val2Preds := make(map[*Value]int32) // Used to accumulate dependency graph of selection operations for topological ordering.
  1339  
  1340  	// Step 2: transform or accumulate selection operations for rewrite in topological order.
  1341  	//
  1342  	// Aggregate types that have already (in earlier phases) been transformed must be lowered comprehensively to finish
  1343  	// the transformation (user-defined structs and arrays, slices, strings, interfaces, complex, 64-bit on 32-bit architectures),
  1344  	//
  1345  	// Any select-for-addressing applied to call results can be transformed directly.
  1346  	for _, b := range f.Blocks {
  1347  		for _, v := range b.Values {
  1348  			// Accumulate chains of selectors for processing in topological order
  1349  			switch v.Op {
  1350  			case OpStructSelect, OpArraySelect,
  1351  				OpIData, OpITab,
  1352  				OpStringPtr, OpStringLen,
  1353  				OpSlicePtr, OpSliceLen, OpSliceCap, OpSlicePtrUnchecked,
  1354  				OpComplexReal, OpComplexImag,
  1355  				OpInt64Hi, OpInt64Lo:
  1356  				w := v.Args[0]
  1357  				switch w.Op {
  1358  				case OpStructSelect, OpArraySelect, OpSelectN, OpArg:
  1359  					val2Preds[w] += 1
  1360  					if x.debug > 1 {
  1361  						x.Printf("v2p[%s] = %d\n", w.LongString(), val2Preds[w])
  1362  					}
  1363  				}
  1364  				fallthrough
  1365  
  1366  			case OpSelectN:
  1367  				if _, ok := val2Preds[v]; !ok {
  1368  					val2Preds[v] = 0
  1369  					if x.debug > 1 {
  1370  						x.Printf("v2p[%s] = %d\n", v.LongString(), val2Preds[v])
  1371  					}
  1372  				}
  1373  
  1374  			case OpArg:
  1375  				if !x.isAlreadyExpandedAggregateType(v.Type) {
  1376  					continue
  1377  				}
  1378  				if _, ok := val2Preds[v]; !ok {
  1379  					val2Preds[v] = 0
  1380  					if x.debug > 1 {
  1381  						x.Printf("v2p[%s] = %d\n", v.LongString(), val2Preds[v])
  1382  					}
  1383  				}
  1384  
  1385  			case OpSelectNAddr:
  1386  				// Do these directly, there are no chains of selectors.
  1387  				call := v.Args[0]
  1388  				which := v.AuxInt
  1389  				aux := call.Aux.(*AuxCall)
  1390  				pt := v.Type
  1391  				off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt)
  1392  				v.copyOf(off)
  1393  			}
  1394  		}
  1395  	}
  1396  
  1397  	// Step 3: Compute topological order of selectors,
  1398  	// then process it in reverse to eliminate duplicates,
  1399  	// then forwards to rewrite selectors.
  1400  	//
  1401  	// All chains of selectors end up in same block as the call.
  1402  
  1403  	// Compilation must be deterministic, so sort after extracting first zeroes from map.
  1404  	// Sorting allows dominators-last order within each batch,
  1405  	// so that the backwards scan for duplicates will most often find copies from dominating blocks (it is best-effort).
  1406  	var toProcess []*Value
  1407  	less := func(i, j int) bool {
  1408  		vi, vj := toProcess[i], toProcess[j]
  1409  		bi, bj := vi.Block, vj.Block
  1410  		if bi == bj {
  1411  			return vi.ID < vj.ID
  1412  		}
  1413  		return x.sdom.domorder(bi) > x.sdom.domorder(bj) // reverse the order to put dominators last.
  1414  	}
  1415  
  1416  	// Accumulate order in allOrdered
  1417  	var allOrdered []*Value
  1418  	for v, n := range val2Preds {
  1419  		if n == 0 {
  1420  			allOrdered = append(allOrdered, v)
  1421  		}
  1422  	}
  1423  	last := 0 // allOrdered[0:last] has been top-sorted and processed
  1424  	for len(val2Preds) > 0 {
  1425  		toProcess = allOrdered[last:]
  1426  		last = len(allOrdered)
  1427  		sort.SliceStable(toProcess, less)
  1428  		for _, v := range toProcess {
  1429  			delete(val2Preds, v)
  1430  			if v.Op == OpArg {
  1431  				continue // no Args[0], hence done.
  1432  			}
  1433  			w := v.Args[0]
  1434  			n, ok := val2Preds[w]
  1435  			if !ok {
  1436  				continue
  1437  			}
  1438  			if n == 1 {
  1439  				allOrdered = append(allOrdered, w)
  1440  				delete(val2Preds, w)
  1441  				continue
  1442  			}
  1443  			val2Preds[w] = n - 1
  1444  		}
  1445  	}
  1446  
  1447  	x.commonSelectors = make(map[selKey]*Value)
  1448  	// Rewrite duplicate selectors as copies where possible.
  1449  	for i := len(allOrdered) - 1; i >= 0; i-- {
  1450  		v := allOrdered[i]
  1451  		if v.Op == OpArg {
  1452  			continue
  1453  		}
  1454  		w := v.Args[0]
  1455  		if w.Op == OpCopy {
  1456  			for w.Op == OpCopy {
  1457  				w = w.Args[0]
  1458  			}
  1459  			v.SetArg(0, w)
  1460  		}
  1461  		typ := v.Type
  1462  		if typ.IsMemory() {
  1463  			continue // handled elsewhere, not an indexable result
  1464  		}
  1465  		size := typ.Size()
  1466  		offset := int64(0)
  1467  		switch v.Op {
  1468  		case OpStructSelect:
  1469  			if w.Type.Kind() == types.TSTRUCT {
  1470  				offset = w.Type.FieldOff(int(v.AuxInt))
  1471  			} else { // Immediate interface data artifact, offset is zero.
  1472  				f.Fatalf("Expand calls interface data problem, func %s, v=%s, w=%s\n", f.Name, v.LongString(), w.LongString())
  1473  			}
  1474  		case OpArraySelect:
  1475  			offset = size * v.AuxInt
  1476  		case OpSelectN:
  1477  			offset = v.AuxInt // offset is just a key, really.
  1478  		case OpInt64Hi:
  1479  			offset = x.hiOffset
  1480  		case OpInt64Lo:
  1481  			offset = x.lowOffset
  1482  		case OpStringLen, OpSliceLen, OpIData:
  1483  			offset = x.ptrSize
  1484  		case OpSliceCap:
  1485  			offset = 2 * x.ptrSize
  1486  		case OpComplexImag:
  1487  			offset = size
  1488  		}
  1489  		sk := selKey{from: w, size: size, offsetOrIndex: offset, typ: typ}
  1490  		dupe := x.commonSelectors[sk]
  1491  		if dupe == nil {
  1492  			x.commonSelectors[sk] = v
  1493  		} else if x.sdom.IsAncestorEq(dupe.Block, v.Block) {
  1494  			if x.debug > 1 {
  1495  				x.Printf("Duplicate, make %s copy of %s\n", v, dupe)
  1496  			}
  1497  			v.copyOf(dupe)
  1498  		} else {
  1499  			// Because values are processed in dominator order, the old common[s] will never dominate after a miss is seen.
  1500  			// Installing the new value might match some future values.
  1501  			x.commonSelectors[sk] = v
  1502  		}
  1503  	}
  1504  
  1505  	// Indices of entries in f.Names that need to be deleted.
  1506  	var toDelete []namedVal
  1507  
  1508  	// Rewrite selectors.
  1509  	for i, v := range allOrdered {
  1510  		if x.debug > 1 {
  1511  			b := v.Block
  1512  			x.Printf("allOrdered[%d] = b%d, %s, uses=%d\n", i, b.ID, v.LongString(), v.Uses)
  1513  		}
  1514  		if v.Uses == 0 {
  1515  			x.invalidateRecursively(v)
  1516  			continue
  1517  		}
  1518  		if v.Op == OpCopy {
  1519  			continue
  1520  		}
  1521  		locs := x.rewriteSelect(v, v, 0, 0)
  1522  		// Install new names.
  1523  		if v.Type.IsMemory() {
  1524  			continue
  1525  		}
  1526  		// Leaf types may have debug locations
  1527  		if !x.isAlreadyExpandedAggregateType(v.Type) {
  1528  			for _, l := range locs {
  1529  				if _, ok := f.NamedValues[*l]; !ok {
  1530  					f.Names = append(f.Names, l)
  1531  				}
  1532  				f.NamedValues[*l] = append(f.NamedValues[*l], v)
  1533  			}
  1534  			continue
  1535  		}
  1536  		if ns, ok := x.namedSelects[v]; ok {
  1537  			// Not-leaf types that had debug locations need to lose them.
  1538  
  1539  			toDelete = append(toDelete, ns...)
  1540  		}
  1541  	}
  1542  
  1543  	deleteNamedVals(f, toDelete)
  1544  
  1545  	// Step 4: rewrite the calls themselves, correcting the type.
  1546  	for _, b := range f.Blocks {
  1547  		for _, v := range b.Values {
  1548  			switch v.Op {
  1549  			case OpArg:
  1550  				x.rewriteArgToMemOrRegs(v)
  1551  			case OpStaticLECall:
  1552  				v.Op = OpStaticCall
  1553  				rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
  1554  				v.Type = types.NewResults(append(rts, types.TypeMem))
  1555  			case OpTailLECall:
  1556  				v.Op = OpTailCall
  1557  				rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
  1558  				v.Type = types.NewResults(append(rts, types.TypeMem))
  1559  			case OpClosureLECall:
  1560  				v.Op = OpClosureCall
  1561  				rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
  1562  				v.Type = types.NewResults(append(rts, types.TypeMem))
  1563  			case OpInterLECall:
  1564  				v.Op = OpInterCall
  1565  				rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
  1566  				v.Type = types.NewResults(append(rts, types.TypeMem))
  1567  			}
  1568  		}
  1569  	}
  1570  
  1571  	// Step 5: dedup OpArgXXXReg values. Mostly it is already dedup'd by commonArgs,
  1572  	// but there are cases that we have same OpArgXXXReg values with different types.
  1573  	// E.g. string is sometimes decomposed as { *int8, int }, sometimes as { unsafe.Pointer, uintptr }.
  1574  	// (Can we avoid that?)
  1575  	var IArg, FArg [32]*Value
  1576  	for _, v := range f.Entry.Values {
  1577  		switch v.Op {
  1578  		case OpArgIntReg:
  1579  			i := v.AuxInt
  1580  			if w := IArg[i]; w != nil {
  1581  				if w.Type.Size() != v.Type.Size() {
  1582  					f.Fatalf("incompatible OpArgIntReg [%d]: %s and %s", i, v.LongString(), w.LongString())
  1583  				}
  1584  				if w.Type.IsUnsafePtr() && !v.Type.IsUnsafePtr() {
  1585  					// Update unsafe.Pointer type if we know the actual pointer type.
  1586  					w.Type = v.Type
  1587  				}
  1588  				// TODO: don't dedup pointer and scalar? Rewrite to OpConvert? Can it happen?
  1589  				v.copyOf(w)
  1590  			} else {
  1591  				IArg[i] = v
  1592  			}
  1593  		case OpArgFloatReg:
  1594  			i := v.AuxInt
  1595  			if w := FArg[i]; w != nil {
  1596  				if w.Type.Size() != v.Type.Size() {
  1597  					f.Fatalf("incompatible OpArgFloatReg [%d]: %v and %v", i, v, w)
  1598  				}
  1599  				v.copyOf(w)
  1600  			} else {
  1601  				FArg[i] = v
  1602  			}
  1603  		}
  1604  	}
  1605  
  1606  	// Step 6: elide any copies introduced.
  1607  	// Update named values.
  1608  	for _, name := range f.Names {
  1609  		values := f.NamedValues[*name]
  1610  		for i, v := range values {
  1611  			if v.Op == OpCopy {
  1612  				a := v.Args[0]
  1613  				for a.Op == OpCopy {
  1614  					a = a.Args[0]
  1615  				}
  1616  				values[i] = a
  1617  			}
  1618  		}
  1619  	}
  1620  	for _, b := range f.Blocks {
  1621  		for _, v := range b.Values {
  1622  			for i, a := range v.Args {
  1623  				if a.Op != OpCopy {
  1624  					continue
  1625  				}
  1626  				aa := copySource(a)
  1627  				v.SetArg(i, aa)
  1628  				for a.Uses == 0 {
  1629  					b := a.Args[0]
  1630  					x.invalidateRecursively(a)
  1631  					a = b
  1632  				}
  1633  			}
  1634  		}
  1635  	}
  1636  
  1637  	// Rewriting can attach lines to values that are unlikely to survive code generation, so move them to a use.
  1638  	for _, b := range f.Blocks {
  1639  		for _, v := range b.Values {
  1640  			for _, a := range v.Args {
  1641  				if a.Pos.IsStmt() != src.PosIsStmt {
  1642  					continue
  1643  				}
  1644  				if a.Type.IsMemory() {
  1645  					continue
  1646  				}
  1647  				if a.Pos.Line() != v.Pos.Line() {
  1648  					continue
  1649  				}
  1650  				if !a.Pos.SameFile(v.Pos) {
  1651  					continue
  1652  				}
  1653  				switch a.Op {
  1654  				case OpArgIntReg, OpArgFloatReg, OpSelectN:
  1655  					v.Pos = v.Pos.WithIsStmt()
  1656  					a.Pos = a.Pos.WithDefaultStmt()
  1657  				}
  1658  			}
  1659  		}
  1660  	}
  1661  }
  1662  
  1663  // rewriteArgToMemOrRegs converts OpArg v in-place into the register version of v,
  1664  // if that is appropriate.
  1665  func (x *expandState) rewriteArgToMemOrRegs(v *Value) *Value {
  1666  	if x.debug > 1 {
  1667  		x.indent(3)
  1668  		defer x.indent(-3)
  1669  		x.Printf("rewriteArgToMemOrRegs(%s)\n", v.LongString())
  1670  	}
  1671  	pa := x.prAssignForArg(v)
  1672  	switch len(pa.Registers) {
  1673  	case 0:
  1674  		frameOff := v.Aux.(*ir.Name).FrameOffset()
  1675  		if pa.Offset() != int32(frameOff+x.f.ABISelf.LocalsOffset()) {
  1676  			panic(fmt.Errorf("Parameter assignment %d and OpArg.Aux frameOffset %d disagree, op=%s",
  1677  				pa.Offset(), frameOff, v.LongString()))
  1678  		}
  1679  	case 1:
  1680  		t := v.Type
  1681  		key := selKey{v, 0, t.Size(), t}
  1682  		w := x.commonArgs[key]
  1683  		if w != nil && w.Uses != 0 { // do not reuse dead value
  1684  			v.copyOf(w)
  1685  			break
  1686  		}
  1687  		r := pa.Registers[0]
  1688  		var i int64
  1689  		v.Op, i = ArgOpAndRegisterFor(r, x.f.ABISelf)
  1690  		v.Aux = &AuxNameOffset{v.Aux.(*ir.Name), 0}
  1691  		v.AuxInt = i
  1692  		x.commonArgs[key] = v
  1693  
  1694  	default:
  1695  		panic(badVal("Saw unexpanded OpArg", v))
  1696  	}
  1697  	if x.debug > 1 {
  1698  		x.Printf("-->%s\n", v.LongString())
  1699  	}
  1700  	return v
  1701  }
  1702  
  1703  // newArgToMemOrRegs either rewrites toReplace into an OpArg referencing memory or into an OpArgXXXReg to a register,
  1704  // or rewrites it into a copy of the appropriate OpArgXXX.  The actual OpArgXXX is determined by combining baseArg (an OpArg)
  1705  // with offset, regOffset, and t to determine which portion of it to reference (either all or a part, in memory or in registers).
  1706  func (x *expandState) newArgToMemOrRegs(baseArg, toReplace *Value, offset int64, regOffset Abi1RO, t *types.Type, pos src.XPos) *Value {
  1707  	if x.debug > 1 {
  1708  		x.indent(3)
  1709  		defer x.indent(-3)
  1710  		x.Printf("newArgToMemOrRegs(base=%s; toReplace=%s; t=%s; memOff=%d; regOff=%d)\n", baseArg.String(), toReplace.LongString(), t.String(), offset, regOffset)
  1711  	}
  1712  	key := selKey{baseArg, offset, t.Size(), t}
  1713  	w := x.commonArgs[key]
  1714  	if w != nil && w.Uses != 0 { // do not reuse dead value
  1715  		if toReplace != nil {
  1716  			toReplace.copyOf(w)
  1717  			if x.debug > 1 {
  1718  				x.Printf("...replace %s\n", toReplace.LongString())
  1719  			}
  1720  		}
  1721  		if x.debug > 1 {
  1722  			x.Printf("-->%s\n", w.LongString())
  1723  		}
  1724  		return w
  1725  	}
  1726  
  1727  	pa := x.prAssignForArg(baseArg)
  1728  	if len(pa.Registers) == 0 { // Arg is on stack
  1729  		frameOff := baseArg.Aux.(*ir.Name).FrameOffset()
  1730  		if pa.Offset() != int32(frameOff+x.f.ABISelf.LocalsOffset()) {
  1731  			panic(fmt.Errorf("Parameter assignment %d and OpArg.Aux frameOffset %d disagree, op=%s",
  1732  				pa.Offset(), frameOff, baseArg.LongString()))
  1733  		}
  1734  		aux := baseArg.Aux
  1735  		auxInt := baseArg.AuxInt + offset
  1736  		if toReplace != nil && toReplace.Block == baseArg.Block {
  1737  			toReplace.reset(OpArg)
  1738  			toReplace.Aux = aux
  1739  			toReplace.AuxInt = auxInt
  1740  			toReplace.Type = t
  1741  			w = toReplace
  1742  		} else {
  1743  			w = baseArg.Block.NewValue0IA(pos, OpArg, t, auxInt, aux)
  1744  		}
  1745  		x.commonArgs[key] = w
  1746  		if toReplace != nil {
  1747  			toReplace.copyOf(w)
  1748  		}
  1749  		if x.debug > 1 {
  1750  			x.Printf("-->%s\n", w.LongString())
  1751  		}
  1752  		return w
  1753  	}
  1754  	// Arg is in registers
  1755  	r := pa.Registers[regOffset]
  1756  	op, auxInt := ArgOpAndRegisterFor(r, x.f.ABISelf)
  1757  	if op == OpArgIntReg && t.IsFloat() || op == OpArgFloatReg && t.IsInteger() {
  1758  		fmt.Printf("pa=%v\nx.f.OwnAux.abiInfo=%s\n",
  1759  			pa.ToString(x.f.ABISelf, true),
  1760  			x.f.OwnAux.abiInfo.String())
  1761  		panic(fmt.Errorf("Op/Type mismatch, op=%s, type=%s", op.String(), t.String()))
  1762  	}
  1763  	if baseArg.AuxInt != 0 {
  1764  		base.Fatalf("BaseArg %s bound to registers has non-zero AuxInt", baseArg.LongString())
  1765  	}
  1766  	aux := &AuxNameOffset{baseArg.Aux.(*ir.Name), offset}
  1767  	if toReplace != nil && toReplace.Block == baseArg.Block {
  1768  		toReplace.reset(op)
  1769  		toReplace.Aux = aux
  1770  		toReplace.AuxInt = auxInt
  1771  		toReplace.Type = t
  1772  		w = toReplace
  1773  	} else {
  1774  		w = baseArg.Block.NewValue0IA(pos, op, t, auxInt, aux)
  1775  	}
  1776  	x.commonArgs[key] = w
  1777  	if toReplace != nil {
  1778  		toReplace.copyOf(w)
  1779  	}
  1780  	if x.debug > 1 {
  1781  		x.Printf("-->%s\n", w.LongString())
  1782  	}
  1783  	return w
  1784  
  1785  }
  1786  
  1787  // argOpAndRegisterFor converts an abi register index into an ssa Op and corresponding
  1788  // arg register index.
  1789  func ArgOpAndRegisterFor(r abi.RegIndex, abiConfig *abi.ABIConfig) (Op, int64) {
  1790  	i := abiConfig.FloatIndexFor(r)
  1791  	if i >= 0 { // float PR
  1792  		return OpArgFloatReg, i
  1793  	}
  1794  	return OpArgIntReg, int64(r)
  1795  }
  1796  

View as plain text