Source file src/cmd/compile/internal/amd64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package amd64
     6  
     7  import (
     8  	"fmt"
     9  	"internal/buildcfg"
    10  	"math"
    11  
    12  	"cmd/compile/internal/base"
    13  	"cmd/compile/internal/ir"
    14  	"cmd/compile/internal/logopt"
    15  	"cmd/compile/internal/objw"
    16  	"cmd/compile/internal/ssa"
    17  	"cmd/compile/internal/ssagen"
    18  	"cmd/compile/internal/types"
    19  	"cmd/internal/obj"
    20  	"cmd/internal/obj/x86"
    21  )
    22  
    23  // markMoves marks any MOVXconst ops that need to avoid clobbering flags.
    24  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
    25  	flive := b.FlagsLiveAtEnd
    26  	for _, c := range b.ControlValues() {
    27  		flive = c.Type.IsFlags() || flive
    28  	}
    29  	for i := len(b.Values) - 1; i >= 0; i-- {
    30  		v := b.Values[i]
    31  		if flive && (v.Op == ssa.OpAMD64MOVLconst || v.Op == ssa.OpAMD64MOVQconst) {
    32  			// The "mark" is any non-nil Aux value.
    33  			v.Aux = v
    34  		}
    35  		if v.Type.IsFlags() {
    36  			flive = false
    37  		}
    38  		for _, a := range v.Args {
    39  			if a.Type.IsFlags() {
    40  				flive = true
    41  			}
    42  		}
    43  	}
    44  }
    45  
    46  // loadByType returns the load instruction of the given type.
    47  func loadByType(t *types.Type) obj.As {
    48  	// Avoid partial register write
    49  	if !t.IsFloat() {
    50  		switch t.Size() {
    51  		case 1:
    52  			return x86.AMOVBLZX
    53  		case 2:
    54  			return x86.AMOVWLZX
    55  		}
    56  	}
    57  	// Otherwise, there's no difference between load and store opcodes.
    58  	return storeByType(t)
    59  }
    60  
    61  // storeByType returns the store instruction of the given type.
    62  func storeByType(t *types.Type) obj.As {
    63  	width := t.Size()
    64  	if t.IsFloat() {
    65  		switch width {
    66  		case 4:
    67  			return x86.AMOVSS
    68  		case 8:
    69  			return x86.AMOVSD
    70  		}
    71  	} else {
    72  		switch width {
    73  		case 1:
    74  			return x86.AMOVB
    75  		case 2:
    76  			return x86.AMOVW
    77  		case 4:
    78  			return x86.AMOVL
    79  		case 8:
    80  			return x86.AMOVQ
    81  		}
    82  	}
    83  	panic(fmt.Sprintf("bad store type %v", t))
    84  }
    85  
    86  // moveByType returns the reg->reg move instruction of the given type.
    87  func moveByType(t *types.Type) obj.As {
    88  	if t.IsFloat() {
    89  		// Moving the whole sse2 register is faster
    90  		// than moving just the correct low portion of it.
    91  		// There is no xmm->xmm move with 1 byte opcode,
    92  		// so use movups, which has 2 byte opcode.
    93  		return x86.AMOVUPS
    94  	} else {
    95  		switch t.Size() {
    96  		case 1:
    97  			// Avoids partial register write
    98  			return x86.AMOVL
    99  		case 2:
   100  			return x86.AMOVL
   101  		case 4:
   102  			return x86.AMOVL
   103  		case 8:
   104  			return x86.AMOVQ
   105  		case 16:
   106  			return x86.AMOVUPS // int128s are in SSE registers
   107  		default:
   108  			panic(fmt.Sprintf("bad int register width %d:%v", t.Size(), t))
   109  		}
   110  	}
   111  }
   112  
   113  // opregreg emits instructions for
   114  //     dest := dest(To) op src(From)
   115  // and also returns the created obj.Prog so it
   116  // may be further adjusted (offset, scale, etc).
   117  func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {
   118  	p := s.Prog(op)
   119  	p.From.Type = obj.TYPE_REG
   120  	p.To.Type = obj.TYPE_REG
   121  	p.To.Reg = dest
   122  	p.From.Reg = src
   123  	return p
   124  }
   125  
   126  // memIdx fills out a as an indexed memory reference for v.
   127  // It assumes that the base register and the index register
   128  // are v.Args[0].Reg() and v.Args[1].Reg(), respectively.
   129  // The caller must still use gc.AddAux/gc.AddAux2 to handle v.Aux as necessary.
   130  func memIdx(a *obj.Addr, v *ssa.Value) {
   131  	r, i := v.Args[0].Reg(), v.Args[1].Reg()
   132  	a.Type = obj.TYPE_MEM
   133  	a.Scale = v.Op.Scale()
   134  	if a.Scale == 1 && i == x86.REG_SP {
   135  		r, i = i, r
   136  	}
   137  	a.Reg = r
   138  	a.Index = i
   139  }
   140  
   141  // DUFFZERO consists of repeated blocks of 4 MOVUPSs + LEAQ,
   142  // See runtime/mkduff.go.
   143  func duffStart(size int64) int64 {
   144  	x, _ := duff(size)
   145  	return x
   146  }
   147  func duffAdj(size int64) int64 {
   148  	_, x := duff(size)
   149  	return x
   150  }
   151  
   152  // duff returns the offset (from duffzero, in bytes) and pointer adjust (in bytes)
   153  // required to use the duffzero mechanism for a block of the given size.
   154  func duff(size int64) (int64, int64) {
   155  	if size < 32 || size > 1024 || size%dzClearStep != 0 {
   156  		panic("bad duffzero size")
   157  	}
   158  	steps := size / dzClearStep
   159  	blocks := steps / dzBlockLen
   160  	steps %= dzBlockLen
   161  	off := dzBlockSize * (dzBlocks - blocks)
   162  	var adj int64
   163  	if steps != 0 {
   164  		off -= dzLeaqSize
   165  		off -= dzMovSize * steps
   166  		adj -= dzClearStep * (dzBlockLen - steps)
   167  	}
   168  	return off, adj
   169  }
   170  
   171  func getgFromTLS(s *ssagen.State, r int16) {
   172  	// See the comments in cmd/internal/obj/x86/obj6.go
   173  	// near CanUse1InsnTLS for a detailed explanation of these instructions.
   174  	if x86.CanUse1InsnTLS(base.Ctxt) {
   175  		// MOVQ (TLS), r
   176  		p := s.Prog(x86.AMOVQ)
   177  		p.From.Type = obj.TYPE_MEM
   178  		p.From.Reg = x86.REG_TLS
   179  		p.To.Type = obj.TYPE_REG
   180  		p.To.Reg = r
   181  	} else {
   182  		// MOVQ TLS, r
   183  		// MOVQ (r)(TLS*1), r
   184  		p := s.Prog(x86.AMOVQ)
   185  		p.From.Type = obj.TYPE_REG
   186  		p.From.Reg = x86.REG_TLS
   187  		p.To.Type = obj.TYPE_REG
   188  		p.To.Reg = r
   189  		q := s.Prog(x86.AMOVQ)
   190  		q.From.Type = obj.TYPE_MEM
   191  		q.From.Reg = r
   192  		q.From.Index = x86.REG_TLS
   193  		q.From.Scale = 1
   194  		q.To.Type = obj.TYPE_REG
   195  		q.To.Reg = r
   196  	}
   197  }
   198  
   199  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   200  	switch v.Op {
   201  	case ssa.OpAMD64VFMADD231SD:
   202  		p := s.Prog(v.Op.Asm())
   203  		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[2].Reg()}
   204  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   205  		p.SetFrom3Reg(v.Args[1].Reg())
   206  	case ssa.OpAMD64ADDQ, ssa.OpAMD64ADDL:
   207  		r := v.Reg()
   208  		r1 := v.Args[0].Reg()
   209  		r2 := v.Args[1].Reg()
   210  		switch {
   211  		case r == r1:
   212  			p := s.Prog(v.Op.Asm())
   213  			p.From.Type = obj.TYPE_REG
   214  			p.From.Reg = r2
   215  			p.To.Type = obj.TYPE_REG
   216  			p.To.Reg = r
   217  		case r == r2:
   218  			p := s.Prog(v.Op.Asm())
   219  			p.From.Type = obj.TYPE_REG
   220  			p.From.Reg = r1
   221  			p.To.Type = obj.TYPE_REG
   222  			p.To.Reg = r
   223  		default:
   224  			var asm obj.As
   225  			if v.Op == ssa.OpAMD64ADDQ {
   226  				asm = x86.ALEAQ
   227  			} else {
   228  				asm = x86.ALEAL
   229  			}
   230  			p := s.Prog(asm)
   231  			p.From.Type = obj.TYPE_MEM
   232  			p.From.Reg = r1
   233  			p.From.Scale = 1
   234  			p.From.Index = r2
   235  			p.To.Type = obj.TYPE_REG
   236  			p.To.Reg = r
   237  		}
   238  	// 2-address opcode arithmetic
   239  	case ssa.OpAMD64SUBQ, ssa.OpAMD64SUBL,
   240  		ssa.OpAMD64MULQ, ssa.OpAMD64MULL,
   241  		ssa.OpAMD64ANDQ, ssa.OpAMD64ANDL,
   242  		ssa.OpAMD64ORQ, ssa.OpAMD64ORL,
   243  		ssa.OpAMD64XORQ, ssa.OpAMD64XORL,
   244  		ssa.OpAMD64SHLQ, ssa.OpAMD64SHLL,
   245  		ssa.OpAMD64SHRQ, ssa.OpAMD64SHRL, ssa.OpAMD64SHRW, ssa.OpAMD64SHRB,
   246  		ssa.OpAMD64SARQ, ssa.OpAMD64SARL, ssa.OpAMD64SARW, ssa.OpAMD64SARB,
   247  		ssa.OpAMD64ROLQ, ssa.OpAMD64ROLL, ssa.OpAMD64ROLW, ssa.OpAMD64ROLB,
   248  		ssa.OpAMD64RORQ, ssa.OpAMD64RORL, ssa.OpAMD64RORW, ssa.OpAMD64RORB,
   249  		ssa.OpAMD64ADDSS, ssa.OpAMD64ADDSD, ssa.OpAMD64SUBSS, ssa.OpAMD64SUBSD,
   250  		ssa.OpAMD64MULSS, ssa.OpAMD64MULSD, ssa.OpAMD64DIVSS, ssa.OpAMD64DIVSD,
   251  		ssa.OpAMD64PXOR,
   252  		ssa.OpAMD64BTSL, ssa.OpAMD64BTSQ,
   253  		ssa.OpAMD64BTCL, ssa.OpAMD64BTCQ,
   254  		ssa.OpAMD64BTRL, ssa.OpAMD64BTRQ:
   255  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   256  
   257  	case ssa.OpAMD64SHRDQ, ssa.OpAMD64SHLDQ:
   258  		p := s.Prog(v.Op.Asm())
   259  		lo, hi, bits := v.Args[0].Reg(), v.Args[1].Reg(), v.Args[2].Reg()
   260  		p.From.Type = obj.TYPE_REG
   261  		p.From.Reg = bits
   262  		p.To.Type = obj.TYPE_REG
   263  		p.To.Reg = lo
   264  		p.SetFrom3Reg(hi)
   265  
   266  	case ssa.OpAMD64BLSIQ, ssa.OpAMD64BLSIL,
   267  		ssa.OpAMD64BLSMSKQ, ssa.OpAMD64BLSMSKL,
   268  		ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL,
   269  		ssa.OpAMD64TZCNTQ, ssa.OpAMD64TZCNTL:
   270  		p := s.Prog(v.Op.Asm())
   271  		p.From.Type = obj.TYPE_REG
   272  		p.From.Reg = v.Args[0].Reg()
   273  		p.To.Type = obj.TYPE_REG
   274  		p.To.Reg = v.Reg()
   275  
   276  	case ssa.OpAMD64ANDNQ, ssa.OpAMD64ANDNL:
   277  		p := s.Prog(v.Op.Asm())
   278  		p.From.Type = obj.TYPE_REG
   279  		p.From.Reg = v.Args[0].Reg()
   280  		p.To.Type = obj.TYPE_REG
   281  		p.To.Reg = v.Reg()
   282  		p.SetFrom3Reg(v.Args[1].Reg())
   283  
   284  	case ssa.OpAMD64DIVQU, ssa.OpAMD64DIVLU, ssa.OpAMD64DIVWU:
   285  		// Arg[0] (the dividend) is in AX.
   286  		// Arg[1] (the divisor) can be in any other register.
   287  		// Result[0] (the quotient) is in AX.
   288  		// Result[1] (the remainder) is in DX.
   289  		r := v.Args[1].Reg()
   290  
   291  		// Zero extend dividend.
   292  		c := s.Prog(x86.AXORL)
   293  		c.From.Type = obj.TYPE_REG
   294  		c.From.Reg = x86.REG_DX
   295  		c.To.Type = obj.TYPE_REG
   296  		c.To.Reg = x86.REG_DX
   297  
   298  		// Issue divide.
   299  		p := s.Prog(v.Op.Asm())
   300  		p.From.Type = obj.TYPE_REG
   301  		p.From.Reg = r
   302  
   303  	case ssa.OpAMD64DIVQ, ssa.OpAMD64DIVL, ssa.OpAMD64DIVW:
   304  		// Arg[0] (the dividend) is in AX.
   305  		// Arg[1] (the divisor) can be in any other register.
   306  		// Result[0] (the quotient) is in AX.
   307  		// Result[1] (the remainder) is in DX.
   308  		r := v.Args[1].Reg()
   309  		var j1 *obj.Prog
   310  
   311  		// CPU faults upon signed overflow, which occurs when the most
   312  		// negative int is divided by -1. Handle divide by -1 as a special case.
   313  		if ssa.DivisionNeedsFixUp(v) {
   314  			var c *obj.Prog
   315  			switch v.Op {
   316  			case ssa.OpAMD64DIVQ:
   317  				c = s.Prog(x86.ACMPQ)
   318  			case ssa.OpAMD64DIVL:
   319  				c = s.Prog(x86.ACMPL)
   320  			case ssa.OpAMD64DIVW:
   321  				c = s.Prog(x86.ACMPW)
   322  			}
   323  			c.From.Type = obj.TYPE_REG
   324  			c.From.Reg = r
   325  			c.To.Type = obj.TYPE_CONST
   326  			c.To.Offset = -1
   327  			j1 = s.Prog(x86.AJEQ)
   328  			j1.To.Type = obj.TYPE_BRANCH
   329  		}
   330  
   331  		// Sign extend dividend.
   332  		switch v.Op {
   333  		case ssa.OpAMD64DIVQ:
   334  			s.Prog(x86.ACQO)
   335  		case ssa.OpAMD64DIVL:
   336  			s.Prog(x86.ACDQ)
   337  		case ssa.OpAMD64DIVW:
   338  			s.Prog(x86.ACWD)
   339  		}
   340  
   341  		// Issue divide.
   342  		p := s.Prog(v.Op.Asm())
   343  		p.From.Type = obj.TYPE_REG
   344  		p.From.Reg = r
   345  
   346  		if j1 != nil {
   347  			// Skip over -1 fixup code.
   348  			j2 := s.Prog(obj.AJMP)
   349  			j2.To.Type = obj.TYPE_BRANCH
   350  
   351  			// Issue -1 fixup code.
   352  			// n / -1 = -n
   353  			var n1 *obj.Prog
   354  			switch v.Op {
   355  			case ssa.OpAMD64DIVQ:
   356  				n1 = s.Prog(x86.ANEGQ)
   357  			case ssa.OpAMD64DIVL:
   358  				n1 = s.Prog(x86.ANEGL)
   359  			case ssa.OpAMD64DIVW:
   360  				n1 = s.Prog(x86.ANEGW)
   361  			}
   362  			n1.To.Type = obj.TYPE_REG
   363  			n1.To.Reg = x86.REG_AX
   364  
   365  			// n % -1 == 0
   366  			n2 := s.Prog(x86.AXORL)
   367  			n2.From.Type = obj.TYPE_REG
   368  			n2.From.Reg = x86.REG_DX
   369  			n2.To.Type = obj.TYPE_REG
   370  			n2.To.Reg = x86.REG_DX
   371  
   372  			// TODO(khr): issue only the -1 fixup code we need.
   373  			// For instance, if only the quotient is used, no point in zeroing the remainder.
   374  
   375  			j1.To.SetTarget(n1)
   376  			j2.To.SetTarget(s.Pc())
   377  		}
   378  
   379  	case ssa.OpAMD64HMULQ, ssa.OpAMD64HMULL, ssa.OpAMD64HMULQU, ssa.OpAMD64HMULLU:
   380  		// the frontend rewrites constant division by 8/16/32 bit integers into
   381  		// HMUL by a constant
   382  		// SSA rewrites generate the 64 bit versions
   383  
   384  		// Arg[0] is already in AX as it's the only register we allow
   385  		// and DX is the only output we care about (the high bits)
   386  		p := s.Prog(v.Op.Asm())
   387  		p.From.Type = obj.TYPE_REG
   388  		p.From.Reg = v.Args[1].Reg()
   389  
   390  		// IMULB puts the high portion in AH instead of DL,
   391  		// so move it to DL for consistency
   392  		if v.Type.Size() == 1 {
   393  			m := s.Prog(x86.AMOVB)
   394  			m.From.Type = obj.TYPE_REG
   395  			m.From.Reg = x86.REG_AH
   396  			m.To.Type = obj.TYPE_REG
   397  			m.To.Reg = x86.REG_DX
   398  		}
   399  
   400  	case ssa.OpAMD64MULQU, ssa.OpAMD64MULLU:
   401  		// Arg[0] is already in AX as it's the only register we allow
   402  		// results lo in AX
   403  		p := s.Prog(v.Op.Asm())
   404  		p.From.Type = obj.TYPE_REG
   405  		p.From.Reg = v.Args[1].Reg()
   406  
   407  	case ssa.OpAMD64MULQU2:
   408  		// Arg[0] is already in AX as it's the only register we allow
   409  		// results hi in DX, lo in AX
   410  		p := s.Prog(v.Op.Asm())
   411  		p.From.Type = obj.TYPE_REG
   412  		p.From.Reg = v.Args[1].Reg()
   413  
   414  	case ssa.OpAMD64DIVQU2:
   415  		// Arg[0], Arg[1] are already in Dx, AX, as they're the only registers we allow
   416  		// results q in AX, r in DX
   417  		p := s.Prog(v.Op.Asm())
   418  		p.From.Type = obj.TYPE_REG
   419  		p.From.Reg = v.Args[2].Reg()
   420  
   421  	case ssa.OpAMD64AVGQU:
   422  		// compute (x+y)/2 unsigned.
   423  		// Do a 64-bit add, the overflow goes into the carry.
   424  		// Shift right once and pull the carry back into the 63rd bit.
   425  		p := s.Prog(x86.AADDQ)
   426  		p.From.Type = obj.TYPE_REG
   427  		p.To.Type = obj.TYPE_REG
   428  		p.To.Reg = v.Reg()
   429  		p.From.Reg = v.Args[1].Reg()
   430  		p = s.Prog(x86.ARCRQ)
   431  		p.From.Type = obj.TYPE_CONST
   432  		p.From.Offset = 1
   433  		p.To.Type = obj.TYPE_REG
   434  		p.To.Reg = v.Reg()
   435  
   436  	case ssa.OpAMD64ADDQcarry, ssa.OpAMD64ADCQ:
   437  		r := v.Reg0()
   438  		r0 := v.Args[0].Reg()
   439  		r1 := v.Args[1].Reg()
   440  		switch r {
   441  		case r0:
   442  			p := s.Prog(v.Op.Asm())
   443  			p.From.Type = obj.TYPE_REG
   444  			p.From.Reg = r1
   445  			p.To.Type = obj.TYPE_REG
   446  			p.To.Reg = r
   447  		case r1:
   448  			p := s.Prog(v.Op.Asm())
   449  			p.From.Type = obj.TYPE_REG
   450  			p.From.Reg = r0
   451  			p.To.Type = obj.TYPE_REG
   452  			p.To.Reg = r
   453  		default:
   454  			v.Fatalf("output not in same register as an input %s", v.LongString())
   455  		}
   456  
   457  	case ssa.OpAMD64SUBQborrow, ssa.OpAMD64SBBQ:
   458  		p := s.Prog(v.Op.Asm())
   459  		p.From.Type = obj.TYPE_REG
   460  		p.From.Reg = v.Args[1].Reg()
   461  		p.To.Type = obj.TYPE_REG
   462  		p.To.Reg = v.Reg0()
   463  
   464  	case ssa.OpAMD64ADDQconstcarry, ssa.OpAMD64ADCQconst, ssa.OpAMD64SUBQconstborrow, ssa.OpAMD64SBBQconst:
   465  		p := s.Prog(v.Op.Asm())
   466  		p.From.Type = obj.TYPE_CONST
   467  		p.From.Offset = v.AuxInt
   468  		p.To.Type = obj.TYPE_REG
   469  		p.To.Reg = v.Reg0()
   470  
   471  	case ssa.OpAMD64ADDQconst, ssa.OpAMD64ADDLconst:
   472  		r := v.Reg()
   473  		a := v.Args[0].Reg()
   474  		if r == a {
   475  			switch v.AuxInt {
   476  			case 1:
   477  				var asm obj.As
   478  				// Software optimization manual recommends add $1,reg.
   479  				// But inc/dec is 1 byte smaller. ICC always uses inc
   480  				// Clang/GCC choose depending on flags, but prefer add.
   481  				// Experiments show that inc/dec is both a little faster
   482  				// and make a binary a little smaller.
   483  				if v.Op == ssa.OpAMD64ADDQconst {
   484  					asm = x86.AINCQ
   485  				} else {
   486  					asm = x86.AINCL
   487  				}
   488  				p := s.Prog(asm)
   489  				p.To.Type = obj.TYPE_REG
   490  				p.To.Reg = r
   491  				return
   492  			case -1:
   493  				var asm obj.As
   494  				if v.Op == ssa.OpAMD64ADDQconst {
   495  					asm = x86.ADECQ
   496  				} else {
   497  					asm = x86.ADECL
   498  				}
   499  				p := s.Prog(asm)
   500  				p.To.Type = obj.TYPE_REG
   501  				p.To.Reg = r
   502  				return
   503  			case 0x80:
   504  				// 'SUBQ $-0x80, r' is shorter to encode than
   505  				// and functionally equivalent to 'ADDQ $0x80, r'.
   506  				asm := x86.ASUBL
   507  				if v.Op == ssa.OpAMD64ADDQconst {
   508  					asm = x86.ASUBQ
   509  				}
   510  				p := s.Prog(asm)
   511  				p.From.Type = obj.TYPE_CONST
   512  				p.From.Offset = -0x80
   513  				p.To.Type = obj.TYPE_REG
   514  				p.To.Reg = r
   515  				return
   516  
   517  			}
   518  			p := s.Prog(v.Op.Asm())
   519  			p.From.Type = obj.TYPE_CONST
   520  			p.From.Offset = v.AuxInt
   521  			p.To.Type = obj.TYPE_REG
   522  			p.To.Reg = r
   523  			return
   524  		}
   525  		var asm obj.As
   526  		if v.Op == ssa.OpAMD64ADDQconst {
   527  			asm = x86.ALEAQ
   528  		} else {
   529  			asm = x86.ALEAL
   530  		}
   531  		p := s.Prog(asm)
   532  		p.From.Type = obj.TYPE_MEM
   533  		p.From.Reg = a
   534  		p.From.Offset = v.AuxInt
   535  		p.To.Type = obj.TYPE_REG
   536  		p.To.Reg = r
   537  
   538  	case ssa.OpAMD64CMOVQEQ, ssa.OpAMD64CMOVLEQ, ssa.OpAMD64CMOVWEQ,
   539  		ssa.OpAMD64CMOVQLT, ssa.OpAMD64CMOVLLT, ssa.OpAMD64CMOVWLT,
   540  		ssa.OpAMD64CMOVQNE, ssa.OpAMD64CMOVLNE, ssa.OpAMD64CMOVWNE,
   541  		ssa.OpAMD64CMOVQGT, ssa.OpAMD64CMOVLGT, ssa.OpAMD64CMOVWGT,
   542  		ssa.OpAMD64CMOVQLE, ssa.OpAMD64CMOVLLE, ssa.OpAMD64CMOVWLE,
   543  		ssa.OpAMD64CMOVQGE, ssa.OpAMD64CMOVLGE, ssa.OpAMD64CMOVWGE,
   544  		ssa.OpAMD64CMOVQHI, ssa.OpAMD64CMOVLHI, ssa.OpAMD64CMOVWHI,
   545  		ssa.OpAMD64CMOVQLS, ssa.OpAMD64CMOVLLS, ssa.OpAMD64CMOVWLS,
   546  		ssa.OpAMD64CMOVQCC, ssa.OpAMD64CMOVLCC, ssa.OpAMD64CMOVWCC,
   547  		ssa.OpAMD64CMOVQCS, ssa.OpAMD64CMOVLCS, ssa.OpAMD64CMOVWCS,
   548  		ssa.OpAMD64CMOVQGTF, ssa.OpAMD64CMOVLGTF, ssa.OpAMD64CMOVWGTF,
   549  		ssa.OpAMD64CMOVQGEF, ssa.OpAMD64CMOVLGEF, ssa.OpAMD64CMOVWGEF:
   550  		p := s.Prog(v.Op.Asm())
   551  		p.From.Type = obj.TYPE_REG
   552  		p.From.Reg = v.Args[1].Reg()
   553  		p.To.Type = obj.TYPE_REG
   554  		p.To.Reg = v.Reg()
   555  
   556  	case ssa.OpAMD64CMOVQNEF, ssa.OpAMD64CMOVLNEF, ssa.OpAMD64CMOVWNEF:
   557  		// Flag condition: ^ZERO || PARITY
   558  		// Generate:
   559  		//   CMOV*NE  SRC,DST
   560  		//   CMOV*PS  SRC,DST
   561  		p := s.Prog(v.Op.Asm())
   562  		p.From.Type = obj.TYPE_REG
   563  		p.From.Reg = v.Args[1].Reg()
   564  		p.To.Type = obj.TYPE_REG
   565  		p.To.Reg = v.Reg()
   566  		var q *obj.Prog
   567  		if v.Op == ssa.OpAMD64CMOVQNEF {
   568  			q = s.Prog(x86.ACMOVQPS)
   569  		} else if v.Op == ssa.OpAMD64CMOVLNEF {
   570  			q = s.Prog(x86.ACMOVLPS)
   571  		} else {
   572  			q = s.Prog(x86.ACMOVWPS)
   573  		}
   574  		q.From.Type = obj.TYPE_REG
   575  		q.From.Reg = v.Args[1].Reg()
   576  		q.To.Type = obj.TYPE_REG
   577  		q.To.Reg = v.Reg()
   578  
   579  	case ssa.OpAMD64CMOVQEQF, ssa.OpAMD64CMOVLEQF, ssa.OpAMD64CMOVWEQF:
   580  		// Flag condition: ZERO && !PARITY
   581  		// Generate:
   582  		//   MOV      SRC,AX
   583  		//   CMOV*NE  DST,AX
   584  		//   CMOV*PC  AX,DST
   585  		//
   586  		// TODO(rasky): we could generate:
   587  		//   CMOV*NE  DST,SRC
   588  		//   CMOV*PC  SRC,DST
   589  		// But this requires a way for regalloc to know that SRC might be
   590  		// clobbered by this instruction.
   591  		if v.Args[1].Reg() != x86.REG_AX {
   592  			opregreg(s, moveByType(v.Type), x86.REG_AX, v.Args[1].Reg())
   593  		}
   594  		p := s.Prog(v.Op.Asm())
   595  		p.From.Type = obj.TYPE_REG
   596  		p.From.Reg = v.Reg()
   597  		p.To.Type = obj.TYPE_REG
   598  		p.To.Reg = x86.REG_AX
   599  		var q *obj.Prog
   600  		if v.Op == ssa.OpAMD64CMOVQEQF {
   601  			q = s.Prog(x86.ACMOVQPC)
   602  		} else if v.Op == ssa.OpAMD64CMOVLEQF {
   603  			q = s.Prog(x86.ACMOVLPC)
   604  		} else {
   605  			q = s.Prog(x86.ACMOVWPC)
   606  		}
   607  		q.From.Type = obj.TYPE_REG
   608  		q.From.Reg = x86.REG_AX
   609  		q.To.Type = obj.TYPE_REG
   610  		q.To.Reg = v.Reg()
   611  
   612  	case ssa.OpAMD64MULQconst, ssa.OpAMD64MULLconst:
   613  		r := v.Reg()
   614  		p := s.Prog(v.Op.Asm())
   615  		p.From.Type = obj.TYPE_CONST
   616  		p.From.Offset = v.AuxInt
   617  		p.To.Type = obj.TYPE_REG
   618  		p.To.Reg = r
   619  		p.SetFrom3Reg(v.Args[0].Reg())
   620  
   621  	case ssa.OpAMD64ANDQconst:
   622  		asm := v.Op.Asm()
   623  		// If the constant is positive and fits into 32 bits, use ANDL.
   624  		// This saves a few bytes of encoding.
   625  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   626  			asm = x86.AANDL
   627  		}
   628  		p := s.Prog(asm)
   629  		p.From.Type = obj.TYPE_CONST
   630  		p.From.Offset = v.AuxInt
   631  		p.To.Type = obj.TYPE_REG
   632  		p.To.Reg = v.Reg()
   633  
   634  	case ssa.OpAMD64SUBQconst, ssa.OpAMD64SUBLconst,
   635  		ssa.OpAMD64ANDLconst,
   636  		ssa.OpAMD64ORQconst, ssa.OpAMD64ORLconst,
   637  		ssa.OpAMD64XORQconst, ssa.OpAMD64XORLconst,
   638  		ssa.OpAMD64SHLQconst, ssa.OpAMD64SHLLconst,
   639  		ssa.OpAMD64SHRQconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SHRWconst, ssa.OpAMD64SHRBconst,
   640  		ssa.OpAMD64SARQconst, ssa.OpAMD64SARLconst, ssa.OpAMD64SARWconst, ssa.OpAMD64SARBconst,
   641  		ssa.OpAMD64ROLQconst, ssa.OpAMD64ROLLconst, ssa.OpAMD64ROLWconst, ssa.OpAMD64ROLBconst:
   642  		p := s.Prog(v.Op.Asm())
   643  		p.From.Type = obj.TYPE_CONST
   644  		p.From.Offset = v.AuxInt
   645  		p.To.Type = obj.TYPE_REG
   646  		p.To.Reg = v.Reg()
   647  	case ssa.OpAMD64SBBQcarrymask, ssa.OpAMD64SBBLcarrymask:
   648  		r := v.Reg()
   649  		p := s.Prog(v.Op.Asm())
   650  		p.From.Type = obj.TYPE_REG
   651  		p.From.Reg = r
   652  		p.To.Type = obj.TYPE_REG
   653  		p.To.Reg = r
   654  	case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8,
   655  		ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8,
   656  		ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   657  		p := s.Prog(v.Op.Asm())
   658  		memIdx(&p.From, v)
   659  		o := v.Reg()
   660  		p.To.Type = obj.TYPE_REG
   661  		p.To.Reg = o
   662  		if v.AuxInt != 0 && v.Aux == nil {
   663  			// Emit an additional LEA to add the displacement instead of creating a slow 3 operand LEA.
   664  			switch v.Op {
   665  			case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8:
   666  				p = s.Prog(x86.ALEAQ)
   667  			case ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8:
   668  				p = s.Prog(x86.ALEAL)
   669  			case ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   670  				p = s.Prog(x86.ALEAW)
   671  			}
   672  			p.From.Type = obj.TYPE_MEM
   673  			p.From.Reg = o
   674  			p.To.Type = obj.TYPE_REG
   675  			p.To.Reg = o
   676  		}
   677  		ssagen.AddAux(&p.From, v)
   678  	case ssa.OpAMD64LEAQ, ssa.OpAMD64LEAL, ssa.OpAMD64LEAW:
   679  		p := s.Prog(v.Op.Asm())
   680  		p.From.Type = obj.TYPE_MEM
   681  		p.From.Reg = v.Args[0].Reg()
   682  		ssagen.AddAux(&p.From, v)
   683  		p.To.Type = obj.TYPE_REG
   684  		p.To.Reg = v.Reg()
   685  	case ssa.OpAMD64CMPQ, ssa.OpAMD64CMPL, ssa.OpAMD64CMPW, ssa.OpAMD64CMPB,
   686  		ssa.OpAMD64TESTQ, ssa.OpAMD64TESTL, ssa.OpAMD64TESTW, ssa.OpAMD64TESTB,
   687  		ssa.OpAMD64BTL, ssa.OpAMD64BTQ:
   688  		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
   689  	case ssa.OpAMD64UCOMISS, ssa.OpAMD64UCOMISD:
   690  		// Go assembler has swapped operands for UCOMISx relative to CMP,
   691  		// must account for that right here.
   692  		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg())
   693  	case ssa.OpAMD64CMPQconst, ssa.OpAMD64CMPLconst, ssa.OpAMD64CMPWconst, ssa.OpAMD64CMPBconst:
   694  		p := s.Prog(v.Op.Asm())
   695  		p.From.Type = obj.TYPE_REG
   696  		p.From.Reg = v.Args[0].Reg()
   697  		p.To.Type = obj.TYPE_CONST
   698  		p.To.Offset = v.AuxInt
   699  	case ssa.OpAMD64BTLconst, ssa.OpAMD64BTQconst,
   700  		ssa.OpAMD64TESTQconst, ssa.OpAMD64TESTLconst, ssa.OpAMD64TESTWconst, ssa.OpAMD64TESTBconst,
   701  		ssa.OpAMD64BTSLconst, ssa.OpAMD64BTSQconst,
   702  		ssa.OpAMD64BTCLconst, ssa.OpAMD64BTCQconst,
   703  		ssa.OpAMD64BTRLconst, ssa.OpAMD64BTRQconst:
   704  		op := v.Op
   705  		if op == ssa.OpAMD64BTQconst && v.AuxInt < 32 {
   706  			// Emit 32-bit version because it's shorter
   707  			op = ssa.OpAMD64BTLconst
   708  		}
   709  		p := s.Prog(op.Asm())
   710  		p.From.Type = obj.TYPE_CONST
   711  		p.From.Offset = v.AuxInt
   712  		p.To.Type = obj.TYPE_REG
   713  		p.To.Reg = v.Args[0].Reg()
   714  	case ssa.OpAMD64CMPQload, ssa.OpAMD64CMPLload, ssa.OpAMD64CMPWload, ssa.OpAMD64CMPBload:
   715  		p := s.Prog(v.Op.Asm())
   716  		p.From.Type = obj.TYPE_MEM
   717  		p.From.Reg = v.Args[0].Reg()
   718  		ssagen.AddAux(&p.From, v)
   719  		p.To.Type = obj.TYPE_REG
   720  		p.To.Reg = v.Args[1].Reg()
   721  	case ssa.OpAMD64CMPQconstload, ssa.OpAMD64CMPLconstload, ssa.OpAMD64CMPWconstload, ssa.OpAMD64CMPBconstload:
   722  		sc := v.AuxValAndOff()
   723  		p := s.Prog(v.Op.Asm())
   724  		p.From.Type = obj.TYPE_MEM
   725  		p.From.Reg = v.Args[0].Reg()
   726  		ssagen.AddAux2(&p.From, v, sc.Off64())
   727  		p.To.Type = obj.TYPE_CONST
   728  		p.To.Offset = sc.Val64()
   729  	case ssa.OpAMD64CMPQloadidx8, ssa.OpAMD64CMPQloadidx1, ssa.OpAMD64CMPLloadidx4, ssa.OpAMD64CMPLloadidx1, ssa.OpAMD64CMPWloadidx2, ssa.OpAMD64CMPWloadidx1, ssa.OpAMD64CMPBloadidx1:
   730  		p := s.Prog(v.Op.Asm())
   731  		memIdx(&p.From, v)
   732  		ssagen.AddAux(&p.From, v)
   733  		p.To.Type = obj.TYPE_REG
   734  		p.To.Reg = v.Args[2].Reg()
   735  	case ssa.OpAMD64CMPQconstloadidx8, ssa.OpAMD64CMPQconstloadidx1, ssa.OpAMD64CMPLconstloadidx4, ssa.OpAMD64CMPLconstloadidx1, ssa.OpAMD64CMPWconstloadidx2, ssa.OpAMD64CMPWconstloadidx1, ssa.OpAMD64CMPBconstloadidx1:
   736  		sc := v.AuxValAndOff()
   737  		p := s.Prog(v.Op.Asm())
   738  		memIdx(&p.From, v)
   739  		ssagen.AddAux2(&p.From, v, sc.Off64())
   740  		p.To.Type = obj.TYPE_CONST
   741  		p.To.Offset = sc.Val64()
   742  	case ssa.OpAMD64MOVLconst, ssa.OpAMD64MOVQconst:
   743  		x := v.Reg()
   744  
   745  		// If flags aren't live (indicated by v.Aux == nil),
   746  		// then we can rewrite MOV $0, AX into XOR AX, AX.
   747  		if v.AuxInt == 0 && v.Aux == nil {
   748  			p := s.Prog(x86.AXORL)
   749  			p.From.Type = obj.TYPE_REG
   750  			p.From.Reg = x
   751  			p.To.Type = obj.TYPE_REG
   752  			p.To.Reg = x
   753  			break
   754  		}
   755  
   756  		asm := v.Op.Asm()
   757  		// Use MOVL to move a small constant into a register
   758  		// when the constant is positive and fits into 32 bits.
   759  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   760  			// The upper 32bit are zeroed automatically when using MOVL.
   761  			asm = x86.AMOVL
   762  		}
   763  		p := s.Prog(asm)
   764  		p.From.Type = obj.TYPE_CONST
   765  		p.From.Offset = v.AuxInt
   766  		p.To.Type = obj.TYPE_REG
   767  		p.To.Reg = x
   768  	case ssa.OpAMD64MOVSSconst, ssa.OpAMD64MOVSDconst:
   769  		x := v.Reg()
   770  		p := s.Prog(v.Op.Asm())
   771  		p.From.Type = obj.TYPE_FCONST
   772  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   773  		p.To.Type = obj.TYPE_REG
   774  		p.To.Reg = x
   775  	case ssa.OpAMD64MOVQload, ssa.OpAMD64MOVLload, ssa.OpAMD64MOVWload, ssa.OpAMD64MOVBload, ssa.OpAMD64MOVOload,
   776  		ssa.OpAMD64MOVSSload, ssa.OpAMD64MOVSDload, ssa.OpAMD64MOVBQSXload, ssa.OpAMD64MOVWQSXload, ssa.OpAMD64MOVLQSXload,
   777  		ssa.OpAMD64MOVBEQload, ssa.OpAMD64MOVBELload:
   778  		p := s.Prog(v.Op.Asm())
   779  		p.From.Type = obj.TYPE_MEM
   780  		p.From.Reg = v.Args[0].Reg()
   781  		ssagen.AddAux(&p.From, v)
   782  		p.To.Type = obj.TYPE_REG
   783  		p.To.Reg = v.Reg()
   784  	case ssa.OpAMD64MOVBloadidx1, ssa.OpAMD64MOVWloadidx1, ssa.OpAMD64MOVLloadidx1, ssa.OpAMD64MOVQloadidx1, ssa.OpAMD64MOVSSloadidx1, ssa.OpAMD64MOVSDloadidx1,
   785  		ssa.OpAMD64MOVQloadidx8, ssa.OpAMD64MOVSDloadidx8, ssa.OpAMD64MOVLloadidx8, ssa.OpAMD64MOVLloadidx4, ssa.OpAMD64MOVSSloadidx4, ssa.OpAMD64MOVWloadidx2:
   786  		p := s.Prog(v.Op.Asm())
   787  		memIdx(&p.From, v)
   788  		ssagen.AddAux(&p.From, v)
   789  		p.To.Type = obj.TYPE_REG
   790  		p.To.Reg = v.Reg()
   791  	case ssa.OpAMD64MOVQstore, ssa.OpAMD64MOVSSstore, ssa.OpAMD64MOVSDstore, ssa.OpAMD64MOVLstore, ssa.OpAMD64MOVWstore, ssa.OpAMD64MOVBstore, ssa.OpAMD64MOVOstore,
   792  		ssa.OpAMD64ADDQmodify, ssa.OpAMD64SUBQmodify, ssa.OpAMD64ANDQmodify, ssa.OpAMD64ORQmodify, ssa.OpAMD64XORQmodify,
   793  		ssa.OpAMD64ADDLmodify, ssa.OpAMD64SUBLmodify, ssa.OpAMD64ANDLmodify, ssa.OpAMD64ORLmodify, ssa.OpAMD64XORLmodify,
   794  		ssa.OpAMD64MOVBEQstore, ssa.OpAMD64MOVBELstore:
   795  		p := s.Prog(v.Op.Asm())
   796  		p.From.Type = obj.TYPE_REG
   797  		p.From.Reg = v.Args[1].Reg()
   798  		p.To.Type = obj.TYPE_MEM
   799  		p.To.Reg = v.Args[0].Reg()
   800  		ssagen.AddAux(&p.To, v)
   801  	case ssa.OpAMD64MOVBstoreidx1, ssa.OpAMD64MOVWstoreidx1, ssa.OpAMD64MOVLstoreidx1, ssa.OpAMD64MOVQstoreidx1, ssa.OpAMD64MOVSSstoreidx1, ssa.OpAMD64MOVSDstoreidx1,
   802  		ssa.OpAMD64MOVQstoreidx8, ssa.OpAMD64MOVSDstoreidx8, ssa.OpAMD64MOVLstoreidx8, ssa.OpAMD64MOVSSstoreidx4, ssa.OpAMD64MOVLstoreidx4, ssa.OpAMD64MOVWstoreidx2,
   803  		ssa.OpAMD64ADDLmodifyidx1, ssa.OpAMD64ADDLmodifyidx4, ssa.OpAMD64ADDLmodifyidx8, ssa.OpAMD64ADDQmodifyidx1, ssa.OpAMD64ADDQmodifyidx8,
   804  		ssa.OpAMD64SUBLmodifyidx1, ssa.OpAMD64SUBLmodifyidx4, ssa.OpAMD64SUBLmodifyidx8, ssa.OpAMD64SUBQmodifyidx1, ssa.OpAMD64SUBQmodifyidx8,
   805  		ssa.OpAMD64ANDLmodifyidx1, ssa.OpAMD64ANDLmodifyidx4, ssa.OpAMD64ANDLmodifyidx8, ssa.OpAMD64ANDQmodifyidx1, ssa.OpAMD64ANDQmodifyidx8,
   806  		ssa.OpAMD64ORLmodifyidx1, ssa.OpAMD64ORLmodifyidx4, ssa.OpAMD64ORLmodifyidx8, ssa.OpAMD64ORQmodifyidx1, ssa.OpAMD64ORQmodifyidx8,
   807  		ssa.OpAMD64XORLmodifyidx1, ssa.OpAMD64XORLmodifyidx4, ssa.OpAMD64XORLmodifyidx8, ssa.OpAMD64XORQmodifyidx1, ssa.OpAMD64XORQmodifyidx8:
   808  		p := s.Prog(v.Op.Asm())
   809  		p.From.Type = obj.TYPE_REG
   810  		p.From.Reg = v.Args[2].Reg()
   811  		memIdx(&p.To, v)
   812  		ssagen.AddAux(&p.To, v)
   813  	case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify:
   814  		sc := v.AuxValAndOff()
   815  		off := sc.Off64()
   816  		val := sc.Val()
   817  		if val == 1 || val == -1 {
   818  			var asm obj.As
   819  			if v.Op == ssa.OpAMD64ADDQconstmodify {
   820  				if val == 1 {
   821  					asm = x86.AINCQ
   822  				} else {
   823  					asm = x86.ADECQ
   824  				}
   825  			} else {
   826  				if val == 1 {
   827  					asm = x86.AINCL
   828  				} else {
   829  					asm = x86.ADECL
   830  				}
   831  			}
   832  			p := s.Prog(asm)
   833  			p.To.Type = obj.TYPE_MEM
   834  			p.To.Reg = v.Args[0].Reg()
   835  			ssagen.AddAux2(&p.To, v, off)
   836  			break
   837  		}
   838  		fallthrough
   839  	case ssa.OpAMD64ANDQconstmodify, ssa.OpAMD64ANDLconstmodify, ssa.OpAMD64ORQconstmodify, ssa.OpAMD64ORLconstmodify,
   840  		ssa.OpAMD64XORQconstmodify, ssa.OpAMD64XORLconstmodify:
   841  		sc := v.AuxValAndOff()
   842  		off := sc.Off64()
   843  		val := sc.Val64()
   844  		p := s.Prog(v.Op.Asm())
   845  		p.From.Type = obj.TYPE_CONST
   846  		p.From.Offset = val
   847  		p.To.Type = obj.TYPE_MEM
   848  		p.To.Reg = v.Args[0].Reg()
   849  		ssagen.AddAux2(&p.To, v, off)
   850  
   851  	case ssa.OpAMD64MOVQstoreconst, ssa.OpAMD64MOVLstoreconst, ssa.OpAMD64MOVWstoreconst, ssa.OpAMD64MOVBstoreconst:
   852  		p := s.Prog(v.Op.Asm())
   853  		p.From.Type = obj.TYPE_CONST
   854  		sc := v.AuxValAndOff()
   855  		p.From.Offset = sc.Val64()
   856  		p.To.Type = obj.TYPE_MEM
   857  		p.To.Reg = v.Args[0].Reg()
   858  		ssagen.AddAux2(&p.To, v, sc.Off64())
   859  	case ssa.OpAMD64MOVOstoreconst:
   860  		sc := v.AuxValAndOff()
   861  		if sc.Val() != 0 {
   862  			v.Fatalf("MOVO for non zero constants not implemented: %s", v.LongString())
   863  		}
   864  
   865  		if s.ABI != obj.ABIInternal {
   866  			// zero X15 manually
   867  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   868  		}
   869  		p := s.Prog(v.Op.Asm())
   870  		p.From.Type = obj.TYPE_REG
   871  		p.From.Reg = x86.REG_X15
   872  		p.To.Type = obj.TYPE_MEM
   873  		p.To.Reg = v.Args[0].Reg()
   874  		ssagen.AddAux2(&p.To, v, sc.Off64())
   875  
   876  	case ssa.OpAMD64MOVQstoreconstidx1, ssa.OpAMD64MOVQstoreconstidx8, ssa.OpAMD64MOVLstoreconstidx1, ssa.OpAMD64MOVLstoreconstidx4, ssa.OpAMD64MOVWstoreconstidx1, ssa.OpAMD64MOVWstoreconstidx2, ssa.OpAMD64MOVBstoreconstidx1,
   877  		ssa.OpAMD64ADDLconstmodifyidx1, ssa.OpAMD64ADDLconstmodifyidx4, ssa.OpAMD64ADDLconstmodifyidx8, ssa.OpAMD64ADDQconstmodifyidx1, ssa.OpAMD64ADDQconstmodifyidx8,
   878  		ssa.OpAMD64ANDLconstmodifyidx1, ssa.OpAMD64ANDLconstmodifyidx4, ssa.OpAMD64ANDLconstmodifyidx8, ssa.OpAMD64ANDQconstmodifyidx1, ssa.OpAMD64ANDQconstmodifyidx8,
   879  		ssa.OpAMD64ORLconstmodifyidx1, ssa.OpAMD64ORLconstmodifyidx4, ssa.OpAMD64ORLconstmodifyidx8, ssa.OpAMD64ORQconstmodifyidx1, ssa.OpAMD64ORQconstmodifyidx8,
   880  		ssa.OpAMD64XORLconstmodifyidx1, ssa.OpAMD64XORLconstmodifyidx4, ssa.OpAMD64XORLconstmodifyidx8, ssa.OpAMD64XORQconstmodifyidx1, ssa.OpAMD64XORQconstmodifyidx8:
   881  		p := s.Prog(v.Op.Asm())
   882  		p.From.Type = obj.TYPE_CONST
   883  		sc := v.AuxValAndOff()
   884  		p.From.Offset = sc.Val64()
   885  		switch {
   886  		case p.As == x86.AADDQ && p.From.Offset == 1:
   887  			p.As = x86.AINCQ
   888  			p.From.Type = obj.TYPE_NONE
   889  		case p.As == x86.AADDQ && p.From.Offset == -1:
   890  			p.As = x86.ADECQ
   891  			p.From.Type = obj.TYPE_NONE
   892  		case p.As == x86.AADDL && p.From.Offset == 1:
   893  			p.As = x86.AINCL
   894  			p.From.Type = obj.TYPE_NONE
   895  		case p.As == x86.AADDL && p.From.Offset == -1:
   896  			p.As = x86.ADECL
   897  			p.From.Type = obj.TYPE_NONE
   898  		}
   899  		memIdx(&p.To, v)
   900  		ssagen.AddAux2(&p.To, v, sc.Off64())
   901  	case ssa.OpAMD64MOVLQSX, ssa.OpAMD64MOVWQSX, ssa.OpAMD64MOVBQSX, ssa.OpAMD64MOVLQZX, ssa.OpAMD64MOVWQZX, ssa.OpAMD64MOVBQZX,
   902  		ssa.OpAMD64CVTTSS2SL, ssa.OpAMD64CVTTSD2SL, ssa.OpAMD64CVTTSS2SQ, ssa.OpAMD64CVTTSD2SQ,
   903  		ssa.OpAMD64CVTSS2SD, ssa.OpAMD64CVTSD2SS:
   904  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())
   905  	case ssa.OpAMD64CVTSL2SD, ssa.OpAMD64CVTSQ2SD, ssa.OpAMD64CVTSQ2SS, ssa.OpAMD64CVTSL2SS:
   906  		r := v.Reg()
   907  		// Break false dependency on destination register.
   908  		opregreg(s, x86.AXORPS, r, r)
   909  		opregreg(s, v.Op.Asm(), r, v.Args[0].Reg())
   910  	case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i, ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
   911  		var p *obj.Prog
   912  		switch v.Op {
   913  		case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i:
   914  			p = s.Prog(x86.AMOVQ)
   915  		case ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
   916  			p = s.Prog(x86.AMOVL)
   917  		}
   918  		p.From.Type = obj.TYPE_REG
   919  		p.From.Reg = v.Args[0].Reg()
   920  		p.To.Type = obj.TYPE_REG
   921  		p.To.Reg = v.Reg()
   922  	case ssa.OpAMD64ADDQload, ssa.OpAMD64ADDLload, ssa.OpAMD64SUBQload, ssa.OpAMD64SUBLload,
   923  		ssa.OpAMD64ANDQload, ssa.OpAMD64ANDLload, ssa.OpAMD64ORQload, ssa.OpAMD64ORLload,
   924  		ssa.OpAMD64XORQload, ssa.OpAMD64XORLload, ssa.OpAMD64ADDSDload, ssa.OpAMD64ADDSSload,
   925  		ssa.OpAMD64SUBSDload, ssa.OpAMD64SUBSSload, ssa.OpAMD64MULSDload, ssa.OpAMD64MULSSload,
   926  		ssa.OpAMD64DIVSDload, ssa.OpAMD64DIVSSload:
   927  		p := s.Prog(v.Op.Asm())
   928  		p.From.Type = obj.TYPE_MEM
   929  		p.From.Reg = v.Args[1].Reg()
   930  		ssagen.AddAux(&p.From, v)
   931  		p.To.Type = obj.TYPE_REG
   932  		p.To.Reg = v.Reg()
   933  	case ssa.OpAMD64ADDLloadidx1, ssa.OpAMD64ADDLloadidx4, ssa.OpAMD64ADDLloadidx8, ssa.OpAMD64ADDQloadidx1, ssa.OpAMD64ADDQloadidx8,
   934  		ssa.OpAMD64SUBLloadidx1, ssa.OpAMD64SUBLloadidx4, ssa.OpAMD64SUBLloadidx8, ssa.OpAMD64SUBQloadidx1, ssa.OpAMD64SUBQloadidx8,
   935  		ssa.OpAMD64ANDLloadidx1, ssa.OpAMD64ANDLloadidx4, ssa.OpAMD64ANDLloadidx8, ssa.OpAMD64ANDQloadidx1, ssa.OpAMD64ANDQloadidx8,
   936  		ssa.OpAMD64ORLloadidx1, ssa.OpAMD64ORLloadidx4, ssa.OpAMD64ORLloadidx8, ssa.OpAMD64ORQloadidx1, ssa.OpAMD64ORQloadidx8,
   937  		ssa.OpAMD64XORLloadidx1, ssa.OpAMD64XORLloadidx4, ssa.OpAMD64XORLloadidx8, ssa.OpAMD64XORQloadidx1, ssa.OpAMD64XORQloadidx8,
   938  		ssa.OpAMD64ADDSSloadidx1, ssa.OpAMD64ADDSSloadidx4, ssa.OpAMD64ADDSDloadidx1, ssa.OpAMD64ADDSDloadidx8,
   939  		ssa.OpAMD64SUBSSloadidx1, ssa.OpAMD64SUBSSloadidx4, ssa.OpAMD64SUBSDloadidx1, ssa.OpAMD64SUBSDloadidx8,
   940  		ssa.OpAMD64MULSSloadidx1, ssa.OpAMD64MULSSloadidx4, ssa.OpAMD64MULSDloadidx1, ssa.OpAMD64MULSDloadidx8,
   941  		ssa.OpAMD64DIVSSloadidx1, ssa.OpAMD64DIVSSloadidx4, ssa.OpAMD64DIVSDloadidx1, ssa.OpAMD64DIVSDloadidx8:
   942  		p := s.Prog(v.Op.Asm())
   943  
   944  		r, i := v.Args[1].Reg(), v.Args[2].Reg()
   945  		p.From.Type = obj.TYPE_MEM
   946  		p.From.Scale = v.Op.Scale()
   947  		if p.From.Scale == 1 && i == x86.REG_SP {
   948  			r, i = i, r
   949  		}
   950  		p.From.Reg = r
   951  		p.From.Index = i
   952  
   953  		ssagen.AddAux(&p.From, v)
   954  		p.To.Type = obj.TYPE_REG
   955  		p.To.Reg = v.Reg()
   956  	case ssa.OpAMD64DUFFZERO:
   957  		if s.ABI != obj.ABIInternal {
   958  			// zero X15 manually
   959  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   960  		}
   961  		off := duffStart(v.AuxInt)
   962  		adj := duffAdj(v.AuxInt)
   963  		var p *obj.Prog
   964  		if adj != 0 {
   965  			p = s.Prog(x86.ALEAQ)
   966  			p.From.Type = obj.TYPE_MEM
   967  			p.From.Offset = adj
   968  			p.From.Reg = x86.REG_DI
   969  			p.To.Type = obj.TYPE_REG
   970  			p.To.Reg = x86.REG_DI
   971  		}
   972  		p = s.Prog(obj.ADUFFZERO)
   973  		p.To.Type = obj.TYPE_ADDR
   974  		p.To.Sym = ir.Syms.Duffzero
   975  		p.To.Offset = off
   976  	case ssa.OpAMD64DUFFCOPY:
   977  		p := s.Prog(obj.ADUFFCOPY)
   978  		p.To.Type = obj.TYPE_ADDR
   979  		p.To.Sym = ir.Syms.Duffcopy
   980  		if v.AuxInt%16 != 0 {
   981  			v.Fatalf("bad DUFFCOPY AuxInt %v", v.AuxInt)
   982  		}
   983  		p.To.Offset = 14 * (64 - v.AuxInt/16)
   984  		// 14 and 64 are magic constants.  14 is the number of bytes to encode:
   985  		//	MOVUPS	(SI), X0
   986  		//	ADDQ	$16, SI
   987  		//	MOVUPS	X0, (DI)
   988  		//	ADDQ	$16, DI
   989  		// and 64 is the number of such blocks. See src/runtime/duff_amd64.s:duffcopy.
   990  
   991  	case ssa.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?
   992  		if v.Type.IsMemory() {
   993  			return
   994  		}
   995  		x := v.Args[0].Reg()
   996  		y := v.Reg()
   997  		if x != y {
   998  			opregreg(s, moveByType(v.Type), y, x)
   999  		}
  1000  	case ssa.OpLoadReg:
  1001  		if v.Type.IsFlags() {
  1002  			v.Fatalf("load flags not implemented: %v", v.LongString())
  1003  			return
  1004  		}
  1005  		p := s.Prog(loadByType(v.Type))
  1006  		ssagen.AddrAuto(&p.From, v.Args[0])
  1007  		p.To.Type = obj.TYPE_REG
  1008  		p.To.Reg = v.Reg()
  1009  
  1010  	case ssa.OpStoreReg:
  1011  		if v.Type.IsFlags() {
  1012  			v.Fatalf("store flags not implemented: %v", v.LongString())
  1013  			return
  1014  		}
  1015  		p := s.Prog(storeByType(v.Type))
  1016  		p.From.Type = obj.TYPE_REG
  1017  		p.From.Reg = v.Args[0].Reg()
  1018  		ssagen.AddrAuto(&p.To, v)
  1019  	case ssa.OpAMD64LoweredHasCPUFeature:
  1020  		p := s.Prog(x86.AMOVBQZX)
  1021  		p.From.Type = obj.TYPE_MEM
  1022  		ssagen.AddAux(&p.From, v)
  1023  		p.To.Type = obj.TYPE_REG
  1024  		p.To.Reg = v.Reg()
  1025  	case ssa.OpArgIntReg, ssa.OpArgFloatReg:
  1026  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
  1027  		// The loop only runs once.
  1028  		for _, ap := range v.Block.Func.RegArgs {
  1029  			// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.
  1030  			addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)
  1031  			s.FuncInfo().AddSpill(
  1032  				obj.RegSpill{Reg: ap.Reg, Addr: addr, Unspill: loadByType(ap.Type), Spill: storeByType(ap.Type)})
  1033  		}
  1034  		v.Block.Func.RegArgs = nil
  1035  		ssagen.CheckArgReg(v)
  1036  	case ssa.OpAMD64LoweredGetClosurePtr:
  1037  		// Closure pointer is DX.
  1038  		ssagen.CheckLoweredGetClosurePtr(v)
  1039  	case ssa.OpAMD64LoweredGetG:
  1040  		if s.ABI == obj.ABIInternal {
  1041  			v.Fatalf("LoweredGetG should not appear in ABIInternal")
  1042  		}
  1043  		r := v.Reg()
  1044  		getgFromTLS(s, r)
  1045  	case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail:
  1046  		if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
  1047  			// zeroing X15 when entering ABIInternal from ABI0
  1048  			if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
  1049  				opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1050  			}
  1051  			// set G register from TLS
  1052  			getgFromTLS(s, x86.REG_R14)
  1053  		}
  1054  		if v.Op == ssa.OpAMD64CALLtail {
  1055  			s.TailCall(v)
  1056  			break
  1057  		}
  1058  		s.Call(v)
  1059  		if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
  1060  			// zeroing X15 when entering ABIInternal from ABI0
  1061  			if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
  1062  				opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1063  			}
  1064  			// set G register from TLS
  1065  			getgFromTLS(s, x86.REG_R14)
  1066  		}
  1067  	case ssa.OpAMD64CALLclosure, ssa.OpAMD64CALLinter:
  1068  		s.Call(v)
  1069  
  1070  	case ssa.OpAMD64LoweredGetCallerPC:
  1071  		p := s.Prog(x86.AMOVQ)
  1072  		p.From.Type = obj.TYPE_MEM
  1073  		p.From.Offset = -8 // PC is stored 8 bytes below first parameter.
  1074  		p.From.Name = obj.NAME_PARAM
  1075  		p.To.Type = obj.TYPE_REG
  1076  		p.To.Reg = v.Reg()
  1077  
  1078  	case ssa.OpAMD64LoweredGetCallerSP:
  1079  		// caller's SP is the address of the first arg
  1080  		mov := x86.AMOVQ
  1081  		if types.PtrSize == 4 {
  1082  			mov = x86.AMOVL
  1083  		}
  1084  		p := s.Prog(mov)
  1085  		p.From.Type = obj.TYPE_ADDR
  1086  		p.From.Offset = -base.Ctxt.FixedFrameSize() // 0 on amd64, just to be consistent with other architectures
  1087  		p.From.Name = obj.NAME_PARAM
  1088  		p.To.Type = obj.TYPE_REG
  1089  		p.To.Reg = v.Reg()
  1090  
  1091  	case ssa.OpAMD64LoweredWB:
  1092  		p := s.Prog(obj.ACALL)
  1093  		p.To.Type = obj.TYPE_MEM
  1094  		p.To.Name = obj.NAME_EXTERN
  1095  		// arg0 is in DI. Set sym to match where regalloc put arg1.
  1096  		p.To.Sym = ssagen.GCWriteBarrierReg[v.Args[1].Reg()]
  1097  
  1098  	case ssa.OpAMD64LoweredPanicBoundsA, ssa.OpAMD64LoweredPanicBoundsB, ssa.OpAMD64LoweredPanicBoundsC:
  1099  		p := s.Prog(obj.ACALL)
  1100  		p.To.Type = obj.TYPE_MEM
  1101  		p.To.Name = obj.NAME_EXTERN
  1102  		p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt]
  1103  		s.UseArgs(int64(2 * types.PtrSize)) // space used in callee args area by assembly stubs
  1104  
  1105  	case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL,
  1106  		ssa.OpAMD64BSWAPQ, ssa.OpAMD64BSWAPL,
  1107  		ssa.OpAMD64NOTQ, ssa.OpAMD64NOTL:
  1108  		p := s.Prog(v.Op.Asm())
  1109  		p.To.Type = obj.TYPE_REG
  1110  		p.To.Reg = v.Reg()
  1111  
  1112  	case ssa.OpAMD64NEGLflags:
  1113  		p := s.Prog(v.Op.Asm())
  1114  		p.To.Type = obj.TYPE_REG
  1115  		p.To.Reg = v.Reg0()
  1116  
  1117  	case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ, ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1118  		p := s.Prog(v.Op.Asm())
  1119  		p.From.Type = obj.TYPE_REG
  1120  		p.From.Reg = v.Args[0].Reg()
  1121  		p.To.Type = obj.TYPE_REG
  1122  		switch v.Op {
  1123  		case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ:
  1124  			p.To.Reg = v.Reg0()
  1125  		case ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1126  			p.To.Reg = v.Reg()
  1127  		}
  1128  	case ssa.OpAMD64ROUNDSD:
  1129  		p := s.Prog(v.Op.Asm())
  1130  		val := v.AuxInt
  1131  		// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc
  1132  		if val < 0 || val > 3 {
  1133  			v.Fatalf("Invalid rounding mode")
  1134  		}
  1135  		p.From.Offset = val
  1136  		p.From.Type = obj.TYPE_CONST
  1137  		p.SetFrom3Reg(v.Args[0].Reg())
  1138  		p.To.Type = obj.TYPE_REG
  1139  		p.To.Reg = v.Reg()
  1140  	case ssa.OpAMD64POPCNTQ, ssa.OpAMD64POPCNTL:
  1141  		if v.Args[0].Reg() != v.Reg() {
  1142  			// POPCNT on Intel has a false dependency on the destination register.
  1143  			// Xor register with itself to break the dependency.
  1144  			p := s.Prog(x86.AXORL)
  1145  			p.From.Type = obj.TYPE_REG
  1146  			p.From.Reg = v.Reg()
  1147  			p.To.Type = obj.TYPE_REG
  1148  			p.To.Reg = v.Reg()
  1149  		}
  1150  		p := s.Prog(v.Op.Asm())
  1151  		p.From.Type = obj.TYPE_REG
  1152  		p.From.Reg = v.Args[0].Reg()
  1153  		p.To.Type = obj.TYPE_REG
  1154  		p.To.Reg = v.Reg()
  1155  
  1156  	case ssa.OpAMD64SETEQ, ssa.OpAMD64SETNE,
  1157  		ssa.OpAMD64SETL, ssa.OpAMD64SETLE,
  1158  		ssa.OpAMD64SETG, ssa.OpAMD64SETGE,
  1159  		ssa.OpAMD64SETGF, ssa.OpAMD64SETGEF,
  1160  		ssa.OpAMD64SETB, ssa.OpAMD64SETBE,
  1161  		ssa.OpAMD64SETORD, ssa.OpAMD64SETNAN,
  1162  		ssa.OpAMD64SETA, ssa.OpAMD64SETAE,
  1163  		ssa.OpAMD64SETO:
  1164  		p := s.Prog(v.Op.Asm())
  1165  		p.To.Type = obj.TYPE_REG
  1166  		p.To.Reg = v.Reg()
  1167  
  1168  	case ssa.OpAMD64SETEQstore, ssa.OpAMD64SETNEstore,
  1169  		ssa.OpAMD64SETLstore, ssa.OpAMD64SETLEstore,
  1170  		ssa.OpAMD64SETGstore, ssa.OpAMD64SETGEstore,
  1171  		ssa.OpAMD64SETBstore, ssa.OpAMD64SETBEstore,
  1172  		ssa.OpAMD64SETAstore, ssa.OpAMD64SETAEstore:
  1173  		p := s.Prog(v.Op.Asm())
  1174  		p.To.Type = obj.TYPE_MEM
  1175  		p.To.Reg = v.Args[0].Reg()
  1176  		ssagen.AddAux(&p.To, v)
  1177  
  1178  	case ssa.OpAMD64SETNEF:
  1179  		p := s.Prog(v.Op.Asm())
  1180  		p.To.Type = obj.TYPE_REG
  1181  		p.To.Reg = v.Reg()
  1182  		q := s.Prog(x86.ASETPS)
  1183  		q.To.Type = obj.TYPE_REG
  1184  		q.To.Reg = x86.REG_AX
  1185  		// ORL avoids partial register write and is smaller than ORQ, used by old compiler
  1186  		opregreg(s, x86.AORL, v.Reg(), x86.REG_AX)
  1187  
  1188  	case ssa.OpAMD64SETEQF:
  1189  		p := s.Prog(v.Op.Asm())
  1190  		p.To.Type = obj.TYPE_REG
  1191  		p.To.Reg = v.Reg()
  1192  		q := s.Prog(x86.ASETPC)
  1193  		q.To.Type = obj.TYPE_REG
  1194  		q.To.Reg = x86.REG_AX
  1195  		// ANDL avoids partial register write and is smaller than ANDQ, used by old compiler
  1196  		opregreg(s, x86.AANDL, v.Reg(), x86.REG_AX)
  1197  
  1198  	case ssa.OpAMD64InvertFlags:
  1199  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1200  	case ssa.OpAMD64FlagEQ, ssa.OpAMD64FlagLT_ULT, ssa.OpAMD64FlagLT_UGT, ssa.OpAMD64FlagGT_ULT, ssa.OpAMD64FlagGT_UGT:
  1201  		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
  1202  	case ssa.OpAMD64AddTupleFirst32, ssa.OpAMD64AddTupleFirst64:
  1203  		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())
  1204  	case ssa.OpAMD64REPSTOSQ:
  1205  		s.Prog(x86.AREP)
  1206  		s.Prog(x86.ASTOSQ)
  1207  	case ssa.OpAMD64REPMOVSQ:
  1208  		s.Prog(x86.AREP)
  1209  		s.Prog(x86.AMOVSQ)
  1210  	case ssa.OpAMD64LoweredNilCheck:
  1211  		// Issue a load which will fault if the input is nil.
  1212  		// TODO: We currently use the 2-byte instruction TESTB AX, (reg).
  1213  		// Should we use the 3-byte TESTB $0, (reg) instead? It is larger
  1214  		// but it doesn't have false dependency on AX.
  1215  		// Or maybe allocate an output register and use MOVL (reg),reg2 ?
  1216  		// That trades clobbering flags for clobbering a register.
  1217  		p := s.Prog(x86.ATESTB)
  1218  		p.From.Type = obj.TYPE_REG
  1219  		p.From.Reg = x86.REG_AX
  1220  		p.To.Type = obj.TYPE_MEM
  1221  		p.To.Reg = v.Args[0].Reg()
  1222  		if logopt.Enabled() {
  1223  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1224  		}
  1225  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
  1226  			base.WarnfAt(v.Pos, "generated nil check")
  1227  		}
  1228  	case ssa.OpAMD64MOVBatomicload, ssa.OpAMD64MOVLatomicload, ssa.OpAMD64MOVQatomicload:
  1229  		p := s.Prog(v.Op.Asm())
  1230  		p.From.Type = obj.TYPE_MEM
  1231  		p.From.Reg = v.Args[0].Reg()
  1232  		ssagen.AddAux(&p.From, v)
  1233  		p.To.Type = obj.TYPE_REG
  1234  		p.To.Reg = v.Reg0()
  1235  	case ssa.OpAMD64XCHGB, ssa.OpAMD64XCHGL, ssa.OpAMD64XCHGQ:
  1236  		p := s.Prog(v.Op.Asm())
  1237  		p.From.Type = obj.TYPE_REG
  1238  		p.From.Reg = v.Reg0()
  1239  		p.To.Type = obj.TYPE_MEM
  1240  		p.To.Reg = v.Args[1].Reg()
  1241  		ssagen.AddAux(&p.To, v)
  1242  	case ssa.OpAMD64XADDLlock, ssa.OpAMD64XADDQlock:
  1243  		s.Prog(x86.ALOCK)
  1244  		p := s.Prog(v.Op.Asm())
  1245  		p.From.Type = obj.TYPE_REG
  1246  		p.From.Reg = v.Reg0()
  1247  		p.To.Type = obj.TYPE_MEM
  1248  		p.To.Reg = v.Args[1].Reg()
  1249  		ssagen.AddAux(&p.To, v)
  1250  	case ssa.OpAMD64CMPXCHGLlock, ssa.OpAMD64CMPXCHGQlock:
  1251  		if v.Args[1].Reg() != x86.REG_AX {
  1252  			v.Fatalf("input[1] not in AX %s", v.LongString())
  1253  		}
  1254  		s.Prog(x86.ALOCK)
  1255  		p := s.Prog(v.Op.Asm())
  1256  		p.From.Type = obj.TYPE_REG
  1257  		p.From.Reg = v.Args[2].Reg()
  1258  		p.To.Type = obj.TYPE_MEM
  1259  		p.To.Reg = v.Args[0].Reg()
  1260  		ssagen.AddAux(&p.To, v)
  1261  		p = s.Prog(x86.ASETEQ)
  1262  		p.To.Type = obj.TYPE_REG
  1263  		p.To.Reg = v.Reg0()
  1264  	case ssa.OpAMD64ANDBlock, ssa.OpAMD64ANDLlock, ssa.OpAMD64ORBlock, ssa.OpAMD64ORLlock:
  1265  		s.Prog(x86.ALOCK)
  1266  		p := s.Prog(v.Op.Asm())
  1267  		p.From.Type = obj.TYPE_REG
  1268  		p.From.Reg = v.Args[1].Reg()
  1269  		p.To.Type = obj.TYPE_MEM
  1270  		p.To.Reg = v.Args[0].Reg()
  1271  		ssagen.AddAux(&p.To, v)
  1272  	case ssa.OpAMD64PrefetchT0, ssa.OpAMD64PrefetchNTA:
  1273  		p := s.Prog(v.Op.Asm())
  1274  		p.From.Type = obj.TYPE_MEM
  1275  		p.From.Reg = v.Args[0].Reg()
  1276  	case ssa.OpClobber:
  1277  		p := s.Prog(x86.AMOVL)
  1278  		p.From.Type = obj.TYPE_CONST
  1279  		p.From.Offset = 0xdeaddead
  1280  		p.To.Type = obj.TYPE_MEM
  1281  		p.To.Reg = x86.REG_SP
  1282  		ssagen.AddAux(&p.To, v)
  1283  		p = s.Prog(x86.AMOVL)
  1284  		p.From.Type = obj.TYPE_CONST
  1285  		p.From.Offset = 0xdeaddead
  1286  		p.To.Type = obj.TYPE_MEM
  1287  		p.To.Reg = x86.REG_SP
  1288  		ssagen.AddAux(&p.To, v)
  1289  		p.To.Offset += 4
  1290  	case ssa.OpClobberReg:
  1291  		x := uint64(0xdeaddeaddeaddead)
  1292  		p := s.Prog(x86.AMOVQ)
  1293  		p.From.Type = obj.TYPE_CONST
  1294  		p.From.Offset = int64(x)
  1295  		p.To.Type = obj.TYPE_REG
  1296  		p.To.Reg = v.Reg()
  1297  	default:
  1298  		v.Fatalf("genValue not implemented: %s", v.LongString())
  1299  	}
  1300  }
  1301  
  1302  var blockJump = [...]struct {
  1303  	asm, invasm obj.As
  1304  }{
  1305  	ssa.BlockAMD64EQ:  {x86.AJEQ, x86.AJNE},
  1306  	ssa.BlockAMD64NE:  {x86.AJNE, x86.AJEQ},
  1307  	ssa.BlockAMD64LT:  {x86.AJLT, x86.AJGE},
  1308  	ssa.BlockAMD64GE:  {x86.AJGE, x86.AJLT},
  1309  	ssa.BlockAMD64LE:  {x86.AJLE, x86.AJGT},
  1310  	ssa.BlockAMD64GT:  {x86.AJGT, x86.AJLE},
  1311  	ssa.BlockAMD64OS:  {x86.AJOS, x86.AJOC},
  1312  	ssa.BlockAMD64OC:  {x86.AJOC, x86.AJOS},
  1313  	ssa.BlockAMD64ULT: {x86.AJCS, x86.AJCC},
  1314  	ssa.BlockAMD64UGE: {x86.AJCC, x86.AJCS},
  1315  	ssa.BlockAMD64UGT: {x86.AJHI, x86.AJLS},
  1316  	ssa.BlockAMD64ULE: {x86.AJLS, x86.AJHI},
  1317  	ssa.BlockAMD64ORD: {x86.AJPC, x86.AJPS},
  1318  	ssa.BlockAMD64NAN: {x86.AJPS, x86.AJPC},
  1319  }
  1320  
  1321  var eqfJumps = [2][2]ssagen.IndexJump{
  1322  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0]
  1323  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1]
  1324  }
  1325  var nefJumps = [2][2]ssagen.IndexJump{
  1326  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0]
  1327  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1]
  1328  }
  1329  
  1330  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  1331  	switch b.Kind {
  1332  	case ssa.BlockPlain:
  1333  		if b.Succs[0].Block() != next {
  1334  			p := s.Prog(obj.AJMP)
  1335  			p.To.Type = obj.TYPE_BRANCH
  1336  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1337  		}
  1338  	case ssa.BlockDefer:
  1339  		// defer returns in rax:
  1340  		// 0 if we should continue executing
  1341  		// 1 if we should jump to deferreturn call
  1342  		p := s.Prog(x86.ATESTL)
  1343  		p.From.Type = obj.TYPE_REG
  1344  		p.From.Reg = x86.REG_AX
  1345  		p.To.Type = obj.TYPE_REG
  1346  		p.To.Reg = x86.REG_AX
  1347  		p = s.Prog(x86.AJNE)
  1348  		p.To.Type = obj.TYPE_BRANCH
  1349  		s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()})
  1350  		if b.Succs[0].Block() != next {
  1351  			p := s.Prog(obj.AJMP)
  1352  			p.To.Type = obj.TYPE_BRANCH
  1353  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1354  		}
  1355  	case ssa.BlockExit, ssa.BlockRetJmp:
  1356  	case ssa.BlockRet:
  1357  		s.Prog(obj.ARET)
  1358  
  1359  	case ssa.BlockAMD64EQF:
  1360  		s.CombJump(b, next, &eqfJumps)
  1361  
  1362  	case ssa.BlockAMD64NEF:
  1363  		s.CombJump(b, next, &nefJumps)
  1364  
  1365  	case ssa.BlockAMD64EQ, ssa.BlockAMD64NE,
  1366  		ssa.BlockAMD64LT, ssa.BlockAMD64GE,
  1367  		ssa.BlockAMD64LE, ssa.BlockAMD64GT,
  1368  		ssa.BlockAMD64OS, ssa.BlockAMD64OC,
  1369  		ssa.BlockAMD64ULT, ssa.BlockAMD64UGT,
  1370  		ssa.BlockAMD64ULE, ssa.BlockAMD64UGE:
  1371  		jmp := blockJump[b.Kind]
  1372  		switch next {
  1373  		case b.Succs[0].Block():
  1374  			s.Br(jmp.invasm, b.Succs[1].Block())
  1375  		case b.Succs[1].Block():
  1376  			s.Br(jmp.asm, b.Succs[0].Block())
  1377  		default:
  1378  			if b.Likely != ssa.BranchUnlikely {
  1379  				s.Br(jmp.asm, b.Succs[0].Block())
  1380  				s.Br(obj.AJMP, b.Succs[1].Block())
  1381  			} else {
  1382  				s.Br(jmp.invasm, b.Succs[1].Block())
  1383  				s.Br(obj.AJMP, b.Succs[0].Block())
  1384  			}
  1385  		}
  1386  
  1387  	default:
  1388  		b.Fatalf("branch not implemented: %s", b.LongString())
  1389  	}
  1390  }
  1391  
  1392  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1393  	p := s.Prog(loadByType(t))
  1394  	p.From.Type = obj.TYPE_MEM
  1395  	p.From.Name = obj.NAME_AUTO
  1396  	p.From.Sym = n.Linksym()
  1397  	p.From.Offset = n.FrameOffset() + off
  1398  	p.To.Type = obj.TYPE_REG
  1399  	p.To.Reg = reg
  1400  	return p
  1401  }
  1402  
  1403  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1404  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  1405  	p.To.Name = obj.NAME_PARAM
  1406  	p.To.Sym = n.Linksym()
  1407  	p.Pos = p.Pos.WithNotStmt()
  1408  	return p
  1409  }
  1410  

View as plain text