unified.go

     1  // UNREVIEWED
     2  
     3  // Copyright 2021 The Go Authors. All rights reserved.
     4  // Use of this source code is governed by a BSD-style
     5  // license that can be found in the LICENSE file.
     6  
     7  package noder
     8  
     9  import (
    10  	"bytes"
    11  	"fmt"
    12  	"internal/goversion"
    13  	"io"
    14  	"runtime"
    15  	"sort"
    16  
    17  	"cmd/compile/internal/base"
    18  	"cmd/compile/internal/inline"
    19  	"cmd/compile/internal/ir"
    20  	"cmd/compile/internal/typecheck"
    21  	"cmd/compile/internal/types"
    22  	"cmd/compile/internal/types2"
    23  	"cmd/internal/src"
    24  )
    25  
    26  // localPkgReader holds the package reader used for reading the local
    27  // package. It exists so the unified IR linker can refer back to it
    28  // later.
    29  var localPkgReader *pkgReader
    30  
    31  // unified construct the local package's IR from syntax's AST.
    32  //
    33  // The pipeline contains 2 steps:
    34  //
    35  // (1) Generate package export data "stub".
    36  //
    37  // (2) Generate package IR from package export data.
    38  //
    39  // The package data "stub" at step (1) contains everything from the local package,
    40  // but nothing that have been imported. When we're actually writing out export data
    41  // to the output files (see writeNewExport function), we run the "linker", which does
    42  // a few things:
    43  //
    44  // + Updates compiler extensions data (e.g., inlining cost, escape analysis results).
    45  //
    46  // + Handles re-exporting any transitive dependencies.
    47  //
    48  // + Prunes out any unnecessary details (e.g., non-inlineable functions, because any
    49  //   downstream importers only care about inlinable functions).
    50  //
    51  // The source files are typechecked twice, once before writing export data
    52  // using types2 checker, once after read export data using gc/typecheck.
    53  // This duplication of work will go away once we always use types2 checker,
    54  // we can remove the gc/typecheck pass. The reason it is still here:
    55  //
    56  // + It reduces engineering costs in maintaining a fork of typecheck
    57  //   (e.g., no need to backport fixes like CL 327651).
    58  //
    59  // + It makes it easier to pass toolstash -cmp.
    60  //
    61  // + Historically, we would always re-run the typechecker after import, even though
    62  //   we know the imported data is valid. It's not ideal, but also not causing any
    63  //   problem either.
    64  //
    65  // + There's still transformation that being done during gc/typecheck, like rewriting
    66  //   multi-valued function call, or transform ir.OINDEX -> ir.OINDEXMAP.
    67  //
    68  // Using syntax+types2 tree, which already has a complete representation of generics,
    69  // the unified IR has the full typed AST for doing introspection during step (1).
    70  // In other words, we have all necessary information to build the generic IR form
    71  // (see writer.captureVars for an example).
    72  func unified(noders []*noder) {
    73  	inline.NewInline = InlineCall
    74  
    75  	if !quirksMode() {
    76  		writeNewExportFunc = writeNewExport
    77  	} else if base.Flag.G != 0 {
    78  		base.Errorf("cannot use -G and -d=quirksmode together")
    79  	}
    80  
    81  	newReadImportFunc = func(data string, pkg1 *types.Pkg, ctxt *types2.Context, packages map[string]*types2.Package) (pkg2 *types2.Package, err error) {
    82  		pr := newPkgDecoder(pkg1.Path, data)
    83  
    84  		// Read package descriptors for both types2 and compiler backend.
    85  		readPackage(newPkgReader(pr), pkg1)
    86  		pkg2 = readPackage2(ctxt, packages, pr)
    87  		return
    88  	}
    89  
    90  	data := writePkgStub(noders)
    91  
    92  	// We already passed base.Flag.Lang to types2 to handle validating
    93  	// the user's source code. Bump it up now to the current version and
    94  	// re-parse, so typecheck doesn't complain if we construct IR that
    95  	// utilizes newer Go features.
    96  	base.Flag.Lang = fmt.Sprintf("go1.%d", goversion.Version)
    97  	types.ParseLangFlag()
    98  
    99  	assert(types.LocalPkg.Path == "")
   100  	types.LocalPkg.Height = 0 // reset so pkgReader.pkgIdx doesn't complain
   101  	target := typecheck.Target
   102  
   103  	typecheck.TypecheckAllowed = true
   104  
   105  	localPkgReader = newPkgReader(newPkgDecoder(types.LocalPkg.Path, data))
   106  	readPackage(localPkgReader, types.LocalPkg)
   107  
   108  	r := localPkgReader.newReader(relocMeta, privateRootIdx, syncPrivate)
   109  	r.pkgInit(types.LocalPkg, target)
   110  
   111  	// Type-check any top-level assignments. We ignore non-assignments
   112  	// here because other declarations are typechecked as they're
   113  	// constructed.
   114  	for i, ndecls := 0, len(target.Decls); i < ndecls; i++ {
   115  		switch n := target.Decls[i]; n.Op() {
   116  		case ir.OAS, ir.OAS2:
   117  			target.Decls[i] = typecheck.Stmt(n)
   118  		}
   119  	}
   120  
   121  	// Don't use range--bodyIdx can add closures to todoBodies.
   122  	for len(todoBodies) > 0 {
   123  		// The order we expand bodies doesn't matter, so pop from the end
   124  		// to reduce todoBodies reallocations if it grows further.
   125  		fn := todoBodies[len(todoBodies)-1]
   126  		todoBodies = todoBodies[:len(todoBodies)-1]
   127  
   128  		pri, ok := bodyReader[fn]
   129  		assert(ok)
   130  		pri.funcBody(fn)
   131  
   132  		// Instantiated generic function: add to Decls for typechecking
   133  		// and compilation.
   134  		if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   135  			target.Decls = append(target.Decls, fn)
   136  		}
   137  	}
   138  	todoBodies = nil
   139  	todoBodiesDone = true
   140  
   141  	// Check that nothing snuck past typechecking.
   142  	for _, n := range target.Decls {
   143  		if n.Typecheck() == 0 {
   144  			base.FatalfAt(n.Pos(), "missed typecheck: %v", n)
   145  		}
   146  
   147  		// For functions, check that at least their first statement (if
   148  		// any) was typechecked too.
   149  		if fn, ok := n.(*ir.Func); ok && len(fn.Body) != 0 {
   150  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   151  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   152  			}
   153  		}
   154  	}
   155  
   156  	base.ExitIfErrors() // just in case
   157  }
   158  
   159  // writePkgStub type checks the given parsed source files,
   160  // writes an export data package stub representing them,
   161  // and returns the result.
   162  func writePkgStub(noders []*noder) string {
   163  	m, pkg, info := checkFiles(noders)
   164  
   165  	pw := newPkgWriter(m, pkg, info)
   166  
   167  	pw.collectDecls(noders)
   168  
   169  	publicRootWriter := pw.newWriter(relocMeta, syncPublic)
   170  	privateRootWriter := pw.newWriter(relocMeta, syncPrivate)
   171  
   172  	assert(publicRootWriter.idx == publicRootIdx)
   173  	assert(privateRootWriter.idx == privateRootIdx)
   174  
   175  	{
   176  		w := publicRootWriter
   177  		w.pkg(pkg)
   178  		w.bool(false) // has init; XXX
   179  
   180  		scope := pkg.Scope()
   181  		names := scope.Names()
   182  		w.len(len(names))
   183  		for _, name := range scope.Names() {
   184  			w.obj(scope.Lookup(name), nil)
   185  		}
   186  
   187  		w.sync(syncEOF)
   188  		w.flush()
   189  	}
   190  
   191  	{
   192  		w := privateRootWriter
   193  		w.pkgInit(noders)
   194  		w.flush()
   195  	}
   196  
   197  	var sb bytes.Buffer // TODO(mdempsky): strings.Builder after #44505 is resolved
   198  	pw.dump(&sb)
   199  
   200  	// At this point, we're done with types2. Make sure the package is
   201  	// garbage collected.
   202  	freePackage(pkg)
   203  
   204  	return sb.String()
   205  }
   206  
   207  // freePackage ensures the given package is garbage collected.
   208  func freePackage(pkg *types2.Package) {
   209  	// The GC test below relies on a precise GC that runs finalizers as
   210  	// soon as objects are unreachable. Our implementation provides
   211  	// this, but other/older implementations may not (e.g., Go 1.4 does
   212  	// not because of #22350). To avoid imposing unnecessary
   213  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   214  	// during bootstrapping.
   215  	if base.CompilerBootstrap {
   216  		return
   217  	}
   218  
   219  	// Set a finalizer on pkg so we can detect if/when it's collected.
   220  	done := make(chan struct{})
   221  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   222  
   223  	// Important: objects involved in cycles are not finalized, so zero
   224  	// out pkg to break its cycles and allow the finalizer to run.
   225  	*pkg = types2.Package{}
   226  
   227  	// It typically takes just 1 or 2 cycles to release pkg, but it
   228  	// doesn't hurt to try a few more times.
   229  	for i := 0; i < 10; i++ {
   230  		select {
   231  		case <-done:
   232  			return
   233  		default:
   234  			runtime.GC()
   235  		}
   236  	}
   237  
   238  	base.Fatalf("package never finalized")
   239  }
   240  
   241  func readPackage(pr *pkgReader, importpkg *types.Pkg) {
   242  	r := pr.newReader(relocMeta, publicRootIdx, syncPublic)
   243  
   244  	pkg := r.pkg()
   245  	assert(pkg == importpkg)
   246  
   247  	if r.bool() {
   248  		sym := pkg.Lookup(".inittask")
   249  		task := ir.NewNameAt(src.NoXPos, sym)
   250  		task.Class = ir.PEXTERN
   251  		sym.Def = task
   252  	}
   253  
   254  	for i, n := 0, r.len(); i < n; i++ {
   255  		r.sync(syncObject)
   256  		assert(!r.bool())
   257  		idx := r.reloc(relocObj)
   258  		assert(r.len() == 0)
   259  
   260  		path, name, code := r.p.peekObj(idx)
   261  		if code != objStub {
   262  			objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil}
   263  		}
   264  	}
   265  }
   266  
   267  func writeNewExport(out io.Writer) {
   268  	l := linker{
   269  		pw: newPkgEncoder(),
   270  
   271  		pkgs:  make(map[string]int),
   272  		decls: make(map[*types.Sym]int),
   273  	}
   274  
   275  	publicRootWriter := l.pw.newEncoder(relocMeta, syncPublic)
   276  	assert(publicRootWriter.idx == publicRootIdx)
   277  
   278  	var selfPkgIdx int
   279  
   280  	{
   281  		pr := localPkgReader
   282  		r := pr.newDecoder(relocMeta, publicRootIdx, syncPublic)
   283  
   284  		r.sync(syncPkg)
   285  		selfPkgIdx = l.relocIdx(pr, relocPkg, r.reloc(relocPkg))
   286  
   287  		r.bool() // has init
   288  
   289  		for i, n := 0, r.len(); i < n; i++ {
   290  			r.sync(syncObject)
   291  			assert(!r.bool())
   292  			idx := r.reloc(relocObj)
   293  			assert(r.len() == 0)
   294  
   295  			xpath, xname, xtag := pr.peekObj(idx)
   296  			assert(xpath == pr.pkgPath)
   297  			assert(xtag != objStub)
   298  
   299  			if types.IsExported(xname) {
   300  				l.relocIdx(pr, relocObj, idx)
   301  			}
   302  		}
   303  
   304  		r.sync(syncEOF)
   305  	}
   306  
   307  	{
   308  		var idxs []int
   309  		for _, idx := range l.decls {
   310  			idxs = append(idxs, idx)
   311  		}
   312  		sort.Ints(idxs)
   313  
   314  		w := publicRootWriter
   315  
   316  		w.sync(syncPkg)
   317  		w.reloc(relocPkg, selfPkgIdx)
   318  
   319  		w.bool(typecheck.Lookup(".inittask").Def != nil)
   320  
   321  		w.len(len(idxs))
   322  		for _, idx := range idxs {
   323  			w.sync(syncObject)
   324  			w.bool(false)
   325  			w.reloc(relocObj, idx)
   326  			w.len(0)
   327  		}
   328  
   329  		w.sync(syncEOF)
   330  		w.flush()
   331  	}
   332  
   333  	l.pw.dump(out)
   334  }
   335
View as plain text