outbuf.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ld
     6  
     7  import (
     8  	"cmd/internal/sys"
     9  	"cmd/link/internal/loader"
    10  	"encoding/binary"
    11  	"errors"
    12  	"log"
    13  	"os"
    14  )
    15  
    16  // If fallocate is not supported on this platform, return this error. The error
    17  // is ignored where needed, and OutBuf writes to heap memory.
    18  var errNoFallocate = errors.New("operation not supported")
    19  
    20  const outbufMode = 0775
    21  
    22  // OutBuf is a buffered file writer.
    23  //
    24  // It is simlar to the Writer in cmd/internal/bio with a few small differences.
    25  //
    26  // First, it tracks the output architecture and uses it to provide
    27  // endian helpers.
    28  //
    29  // Second, it provides a very cheap offset counter that doesn't require
    30  // any system calls to read the value.
    31  //
    32  // Third, it also mmaps the output file (if available). The intended usage is:
    33  // - Mmap the output file
    34  // - Write the content
    35  // - possibly apply any edits in the output buffer
    36  // - possibly write more content to the file. These writes take place in a heap
    37  //   backed buffer that will get synced to disk.
    38  // - Munmap the output file
    39  //
    40  // And finally, it provides a mechanism by which you can multithread the
    41  // writing of output files. This mechanism is accomplished by copying a OutBuf,
    42  // and using it in the thread/goroutine.
    43  //
    44  // Parallel OutBuf is intended to be used like:
    45  //
    46  //  func write(out *OutBuf) {
    47  //    var wg sync.WaitGroup
    48  //    for i := 0; i < 10; i++ {
    49  //      wg.Add(1)
    50  //      view, err := out.View(start[i])
    51  //      if err != nil {
    52  //         // handle output
    53  //         continue
    54  //      }
    55  //      go func(out *OutBuf, i int) {
    56  //        // do output
    57  //        wg.Done()
    58  //      }(view, i)
    59  //    }
    60  //    wg.Wait()
    61  //  }
    62  type OutBuf struct {
    63  	arch *sys.Arch
    64  	off  int64
    65  
    66  	buf  []byte // backing store of mmap'd output file
    67  	heap []byte // backing store for non-mmapped data
    68  
    69  	name   string
    70  	f      *os.File
    71  	encbuf [8]byte // temp buffer used by WriteN methods
    72  	isView bool    // true if created from View()
    73  }
    74  
    75  func (out *OutBuf) Open(name string) error {
    76  	if out.f != nil {
    77  		return errors.New("cannot open more than one file")
    78  	}
    79  	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, outbufMode)
    80  	if err != nil {
    81  		return err
    82  	}
    83  	out.off = 0
    84  	out.name = name
    85  	out.f = f
    86  	return nil
    87  }
    88  
    89  func NewOutBuf(arch *sys.Arch) *OutBuf {
    90  	return &OutBuf{
    91  		arch: arch,
    92  	}
    93  }
    94  
    95  var viewError = errors.New("output not mmapped")
    96  
    97  func (out *OutBuf) View(start uint64) (*OutBuf, error) {
    98  	return &OutBuf{
    99  		arch:   out.arch,
   100  		name:   out.name,
   101  		buf:    out.buf,
   102  		heap:   out.heap,
   103  		off:    int64(start),
   104  		isView: true,
   105  	}, nil
   106  }
   107  
   108  var viewCloseError = errors.New("cannot Close OutBuf from View")
   109  
   110  func (out *OutBuf) Close() error {
   111  	if out.isView {
   112  		return viewCloseError
   113  	}
   114  	if out.isMmapped() {
   115  		out.copyHeap()
   116  		out.purgeSignatureCache()
   117  		out.munmap()
   118  	}
   119  	if out.f == nil {
   120  		return nil
   121  	}
   122  	if len(out.heap) != 0 {
   123  		if _, err := out.f.Write(out.heap); err != nil {
   124  			return err
   125  		}
   126  	}
   127  	if err := out.f.Close(); err != nil {
   128  		return err
   129  	}
   130  	out.f = nil
   131  	return nil
   132  }
   133  
   134  // ErrorClose closes the output file (if any).
   135  // It is supposed to be called only at exit on error, so it doesn't do
   136  // any clean up or buffer flushing, just closes the file.
   137  func (out *OutBuf) ErrorClose() {
   138  	if out.isView {
   139  		panic(viewCloseError)
   140  	}
   141  	if out.f == nil {
   142  		return
   143  	}
   144  	out.f.Close() // best effort, ignore error
   145  	out.f = nil
   146  }
   147  
   148  // isMmapped returns true if the OutBuf is mmaped.
   149  func (out *OutBuf) isMmapped() bool {
   150  	return len(out.buf) != 0
   151  }
   152  
   153  // Data returns the whole written OutBuf as a byte slice.
   154  func (out *OutBuf) Data() []byte {
   155  	if out.isMmapped() {
   156  		out.copyHeap()
   157  		return out.buf
   158  	}
   159  	return out.heap
   160  }
   161  
   162  // copyHeap copies the heap to the mmapped section of memory, returning true if
   163  // a copy takes place.
   164  func (out *OutBuf) copyHeap() bool {
   165  	if !out.isMmapped() { // only valuable for mmapped OutBufs.
   166  		return false
   167  	}
   168  	if out.isView {
   169  		panic("can't copyHeap a view")
   170  	}
   171  
   172  	bufLen := len(out.buf)
   173  	heapLen := len(out.heap)
   174  	total := uint64(bufLen + heapLen)
   175  	if heapLen != 0 {
   176  		if err := out.Mmap(total); err != nil { // Mmap will copy out.heap over to out.buf
   177  			Exitf("mapping output file failed: %v", err)
   178  		}
   179  	}
   180  	return true
   181  }
   182  
   183  // maxOutBufHeapLen limits the growth of the heap area.
   184  const maxOutBufHeapLen = 10 << 20
   185  
   186  // writeLoc determines the write location if a buffer is mmaped.
   187  // We maintain two write buffers, an mmapped section, and a heap section for
   188  // writing. When the mmapped section is full, we switch over the heap memory
   189  // for writing.
   190  func (out *OutBuf) writeLoc(lenToWrite int64) (int64, []byte) {
   191  	// See if we have enough space in the mmaped area.
   192  	bufLen := int64(len(out.buf))
   193  	if out.off+lenToWrite <= bufLen {
   194  		return out.off, out.buf
   195  	}
   196  
   197  	// Not enough space in the mmaped area, write to heap area instead.
   198  	heapPos := out.off - bufLen
   199  	heapLen := int64(len(out.heap))
   200  	lenNeeded := heapPos + lenToWrite
   201  	if lenNeeded > heapLen { // do we need to grow the heap storage?
   202  		// The heap variables aren't protected by a mutex. For now, just bomb if you
   203  		// try to use OutBuf in parallel. (Note this probably could be fixed.)
   204  		if out.isView {
   205  			panic("cannot write to heap in parallel")
   206  		}
   207  		// See if our heap would grow to be too large, and if so, copy it to the end
   208  		// of the mmapped area.
   209  		if heapLen > maxOutBufHeapLen && out.copyHeap() {
   210  			heapPos -= heapLen
   211  			lenNeeded = heapPos + lenToWrite
   212  			heapLen = 0
   213  		}
   214  		out.heap = append(out.heap, make([]byte, lenNeeded-heapLen)...)
   215  	}
   216  	return heapPos, out.heap
   217  }
   218  
   219  func (out *OutBuf) SeekSet(p int64) {
   220  	out.off = p
   221  }
   222  
   223  func (out *OutBuf) Offset() int64 {
   224  	return out.off
   225  }
   226  
   227  // Write writes the contents of v to the buffer.
   228  func (out *OutBuf) Write(v []byte) (int, error) {
   229  	n := len(v)
   230  	pos, buf := out.writeLoc(int64(n))
   231  	copy(buf[pos:], v)
   232  	out.off += int64(n)
   233  	return n, nil
   234  }
   235  
   236  func (out *OutBuf) Write8(v uint8) {
   237  	pos, buf := out.writeLoc(1)
   238  	buf[pos] = v
   239  	out.off++
   240  }
   241  
   242  // WriteByte is an alias for Write8 to fulfill the io.ByteWriter interface.
   243  func (out *OutBuf) WriteByte(v byte) error {
   244  	out.Write8(v)
   245  	return nil
   246  }
   247  
   248  func (out *OutBuf) Write16(v uint16) {
   249  	out.arch.ByteOrder.PutUint16(out.encbuf[:], v)
   250  	out.Write(out.encbuf[:2])
   251  }
   252  
   253  func (out *OutBuf) Write32(v uint32) {
   254  	out.arch.ByteOrder.PutUint32(out.encbuf[:], v)
   255  	out.Write(out.encbuf[:4])
   256  }
   257  
   258  func (out *OutBuf) Write32b(v uint32) {
   259  	binary.BigEndian.PutUint32(out.encbuf[:], v)
   260  	out.Write(out.encbuf[:4])
   261  }
   262  
   263  func (out *OutBuf) Write64(v uint64) {
   264  	out.arch.ByteOrder.PutUint64(out.encbuf[:], v)
   265  	out.Write(out.encbuf[:8])
   266  }
   267  
   268  func (out *OutBuf) Write64b(v uint64) {
   269  	binary.BigEndian.PutUint64(out.encbuf[:], v)
   270  	out.Write(out.encbuf[:8])
   271  }
   272  
   273  func (out *OutBuf) WriteString(s string) {
   274  	pos, buf := out.writeLoc(int64(len(s)))
   275  	n := copy(buf[pos:], s)
   276  	if n != len(s) {
   277  		log.Fatalf("WriteString truncated. buffer size: %d, offset: %d, len(s)=%d", len(out.buf), out.off, len(s))
   278  	}
   279  	out.off += int64(n)
   280  }
   281  
   282  // WriteStringN writes the first n bytes of s.
   283  // If n is larger than len(s) then it is padded with zero bytes.
   284  func (out *OutBuf) WriteStringN(s string, n int) {
   285  	out.WriteStringPad(s, n, zeros[:])
   286  }
   287  
   288  // WriteStringPad writes the first n bytes of s.
   289  // If n is larger than len(s) then it is padded with the bytes in pad (repeated as needed).
   290  func (out *OutBuf) WriteStringPad(s string, n int, pad []byte) {
   291  	if len(s) >= n {
   292  		out.WriteString(s[:n])
   293  	} else {
   294  		out.WriteString(s)
   295  		n -= len(s)
   296  		for n > len(pad) {
   297  			out.Write(pad)
   298  			n -= len(pad)
   299  
   300  		}
   301  		out.Write(pad[:n])
   302  	}
   303  }
   304  
   305  // WriteSym writes the content of a Symbol, and returns the output buffer
   306  // that we just wrote, so we can apply further edit to the symbol content.
   307  // For generator symbols, it also sets the symbol's Data to the output
   308  // buffer.
   309  func (out *OutBuf) WriteSym(ldr *loader.Loader, s loader.Sym) []byte {
   310  	if !ldr.IsGeneratedSym(s) {
   311  		P := ldr.Data(s)
   312  		n := int64(len(P))
   313  		pos, buf := out.writeLoc(n)
   314  		copy(buf[pos:], P)
   315  		out.off += n
   316  		ldr.FreeData(s)
   317  		return buf[pos : pos+n]
   318  	} else {
   319  		n := ldr.SymSize(s)
   320  		pos, buf := out.writeLoc(n)
   321  		out.off += n
   322  		ldr.MakeSymbolUpdater(s).SetData(buf[pos : pos+n])
   323  		return buf[pos : pos+n]
   324  	}
   325  }
   326
View as plain text