fetch.go

     1  // Copyright 2014 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package driver
    16  
    17  import (
    18  	"bytes"
    19  	"fmt"
    20  	"io"
    21  	"io/ioutil"
    22  	"net/http"
    23  	"net/url"
    24  	"os"
    25  	"os/exec"
    26  	"path/filepath"
    27  	"runtime"
    28  	"strconv"
    29  	"strings"
    30  	"sync"
    31  	"time"
    32  
    33  	"github.com/google/pprof/internal/measurement"
    34  	"github.com/google/pprof/internal/plugin"
    35  	"github.com/google/pprof/profile"
    36  )
    37  
    38  // fetchProfiles fetches and symbolizes the profiles specified by s.
    39  // It will merge all the profiles it is able to retrieve, even if
    40  // there are some failures. It will return an error if it is unable to
    41  // fetch any profiles.
    42  func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
    43  	sources := make([]profileSource, 0, len(s.Sources))
    44  	for _, src := range s.Sources {
    45  		sources = append(sources, profileSource{
    46  			addr:   src,
    47  			source: s,
    48  		})
    49  	}
    50  
    51  	bases := make([]profileSource, 0, len(s.Base))
    52  	for _, src := range s.Base {
    53  		bases = append(bases, profileSource{
    54  			addr:   src,
    55  			source: s,
    56  		})
    57  	}
    58  
    59  	p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport)
    60  	if err != nil {
    61  		return nil, err
    62  	}
    63  
    64  	if pbase != nil {
    65  		if s.DiffBase {
    66  			pbase.SetLabel("pprof::base", []string{"true"})
    67  		}
    68  		if s.Normalize {
    69  			err := p.Normalize(pbase)
    70  			if err != nil {
    71  				return nil, err
    72  			}
    73  		}
    74  		pbase.Scale(-1)
    75  		p, m, err = combineProfiles([]*profile.Profile{p, pbase}, []plugin.MappingSources{m, mbase})
    76  		if err != nil {
    77  			return nil, err
    78  		}
    79  	}
    80  
    81  	// Symbolize the merged profile.
    82  	if err := o.Sym.Symbolize(s.Symbolize, m, p); err != nil {
    83  		return nil, err
    84  	}
    85  	p.RemoveUninteresting()
    86  	unsourceMappings(p)
    87  
    88  	if s.Comment != "" {
    89  		p.Comments = append(p.Comments, s.Comment)
    90  	}
    91  
    92  	// Save a copy of the merged profile if there is at least one remote source.
    93  	if save {
    94  		dir, err := setTmpDir(o.UI)
    95  		if err != nil {
    96  			return nil, err
    97  		}
    98  
    99  		prefix := "pprof."
   100  		if len(p.Mapping) > 0 && p.Mapping[0].File != "" {
   101  			prefix += filepath.Base(p.Mapping[0].File) + "."
   102  		}
   103  		for _, s := range p.SampleType {
   104  			prefix += s.Type + "."
   105  		}
   106  
   107  		tempFile, err := newTempFile(dir, prefix, ".pb.gz")
   108  		if err == nil {
   109  			if err = p.Write(tempFile); err == nil {
   110  				o.UI.PrintErr("Saved profile in ", tempFile.Name())
   111  			}
   112  		}
   113  		if err != nil {
   114  			o.UI.PrintErr("Could not save profile: ", err)
   115  		}
   116  	}
   117  
   118  	if err := p.CheckValid(); err != nil {
   119  		return nil, err
   120  	}
   121  
   122  	return p, nil
   123  }
   124  
   125  func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
   126  	wg := sync.WaitGroup{}
   127  	wg.Add(2)
   128  	var psrc, pbase *profile.Profile
   129  	var msrc, mbase plugin.MappingSources
   130  	var savesrc, savebase bool
   131  	var errsrc, errbase error
   132  	var countsrc, countbase int
   133  	go func() {
   134  		defer wg.Done()
   135  		psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr)
   136  	}()
   137  	go func() {
   138  		defer wg.Done()
   139  		pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr)
   140  	}()
   141  	wg.Wait()
   142  	save := savesrc || savebase
   143  
   144  	if errsrc != nil {
   145  		return nil, nil, nil, nil, false, fmt.Errorf("problem fetching source profiles: %v", errsrc)
   146  	}
   147  	if errbase != nil {
   148  		return nil, nil, nil, nil, false, fmt.Errorf("problem fetching base profiles: %v,", errbase)
   149  	}
   150  	if countsrc == 0 {
   151  		return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any source profiles")
   152  	}
   153  	if countbase == 0 && len(bases) > 0 {
   154  		return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any base profiles")
   155  	}
   156  	if want, got := len(sources), countsrc; want != got {
   157  		ui.PrintErr(fmt.Sprintf("Fetched %d source profiles out of %d", got, want))
   158  	}
   159  	if want, got := len(bases), countbase; want != got {
   160  		ui.PrintErr(fmt.Sprintf("Fetched %d base profiles out of %d", got, want))
   161  	}
   162  
   163  	return psrc, pbase, msrc, mbase, save, nil
   164  }
   165  
   166  // chunkedGrab fetches the profiles described in source and merges them into
   167  // a single profile. It fetches a chunk of profiles concurrently, with a maximum
   168  // chunk size to limit its memory usage.
   169  func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
   170  	const chunkSize = 64
   171  
   172  	var p *profile.Profile
   173  	var msrc plugin.MappingSources
   174  	var save bool
   175  	var count int
   176  
   177  	for start := 0; start < len(sources); start += chunkSize {
   178  		end := start + chunkSize
   179  		if end > len(sources) {
   180  			end = len(sources)
   181  		}
   182  		chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr)
   183  		switch {
   184  		case chunkErr != nil:
   185  			return nil, nil, false, 0, chunkErr
   186  		case chunkP == nil:
   187  			continue
   188  		case p == nil:
   189  			p, msrc, save, count = chunkP, chunkMsrc, chunkSave, chunkCount
   190  		default:
   191  			p, msrc, chunkErr = combineProfiles([]*profile.Profile{p, chunkP}, []plugin.MappingSources{msrc, chunkMsrc})
   192  			if chunkErr != nil {
   193  				return nil, nil, false, 0, chunkErr
   194  			}
   195  			if chunkSave {
   196  				save = true
   197  			}
   198  			count += chunkCount
   199  		}
   200  	}
   201  
   202  	return p, msrc, save, count, nil
   203  }
   204  
   205  // concurrentGrab fetches multiple profiles concurrently
   206  func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
   207  	wg := sync.WaitGroup{}
   208  	wg.Add(len(sources))
   209  	for i := range sources {
   210  		go func(s *profileSource) {
   211  			defer wg.Done()
   212  			s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr)
   213  		}(&sources[i])
   214  	}
   215  	wg.Wait()
   216  
   217  	var save bool
   218  	profiles := make([]*profile.Profile, 0, len(sources))
   219  	msrcs := make([]plugin.MappingSources, 0, len(sources))
   220  	for i := range sources {
   221  		s := &sources[i]
   222  		if err := s.err; err != nil {
   223  			ui.PrintErr(s.addr + ": " + err.Error())
   224  			continue
   225  		}
   226  		save = save || s.remote
   227  		profiles = append(profiles, s.p)
   228  		msrcs = append(msrcs, s.msrc)
   229  		*s = profileSource{}
   230  	}
   231  
   232  	if len(profiles) == 0 {
   233  		return nil, nil, false, 0, nil
   234  	}
   235  
   236  	p, msrc, err := combineProfiles(profiles, msrcs)
   237  	if err != nil {
   238  		return nil, nil, false, 0, err
   239  	}
   240  	return p, msrc, save, len(profiles), nil
   241  }
   242  
   243  func combineProfiles(profiles []*profile.Profile, msrcs []plugin.MappingSources) (*profile.Profile, plugin.MappingSources, error) {
   244  	// Merge profiles.
   245  	if err := measurement.ScaleProfiles(profiles); err != nil {
   246  		return nil, nil, err
   247  	}
   248  
   249  	p, err := profile.Merge(profiles)
   250  	if err != nil {
   251  		return nil, nil, err
   252  	}
   253  
   254  	// Combine mapping sources.
   255  	msrc := make(plugin.MappingSources)
   256  	for _, ms := range msrcs {
   257  		for m, s := range ms {
   258  			msrc[m] = append(msrc[m], s...)
   259  		}
   260  	}
   261  	return p, msrc, nil
   262  }
   263  
   264  type profileSource struct {
   265  	addr   string
   266  	source *source
   267  
   268  	p      *profile.Profile
   269  	msrc   plugin.MappingSources
   270  	remote bool
   271  	err    error
   272  }
   273  
   274  func homeEnv() string {
   275  	switch runtime.GOOS {
   276  	case "windows":
   277  		return "USERPROFILE"
   278  	case "plan9":
   279  		return "home"
   280  	default:
   281  		return "HOME"
   282  	}
   283  }
   284  
   285  // setTmpDir prepares the directory to use to save profiles retrieved
   286  // remotely. It is selected from PPROF_TMPDIR, defaults to $HOME/pprof, and, if
   287  // $HOME is not set, falls back to os.TempDir().
   288  func setTmpDir(ui plugin.UI) (string, error) {
   289  	var dirs []string
   290  	if profileDir := os.Getenv("PPROF_TMPDIR"); profileDir != "" {
   291  		dirs = append(dirs, profileDir)
   292  	}
   293  	if homeDir := os.Getenv(homeEnv()); homeDir != "" {
   294  		dirs = append(dirs, filepath.Join(homeDir, "pprof"))
   295  	}
   296  	dirs = append(dirs, os.TempDir())
   297  	for _, tmpDir := range dirs {
   298  		if err := os.MkdirAll(tmpDir, 0755); err != nil {
   299  			ui.PrintErr("Could not use temp dir ", tmpDir, ": ", err.Error())
   300  			continue
   301  		}
   302  		return tmpDir, nil
   303  	}
   304  	return "", fmt.Errorf("failed to identify temp dir")
   305  }
   306  
   307  const testSourceAddress = "pproftest.local"
   308  
   309  // grabProfile fetches a profile. Returns the profile, sources for the
   310  // profile mappings, a bool indicating if the profile was fetched
   311  // remotely, and an error.
   312  func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
   313  	var src string
   314  	duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second
   315  	if fetcher != nil {
   316  		p, src, err = fetcher.Fetch(source, duration, timeout)
   317  		if err != nil {
   318  			return
   319  		}
   320  	}
   321  	if err != nil || p == nil {
   322  		// Fetch the profile over HTTP or from a file.
   323  		p, src, err = fetch(source, duration, timeout, ui, tr)
   324  		if err != nil {
   325  			return
   326  		}
   327  	}
   328  
   329  	if err = p.CheckValid(); err != nil {
   330  		return
   331  	}
   332  
   333  	// Update the binary locations from command line and paths.
   334  	locateBinaries(p, s, obj, ui)
   335  
   336  	// Collect the source URL for all mappings.
   337  	if src != "" {
   338  		msrc = collectMappingSources(p, src)
   339  		remote = true
   340  		if strings.HasPrefix(src, "http://"+testSourceAddress) {
   341  			// Treat test inputs as local to avoid saving
   342  			// testcase profiles during driver testing.
   343  			remote = false
   344  		}
   345  	}
   346  	return
   347  }
   348  
   349  // collectMappingSources saves the mapping sources of a profile.
   350  func collectMappingSources(p *profile.Profile, source string) plugin.MappingSources {
   351  	ms := plugin.MappingSources{}
   352  	for _, m := range p.Mapping {
   353  		src := struct {
   354  			Source string
   355  			Start  uint64
   356  		}{
   357  			source, m.Start,
   358  		}
   359  		key := m.BuildID
   360  		if key == "" {
   361  			key = m.File
   362  		}
   363  		if key == "" {
   364  			// If there is no build id or source file, use the source as the
   365  			// mapping file. This will enable remote symbolization for this
   366  			// mapping, in particular for Go profiles on the legacy format.
   367  			// The source is reset back to empty string by unsourceMapping
   368  			// which is called after symbolization is finished.
   369  			m.File = source
   370  			key = source
   371  		}
   372  		ms[key] = append(ms[key], src)
   373  	}
   374  	return ms
   375  }
   376  
   377  // unsourceMappings iterates over the mappings in a profile and replaces file
   378  // set to the remote source URL by collectMappingSources back to empty string.
   379  func unsourceMappings(p *profile.Profile) {
   380  	for _, m := range p.Mapping {
   381  		if m.BuildID == "" {
   382  			if u, err := url.Parse(m.File); err == nil && u.IsAbs() {
   383  				m.File = ""
   384  			}
   385  		}
   386  	}
   387  }
   388  
   389  // locateBinaries searches for binary files listed in the profile and, if found,
   390  // updates the profile accordingly.
   391  func locateBinaries(p *profile.Profile, s *source, obj plugin.ObjTool, ui plugin.UI) {
   392  	// Construct search path to examine
   393  	searchPath := os.Getenv("PPROF_BINARY_PATH")
   394  	if searchPath == "" {
   395  		// Use $HOME/pprof/binaries as default directory for local symbolization binaries
   396  		searchPath = filepath.Join(os.Getenv(homeEnv()), "pprof", "binaries")
   397  	}
   398  mapping:
   399  	for _, m := range p.Mapping {
   400  		var baseName string
   401  		if m.File != "" {
   402  			baseName = filepath.Base(m.File)
   403  		}
   404  
   405  		for _, path := range filepath.SplitList(searchPath) {
   406  			var fileNames []string
   407  			if m.BuildID != "" {
   408  				fileNames = []string{filepath.Join(path, m.BuildID, baseName)}
   409  				if matches, err := filepath.Glob(filepath.Join(path, m.BuildID, "*")); err == nil {
   410  					fileNames = append(fileNames, matches...)
   411  				}
   412  				fileNames = append(fileNames, filepath.Join(path, m.File, m.BuildID)) // perf path format
   413  			}
   414  			if m.File != "" {
   415  				// Try both the basename and the full path, to support the same directory
   416  				// structure as the perf symfs option.
   417  				if baseName != "" {
   418  					fileNames = append(fileNames, filepath.Join(path, baseName))
   419  				}
   420  				fileNames = append(fileNames, filepath.Join(path, m.File))
   421  			}
   422  			for _, name := range fileNames {
   423  				if f, err := obj.Open(name, m.Start, m.Limit, m.Offset); err == nil {
   424  					defer f.Close()
   425  					fileBuildID := f.BuildID()
   426  					if m.BuildID != "" && m.BuildID != fileBuildID {
   427  						ui.PrintErr("Ignoring local file " + name + ": build-id mismatch (" + m.BuildID + " != " + fileBuildID + ")")
   428  					} else {
   429  						m.File = name
   430  						continue mapping
   431  					}
   432  				}
   433  			}
   434  		}
   435  	}
   436  	if len(p.Mapping) == 0 {
   437  		// If there are no mappings, add a fake mapping to attempt symbolization.
   438  		// This is useful for some profiles generated by the golang runtime, which
   439  		// do not include any mappings. Symbolization with a fake mapping will only
   440  		// be successful against a non-PIE binary.
   441  		m := &profile.Mapping{ID: 1}
   442  		p.Mapping = []*profile.Mapping{m}
   443  		for _, l := range p.Location {
   444  			l.Mapping = m
   445  		}
   446  	}
   447  	// Replace executable filename/buildID with the overrides from source.
   448  	// Assumes the executable is the first Mapping entry.
   449  	if execName, buildID := s.ExecName, s.BuildID; execName != "" || buildID != "" {
   450  		m := p.Mapping[0]
   451  		if execName != "" {
   452  			m.File = execName
   453  		}
   454  		if buildID != "" {
   455  			m.BuildID = buildID
   456  		}
   457  	}
   458  }
   459  
   460  // fetch fetches a profile from source, within the timeout specified,
   461  // producing messages through the ui. It returns the profile and the
   462  // url of the actual source of the profile for remote profiles.
   463  func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) {
   464  	var f io.ReadCloser
   465  
   466  	if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" {
   467  		ui.Print("Fetching profile over HTTP from " + sourceURL)
   468  		if duration > 0 {
   469  			ui.Print(fmt.Sprintf("Please wait... (%v)", duration))
   470  		}
   471  		f, err = fetchURL(sourceURL, timeout, tr)
   472  		src = sourceURL
   473  	} else if isPerfFile(source) {
   474  		f, err = convertPerfData(source, ui)
   475  	} else {
   476  		f, err = os.Open(source)
   477  	}
   478  	if err == nil {
   479  		defer f.Close()
   480  		p, err = profile.Parse(f)
   481  	}
   482  	return
   483  }
   484  
   485  // fetchURL fetches a profile from a URL using HTTP.
   486  func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) {
   487  	client := &http.Client{
   488  		Transport: tr,
   489  		Timeout:   timeout + 5*time.Second,
   490  	}
   491  	resp, err := client.Get(source)
   492  	if err != nil {
   493  		return nil, fmt.Errorf("http fetch: %v", err)
   494  	}
   495  	if resp.StatusCode != http.StatusOK {
   496  		defer resp.Body.Close()
   497  		return nil, statusCodeError(resp)
   498  	}
   499  
   500  	return resp.Body, nil
   501  }
   502  
   503  func statusCodeError(resp *http.Response) error {
   504  	if resp.Header.Get("X-Go-Pprof") != "" && strings.Contains(resp.Header.Get("Content-Type"), "text/plain") {
   505  		// error is from pprof endpoint
   506  		if body, err := ioutil.ReadAll(resp.Body); err == nil {
   507  			return fmt.Errorf("server response: %s - %s", resp.Status, body)
   508  		}
   509  	}
   510  	return fmt.Errorf("server response: %s", resp.Status)
   511  }
   512  
   513  // isPerfFile checks if a file is in perf.data format. It also returns false
   514  // if it encounters an error during the check.
   515  func isPerfFile(path string) bool {
   516  	sourceFile, openErr := os.Open(path)
   517  	if openErr != nil {
   518  		return false
   519  	}
   520  	defer sourceFile.Close()
   521  
   522  	// If the file is the output of a perf record command, it should begin
   523  	// with the string PERFILE2.
   524  	perfHeader := []byte("PERFILE2")
   525  	actualHeader := make([]byte, len(perfHeader))
   526  	if _, readErr := sourceFile.Read(actualHeader); readErr != nil {
   527  		return false
   528  	}
   529  	return bytes.Equal(actualHeader, perfHeader)
   530  }
   531  
   532  // convertPerfData converts the file at path which should be in perf.data format
   533  // using the perf_to_profile tool and returns the file containing the
   534  // profile.proto formatted data.
   535  func convertPerfData(perfPath string, ui plugin.UI) (*os.File, error) {
   536  	ui.Print(fmt.Sprintf(
   537  		"Converting %s to a profile.proto... (May take a few minutes)",
   538  		perfPath))
   539  	profile, err := newTempFile(os.TempDir(), "pprof_", ".pb.gz")
   540  	if err != nil {
   541  		return nil, err
   542  	}
   543  	deferDeleteTempFile(profile.Name())
   544  	cmd := exec.Command("perf_to_profile", "-i", perfPath, "-o", profile.Name(), "-f")
   545  	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
   546  	if err := cmd.Run(); err != nil {
   547  		profile.Close()
   548  		return nil, fmt.Errorf("failed to convert perf.data file. Try github.com/google/perf_data_converter: %v", err)
   549  	}
   550  	return profile, nil
   551  }
   552  
   553  // adjustURL validates if a profile source is a URL and returns an
   554  // cleaned up URL and the timeout to use for retrieval over HTTP.
   555  // If the source cannot be recognized as a URL it returns an empty string.
   556  func adjustURL(source string, duration, timeout time.Duration) (string, time.Duration) {
   557  	u, err := url.Parse(source)
   558  	if err != nil || (u.Host == "" && u.Scheme != "" && u.Scheme != "file") {
   559  		// Try adding http:// to catch sources of the form hostname:port/path.
   560  		// url.Parse treats "hostname" as the scheme.
   561  		u, err = url.Parse("http://" + source)
   562  	}
   563  	if err != nil || u.Host == "" {
   564  		return "", 0
   565  	}
   566  
   567  	// Apply duration/timeout overrides to URL.
   568  	values := u.Query()
   569  	if duration > 0 {
   570  		values.Set("seconds", fmt.Sprint(int(duration.Seconds())))
   571  	} else {
   572  		if urlSeconds := values.Get("seconds"); urlSeconds != "" {
   573  			if us, err := strconv.ParseInt(urlSeconds, 10, 32); err == nil {
   574  				duration = time.Duration(us) * time.Second
   575  			}
   576  		}
   577  	}
   578  	if timeout <= 0 {
   579  		if duration > 0 {
   580  			timeout = duration + duration/2
   581  		} else {
   582  			timeout = 60 * time.Second
   583  		}
   584  	}
   585  	u.RawQuery = values.Encode()
   586  	return u.String(), timeout
   587  }
   588
View as plain text