fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/trace"
    30  	"cmd/internal/par"
    31  	"cmd/internal/robustio"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    39  
    40  // Download downloads the specific module version to the
    41  // local download cache and returns the name of the directory
    42  // corresponding to the root of the module's file tree.
    43  func (f *Fetcher) Download(ctx context.Context, mod module.Version) (dir string, err error) {
    44  	if gover.IsToolchain(mod.Path) {
    45  		return "", ErrToolchain
    46  	}
    47  	if err := checkCacheDir(ctx); err != nil {
    48  		base.Fatal(err)
    49  	}
    50  
    51  	// The par.Cache here avoids duplicate work.
    52  	return f.downloadCache.Do(mod, func() (string, error) {
    53  		dir, err := f.download(ctx, mod)
    54  		if err != nil {
    55  			return "", err
    56  		}
    57  		f.checkMod(ctx, mod)
    58  
    59  		// If go.mod exists (not an old legacy module), check version is not too new.
    60  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    61  			goVersion := gover.GoModLookup(data, "go")
    62  			if gover.Compare(goVersion, gover.Local()) > 0 {
    63  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    64  			}
    65  		} else if !errors.Is(err, fs.ErrNotExist) {
    66  			return "", err
    67  		}
    68  
    69  		return dir, nil
    70  	})
    71  }
    72  
    73  // Unzip is like Download but is given the explicit zip file to use,
    74  // rather than downloading it. This is used for the GOFIPS140 zip files,
    75  // which ship in the Go distribution itself.
    76  func (f *Fetcher) Unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
    77  	if err := checkCacheDir(ctx); err != nil {
    78  		base.Fatal(err)
    79  	}
    80  
    81  	return f.downloadCache.Do(mod, func() (string, error) {
    82  		ctx, span := trace.StartSpan(ctx, "modfetch.Unzip "+mod.String())
    83  		defer span.Done()
    84  
    85  		dir, err = DownloadDir(ctx, mod)
    86  		if err == nil {
    87  			// The directory has already been completely extracted (no .partial file exists).
    88  			return dir, nil
    89  		} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    90  			return "", err
    91  		}
    92  
    93  		return unzip(ctx, mod, zipfile)
    94  	})
    95  }
    96  
    97  func (f *Fetcher) download(ctx context.Context, mod module.Version) (dir string, err error) {
    98  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    99  	defer span.Done()
   100  
   101  	dir, err = DownloadDir(ctx, mod)
   102  	if err == nil {
   103  		// The directory has already been completely extracted (no .partial file exists).
   104  		return dir, nil
   105  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
   106  		return "", err
   107  	}
   108  
   109  	// To avoid cluttering the cache with extraneous files,
   110  	// DownloadZip uses the same lockfile as Download.
   111  	// Invoke DownloadZip before locking the file.
   112  	zipfile, err := f.DownloadZip(ctx, mod)
   113  	if err != nil {
   114  		return "", err
   115  	}
   116  
   117  	return unzip(ctx, mod, zipfile)
   118  }
   119  
   120  func unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
   121  	unlock, err := lockVersion(ctx, mod)
   122  	if err != nil {
   123  		return "", err
   124  	}
   125  	defer unlock()
   126  
   127  	ctx, span := trace.StartSpan(ctx, "unzip "+zipfile)
   128  	defer span.Done()
   129  
   130  	// Check whether the directory was populated while we were waiting on the lock.
   131  	dir, dirErr := DownloadDir(ctx, mod)
   132  	if dirErr == nil {
   133  		return dir, nil
   134  	}
   135  	_, dirExists := dirErr.(*DownloadDirPartialError)
   136  
   137  	// Clean up any remaining temporary directories created by old versions
   138  	// (before 1.16), as well as partially extracted directories (indicated by
   139  	// DownloadDirPartialError, usually because of a .partial file). This is only
   140  	// safe to do because the lock file ensures that their writers are no longer
   141  	// active.
   142  	parentDir := filepath.Dir(dir)
   143  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   144  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   145  		for _, path := range old {
   146  			RemoveAll(path) // best effort
   147  		}
   148  	}
   149  	if dirExists {
   150  		if err := RemoveAll(dir); err != nil {
   151  			return "", err
   152  		}
   153  	}
   154  
   155  	partialPath, err := CachePath(ctx, mod, "partial")
   156  	if err != nil {
   157  		return "", err
   158  	}
   159  
   160  	// Extract the module zip directory at its final location.
   161  	//
   162  	// To prevent other processes from reading the directory if we crash,
   163  	// create a .partial file before extracting the directory, and delete
   164  	// the .partial file afterward (all while holding the lock).
   165  	//
   166  	// Before Go 1.16, we extracted to a temporary directory with a random name
   167  	// then renamed it into place with os.Rename. On Windows, this failed with
   168  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   169  	// opened files in the temporary directory.
   170  	//
   171  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   172  	// partially extracted directories. 'go mod verify' can detect this,
   173  	// and 'go clean -modcache' can fix it.
   174  	if err := os.MkdirAll(parentDir, 0o777); err != nil {
   175  		return "", err
   176  	}
   177  	if err := os.WriteFile(partialPath, nil, 0o666); err != nil {
   178  		return "", err
   179  	}
   180  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   181  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   182  		if rmErr := RemoveAll(dir); rmErr == nil {
   183  			os.Remove(partialPath)
   184  		}
   185  		return "", err
   186  	}
   187  	if err := os.Remove(partialPath); err != nil {
   188  		return "", err
   189  	}
   190  
   191  	if !cfg.ModCacheRW {
   192  		makeDirsReadOnly(dir)
   193  	}
   194  	return dir, nil
   195  }
   196  
   197  var downloadZipCache par.ErrCache[module.Version, string]
   198  
   199  // DownloadZip downloads the specific module version to the
   200  // local zip cache and returns the name of the zip file.
   201  func (f *Fetcher) DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   202  	// The par.Cache here avoids duplicate work.
   203  	return downloadZipCache.Do(mod, func() (string, error) {
   204  		zipfile, err := CachePath(ctx, mod, "zip")
   205  		if err != nil {
   206  			return "", err
   207  		}
   208  		ziphashfile := zipfile + "hash"
   209  
   210  		// Return early if the zip and ziphash files exist.
   211  		if _, err := os.Stat(zipfile); err == nil {
   212  			if _, err := os.Stat(ziphashfile); err == nil {
   213  				if !HaveSum(f, mod) {
   214  					f.checkMod(ctx, mod)
   215  				}
   216  				return zipfile, nil
   217  			}
   218  		}
   219  
   220  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   221  		if cfg.CmdName != "mod download" {
   222  			vers := mod.Version
   223  			if mod.Path == "golang.org/toolchain" {
   224  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   225  				_, vers, _ = strings.Cut(vers, "-")
   226  				if i := strings.LastIndex(vers, "."); i >= 0 {
   227  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   228  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   229  				}
   230  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   231  			} else {
   232  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   233  			}
   234  		}
   235  		unlock, err := lockVersion(ctx, mod)
   236  		if err != nil {
   237  			return "", err
   238  		}
   239  		defer unlock()
   240  
   241  		if err := f.downloadZip(ctx, mod, zipfile); err != nil {
   242  			return "", err
   243  		}
   244  		return zipfile, nil
   245  	})
   246  }
   247  
   248  func (f *Fetcher) downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   249  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   250  	defer span.Done()
   251  
   252  	// Double-check that the zipfile was not created while we were waiting for
   253  	// the lock in DownloadZip.
   254  	ziphashfile := zipfile + "hash"
   255  	var zipExists, ziphashExists bool
   256  	if _, err := os.Stat(zipfile); err == nil {
   257  		zipExists = true
   258  	}
   259  	if _, err := os.Stat(ziphashfile); err == nil {
   260  		ziphashExists = true
   261  	}
   262  	if zipExists && ziphashExists {
   263  		return nil
   264  	}
   265  
   266  	// Create parent directories.
   267  	if err := os.MkdirAll(filepath.Dir(zipfile), 0o777); err != nil {
   268  		return err
   269  	}
   270  
   271  	// Clean up any remaining tempfiles from previous runs.
   272  	// This is only safe to do because the lock file ensures that their
   273  	// writers are no longer active.
   274  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   275  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   276  		for _, path := range old {
   277  			os.Remove(path) // best effort
   278  		}
   279  	}
   280  
   281  	// If the zip file exists, the ziphash file must have been deleted
   282  	// or lost after a file system crash. Re-hash the zip without downloading.
   283  	if zipExists {
   284  		return hashZip(f, mod, zipfile, ziphashfile)
   285  	}
   286  
   287  	// From here to the os.Rename call below is functionally almost equivalent to
   288  	// renameio.WriteToFile, with one key difference: we want to validate the
   289  	// contents of the file (by hashing it) before we commit it. Because the file
   290  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   291  	// validate it: we can't just tee the stream as we write it.
   292  	file, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0o666)
   293  	if err != nil {
   294  		return err
   295  	}
   296  	defer func() {
   297  		if err != nil {
   298  			file.Close()
   299  			os.Remove(file.Name())
   300  		}
   301  	}()
   302  
   303  	var unrecoverableErr error
   304  	err = TryProxies(func(proxy string) error {
   305  		if unrecoverableErr != nil {
   306  			return unrecoverableErr
   307  		}
   308  		repo := f.Lookup(ctx, proxy, mod.Path)
   309  		err := repo.Zip(ctx, file, mod.Version)
   310  		if err != nil {
   311  			// Zip may have partially written to f before failing.
   312  			// (Perhaps the server crashed while sending the file?)
   313  			// Since we allow fallback on error in some cases, we need to fix up the
   314  			// file to be empty again for the next attempt.
   315  			if _, err := file.Seek(0, io.SeekStart); err != nil {
   316  				unrecoverableErr = err
   317  				return err
   318  			}
   319  			if err := file.Truncate(0); err != nil {
   320  				unrecoverableErr = err
   321  				return err
   322  			}
   323  		}
   324  		return err
   325  	})
   326  	if err != nil {
   327  		return err
   328  	}
   329  
   330  	// Double-check that the paths within the zip file are well-formed.
   331  	//
   332  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   333  	fi, err := file.Stat()
   334  	if err != nil {
   335  		return err
   336  	}
   337  	z, err := zip.NewReader(file, fi.Size())
   338  	if err != nil {
   339  		return err
   340  	}
   341  	prefix := mod.Path + "@" + mod.Version + "/"
   342  	for _, zf := range z.File {
   343  		if !strings.HasPrefix(zf.Name, prefix) {
   344  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], zf.Name)
   345  		}
   346  	}
   347  
   348  	if err := file.Close(); err != nil {
   349  		return err
   350  	}
   351  
   352  	// Hash the zip file and check the sum before renaming to the final location.
   353  	if err := hashZip(f, mod, file.Name(), ziphashfile); err != nil {
   354  		return err
   355  	}
   356  	if err := os.Rename(file.Name(), zipfile); err != nil {
   357  		return err
   358  	}
   359  
   360  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   361  
   362  	return nil
   363  }
   364  
   365  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   366  // overwriting that file if it exists.
   367  //
   368  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   369  // an error and does not write ziphashfile.
   370  func hashZip(f *Fetcher, mod module.Version, zipfile, ziphashfile string) (err error) {
   371  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   372  	if err != nil {
   373  		return err
   374  	}
   375  	if err := checkModSum(f, mod, hash); err != nil {
   376  		return err
   377  	}
   378  	hf, err := lockedfile.Create(ziphashfile)
   379  	if err != nil {
   380  		return err
   381  	}
   382  	defer func() {
   383  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   384  			err = closeErr
   385  		}
   386  	}()
   387  	if err := hf.Truncate(int64(len(hash))); err != nil {
   388  		return err
   389  	}
   390  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   391  		return err
   392  	}
   393  	return nil
   394  }
   395  
   396  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   397  // and its transitive contents.
   398  func makeDirsReadOnly(dir string) {
   399  	type pathMode struct {
   400  		path string
   401  		mode fs.FileMode
   402  	}
   403  	var dirs []pathMode // in lexical order
   404  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   405  		if err == nil && d.IsDir() {
   406  			info, err := d.Info()
   407  			if err == nil && info.Mode()&0o222 != 0 {
   408  				dirs = append(dirs, pathMode{path, info.Mode()})
   409  			}
   410  		}
   411  		return nil
   412  	})
   413  
   414  	// Run over list backward to chmod children before parents.
   415  	for i := len(dirs) - 1; i >= 0; i-- {
   416  		os.Chmod(dirs[i].path, dirs[i].mode&^0o222)
   417  	}
   418  }
   419  
   420  // RemoveAll removes a directory written by Download or Unzip, first applying
   421  // any permission changes needed to do so.
   422  func RemoveAll(dir string) error {
   423  	// Module cache has 0555 directories; make them writable in order to remove content.
   424  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   425  		if err != nil {
   426  			return nil // ignore errors walking in file system
   427  		}
   428  		if info.IsDir() {
   429  			os.Chmod(path, 0o777)
   430  		}
   431  		return nil
   432  	})
   433  	return robustio.RemoveAll(dir)
   434  }
   435  
   436  // The GoSumFile, WorkspaceGoSumFiles, and goSum are global state that must not be
   437  // accessed by any of the exported functions of this package after they return, because
   438  // they can be modified by the non-thread-safe SetState function.
   439  
   440  type modSum struct {
   441  	mod module.Version
   442  	sum string
   443  }
   444  
   445  type sumState struct {
   446  	m         map[module.Version][]string            // content of go.sum file
   447  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   448  	status    map[modSum]modSumStatus                // state of sums in m
   449  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   450  	enabled   bool                                   // whether to use go.sum at all
   451  }
   452  
   453  type modSumStatus struct {
   454  	used, dirty bool
   455  }
   456  
   457  // Fetcher holds a snapshot of the global state of the modfetch package.
   458  type Fetcher struct {
   459  	// path to go.sum; set by package modload
   460  	goSumFile string
   461  	// path to module go.sums in workspace; set by package modload
   462  	workspaceGoSumFiles []string
   463  	// The Lookup cache is used cache the work done by Lookup.
   464  	// It is important that the global functions of this package that access it do not
   465  	// do so after they return.
   466  	lookupCache *par.Cache[lookupCacheKey, Repo]
   467  	// The downloadCache is used to cache the operation of downloading a module to disk
   468  	// (if it's not already downloaded) and getting the directory it was downloaded to.
   469  	// It is important that downloadCache must not be accessed by any of the exported
   470  	// functions of this package after they return, because it can be modified by the
   471  	// non-thread-safe SetState function.
   472  	downloadCache *par.ErrCache[module.Version, string] // version → directory;
   473  
   474  	mu       sync.Mutex
   475  	sumState sumState
   476  }
   477  
   478  func NewFetcher() *Fetcher {
   479  	f := new(Fetcher)
   480  	f.lookupCache = new(par.Cache[lookupCacheKey, Repo])
   481  	f.downloadCache = new(par.ErrCache[module.Version, string])
   482  	return f
   483  }
   484  
   485  func (f *Fetcher) GoSumFile() string {
   486  	return f.goSumFile
   487  }
   488  
   489  func (f *Fetcher) SetGoSumFile(str string) {
   490  	f.goSumFile = str
   491  }
   492  
   493  func (f *Fetcher) AddWorkspaceGoSumFile(file string) {
   494  	f.workspaceGoSumFiles = append(f.workspaceGoSumFiles, file)
   495  }
   496  
   497  // Reset resets globals in the modfetch package, so previous loads don't affect
   498  // contents of go.sum files.
   499  func (f *Fetcher) Reset() {
   500  	f.SetState(NewFetcher())
   501  }
   502  
   503  // SetState sets the global state of the modfetch package to the newState, and returns the previous
   504  // global state. newState should have been returned by SetState, or be an empty State.
   505  // There should be no concurrent calls to any of the exported functions of this package with
   506  // a call to SetState because it will modify the global state in a non-thread-safe way.
   507  func (f *Fetcher) SetState(newState *Fetcher) (oldState *Fetcher) {
   508  	if newState.lookupCache == nil {
   509  		newState.lookupCache = new(par.Cache[lookupCacheKey, Repo])
   510  	}
   511  	if newState.downloadCache == nil {
   512  		newState.downloadCache = new(par.ErrCache[module.Version, string])
   513  	}
   514  
   515  	f.mu.Lock()
   516  	defer f.mu.Unlock()
   517  
   518  	oldState = &Fetcher{
   519  		goSumFile:           f.goSumFile,
   520  		workspaceGoSumFiles: f.workspaceGoSumFiles,
   521  		lookupCache:         f.lookupCache,
   522  		downloadCache:       f.downloadCache,
   523  		sumState:            f.sumState,
   524  	}
   525  
   526  	f.SetGoSumFile(newState.goSumFile)
   527  	f.workspaceGoSumFiles = newState.workspaceGoSumFiles
   528  	// Uses of lookupCache and downloadCache both can call checkModSum,
   529  	// which in turn sets the used bit on goSum.status for modules.
   530  	// Set (or reset) them so used can be computed properly.
   531  	f.lookupCache = newState.lookupCache
   532  	f.downloadCache = newState.downloadCache
   533  	// Set, or reset all fields on goSum. If being reset to empty, it will be initialized later.
   534  	f.sumState = newState.sumState
   535  
   536  	return oldState
   537  }
   538  
   539  // initGoSum initializes the go.sum data.
   540  // The boolean it returns reports whether the
   541  // use of go.sum is now enabled.
   542  // The goSum lock must be held.
   543  func (f *Fetcher) initGoSum() (bool, error) {
   544  	if f.goSumFile == "" {
   545  		return false, nil
   546  	}
   547  	if f.sumState.m != nil {
   548  		return true, nil
   549  	}
   550  
   551  	f.sumState.m = make(map[module.Version][]string)
   552  	f.sumState.status = make(map[modSum]modSumStatus)
   553  	f.sumState.w = make(map[string]map[module.Version][]string)
   554  
   555  	for _, fn := range f.workspaceGoSumFiles {
   556  		f.sumState.w[fn] = make(map[module.Version][]string)
   557  		_, err := readGoSumFile(f.sumState.w[fn], fn)
   558  		if err != nil {
   559  			return false, err
   560  		}
   561  	}
   562  
   563  	enabled, err := readGoSumFile(f.sumState.m, f.goSumFile)
   564  	f.sumState.enabled = enabled
   565  	return enabled, err
   566  }
   567  
   568  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   569  	var (
   570  		data []byte
   571  		err  error
   572  	)
   573  	if fsys.Replaced(file) {
   574  		// Don't lock go.sum if it's part of the overlay.
   575  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   576  		// in the overlay. See #44700.
   577  		data, err = os.ReadFile(fsys.Actual(file))
   578  	} else {
   579  		data, err = lockedfile.Read(file)
   580  	}
   581  	if err != nil && !os.IsNotExist(err) {
   582  		return false, err
   583  	}
   584  	readGoSum(dst, file, data)
   585  
   586  	return true, nil
   587  }
   588  
   589  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   590  // A bug caused us to write these into go.sum files for non-modules.
   591  // We detect and remove them.
   592  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   593  
   594  // readGoSum parses data, which is the content of file,
   595  // and adds it to goSum.m. The goSum lock must be held.
   596  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   597  	lineno := 0
   598  	for len(data) > 0 {
   599  		var line []byte
   600  		lineno++
   601  		i := bytes.IndexByte(data, '\n')
   602  		if i < 0 {
   603  			line, data = data, nil
   604  		} else {
   605  			line, data = data[:i], data[i+1:]
   606  		}
   607  		f := strings.Fields(string(line))
   608  		if len(f) == 0 {
   609  			// blank line; skip it
   610  			continue
   611  		}
   612  		if len(f) != 3 {
   613  			if cfg.CmdName == "mod tidy" {
   614  				// ignore malformed line so that go mod tidy can fix go.sum
   615  				continue
   616  			} else {
   617  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   618  			}
   619  		}
   620  		if f[2] == emptyGoModHash {
   621  			// Old bug; drop it.
   622  			continue
   623  		}
   624  		mod := module.Version{Path: f[0], Version: f[1]}
   625  		dst[mod] = append(dst[mod], f[2])
   626  	}
   627  }
   628  
   629  // HaveSum returns true if the go.sum file contains an entry for mod.
   630  // The entry's hash must be generated with a known hash algorithm.
   631  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   632  // .mod and .zip files.
   633  func HaveSum(f *Fetcher, mod module.Version) bool {
   634  	f.mu.Lock()
   635  	defer f.mu.Unlock()
   636  	inited, err := f.initGoSum()
   637  	if err != nil || !inited {
   638  		return false
   639  	}
   640  	for _, goSums := range f.sumState.w {
   641  		for _, h := range goSums[mod] {
   642  			if !strings.HasPrefix(h, "h1:") {
   643  				continue
   644  			}
   645  			if !f.sumState.status[modSum{mod, h}].dirty {
   646  				return true
   647  			}
   648  		}
   649  	}
   650  	for _, h := range f.sumState.m[mod] {
   651  		if !strings.HasPrefix(h, "h1:") {
   652  			continue
   653  		}
   654  		if !f.sumState.status[modSum{mod, h}].dirty {
   655  			return true
   656  		}
   657  	}
   658  	return false
   659  }
   660  
   661  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   662  // The boolean reports true if an entry was found or
   663  // false if no entry found or two conflicting sums are found.
   664  // The entry's hash must be generated with a known hash algorithm.
   665  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   666  // .mod and .zip files.
   667  func (f *Fetcher) RecordedSum(mod module.Version) (sum string, ok bool) {
   668  	f.mu.Lock()
   669  	defer f.mu.Unlock()
   670  	inited, err := f.initGoSum()
   671  	foundSum := ""
   672  	if err != nil || !inited {
   673  		return "", false
   674  	}
   675  	for _, goSums := range f.sumState.w {
   676  		for _, h := range goSums[mod] {
   677  			if !strings.HasPrefix(h, "h1:") {
   678  				continue
   679  			}
   680  			if !f.sumState.status[modSum{mod, h}].dirty {
   681  				if foundSum != "" && foundSum != h { // conflicting sums exist
   682  					return "", false
   683  				}
   684  				foundSum = h
   685  			}
   686  		}
   687  	}
   688  	for _, h := range f.sumState.m[mod] {
   689  		if !strings.HasPrefix(h, "h1:") {
   690  			continue
   691  		}
   692  		if !f.sumState.status[modSum{mod, h}].dirty {
   693  			if foundSum != "" && foundSum != h { // conflicting sums exist
   694  				return "", false
   695  			}
   696  			foundSum = h
   697  		}
   698  	}
   699  	return foundSum, true
   700  }
   701  
   702  // checkMod checks the given module's checksum and Go version.
   703  func (f *Fetcher) checkMod(ctx context.Context, mod module.Version) {
   704  	// Do the file I/O before acquiring the go.sum lock.
   705  	ziphash, err := CachePath(ctx, mod, "ziphash")
   706  	if err != nil {
   707  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   708  	}
   709  	data, err := lockedfile.Read(ziphash)
   710  	if err != nil {
   711  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   712  	}
   713  	data = bytes.TrimSpace(data)
   714  	if !isValidSum(data) {
   715  		// Recreate ziphash file from zip file and use that to check the mod sum.
   716  		zip, err := CachePath(ctx, mod, "zip")
   717  		if err != nil {
   718  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   719  		}
   720  		err = hashZip(f, mod, zip, ziphash)
   721  		if err != nil {
   722  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   723  		}
   724  		return
   725  	}
   726  	h := string(data)
   727  	if !strings.HasPrefix(h, "h1:") {
   728  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   729  	}
   730  
   731  	if err := checkModSum(f, mod, h); err != nil {
   732  		base.Fatalf("%s", err)
   733  	}
   734  }
   735  
   736  // goModSum returns the checksum for the go.mod contents.
   737  func goModSum(data []byte) (string, error) {
   738  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   739  		return io.NopCloser(bytes.NewReader(data)), nil
   740  	})
   741  }
   742  
   743  // checkGoMod checks the given module's go.mod checksum;
   744  // data is the go.mod content.
   745  func checkGoMod(f *Fetcher, path, version string, data []byte) error {
   746  	h, err := goModSum(data)
   747  	if err != nil {
   748  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   749  	}
   750  
   751  	return checkModSum(f, module.Version{Path: path, Version: version + "/go.mod"}, h)
   752  }
   753  
   754  // checkModSum checks that the recorded checksum for mod is h.
   755  //
   756  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   757  // for the module's go.mod file only.
   758  func checkModSum(f *Fetcher, mod module.Version, h string) error {
   759  	// We lock goSum when manipulating it,
   760  	// but we arrange to release the lock when calling checkSumDB,
   761  	// so that parallel calls to checkModHash can execute parallel calls
   762  	// to checkSumDB.
   763  
   764  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   765  	f.mu.Lock()
   766  	inited, err := f.initGoSum()
   767  	if err != nil {
   768  		f.mu.Unlock()
   769  		return err
   770  	}
   771  	done := inited && haveModSumLocked(f, mod, h)
   772  	if inited {
   773  		st := f.sumState.status[modSum{mod, h}]
   774  		st.used = true
   775  		f.sumState.status[modSum{mod, h}] = st
   776  	}
   777  	f.mu.Unlock()
   778  
   779  	if done {
   780  		return nil
   781  	}
   782  
   783  	// Not listed, so we want to add them.
   784  	// Consult checksum database if appropriate.
   785  	if useSumDB(mod) {
   786  		// Calls base.Fatalf if mismatch detected.
   787  		if err := checkSumDB(mod, h); err != nil {
   788  			return err
   789  		}
   790  	}
   791  
   792  	// Add mod+h to go.sum, if it hasn't appeared already.
   793  	if inited {
   794  		f.mu.Lock()
   795  		addModSumLocked(f, mod, h)
   796  		st := f.sumState.status[modSum{mod, h}]
   797  		st.dirty = true
   798  		f.sumState.status[modSum{mod, h}] = st
   799  		f.mu.Unlock()
   800  	}
   801  	return nil
   802  }
   803  
   804  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   805  // If it finds a conflicting pair instead, it calls base.Fatalf.
   806  // goSum.mu must be locked.
   807  func haveModSumLocked(f *Fetcher, mod module.Version, h string) bool {
   808  	sumFileName := "go.sum"
   809  	if strings.HasSuffix(f.goSumFile, "go.work.sum") {
   810  		sumFileName = "go.work.sum"
   811  	}
   812  	for _, vh := range f.sumState.m[mod] {
   813  		if h == vh {
   814  			return true
   815  		}
   816  		if strings.HasPrefix(vh, "h1:") {
   817  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   818  		}
   819  	}
   820  	// Also check workspace sums.
   821  	foundMatch := false
   822  	// Check sums from all files in case there are conflicts between
   823  	// the files.
   824  	for goSumFile, goSums := range f.sumState.w {
   825  		for _, vh := range goSums[mod] {
   826  			if h == vh {
   827  				foundMatch = true
   828  			} else if strings.HasPrefix(vh, "h1:") {
   829  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   830  			}
   831  		}
   832  	}
   833  	return foundMatch
   834  }
   835  
   836  // addModSumLocked adds the pair mod,h to go.sum.
   837  // goSum.mu must be locked.
   838  func addModSumLocked(f *Fetcher, mod module.Version, h string) {
   839  	if haveModSumLocked(f, mod, h) {
   840  		return
   841  	}
   842  	if len(f.sumState.m[mod]) > 0 {
   843  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(f.sumState.m[mod], ", "), h)
   844  	}
   845  	f.sumState.m[mod] = append(f.sumState.m[mod], h)
   846  }
   847  
   848  // checkSumDB checks the mod, h pair against the Go checksum database.
   849  // It calls base.Fatalf if the hash is to be rejected.
   850  func checkSumDB(mod module.Version, h string) error {
   851  	modWithoutSuffix := mod
   852  	noun := "module"
   853  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   854  		noun = "go.mod"
   855  		modWithoutSuffix.Version = before
   856  	}
   857  
   858  	db, lines, err := lookupSumDB(mod)
   859  	if err != nil {
   860  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   861  	}
   862  
   863  	have := mod.Path + " " + mod.Version + " " + h
   864  	prefix := mod.Path + " " + mod.Version + " h1:"
   865  	for _, line := range lines {
   866  		if line == have {
   867  			return nil
   868  		}
   869  		if strings.HasPrefix(line, prefix) {
   870  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   871  		}
   872  	}
   873  	return nil
   874  }
   875  
   876  // Sum returns the checksum for the downloaded copy of the given module,
   877  // if present in the download cache.
   878  func Sum(ctx context.Context, mod module.Version) string {
   879  	if cfg.GOMODCACHE == "" {
   880  		// Do not use current directory.
   881  		return ""
   882  	}
   883  
   884  	ziphash, err := CachePath(ctx, mod, "ziphash")
   885  	if err != nil {
   886  		return ""
   887  	}
   888  	data, err := lockedfile.Read(ziphash)
   889  	if err != nil {
   890  		return ""
   891  	}
   892  	data = bytes.TrimSpace(data)
   893  	if !isValidSum(data) {
   894  		return ""
   895  	}
   896  	return string(data)
   897  }
   898  
   899  // isValidSum returns true if data is the valid contents of a zip hash file.
   900  // Certain critical files are written to disk by first truncating
   901  // then writing the actual bytes, so that if the write fails
   902  // the corrupt file should contain at least one of the null
   903  // bytes written by the truncate operation.
   904  func isValidSum(data []byte) bool {
   905  	if bytes.IndexByte(data, '\000') >= 0 {
   906  		return false
   907  	}
   908  
   909  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   910  		return false
   911  	}
   912  
   913  	return true
   914  }
   915  
   916  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   917  
   918  // WriteGoSum writes the go.sum file if it needs to be updated.
   919  //
   920  // keep is used to check whether a newly added sum should be saved in go.sum.
   921  // It should have entries for both module content sums and go.mod sums
   922  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   923  // have been marked for deletion with TrimGoSum.
   924  func (f *Fetcher) WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   925  	f.mu.Lock()
   926  	defer f.mu.Unlock()
   927  
   928  	// If we haven't read the go.sum file yet, don't bother writing it.
   929  	if !f.sumState.enabled {
   930  		return nil
   931  	}
   932  
   933  	// Check whether we need to add sums for which keep[m] is true or remove
   934  	// unused sums marked with TrimGoSum. If there are no changes to make,
   935  	// just return without opening go.sum.
   936  	dirty := false
   937  Outer:
   938  	for m, hs := range f.sumState.m {
   939  		for _, h := range hs {
   940  			st := f.sumState.status[modSum{m, h}]
   941  			if st.dirty && (!st.used || keep[m]) {
   942  				dirty = true
   943  				break Outer
   944  			}
   945  		}
   946  	}
   947  	if !dirty {
   948  		return nil
   949  	}
   950  	if readonly {
   951  		return ErrGoSumDirty
   952  	}
   953  	if fsys.Replaced(f.goSumFile) {
   954  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   955  	}
   956  
   957  	// Make a best-effort attempt to acquire the side lock, only to exclude
   958  	// previous versions of the 'go' command from making simultaneous edits.
   959  	if unlock, err := SideLock(ctx); err == nil {
   960  		defer unlock()
   961  	}
   962  
   963  	err := lockedfile.Transform(f.goSumFile, func(data []byte) ([]byte, error) {
   964  		tidyGoSum := tidyGoSum(f, data, keep)
   965  		return tidyGoSum, nil
   966  	})
   967  	if err != nil {
   968  		return fmt.Errorf("updating go.sum: %w", err)
   969  	}
   970  
   971  	f.sumState.status = make(map[modSum]modSumStatus)
   972  	f.sumState.overwrite = false
   973  	return nil
   974  }
   975  
   976  // TidyGoSum returns a tidy version of the go.sum file.
   977  // A missing go.sum file is treated as if empty.
   978  func (f *Fetcher) TidyGoSum(keep map[module.Version]bool) (before, after []byte) {
   979  	f.mu.Lock()
   980  	defer f.mu.Unlock()
   981  	before, err := lockedfile.Read(f.goSumFile)
   982  	if err != nil && !errors.Is(err, fs.ErrNotExist) {
   983  		base.Fatalf("reading go.sum: %v", err)
   984  	}
   985  	after = tidyGoSum(f, before, keep)
   986  	return before, after
   987  }
   988  
   989  // tidyGoSum returns a tidy version of the go.sum file.
   990  // The goSum lock must be held.
   991  func tidyGoSum(f *Fetcher, data []byte, keep map[module.Version]bool) []byte {
   992  	if !f.sumState.overwrite {
   993  		// Incorporate any sums added by other processes in the meantime.
   994  		// Add only the sums that we actually checked: the user may have edited or
   995  		// truncated the file to remove erroneous hashes, and we shouldn't restore
   996  		// them without good reason.
   997  		f.sumState.m = make(map[module.Version][]string, len(f.sumState.m))
   998  		readGoSum(f.sumState.m, f.goSumFile, data)
   999  		for ms, st := range f.sumState.status {
  1000  			if st.used && !sumInWorkspaceModulesLocked(f, ms.mod) {
  1001  				addModSumLocked(f, ms.mod, ms.sum)
  1002  			}
  1003  		}
  1004  	}
  1005  
  1006  	mods := make([]module.Version, 0, len(f.sumState.m))
  1007  	for m := range f.sumState.m {
  1008  		mods = append(mods, m)
  1009  	}
  1010  	module.Sort(mods)
  1011  
  1012  	var buf bytes.Buffer
  1013  	for _, m := range mods {
  1014  		list := f.sumState.m[m]
  1015  		sort.Strings(list)
  1016  		str.Uniq(&list)
  1017  		for _, h := range list {
  1018  			st := f.sumState.status[modSum{m, h}]
  1019  			if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(f, m) {
  1020  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
  1021  			}
  1022  		}
  1023  	}
  1024  	return buf.Bytes()
  1025  }
  1026  
  1027  func sumInWorkspaceModulesLocked(f *Fetcher, m module.Version) bool {
  1028  	for _, goSums := range f.sumState.w {
  1029  		if _, ok := goSums[m]; ok {
  1030  			return true
  1031  		}
  1032  	}
  1033  	return false
  1034  }
  1035  
  1036  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
  1037  // builds.
  1038  //
  1039  // keep is used to check whether a sum should be retained in go.mod. It should
  1040  // have entries for both module content sums and go.mod sums (version ends
  1041  // with "/go.mod").
  1042  func (f *Fetcher) TrimGoSum(keep map[module.Version]bool) {
  1043  	f.mu.Lock()
  1044  	defer f.mu.Unlock()
  1045  	inited, err := f.initGoSum()
  1046  	if err != nil {
  1047  		base.Fatalf("%s", err)
  1048  	}
  1049  	if !inited {
  1050  		return
  1051  	}
  1052  
  1053  	for m, hs := range f.sumState.m {
  1054  		if !keep[m] {
  1055  			for _, h := range hs {
  1056  				f.sumState.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
  1057  			}
  1058  			f.sumState.overwrite = true
  1059  		}
  1060  	}
  1061  }
  1062  
  1063  const goSumMismatch = `
  1064  
  1065  SECURITY ERROR
  1066  This download does NOT match an earlier download recorded in go.sum.
  1067  The bits may have been replaced on the origin server, or an attacker may
  1068  have intercepted the download attempt.
  1069  
  1070  For more information, see 'go help module-auth'.
  1071  `
  1072  
  1073  const sumdbMismatch = `
  1074  
  1075  SECURITY ERROR
  1076  This download does NOT match the one reported by the checksum server.
  1077  The bits may have been replaced on the origin server, or an attacker may
  1078  have intercepted the download attempt.
  1079  
  1080  For more information, see 'go help module-auth'.
  1081  `
  1082  
  1083  const hashVersionMismatch = `
  1084  
  1085  SECURITY WARNING
  1086  This download is listed in go.sum, but using an unknown hash algorithm.
  1087  The download cannot be verified.
  1088  
  1089  For more information, see 'go help module-auth'.
  1090  
  1091  `
  1092  
  1093  var HelpModuleAuth = &base.Command{
  1094  	UsageLine: "module-auth",
  1095  	Short:     "module authentication using go.sum",
  1096  	Long: `
  1097  When the go command downloads a module zip file or go.mod file into the
  1098  module cache, it computes a cryptographic hash and compares it with a known
  1099  value to verify the file hasn't changed since it was first downloaded. Known
  1100  hashes are stored in a file in the module root directory named go.sum. Hashes
  1101  may also be downloaded from the checksum database depending on the values of
  1102  GOSUMDB, GOPRIVATE, and GONOSUMDB.
  1103  
  1104  For details, see https://golang.org/ref/mod#authenticating.
  1105  `,
  1106  }
  1107  
  1108  var HelpPrivate = &base.Command{
  1109  	UsageLine: "private",
  1110  	Short:     "configuration for downloading non-public code",
  1111  	Long: `
  1112  The go command defaults to downloading modules from the public Go module
  1113  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1114  regardless of source, against the public Go checksum database at sum.golang.org.
  1115  These defaults work well for publicly available source code.
  1116  
  1117  The GOPRIVATE environment variable controls which modules the go command
  1118  considers to be private (not available publicly) and should therefore not use
  1119  the proxy or checksum database. The variable is a comma-separated list of
  1120  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1121  For example,
  1122  
  1123  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1124  
  1125  causes the go command to treat as private any module with a path prefix
  1126  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1127  and rsc.io/private/quux.
  1128  
  1129  For fine-grained control over module download and validation, the GONOPROXY
  1130  and GONOSUMDB environment variables accept the same kind of glob list
  1131  and override GOPRIVATE for the specific decision of whether to use the proxy
  1132  and checksum database, respectively.
  1133  
  1134  For example, if a company ran a module proxy serving private modules,
  1135  users would configure go using:
  1136  
  1137  	GOPRIVATE=*.corp.example.com
  1138  	GOPROXY=proxy.example.com
  1139  	GONOPROXY=none
  1140  
  1141  The GOPRIVATE variable is also used to define the "public" and "private"
  1142  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1143  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1144  instead of module paths.
  1145  
  1146  The 'go env -w' command (see 'go help env') can be used to set these variables
  1147  for future go command invocations.
  1148  
  1149  For more details, see https://golang.org/ref/mod#private-modules.
  1150  `,
  1151  }
  1152
View as plain text