crash_test.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.runtimesecret && linux
     6  
     7  package secret
     8  
     9  import (
    10  	"bytes"
    11  	"debug/elf"
    12  	"fmt"
    13  	"internal/testenv"
    14  	"io"
    15  	"os"
    16  	"os/exec"
    17  	"path/filepath"
    18  	"runtime"
    19  	"strings"
    20  	"syscall"
    21  	"testing"
    22  )
    23  
    24  // Copied from runtime/runtime-gdb_unix_test.go
    25  func canGenerateCore(t *testing.T) bool {
    26  	// Ensure there is enough RLIMIT_CORE available to generate a full core.
    27  	var lim syscall.Rlimit
    28  	err := syscall.Getrlimit(syscall.RLIMIT_CORE, &lim)
    29  	if err != nil {
    30  		t.Fatalf("error getting rlimit: %v", err)
    31  	}
    32  	// Minimum RLIMIT_CORE max to allow. This is a conservative estimate.
    33  	// Most systems allow infinity.
    34  	const minRlimitCore = 100 << 20 // 100 MB
    35  	if lim.Max < minRlimitCore {
    36  		t.Skipf("RLIMIT_CORE max too low: %#+v", lim)
    37  	}
    38  
    39  	// Make sure core pattern will send core to the current directory.
    40  	b, err := os.ReadFile("/proc/sys/kernel/core_pattern")
    41  	if err != nil {
    42  		t.Fatalf("error reading core_pattern: %v", err)
    43  	}
    44  	if string(b) != "core\n" {
    45  		t.Skipf("Unexpected core pattern %q", string(b))
    46  	}
    47  
    48  	coreUsesPID := false
    49  	b, err = os.ReadFile("/proc/sys/kernel/core_uses_pid")
    50  	if err == nil {
    51  		switch string(bytes.TrimSpace(b)) {
    52  		case "0":
    53  		case "1":
    54  			coreUsesPID = true
    55  		default:
    56  			t.Skipf("unexpected core_uses_pid value %q", string(b))
    57  		}
    58  	}
    59  	return coreUsesPID
    60  }
    61  
    62  func TestCore(t *testing.T) {
    63  	// use secret, grab a coredump, rummage through
    64  	// it, trying to find our secret.
    65  
    66  	switch runtime.GOARCH {
    67  	case "amd64", "arm64":
    68  	default:
    69  		t.Skip("unsupported arch")
    70  	}
    71  	coreUsesPid := canGenerateCore(t)
    72  
    73  	// Build our crashing program
    74  	// Because we need assembly files to properly dirty our state
    75  	// we need to construct a package in our temporary directory.
    76  	tmpDir := t.TempDir()
    77  	// copy our base source
    78  	err := copyToDir("./testdata/crash.go", tmpDir, nil)
    79  	if err != nil {
    80  		t.Fatalf("error copying directory %v", err)
    81  	}
    82  	// Copy our testing assembly files. Use the ones from the package
    83  	// to assure that they are always in sync
    84  	err = copyToDir("./asm_amd64.s", tmpDir, nil)
    85  	if err != nil {
    86  		t.Fatalf("error copying file %v", err)
    87  	}
    88  	err = copyToDir("./asm_arm64.s", tmpDir, nil)
    89  	if err != nil {
    90  		t.Fatalf("error copying file %v", err)
    91  	}
    92  	err = copyToDir("./stubs.go", tmpDir, func(s string) string {
    93  		return strings.Replace(s, "package secret", "package main", 1)
    94  	})
    95  	if err != nil {
    96  		t.Fatalf("error copying file %v", err)
    97  	}
    98  
    99  	// the crashing package will live out of tree, so its source files
   100  	// cannot refer to our internal packages. However, the assembly files
   101  	// can refer to internal names and we can pass the missing offsets as
   102  	// a small generated file
   103  	offsets := `
   104  	package main
   105  	const (
   106  		offsetX86HasAVX    = %v
   107  		offsetX86HasAVX512 = %v
   108  	)
   109  	`
   110  	err = os.WriteFile(filepath.Join(tmpDir, "offsets.go"), []byte(fmt.Sprintf(offsets, offsetX86HasAVX, offsetX86HasAVX512)), 0666)
   111  	if err != nil {
   112  		t.Fatalf("error writing offset file %v", err)
   113  	}
   114  
   115  	// generate go.mod file
   116  	cmd := exec.Command(testenv.GoToolPath(t), "mod", "init", "crashtest")
   117  	cmd.Dir = tmpDir
   118  	out, err := testenv.CleanCmdEnv(cmd).CombinedOutput()
   119  	if err != nil {
   120  		t.Fatalf("error initing module %v\n%s", err, out)
   121  	}
   122  
   123  	cmd = exec.Command(testenv.GoToolPath(t), "build", "-o", filepath.Join(tmpDir, "a.exe"))
   124  	cmd.Dir = tmpDir
   125  	out, err = testenv.CleanCmdEnv(cmd).CombinedOutput()
   126  	if err != nil {
   127  		t.Fatalf("error building source %v\n%s", err, out)
   128  	}
   129  
   130  	// Start the test binary.
   131  	cmd = testenv.CommandContext(t, t.Context(), "./a.exe")
   132  	cmd.Dir = tmpDir
   133  	var stdout strings.Builder
   134  	cmd.Stdout = &stdout
   135  	cmd.Stderr = &stdout
   136  
   137  	err = cmd.Run()
   138  	// For debugging.
   139  	t.Logf("\n\n\n--- START SUBPROCESS ---\n\n\n%s\n\n--- END SUBPROCESS ---\n\n\n", stdout.String())
   140  	if err == nil {
   141  		t.Fatalf("test binary did not crash")
   142  	}
   143  	eErr, ok := err.(*exec.ExitError)
   144  	if !ok {
   145  		t.Fatalf("error is not exit error: %v", err)
   146  	}
   147  	if eErr.Exited() {
   148  		t.Fatalf("process exited instead of being terminated: %v", eErr)
   149  	}
   150  
   151  	rummage(t, tmpDir, eErr.Pid(), coreUsesPid)
   152  }
   153  
   154  func copyToDir(name string, dir string, replace func(string) string) error {
   155  	f, err := os.ReadFile(name)
   156  	if err != nil {
   157  		return err
   158  	}
   159  	if replace != nil {
   160  		f = []byte(replace(string(f)))
   161  	}
   162  	return os.WriteFile(filepath.Join(dir, filepath.Base(name)), f, 0666)
   163  }
   164  
   165  type violation struct {
   166  	id  byte   // secret ID
   167  	off uint64 // offset in core dump
   168  }
   169  
   170  // A secret value that should never appear in a core dump,
   171  // except for this global variable itself.
   172  // The first byte of the secret is variable, to track
   173  // different instances of it.
   174  //
   175  // If this value is changed, update ./internal/crashsecret/main.go
   176  // TODO: this is little-endian specific.
   177  var secretStore = [8]byte{
   178  	0x00,
   179  	0x81,
   180  	0xa0,
   181  	0xc6,
   182  	0xb3,
   183  	0x01,
   184  	0x66,
   185  	0x53,
   186  }
   187  
   188  func rummage(t *testing.T, tmpDir string, pid int, coreUsesPid bool) {
   189  	coreFileName := "core"
   190  	if coreUsesPid {
   191  		coreFileName += fmt.Sprintf(".%d", pid)
   192  	}
   193  	core, err := os.Open(filepath.Join(tmpDir, coreFileName))
   194  	if err != nil {
   195  		t.Fatalf("core file not found: %v", err)
   196  	}
   197  	b, err := io.ReadAll(core)
   198  	if err != nil {
   199  		t.Fatalf("can't read core file: %v", err)
   200  	}
   201  
   202  	// Open elf view onto core file.
   203  	coreElf, err := elf.NewFile(core)
   204  	if err != nil {
   205  		t.Fatalf("can't parse core file: %v", err)
   206  	}
   207  
   208  	// Look for any places that have the secret.
   209  	var violations []violation // core file offsets where we found a secret
   210  	i := 0
   211  	for {
   212  		j := bytes.Index(b[i:], secretStore[1:])
   213  		if j < 0 {
   214  			break
   215  		}
   216  		j--
   217  		i += j
   218  
   219  		t.Errorf("secret %d found at offset %x in core file", b[i], i)
   220  		violations = append(violations, violation{
   221  			id:  b[i],
   222  			off: uint64(i),
   223  		})
   224  
   225  		i += len(secretStore)
   226  	}
   227  
   228  	// Get more specific data about where in the core we found the secrets.
   229  	regions := elfRegions(t, core, coreElf)
   230  	for _, r := range regions {
   231  		for _, v := range violations {
   232  			if v.off >= r.min && v.off < r.max {
   233  				var addr string
   234  				if r.addrMin != 0 {
   235  					addr = fmt.Sprintf(" addr=%x", r.addrMin+(v.off-r.min))
   236  				}
   237  				t.Logf("additional info: secret %d at offset %x in %s%s", v.id, v.off-r.min, r.name, addr)
   238  			}
   239  		}
   240  	}
   241  }
   242  
   243  type elfRegion struct {
   244  	name             string
   245  	min, max         uint64 // core file offset range
   246  	addrMin, addrMax uint64 // inferior address range (or 0,0 if no address, like registers)
   247  }
   248  
   249  func elfRegions(t *testing.T, core *os.File, coreElf *elf.File) []elfRegion {
   250  	var regions []elfRegion
   251  	for _, p := range coreElf.Progs {
   252  		regions = append(regions, elfRegion{
   253  			name:    fmt.Sprintf("%s[%s]", p.Type, p.Flags),
   254  			min:     p.Off,
   255  			max:     p.Off + min(p.Filesz, p.Memsz),
   256  			addrMin: p.Vaddr,
   257  			addrMax: p.Vaddr + min(p.Filesz, p.Memsz),
   258  		})
   259  	}
   260  
   261  	// TODO(dmo): parse thread regions for arm64.
   262  	// This doesn't invalidate the test, it just makes it harder to figure
   263  	// out where we're leaking stuff.
   264  	if runtime.GOARCH == "amd64" {
   265  		regions = append(regions, threadRegions(t, core, coreElf)...)
   266  	}
   267  
   268  	for i, r1 := range regions {
   269  		for j, r2 := range regions {
   270  			if i == j {
   271  				continue
   272  			}
   273  			if r1.max <= r2.min || r2.max <= r1.min {
   274  				continue
   275  			}
   276  			t.Fatalf("overlapping regions %v %v", r1, r2)
   277  		}
   278  	}
   279  
   280  	return regions
   281  }
   282  
   283  func threadRegions(t *testing.T, core *os.File, coreElf *elf.File) []elfRegion {
   284  	var regions []elfRegion
   285  
   286  	for _, prog := range coreElf.Progs {
   287  		if prog.Type != elf.PT_NOTE {
   288  			continue
   289  		}
   290  
   291  		b := make([]byte, prog.Filesz)
   292  		_, err := core.ReadAt(b, int64(prog.Off))
   293  		if err != nil {
   294  			t.Fatalf("can't read core file %v", err)
   295  		}
   296  		prefix := "unk"
   297  		b0 := b
   298  		for len(b) > 0 {
   299  			namesz := coreElf.ByteOrder.Uint32(b)
   300  			b = b[4:]
   301  			descsz := coreElf.ByteOrder.Uint32(b)
   302  			b = b[4:]
   303  			typ := elf.NType(coreElf.ByteOrder.Uint32(b))
   304  			b = b[4:]
   305  			name := string(b[:namesz-1])
   306  			b = b[(namesz+3)/4*4:]
   307  			off := prog.Off + uint64(len(b0)-len(b))
   308  			desc := b[:descsz]
   309  			b = b[(descsz+3)/4*4:]
   310  
   311  			if name != "CORE" && name != "LINUX" {
   312  				continue
   313  			}
   314  			end := off + uint64(len(desc))
   315  			// Note: amd64 specific
   316  			// See /usr/include/x86_64-linux-gnu/bits/sigcontext.h
   317  			//
   318  			//   struct _fpstate
   319  			switch typ {
   320  			case elf.NT_PRSTATUS:
   321  				pid := coreElf.ByteOrder.Uint32(desc[32:36])
   322  				prefix = fmt.Sprintf("thread%d: ", pid)
   323  				regions = append(regions, elfRegion{
   324  					name: prefix + "prstatus header",
   325  					min:  off,
   326  					max:  off + 112,
   327  				})
   328  				off += 112
   329  				greg := []string{
   330  					"r15",
   331  					"r14",
   332  					"r13",
   333  					"r12",
   334  					"rbp",
   335  					"rbx",
   336  					"r11",
   337  					"r10",
   338  					"r9",
   339  					"r8",
   340  					"rax",
   341  					"rcx",
   342  					"rdx",
   343  					"rsi",
   344  					"rdi",
   345  					"orig_rax",
   346  					"rip",
   347  					"cs",
   348  					"eflags",
   349  					"rsp",
   350  					"ss",
   351  					"fs_base",
   352  					"gs_base",
   353  					"ds",
   354  					"es",
   355  					"fs",
   356  					"gs",
   357  				}
   358  				for _, r := range greg {
   359  					regions = append(regions, elfRegion{
   360  						name: prefix + r,
   361  						min:  off,
   362  						max:  off + 8,
   363  					})
   364  					off += 8
   365  				}
   366  				regions = append(regions, elfRegion{
   367  					name: prefix + "prstatus footer",
   368  					min:  off,
   369  					max:  off + 8,
   370  				})
   371  				off += 8
   372  			case elf.NT_FPREGSET:
   373  				regions = append(regions, elfRegion{
   374  					name: prefix + "fpregset header",
   375  					min:  off,
   376  					max:  off + 32,
   377  				})
   378  				off += 32
   379  				for i := 0; i < 8; i++ {
   380  					regions = append(regions, elfRegion{
   381  						name: prefix + fmt.Sprintf("mmx%d", i),
   382  						min:  off,
   383  						max:  off + 16,
   384  					})
   385  					off += 16
   386  					// They are long double (10 bytes), but
   387  					// stored in 16-byte slots.
   388  				}
   389  				for i := 0; i < 16; i++ {
   390  					regions = append(regions, elfRegion{
   391  						name: prefix + fmt.Sprintf("xmm%d", i),
   392  						min:  off,
   393  						max:  off + 16,
   394  					})
   395  					off += 16
   396  				}
   397  				regions = append(regions, elfRegion{
   398  					name: prefix + "fpregset footer",
   399  					min:  off,
   400  					max:  off + 96,
   401  				})
   402  				off += 96
   403  				/*
   404  					case NT_X86_XSTATE: // aka NT_PRPSINFO+511
   405  						// legacy: 512 bytes
   406  						// xsave header: 64 bytes
   407  						fmt.Printf("hdr %v\n", desc[512:][:64])
   408  						// ymm high128: 256 bytes
   409  
   410  						println(len(desc))
   411  						fallthrough
   412  				*/
   413  			default:
   414  				regions = append(regions, elfRegion{
   415  					name: fmt.Sprintf("%s/%s", name, typ),
   416  					min:  off,
   417  					max:  off + uint64(len(desc)),
   418  				})
   419  				off += uint64(len(desc))
   420  			}
   421  			if off != end {
   422  				t.Fatalf("note section incomplete")
   423  			}
   424  		}
   425  	}
   426  	return regions
   427  }
   428
View as plain text