Source file src/runtime/metrics.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  // Metrics implementation exported to runtime/metrics.
     8  
     9  import (
    10  	"internal/godebugs"
    11  	"internal/runtime/atomic"
    12  	"internal/runtime/gc"
    13  	"unsafe"
    14  )
    15  
    16  var (
    17  	// metrics is a map of runtime/metrics keys to data used by the runtime
    18  	// to sample each metric's value. metricsInit indicates it has been
    19  	// initialized.
    20  	//
    21  	// These fields are protected by metricsSema which should be
    22  	// locked/unlocked with metricsLock() / metricsUnlock().
    23  	metricsSema uint32 = 1
    24  	metricsInit bool
    25  	metrics     map[string]metricData
    26  
    27  	sizeClassBuckets []float64
    28  	timeHistBuckets  []float64
    29  )
    30  
    31  type metricData struct {
    32  	// deps is the set of runtime statistics that this metric
    33  	// depends on. Before compute is called, the statAggregate
    34  	// which will be passed must ensure() these dependencies.
    35  	deps statDepSet
    36  
    37  	// compute is a function that populates a metricValue
    38  	// given a populated statAggregate structure.
    39  	compute func(in *statAggregate, out *metricValue)
    40  }
    41  
    42  func metricsLock() {
    43  	// Acquire the metricsSema but with handoff. Operations are typically
    44  	// expensive enough that queueing up goroutines and handing off between
    45  	// them will be noticeably better-behaved.
    46  	semacquire1(&metricsSema, true, 0, 0, waitReasonSemacquire)
    47  	if raceenabled {
    48  		raceacquire(unsafe.Pointer(&metricsSema))
    49  	}
    50  }
    51  
    52  func metricsUnlock() {
    53  	if raceenabled {
    54  		racerelease(unsafe.Pointer(&metricsSema))
    55  	}
    56  	semrelease(&metricsSema)
    57  }
    58  
    59  // initMetrics initializes the metrics map if it hasn't been yet.
    60  //
    61  // metricsSema must be held.
    62  func initMetrics() {
    63  	if metricsInit {
    64  		return
    65  	}
    66  
    67  	sizeClassBuckets = make([]float64, gc.NumSizeClasses, gc.NumSizeClasses+1)
    68  	// Skip size class 0 which is a stand-in for large objects, but large
    69  	// objects are tracked separately (and they actually get placed in
    70  	// the last bucket, not the first).
    71  	sizeClassBuckets[0] = 1 // The smallest allocation is 1 byte in size.
    72  	for i := 1; i < gc.NumSizeClasses; i++ {
    73  		// Size classes have an inclusive upper-bound
    74  		// and exclusive lower bound (e.g. 48-byte size class is
    75  		// (32, 48]) whereas we want and inclusive lower-bound
    76  		// and exclusive upper-bound (e.g. 48-byte size class is
    77  		// [33, 49)). We can achieve this by shifting all bucket
    78  		// boundaries up by 1.
    79  		//
    80  		// Also, a float64 can precisely represent integers with
    81  		// value up to 2^53 and size classes are relatively small
    82  		// (nowhere near 2^48 even) so this will give us exact
    83  		// boundaries.
    84  		sizeClassBuckets[i] = float64(gc.SizeClassToSize[i] + 1)
    85  	}
    86  	sizeClassBuckets = append(sizeClassBuckets, float64Inf())
    87  
    88  	timeHistBuckets = timeHistogramMetricsBuckets()
    89  	metrics = map[string]metricData{
    90  		"/cgo/go-to-c-calls:calls": {
    91  			compute: func(_ *statAggregate, out *metricValue) {
    92  				out.kind = metricKindUint64
    93  				out.scalar = uint64(NumCgoCall())
    94  			},
    95  		},
    96  		"/cpu/classes/gc/mark/assist:cpu-seconds": {
    97  			deps: makeStatDepSet(cpuStatsDep),
    98  			compute: func(in *statAggregate, out *metricValue) {
    99  				out.kind = metricKindFloat64
   100  				out.scalar = float64bits(nsToSec(in.cpuStats.GCAssistTime))
   101  			},
   102  		},
   103  		"/cpu/classes/gc/mark/dedicated:cpu-seconds": {
   104  			deps: makeStatDepSet(cpuStatsDep),
   105  			compute: func(in *statAggregate, out *metricValue) {
   106  				out.kind = metricKindFloat64
   107  				out.scalar = float64bits(nsToSec(in.cpuStats.GCDedicatedTime))
   108  			},
   109  		},
   110  		"/cpu/classes/gc/mark/idle:cpu-seconds": {
   111  			deps: makeStatDepSet(cpuStatsDep),
   112  			compute: func(in *statAggregate, out *metricValue) {
   113  				out.kind = metricKindFloat64
   114  				out.scalar = float64bits(nsToSec(in.cpuStats.GCIdleTime))
   115  			},
   116  		},
   117  		"/cpu/classes/gc/pause:cpu-seconds": {
   118  			deps: makeStatDepSet(cpuStatsDep),
   119  			compute: func(in *statAggregate, out *metricValue) {
   120  				out.kind = metricKindFloat64
   121  				out.scalar = float64bits(nsToSec(in.cpuStats.GCPauseTime))
   122  			},
   123  		},
   124  		"/cpu/classes/gc/total:cpu-seconds": {
   125  			deps: makeStatDepSet(cpuStatsDep),
   126  			compute: func(in *statAggregate, out *metricValue) {
   127  				out.kind = metricKindFloat64
   128  				out.scalar = float64bits(nsToSec(in.cpuStats.GCTotalTime))
   129  			},
   130  		},
   131  		"/cpu/classes/idle:cpu-seconds": {
   132  			deps: makeStatDepSet(cpuStatsDep),
   133  			compute: func(in *statAggregate, out *metricValue) {
   134  				out.kind = metricKindFloat64
   135  				out.scalar = float64bits(nsToSec(in.cpuStats.IdleTime))
   136  			},
   137  		},
   138  		"/cpu/classes/scavenge/assist:cpu-seconds": {
   139  			deps: makeStatDepSet(cpuStatsDep),
   140  			compute: func(in *statAggregate, out *metricValue) {
   141  				out.kind = metricKindFloat64
   142  				out.scalar = float64bits(nsToSec(in.cpuStats.ScavengeAssistTime))
   143  			},
   144  		},
   145  		"/cpu/classes/scavenge/background:cpu-seconds": {
   146  			deps: makeStatDepSet(cpuStatsDep),
   147  			compute: func(in *statAggregate, out *metricValue) {
   148  				out.kind = metricKindFloat64
   149  				out.scalar = float64bits(nsToSec(in.cpuStats.ScavengeBgTime))
   150  			},
   151  		},
   152  		"/cpu/classes/scavenge/total:cpu-seconds": {
   153  			deps: makeStatDepSet(cpuStatsDep),
   154  			compute: func(in *statAggregate, out *metricValue) {
   155  				out.kind = metricKindFloat64
   156  				out.scalar = float64bits(nsToSec(in.cpuStats.ScavengeTotalTime))
   157  			},
   158  		},
   159  		"/cpu/classes/total:cpu-seconds": {
   160  			deps: makeStatDepSet(cpuStatsDep),
   161  			compute: func(in *statAggregate, out *metricValue) {
   162  				out.kind = metricKindFloat64
   163  				out.scalar = float64bits(nsToSec(in.cpuStats.TotalTime))
   164  			},
   165  		},
   166  		"/cpu/classes/user:cpu-seconds": {
   167  			deps: makeStatDepSet(cpuStatsDep),
   168  			compute: func(in *statAggregate, out *metricValue) {
   169  				out.kind = metricKindFloat64
   170  				out.scalar = float64bits(nsToSec(in.cpuStats.UserTime))
   171  			},
   172  		},
   173  		"/gc/cleanups/executed:cleanups": {
   174  			deps: makeStatDepSet(finalStatsDep),
   175  			compute: func(in *statAggregate, out *metricValue) {
   176  				out.kind = metricKindUint64
   177  				out.scalar = in.finalStats.cleanupsExecuted
   178  			},
   179  		},
   180  		"/gc/cleanups/queued:cleanups": {
   181  			deps: makeStatDepSet(finalStatsDep),
   182  			compute: func(in *statAggregate, out *metricValue) {
   183  				out.kind = metricKindUint64
   184  				out.scalar = in.finalStats.cleanupsQueued
   185  			},
   186  		},
   187  		"/gc/cycles/automatic:gc-cycles": {
   188  			deps: makeStatDepSet(sysStatsDep),
   189  			compute: func(in *statAggregate, out *metricValue) {
   190  				out.kind = metricKindUint64
   191  				out.scalar = in.sysStats.gcCyclesDone - in.sysStats.gcCyclesForced
   192  			},
   193  		},
   194  		"/gc/cycles/forced:gc-cycles": {
   195  			deps: makeStatDepSet(sysStatsDep),
   196  			compute: func(in *statAggregate, out *metricValue) {
   197  				out.kind = metricKindUint64
   198  				out.scalar = in.sysStats.gcCyclesForced
   199  			},
   200  		},
   201  		"/gc/cycles/total:gc-cycles": {
   202  			deps: makeStatDepSet(sysStatsDep),
   203  			compute: func(in *statAggregate, out *metricValue) {
   204  				out.kind = metricKindUint64
   205  				out.scalar = in.sysStats.gcCyclesDone
   206  			},
   207  		},
   208  		"/gc/finalizers/executed:finalizers": {
   209  			deps: makeStatDepSet(finalStatsDep),
   210  			compute: func(in *statAggregate, out *metricValue) {
   211  				out.kind = metricKindUint64
   212  				out.scalar = in.finalStats.finalizersExecuted
   213  			},
   214  		},
   215  		"/gc/finalizers/queued:finalizers": {
   216  			deps: makeStatDepSet(finalStatsDep),
   217  			compute: func(in *statAggregate, out *metricValue) {
   218  				out.kind = metricKindUint64
   219  				out.scalar = in.finalStats.finalizersQueued
   220  			},
   221  		},
   222  		"/gc/scan/globals:bytes": {
   223  			deps: makeStatDepSet(gcStatsDep),
   224  			compute: func(in *statAggregate, out *metricValue) {
   225  				out.kind = metricKindUint64
   226  				out.scalar = in.gcStats.globalsScan
   227  			},
   228  		},
   229  		"/gc/scan/heap:bytes": {
   230  			deps: makeStatDepSet(gcStatsDep),
   231  			compute: func(in *statAggregate, out *metricValue) {
   232  				out.kind = metricKindUint64
   233  				out.scalar = in.gcStats.heapScan
   234  			},
   235  		},
   236  		"/gc/scan/stack:bytes": {
   237  			deps: makeStatDepSet(gcStatsDep),
   238  			compute: func(in *statAggregate, out *metricValue) {
   239  				out.kind = metricKindUint64
   240  				out.scalar = in.gcStats.stackScan
   241  			},
   242  		},
   243  		"/gc/scan/total:bytes": {
   244  			deps: makeStatDepSet(gcStatsDep),
   245  			compute: func(in *statAggregate, out *metricValue) {
   246  				out.kind = metricKindUint64
   247  				out.scalar = in.gcStats.totalScan
   248  			},
   249  		},
   250  		"/gc/heap/allocs-by-size:bytes": {
   251  			deps: makeStatDepSet(heapStatsDep),
   252  			compute: func(in *statAggregate, out *metricValue) {
   253  				hist := out.float64HistOrInit(sizeClassBuckets)
   254  				hist.counts[len(hist.counts)-1] = in.heapStats.largeAllocCount
   255  				// Cut off the first index which is ostensibly for size class 0,
   256  				// but large objects are tracked separately so it's actually unused.
   257  				for i, count := range in.heapStats.smallAllocCount[1:] {
   258  					hist.counts[i] = count
   259  				}
   260  			},
   261  		},
   262  		"/gc/heap/allocs:bytes": {
   263  			deps: makeStatDepSet(heapStatsDep),
   264  			compute: func(in *statAggregate, out *metricValue) {
   265  				out.kind = metricKindUint64
   266  				out.scalar = in.heapStats.totalAllocated
   267  			},
   268  		},
   269  		"/gc/heap/allocs:objects": {
   270  			deps: makeStatDepSet(heapStatsDep),
   271  			compute: func(in *statAggregate, out *metricValue) {
   272  				out.kind = metricKindUint64
   273  				out.scalar = in.heapStats.totalAllocs
   274  			},
   275  		},
   276  		"/gc/heap/frees-by-size:bytes": {
   277  			deps: makeStatDepSet(heapStatsDep),
   278  			compute: func(in *statAggregate, out *metricValue) {
   279  				hist := out.float64HistOrInit(sizeClassBuckets)
   280  				hist.counts[len(hist.counts)-1] = in.heapStats.largeFreeCount
   281  				// Cut off the first index which is ostensibly for size class 0,
   282  				// but large objects are tracked separately so it's actually unused.
   283  				for i, count := range in.heapStats.smallFreeCount[1:] {
   284  					hist.counts[i] = count
   285  				}
   286  			},
   287  		},
   288  		"/gc/heap/frees:bytes": {
   289  			deps: makeStatDepSet(heapStatsDep),
   290  			compute: func(in *statAggregate, out *metricValue) {
   291  				out.kind = metricKindUint64
   292  				out.scalar = in.heapStats.totalFreed
   293  			},
   294  		},
   295  		"/gc/heap/frees:objects": {
   296  			deps: makeStatDepSet(heapStatsDep),
   297  			compute: func(in *statAggregate, out *metricValue) {
   298  				out.kind = metricKindUint64
   299  				out.scalar = in.heapStats.totalFrees
   300  			},
   301  		},
   302  		"/gc/heap/goal:bytes": {
   303  			deps: makeStatDepSet(sysStatsDep),
   304  			compute: func(in *statAggregate, out *metricValue) {
   305  				out.kind = metricKindUint64
   306  				out.scalar = in.sysStats.heapGoal
   307  			},
   308  		},
   309  		"/gc/gomemlimit:bytes": {
   310  			compute: func(in *statAggregate, out *metricValue) {
   311  				out.kind = metricKindUint64
   312  				out.scalar = uint64(gcController.memoryLimit.Load())
   313  			},
   314  		},
   315  		"/gc/gogc:percent": {
   316  			compute: func(in *statAggregate, out *metricValue) {
   317  				out.kind = metricKindUint64
   318  				out.scalar = uint64(gcController.gcPercent.Load())
   319  			},
   320  		},
   321  		"/gc/heap/live:bytes": {
   322  			deps: makeStatDepSet(heapStatsDep),
   323  			compute: func(in *statAggregate, out *metricValue) {
   324  				out.kind = metricKindUint64
   325  				out.scalar = gcController.heapMarked
   326  			},
   327  		},
   328  		"/gc/heap/objects:objects": {
   329  			deps: makeStatDepSet(heapStatsDep),
   330  			compute: func(in *statAggregate, out *metricValue) {
   331  				out.kind = metricKindUint64
   332  				out.scalar = in.heapStats.numObjects
   333  			},
   334  		},
   335  		"/gc/heap/tiny/allocs:objects": {
   336  			deps: makeStatDepSet(heapStatsDep),
   337  			compute: func(in *statAggregate, out *metricValue) {
   338  				out.kind = metricKindUint64
   339  				out.scalar = in.heapStats.tinyAllocCount
   340  			},
   341  		},
   342  		"/gc/limiter/last-enabled:gc-cycle": {
   343  			compute: func(_ *statAggregate, out *metricValue) {
   344  				out.kind = metricKindUint64
   345  				out.scalar = uint64(gcCPULimiter.lastEnabledCycle.Load())
   346  			},
   347  		},
   348  		"/gc/pauses:seconds": {
   349  			compute: func(_ *statAggregate, out *metricValue) {
   350  				// N.B. this is identical to /sched/pauses/total/gc:seconds.
   351  				sched.stwTotalTimeGC.write(out)
   352  			},
   353  		},
   354  		"/gc/stack/starting-size:bytes": {
   355  			compute: func(in *statAggregate, out *metricValue) {
   356  				out.kind = metricKindUint64
   357  				out.scalar = uint64(startingStackSize)
   358  			},
   359  		},
   360  		"/memory/classes/heap/free:bytes": {
   361  			deps: makeStatDepSet(heapStatsDep),
   362  			compute: func(in *statAggregate, out *metricValue) {
   363  				out.kind = metricKindUint64
   364  				out.scalar = uint64(in.heapStats.committed - in.heapStats.inHeap -
   365  					in.heapStats.inStacks - in.heapStats.inWorkBufs)
   366  			},
   367  		},
   368  		"/memory/classes/heap/objects:bytes": {
   369  			deps: makeStatDepSet(heapStatsDep),
   370  			compute: func(in *statAggregate, out *metricValue) {
   371  				out.kind = metricKindUint64
   372  				out.scalar = in.heapStats.inObjects
   373  			},
   374  		},
   375  		"/memory/classes/heap/released:bytes": {
   376  			deps: makeStatDepSet(heapStatsDep),
   377  			compute: func(in *statAggregate, out *metricValue) {
   378  				out.kind = metricKindUint64
   379  				out.scalar = uint64(in.heapStats.released)
   380  			},
   381  		},
   382  		"/memory/classes/heap/stacks:bytes": {
   383  			deps: makeStatDepSet(heapStatsDep),
   384  			compute: func(in *statAggregate, out *metricValue) {
   385  				out.kind = metricKindUint64
   386  				out.scalar = uint64(in.heapStats.inStacks)
   387  			},
   388  		},
   389  		"/memory/classes/heap/unused:bytes": {
   390  			deps: makeStatDepSet(heapStatsDep),
   391  			compute: func(in *statAggregate, out *metricValue) {
   392  				out.kind = metricKindUint64
   393  				out.scalar = uint64(in.heapStats.inHeap) - in.heapStats.inObjects
   394  			},
   395  		},
   396  		"/memory/classes/metadata/mcache/free:bytes": {
   397  			deps: makeStatDepSet(sysStatsDep),
   398  			compute: func(in *statAggregate, out *metricValue) {
   399  				out.kind = metricKindUint64
   400  				out.scalar = in.sysStats.mCacheSys - in.sysStats.mCacheInUse
   401  			},
   402  		},
   403  		"/memory/classes/metadata/mcache/inuse:bytes": {
   404  			deps: makeStatDepSet(sysStatsDep),
   405  			compute: func(in *statAggregate, out *metricValue) {
   406  				out.kind = metricKindUint64
   407  				out.scalar = in.sysStats.mCacheInUse
   408  			},
   409  		},
   410  		"/memory/classes/metadata/mspan/free:bytes": {
   411  			deps: makeStatDepSet(sysStatsDep),
   412  			compute: func(in *statAggregate, out *metricValue) {
   413  				out.kind = metricKindUint64
   414  				out.scalar = in.sysStats.mSpanSys - in.sysStats.mSpanInUse
   415  			},
   416  		},
   417  		"/memory/classes/metadata/mspan/inuse:bytes": {
   418  			deps: makeStatDepSet(sysStatsDep),
   419  			compute: func(in *statAggregate, out *metricValue) {
   420  				out.kind = metricKindUint64
   421  				out.scalar = in.sysStats.mSpanInUse
   422  			},
   423  		},
   424  		"/memory/classes/metadata/other:bytes": {
   425  			deps: makeStatDepSet(heapStatsDep, sysStatsDep),
   426  			compute: func(in *statAggregate, out *metricValue) {
   427  				out.kind = metricKindUint64
   428  				out.scalar = uint64(in.heapStats.inWorkBufs) + in.sysStats.gcMiscSys
   429  			},
   430  		},
   431  		"/memory/classes/os-stacks:bytes": {
   432  			deps: makeStatDepSet(sysStatsDep),
   433  			compute: func(in *statAggregate, out *metricValue) {
   434  				out.kind = metricKindUint64
   435  				out.scalar = in.sysStats.stacksSys
   436  			},
   437  		},
   438  		"/memory/classes/other:bytes": {
   439  			deps: makeStatDepSet(sysStatsDep),
   440  			compute: func(in *statAggregate, out *metricValue) {
   441  				out.kind = metricKindUint64
   442  				out.scalar = in.sysStats.otherSys
   443  			},
   444  		},
   445  		"/memory/classes/profiling/buckets:bytes": {
   446  			deps: makeStatDepSet(sysStatsDep),
   447  			compute: func(in *statAggregate, out *metricValue) {
   448  				out.kind = metricKindUint64
   449  				out.scalar = in.sysStats.buckHashSys
   450  			},
   451  		},
   452  		"/memory/classes/total:bytes": {
   453  			deps: makeStatDepSet(heapStatsDep, sysStatsDep),
   454  			compute: func(in *statAggregate, out *metricValue) {
   455  				out.kind = metricKindUint64
   456  				out.scalar = uint64(in.heapStats.committed+in.heapStats.released) +
   457  					in.sysStats.stacksSys + in.sysStats.mSpanSys +
   458  					in.sysStats.mCacheSys + in.sysStats.buckHashSys +
   459  					in.sysStats.gcMiscSys + in.sysStats.otherSys
   460  			},
   461  		},
   462  		"/sched/gomaxprocs:threads": {
   463  			compute: func(_ *statAggregate, out *metricValue) {
   464  				out.kind = metricKindUint64
   465  				out.scalar = uint64(gomaxprocs)
   466  			},
   467  		},
   468  		"/sched/goroutines:goroutines": {
   469  			deps: makeStatDepSet(schedStatsDep),
   470  			compute: func(in *statAggregate, out *metricValue) {
   471  				out.kind = metricKindUint64
   472  				out.scalar = uint64(in.schedStats.gTotal)
   473  			},
   474  		},
   475  		"/sched/goroutines/not-in-go:goroutines": {
   476  			deps: makeStatDepSet(schedStatsDep),
   477  			compute: func(in *statAggregate, out *metricValue) {
   478  				out.kind = metricKindUint64
   479  				out.scalar = uint64(in.schedStats.gNonGo)
   480  			},
   481  		},
   482  		"/sched/goroutines/running:goroutines": {
   483  			deps: makeStatDepSet(schedStatsDep),
   484  			compute: func(in *statAggregate, out *metricValue) {
   485  				out.kind = metricKindUint64
   486  				out.scalar = uint64(in.schedStats.gRunning)
   487  			},
   488  		},
   489  		"/sched/goroutines/runnable:goroutines": {
   490  			deps: makeStatDepSet(schedStatsDep),
   491  			compute: func(in *statAggregate, out *metricValue) {
   492  				out.kind = metricKindUint64
   493  				out.scalar = uint64(in.schedStats.gRunnable)
   494  			},
   495  		},
   496  		"/sched/goroutines/waiting:goroutines": {
   497  			deps: makeStatDepSet(schedStatsDep),
   498  			compute: func(in *statAggregate, out *metricValue) {
   499  				out.kind = metricKindUint64
   500  				out.scalar = uint64(in.schedStats.gWaiting)
   501  			},
   502  		},
   503  		"/sched/goroutines-created:goroutines": {
   504  			deps: makeStatDepSet(schedStatsDep),
   505  			compute: func(in *statAggregate, out *metricValue) {
   506  				out.kind = metricKindUint64
   507  				out.scalar = uint64(in.schedStats.gCreated)
   508  			},
   509  		},
   510  		"/sched/latencies:seconds": {
   511  			compute: func(_ *statAggregate, out *metricValue) {
   512  				sched.timeToRun.write(out)
   513  			},
   514  		},
   515  		"/sched/pauses/stopping/gc:seconds": {
   516  			compute: func(_ *statAggregate, out *metricValue) {
   517  				sched.stwStoppingTimeGC.write(out)
   518  			},
   519  		},
   520  		"/sched/pauses/stopping/other:seconds": {
   521  			compute: func(_ *statAggregate, out *metricValue) {
   522  				sched.stwStoppingTimeOther.write(out)
   523  			},
   524  		},
   525  		"/sched/pauses/total/gc:seconds": {
   526  			compute: func(_ *statAggregate, out *metricValue) {
   527  				sched.stwTotalTimeGC.write(out)
   528  			},
   529  		},
   530  		"/sched/pauses/total/other:seconds": {
   531  			compute: func(_ *statAggregate, out *metricValue) {
   532  				sched.stwTotalTimeOther.write(out)
   533  			},
   534  		},
   535  		"/sched/threads/total:threads": {
   536  			deps: makeStatDepSet(schedStatsDep),
   537  			compute: func(in *statAggregate, out *metricValue) {
   538  				out.kind = metricKindUint64
   539  				out.scalar = uint64(in.schedStats.threads)
   540  			},
   541  		},
   542  		"/sync/mutex/wait/total:seconds": {
   543  			compute: func(_ *statAggregate, out *metricValue) {
   544  				out.kind = metricKindFloat64
   545  				out.scalar = float64bits(nsToSec(totalMutexWaitTimeNanos()))
   546  			},
   547  		},
   548  	}
   549  
   550  	for _, info := range godebugs.All {
   551  		if !info.Opaque {
   552  			metrics["/godebug/non-default-behavior/"+info.Name+":events"] = metricData{compute: compute0}
   553  		}
   554  	}
   555  
   556  	metricsInit = true
   557  }
   558  
   559  func compute0(_ *statAggregate, out *metricValue) {
   560  	out.kind = metricKindUint64
   561  	out.scalar = 0
   562  }
   563  
   564  type metricReader func() uint64
   565  
   566  func (f metricReader) compute(_ *statAggregate, out *metricValue) {
   567  	out.kind = metricKindUint64
   568  	out.scalar = f()
   569  }
   570  
   571  //go:linkname godebug_registerMetric internal/godebug.registerMetric
   572  func godebug_registerMetric(name string, read func() uint64) {
   573  	metricsLock()
   574  	initMetrics()
   575  	d, ok := metrics[name]
   576  	if !ok {
   577  		throw("runtime: unexpected metric registration for " + name)
   578  	}
   579  	d.compute = metricReader(read).compute
   580  	metrics[name] = d
   581  	metricsUnlock()
   582  }
   583  
   584  // statDep is a dependency on a group of statistics
   585  // that a metric might have.
   586  type statDep uint
   587  
   588  const (
   589  	heapStatsDep  statDep = iota // corresponds to heapStatsAggregate
   590  	sysStatsDep                  // corresponds to sysStatsAggregate
   591  	cpuStatsDep                  // corresponds to cpuStatsAggregate
   592  	gcStatsDep                   // corresponds to gcStatsAggregate
   593  	finalStatsDep                // corresponds to finalStatsAggregate
   594  	schedStatsDep                // corresponds to schedStatsAggregate
   595  	numStatsDeps
   596  )
   597  
   598  // statDepSet represents a set of statDeps.
   599  //
   600  // Under the hood, it's a bitmap.
   601  type statDepSet [1]uint64
   602  
   603  // makeStatDepSet creates a new statDepSet from a list of statDeps.
   604  func makeStatDepSet(deps ...statDep) statDepSet {
   605  	var s statDepSet
   606  	for _, d := range deps {
   607  		s[d/64] |= 1 << (d % 64)
   608  	}
   609  	return s
   610  }
   611  
   612  // difference returns set difference of s from b as a new set.
   613  func (s statDepSet) difference(b statDepSet) statDepSet {
   614  	var c statDepSet
   615  	for i := range s {
   616  		c[i] = s[i] &^ b[i]
   617  	}
   618  	return c
   619  }
   620  
   621  // union returns the union of the two sets as a new set.
   622  func (s statDepSet) union(b statDepSet) statDepSet {
   623  	var c statDepSet
   624  	for i := range s {
   625  		c[i] = s[i] | b[i]
   626  	}
   627  	return c
   628  }
   629  
   630  // empty returns true if there are no dependencies in the set.
   631  func (s *statDepSet) empty() bool {
   632  	for _, c := range s {
   633  		if c != 0 {
   634  			return false
   635  		}
   636  	}
   637  	return true
   638  }
   639  
   640  // has returns true if the set contains a given statDep.
   641  func (s *statDepSet) has(d statDep) bool {
   642  	return s[d/64]&(1<<(d%64)) != 0
   643  }
   644  
   645  // heapStatsAggregate represents memory stats obtained from the
   646  // runtime. This set of stats is grouped together because they
   647  // depend on each other in some way to make sense of the runtime's
   648  // current heap memory use. They're also sharded across Ps, so it
   649  // makes sense to grab them all at once.
   650  type heapStatsAggregate struct {
   651  	heapStatsDelta
   652  
   653  	// Derived from values in heapStatsDelta.
   654  
   655  	// inObjects is the bytes of memory occupied by objects,
   656  	inObjects uint64
   657  
   658  	// numObjects is the number of live objects in the heap.
   659  	numObjects uint64
   660  
   661  	// totalAllocated is the total bytes of heap objects allocated
   662  	// over the lifetime of the program.
   663  	totalAllocated uint64
   664  
   665  	// totalFreed is the total bytes of heap objects freed
   666  	// over the lifetime of the program.
   667  	totalFreed uint64
   668  
   669  	// totalAllocs is the number of heap objects allocated over
   670  	// the lifetime of the program.
   671  	totalAllocs uint64
   672  
   673  	// totalFrees is the number of heap objects freed over
   674  	// the lifetime of the program.
   675  	totalFrees uint64
   676  }
   677  
   678  // compute populates the heapStatsAggregate with values from the runtime.
   679  func (a *heapStatsAggregate) compute() {
   680  	memstats.heapStats.read(&a.heapStatsDelta)
   681  
   682  	// Calculate derived stats.
   683  	a.totalAllocs = a.largeAllocCount
   684  	a.totalFrees = a.largeFreeCount
   685  	a.totalAllocated = a.largeAlloc
   686  	a.totalFreed = a.largeFree
   687  	for i := range a.smallAllocCount {
   688  		na := a.smallAllocCount[i]
   689  		nf := a.smallFreeCount[i]
   690  		a.totalAllocs += na
   691  		a.totalFrees += nf
   692  		a.totalAllocated += na * uint64(gc.SizeClassToSize[i])
   693  		a.totalFreed += nf * uint64(gc.SizeClassToSize[i])
   694  	}
   695  	a.inObjects = a.totalAllocated - a.totalFreed
   696  	a.numObjects = a.totalAllocs - a.totalFrees
   697  }
   698  
   699  // sysStatsAggregate represents system memory stats obtained
   700  // from the runtime. This set of stats is grouped together because
   701  // they're all relatively cheap to acquire and generally independent
   702  // of one another and other runtime memory stats. The fact that they
   703  // may be acquired at different times, especially with respect to
   704  // heapStatsAggregate, means there could be some skew, but because of
   705  // these stats are independent, there's no real consistency issue here.
   706  type sysStatsAggregate struct {
   707  	stacksSys      uint64
   708  	mSpanSys       uint64
   709  	mSpanInUse     uint64
   710  	mCacheSys      uint64
   711  	mCacheInUse    uint64
   712  	buckHashSys    uint64
   713  	gcMiscSys      uint64
   714  	otherSys       uint64
   715  	heapGoal       uint64
   716  	gcCyclesDone   uint64
   717  	gcCyclesForced uint64
   718  }
   719  
   720  // compute populates the sysStatsAggregate with values from the runtime.
   721  func (a *sysStatsAggregate) compute() {
   722  	a.stacksSys = memstats.stacks_sys.load()
   723  	a.buckHashSys = memstats.buckhash_sys.load()
   724  	a.gcMiscSys = memstats.gcMiscSys.load()
   725  	a.otherSys = memstats.other_sys.load()
   726  	a.heapGoal = gcController.heapGoal()
   727  	a.gcCyclesDone = uint64(memstats.numgc)
   728  	a.gcCyclesForced = uint64(memstats.numforcedgc)
   729  
   730  	systemstack(func() {
   731  		lock(&mheap_.lock)
   732  		a.mSpanSys = memstats.mspan_sys.load()
   733  		a.mSpanInUse = uint64(mheap_.spanalloc.inuse)
   734  		a.mCacheSys = memstats.mcache_sys.load()
   735  		a.mCacheInUse = uint64(mheap_.cachealloc.inuse)
   736  		unlock(&mheap_.lock)
   737  	})
   738  }
   739  
   740  // cpuStatsAggregate represents CPU stats obtained from the runtime
   741  // acquired together to avoid skew and inconsistencies.
   742  type cpuStatsAggregate struct {
   743  	cpuStats
   744  }
   745  
   746  // compute populates the cpuStatsAggregate with values from the runtime.
   747  func (a *cpuStatsAggregate) compute() {
   748  	a.cpuStats = work.cpuStats
   749  	// TODO(mknyszek): Update the CPU stats again so that we're not
   750  	// just relying on the STW snapshot. The issue here is that currently
   751  	// this will cause non-monotonicity in the "user" CPU time metric.
   752  	//
   753  	// a.cpuStats.accumulate(nanotime(), gcphase == _GCmark)
   754  }
   755  
   756  // gcStatsAggregate represents various GC stats obtained from the runtime
   757  // acquired together to avoid skew and inconsistencies.
   758  type gcStatsAggregate struct {
   759  	heapScan    uint64
   760  	stackScan   uint64
   761  	globalsScan uint64
   762  	totalScan   uint64
   763  }
   764  
   765  // compute populates the gcStatsAggregate with values from the runtime.
   766  func (a *gcStatsAggregate) compute() {
   767  	a.heapScan = gcController.heapScan.Load()
   768  	a.stackScan = gcController.lastStackScan.Load()
   769  	a.globalsScan = gcController.globalsScan.Load()
   770  	a.totalScan = a.heapScan + a.stackScan + a.globalsScan
   771  }
   772  
   773  // finalStatsAggregate represents various finalizer/cleanup stats obtained
   774  // from the runtime acquired together to avoid skew and inconsistencies.
   775  type finalStatsAggregate struct {
   776  	finalizersQueued   uint64
   777  	finalizersExecuted uint64
   778  	cleanupsQueued     uint64
   779  	cleanupsExecuted   uint64
   780  }
   781  
   782  // compute populates the finalStatsAggregate with values from the runtime.
   783  func (a *finalStatsAggregate) compute() {
   784  	a.finalizersQueued, a.finalizersExecuted = finReadQueueStats()
   785  	a.cleanupsQueued, a.cleanupsExecuted = gcCleanups.readQueueStats()
   786  }
   787  
   788  // schedStatsAggregate contains stats about the scheduler, including
   789  // an approximate count of goroutines in each state.
   790  type schedStatsAggregate struct {
   791  	gTotal    uint64
   792  	gRunning  uint64
   793  	gRunnable uint64
   794  	gNonGo    uint64
   795  	gWaiting  uint64
   796  	gCreated  uint64
   797  	threads   uint64
   798  }
   799  
   800  // compute populates the schedStatsAggregate with values from the runtime.
   801  func (a *schedStatsAggregate) compute() {
   802  	// Lock the scheduler so the global run queue can't change and
   803  	// the number of Ps can't change. This doesn't prevent the
   804  	// local run queues from changing, so the results are still
   805  	// approximate.
   806  	lock(&sched.lock)
   807  
   808  	// The total count of threads owned by Go is the number of Ms
   809  	// minus extra Ms on the list or in use.
   810  	a.threads = uint64(mcount()) - uint64(extraMInUse.Load()) - uint64(extraMLength.Load())
   811  
   812  	// Collect running/runnable from per-P run queues.
   813  	a.gCreated += sched.goroutinesCreated.Load()
   814  	for _, p := range allp {
   815  		if p == nil || p.status == _Pdead {
   816  			break
   817  		}
   818  		a.gCreated += p.goroutinesCreated
   819  		switch p.status {
   820  		case _Prunning:
   821  			a.gRunning++
   822  		case _Psyscall:
   823  			a.gNonGo++
   824  		case _Pgcstop:
   825  			// The world is stopping or stopped.
   826  			// This is fine. The results will be
   827  			// slightly odd since nothing else
   828  			// is running, but it will be accurate.
   829  		}
   830  
   831  		for {
   832  			h := atomic.Load(&p.runqhead)
   833  			t := atomic.Load(&p.runqtail)
   834  			next := atomic.Loaduintptr((*uintptr)(&p.runnext))
   835  			runnable := int32(t - h)
   836  			if atomic.Load(&p.runqhead) != h || runnable < 0 {
   837  				continue
   838  			}
   839  			if next != 0 {
   840  				runnable++
   841  			}
   842  			a.gRunnable += uint64(runnable)
   843  			break
   844  		}
   845  	}
   846  
   847  	// Global run queue.
   848  	a.gRunnable += uint64(sched.runq.size)
   849  
   850  	// Account for Gs that are in _Gsyscall without a P in _Psyscall.
   851  	nGsyscallNoP := sched.nGsyscallNoP.Load()
   852  
   853  	// nGsyscallNoP can go negative during temporary races.
   854  	if nGsyscallNoP >= 0 {
   855  		a.gNonGo += uint64(nGsyscallNoP)
   856  	}
   857  
   858  	// Compute the number of blocked goroutines. We have to
   859  	// include system goroutines in this count because we included
   860  	// them above.
   861  	a.gTotal = uint64(gcount(true))
   862  	a.gWaiting = a.gTotal - (a.gRunning + a.gRunnable + a.gNonGo)
   863  	if a.gWaiting < 0 {
   864  		a.gWaiting = 0
   865  	}
   866  
   867  	unlock(&sched.lock)
   868  }
   869  
   870  // nsToSec takes a duration in nanoseconds and converts it to seconds as
   871  // a float64.
   872  func nsToSec(ns int64) float64 {
   873  	return float64(ns) / 1e9
   874  }
   875  
   876  // statAggregate is the main driver of the metrics implementation.
   877  //
   878  // It contains multiple aggregates of runtime statistics, as well
   879  // as a set of these aggregates that it has populated. The aggregates
   880  // are populated lazily by its ensure method.
   881  type statAggregate struct {
   882  	ensured    statDepSet
   883  	heapStats  heapStatsAggregate
   884  	sysStats   sysStatsAggregate
   885  	cpuStats   cpuStatsAggregate
   886  	gcStats    gcStatsAggregate
   887  	finalStats finalStatsAggregate
   888  	schedStats schedStatsAggregate
   889  }
   890  
   891  // ensure populates statistics aggregates determined by deps if they
   892  // haven't yet been populated.
   893  func (a *statAggregate) ensure(deps *statDepSet) {
   894  	missing := deps.difference(a.ensured)
   895  	if missing.empty() {
   896  		return
   897  	}
   898  	for i := statDep(0); i < numStatsDeps; i++ {
   899  		if !missing.has(i) {
   900  			continue
   901  		}
   902  		switch i {
   903  		case heapStatsDep:
   904  			a.heapStats.compute()
   905  		case sysStatsDep:
   906  			a.sysStats.compute()
   907  		case cpuStatsDep:
   908  			a.cpuStats.compute()
   909  		case gcStatsDep:
   910  			a.gcStats.compute()
   911  		case finalStatsDep:
   912  			a.finalStats.compute()
   913  		case schedStatsDep:
   914  			a.schedStats.compute()
   915  		}
   916  	}
   917  	a.ensured = a.ensured.union(missing)
   918  }
   919  
   920  // metricKind is a runtime copy of runtime/metrics.ValueKind and
   921  // must be kept structurally identical to that type.
   922  type metricKind int
   923  
   924  const (
   925  	// These values must be kept identical to their corresponding Kind* values
   926  	// in the runtime/metrics package.
   927  	metricKindBad metricKind = iota
   928  	metricKindUint64
   929  	metricKindFloat64
   930  	metricKindFloat64Histogram
   931  )
   932  
   933  // metricSample is a runtime copy of runtime/metrics.Sample and
   934  // must be kept structurally identical to that type.
   935  type metricSample struct {
   936  	name  string
   937  	value metricValue
   938  }
   939  
   940  // metricValue is a runtime copy of runtime/metrics.Sample and
   941  // must be kept structurally identical to that type.
   942  type metricValue struct {
   943  	kind    metricKind
   944  	scalar  uint64         // contains scalar values for scalar Kinds.
   945  	pointer unsafe.Pointer // contains non-scalar values.
   946  }
   947  
   948  // float64HistOrInit tries to pull out an existing float64Histogram
   949  // from the value, but if none exists, then it allocates one with
   950  // the given buckets.
   951  func (v *metricValue) float64HistOrInit(buckets []float64) *metricFloat64Histogram {
   952  	var hist *metricFloat64Histogram
   953  	if v.kind == metricKindFloat64Histogram && v.pointer != nil {
   954  		hist = (*metricFloat64Histogram)(v.pointer)
   955  	} else {
   956  		v.kind = metricKindFloat64Histogram
   957  		hist = new(metricFloat64Histogram)
   958  		v.pointer = unsafe.Pointer(hist)
   959  	}
   960  	hist.buckets = buckets
   961  	if len(hist.counts) != len(hist.buckets)-1 {
   962  		hist.counts = make([]uint64, len(buckets)-1)
   963  	}
   964  	return hist
   965  }
   966  
   967  // metricFloat64Histogram is a runtime copy of runtime/metrics.Float64Histogram
   968  // and must be kept structurally identical to that type.
   969  type metricFloat64Histogram struct {
   970  	counts  []uint64
   971  	buckets []float64
   972  }
   973  
   974  // agg is used by readMetrics, and is protected by metricsSema.
   975  //
   976  // Managed as a global variable because its pointer will be
   977  // an argument to a dynamically-defined function, and we'd
   978  // like to avoid it escaping to the heap.
   979  var agg statAggregate
   980  
   981  type metricName struct {
   982  	name string
   983  	kind metricKind
   984  }
   985  
   986  // readMetricNames is the implementation of runtime/metrics.readMetricNames,
   987  // used by the runtime/metrics test and otherwise unreferenced.
   988  //
   989  //go:linkname readMetricNames runtime/metrics_test.runtime_readMetricNames
   990  func readMetricNames() []string {
   991  	metricsLock()
   992  	initMetrics()
   993  	n := len(metrics)
   994  	metricsUnlock()
   995  
   996  	list := make([]string, 0, n)
   997  
   998  	metricsLock()
   999  	for name := range metrics {
  1000  		list = append(list, name)
  1001  	}
  1002  	metricsUnlock()
  1003  
  1004  	return list
  1005  }
  1006  
  1007  // readMetrics is the implementation of runtime/metrics.Read.
  1008  //
  1009  //go:linkname readMetrics runtime/metrics.runtime_readMetrics
  1010  func readMetrics(samplesp unsafe.Pointer, len int, cap int) {
  1011  	metricsLock()
  1012  
  1013  	// Ensure the map is initialized.
  1014  	initMetrics()
  1015  
  1016  	// Read the metrics.
  1017  	readMetricsLocked(samplesp, len, cap)
  1018  	metricsUnlock()
  1019  }
  1020  
  1021  // readMetricsLocked is the internal, locked portion of readMetrics.
  1022  //
  1023  // Broken out for more robust testing. metricsLock must be held and
  1024  // initMetrics must have been called already.
  1025  func readMetricsLocked(samplesp unsafe.Pointer, len int, cap int) {
  1026  	// Construct a slice from the args.
  1027  	sl := slice{samplesp, len, cap}
  1028  	samples := *(*[]metricSample)(unsafe.Pointer(&sl))
  1029  
  1030  	// Clear agg defensively.
  1031  	agg = statAggregate{}
  1032  
  1033  	// Sample.
  1034  	for i := range samples {
  1035  		sample := &samples[i]
  1036  		data, ok := metrics[sample.name]
  1037  		if !ok {
  1038  			sample.value.kind = metricKindBad
  1039  			continue
  1040  		}
  1041  		// Ensure we have all the stats we need.
  1042  		// agg is populated lazily.
  1043  		agg.ensure(&data.deps)
  1044  
  1045  		// Compute the value based on the stats we have.
  1046  		data.compute(&agg, &sample.value)
  1047  	}
  1048  }
  1049  

View as plain text