metrics.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"path/filepath"
    11  	"regexp"
    12  	"simd/archsimd/_gen/specgen"
    13  	"slices"
    14  	"strings"
    15  
    16  	"gopkg.in/yaml.v3"
    17  )
    18  
    19  // specOp is one spec operation -- one function in package spec. It is the grain
    20  // spec is authored at, and the only grain at which "implemented nowhere" means
    21  // anything: one operation expands into many API names and many (receiver, name)
    22  // pairs. The API has no corresponding notion, so this is a spec-side diagnostic
    23  // rather than a fourth coverage grain (SPEC-TRANSITION.md §1.3).
    24  type specOp struct {
    25  	name     string // the operation's name in package spec
    26  	covPairs int    // pairs it generates that the API also declares
    27  }
    28  
    29  // Spec and the API are each a map from name to the set of receivers that name
    30  // is declared on. A relation is how those two sets stand for one name, and
    31  // walking the union of the two key sets classifies every name into exactly one
    32  // -- which the older "divergence classes" did not do, since a name whose sets
    33  // cross contributes to both sides at once.
    34  type relation int
    35  
    36  const (
    37  	relEqual    relation = iota // api = spec
    38  	relSubset                   // api is a proper subset of spec: spec covers more receivers
    39  	relSuperset                 // api is a proper superset of spec: spec covers fewer
    40  	relOverlap                  // they share receivers, neither contains the other
    41  	relDisjoint                 // they share the name and not one receiver
    42  	relSpecOnly                 // api = ∅: no target declares the name
    43  	relAPIOnly                  // spec = ∅: spec has never defined the name
    44  	numRelations
    45  )
    46  
    47  // shared reports whether both sides define the name. The api-only receivers of
    48  // a shared name are defects; under spec = ∅ they are ordinary coverage.
    49  func (r relation) shared() bool { return r != relSpecOnly && r != relAPIOnly }
    50  
    51  var relationName = [numRelations]string{
    52  	relEqual:    "api = spec",
    53  	relSubset:   "api ⊂ spec",
    54  	relSuperset: "api ⊃ spec",
    55  	relOverlap:  "overlapping",
    56  	relDisjoint: "disjoint",
    57  	relSpecOnly: "api = ∅",
    58  	relAPIOnly:  "spec = ∅",
    59  }
    60  
    61  // nameRel is one name, and how the two receiver sets stand under it.
    62  type nameRel struct {
    63  	name      string
    64  	rel       relation
    65  	api, spec int      // receiver-set sizes
    66  	apiOnly   []string // receivers only the API declares, sorted
    67  	specOnly  int      // receivers only spec generates
    68  }
    69  
    70  // relTotal is one row of the relation summary: how many names stand in it, and
    71  // how many receivers each side holds alone across them.
    72  type relTotal struct{ names, apiOnly, specOnly int }
    73  
    74  // fileCount pairs a filename with a count, for per-file breakdowns.
    75  type fileCount struct {
    76  	file string
    77  	n    int
    78  }
    79  
    80  // metrics is the whole report as data. Every field is a tally or a pivot, so
    81  // the summary and the sections below it read the same numbers and cannot
    82  // disagree.
    83  type metrics struct {
    84  	facts []fact
    85  	decls []*decl // the facts' declarations, deduplicated across configurations
    86  	spec  map[methodKey]*specgen.Func
    87  
    88  	// Coverage, at three grains and split two ways.
    89  	covTriple *tally[archMethodKey]
    90  	covPair   *tally[methodKey]
    91  	covName   *tally[string]
    92  	covTarget *pivot[archMethodKey]
    93  	covGen    *pivot[methodKey]
    94  	covMulti  *tally[string] // names emitted by more than one tool
    95  	covCommon *tally[string] // simd_emulated.go's methods, spec-common's scope
    96  
    97  	specPairs, specNames int
    98  
    99  	// Divergence classes
   100  	ops                   []specOp               // spec source functions, sorted by name
   101  	specOnly              []string               // spec operations no target implements
   102  	names                 []nameRel              // every name either side defines, sorted
   103  	rel                   [numRelations]relTotal // m.names summed by relation
   104  	apiOnly, apiOnlyDecls int                    // the defect: over shared names only
   105  
   106  	// Agreement with spec where the two overlap. Names are split the way
   107  	// specdoc.Fill treats them: a named argument that disagrees with spec is a
   108  	// mismatch, and an unnamed one is a name Fill will fill in from spec.
   109  	agree, typeOK               *tally[declKey]
   110  	paramNameOK, resultNameOK   *tally[declKey]
   111  	paramsFilled, resultsFilled *tally[declKey]
   112  
   113  	// The invariant, checked without reference to spec.
   114  	consistent, typeSame, paramNameSame, resultNameSame *tally[methodKey]
   115  
   116  	// Documentation.
   117  	fromSpec           *tally[declKey]
   118  	bounded            *tally[declKey]
   119  	handConflict       int
   120  	handConflictDecls  []string // the declarations behind handConflict, sorted
   121  	handConflictByFile []fileCount
   122  	crossPkg           *tally[methodKey]
   123  	mirror             *tally[methodKey]
   124  
   125  	// The inputs being retired.
   126  	categories                       map[string]int
   127  	catEntries, catUniq, catExported int
   128  	catCompilerOnly                  int
   129  	catExportedFields                map[string]int
   130  	comments                         int
   131  	commentsSpecCovered              int
   132  	commentsSpecCoveredNames         []string // the entries behind commentsSpecCovered
   133  	commutDrift                      int
   134  }
   135  
   136  func measure(api []*decl, specFuncs []*specgen.Func, genDir string) *metrics {
   137  	m := &metrics{spec: map[methodKey]*specgen.Func{}}
   138  
   139  	specNames := map[string]bool{}
   140  	for _, f := range specFuncs {
   141  		m.spec[specPair(f)] = f
   142  		specNames[f.Name] = true
   143  	}
   144  	m.specPairs, m.specNames = len(m.spec), len(specNames)
   145  
   146  	// The fact table: one row per declaration per configuration it builds in.
   147  	for _, d := range api {
   148  		if nonSpecDecl(d) {
   149  			continue
   150  		}
   151  		m.decls = append(m.decls, d)
   152  		for _, t := range d.targets {
   153  			m.facts = append(m.facts, fact{d, t})
   154  		}
   155  	}
   156  
   157  	covered := func(f fact) bool { return m.spec[f.methodKey()] != nil }
   158  
   159  	// 1. Coverage. Same predicate, three grains, two dimensions.
   160  	m.covTriple = tallyBy(m.facts, fact.archMethodKey, covered)
   161  	m.covPair = tallyBy(m.facts, func(f fact) methodKey { return f.methodKey() }, covered)
   162  	m.covName = tallyBy(m.facts, func(f fact) string { return f.name }, func(f fact) bool { return specNames[f.name] })
   163  	m.covTarget = pivotBy(m.facts, func(f fact) string { return f.target }, fact.archMethodKey, covered)
   164  	m.covGen = pivotBy(m.facts, func(f fact) string { return f.gen }, func(f fact) methodKey { return f.methodKey() }, covered)
   165  	for _, g := range m.covGen.dims() {
   166  		if t, _, _ := strings.Cut(g, "/"); t != g {
   167  			m.covGen.fold(g, t)
   168  		}
   169  	}
   170  
   171  	// Names emitted by more than one tool are where the invariant is most
   172  	// exposed, because no single generator can see the conflict. simdgen's own
   173  	// runs share their inputs, so they do not count as more than one.
   174  	tools := groupBy(m.facts, func(f fact) string { return f.name })
   175  	m.covMulti = new(tally[string])
   176  	for name, fs := range tools {
   177  		seen := map[string]bool{}
   178  		for _, f := range fs {
   179  			if f.gen != "hand-written" {
   180  				seen[f.tool()] = true
   181  			}
   182  		}
   183  		if len(seen) > 1 {
   184  			m.covMulti.add(name, specNames[name])
   185  		}
   186  	}
   187  	m.covCommon = new(tally[string])
   188  	for _, n := range commonMethods(api) {
   189  		m.covCommon.add(n, m.covName.done(n))
   190  	}
   191  
   192  	// 2. Divergence classes.
   193  	byOp := map[string]*specOp{}
   194  	for _, f := range specFuncs {
   195  		src, _, _ := f.SpecFunc()
   196  		op := byOp[src]
   197  		if op == nil {
   198  			op = &specOp{name: src}
   199  			byOp[src] = op
   200  		}
   201  		if m.covPair.has(specPair(f)) {
   202  			op.covPairs++
   203  		}
   204  	}
   205  	for _, op := range byOp {
   206  		if op.covPairs == 0 {
   207  			// Operations implemented by spec but not by the API on any platform
   208  			// is measured at the grain spec is authored at: the source
   209  			// function. A spec operation with no covered declaration anywhere
   210  			// is likely a defect, because that takes four simultaneous
   211  			// omissions to explain away.
   212  			m.specOnly = append(m.specOnly, op.name)
   213  		}
   214  		m.ops = append(m.ops, *op)
   215  	}
   216  	slices.Sort(m.specOnly)
   217  	slices.SortFunc(m.ops, func(a, b specOp) int { return cmp.Compare(a.name, b.name) })
   218  	// The two maps the rest of this section is about, and the union of their
   219  	// keys. Everything below -- the defect count, the review queue, and the
   220  	// coverage gap -- is a statement about how one name's two receiver sets
   221  	// stand, so they are derived here once rather than counted three ways.
   222  	apiRecv, specRecv := map[string]map[string]bool{}, map[string]map[string]bool{}
   223  	add := func(m map[string]map[string]bool, name, recv string) {
   224  		if m[name] == nil {
   225  			m[name] = map[string]bool{}
   226  		}
   227  		m[name][recv] = true
   228  	}
   229  	for _, d := range m.decls {
   230  		add(apiRecv, d.name, d.recv)
   231  	}
   232  	for p := range m.spec {
   233  		if !nonSpecOp(p.name) {
   234  			add(specRecv, p.name, p.recv)
   235  		}
   236  	}
   237  	for name := range apiRecv {
   238  		m.names = append(m.names, nameRelation(name, apiRecv[name], specRecv[name]))
   239  	}
   240  	for name := range specRecv {
   241  		if apiRecv[name] == nil {
   242  			m.names = append(m.names, nameRelation(name, nil, specRecv[name]))
   243  		}
   244  	}
   245  	slices.SortFunc(m.names, func(a, b nameRel) int { return cmp.Compare(a.name, b.name) })
   246  	for _, r := range m.names {
   247  		t := &m.rel[r.rel]
   248  		t.names++
   249  		t.apiOnly += len(r.apiOnly)
   250  		t.specOnly += r.specOnly
   251  		if r.rel.shared() {
   252  			m.apiOnly += len(r.apiOnly)
   253  		}
   254  	}
   255  	// The same api-only receivers counted as declarations rather than pairs: one
   256  	// pair can be declared once per target.
   257  	m.apiOnlyDecls = countBy(filter(m.facts, func(f fact) bool {
   258  		return !covered(f) && specNames[f.name]
   259  	}), fact.declKey)
   260  
   261  	// 3. Agreement with spec, over the declarations spec covers.
   262  	overlap := filter(m.facts, covered)
   263  	specIn := func(f fact) []param { return specParams(m.spec[f.methodKey()].In) }
   264  	specOut := func(f fact) []param { return specParams(m.spec[f.methodKey()].Out) }
   265  	typeOK := func(f fact) bool {
   266  		return equalBy(f.in, specIn(f), paramType) && equalBy(f.out, specOut(f), paramType)
   267  	}
   268  	resultNameOK := func(f fact) bool { return !namesClash(f.out, specOut(f)) }
   269  	paramNameOK := func(f fact) bool { return !namesClash(f.in, specIn(f)) }
   270  	resultsFilled := func(f fact) bool { return !namesMissing(f.out, specOut(f)) }
   271  	paramsFilled := func(f fact) bool { return !namesMissing(f.in, specIn(f)) }
   272  
   273  	m.typeOK = tallyBy(overlap, fact.declKey, typeOK)
   274  	m.resultNameOK = tallyBy(overlap, fact.declKey, resultNameOK)
   275  	m.paramNameOK = tallyBy(overlap, fact.declKey, paramNameOK)
   276  	m.resultsFilled = tallyBy(overlap, fact.declKey, resultsFilled)
   277  	m.paramsFilled = tallyBy(overlap, fact.declKey, paramsFilled)
   278  	m.agree = tallyBy(overlap, fact.declKey, func(f fact) bool {
   279  		return typeOK(f) && resultNameOK(f) && paramNameOK(f)
   280  	})
   281  
   282  	// 4. The invariant: where a pair is declared more than once, do the
   283  	// declarations agree with each other? An unnamed argument agrees with any
   284  	// name, since only names that are used -- by a doc or a body -- need to agree.
   285  	m.consistent = new(tally[methodKey])
   286  	m.typeSame = new(tally[methodKey])
   287  	m.paramNameSame = new(tally[methodKey])
   288  	m.resultNameSame = new(tally[methodKey])
   289  	for p, ds := range groupBy(m.decls, (*decl).methodKey) {
   290  		if len(ds) < 2 {
   291  			continue
   292  		}
   293  		params := func(d *decl) []param { return d.in }
   294  		results := func(d *decl) []param { return d.out }
   295  		typesAgree := func(get func(*decl) []param) bool {
   296  			return !slices.ContainsFunc(ds[1:], func(d *decl) bool {
   297  				return !equalBy(get(ds[0]), get(d), paramType)
   298  			})
   299  		}
   300  		namesAgree := func(get func(*decl) []param) bool {
   301  			lists := make([][]param, len(ds))
   302  			for i, d := range ds {
   303  				lists[i] = get(d)
   304  			}
   305  			return namesConsistent(lists)
   306  		}
   307  		t := typesAgree(params) && typesAgree(results)
   308  		pn, rn := namesAgree(params), namesAgree(results)
   309  		m.typeSame.add(p, t)
   310  		m.paramNameSame.add(p, pn)
   311  		m.resultNameSame.add(p, rn)
   312  		m.consistent.add(p, t && pn && rn)
   313  	}
   314  
   315  	// 5. Documentation.
   316  	// A declaration is sourced from spec when its doc begins with spec's text --
   317  	// a prefix test, because Fill leaves the generator's notes below what it
   318  	// injects. Prose that disagrees instead is a conflict for doc-triage; a missing
   319  	// doc is not, since Fill would simply add one.
   320  	documented := filter(overlap, func(f fact) bool { return len(m.spec[f.methodKey()].Doc) > 0 })
   321  	saysSpec := func(f fact) bool {
   322  		specDoc, err := specgen.FormatComment(m.spec[f.methodKey()].Doc)
   323  		if err != nil {
   324  			return false
   325  		}
   326  		return strings.HasPrefix(f.doc, strings.TrimRight(specDoc, "\n"))
   327  	}
   328  	m.fromSpec = tallyBy(documented, fact.declKey, saysSpec)
   329  	handConflicts := filter(documented, func(f fact) bool {
   330  		return f.gen == "hand-written" && f.doc != "" && !saysSpec(f)
   331  	})
   332  	m.handConflict = countBy(handConflicts, fact.declKey)
   333  	byFile := map[string]int{}
   334  	seenByFile := map[declKey]bool{}
   335  	for _, f := range handConflicts {
   336  		dk := f.declKey()
   337  		if !seenByFile[dk] {
   338  			seenByFile[dk] = true
   339  			byFile[f.file]++
   340  			m.handConflictDecls = append(m.handConflictDecls, fmt.Sprint(dk))
   341  		}
   342  	}
   343  	slices.Sort(m.handConflictDecls)
   344  	for _, k := range sortedKeys(byFile) {
   345  		m.handConflictByFile = append(m.handConflictByFile, fileCount{k, byFile[k]})
   346  	}
   347  
   348  	m.bounded = tallyBy(filter(m.facts, func(f fact) bool { return f.doc != "" }),
   349  		fact.declKey, func(f fact) bool { return f.hasSep })
   350  
   351  	// simd and archsimd document the same operations from different sources.
   352  	// Target-specific implementation notes (such as Asm: and CPU Feature:)
   353  	// and directives are ignored when comparing docs.
   354  	m.crossPkg = new(tally[methodKey])
   355  	docs := map[string]map[methodKey]string{"simd": {}, "archsimd": {}}
   356  	for _, d := range m.decls {
   357  		if d.doc != "" {
   358  			docs[d.pkg][d.methodKey()] = d.doc
   359  		}
   360  	}
   361  	for p, sd := range docs["simd"] {
   362  		if ad, ok := docs["archsimd"][p]; ok {
   363  			ssd, sad := stripDocNotes(sd), stripDocNotes(ad)
   364  			m.crossPkg.add(p, ssd != "" && ssd == sad)
   365  		}
   366  	}
   367  
   368  	// The hand-maintained mirror in package simd. This one deliberately spans
   369  	// the whole of both files, residue included: midway owns those docs too.
   370  	m.mirror = new(tally[methodKey])
   371  	stubs, emul := map[methodKey]string{}, map[methodKey]string{}
   372  	for _, d := range api {
   373  		switch d.file {
   374  		case "simd_stubs.go":
   375  			stubs[d.methodKey()] = d.doc
   376  		case "simd_emulated.go":
   377  			emul[d.methodKey()] = d.doc
   378  		}
   379  	}
   380  	for p, s := range stubs {
   381  		if e, ok := emul[p]; ok {
   382  			m.mirror.add(p, s == e)
   383  		}
   384  	}
   385  
   386  	// 7. The inputs being retired.
   387  	cat := makeCategoriesInventory(genDir)
   388  	m.categories = cat.allFields
   389  	m.catEntries = cat.entries
   390  	m.catUniq = cat.uniq
   391  	m.catExported = cat.exported
   392  	m.catCompilerOnly = cat.compilerOnly
   393  	m.catExportedFields = cat.exportedFields
   394  	m.comments = countMidwayComments(genDir)
   395  
   396  	// Comments.yaml entries that spec already covers, by name.
   397  	root := readYAML(commentsPath(genDir))
   398  	eachEntry(root, func(k, v *yaml.Node) {
   399  		if isString(v) && specNames[k.Value] {
   400  			m.commentsSpecCovered++
   401  			m.commentsSpecCoveredNames = append(m.commentsSpecCoveredNames, k.Value)
   402  		}
   403  	})
   404  
   405  	// Commutativity drift: spec vs categories.yaml exported entries.
   406  	m.commutDrift = commutativityDrift(genDir, m.spec)
   407  	return m
   408  }
   409  
   410  // nameRelation classifies one name by how its two receiver sets stand.
   411  func nameRelation(name string, api, spec map[string]bool) nameRel {
   412  	r := nameRel{name: name, api: len(api), spec: len(spec)}
   413  	shared := 0
   414  	for recv := range api {
   415  		if spec[recv] {
   416  			shared++
   417  		} else {
   418  			r.apiOnly = append(r.apiOnly, recv)
   419  		}
   420  	}
   421  	slices.Sort(r.apiOnly)
   422  	r.specOnly = len(spec) - shared
   423  	switch {
   424  	case len(spec) == 0:
   425  		r.rel = relAPIOnly
   426  	case len(api) == 0:
   427  		r.rel = relSpecOnly
   428  	case len(r.apiOnly) == 0 && r.specOnly == 0:
   429  		r.rel = relEqual
   430  	case len(r.apiOnly) == 0:
   431  		r.rel = relSubset
   432  	case r.specOnly == 0:
   433  		r.rel = relSuperset
   434  	case shared == 0:
   435  		r.rel = relDisjoint
   436  	default:
   437  		r.rel = relOverlap
   438  	}
   439  	return r
   440  }
   441  
   442  // commutativityDrift counts exported categories.yaml entries whose commutative field
   443  // disagrees with spec. Some go: values are regexps, so each is tried as one.
   444  func commutativityDrift(genDir string, spec map[methodKey]*specgen.Func) int {
   445  	var n int
   446  	paths, _ := filepath.Glob(filepath.Join(genDir, "simdgen", "ops", "*", "categories.yaml"))
   447  	slices.Sort(paths)
   448  	for _, p := range paths {
   449  		root := readYAML(p)
   450  		if root == nil || root.Kind != yaml.SequenceNode {
   451  			continue
   452  		}
   453  		for _, entry := range root.Content {
   454  			if entry.Kind != yaml.MappingNode {
   455  				continue
   456  			}
   457  			goName := mapValue(entry, "go")
   458  			if goName == nil {
   459  				continue
   460  			}
   461  			isExported := goName.Value != "" && !(goName.Value[0] >= 'a' && goName.Value[0] <= 'z')
   462  			if !isExported || mapValue(entry, "noTypes") != nil {
   463  				continue
   464  			}
   465  			commNode := mapValue(entry, "commutative")
   466  			if commNode == nil {
   467  				continue
   468  			}
   469  			catCommut := commNode.Value == "true"
   470  			re, err := regexp.Compile("^" + goName.Value + "$")
   471  			if err != nil {
   472  				continue
   473  			}
   474  			for p, f := range spec {
   475  				if re.MatchString(p.name) && f.Commutative != catCommut {
   476  					n++
   477  					break
   478  				}
   479  			}
   480  		}
   481  	}
   482  	return n
   483  }
   484  
   485  func filter[T any](s []T, keep func(T) bool) []T {
   486  	var out []T
   487  	for _, v := range s {
   488  		if keep(v) {
   489  			out = append(out, v)
   490  		}
   491  	}
   492  	return out
   493  }
   494  
   495  // specPair returns the (receiver, name) key for a spec function.
   496  func specPair(f *specgen.Func) methodKey {
   497  	var recv string
   498  	if f.Recv.Type != nil {
   499  		recv = f.Recv.Type.String()
   500  	}
   501  	return methodKey{recv, f.Name}
   502  }
   503  
   504  // stripDocNotes returns doc with target-specific implementation notes and
   505  // directives removed, leaving only spec-owned paragraphs.
   506  func stripDocNotes(doc string) string {
   507  	var keptParas []string
   508  	var curPara []string
   509  	flush := func() bool {
   510  		if len(curPara) == 0 {
   511  			return true
   512  		}
   513  		first := strings.TrimSpace(curPara[0])
   514  		if strings.HasPrefix(first, "//go:") || strings.HasPrefix(first, "//line ") {
   515  			return false
   516  		}
   517  		text := strings.TrimPrefix(first, "// ")
   518  		text = strings.TrimPrefix(text, "//")
   519  		if specgen.HasNotePrefix(text) {
   520  			return false
   521  		}
   522  		keptParas = append(keptParas, strings.Join(curPara, "\n"))
   523  		curPara = nil
   524  		return true
   525  	}
   526  
   527  	for _, line := range strings.Split(doc, "\n") {
   528  		trimmed := strings.TrimSpace(line)
   529  		if strings.HasPrefix(trimmed, "//go:") || strings.HasPrefix(trimmed, "//line ") {
   530  			if !flush() {
   531  				return strings.Join(keptParas, "\n//\n")
   532  			}
   533  			break
   534  		}
   535  		if trimmed == "//" {
   536  			if !flush() {
   537  				return strings.Join(keptParas, "\n//\n")
   538  			}
   539  			continue
   540  		}
   541  		curPara = append(curPara, line)
   542  	}
   543  	flush()
   544  	return strings.Join(keptParas, "\n//\n")
   545  }
   546
View as plain text