input.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"cmp"
     9  	"errors"
    10  	"go/ast"
    11  	"go/token"
    12  	"go/types"
    13  	"io/fs"
    14  	"log"
    15  	"os"
    16  	"path/filepath"
    17  	"slices"
    18  	"strings"
    19  
    20  	"golang.org/x/tools/go/packages"
    21  	"gopkg.in/yaml.v3"
    22  
    23  	"simd/archsimd/_gen/gentools"
    24  )
    25  
    26  // buildConfigs are the configurations we report API coverage for. The API is spread
    27  // across files selected by build constraints, and the only authority on which
    28  // files belong to which configuration is the go command -- so rather than
    29  // interpreting //go:build lines here, we ask go/packages once per configuration
    30  // and use the file set it hands back.
    31  //
    32  // SVE is a target in SPEC-TRANSITION.md's sense but not a GOARCH: its files are
    33  // built for arm64 like NEON's. It separates out along the generator dimension
    34  // instead, because simdgen records its -arch in the generated-by header.
    35  var buildConfigs = []struct{ name, goos, goarch string }{
    36  	{"amd64", "linux", "amd64"},
    37  	{"arm64", "linux", "arm64"},
    38  	{"wasm", "js", "wasm"},
    39  	// Package simd falls back to a pure-Go implementation on architectures with
    40  	// no SIMD support. 386 is an arbitrary representative of those.
    41  	{"portable", "linux", "386"},
    42  }
    43  
    44  // A decl is one exported function or method declaration in the public API.
    45  type decl struct {
    46  	pkg     string   // "simd" or "archsimd"
    47  	file    string   // path relative to the simd directory
    48  	gen     string   // generator that emitted it, or "hand-written"
    49  	targets []string // names of build configurations it compiles in
    50  	recv    string   // receiver type name, empty for plain functions
    51  	name    string
    52  	in      []param
    53  	out     []param
    54  
    55  	doc    string // doc comment text, comment markers stripped
    56  	hasSep bool   // doc contains a bare "//" separator line
    57  }
    58  
    59  type param struct{ name, typ string }
    60  
    61  // The grains the report counts at, coarsest last. Spec describes an operation
    62  // once, the API declares it once per target, and a name collapses receivers
    63  // too; the gaps between the three are diagnostic (SPEC-TRANSITION.md §1.2).
    64  type (
    65  	// methodKey is the (receiver, name) key spec is unique on.
    66  	methodKey struct{ recv, name string }
    67  
    68  	// archMethodKey is a method as it appears in one build configuration.
    69  	//
    70  	// It carries the package because simd and archsimd share their scalable
    71  	// type names.
    72  	archMethodKey struct {
    73  		target string
    74  		pkg    string
    75  		methodKey
    76  	}
    77  
    78  	// declKey identifies one declaration in the source, which is what collapses
    79  	// the several configurations it may build in.
    80  	declKey struct {
    81  		file string
    82  		methodKey
    83  	}
    84  )
    85  
    86  // A fact is one declaration as seen in one build configuration: the grain of
    87  // the fact table, and the row type every figure in the report pivots over.
    88  type fact struct {
    89  	*decl
    90  	target string
    91  }
    92  
    93  func (k methodKey) String() string { return k.recv + "." + k.name }
    94  func (k declKey) String() string   { return k.file + ": " + k.methodKey.String() }
    95  
    96  func (d *decl) methodKey() methodKey        { return methodKey{d.recv, d.name} }
    97  func (f fact) declKey() declKey             { return declKey{f.file, f.methodKey()} }
    98  func (f fact) archMethodKey() archMethodKey { return archMethodKey{f.target, f.pkg, f.methodKey()} }
    99  
   100  // tool strips the invocation from a generator label, so that simdgen's three
   101  // runs count as one tool.
   102  func (f fact) tool() string { t, _, _ := strings.Cut(f.gen, "/"); return t }
   103  
   104  // nonSpecOp reports whether name is an operation that will never come from spec:
   105  // specifically architecture bridge operations ToArch and FromArch.
   106  // They are excluded from the fact table, and
   107  // so from every denominator. See SPEC-TRANSITION.md §2.7.
   108  func nonSpecOp(name string) bool {
   109  	return name == "ToArch" || strings.HasSuffix(name, "FromArch")
   110  }
   111  
   112  // nonSpecDecl reports whether d is a declaration that will never come from
   113  // spec, so that it is excluded from the fact table and every denominator:
   114  // a [nonSpecOp], or a declaration that is not a vector operation at all.
   115  // These are:
   116  //
   117  //   - CPU feature checks, the methods of archsimd.X86Features and
   118  //     archsimd.ARM64Features (such as X86.AVX512 and ARM64.SVE);
   119  //   - package simd's queries about the implementation, Emulated, VectorBitSize
   120  //     and HasHardwareCarrylessMultiply, which the generator README lists as
   121  //     the one addition to the intersection of the archsimd APIs;
   122  //   - archsimd.ClearAVXUpperBits, a hint to the hardware with no operands;
   123  //   - the amd64 Merge methods, which are deprecated in favor of IfElse, which
   124  //     spec covers.
   125  func nonSpecDecl(d *decl) bool {
   126  	switch {
   127  	case nonSpecOp(d.name):
   128  		return true
   129  	case d.recv == "X86Features" || d.recv == "ARM64Features":
   130  		return true
   131  	case d.recv == "" && nonSpecFuncs[d.name]:
   132  		return true
   133  	case d.recv != "" && d.name == "Merge":
   134  		return true
   135  	}
   136  	return false
   137  }
   138  
   139  var nonSpecFuncs = map[string]bool{
   140  	"Emulated":                     true,
   141  	"VectorBitSize":                true,
   142  	"HasHardwareCarrylessMultiply": true,
   143  	"ClearAVXUpperBits":            true,
   144  }
   145  
   146  // loadOverlay builds a packages.Config.Overlay map from the scratch overlay
   147  // directory if one is active.
   148  func loadOverlay(genOpts *gentools.Options) map[string][]byte {
   149  	if genOpts == nil {
   150  		return nil
   151  	}
   152  	dir := genOpts.OverlayDir()
   153  	if dir == "" {
   154  		return nil
   155  	}
   156  	root := filepath.Join(dir, "src")
   157  	overlay := make(map[string][]byte)
   158  	_ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
   159  		if err != nil || d.IsDir() {
   160  			return nil
   161  		}
   162  		rel, err := filepath.Rel(root, path)
   163  		if err != nil {
   164  			return nil
   165  		}
   166  		slashRel := filepath.ToSlash(rel)
   167  		if !strings.HasPrefix(slashRel, "simd/") {
   168  			return nil
   169  		}
   170  		b, err := genOpts.ReadFile(rel)
   171  		if err != nil {
   172  			return nil
   173  		}
   174  		gorootPath := filepath.Join(genOpts.GOROOT, "src", filepath.FromSlash(rel))
   175  		overlay[gorootPath] = b
   176  		return nil
   177  	})
   178  	return overlay
   179  }
   180  
   181  func loadAPI(genOpts *gentools.Options, simdDir string) []*decl {
   182  	fset := token.NewFileSet()
   183  	byKey := map[declKey]*decl{}
   184  	overlay := loadOverlay(genOpts)
   185  
   186  	for _, bc := range buildConfigs {
   187  		cfg := &packages.Config{
   188  			// Syntax only: we compare declarations across configurations, which
   189  			// type-checking would not help with and would slow down. See
   190  			// SPEC-TRANSITION.md §2.4.
   191  			Mode:    packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedSyntax,
   192  			Dir:     simdDir,
   193  			Fset:    fset,
   194  			Env:     append(os.Environ(), "GOOS="+bc.goos, "GOARCH="+bc.goarch, "GOEXPERIMENT=simd"),
   195  			Overlay: overlay,
   196  		}
   197  		pkgs, err := packages.Load(cfg, ".", "./archsimd")
   198  		if err != nil {
   199  			log.Fatalf("loading simd packages for %s/%s: %s", bc.goos, bc.goarch, err)
   200  		}
   201  		for _, p := range pkgs {
   202  			for _, e := range p.Errors {
   203  				log.Fatalf("loading %s for %s/%s: %s", p.PkgPath, bc.goos, bc.goarch, e)
   204  			}
   205  			for _, f := range p.Syntax {
   206  				file := fset.Position(f.Pos()).Filename
   207  				if rel, err := filepath.Rel(simdDir, file); err == nil {
   208  					file = filepath.ToSlash(rel)
   209  				}
   210  				gen := generatorOf(f)
   211  				for _, d := range f.Decls {
   212  					fd, ok := d.(*ast.FuncDecl)
   213  					if !ok || !fd.Name.IsExported() {
   214  						continue
   215  					}
   216  					dec := declOf(fd, p.Name, file, gen)
   217  					key := declKey{file, dec.methodKey()}
   218  					if prev, ok := byKey[key]; ok {
   219  						dec = prev
   220  					} else {
   221  						byKey[key] = dec
   222  					}
   223  					dec.targets = append(dec.targets, bc.name)
   224  				}
   225  			}
   226  		}
   227  	}
   228  
   229  	decls := make([]*decl, 0, len(byKey))
   230  	for _, d := range byKey {
   231  		slices.Sort(d.targets)
   232  		d.targets = slices.Compact(d.targets)
   233  		decls = append(decls, d)
   234  	}
   235  	slices.SortFunc(decls, func(a, b *decl) int {
   236  		return cmp.Or(cmp.Compare(a.file, b.file), cmp.Compare(a.recv, b.recv), cmp.Compare(a.name, b.name))
   237  	})
   238  	return decls
   239  }
   240  
   241  func declOf(fd *ast.FuncDecl, pkg, file, gen string) *decl {
   242  	d := &decl{pkg: pkg, file: file, gen: gen, name: fd.Name.Name}
   243  	if fd.Recv != nil && len(fd.Recv.List) == 1 {
   244  		// Receivers in this API are always by value and never generic.
   245  		if id, ok := fd.Recv.List[0].Type.(*ast.Ident); ok {
   246  			d.recv = id.Name
   247  		} else {
   248  			d.recv = types.ExprString(fd.Recv.List[0].Type)
   249  		}
   250  	}
   251  	d.in = fields(fd.Type.Params)
   252  	d.out = fields(fd.Type.Results)
   253  	if fd.Doc != nil {
   254  		var lines []string
   255  		for _, c := range fd.Doc.List {
   256  			lines = append(lines, c.Text)
   257  			if strings.TrimSpace(c.Text) == "//" {
   258  				d.hasSep = true
   259  			}
   260  		}
   261  		d.doc = strings.Join(lines, "\n")
   262  	}
   263  	return d
   264  }
   265  
   266  // fields flattens a parameter or result list, preserving declaration order and
   267  // expanding grouped names ("x, y Vec") into one param each.
   268  func fields(fl *ast.FieldList) []param {
   269  	if fl == nil {
   270  		return nil
   271  	}
   272  	var ps []param
   273  	for _, f := range fl.List {
   274  		typ := types.ExprString(f.Type)
   275  		if len(f.Names) == 0 {
   276  			ps = append(ps, param{"", typ})
   277  			continue
   278  		}
   279  		for _, n := range f.Names {
   280  			ps = append(ps, param{n.Name, typ})
   281  		}
   282  	}
   283  	return ps
   284  }
   285  
   286  // generatorOf returns the tool named in a "Code generated by" header, or
   287  // "hand-written" if there is none.
   288  //
   289  // Since simdgen runs separately per target, for simdgen this returns
   290  // "simdgen/<arch>", where arch is the -arch flag passed to simdgen.
   291  func generatorOf(f *ast.File) string {
   292  	for _, cg := range f.Comments {
   293  		for _, c := range cg.List {
   294  			if !strings.Contains(c.Text, "Code generated by") {
   295  				continue
   296  			}
   297  			for _, tool := range []string{"simdgen", "tmplgen", "wasmgen", "midway", "refgen"} {
   298  				if !strings.Contains(c.Text, tool) {
   299  					continue
   300  				}
   301  				if args := strings.Fields(c.Text); tool == "simdgen" {
   302  					for i, a := range args {
   303  						if a == "-arch" && i+1 < len(args) {
   304  							return tool + "/" + args[i+1]
   305  						}
   306  					}
   307  				}
   308  				return tool
   309  			}
   310  			return "generated"
   311  		}
   312  		break // only the leading comment block can carry the header
   313  	}
   314  	return "hand-written"
   315  }
   316  
   317  // readYAML parses one YAML document into a node tree and returns its root
   318  // content node. The generators' inputs carry custom tags (!sum, !string, and
   319  // the merge keys in comments.yaml), but a node tree keeps a tag as data rather
   320  // than trying to resolve it, so this reads them without knowing their schemas.
   321  //
   322  // A missing file counts as empty: these inputs are deleted by comments-yaml and cat-delete,
   323  // and their figures should read zero rather than crash once they are gone. A
   324  // malformed file is fatal, because that is a real problem with the input.
   325  func readYAML(path string) *yaml.Node {
   326  	b, err := os.ReadFile(path)
   327  	if errors.Is(err, fs.ErrNotExist) {
   328  		return nil
   329  	} else if err != nil {
   330  		log.Fatal(err)
   331  	}
   332  	var doc yaml.Node
   333  	if err := yaml.Unmarshal(b, &doc); err != nil {
   334  		log.Fatalf("%s: %s", path, err)
   335  	}
   336  	if doc.Kind == yaml.DocumentNode && len(doc.Content) == 1 {
   337  		return doc.Content[0]
   338  	}
   339  	return &doc
   340  }
   341  
   342  // eachEntry calls fn for every key/value pair of every mapping under n, depth
   343  // first.
   344  //
   345  // It does not follow aliases, so an entry merged into several mappings with
   346  // "<<: *anchor" is counted once, where it is defined, rather than once per
   347  // merge.
   348  func eachEntry(n *yaml.Node, fn func(key, val *yaml.Node)) {
   349  	if n == nil || n.Kind == yaml.AliasNode {
   350  		return
   351  	}
   352  	if n.Kind == yaml.MappingNode {
   353  		for i := 0; i+1 < len(n.Content); i += 2 {
   354  			fn(n.Content[i], n.Content[i+1])
   355  		}
   356  	}
   357  	for _, c := range n.Content {
   358  		eachEntry(c, fn)
   359  	}
   360  }
   361  
   362  // mapValue returns the value of key in a mapping node, or nil.
   363  func mapValue(n *yaml.Node, key string) *yaml.Node {
   364  	if n == nil || n.Kind != yaml.MappingNode {
   365  		return nil
   366  	}
   367  	for i := 0; i+1 < len(n.Content); i += 2 {
   368  		if n.Content[i].Value == key {
   369  			return n.Content[i+1]
   370  		}
   371  	}
   372  	return nil
   373  }
   374  
   375  // isString reports whether n is a string scalar, which in these files means a
   376  // documentation string. Block scalars ("|-") and quoted scalars both qualify.
   377  func isString(n *yaml.Node) bool {
   378  	return n != nil && n.Kind == yaml.ScalarNode && n.ShortTag() == "!!str"
   379  }
   380  
   381  // categoriesInventory holds the inventory of categories.yaml.
   382  type categoriesInventory struct {
   383  	allFields      map[string]int // field → count across all entries
   384  	exportedFields map[string]int // field → count across exported entries only
   385  	entries        int
   386  	uniq           int
   387  	exported       int
   388  	compilerOnly   int
   389  }
   390  
   391  func makeCategoriesInventory(genDir string) categoriesInventory {
   392  	r := categoriesInventory{
   393  		allFields:      map[string]int{},
   394  		exportedFields: map[string]int{},
   395  	}
   396  	names := map[string]bool{}
   397  	paths, _ := filepath.Glob(filepath.Join(genDir, "simdgen", "ops", "*", "categories.yaml"))
   398  	slices.Sort(paths)
   399  	for _, p := range paths {
   400  		root := readYAML(p)
   401  		if root == nil || root.Kind != yaml.SequenceNode {
   402  			continue
   403  		}
   404  		for _, entry := range root.Content {
   405  			if entry.Kind != yaml.MappingNode {
   406  				continue
   407  			}
   408  			goName := mapValue(entry, "go")
   409  			if goName == nil {
   410  				continue
   411  			}
   412  			r.entries++
   413  			names[goName.Value] = true
   414  			isExported := goName.Value != "" && !(goName.Value[0] >= 'a' && goName.Value[0] <= 'z')
   415  			hasNoTypes := mapValue(entry, "noTypes") != nil
   416  			if !isExported || hasNoTypes {
   417  				r.compilerOnly++
   418  			}
   419  			eachEntry(entry, func(k, _ *yaml.Node) {
   420  				r.allFields[k.Value]++
   421  				if isExported && !hasNoTypes {
   422  					r.exportedFields[k.Value]++
   423  				}
   424  			})
   425  		}
   426  	}
   427  	for n := range names {
   428  		r.uniq++
   429  		if n != "" && !(n[0] >= 'a' && n[0] <= 'z') {
   430  			r.exported++
   431  		}
   432  	}
   433  	return r
   434  }
   435  
   436  // countMidwayComments counts the doc-bearing entries in midway/comments.yaml:
   437  // every key whose value is a string, wherever it appears.
   438  func countMidwayComments(genDir string) (n int) {
   439  	eachEntry(readYAML(commentsPath(genDir)), func(_, v *yaml.Node) {
   440  		if isString(v) {
   441  			n++
   442  		}
   443  	})
   444  	return n
   445  }
   446  
   447  // commonMethods returns the method names defined in simd_emulated.go,
   448  // less [nonSpecDecl]. That set is spec-common's scope, and the portable
   449  // core a new architecture implements first.
   450  func commonMethods(api []*decl) []string {
   451  	seen := map[string]bool{}
   452  	var names []string
   453  	for _, d := range api {
   454  		if d.file == "simd_emulated.go" && d.recv != "" && !nonSpecDecl(d) {
   455  			if !seen[d.name] {
   456  				seen[d.name] = true
   457  				names = append(names, d.name)
   458  			}
   459  		}
   460  	}
   461  	slices.Sort(names)
   462  	return names
   463  }
   464  
   465  func commentsPath(genDir string) string {
   466  	return filepath.Join(genDir, "midway", "comments.yaml")
   467  }
   468
View as plain text