narrow.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"simd/archsimd/_gen/sgutil"
    10  	"text/template"
    11  )
    12  
    13  // narrowConversion is a narrowing integer conversion that amd64 emulates (see
    14  // converts_emulated_amd64.go).
    15  type narrowConversion struct {
    16  	Base    string // base type of the input, "Int" or "Uint"
    17  	EWidth  int    // element width of the input
    18  	Vwidth  int    // width of the input and result vectors
    19  	OBase   string // base type of the result
    20  	OEWidth int    // element width of the result
    21  	CPU     string // CPU feature the emulation requires
    22  }
    23  
    24  // VType returns the input vector type, e.g., Int16x8.
    25  func (c narrowConversion) VType() string {
    26  	return fmt.Sprintf("%s%dx%d", c.Base, c.EWidth, c.Count())
    27  }
    28  
    29  // Count returns the number of elements of the input.
    30  func (c narrowConversion) Count() int {
    31  	return c.Vwidth / c.EWidth
    32  }
    33  
    34  // OVType returns the result vector type, e.g., Int8x16.
    35  func (c narrowConversion) OVType() string {
    36  	return fmt.Sprintf("%s%dx%d", c.OBase, c.OEWidth, c.Vwidth/c.OEWidth)
    37  }
    38  
    39  // Pack returns the name of the operation that packs two vectors of type
    40  // c.VType into one vector of type c.OVType with saturation. VPACKSSWB and
    41  // VPACKUSWB are internal, since there is no exported concatenating conversion
    42  // to 8-bit elements.
    43  func (c narrowConversion) Pack() string {
    44  	if c.OEWidth == 8 {
    45  		return fmt.Sprintf("concatSaturateTo%s8", c.OBase)
    46  	}
    47  	return fmt.Sprintf("ConcatSaturateTo%s%d", c.OBase, c.OEWidth)
    48  }
    49  
    50  // truncTemplate emulates the 2:1 TruncTo conversions on 128- and 256-bit
    51  // vectors. The signed forms use the unsigned forms. The unsigned forms
    52  // shuffle the low part of each element into place, within each 128-bit group
    53  // at 256 bits.
    54  var truncTemplate = sgutil.TemplateNamed("truncTemplate", `
    55  // Emulated, CPU Feature: {{.CPU}}
    56  func (x {{.VType}}) TruncTo{{.OBase}}{{.OEWidth}}() (z {{.OVType}}) {
    57  {{- if eq .Base "Int"}}
    58  	return x.ToBits().TruncToUint{{.OEWidth}}().BitsToInt{{.OEWidth}}()
    59  {{- else if eq .Vwidth 128}}
    60  {{-  if eq .EWidth 64}}
    61  	var zero Uint32x4
    62  	return x.ReshapeToUint32s().ConcatPermuteScalars(0, 2, 4, 4, zero)
    63  {{-  else}}
    64  	return x.ReshapeToUint8s().LookupModOrZero(LoadInt8x16Array(&trunc{{.EWidth}}To{{.OEWidth}}Idx)){{if ne .OEWidth 8}}.ReshapeToUint{{.OEWidth}}s(){{end}}
    65  {{-  end}}
    66  {{- else}}
    67  	if X86.AVX512() {
    68  		return x.truncToUint{{.OEWidth}}()
    69  	}
    70  {{-  if eq .EWidth 64}}
    71  	var zero Uint32x8
    72  	g := x.ReshapeToUint32s().ConcatPermuteScalarsGrouped(0, 2, 4, 4, zero)
    73  {{-  else}}
    74  	g := x.ReshapeToUint8s().LookupModOrZeroGrouped(LoadInt8x32Array(&trunc{{.EWidth}}To{{.OEWidth}}GroupedIdx))
    75  {{-  end}}
    76  	return gatherLo64(g.ReshapeToUint64s()).ReshapeToUint{{.OEWidth}}s()
    77  {{- end}}
    78  }
    79  `)
    80  
    81  // saturateTemplate emulates the 2:1 SaturateTo conversions on 128- and 256-bit
    82  // vectors.
    83  var saturateTemplate = sgutil.TemplateNamed("saturateTemplate", `
    84  // Emulated, CPU Feature: {{.CPU}}
    85  func (x {{.VType}}) SaturateTo{{.OBase}}{{.OEWidth}}() (z {{.OVType}}) {
    86  {{- if eq .Base "Uint"}}
    87  	if X86.AVX512() {
    88  		return x.saturateToUint{{.OEWidth}}()
    89  	}
    90  {{-  if eq .EWidth 64}}
    91  	// x fits in a uint32 if its high 32 bits are zero.
    92  	var zero {{.VType}}
    93  	hi := LoadInt64x{{.Count}}Array(&maxUint32x{{.Count}}).ToBits()
    94  	return x.IfElse(x.ShiftAllRight(32).Equal(zero), hi).TruncToUint32()
    95  {{-  else}}
    96  	// After the Min, x is non-negative as an int{{.EWidth}}, so the signed-to-unsigned
    97  	// saturation doesn't change it.
    98  	x = x.Min(Load{{.VType}}Array(&maxUint{{.OEWidth}}x{{.Count}}))
    99  	return x.BitsToInt{{.EWidth}}().SaturateToUint{{.OEWidth}}()
   100  {{-  end}}
   101  {{- else if and (eq .EWidth 64) (eq .OBase "Int")}}
   102  	if X86.AVX512() {
   103  		return x.saturateToInt32()
   104  	}
   105  	hi, lo := Load{{.VType}}Array(&maxInt32x{{.Count}}), Load{{.VType}}Array(&minInt32x{{.Count}})
   106  	x = hi.IfElse(x.Greater(hi), x)
   107  	x = lo.IfElse(x.Less(lo), x)
   108  	return x.TruncToInt32()
   109  {{- else if eq .EWidth 64}}
   110  	var zero {{.VType}}
   111  	if X86.AVX512() {
   112  		return x.Max(zero).ToBits().saturateToUint32()
   113  	}
   114  	hi := Load{{.VType}}Array(&maxUint32x{{.Count}})
   115  	x = x.Masked(x.Greater(zero))
   116  	x = hi.IfElse(x.Greater(hi), x)
   117  	return x.ToBits().TruncToUint32()
   118  {{- else if eq .Vwidth 128}}
   119  	var zero {{.VType}}
   120  	return x.{{.Pack}}(zero)
   121  {{- else if eq .OBase "Int"}}
   122  	if X86.AVX512() {
   123  		return x.saturateToInt{{.OEWidth}}()
   124  	}
   125  	var zero {{.VType}}
   126  	g := x.{{.Pack}}Grouped(zero)
   127  	return gatherLo64(g.ToBits().ReshapeToUint64s()).ReshapeToUint{{.OEWidth}}s().BitsToInt{{.OEWidth}}()
   128  {{- else}}
   129  	var zero {{.VType}}
   130  	g := x.{{.Pack}}Grouped(zero)
   131  	return gatherLo64(g.ReshapeToUint64s()).ReshapeToUint{{.OEWidth}}s()
   132  {{- end}}
   133  }
   134  `)
   135  
   136  // saturateSignedToUnsignedTemplate implements the SaturateToUint conversions
   137  // from signed integers that have no AVX-512 instruction and aren't emulated by
   138  // saturateTemplate. These clamp negative values to 0 and then use unsigned
   139  // saturation.
   140  var saturateSignedToUnsignedTemplate = sgutil.TemplateNamed("saturateSignedToUnsignedTemplate", `
   141  // Emulated, CPU Feature: {{.CPU}}
   142  func (x {{.VType}}) SaturateToUint{{.OEWidth}}() (z {{.OVType}}) {
   143  	var zero {{.VType}}
   144  	return x.Max(zero).ToBits().SaturateToUint{{.OEWidth}}()
   145  }
   146  `)
   147  
   148  // narrowConversions writes filename, which contains the amd64 narrowing
   149  // integer conversions that are emulated using other operations.
   150  func narrowConversions(filename string) {
   151  	buf := files.NewGoFile(filename)
   152  	prologue("tmplgen", "amd64", buf)
   153  	exec := func(t *template.Template, c narrowConversion) {
   154  		if err := t.Execute(buf, c); err != nil {
   155  			panic(fmt.Errorf("template execute failed, %v", err))
   156  		}
   157  	}
   158  	cpu := map[int]string{128: "AVX", 256: "AVX2"}
   159  	for _, vw := range []int{128, 256} {
   160  		for _, w := range []int{16, 32, 64} {
   161  			for _, base := range []string{"Int", "Uint"} {
   162  				exec(truncTemplate, narrowConversion{base, w, vw, base, w / 2, cpu[vw]})
   163  			}
   164  		}
   165  	}
   166  	for _, vw := range []int{128, 256} {
   167  		for _, w := range []int{16, 32, 64} {
   168  			exec(saturateTemplate, narrowConversion{"Int", w, vw, "Int", w / 2, cpu[vw]})
   169  			exec(saturateTemplate, narrowConversion{"Int", w, vw, "Uint", w / 2, cpu[vw]})
   170  			exec(saturateTemplate, narrowConversion{"Uint", w, vw, "Uint", w / 2, cpu[vw]})
   171  		}
   172  	}
   173  	for _, ow := range []int{8, 16, 32} {
   174  		for _, w := range []int{16, 32, 64} {
   175  			for _, vw := range []int{128, 256, 512} {
   176  				if w <= ow || (w == 2*ow && vw < 512) {
   177  					// Same or wider result, or emulated by saturateTemplate.
   178  					continue
   179  				}
   180  				exec(saturateSignedToUnsignedTemplate, narrowConversion{"Int", w, vw, "Uint", ow, "AVX512"})
   181  			}
   182  		}
   183  	}
   184  }
   185
View as plain text