go:generate
the story of code generation
Neven Miculinić
Motivation
-
Boilerplate code
-
Missing language features (generic, function decorators)
-
"higher order libraries"
-
automatic language bindings generation
About me
-
2013 - now: CS student
-
master thesis: using machine learning in improving DNA sequencing precision (MinION)
-
-
2017 - now: DevOps at Ozone
-
writing python & go code
-
deploying using kubernetes
-
-
2016 wrote AI Battleground platform
-
open source Java-based platform for autonomous programmed bots playing a predetermined game
-
-
2015: Facebook Summer intern
- worked on ML pipeline for spam detection
Outline
-
Various premade code generator (3 examples)
-
Building our own via template/text (1 example)
-
genny as an alternative to template/text (1 example)
-
Parsing go's
ast and generating adequate code (grand finale)
Rules for generated files
- deterministic
- starting line comment should match regex:
- ^// Code generated .* DO NOT EDIT.$
-
Commited to the repo - Aim to replace various make/bash scripts with
go specific toolchain - Name generated files similar to the input
Stringer
package painkiller
//go:generate stringer -type=Pill
type Pill int
const (
Placebo Pill = iota
Aspirin
Ibuprofen
Paracetamol
Acetaminophen = Paracetamol
)
Stringer
// Code generated by "stringer -type=Pill"; DO NOT EDIT.
package painkiller
import "strconv"
const _Pill_name = "PlaceboAspirinIbuprofenParacetamol"
var _Pill_index = [...]uint8{0, 7, 14, 23, 34}
func (i Pill) String() string {
if i < 0 || i >= Pill(len(_Pill_index)-1) {
return "Pill(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Pill_name[_Pill_index[i]:_Pill_index[i+1]]
}
goderive
package keys
//go:generate goderive .
import (
"strconv"
)
func printMap(m map[string]int) {
for _, k := range deriveSort(deriveKeys(m)) {
println(k + ":" + strconv.Itoa(m[k]))
}
}
goderive
// Code generated by goderive DO NOT EDIT.
package keys
import (
"sort"
)
// deriveSort sorts the slice inplace and also returns it.
func deriveSort(list []string) []string {
sort.Strings(list)
return list
}
// deriveKeys returns the keys of the input map as a slice.
func deriveKeys(m map[string]int) []string {
keys := make([]string, 0, len(m))
for key := range m {
keys = append(keys, key)
}
return keys
}
gRPC demo
syntax = "proto3";
package summer;
option go_package = "summer";
message SumRequest {
repeated int32 a = 1;
}
message SumResponse{
int32 sum = 1;
}
service Math {
rpc Sum(SumRequest) returns(SumResponse);
}
gRPC demo
package summer
/*
go get github.com/gogo/protobuf/protoc-gen-gogoslick
go get github.com/gogo/letmegrpc/protoc-gen-letmegrpc
*/
//go:generate bash -c "protoc -I . --gogoslick_out=plugins=grpc:. --letmegrpc_out=. *.proto"
--letmegrpc_out=. *.proto"
gRPC Demo
Other common tools
xo | generate idiomatic Go code for SQL databases |
go-swagger | a simple yet powerful representation of your RESTful API. |
gotests | Generate Go tests from your source code. |
mockery | A mock code autogenerator for golang |
impl | generates interface stubs |
... | ... |
Build your own
- template/text
- genny
- using go's AST
text/template
text/template
- Mostly used in place of generics
- Or for any kind of simple code generation
- Usually ./bin subfoler holding package main
package template_text
//go:generate bash -c "go run ./bin/* -- $GOFILE"
type FInt interface {
Filter(map[string]int) map[string]int
}
type FString interface {
Filter(map[string]string) map[string]string
}
text/template
// Code generated by bin from example.go. DO NOT EDIT.
package template_text
type EmptyInt struct{}
func (f *EmptyInt) Filter(in map[string]int) map[string]int {
return in
}
type ComposeInt []FInt
func (f *ComposeInt) Filter(in map[string]int) map[string]int {
out := in
for _, filter := range f {
out = filter(out)
}
return out
}
text/template
type EmptyString struct{}
func (f *EmptyString) Filter(in map[string]string) map[string]string {
return in
}
type ComposeString []FString
func (f *ComposeString) Filter(in map[string]string) map[string]string {
out := in
for _, filter := range f {
out = filter(out)
}
return out
}
text/template
const tml = `
type Empty{{ .Name }} struct{}
func (f *Empty{{ .Name }}) Filter(in map[string]{{ .T }}) map[string]{{ .T }} {
return in
}
type Compose{{ .Name }} []{{ .TInterfaceName }}
func (f *Compose{{ .Name }}) Filter(in map[string]{{ .T }}) map[string]{{ .T }} {
out := in
for _, filter := range f {
out = filter(out)
}
return out
}
`
text/template
t, err := template.New("x").Parse(tml)
for _, values := range []struct{
Name string
T string
TInterfaceName string
}{
{
Name:"Int",
T:"int",
TInterfaceName:"FInt",
},
{
Name:"String",
T:"string",
TInterfaceName:"FString",
},
}{
if err := t.Execute(out, values); err != nil {
log.Fatal(err)
}
}
text/template
- Sprig -- many template functions
- https://golang.org/pkg/text/template/
genny
package template_text
//go:generate genny -in=$GOFILE -out=gen-$GOFILE gen "TypeT=string,int"
import "github.com/cheekybits/genny/generic"
type TypeT generic.Type
type EmptyTypeT struct {}
func (* EmptyTypeT) Filter(in map[string]TypeT) map[string]TypeT {
return in
}
genny
// This file was automatically generated by genny.
// Any changes will be lost if this file is regenerated.
// see https://github.com/cheekybits/genny
package template_text
type EmptyString struct{}
func (*EmptyString) Filter(in map[string]string) map[string]string {
return in
}
type EmptyInt struct{}
func (*EmptyInt) Filter(in map[string]int) map[string]int {
return in
}
Filter composition?
go ast
package decorator
import (
_ "context"
xx "context"
)
//go:generate decorator-gen -in=$GOFILE -out=gen-$GOFILE
//@decorate:chan
func Increase(a int, ctx xx.Context) (int,int){
return a + 1, 0
}
go ast
// Code generated by decorator-gen; DO NOT EDIT.
package decorator
import (
xx "context"
)
type IncreaseReq struct {
a int
ctx xx.Context
}
type IncreaseResp struct {
element0 int
element1 int
}
func IncreaseChan(in chan<- IncreaseReq, out <-chan IncreaseResp) {
for it := range in {
element0, element1 := Increase(
it.a,
it.ctx,
)
out <- IncreaseResp{element0, element1}
}
}
go ast
type File struct {
Doc *CommentGroup // associated documentation; or nil
Package token.Pos // position of "package" keyword
Name *Ident // package name
Decls []Decl // top-level declarations; or nil
Scope *Scope // package scope (this file only)
Imports []*ImportSpec // imports in this file
Unresolved []*Ident // unresolved identifiers in this file
Comments []*CommentGroup // list of all comments in the source file
}
go ast
fs := token.NewFileSet()
f, err := parser.ParseFile(fs, *in, nil, parser.ParseComments)
if err != nil {
log.Fatal(err)
}
imports := make(map[string]*ast.ImportSpec, len(f.Imports))
for _, x := range f.Imports {
imports[x.Name.Name] = x
}
usedImports := make(map[string]bool)
elems := make([]string, 0)
go ast
// A FuncDecl node represents a function declaration.
FuncDecl struct {
Doc *CommentGroup // associated documentation; or nil
Recv *FieldList // receiver (methods); or nil (functions)
Name *Ident // function/method name
Type *FuncType // function signature: parameters, results, and position of "func" keyword
Body *BlockStmt // function body; or nil for external (non-Go) function
}
FuncType struct {
Func token.Pos // position of "func" keyword (token.NoPos if there is no "func")
Params *FieldList // (incoming) parameters; non-nil
Results *FieldList // (outgoing) results; or nil
}
// A FieldList represents a list of Fields, enclosed by parentheses or braces.
type FieldList struct {
Opening token.Pos // position of opening parenthesis/brace, if any
List []*Field // field list; or nil
Closing token.Pos // position of closing parenthesis/brace, if any
}
go ast
for _, decl := range f.Decls {
switch decl.(type) {
case *ast.FuncDecl:
decl := decl.(*ast.FuncDecl) // For better autocompletion
if decl.Doc == nil {
continue
}
for _, comm := range decl.Doc.List {
if strings.HasPrefix(comm.Text, "//@decorate:chan") {
elems = append(elems, handleFunc(decl, fs, usedImports, t, quiet))
}
}
default:
}
}
go ast
// Code generated by decorator-gen; DO NOT EDIT.
package decorator
import (
xx "context"
)
type IncreaseReq struct {
a int
ctx xx.Context
}
type IncreaseResp struct {
element0 int
element1 int
}
func IncreaseChan(in chan<- IncreaseReq, out <-chan IncreaseResp) {
for it := range in {
element0, element1 := Increase(
it.a,
it.ctx,
)
out <- IncreaseResp{element0, element1}
}
}
go ast
func handleFunc(
decl *ast.FuncDecl,
fs *token.FileSet,
usedImports map[string]bool,
t *template.Template,
quiet *bool) string {
reqStructName := decl.Name.Name + "Req"
respStructName := decl.Name.Name + "Resp"
out := &bytes.Buffer{}
reqStruct, reqElems := makeStruct(
decl.Type.Params, reqStructName, fs, usedImports
)
respStruct, respElems := makeStruct(
decl.Type.Results, respStructName, fs, usedImports
)
fmt.Fprintln(out, reqStruct)
fmt.Fprintln(out, respStruct)
t.Execute(out, TemplateValues{
Name: decl.Name.Name,
Results: respElems,
Args: reqElems,
})
return out.String()
}
go ast
// Code generated by decorator-gen; DO NOT EDIT.
package decorator
import (
xx "context"
)
type IncreaseReq struct {
a int
ctx xx.Context
}
type IncreaseResp struct {
element0 int
element1 int
}
func IncreaseChan(in chan<- IncreaseReq, out <-chan IncreaseResp) {
for it := range in {
element0, element1 := Increase(
it.a,
it.ctx,
)
out <- IncreaseResp{element0, element1}
}
}
go ast
// makeStruct created struct with all required fields in the FieldList
func makeStruct(
fields *ast.FieldList,
structName string,
fs *token.FileSet,
usedImports map[string]bool,
) (string, []string) {
s := &bytes.Buffer{}
fmt.Fprintf(s, "type %s struct {\n", structName)
ListNames := make([]string, 0, len(fields.List))
for i, inp := range fields.List {
names := []string{}
if len(inp.Names) == 0 {
names = append(names, fmt.Sprintf("element%d", i))
}
for _, name := range inp.Names {
names = append(names, name.Name)
}
ListNames = append(ListNames, names...)
tBuff := &bytes.Buffer{}
printer.Fprint(tBuff, fs, inp.Type)
typeStr := tBuff.String()
fmt.Fprintf(s, "%s %s", strings.Join(names, ","), typeStr)
if strings.Contains(typeStr, ".") {
usedImports[strings.Split(typeStr, ".")[0]] = true
}
}
fmt.Fprintln(s, "}")
return s.String(), ListNames
}
go ast
// Code generated by decorator-gen; DO NOT EDIT.
package decorator
import (
xx "context"
)
type IncreaseReq struct {
a int
ctx xx.Context
}
type IncreaseResp struct {
element0 int
element1 int
}
func IncreaseChan(in chan<- IncreaseReq, out <-chan IncreaseResp) {
for it := range in {
element0, element1 := Increase(
it.a,
it.ctx,
)
out <- IncreaseResp{element0, element1}
}
}
go ast
const tmp = `
func {{ .Name }}Chan(in chan<- {{.Name}}Req, out <-chan {{ .Name }}Resp) {
for it := range in {
{{ .Results | join "," }} := {{ .Name }}(
{{- range $index, $element := .Args }}
it.{{ $element }},
{{- end }}
)
out <- {{ .Name }}Resp{ {{- .Results | join "," -}} }
}
}
`
func handleFunc(...) string {
...
t.Execute(out, TemplateValues{
Name: decl.Name.Name,
Results: respElems,
Args: reqElems,
})
return out.String()
}
go ast
// Final printing
b := &bytes.Buffer{}
fmt.Fprintf(b, "// Code generated by decorator-gen; DO NOT EDIT.\npackage ")
printer.Fprint(b, fs, f.Name)
fmt.Fprintln(b)
fmt.Fprintln(b, "import (")
for imp := range usedImports {
printer.Fprint(b, fs, imports[imp])
fmt.Fprintln(b)
}
fmt.Fprintln(b, ")")
for _, x := range elems {
fmt.Fprintln(b, x)
}
go ast
// Runing through go-fmt
s, err := format.Source(b.Bytes())
// bunch of error handling later
if err != nil {
log.Fatal(err)
}
outFile, err := os.Create(*out)
if err != nil {
log.Fatal(err)
}
if _, err := outFile.Write(s); err != nil {
outFile.Close()
log.Fatal(err)
}
if err := outFile.Close(); err != nil {
log.Fatal(err)
}
go ast
// Code generated by decorator-gen; DO NOT EDIT.
package decorator
import (
xx "context"
)
type IncreaseReq struct {
a int
ctx xx.Context
}
type IncreaseResp struct {
element0 int
element1 int
}
func IncreaseChan(in chan<- IncreaseReq, out <-chan IncreaseResp) {
for it := range in {
element0, element1 := Increase(
it.a,
it.ctx,
)
out <- IncreaseResp{element0, element1}
}
}
Takeaway
-
DRY
-
Reduce boilerplate code
-
Or even have pregenerated boilerplate
-
Filling the gap for generics/decorator/...
Acknowledgments
-
Matej Baćo
-
Davor Kapša
For reviewing the slides, providing constructive feedback and ideas for this talk
Q & A
- https://github.com/nmiculinic/go-generate-talk
- https://slides.com/nmiculinic/go-generate
References part 1
- https://github.com/nmiculinic/go-generate-talk
- https://www.factorio.com/
- https://godoc.org/golang.org/x/tools/cmd/stringer
- https://github.com/awalterschulze/goderive
- https://github.com/gogo/letmegrpc
- https://github.com/gogo/protobuf/
- https://github.com/vektra/mockery
- https://github.com/xo/xo
- https://github.com/cweill/gotests
- https://github.com/josharian/impl
- https://github.com/go-swagger/go-swagger
- https://jeffreykegler.github.io/personal/timeline_v3
- https://golang.org/pkg/text/template/
- https://github.com/Masterminds/sprig
References part 2
- https://github.com/kulshekhar/fungen
- https://github.com/cheekybits/genny
- https://golang.org/pkg/text/template/
- https://golang.org/cmd/go/#hdr-Generate_Go_files_by_processing_source
- justforfunc #25: deeper program analysis with go/parser (https://youtu.be/YRWCa84pykM)
- https://github.com/aibg
go: generate
By Neven Miculinić
go: generate
- 534