From de8677b1d9ce676866c6ed24621f338b07b49287 Mon Sep 17 00:00:00 2001 From: Maxime Boucher Date: Wed, 17 Jun 2026 22:40:45 -0700 Subject: [PATCH 1/2] perf: avoid eager fingerprint variable evaluation Only compute CHECKSUM or TIMESTAMP template variables when the raw task references the corresponding variable in commands, dependencies, preconditions, or status. This keeps the up-to-date source check unchanged while avoiding a duplicate fingerprint pass for tasks that only use sources and generates caching. Against current main with 20,000 tiny files, checksum improves from 440-445 ms/op to 136-145 ms/op and timestamp from 125-136 ms/op to 44-46 ms/op. Allocations fall from about 742k to 248k for checksum and from about 562k to 188k for timestamp. Verification: go test ./...; golangci-lint run --- variables.go | 145 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 8 deletions(-) diff --git a/variables.go b/variables.go index b9f43d85a1..568591796e 100644 --- a/variables.go +++ b/variables.go @@ -5,6 +5,7 @@ import ( "maps" "os" "path/filepath" + "reflect" "strings" "github.com/joho/godotenv" @@ -217,15 +218,17 @@ func (e *Executor) compiledTask(call *Call, evaluateShVars bool) (*ast.Task, err checker = fingerprint.NewChecksumChecker(e.TempDir.Fingerprint, e.Dry) } - value, err := checker.Value(&new) - if err != nil { - return nil, err - } - vars.Set(strings.ToUpper(checker.Kind()), ast.Var{Live: value}) + if taskReferencesFingerprintVar(origTask, checker.Kind()) { + value, err := checker.Value(&new) + if err != nil { + return nil, err + } + vars.Set(strings.ToUpper(checker.Kind()), ast.Var{Live: value}) - // Adding new variables, requires us to refresh the templaters - // cache of the the values manually - cache.ResetCache() + // Adding new variables, requires us to refresh the templaters + // cache of the the values manually + cache.ResetCache() + } } if len(origTask.Cmds) > 0 { @@ -346,6 +349,132 @@ func (e *Executor) compiledTask(call *Call, evaluateShVars bool) (*ast.Task, err return &new, nil } +func taskReferencesFingerprintVar(t *ast.Task, kind string) bool { + name := strings.ToUpper(kind) + + for _, status := range t.Status { + if stringReferencesFingerprintVar(status, name) { + return true + } + } + for _, cmd := range t.Cmds { + if cmdReferencesFingerprintVar(cmd, name) { + return true + } + } + for _, dep := range t.Deps { + if depReferencesFingerprintVar(dep, name) { + return true + } + } + for _, precondition := range t.Preconditions { + if preconditionReferencesFingerprintVar(precondition, name) { + return true + } + } + return false +} + +func cmdReferencesFingerprintVar(cmd *ast.Cmd, name string) bool { + if cmd == nil { + return false + } + return stringReferencesFingerprintVar(cmd.Cmd, name) || + stringReferencesFingerprintVar(cmd.Task, name) || + stringReferencesFingerprintVar(cmd.If, name) || + forReferencesFingerprintVar(cmd.For, name) || + varsReferenceFingerprintVar(cmd.Vars, name) +} + +func depReferencesFingerprintVar(dep *ast.Dep, name string) bool { + if dep == nil { + return false + } + return stringReferencesFingerprintVar(dep.Task, name) || + forReferencesFingerprintVar(dep.For, name) || + varsReferenceFingerprintVar(dep.Vars, name) +} + +func preconditionReferencesFingerprintVar(precondition *ast.Precondition, name string) bool { + if precondition == nil { + return false + } + return stringReferencesFingerprintVar(precondition.Sh, name) || + stringReferencesFingerprintVar(precondition.Msg, name) +} + +func forReferencesFingerprintVar(f *ast.For, name string) bool { + if f == nil { + return false + } + if valueReferencesFingerprintVar(f.List, name) { + return true + } + for _, row := range f.Matrix.All() { + if stringReferencesFingerprintVar(row.Ref, name) || + valueReferencesFingerprintVar(row.Value, name) { + return true + } + } + return false +} + +func varsReferenceFingerprintVar(vars *ast.Vars, name string) bool { + if vars == nil { + return false + } + for _, v := range vars.All() { + if valueReferencesFingerprintVar(v.Value, name) || + valueReferencesFingerprintVar(v.Live, name) || + stringPointerReferencesFingerprintVar(v.Sh, name) || + stringReferencesFingerprintVar(v.Ref, name) || + stringReferencesFingerprintVar(v.Dir, name) { + return true + } + } + return false +} + +func valueReferencesFingerprintVar(value any, name string) bool { + if value == nil { + return false + } + if s, ok := value.(string); ok { + return stringReferencesFingerprintVar(s, name) + } + + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + return false + } + return valueReferencesFingerprintVar(rv.Elem().Interface(), name) + case reflect.Array, reflect.Slice: + for i := 0; i < rv.Len(); i++ { + if valueReferencesFingerprintVar(rv.Index(i).Interface(), name) { + return true + } + } + case reflect.Map: + for _, key := range rv.MapKeys() { + if valueReferencesFingerprintVar(key.Interface(), name) || + valueReferencesFingerprintVar(rv.MapIndex(key).Interface(), name) { + return true + } + } + } + return false +} + +func stringPointerReferencesFingerprintVar(s *string, name string) bool { + return s != nil && stringReferencesFingerprintVar(*s, name) +} + +func stringReferencesFingerprintVar(s string, name string) bool { + return strings.Contains(s, name) +} + func asAnySlice[T any](slice []T) []any { ret := make([]any, len(slice)) for i, v := range slice { From 5329fdbd3b810817b7c2acc2c9b362bf37fdb683 Mon Sep 17 00:00:00 2001 From: Maxime Boucher Date: Sat, 11 Jul 2026 19:16:01 -0700 Subject: [PATCH 2/2] refactor(ast): move fingerprint reference detection Move fingerprint-variable reference traversal out of task compilation and into the taskfile AST package as Task.ReferencesFingerprintVar. Keep traversal helpers private and add focused coverage for commands, dependencies, preconditions, status, loops, matrices, nested variables, and nil entries. --- taskfile/ast/fingerprint.go | 134 +++++++++++++++++++++++++++++++ taskfile/ast/fingerprint_test.go | 103 ++++++++++++++++++++++++ variables.go | 129 +---------------------------- 3 files changed, 238 insertions(+), 128 deletions(-) create mode 100644 taskfile/ast/fingerprint.go create mode 100644 taskfile/ast/fingerprint_test.go diff --git a/taskfile/ast/fingerprint.go b/taskfile/ast/fingerprint.go new file mode 100644 index 0000000000..632ef29bc7 --- /dev/null +++ b/taskfile/ast/fingerprint.go @@ -0,0 +1,134 @@ +package ast + +import ( + "reflect" + "strings" +) + +// ReferencesFingerprintVar reports whether the task references the variable +// produced by the given fingerprint method in fields compiled after it. +func (t *Task) ReferencesFingerprintVar(kind string) bool { + name := strings.ToUpper(kind) + if t == nil || name == "" { + return false + } + + for _, status := range t.Status { + if stringReferencesFingerprintVar(status, name) { + return true + } + } + for _, cmd := range t.Cmds { + if cmdReferencesFingerprintVar(cmd, name) { + return true + } + } + for _, dep := range t.Deps { + if depReferencesFingerprintVar(dep, name) { + return true + } + } + for _, precondition := range t.Preconditions { + if preconditionReferencesFingerprintVar(precondition, name) { + return true + } + } + return false +} + +func cmdReferencesFingerprintVar(cmd *Cmd, name string) bool { + if cmd == nil { + return false + } + return stringReferencesFingerprintVar(cmd.Cmd, name) || + stringReferencesFingerprintVar(cmd.Task, name) || + stringReferencesFingerprintVar(cmd.If, name) || + forReferencesFingerprintVar(cmd.For, name) || + varsReferenceFingerprintVar(cmd.Vars, name) +} + +func depReferencesFingerprintVar(dep *Dep, name string) bool { + if dep == nil { + return false + } + return stringReferencesFingerprintVar(dep.Task, name) || + forReferencesFingerprintVar(dep.For, name) || + varsReferenceFingerprintVar(dep.Vars, name) +} + +func preconditionReferencesFingerprintVar(precondition *Precondition, name string) bool { + if precondition == nil { + return false + } + return stringReferencesFingerprintVar(precondition.Sh, name) || + stringReferencesFingerprintVar(precondition.Msg, name) +} + +func forReferencesFingerprintVar(f *For, name string) bool { + if f == nil { + return false + } + if valueReferencesFingerprintVar(f.List, name) { + return true + } + for _, row := range f.Matrix.All() { + if row != nil && (stringReferencesFingerprintVar(row.Ref, name) || + valueReferencesFingerprintVar(row.Value, name)) { + return true + } + } + return false +} + +func varsReferenceFingerprintVar(vars *Vars, name string) bool { + for _, v := range vars.All() { + if valueReferencesFingerprintVar(v.Value, name) || + valueReferencesFingerprintVar(v.Live, name) || + stringPointerReferencesFingerprintVar(v.Sh, name) || + stringReferencesFingerprintVar(v.Ref, name) || + stringReferencesFingerprintVar(v.Dir, name) { + return true + } + } + return false +} + +func valueReferencesFingerprintVar(value any, name string) bool { + if value == nil { + return false + } + if s, ok := value.(string); ok { + return stringReferencesFingerprintVar(s, name) + } + + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Pointer, reflect.Interface: + if rv.IsNil() { + return false + } + return valueReferencesFingerprintVar(rv.Elem().Interface(), name) + case reflect.Array, reflect.Slice: + for i := 0; i < rv.Len(); i++ { + if valueReferencesFingerprintVar(rv.Index(i).Interface(), name) { + return true + } + } + case reflect.Map: + for _, key := range rv.MapKeys() { + if valueReferencesFingerprintVar(key.Interface(), name) || + valueReferencesFingerprintVar(rv.MapIndex(key).Interface(), name) { + return true + } + } + } + return false +} + +func stringPointerReferencesFingerprintVar(s *string, name string) bool { + return s != nil && stringReferencesFingerprintVar(*s, name) +} + +func stringReferencesFingerprintVar(s string, name string) bool { + return strings.Contains(s, name) +} diff --git a/taskfile/ast/fingerprint_test.go b/taskfile/ast/fingerprint_test.go new file mode 100644 index 0000000000..ee55b22d11 --- /dev/null +++ b/taskfile/ast/fingerprint_test.go @@ -0,0 +1,103 @@ +package ast + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTaskReferencesFingerprintVar(t *testing.T) { + t.Parallel() + + fingerprintRef := "{{.CHECKSUM}}" + tests := []struct { + name string + task *Task + want bool + }{ + { + name: "nil task", + want: false, + }, + { + name: "status", + task: &Task{Status: []string{"test -n " + fingerprintRef}}, + want: true, + }, + { + name: "command", + task: &Task{Cmds: []*Cmd{{Cmd: "echo " + fingerprintRef}}}, + want: true, + }, + { + name: "task call", + task: &Task{Cmds: []*Cmd{{Task: "build-" + fingerprintRef}}}, + want: true, + }, + { + name: "command condition", + task: &Task{Cmds: []*Cmd{{If: "test -n " + fingerprintRef}}}, + want: true, + }, + { + name: "loop list nested value", + task: &Task{Cmds: []*Cmd{{For: &For{List: []any{map[string]any{"value": fingerprintRef}}}}}}, + want: true, + }, + { + name: "loop matrix reference", + task: &Task{Cmds: []*Cmd{{For: &For{Matrix: NewMatrix( + &MatrixElement{Key: "item", Value: &MatrixRow{Ref: fingerprintRef}}, + )}}}}, + want: true, + }, + { + name: "command variable", + task: &Task{Cmds: []*Cmd{{Vars: NewVars( + &VarElement{Key: "VALUE", Value: Var{Value: []any{fingerprintRef}}}, + )}}}, + want: true, + }, + { + name: "dynamic command variable", + task: &Task{Cmds: []*Cmd{{Vars: NewVars( + &VarElement{Key: "VALUE", Value: Var{Sh: &fingerprintRef}}, + )}}}, + want: true, + }, + { + name: "dependency", + task: &Task{Deps: []*Dep{{Task: "build-" + fingerprintRef}}}, + want: true, + }, + { + name: "dependency variable", + task: &Task{Deps: []*Dep{{Vars: NewVars( + &VarElement{Key: "VALUE", Value: Var{Ref: fingerprintRef}}, + )}}}, + want: true, + }, + { + name: "precondition", + task: &Task{Preconditions: []*Precondition{{Msg: fingerprintRef}}}, + want: true, + }, + { + name: "unrelated fields and nil entries", + task: &Task{ + Cmds: []*Cmd{nil, {Cmd: "echo ok"}}, + Deps: []*Dep{nil, {Task: "build"}}, + Preconditions: []*Precondition{nil, {Sh: "test -f output"}}, + Status: []string{"test -f output"}, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.task.ReferencesFingerprintVar("checksum")) + }) + } +} diff --git a/variables.go b/variables.go index 568591796e..900f87c0d9 100644 --- a/variables.go +++ b/variables.go @@ -5,7 +5,6 @@ import ( "maps" "os" "path/filepath" - "reflect" "strings" "github.com/joho/godotenv" @@ -218,7 +217,7 @@ func (e *Executor) compiledTask(call *Call, evaluateShVars bool) (*ast.Task, err checker = fingerprint.NewChecksumChecker(e.TempDir.Fingerprint, e.Dry) } - if taskReferencesFingerprintVar(origTask, checker.Kind()) { + if origTask.ReferencesFingerprintVar(checker.Kind()) { value, err := checker.Value(&new) if err != nil { return nil, err @@ -349,132 +348,6 @@ func (e *Executor) compiledTask(call *Call, evaluateShVars bool) (*ast.Task, err return &new, nil } -func taskReferencesFingerprintVar(t *ast.Task, kind string) bool { - name := strings.ToUpper(kind) - - for _, status := range t.Status { - if stringReferencesFingerprintVar(status, name) { - return true - } - } - for _, cmd := range t.Cmds { - if cmdReferencesFingerprintVar(cmd, name) { - return true - } - } - for _, dep := range t.Deps { - if depReferencesFingerprintVar(dep, name) { - return true - } - } - for _, precondition := range t.Preconditions { - if preconditionReferencesFingerprintVar(precondition, name) { - return true - } - } - return false -} - -func cmdReferencesFingerprintVar(cmd *ast.Cmd, name string) bool { - if cmd == nil { - return false - } - return stringReferencesFingerprintVar(cmd.Cmd, name) || - stringReferencesFingerprintVar(cmd.Task, name) || - stringReferencesFingerprintVar(cmd.If, name) || - forReferencesFingerprintVar(cmd.For, name) || - varsReferenceFingerprintVar(cmd.Vars, name) -} - -func depReferencesFingerprintVar(dep *ast.Dep, name string) bool { - if dep == nil { - return false - } - return stringReferencesFingerprintVar(dep.Task, name) || - forReferencesFingerprintVar(dep.For, name) || - varsReferenceFingerprintVar(dep.Vars, name) -} - -func preconditionReferencesFingerprintVar(precondition *ast.Precondition, name string) bool { - if precondition == nil { - return false - } - return stringReferencesFingerprintVar(precondition.Sh, name) || - stringReferencesFingerprintVar(precondition.Msg, name) -} - -func forReferencesFingerprintVar(f *ast.For, name string) bool { - if f == nil { - return false - } - if valueReferencesFingerprintVar(f.List, name) { - return true - } - for _, row := range f.Matrix.All() { - if stringReferencesFingerprintVar(row.Ref, name) || - valueReferencesFingerprintVar(row.Value, name) { - return true - } - } - return false -} - -func varsReferenceFingerprintVar(vars *ast.Vars, name string) bool { - if vars == nil { - return false - } - for _, v := range vars.All() { - if valueReferencesFingerprintVar(v.Value, name) || - valueReferencesFingerprintVar(v.Live, name) || - stringPointerReferencesFingerprintVar(v.Sh, name) || - stringReferencesFingerprintVar(v.Ref, name) || - stringReferencesFingerprintVar(v.Dir, name) { - return true - } - } - return false -} - -func valueReferencesFingerprintVar(value any, name string) bool { - if value == nil { - return false - } - if s, ok := value.(string); ok { - return stringReferencesFingerprintVar(s, name) - } - - rv := reflect.ValueOf(value) - switch rv.Kind() { - case reflect.Pointer, reflect.Interface: - if rv.IsNil() { - return false - } - return valueReferencesFingerprintVar(rv.Elem().Interface(), name) - case reflect.Array, reflect.Slice: - for i := 0; i < rv.Len(); i++ { - if valueReferencesFingerprintVar(rv.Index(i).Interface(), name) { - return true - } - } - case reflect.Map: - for _, key := range rv.MapKeys() { - if valueReferencesFingerprintVar(key.Interface(), name) || - valueReferencesFingerprintVar(rv.MapIndex(key).Interface(), name) { - return true - } - } - } - return false -} - -func stringPointerReferencesFingerprintVar(s *string, name string) bool { - return s != nil && stringReferencesFingerprintVar(*s, name) -} - -func stringReferencesFingerprintVar(s string, name string) bool { - return strings.Contains(s, name) -} - func asAnySlice[T any](slice []T) []any { ret := make([]any, len(slice)) for i, v := range slice {