v3.3.0

Closes #327

CHANGELOG.md

@@ -1,6 +1,15 @@
 # Changelog
 
-## v3.2.2
+## v3.3.0 - 2021-03-20
+
+- Add support for delegating CLI arguments to commands with `--` and a
+  special `CLI_ARGS` variable
+  ([#327](https://github.com/go-task/task/issues/327)).
+- Add a `--concurrency` (alias `-C`) flag, to limit the number of tasks that
+  run concurrently. This is useful for heavy workloads.
+  ([#345](https://github.com/go-task/task/pull/345)).
+
+## v3.2.2 - 2021-01-12
 
 - Improve performance of `--list` and `--summary` by skipping running shell
   variables for these flags
@@ -13,14 +22,14 @@
 - The install script is now working for ARM platforms
   ([#428](https://github.com/go-task/task/pull/428)).
 
-## v3.2.1
+## v3.2.1 - 2021-01-09
 
 - Fixed some bugs and regressions regarding dynamic variables and directories
   ([#426](https://github.com/go-task/task/issues/426)).
 - The [slim-sprig](https://github.com/go-task/slim-sprig) package was updated
   with the upstream [sprig](https://github.com/Masterminds/sprig).
 
-## v3.2.0
+## v3.2.0 - 2021-01-07
 
 - Fix the `.task` directory being created in the task directory instead of the
   Taskfile directory
@@ -33,7 +42,7 @@
   should be more stable now
   ([#423](https://github.com/go-task/task/pull/423), [#365](https://github.com/go-task/task/issues/365)).
 
-## v3.1.0
+## v3.1.0 - 2021-01-03
 
 - Fix a bug when the checksum up-to-date resolution is used by a task
   with a custom `label:` attribute
@@ -49,7 +58,7 @@
   sentence was in the last line
   ([#403](https://github.com/go-task/task/issues/403)).
 
-## v3.0.1
+## v3.0.1 - 2020-12-26
 
 - Allow use as a library by moving the required packages out of the `internal`
   directory
@@ -59,7 +68,7 @@
 - Fix panic when you have empty tasks in your Taskfile
   ([#338](https://github.com/go-task/task/issues/338), [#362](https://github.com/go-task/task/pull/362)).
 
-## v3.0.0
+## v3.0.0 - 2020-08-16
 
 - On `v3`, all CLI variables will be considered global variables
   ([#336](https://github.com/go-task/task/issues/336), [#341](https://github.com/go-task/task/pull/341))
@@ -102,7 +111,7 @@
   commands are green, errors are red, etc
   ([#207](https://github.com/go-task/task/pull/207)).
 
-## v2.8.1 - 2019-05-20
+## v2.8.1 - 2020-05-20
 
 - Fix error code for the `--help` flag
   ([#300](https://github.com/go-task/task/issues/300), [#330](https://github.com/go-task/task/pull/330)).

Taskfile.yml

@@ -12,6 +12,9 @@ vars:
   GO_PACKAGES:
     sh: go list ./...
 
+env:
+  CGO_ENABLED: '0'
+
 tasks:
   default:
     cmds:
@@ -21,8 +24,6 @@ tasks:
     desc: Installs Task
     cmds:
       - go install -v -ldflags="-w -s -X main.version={{.GIT_COMMIT}}" ./cmd/task
-    env:
-      CGO_ENABLED: '0'
 
   mod:
     desc: Downloads and tidy Go modules

args/args.go

@@ -9,7 +9,7 @@ import (
 // ParseV3 parses command line argument: tasks and global variables
 func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 	var calls []taskfile.Call
-	var globals *taskfile.Vars
+	var globals = &taskfile.Vars{}
 
 	for _, arg := range args {
 		if !strings.Contains(arg, "=") {
@@ -17,9 +17,6 @@ func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 			continue
 		}
 
-		if globals == nil {
-			globals = &taskfile.Vars{}
-		}
 		name, value := splitVar(arg)
 		globals.Set(name, taskfile.Var{Static: value})
 	}
@@ -34,7 +31,7 @@ func ParseV3(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 // ParseV2 parses command line argument: tasks and vars of each task
 func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 	var calls []taskfile.Call
-	var globals *taskfile.Vars
+	var globals = &taskfile.Vars{}
 
 	for _, arg := range args {
 		if !strings.Contains(arg, "=") {
@@ -43,9 +40,6 @@ func ParseV2(args ...string) ([]taskfile.Call, *taskfile.Vars) {
 		}
 
 		if len(calls) < 1 {
-			if globals == nil {
-				globals = &taskfile.Vars{}
-			}
 			name, value := splitVar(arg)
 			globals.Set(name, taskfile.Var{Static: value})
 		} else {

args/args_test.go

@@ -96,7 +96,9 @@ func TestArgsV3(t *testing.T) {
 		t.Run(fmt.Sprintf("TestArgs%d", i+1), func(t *testing.T) {
 			calls, globals := args.ParseV3(test.Args...)
 			assert.Equal(t, test.ExpectedCalls, calls)
-			assert.Equal(t, test.ExpectedGlobals, globals)
+			if test.ExpectedGlobals.Len() > 0 || globals.Len() > 0 {
+				assert.Equal(t, test.ExpectedGlobals, globals)
+			}
 		})
 	}
 }
@@ -198,7 +200,10 @@ func TestArgsV2(t *testing.T) {
 		t.Run(fmt.Sprintf("TestArgs%d", i+1), func(t *testing.T) {
 			calls, globals := args.ParseV2(test.Args...)
 			assert.Equal(t, test.ExpectedCalls, calls)
-			assert.Equal(t, test.ExpectedGlobals, globals)
+			if test.ExpectedGlobals.Len() > 0 || globals.Len() > 0 {
+				assert.Equal(t, test.ExpectedGlobals, globals)
+			}
+
 		})
 	}
 }

cmd/task/task.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"github.com/spf13/pflag"
@@ -65,6 +66,7 @@ func main() {
 		dry         bool
 		summary     bool
 		parallel    bool
+		concurrency int
 		dir         string
 		entrypoint  string
 		output      string
@@ -87,6 +89,7 @@ func main() {
 	pflag.StringVarP(&entrypoint, "taskfile", "t", "", `choose which Taskfile to run. Defaults to "Taskfile.yml"`)
 	pflag.StringVarP(&output, "output", "o", "", "sets output style: [interleaved|group|prefixed]")
 	pflag.BoolVarP(&color, "color", "c", true, "colored output")
+	pflag.IntVarP(&concurrency, "concurrency", "C", 0, "limit number tasks to run concurrently")
 	pflag.Parse()
 
 	if versionFlag {
@@ -122,16 +125,17 @@ func main() {
 	}
 
 	e := task.Executor{
-		Force:      force,
+		Force:       force,
-		Watch:      watch,
+		Watch:       watch,
-		Verbose:    verbose,
+		Verbose:     verbose,
-		Silent:     silent,
+		Silent:      silent,
-		Dir:        dir,
+		Dir:         dir,
-		Dry:        dry,
+		Dry:         dry,
-		Entrypoint: entrypoint,
+		Entrypoint:  entrypoint,
-		Summary:    summary,
+		Summary:     summary,
-		Parallel:   parallel,
+		Parallel:    parallel,
-		Color:      color,
+		Color:       color,
+		Concurrency: concurrency,
 
 		Stdin:  os.Stdin,
 		Stdout: os.Stdout,
@@ -154,14 +158,18 @@ func main() {
 	}
 
 	var (
-		calls   []taskfile.Call
+		calls                 []taskfile.Call
-		globals *taskfile.Vars
+		globals               *taskfile.Vars
+		tasksAndVars, cliArgs = getArgs()
 	)
+
 	if v >= 3.0 {
-		calls, globals = args.ParseV3(pflag.Args()...)
+		calls, globals = args.ParseV3(tasksAndVars...)
 	} else {
-		calls, globals = args.ParseV2(pflag.Args()...)
+		calls, globals = args.ParseV2(tasksAndVars...)
 	}
+
+	globals.Set("CLI_ARGS", taskfile.Var{Static: strings.Join(cliArgs, " ")})
 	e.Taskfile.Vars.Merge(globals)
 
 	ctx := context.Background()
@@ -182,6 +190,22 @@ func main() {
 	}
 }
 
+func getArgs() (tasksAndVars, cliArgs []string) {
+	var (
+		args          = pflag.Args()
+		doubleDashPos = pflag.CommandLine.ArgsLenAtDash()
+	)
+
+	if doubleDashPos != -1 {
+		tasksAndVars = args[:doubleDashPos]
+		cliArgs = args[doubleDashPos:]
+	} else {
+		tasksAndVars = args
+	}
+
+	return
+}
+
 func getSignalContext() context.Context {
 	ch := make(chan os.Signal, 1)
 	signal.Notify(ch, os.Interrupt, os.Kill, syscall.SIGTERM)

concurrency.go

@@ -0,0 +1,25 @@
+package task
+
+func (e *Executor) acquireConcurrencyLimit() func() {
+	if e.concurrencySemaphore == nil {
+		return emptyFunc
+	}
+
+	e.concurrencySemaphore <- struct{}{}
+	return func() {
+		<-e.concurrencySemaphore
+	}
+}
+
+func (e *Executor) releaseConcurrencyLimit() func() {
+	if e.concurrencySemaphore == nil {
+		return emptyFunc
+	}
+
+	<-e.concurrencySemaphore
+	return func() {
+		e.concurrencySemaphore <- struct{}{}
+	}
+}
+
+func emptyFunc() {}

docs/usage.md

@@ -520,6 +520,27 @@ tasks:
 
 This works for all types of variables.
 
+## Forwarding CLI arguments to commands
+
+If `--` is given in the CLI, all following paramaters are added to a
+special `.CLI_ARGS` variable. This is useful to forward arguments to another
+command.
+
+The below example will run `yarn install`.
+
+```bash
+$ task yarn -- install
+```
+
+```yaml
+version: '3'
+
+tasks:
+  yarn:
+    cmds:
+      - yarn {{.CLI_ARGS}}
+```
+
 ## Go's template engine
 
 Task parse commands as [Go's template engine][gotemplate] before executing

go.mod

@@ -7,11 +7,11 @@ require (
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-zglob v0.0.1
 	github.com/radovskyb/watcher v1.0.5
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.5.1
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
-	mvdan.cc/sh/v3 v3.2.1
+	mvdan.cc/sh/v3 v3.2.4
 )
 
 go 1.13

go.sum

@@ -13,7 +13,6 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -33,10 +32,9 @@ github.com/radovskyb/watcher v1.0.5 h1:wqt7gb+HjGacvFoLTKeT44C+XVPxu7bvHvKT1IvZ7
 github.com/radovskyb/watcher v1.0.5/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@@ -48,7 +46,6 @@ golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1
 golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407 h1:5zh5atpUEdIc478E/ebrIaHLKcfVvG6dL/fGv7BcMoM=
 golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -58,5 +55,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 mvdan.cc/editorconfig v0.1.1-0.20200121172147-e40951bde157/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8=
-mvdan.cc/sh/v3 v3.2.1 h1:uQBpiGM+rEuHse3Q+W7ajuJUeOtFVJUN/6GeX4/dUWE=
+mvdan.cc/sh/v3 v3.2.4 h1:+fZaWcXWRjYAvqzEKoDhDM3DkxdDUykU2iw0VMKFe9s=
-mvdan.cc/sh/v3 v3.2.1/go.mod h1:fPQmabBpREM/XQ9YXSU5ZFZ/Sm+PmKP9/vkFHgYKJEI=
+mvdan.cc/sh/v3 v3.2.4/go.mod h1:fPQmabBpREM/XQ9YXSU5ZFZ/Sm+PmKP9/vkFHgYKJEI=

internal/templater/templater.go

@@ -63,7 +63,7 @@ func (r *Templater) ReplaceSlice(strs []string) []string {
 }
 
 func (r *Templater) ReplaceVars(vars *taskfile.Vars) *taskfile.Vars {
-	if r.err != nil || vars == nil || len(vars.Keys) == 0 {
+	if r.err != nil || vars.Len() == 0 {
 		return nil
 	}
 

task.go

@@ -32,16 +32,17 @@ const (
 type Executor struct {
 	Taskfile *taskfile.Taskfile
 
-	Dir        string
+	Dir         string
-	Entrypoint string
+	Entrypoint  string
-	Force      bool
+	Force       bool
-	Watch      bool
+	Watch       bool
-	Verbose    bool
+	Verbose     bool
-	Silent     bool
+	Silent      bool
-	Dry        bool
+	Dry         bool
-	Summary    bool
+	Summary     bool
-	Parallel   bool
+	Parallel    bool
-	Color      bool
+	Color       bool
+	Concurrency int
 
 	Stdin  io.Reader
 	Stdout io.Writer
@@ -54,8 +55,9 @@ type Executor struct {
 
 	taskvars *taskfile.Vars
 
-	taskCallCount map[string]*int32
+	concurrencySemaphore chan struct{}
-	mkdirMutexMap map[string]*sync.Mutex
+	taskCallCount        map[string]*int32
+	mkdirMutexMap        map[string]*sync.Mutex
 }
 
 // Run runs Task
@@ -247,6 +249,10 @@ func (e *Executor) Setup() error {
 		e.taskCallCount[k] = new(int32)
 		e.mkdirMutexMap[k] = &sync.Mutex{}
 	}
+
+	if e.Concurrency > 0 {
+		e.concurrencySemaphore = make(chan struct{}, e.Concurrency)
+	}
 	return nil
 }
 
@@ -260,6 +266,9 @@ func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
 		return &MaximumTaskCallExceededError{task: call.Task}
 	}
 
+	release := e.acquireConcurrencyLimit()
+	defer release()
+
 	if err := e.runDeps(ctx, t); err != nil {
 		return err
 	}
@@ -324,6 +333,9 @@ func (e *Executor) mkdir(t *taskfile.Task) error {
 func (e *Executor) runDeps(ctx context.Context, t *taskfile.Task) error {
 	g, ctx := errgroup.WithContext(ctx)
 
+	reacquire := e.releaseConcurrencyLimit()
+	defer reacquire()
+
 	for _, d := range t.Deps {
 		d := d
 
@@ -344,6 +356,9 @@ func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfi
 
 	switch {
 	case cmd.Task != "":
+		reacquire := e.releaseConcurrencyLimit()
+		defer reacquire()
+
 		err := e.RunTask(ctx, taskfile.Call{Task: cmd.Task, Vars: cmd.Vars})
 		if err != nil {
 			return err

task_test.go

@@ -171,6 +171,22 @@ func TestVarsInvalidTmpl(t *testing.T) {
 	assert.EqualError(t, e.Run(context.Background(), taskfile.Call{Task: target}), expectError, "e.Run(target)")
 }
 
+func TestConcurrency(t *testing.T) {
+	const (
+		dir    = "testdata/concurrency"
+		target = "default"
+	)
+
+	e := &task.Executor{
+		Dir:         dir,
+		Stdout:      ioutil.Discard,
+		Stderr:      ioutil.Discard,
+		Concurrency: 1,
+	}
+	assert.NoError(t, e.Setup(), "e.Setup()")
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: target}), "e.Run(target)")
+}
+
 func TestParams(t *testing.T) {
 	tt := fileContentTest{
 		Dir:       "testdata/params",

taskfile/var.go

@@ -75,7 +75,7 @@ func (vs *Vars) Range(yield func(key string, value Var) error) error {
 // ToCacheMap converts Vars to a map containing only the static
 // variables
 func (vs *Vars) ToCacheMap() (m map[string]interface{}) {
-	m = make(map[string]interface{}, len(vs.Keys))
+	m = make(map[string]interface{}, vs.Len())
 	vs.Range(func(k string, v Var) error {
 		if v.Sh != "" {
 			// Dynamic variable is not yet resolved; trigger
@@ -93,6 +93,14 @@ func (vs *Vars) ToCacheMap() (m map[string]interface{}) {
 	return
 }
 
+// Len returns the size of the map
+func (vs *Vars) Len() int {
+	if vs == nil {
+		return 0
+	}
+	return len(vs.Keys)
+}
+
 // Var represents either a static or dynamic variable.
 type Var struct {
 	Static string

testdata/concurrency/Taskfile.yml

@@ -0,0 +1,32 @@
+version: '2'
+
+tasks:
+  default:
+    deps:
+      - t1
+
+  t1:
+    deps:
+      - t3
+      - t4
+    cmds:
+      - task: t2
+      - echo done 1
+  t2:
+    deps:
+      - t5
+      - t6
+    cmds:
+      - echo done 2
+  t3:
+    cmds:
+      - echo done 3
+  t4:
+    cmds:
+      - echo done 4
+  t5:
+    cmds:
+      - echo done 5
+  t6:
+    cmds:
+      - echo done 6