v2.5.1

Fixes some bugs relatated to commands output handling

.gitignore

@@ -18,3 +18,6 @@
 dist/
 
 .DS_Store
+
+# intellij idea/goland
+.idea/

.travis.yml

@@ -1,8 +1,8 @@
 language: go
 
 go:
-  - 1.10.x
   - 1.11.x
+  - 1.12.x
 
 addons:
   apt:

CHANGELOG.md

@@ -0,0 +1,184 @@
+# Changelog
+
+## 2.5.1 - 2019-04-27
+
+- Fixed some issues with interactive command line tools, where sometimes
+  the output were not being shown, and similar issues
+  ([#114](https://github.com/go-task/task/issues/114), [#190](https://github.com/go-task/task/issues/190), [#200](https://github.com/go-task/task/pull/200)).
+- Upgraded [go-yaml/yaml](https://github.com/go-yaml/yaml) from v2 to v3.
+
+## v2.5.0 - 2019-03-16
+
+- We moved from the taskfile.org domain to the new fancy taskfile.dev domain.
+  While stuff is being redirected, we strongly recommend to everyone that use
+  [this install script](https://taskfile.dev/#/installation?id=install-script)
+  to use the new taskfile.dev domain on scripts from now on.
+- Fixed to the ZSH completion
+  ([#182](https://github.com/go-task/task/pull/182)).
+- Add [`--summary` flag along with `summary:` task attribute](https://taskfile.org/#/usage?id=display-summary-of-task)
+  ([#180](https://github.com/go-task/task/pull/180)).
+
+## v2.4.0 - 2019-02-21
+
+- Allow calling a task of the root Taskfile from an included Taskfile
+  by prefixing it with `:`
+  ([#161](https://github.com/go-task/task/issues/161), [#172](https://github.com/go-task/task/issues/172)),
+- Add flag to override the `output` option
+  ([#173](https://github.com/go-task/task/pull/173));
+- Fix bug where Task was persisting the new checksum on the disk when the Dry
+  Mode is enabled
+  ([#166](https://github.com/go-task/task/issues/166));
+- Fix file timestamp issue when the file name has spaces
+  ([#176](https://github.com/go-task/task/issues/176));
+- Mitigating path expanding issues on Windows
+  ([#170](https://github.com/go-task/task/pull/170)).
+
+## v2.3.0 - 2019-01-02
+
+- On Windows, Task can now be installed using [Scoop](https://scoop.sh/)
+  ([#152](https://github.com/go-task/task/pull/152));
+- Fixed issue with file/directory globing
+  ([#153](https://github.com/go-task/task/issues/153));
+- Added ability to globally set environment variables
+  (
+    [#138](https://github.com/go-task/task/pull/138),
+    [#159](https://github.com/go-task/task/pull/159)
+  ).
+
+## v2.2.1 - 2018-12-09
+
+- This repository now uses Go Modules (#143). We'll still keep the `vendor` directory in sync for some time, though;
+- Fixing a bug when the Taskfile has no tasks but includes another Taskfile (#150);
+- Fix a bug when calling another task or a dependency in an included Taskfile (#151).
+
+## v2.2.0 - 2018-10-25
+
+- Added support for [including other Taskfiles](https://taskfile.org/#/usage?id=including-other-taskfiles) (#98)
+  - This should be considered experimental. For now, only including local files is supported, but support for including remote Taskfiles is being discussed. If you have any feedback, please comment on #98.
+- Task now have a dedicated documentation site: https://taskfile.org
+  - Thanks to [Docsify](https://docsify.js.org/) for making this pretty easy. To check the source code, just take a look at the [docs](https://github.com/go-task/task/tree/master/docs) directory of this repository. Contributions to the documentation is really appreciated.
+
+## v2.1.1 - 2018-09-17
+
+- Fix suggestion to use `task --init` not being shown anymore (when a `Taskfile.yml` is not found)
+- Fix error when using checksum method and no file exists for a source glob (#131)
+- Fix signal handling when the `--watch` flag is given (#132)
+
+## v2.1.0 - 2018-08-19
+
+- Add a `ignore_error` option to task and command (#123)
+- Add a dry run mode (`--dry` flag) (#126)
+
+## v2.0.3 - 2018-06-24
+
+- Expand environment variables on "dir", "sources" and "generates" (#116)
+- Fix YAML merging syntax (#112)
+- Add ZSH completion (#111)
+- Implement new `output` option. Please check out the [documentation](https://github.com/go-task/task#output-syntax)
+
+## v2.0.2 - 2018-05-01
+
+- Fix merging of YAML anchors (#112)
+
+## v2.0.1 - 2018-03-11
+
+- Fixes panic on `task --list`
+
+## v2.0.0 - 2018-03-08
+
+Version 2.0.0 is here, with a new Taskfile format.
+
+Please, make sure to read the [Taskfile versions](https://github.com/go-task/task/blob/master/TASKFILE_VERSIONS.md) document, since it describes in depth what changed for this version.
+
+* New Taskfile version 2 (https://github.com/go-task/task/issues/77)
+* Possibility to have global variables in the `Taskfile.yml` instead of `Taskvars.yml` (https://github.com/go-task/task/issues/66)
+* Small improvements and fixes
+
+## v1.4.4 - 2017-11-19
+
+- Handle SIGINT and SIGTERM (#75);
+- List: print message with there's no task with description;
+- Expand home dir ("~" symbol) on paths (#74);
+- Add Snap as an installation method;
+- Move examples to its own repo;
+- Watch: also walk on tasks called on on "cmds", and not only on "deps";
+- Print logs to stderr instead of stdout (#68);
+- Remove deprecated `set` keyword;
+- Add checksum based status check, alternative to timestamp based.
+
+## v1.4.3 - 2017-09-07
+
+- Allow assigning variables to tasks at run time via CLI (#33)
+- Added suport for multiline variables from sh (#64)
+- Fixes env: remove square braces and evaluate shell (#62)
+- Watch: change watch library and few fixes and improvements
+- When use watching, cancel and restart long running process on file change (#59 and #60)
+
+## v1.4.2 - 2017-07-30
+
+- Flag to set directory of execution
+- Always echo command if is verbose mode
+- Add silent mode to disable echoing of commands
+- Fixes and improvements of variables (#56)
+
+## v1.4.1 - 2017-07-15
+
+- Allow use of YAML for dynamic variables instead of $ prefix
+  - `VAR: {sh: echo Hello}` instead of `VAR: $echo Hello`
+- Add `--list` (or `-l`) flag to print existing tasks
+- OS specific Taskvars file (e.g. `Taskvars_windows.yml`, `Taskvars_linux.yml`, etc)
+- Consider task up-to-date on equal timestamps (#49)
+- Allow absolute path in generates section (#48)
+- Bugfix: allow templating when calling deps (#42)
+- Fix panic for invalid task in cyclic dep detection
+- Better error output for dynamic variables in Taskvars.yml (#41)
+- Allow template evaluation in parameters
+
+## v1.4.0 - 2017-07-06
+
+- Cache dynamic variables
+- Add verbose mode (`-v` flag)
+- Support to task parameters (overriding vars) (#31) (#32)
+- Print command, also when "set:" is specified (#35)
+- Improve task command help text (#35)
+
+## v1.3.1 - 2017-06-14
+
+- Fix glob not working on commands (#28)
+- Add ExeExt template function
+- Add `--init` flag to create a new Taskfile
+- Add status option to prevent task from running (#27)
+- Allow interpolation on `generates` and `sources` attributes (#26)
+
+## v1.3.0 - 2017-04-24
+
+- Migrate from os/exec.Cmd to a native Go sh/bash interpreter
+  - This is a potentially breaking change if you use Windows.
+  - Now, `cmd` is not used anymore on Windows. Always use Bash-like syntax for your commands, even on Windows.
+- Add "ToSlash" and "FromSlash" to template functions
+- Use functions defined on github.com/Masterminds/sprig
+- Do not redirect stdin while running variables commands
+- Using `context` and `errgroup` packages (this will make other tasks to be cancelled, if one returned an error)
+
+## v1.2.0 - 2017-04-02
+
+- More tests and Travis integration
+- Watch a task (experimental)
+- Possibility to call another task
+- Fix "=" not being reconized in variables/environment variables
+- Tasks can now have a description, and help will print them (#10)
+- Task dependencies now run concurrently
+- Support for a default task (#16)
+
+## v1.1.0 - 2017-03-08
+
+- Support for YAML, TOML and JSON (#1)
+- Support running command in another directory (#4)
+- `--force` or `-f` flag to force execution of task even when it's up-to-date
+- Detection of cyclic dependencies (#5)
+- Support for variables (#6, #9, #14)
+- Operation System specific commands and variables (#13)
+
+## v1.0.0 - 2017-02-28
+
+- Add LICENSE file

README.md

@@ -3,10 +3,20 @@
 # Task
 
 Task is a task runner / build tool that aims to be simpler and easier to use
-than, for example, [GNU Make][make].
+than, for example, [GNU Make](https://www.gnu.org/software/make/).
+
+See [taskfile.dev](https://taskfile.dev) for documentation.
 
 ---
 
-See [taskfile.org](https://taskfile.org) for documentation.
+## Sponsors
 
-[make]: https://www.gnu.org/software/make/
+[![Sponsors](https://opencollective.com/task/sponsors.svg?width=890)](https://opencollective.com/task)
+
+## Backers
+
+[![Backers](https://opencollective.com/task/backers.svg?width=890)](https://opencollective.com/task)
+
+## Contributors
+
+[![Contributors](https://opencollective.com/task/contributors.svg?width=890)](https://github.com/go-task/task/graphs/contributors)

Taskfile.yml

@@ -1,5 +1,8 @@
 version: '2'
 
+includes:
+  docs: ./docs
+
 vars:
   GIT_COMMIT:
     sh: git log -n 1 --format=%h
@@ -83,13 +86,3 @@ tasks:
     cmds:
       - echo '{{.GO_PACKAGES}}'
     silent: true
-
-  docs:install:
-    desc: Installs docsify to work the on the documentation site
-    cmds:
-      - npm install docsify-cli -g
-
-  docs:serve:
-    desc: Serves the documentation site locally
-    cmds:
-      - docsify serve docs

cmd/redirector/redirector.go

@@ -0,0 +1,421 @@
+// This small web app is used to redirect from the old taskfile.org domain
+// to the new taskfile.dev without breaking CIs that uses cURL to download
+// "/install.sh" without the -L flag (which follow redirects).
+
+package main
+
+import (
+	"net/http"
+)
+
+func main() {
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/install.sh" {
+			println("Dumping install.sh")
+
+			w.Write(installShContent)
+			return
+		}
+
+		println("Redirecting to https://taskfile.dev" + r.URL.Path)
+
+		w.Header().Set("Location", "https://taskfile.dev"+r.URL.Path)
+		w.WriteHeader(301)
+	})
+
+	println("Listening :8080")
+
+	panic(http.ListenAndServe(":8080", nil))
+}
+
+var installShContent = []byte(`#!/bin/sh
+set -e
+# Code generated by godownloader on 2018-04-07T17:47:38Z. DO NOT EDIT.
+#
+
+usage() {
+  this=$1
+  cat <<EOF
+$this: download go binaries for go-task/task
+
+Usage: $this [-b] bindir [-d] [tag]
+  -b sets bindir or installation directory, Defaults to ./bin
+  -d turns on debug logging
+   [tag] is a tag from
+   https://github.com/go-task/task/releases
+   If tag is missing, then the latest will be used.
+
+ Generated by godownloader
+  https://github.com/goreleaser/godownloader
+
+EOF
+  exit 2
+}
+
+parse_args() {
+  #BINDIR is ./bin unless set be ENV
+  # over-ridden by flag below
+
+  BINDIR=${BINDIR:-./bin}
+  while getopts "b:dh?" arg; do
+    case "$arg" in
+      b) BINDIR="$OPTARG" ;;
+      d) log_set_priority 10 ;;
+      h | \?) usage "$0" ;;
+    esac
+  done
+  shift $((OPTIND - 1))
+  TAG=$1
+}
+# this function wraps all the destructive operations
+# if a curl|bash cuts off the end of the script due to
+# network, either nothing will happen or will syntax error
+# out preventing half-done work
+execute() {
+  tmpdir=$(mktmpdir)
+  log_debug "downloading files into ${tmpdir}"
+  http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
+  http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
+  hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
+  srcdir="${tmpdir}"
+  (cd "${tmpdir}" && untar "${TARBALL}")
+  install -d "${BINDIR}"
+  for binexe in "task" ; do
+    if [ "$OS" = "windows" ]; then
+      binexe="${binexe}.exe"
+    fi
+    install "${srcdir}/${binexe}" "${BINDIR}/"
+    log_info "installed ${BINDIR}/${binexe}"
+  done
+}
+is_supported_platform() {
+  platform=$1
+  found=1
+  case "$platform" in
+    windows/386) found=0 ;;
+    windows/amd64) found=0 ;;
+    darwin/386) found=0 ;;
+    darwin/amd64) found=0 ;;
+    linux/386) found=0 ;;
+    linux/amd64) found=0 ;;
+  esac
+  case "$platform" in
+    darwin/386) found=1 ;;
+  esac
+  return $found
+}
+check_platform() {
+  if is_supported_platform "$PLATFORM"; then
+    # optional logging goes here
+    true
+  else
+    log_crit "platform $PLATFORM is not supported.  Make sure this script is up-to-date and file request at https://github.com/${PREFIX}/issues/new"
+    exit 1
+  fi
+}
+tag_to_version() {
+  if [ -z "${TAG}" ]; then
+    log_info "checking GitHub for latest tag"
+  else
+    log_info "checking GitHub for tag '${TAG}'"
+  fi
+  REALTAG=$(github_release "$OWNER/$REPO" "${TAG}") && true
+  if test -z "$REALTAG"; then
+    log_crit "unable to find '${TAG}' - use 'latest' or see https://github.com/${PREFIX}/releases for details"
+    exit 1
+  fi
+  # if version starts with 'v', remove it
+  TAG="$REALTAG"
+  VERSION=${TAG#v}
+}
+adjust_format() {
+  # change format (tar.gz or zip) based on ARCH
+  case ${ARCH} in
+    windows) FORMAT=zip ;;
+  esac
+  true
+}
+adjust_os() {
+  # adjust archive name based on OS
+  true
+}
+adjust_arch() {
+  # adjust archive name based on ARCH
+  true
+}
+
+cat /dev/null <<EOF
+------------------------------------------------------------------------
+https://github.com/client9/shlib - portable posix shell functions
+Public domain - http://unlicense.org
+https://github.com/client9/shlib/blob/master/LICENSE.md
+but credit (and pull requests) appreciated.
+------------------------------------------------------------------------
+EOF
+is_command() {
+  command -v "$1" >/dev/null
+}
+echoerr() {
+  echo "$@" 1>&2
+}
+log_prefix() {
+  echo "$0"
+}
+_logp=6
+log_set_priority() {
+  _logp="$1"
+}
+log_priority() {
+  if test -z "$1"; then
+    echo "$_logp"
+    return
+  fi
+  [ "$1" -le "$_logp" ]
+}
+log_tag() {
+  case $1 in
+    0) echo "emerg" ;;
+    1) echo "alert" ;;
+    2) echo "crit" ;;
+    3) echo "err" ;;
+    4) echo "warning" ;;
+    5) echo "notice" ;;
+    6) echo "info" ;;
+    7) echo "debug" ;;
+    *) echo "$1" ;;
+  esac
+}
+log_debug() {
+  log_priority 7 || return 0
+  echoerr "$(log_prefix)" "$(log_tag 7)" "$@"
+}
+log_info() {
+  log_priority 6 || return 0
+  echoerr "$(log_prefix)" "$(log_tag 6)" "$@"
+}
+log_err() {
+  log_priority 3 || return 0
+  echoerr "$(log_prefix)" "$(log_tag 3)" "$@"
+}
+log_crit() {
+  log_priority 2 || return 0
+  echoerr "$(log_prefix)" "$(log_tag 2)" "$@"
+}
+uname_os() {
+  os=$(uname -s | tr '[:upper:]' '[:lower:]')
+  case "$os" in
+    msys_nt) os="windows" ;;
+  esac
+  echo "$os"
+}
+uname_arch() {
+  arch=$(uname -m)
+  case $arch in
+    x86_64) arch="amd64" ;;
+    x86) arch="386" ;;
+    i686) arch="386" ;;
+    i386) arch="386" ;;
+    aarch64) arch="arm64" ;;
+    armv5*) arch="arm5" ;;
+    armv6*) arch="arm6" ;;
+    armv7*) arch="arm7" ;;
+  esac
+  echo ${arch}
+}
+uname_os_check() {
+  os=$(uname_os)
+  case "$os" in
+    darwin) return 0 ;;
+    dragonfly) return 0 ;;
+    freebsd) return 0 ;;
+    linux) return 0 ;;
+    android) return 0 ;;
+    nacl) return 0 ;;
+    netbsd) return 0 ;;
+    openbsd) return 0 ;;
+    plan9) return 0 ;;
+    solaris) return 0 ;;
+    windows) return 0 ;;
+  esac
+  log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib"
+  return 1
+}
+uname_arch_check() {
+  arch=$(uname_arch)
+  case "$arch" in
+    386) return 0 ;;
+    amd64) return 0 ;;
+    arm64) return 0 ;;
+    armv5) return 0 ;;
+    armv6) return 0 ;;
+    armv7) return 0 ;;
+    ppc64) return 0 ;;
+    ppc64le) return 0 ;;
+    mips) return 0 ;;
+    mipsle) return 0 ;;
+    mips64) return 0 ;;
+    mips64le) return 0 ;;
+    s390x) return 0 ;;
+    amd64p32) return 0 ;;
+  esac
+  log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value.  Please file bug report at https://github.com/client9/shlib"
+  return 1
+}
+untar() {
+  tarball=$1
+  case "${tarball}" in
+    *.tar.gz | *.tgz) tar -xzf "${tarball}" ;;
+    *.tar) tar -xf "${tarball}" ;;
+    *.zip) unzip "${tarball}" ;;
+    *)
+      log_err "untar unknown archive format for ${tarball}"
+      return 1
+      ;;
+  esac
+}
+mktmpdir() {
+  test -z "$TMPDIR" && TMPDIR="$(mktemp -d)"
+  mkdir -p "${TMPDIR}"
+  echo "${TMPDIR}"
+}
+http_download_curl() {
+  local_file=$1
+  source_url=$2
+  header=$3
+  if [ -z "$header" ]; then
+    code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url")
+  else
+    code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url")
+  fi
+  if [ "$code" != "200" ]; then
+    log_debug "http_download_curl received HTTP status $code"
+    return 1
+  fi
+  return 0
+}
+http_download_wget() {
+  local_file=$1
+  source_url=$2
+  header=$3
+  if [ -z "$header" ]; then
+    wget -q -O "$local_file" "$source_url"
+  else
+    wget -q --header "$header" -O "$local_file" "$source_url"
+  fi
+}
+http_download() {
+  log_debug "http_download $2"
+  if is_command curl; then
+    http_download_curl "$@"
+    return
+  elif is_command wget; then
+    http_download_wget "$@"
+    return
+  fi
+  log_crit "http_download unable to find wget or curl"
+  return 1
+}
+http_copy() {
+  tmp=$(mktemp)
+  http_download "${tmp}" "$1" "$2" || return 1
+  body=$(cat "$tmp")
+  rm -f "${tmp}"
+  echo "$body"
+}
+github_release() {
+  owner_repo=$1
+  version=$2
+  test -z "$version" && version="latest"
+  giturl="https://github.com/${owner_repo}/releases/${version}"
+  json=$(http_copy "$giturl" "Accept:application/json")
+  test -z "$json" && return 1
+  version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//')
+  test -z "$version" && return 1
+  echo "$version"
+}
+hash_sha256() {
+  TARGET=${1:-/dev/stdin}
+  if is_command gsha256sum; then
+    hash=$(gsha256sum "$TARGET") || return 1
+    echo "$hash" | cut -d ' ' -f 1
+  elif is_command sha256sum; then
+    hash=$(sha256sum "$TARGET") || return 1
+    echo "$hash" | cut -d ' ' -f 1
+  elif is_command shasum; then
+    hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1
+    echo "$hash" | cut -d ' ' -f 1
+  elif is_command openssl; then
+    hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1
+    echo "$hash" | cut -d ' ' -f a
+  else
+    log_crit "hash_sha256 unable to find command to compute sha-256 hash"
+    return 1
+  fi
+}
+hash_sha256_verify() {
+  TARGET=$1
+  checksums=$2
+  if [ -z "$checksums" ]; then
+    log_err "hash_sha256_verify checksum file not specified in arg2"
+    return 1
+  fi
+  BASENAME=${TARGET##*/}
+  want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
+  if [ -z "$want" ]; then
+    log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'"
+    return 1
+  fi
+  got=$(hash_sha256 "$TARGET")
+  if [ "$want" != "$got" ]; then
+    log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got"
+    return 1
+  fi
+}
+cat /dev/null <<EOF
+------------------------------------------------------------------------
+End of functions from https://github.com/client9/shlib
+------------------------------------------------------------------------
+EOF
+
+PROJECT_NAME="task"
+OWNER=go-task
+REPO="task"
+BINARY=task
+FORMAT=tar.gz
+OS=$(uname_os)
+ARCH=$(uname_arch)
+PREFIX="$OWNER/$REPO"
+
+# use in logging routines
+log_prefix() {
+	echo "$PREFIX"
+}
+PLATFORM="${OS}/${ARCH}"
+GITHUB_DOWNLOAD=https://github.com/${OWNER}/${REPO}/releases/download
+
+uname_os_check "$OS"
+uname_arch_check "$ARCH"
+
+parse_args "$@"
+
+check_platform
+
+tag_to_version
+
+adjust_format
+
+adjust_os
+
+adjust_arch
+
+log_info "found version: ${VERSION} for ${TAG}/${OS}/${ARCH}"
+
+NAME=${BINARY}_${OS}_${ARCH}
+TARBALL=${NAME}.${FORMAT}
+TARBALL_URL=${GITHUB_DOWNLOAD}/${TAG}/${TARBALL}
+CHECKSUM=task_checksums.txt
+CHECKSUM_URL=${GITHUB_DOWNLOAD}/${TAG}/${CHECKSUM}
+
+
+execute
+`)

cmd/task/task.go

@@ -17,7 +17,7 @@ var (
 	version = "master"
 )
 
-const usage = `Usage: task [-ilfwvsd] [--init] [--list] [--force] [--watch] [--verbose] [--silent] [--dir] [--dry] [task...]
+const usage = `Usage: task [-ilfwvsd] [--init] [--list] [--force] [--watch] [--verbose] [--silent] [--dir] [--dry] [--summary] [task...]
 
 Runs the specified task(s). Falls back to the "default" task if no task name
 was specified, or lists all tasks if an unknown task name was specified.
@@ -56,7 +56,9 @@ func main() {
 		verbose     bool
 		silent      bool
 		dry         bool
+		summary     bool
 		dir         string
+		output      string
 	)
 
 	pflag.BoolVar(&versionFlag, "version", false, "show Task version")
@@ -68,7 +70,9 @@ func main() {
 	pflag.BoolVarP(&verbose, "verbose", "v", false, "enables verbose mode")
 	pflag.BoolVarP(&silent, "silent", "s", false, "disables echoing")
 	pflag.BoolVar(&dry, "dry", false, "compiles and prints tasks in the order that they would be run, without executing them")
+	pflag.BoolVar(&summary, "summary", false, "show summary about a task")
 	pflag.StringVarP(&dir, "dir", "d", "", "sets directory of execution")
+	pflag.StringVarP(&output, "output", "o", "", "sets output style: [interleaved|group|prefixed]")
 	pflag.Parse()
 
 	if versionFlag {
@@ -87,11 +91,6 @@ func main() {
 		return
 	}
 
-	ctx := context.Background()
-	if !watch {
-		ctx = getSignalContext()
-	}
-
 	e := task.Executor{
 		Force:   force,
 		Watch:   watch,
@@ -99,12 +98,13 @@ func main() {
 		Silent:  silent,
 		Dir:     dir,
 		Dry:     dry,
-
+		Summary: summary,
-		Context: ctx,
 
 		Stdin:  os.Stdin,
 		Stdout: os.Stdout,
 		Stderr: os.Stderr,
+
+		OutputStyle: output,
 	}
 	if err := e.Setup(); err != nil {
 		log.Fatal(err)
@@ -126,14 +126,19 @@ func main() {
 		log.Fatal(err)
 	}
 
+	ctx := context.Background()
+	if !watch {
+		ctx = getSignalContext()
+	}
+
 	if status {
-		if err = e.Status(calls...); err != nil {
+		if err = e.Status(ctx, calls...); err != nil {
 			log.Fatal(err)
 		}
 		return
 	}
 
-	if err := e.Run(calls...); err != nil {
+	if err := e.Run(ctx, calls...); err != nil {
 		log.Fatal(err)
 	}
 }

completion/zsh/_task

@@ -5,7 +5,7 @@ function __list() {
     local -a scripts
 
     if [ -f Taskfile.yml ]; then
-        scripts=($(task -l | sed '1d' | sed 's/://' | awk '{ print $2 }'))
+        scripts=($(task -l | sed '1d' | sed 's/^\* //' | awk '{ print $1 }' | sed 's/:$//' | sed 's/:/\\:/'))
         _describe 'script' scripts
     fi
 }

docs/CNAME

@@ -1 +1 @@
-taskfile.org
+taskfile.dev

docs/README.md

@@ -28,8 +28,8 @@ guide to check the full schema documentation and Task features.
 ## Features
 
 - [Easy installation](installation.md): just download a single binary, add to
-  $PATH and you're done! Or you can also install using [Homebrew][homebrew] or
+  $PATH and you're done! Or you can also install using [Homebrew][homebrew],
-  [Snapcraft][snapcraft] if you want;
+  [Snapcraft][snapcraft], or [Scoop][scoop] if you want;
 - Available on CIs: by adding [this simple command](installation.md#install-script)
   to install on your CI script and you're done to use Task as part of your CI pipeline;
 - Truly cross-platform: while most build tools only work well on Linux or macOS,
@@ -38,9 +38,22 @@ guide to check the full schema documentation and Task features.
   if a given set of files haven't changed since last run (based either on its
   timestamp or content).
 
+## Sponsors
+
+[![Sponsors](https://opencollective.com/task/sponsors.svg?width=890)](https://opencollective.com/task)
+
+## Backers
+
+[![Backers](https://opencollective.com/task/backers.svg?width=890)](https://opencollective.com/task)
+
+## Contributors
+
+[![Contributors](https://opencollective.com/task/contributors.svg?width=890)](https://github.com/go-task/task/graphs/contributors)
+
 [make]: https://www.gnu.org/software/make/
 [go]: https://golang.org/
 [yaml]: http://yaml.org/
 [homebrew]: https://brew.sh/
 [snapcraft]: https://snapcraft.io/
+[scoop]: https://scoop.sh/
 [sh]: https://mvdan.cc/sh

docs/Taskfile.yml

@@ -0,0 +1,12 @@
+version: '2'
+
+tasks:
+  install:
+    desc: Installs docsify to work the on the documentation site
+    cmds:
+      - npm install docsify-cli -g
+
+  serve:
+    desc: Serves the documentation site locally
+    cmds:
+      - docsify serve docs

docs/_sidebar.md

@@ -4,5 +4,4 @@
 - [Examples](examples.md)
 - [Releasing Task](releasing_task.md)
 - [Alternative Task Runners](alternative_task_runners.md)
-- [Sponsors and Backers](sponsors_and_backers.md)
 - [![Github](https://icongram.jgog.in/simple/github.svg?color=808080&size=16)Github](https://github.com/go-task/task)

docs/index.html

@@ -33,7 +33,7 @@
         name: 'Task',
         repo: 'go-task/task',
         ga: 'UA-126286662-1',
-        themeColor: '#83d0f2',
+        themeColor: '#00add8',
         loadSidebar: true,
         auto2top: true,
         maxLevel: 3,

docs/installation.md

@@ -25,6 +25,19 @@ right:
 sudo snap install task
 ```
 
+## Scoop
+
+If you're on Windows and have [Scoop][scoop] installed, use `extras` bucket
+to install Task like:
+
+```cmd
+scoop bucket add extras
+scoop install task
+```
+
+This installation method is community owned. After a new release of Task, it
+may take some time until it's available on Scoop.
+
 ## Go
 
 Task now uses [Go Modules](https://github.com/golang/go/wiki/Modules), which
@@ -59,11 +72,11 @@ Both methods requires having the [Go][go] environment properly setup locally.
 ## Install script
 
 We also have a [install script][installscript], which is very useful on
-scanarios like CIs. Many thanks to [godownloader][godownloader] for allowing
+scenarios like CIs. Many thanks to [godownloader][godownloader] for allowing
 easily generating this script.
 
 ```bash
-curl -s https://taskfile.org/install.sh | sh
+curl -sL https://taskfile.dev/install.sh | sh
 ```
 
 > This method will download the binary on the local `./bin` directory by default.
@@ -74,3 +87,4 @@ curl -s https://taskfile.org/install.sh | sh
 [installscript]: https://github.com/go-task/task/blob/master/install-task.sh
 [releases]: https://github.com/go-task/task/releases
 [godownloader]: https://github.com/goreleaser/godownloader
+[scoop]: https://scoop.sh/

docs/releasing_task.md

@@ -1,6 +1,6 @@
 # Releasing Task
 
-The release process of Task is done is done with the help of
+The release process of Task is done with the help of
 [GoReleaser][goreleaser]. You can test the release process locally by calling
 the `test-release` task of the Taskfile.
 
@@ -22,7 +22,14 @@ the binaries:
 
 * Updating the current version on [snapcraft.yaml][snapcraftyaml];
 * Moving both `i386` and `amd64` new artifacts to the stable channel on
-the [Snapscraft dashboard][snapcraftdashboard]
+the [Snapcraft dashboard][snapcraftdashboard]
+
+# Scoop
+
+Scoop is a community owned installation method. Scoop owners usually take care
+of updating versions there by editing
+[this file](https://github.com/lukesampson/scoop-extras/blob/master/bucket/task.json).
+If you think its Task version is outdated, open an issue to let us know.
 
 [goreleaser]: https://goreleaser.com/#continuous_integration
 [homebrewtap]: https://github.com/go-task/homebrew-tap

docs/sponsors_and_backers.md

@@ -1,16 +0,0 @@
-# Sponsors and Backers
-
-## Sponsors
-
-[![Sponsors](https://opencollective.com/task/sponsors.svg?width=890)][opencollective]
-
-## Backers
-
-[![Backers](https://opencollective.com/task/backers.svg?width=890)][opencollective]
-
-## Contributors
-
-[![Contributors](https://opencollective.com/task/contributors.svg?width=890)][contributors]
-
-[opencollective]: https://opencollective.com/task
-[contributors]: https://github.com/go-task/task/graphs/contributors

docs/usage.md

@@ -31,23 +31,41 @@ interpreter. So you can write sh/bash commands and it will work even on
 Windows, where `sh` or `bash` are usually not available. Just remember any
 executable called must be available by the OS or in PATH.
 
-If you ommit a task name, "default" will be assumed.
+If you omit a task name, "default" will be assumed.
 
 ## Environment
 
-You can specify environment variables that are added when running a command:
+You can use `env` to set custom environment variables for a specific task:
 
 ```yaml
 version: '2'
 
 tasks:
-  build:
+  greet:
     cmds:
-      - echo $hallo
+      - echo $GREETING
     env:
-      hallo: welt
+      GREETING: Hey, there!
 ```
 
+Additionally, you can set globally environment variables, that'll be available
+to all tasks:
+
+```yaml
+version: '2'
+
+env:
+  GREETING: Hey, there!
+
+tasks:
+  greet:
+    cmds:
+      - echo $GREETING
+```
+
+> NOTE: `env` supports expansion and and retrieving output from a shell command
+> just like variables, as you can see on the [Variables](#variables) section.
+
 ## Operating System specific tasks
 
 If you add a `Taskfile_{{GOOS}}.yml` you can override or amend your Taskfile
@@ -83,7 +101,7 @@ Keep in mind that the version of the files should match. Also, when redefining
 a task the whole task is replaced, properties of the task are not merged.
 
 It's also possible to have an OS specific `Taskvars.yml` file, like
-`Taskvars_windows.yml`, `Taskfile_linux.yml`, or `Taskvars_darwin.yml`. See the
+`Taskvars_windows.yml`, `Taskvars_linux.yml`, or `Taskvars_darwin.yml`. See the
 [variables section](#variables) below.
 
 ## Including other Taskfiles
@@ -240,6 +258,10 @@ tasks:
 
 The above syntax is also supported in `deps`.
 
+> NOTE: If you want to call a task declared in the root Taskfile from within an
+> [included Taskfile](#including-other-taskfiles), add a leading `:` like this:
+> `task: :task-name`.
+
 ## Prevent unnecessary work
 
 If a task generates something, you can inform Task the source and generated
@@ -455,7 +477,7 @@ Task also adds the following functions:
 - `catLines`: Replaces Unix (\n) and Windows (\r\n) styled newlines with a space.
 - `toSlash`: Does nothing on Unix, but on Windows converts a string from `\`
   path format to `/`.
-- `fromSlash`: Oposite of `toSlash`. Does nothing on Unix, but on Windows
+- `fromSlash`: Opposite of `toSlash`. Does nothing on Unix, but on Windows
   converts a string from `\` path format to `/`.
 - `exeExt`: Returns the right executable extension for the current OS
   (`".exe"` for Windows, `""` for others).
@@ -488,7 +510,7 @@ tasks:
 ## Help
 
 Running `task --list` (or `task -l`) lists all tasks with a description.
-The following taskfile:
+The following Taskfile:
 
 ```yaml
 version: '2'
@@ -520,6 +542,51 @@ would print the following output:
 * test:    Run all the go tests.
 ```
 
+## Display summary of task
+
+Running `task --summary task-name` will show a summary of a task
+The following Taskfile:
+
+```yaml
+version: '2'
+
+tasks:
+  release:
+    deps: [build]
+    summary: |
+      Release your project to github
+
+      It will build your project before starting the release it.
+      Please make sure that you have set GITHUB_TOKEN before starting.
+    cmds:
+      - your-release-tool
+
+  build:
+    cmds:
+      - your-build-tool
+```
+
+with running ``task --summary release`` would print the following output:
+
+```
+task: release
+
+Release your project to github
+
+It will build your project before starting the release it.
+Please make sure that you have set GITHUB_TOKEN before starting.
+
+dependencies:
+ - build
+
+commands:
+ - your-release-tool
+```
+If a summary is missing, the description will be printed. 
+If the task does not have a summary or a description, a warning is printed.
+
+Please note: *showing the summary will not execute the command*.
+
 ## Silent mode
 
 Silent mode disables echoing of commands before Task runs it.
@@ -575,7 +642,7 @@ tasks:
 
 * Or globally with `--silent` or `-s` flag
 
-If you want to suppress stdout instead, just redirect a command to `/dev/null`:
+If you want to suppress STDOUT instead, just redirect a command to `/dev/null`:
 
 ```yaml
 version: '2'
@@ -620,7 +687,7 @@ tasks:
       - echo "Hello World"
 ```
 
-`ignore_error` can also be set for a task, which mean errors will be supressed
+`ignore_error` can also be set for a task, which mean errors will be suppressed
 for all commands. But keep in mind this option won't propagate to other tasks
 called either by `deps` or `cmds`!
 
@@ -686,6 +753,8 @@ $ task default
 [print-baz] baz
 ```
 
+> The `output` option can also be specified by the `--output` or `-o` flags.
+
 ## Watch tasks
 
 If you give a `--watch` or `-w` argument, task will watch for file changes

go.mod

@@ -9,16 +9,15 @@ require (
 	github.com/huandu/xstrings v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53
+	github.com/mattn/go-zglob v0.0.1
 	github.com/mitchellh/go-homedir v1.0.0
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/radovskyb/watcher v1.0.5
-	github.com/radovskyb/watcher v1.0.2
 	github.com/spf13/pflag v1.0.3
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.3.0
 	golang.org/x/crypto v0.0.0-20180830192347-182538f80094 // indirect
 	golang.org/x/net v0.0.0-20180826012351-8a410e7b638d // indirect
 	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
 	golang.org/x/sys v0.0.0-20180831094639-fa5fdf94c789 // indirect
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v3 v3.0.0-20190409140830-cdc409dda467
-	mvdan.cc/sh v0.0.0-20180829163519-3a244a89e2e5
+	mvdan.cc/sh v2.6.4+incompatible
 )

go.sum

@@ -4,6 +4,7 @@ github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88
 github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
 github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
@@ -17,18 +18,19 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU=
+github.com/mattn/go-zglob v0.0.1 h1:xsEx/XUoVlI6yXjqBK062zYhRTZltCNmYPx6v+8DNaY=
-github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
+github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/radovskyb/watcher v1.0.2 h1:9L5TsZUbo1nKhQEQPtICVc+x9UZQ6VPdBepLHyGw/bQ=
+github.com/radovskyb/watcher v1.0.5 h1:wqt7gb+HjGacvFoLTKeT44C+XVPxu7bvHvKT1IvZ7rw=
-github.com/radovskyb/watcher v1.0.2/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
+github.com/radovskyb/watcher v1.0.5/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/crypto v0.0.0-20180830192347-182538f80094 h1:rVTAlhYa4+lCfNxmAIEOGQRoD23UqP72M3+rSWVGDTg=
 golang.org/x/crypto v0.0.0-20180830192347-182538f80094/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I=
@@ -39,7 +41,7 @@ golang.org/x/sys v0.0.0-20180831094639-fa5fdf94c789 h1:T8D7l6WB3tLu+VpKvw06ieD/O
 golang.org/x/sys v0.0.0-20180831094639-fa5fdf94c789/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v3 v3.0.0-20190409140830-cdc409dda467 h1:w3VhdSYz2sIVz54Ta/eDCCfCQ4fQkDgRxMACggArIUw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20190409140830-cdc409dda467/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-mvdan.cc/sh v0.0.0-20180829163519-3a244a89e2e5 h1:FKi9XtQO5aNipfQ/qnnLCoM6gdFwPQY702RRbNRxjK8=
+mvdan.cc/sh v2.6.4+incompatible h1:eD6tDeh0pw+/TOTI1BBEryZ02rD2nMcFsgcvde7jffM=
-mvdan.cc/sh v0.0.0-20180829163519-3a244a89e2e5/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=
+mvdan.cc/sh v2.6.4+incompatible/go.mod h1:IeeQbZq+x2SUGBensq/jge5lLQbS3XT2ktyp3wrt4x8=

init.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 )
 
-const defaultTaskfile = `# https://taskfile.org
+const defaultTaskfile = `# https://taskfile.dev
 
 version: '2'
 

internal/execext/exec.go

@@ -5,9 +5,12 @@ import (
 	"errors"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/interp"
+	"mvdan.cc/sh/shell"
 	"mvdan.cc/sh/syntax"
 )
 
@@ -41,14 +44,10 @@ func RunCommand(ctx context.Context, opts *RunCommandOptions) error {
 	if len(environ) == 0 {
 		environ = os.Environ()
 	}
-	env, err := interp.EnvFromList(environ)
-	if err != nil {
-		return err
-	}
 
 	r, err := interp.New(
 		interp.Dir(opts.Dir),
-		interp.Env(env),
+		interp.Env(expand.ListEnviron(environ...)),
 
 		interp.Module(interp.DefaultExec),
 		interp.Module(interp.OpenDevImpls(interp.DefaultOpen)),
@@ -70,3 +69,18 @@ func IsExitError(err error) bool {
 		return false
 	}
 }
+
+// Expand is a helper to mvdan.cc/shell.Fields that returns the first field
+// if available.
+func Expand(s string) (string, error) {
+	s = filepath.ToSlash(s)
+	s = strings.Replace(s, " ", `\ `, -1)
+	fields, err := shell.Fields(s, nil)
+	if err != nil {
+		return "", err
+	}
+	if len(fields) > 0 {
+		return fields[0], nil
+	}
+	return "", nil
+}

internal/output/group.go

@@ -7,7 +7,7 @@ import (
 
 type Group struct{}
 
-func (Group) WrapWriter(w io.Writer, _ string) io.WriteCloser {
+func (Group) WrapWriter(w io.Writer, _ string) io.Writer {
 	return &groupWriter{writer: w}
 }
 

internal/output/interleaved.go

@@ -6,18 +6,6 @@ import (
 
 type Interleaved struct{}
 
-func (Interleaved) WrapWriter(w io.Writer, _ string) io.WriteCloser {
+func (Interleaved) WrapWriter(w io.Writer, _ string) io.Writer {
-	return nopWriterCloser{w: w}
+	return w
-}
-
-type nopWriterCloser struct {
-	w io.Writer
-}
-
-func (wc nopWriterCloser) Write(p []byte) (int, error) {
-	return wc.w.Write(p)
-}
-
-func (wc nopWriterCloser) Close() error {
-	return nil
 }

internal/output/output.go

@@ -5,5 +5,5 @@ import (
 )
 
 type Output interface {
-	WrapWriter(w io.Writer, prefix string) io.WriteCloser
+	WrapWriter(w io.Writer, prefix string) io.Writer
 }

internal/output/output_test.go

@@ -3,6 +3,7 @@ package output_test
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"testing"
 
 	"github.com/go-task/task/v2/internal/output"
@@ -24,7 +25,7 @@ func TestInterleaved(t *testing.T) {
 func TestGroup(t *testing.T) {
 	var b bytes.Buffer
 	var o output.Output = output.Group{}
-	var w = o.WrapWriter(&b, "")
+	var w = o.WrapWriter(&b, "").(io.WriteCloser)
 
 	fmt.Fprintln(w, "foo\nbar")
 	assert.Equal(t, "", b.String())
@@ -37,7 +38,7 @@ func TestGroup(t *testing.T) {
 func TestPrefixed(t *testing.T) {
 	var b bytes.Buffer
 	var o output.Output = output.Prefixed{}
-	var w = o.WrapWriter(&b, "prefix")
+	var w = o.WrapWriter(&b, "prefix").(io.WriteCloser)
 
 	t.Run("simple use cases", func(t *testing.T) {
 		b.Reset()

internal/output/prefixed.go

@@ -9,7 +9,7 @@ import (
 
 type Prefixed struct{}
 
-func (Prefixed) WrapWriter(w io.Writer, prefix string) io.WriteCloser {
+func (Prefixed) WrapWriter(w io.Writer, prefix string) io.Writer {
 	return &prefixWriter{writer: w, prefix: prefix}
 }
 
@@ -34,12 +34,12 @@ func (pw *prefixWriter) Close() error {
 
 func (pw *prefixWriter) writeOutputLines(force bool) error {
 	for {
-		line, err := pw.buff.ReadString('\n')
+		switch line, err := pw.buff.ReadString('\n'); err {
-		if err == nil {
+		case nil:
 			if err = pw.writeLine(line); err != nil {
 				return err
 			}
-		} else if err == io.EOF {
+		case io.EOF:
 			// if this line was not a complete line, re-add to the buffer
 			if !force && !strings.HasSuffix(line, "\n") {
 				_, err = pw.buff.WriteString(line)
@@ -47,7 +47,7 @@ func (pw *prefixWriter) writeOutputLines(force bool) error {
 			}
 
 			return pw.writeLine(line)
-		} else {
+		default:
 			return err
 		}
 	}

internal/status/checksum.go

@@ -17,6 +17,7 @@ type Checksum struct {
 	Dir     string
 	Task    string
 	Sources []string
+	Dry     bool
 }
 
 // IsUpToDate implements the Checker interface
@@ -36,9 +37,11 @@ func (c *Checksum) IsUpToDate() (bool, error) {
 		return false, nil
 	}
 
-	_ = os.MkdirAll(filepath.Join(c.Dir, ".task", "checksum"), 0755)
+	if !c.Dry {
-	if err = ioutil.WriteFile(checksumFile, []byte(newMd5+"\n"), 0644); err != nil {
+		_ = os.MkdirAll(filepath.Join(c.Dir, ".task", "checksum"), 0755)
-		return false, err
+		if err = ioutil.WriteFile(checksumFile, []byte(newMd5+"\n"), 0644); err != nil {
+			return false, err
+		}
 	}
 	return oldMd5 == newMd5, nil
 }

internal/status/glob.go

@@ -4,8 +4,9 @@ import (
 	"path/filepath"
 	"sort"
 
+	"github.com/go-task/task/v2/internal/execext"
+
 	"github.com/mattn/go-zglob"
-	"mvdan.cc/sh/shell"
 )
 
 func glob(dir string, globs []string) (files []string, err error) {
@@ -13,7 +14,7 @@ func glob(dir string, globs []string) (files []string, err error) {
 		if !filepath.IsAbs(g) {
 			g = filepath.Join(dir, g)
 		}
-		g, err = shell.Expand(g, nil)
+		g, err = execext.Expand(g)
 		if err != nil {
 			return nil, err
 		}

internal/summary/summary.go

@@ -0,0 +1,103 @@
+package summary
+
+import (
+	"strings"
+
+	"github.com/go-task/task/v2/internal/logger"
+	"github.com/go-task/task/v2/internal/taskfile"
+)
+
+func PrintTasks(l *logger.Logger, t *taskfile.Taskfile, c []taskfile.Call) {
+	for i, call := range c {
+		printSpaceBetweenSummaries(l, i)
+		PrintTask(l, t.Tasks[call.Task])
+	}
+}
+
+func printSpaceBetweenSummaries(l *logger.Logger, i int) {
+	spaceRequired := i > 0
+	if !spaceRequired {
+		return
+	}
+
+	l.Outf("")
+	l.Outf("")
+}
+
+func PrintTask(l *logger.Logger, t *taskfile.Task) {
+	printTaskName(l, t)
+	printTaskDescribingText(t, l)
+	printTaskDependencies(l, t)
+	printTaskCommands(l, t)
+}
+
+func printTaskDescribingText(t *taskfile.Task, l *logger.Logger) {
+	if hasSummary(t) {
+		printTaskSummary(l, t)
+	} else if hasDescription(t) {
+		printTaskDescription(l, t)
+	} else {
+		printNoDescriptionOrSummary(l)
+	}
+}
+
+func hasSummary(t *taskfile.Task) bool {
+	return t.Summary != ""
+}
+
+func printTaskSummary(l *logger.Logger, t *taskfile.Task) {
+	lines := strings.Split(t.Summary, "\n")
+	for i, line := range lines {
+		notLastLine := i+1 < len(lines)
+		if notLastLine || line != "" {
+			l.Outf(line)
+		}
+	}
+}
+
+func printTaskName(l *logger.Logger, t *taskfile.Task) {
+	l.Outf("task: %s", t.Task)
+	l.Outf("")
+}
+
+func hasDescription(t *taskfile.Task) bool {
+	return t.Desc != ""
+}
+
+func printTaskDescription(l *logger.Logger, t *taskfile.Task) {
+	l.Outf(t.Desc)
+}
+
+func printNoDescriptionOrSummary(l *logger.Logger) {
+	l.Outf("(task does not have description or summary)")
+}
+
+func printTaskDependencies(l *logger.Logger, t *taskfile.Task) {
+	if len(t.Deps) == 0 {
+		return
+	}
+
+	l.Outf("")
+	l.Outf("dependencies:")
+
+	for _, d := range t.Deps {
+		l.Outf(" - %s", d.Task)
+	}
+}
+
+func printTaskCommands(l *logger.Logger, t *taskfile.Task) {
+	if len(t.Cmds) == 0 {
+		return
+	}
+
+	l.Outf("")
+	l.Outf("commands:")
+	for _, c := range t.Cmds {
+		isCommand := c.Cmd != ""
+		if isCommand {
+			l.Outf(" - %s", c.Cmd)
+		} else {
+			l.Outf(" - Task: %s", c.Task)
+		}
+	}
+}

internal/summary/summary_test.go

@@ -0,0 +1,173 @@
+package summary_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/go-task/task/v2/internal/logger"
+	"github.com/go-task/task/v2/internal/summary"
+	"github.com/go-task/task/v2/internal/taskfile"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPrintsDependenciesIfPresent(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Deps: []*taskfile.Dep{
+			{Task: "dep1"},
+			{Task: "dep2"},
+			{Task: "dep3"},
+		},
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.Contains(t, buffer.String(), "\ndependencies:\n - dep1\n - dep2\n - dep3\n")
+}
+
+func createDummyLogger() (*bytes.Buffer, logger.Logger) {
+	buffer := &bytes.Buffer{}
+	l := logger.Logger{
+		Stderr:  buffer,
+		Stdout:  buffer,
+		Verbose: false,
+	}
+	return buffer, l
+}
+
+func TestDoesNotPrintDependenciesIfMissing(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Deps: []*taskfile.Dep{},
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.NotContains(t, buffer.String(), "dependencies:")
+}
+
+func TestPrintTaskName(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Task: "my-task-name",
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.Contains(t, buffer.String(), "task: my-task-name\n")
+}
+
+func TestPrintTaskCommandsIfPresent(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Cmds: []*taskfile.Cmd{
+			{Cmd: "command-1"},
+			{Cmd: "command-2"},
+			{Task: "task-1"},
+		},
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.Contains(t, buffer.String(), "\ncommands:\n")
+	assert.Contains(t, buffer.String(), "\n - command-1\n")
+	assert.Contains(t, buffer.String(), "\n - command-2\n")
+	assert.Contains(t, buffer.String(), "\n - Task: task-1\n")
+}
+
+func TestDoesNotPrintCommandIfMissing(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Cmds: []*taskfile.Cmd{},
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.NotContains(t, buffer.String(), "commands")
+}
+
+func TestLayout(t *testing.T) {
+	buffer, l := createDummyLogger()
+	task := &taskfile.Task{
+		Task:    "sample-task",
+		Summary: "line1\nline2\nline3\n",
+		Deps: []*taskfile.Dep{
+			{Task: "dependency"},
+		},
+		Cmds: []*taskfile.Cmd{
+			{Cmd: "command"},
+		},
+	}
+
+	summary.PrintTask(&l, task)
+
+	assert.Equal(t, expectedOutput(), buffer.String())
+}
+
+func expectedOutput() string {
+	expected := `task: sample-task
+
+line1
+line2
+line3
+
+dependencies:
+ - dependency
+
+commands:
+ - command
+`
+	return expected
+}
+
+func TestPrintDescriptionAsFallback(t *testing.T) {
+	buffer, l := createDummyLogger()
+	taskWithoutSummary := &taskfile.Task{
+		Desc: "description",
+	}
+
+	taskWithSummary := &taskfile.Task{
+		Desc:    "description",
+		Summary: "summary",
+	}
+	taskWithoutSummaryOrDescription := &taskfile.Task{}
+
+	summary.PrintTask(&l, taskWithoutSummary)
+
+	assert.Contains(t, buffer.String(), "description")
+
+	buffer.Reset()
+	summary.PrintTask(&l, taskWithSummary)
+
+	assert.NotContains(t, buffer.String(), "description")
+
+	buffer.Reset()
+	summary.PrintTask(&l, taskWithoutSummaryOrDescription)
+
+	assert.Contains(t, buffer.String(), "\n(task does not have description or summary)\n")
+
+}
+
+func TestPrintAllWithSpaces(t *testing.T) {
+	buffer, l := createDummyLogger()
+
+	t1 := &taskfile.Task{Task: "t1"}
+	t2 := &taskfile.Task{Task: "t2"}
+	t3 := &taskfile.Task{Task: "t3"}
+
+	tasks := make(taskfile.Tasks, 3)
+	tasks["t1"] = t1
+	tasks["t2"] = t2
+	tasks["t3"] = t3
+
+	summary.PrintTasks(&l,
+		&taskfile.Taskfile{Tasks: tasks},
+		[]taskfile.Call{{Task: "t1"}, {Task: "t2"}, {Task: "t3"}})
+
+	assert.True(t, strings.HasPrefix(buffer.String(), "task: t1"))
+	assert.Contains(t, buffer.String(), "\n(task does not have description or summary)\n\n\ntask: t2")
+	assert.Contains(t, buffer.String(), "\n(task does not have description or summary)\n\n\ntask: t3")
+
+}

internal/taskfile/merge.go

@@ -35,6 +35,13 @@ func Merge(t1, t2 *Taskfile, namespaces ...string) error {
 		t1.Vars[k] = v
 	}
 
+	if t1.Env == nil {
+		t1.Env = make(Vars)
+	}
+	for k, v := range t2.Env {
+		t1.Env[k] = v
+	}
+
 	if t1.Tasks == nil {
 		t1.Tasks = make(Tasks)
 	}
@@ -59,5 +66,8 @@ func Merge(t1, t2 *Taskfile, namespaces ...string) error {
 }
 
 func taskNameWithNamespace(taskName string, namespaces ...string) string {
+	if strings.HasPrefix(taskName, ":") {
+		return strings.TrimPrefix(taskName, ":")
+	}
 	return strings.Join(append(namespaces, taskName), NamespaceSeparator)
 }

internal/taskfile/read/taskfile.go

@@ -9,17 +9,22 @@ import (
 
 	"github.com/go-task/task/v2/internal/taskfile"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
-// ErrIncludedTaskfilesCantHaveIncludes is returned when a included Taskfile contains includes
+var (
-var ErrIncludedTaskfilesCantHaveIncludes = errors.New("task: Included Taskfiles can't have includes. Please, move the include to the main Taskfile")
+	// ErrIncludedTaskfilesCantHaveIncludes is returned when a included Taskfile contains includes
+	ErrIncludedTaskfilesCantHaveIncludes = errors.New("task: Included Taskfiles can't have includes. Please, move the include to the main Taskfile")
+
+	// ErrNoTaskfileFound is returned when Taskfile.yml is not found
+	ErrNoTaskfileFound = errors.New(`task: No Taskfile.yml found. Use "task --init" to create a new one`)
+)
 
 // Taskfile reads a Taskfile for a given directory
 func Taskfile(dir string) (*taskfile.Taskfile, error) {
 	path := filepath.Join(dir, "Taskfile.yml")
 	if _, err := os.Stat(path); err != nil {
-		return nil, fmt.Errorf(`No Taskfile.yml found. Use "task --init" to create a new one`)
+		return nil, ErrNoTaskfileFound
 	}
 	t, err := readTaskfile(path)
 	if err != nil {

internal/taskfile/read/taskvars.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/go-task/task/v2/internal/taskfile"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // Taskvars reads a Taskvars for a given directory

internal/taskfile/task.go

@@ -1,6 +1,6 @@
 package taskfile
 
-// Tasks representas a group of tasks
+// Tasks represents a group of tasks
 type Tasks map[string]*Task
 
 // Task represents a task
@@ -9,6 +9,7 @@ type Task struct {
 	Cmds        []*Cmd
 	Deps        []*Dep
 	Desc        string
+	Summary     string
 	Sources     []string
 	Generates   []string
 	Status      []string

internal/taskfile/taskfile.go

@@ -7,6 +7,7 @@ type Taskfile struct {
 	Output     string
 	Includes   map[string]string
 	Vars       Vars
+	Env        Vars
 	Tasks      Tasks
 }
 
@@ -23,6 +24,7 @@ func (tf *Taskfile) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		Output     string
 		Includes   map[string]string
 		Vars       Vars
+		Env        Vars
 		Tasks      Tasks
 	}
 	if err := unmarshal(&taskfile); err != nil {
@@ -33,6 +35,7 @@ func (tf *Taskfile) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	tf.Output = taskfile.Output
 	tf.Includes = taskfile.Includes
 	tf.Vars = taskfile.Vars
+	tf.Env = taskfile.Env
 	tf.Tasks = taskfile.Tasks
 	if tf.Expansions <= 0 {
 		tf.Expansions = 2

internal/taskfile/taskfile_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/go-task/task/v2/internal/taskfile"
 
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestCmdParse(t *testing.T) {

status.go

@@ -10,13 +10,13 @@ import (
 )
 
 // Status returns an error if any the of given tasks is not up-to-date
-func (e *Executor) Status(calls ...taskfile.Call) error {
+func (e *Executor) Status(ctx context.Context, calls ...taskfile.Call) error {
 	for _, call := range calls {
 		t, err := e.CompiledTask(call)
 		if err != nil {
 			return err
 		}
-		isUpToDate, err := isTaskUpToDate(e.Context, t)
+		isUpToDate, err := e.isTaskUpToDate(ctx, t)
 		if err != nil {
 			return err
 		}
@@ -27,12 +27,12 @@ func (e *Executor) Status(calls ...taskfile.Call) error {
 	return nil
 }
 
-func isTaskUpToDate(ctx context.Context, t *taskfile.Task) (bool, error) {
+func (e *Executor) isTaskUpToDate(ctx context.Context, t *taskfile.Task) (bool, error) {
 	if len(t.Status) > 0 {
-		return isTaskUpToDateStatus(ctx, t)
+		return e.isTaskUpToDateStatus(ctx, t)
 	}
 
-	checker, err := getStatusChecker(t)
+	checker, err := e.getStatusChecker(t)
 	if err != nil {
 		return false, err
 	}
@@ -40,15 +40,15 @@ func isTaskUpToDate(ctx context.Context, t *taskfile.Task) (bool, error) {
 	return checker.IsUpToDate()
 }
 
-func statusOnError(t *taskfile.Task) error {
+func (e *Executor) statusOnError(t *taskfile.Task) error {
-	checker, err := getStatusChecker(t)
+	checker, err := e.getStatusChecker(t)
 	if err != nil {
 		return err
 	}
 	return checker.OnError()
 }
 
-func getStatusChecker(t *taskfile.Task) (status.Checker, error) {
+func (e *Executor) getStatusChecker(t *taskfile.Task) (status.Checker, error) {
 	switch t.Method {
 	case "", "timestamp":
 		return &status.Timestamp{
@@ -61,6 +61,7 @@ func getStatusChecker(t *taskfile.Task) (status.Checker, error) {
 			Dir:     t.Dir,
 			Task:    t.Task,
 			Sources: t.Sources,
+			Dry:     e.Dry,
 		}, nil
 	case "none":
 		return status.None{}, nil
@@ -69,7 +70,7 @@ func getStatusChecker(t *taskfile.Task) (status.Checker, error) {
 	}
 }
 
-func isTaskUpToDateStatus(ctx context.Context, t *taskfile.Task) (bool, error) {
+func (e *Executor) isTaskUpToDateStatus(ctx context.Context, t *taskfile.Task) (bool, error) {
 	for _, s := range t.Status {
 		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
 			Command: s,

task.go

@@ -13,6 +13,7 @@ import (
 	"github.com/go-task/task/v2/internal/execext"
 	"github.com/go-task/task/v2/internal/logger"
 	"github.com/go-task/task/v2/internal/output"
+	"github.com/go-task/task/v2/internal/summary"
 	"github.com/go-task/task/v2/internal/taskfile"
 	"github.com/go-task/task/v2/internal/taskfile/read"
 	"github.com/go-task/task/v2/internal/taskfile/version"
@@ -36,16 +37,16 @@ type Executor struct {
 	Verbose  bool
 	Silent   bool
 	Dry      bool
-
+	Summary  bool
-	Context context.Context
 
 	Stdin  io.Reader
 	Stdout io.Writer
 	Stderr io.Writer
 
-	Logger   *logger.Logger
+	Logger      *logger.Logger
-	Compiler compiler.Compiler
+	Compiler    compiler.Compiler
-	Output   output.Output
+	Output      output.Output
+	OutputStyle string
 
 	taskvars taskfile.Vars
 
@@ -53,7 +54,7 @@ type Executor struct {
 }
 
 // Run runs Task
-func (e *Executor) Run(calls ...taskfile.Call) error {
+func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
 	// check if given tasks exist
 	for _, c := range calls {
 		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
@@ -63,12 +64,17 @@ func (e *Executor) Run(calls ...taskfile.Call) error {
 		}
 	}
 
+	if e.Summary {
+		summary.PrintTasks(e.Logger, e.Taskfile, calls)
+		return nil
+	}
+
 	if e.Watch {
 		return e.watchTasks(calls...)
 	}
 
 	for _, c := range calls {
-		if err := e.RunTask(e.Context, c); err != nil {
+		if err := e.RunTask(ctx, c); err != nil {
 			return err
 		}
 	}
@@ -92,9 +98,6 @@ func (e *Executor) Setup() error {
 		return fmt.Errorf(`task: could not parse taskfile version "%s": %v`, e.Taskfile.Version, err)
 	}
 
-	if e.Context == nil {
-		e.Context = context.Background()
-	}
 	if e.Stdin == nil {
 		e.Stdin = os.Stdin
 	}
@@ -134,6 +137,9 @@ func (e *Executor) Setup() error {
 	if !version.IsV22(v) && len(e.Taskfile.Includes) > 0 {
 		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
 	}
+	if e.OutputStyle != "" {
+		e.Taskfile.Output = e.OutputStyle
+	}
 	switch e.Taskfile.Output {
 	case "", "interleaved":
 		e.Output = output.Interleaved{}
@@ -182,7 +188,7 @@ func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
 	}
 
 	if !e.Force {
-		upToDate, err := isTaskUpToDate(ctx, t)
+		upToDate, err := e.isTaskUpToDate(ctx, t)
 		if err != nil {
 			return err
 		}
@@ -196,7 +202,7 @@ func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
 
 	for i := range t.Cmds {
 		if err := e.runCommand(ctx, t, call, i); err != nil {
-			if err2 := statusOnError(t); err2 != nil {
+			if err2 := e.statusOnError(t); err2 != nil {
 				e.Logger.VerboseErrf("task: error cleaning status on error: %v", err2)
 			}
 
@@ -242,8 +248,18 @@ func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfi
 
 		stdOut := e.Output.WrapWriter(e.Stdout, t.Prefix)
 		stdErr := e.Output.WrapWriter(e.Stderr, t.Prefix)
-		defer stdOut.Close()
+		defer func() {
-		defer stdErr.Close()
+			if _, ok := stdOut.(*os.File); !ok {
+				if closer, ok := stdOut.(io.Closer); ok {
+					closer.Close()
+				}
+			}
+			if _, ok := stdErr.(*os.File); !ok {
+				if closer, ok := stdErr.(io.Closer); ok {
+					closer.Close()
+				}
+			}
+		}()
 
 		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
 			Command: cmd.Cmd,
@@ -268,9 +284,9 @@ func getEnviron(t *taskfile.Task) []string {
 		return nil
 	}
 
-	envs := os.Environ()
+	environ := os.Environ()
 	for k, v := range t.Env.ToStringMap() {
-		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+		environ = append(environ, fmt.Sprintf("%s=%s", k, v))
 	}
-	return envs
+	return environ
 }

task_test.go

@@ -2,6 +2,7 @@ package task_test
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -40,7 +41,7 @@ func (fct fileContentTest) Run(t *testing.T) {
 		Stderr: ioutil.Discard,
 	}
 	assert.NoError(t, e.Setup(), "e.Setup()")
-	assert.NoError(t, e.Run(taskfile.Call{Task: fct.Target}), "e.Run(target)")
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: fct.Target}), "e.Run(target)")
 
 	for name, expectContent := range fct.Files {
 		t.Run(fct.name(name), func(t *testing.T) {
@@ -61,7 +62,8 @@ func TestEnv(t *testing.T) {
 		Target:    "default",
 		TrimSpace: false,
 		Files: map[string]string{
-			"env.txt": "GOOS='linux' GOARCH='amd64' CGO_ENABLED='0'\n",
+			"local.txt":  "GOOS='linux' GOARCH='amd64' CGO_ENABLED='0'\n",
+			"global.txt": "FOO='foo' BAR='overriden' BAZ='baz'\n",
 		},
 	}
 	tt.Run(t)
@@ -177,7 +179,7 @@ func TestVarsInvalidTmpl(t *testing.T) {
 		Stderr: ioutil.Discard,
 	}
 	assert.NoError(t, e.Setup(), "e.Setup()")
-	assert.EqualError(t, e.Run(taskfile.Call{Task: target}), expectError, "e.Run(target)")
+	assert.EqualError(t, e.Run(context.Background(), taskfile.Call{Task: target}), expectError, "e.Run(target)")
 }
 
 func TestParams(t *testing.T) {
@@ -229,7 +231,7 @@ func TestDeps(t *testing.T) {
 		Stderr: ioutil.Discard,
 	}
 	assert.NoError(t, e.Setup())
-	assert.NoError(t, e.Run(taskfile.Call{Task: "default"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "default"}))
 
 	for _, f := range files {
 		f = filepath.Join(dir, f)
@@ -257,14 +259,14 @@ func TestStatus(t *testing.T) {
 		Silent: true,
 	}
 	assert.NoError(t, e.Setup())
-	assert.NoError(t, e.Run(taskfile.Call{Task: "gen-foo"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "gen-foo"}))
 
 	if _, err := os.Stat(file); err != nil {
 		t.Errorf("File should exists: %v", err)
 	}
 
 	e.Silent = false
-	assert.NoError(t, e.Run(taskfile.Call{Task: "gen-foo"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "gen-foo"}))
 
 	if buff.String() != `task: Task "gen-foo" is up to date`+"\n" {
 		t.Errorf("Wrong output message: %s", buff.String())
@@ -272,16 +274,19 @@ func TestStatus(t *testing.T) {
 }
 
 func TestGenerates(t *testing.T) {
-	var srcTask = "sub/src.txt"
+	const (
-	var relTask = "rel.txt"
+		srcTask        = "sub/src.txt"
-	var absTask = "abs.txt"
+		relTask        = "rel.txt"
+		absTask        = "abs.txt"
+		fileWithSpaces = "my text file.txt"
+	)
 
 	// This test does not work with a relative dir.
 	dir, err := filepath.Abs("testdata/generates")
 	assert.NoError(t, err)
 	var srcFile = filepath.Join(dir, srcTask)
 
-	for _, task := range []string{srcTask, relTask, absTask} {
+	for _, task := range []string{srcTask, relTask, absTask, fileWithSpaces} {
 		path := filepath.Join(dir, task)
 		_ = os.Remove(path)
 		if _, err := os.Stat(path); err == nil {
@@ -297,13 +302,13 @@ func TestGenerates(t *testing.T) {
 	}
 	assert.NoError(t, e.Setup())
 
-	for _, theTask := range []string{relTask, absTask} {
+	for _, theTask := range []string{relTask, absTask, fileWithSpaces} {
 		var destFile = filepath.Join(dir, theTask)
 		var upToDate = fmt.Sprintf("task: Task \"%s\" is up to date\n", srcTask) +
 			fmt.Sprintf("task: Task \"%s\" is up to date\n", theTask)
 
 		// Run task for the first time.
-		assert.NoError(t, e.Run(taskfile.Call{Task: theTask}))
+		assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: theTask}))
 
 		if _, err := os.Stat(srcFile); err != nil {
 			t.Errorf("File should exists: %v", err)
@@ -318,7 +323,7 @@ func TestGenerates(t *testing.T) {
 		buff.Reset()
 
 		// Re-run task to ensure it's now found to be up-to-date.
-		assert.NoError(t, e.Run(taskfile.Call{Task: theTask}))
+		assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: theTask}))
 		if buff.String() != upToDate {
 			t.Errorf("Wrong output message: %s", buff.String())
 		}
@@ -349,14 +354,14 @@ func TestStatusChecksum(t *testing.T) {
 	}
 	assert.NoError(t, e.Setup())
 
-	assert.NoError(t, e.Run(taskfile.Call{Task: "build"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "build"}))
 	for _, f := range files {
 		_, err := os.Stat(filepath.Join(dir, f))
 		assert.NoError(t, err)
 	}
 
 	buff.Reset()
-	assert.NoError(t, e.Run(taskfile.Call{Task: "build"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "build"}))
 	assert.Equal(t, `task: Task "build" is up to date`+"\n", buff.String())
 }
 
@@ -387,7 +392,7 @@ func TestCyclicDep(t *testing.T) {
 		Stderr: ioutil.Discard,
 	}
 	assert.NoError(t, e.Setup())
-	assert.IsType(t, &task.MaximumTaskCallExceededError{}, e.Run(taskfile.Call{Task: "task-1"}))
+	assert.IsType(t, &task.MaximumTaskCallExceededError{}, e.Run(context.Background(), taskfile.Call{Task: "task-1"}))
 }
 
 func TestTaskVersion(t *testing.T) {
@@ -423,10 +428,10 @@ func TestTaskIgnoreErrors(t *testing.T) {
 	}
 	assert.NoError(t, e.Setup())
 
-	assert.NoError(t, e.Run(taskfile.Call{Task: "task-should-pass"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "task-should-pass"}))
-	assert.Error(t, e.Run(taskfile.Call{Task: "task-should-fail"}))
+	assert.Error(t, e.Run(context.Background(), taskfile.Call{Task: "task-should-fail"}))
-	assert.NoError(t, e.Run(taskfile.Call{Task: "cmd-should-pass"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "cmd-should-pass"}))
-	assert.Error(t, e.Run(taskfile.Call{Task: "cmd-should-fail"}))
+	assert.Error(t, e.Run(context.Background(), taskfile.Call{Task: "cmd-should-fail"}))
 }
 
 func TestExpand(t *testing.T) {
@@ -444,7 +449,7 @@ func TestExpand(t *testing.T) {
 		Stderr: &buff,
 	}
 	assert.NoError(t, e.Setup())
-	assert.NoError(t, e.Run(taskfile.Call{Task: "pwd"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "pwd"}))
 	assert.Equal(t, home, strings.TrimSpace(buff.String()))
 }
 
@@ -463,7 +468,7 @@ func TestDry(t *testing.T) {
 		Dry:    true,
 	}
 	assert.NoError(t, e.Setup())
-	assert.NoError(t, e.Run(taskfile.Call{Task: "build"}))
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "build"}))
 
 	assert.Equal(t, "touch file.txt", strings.TrimSpace(buff.String()))
 	if _, err := os.Stat(file); err == nil {
@@ -471,6 +476,32 @@ func TestDry(t *testing.T) {
 	}
 }
 
+// TestDryChecksum tests if the checksum file is not being written to disk
+// if the dry mode is enabled.
+func TestDryChecksum(t *testing.T) {
+	const dir = "testdata/dry_checksum"
+
+	checksumFile := filepath.Join(dir, ".task/checksum/default")
+	_ = os.Remove(checksumFile)
+
+	e := task.Executor{
+		Dir:    dir,
+		Stdout: ioutil.Discard,
+		Stderr: ioutil.Discard,
+		Dry:    true,
+	}
+	assert.NoError(t, e.Setup())
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "default"}))
+
+	_, err := os.Stat(checksumFile)
+	assert.Error(t, err, "checksum file should not exist")
+
+	e.Dry = false
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "default"}))
+	_, err = os.Stat(checksumFile)
+	assert.NoError(t, err, "checksum file should exist")
+}
+
 func TestIncludes(t *testing.T) {
 	tt := fileContentTest{
 		Dir:       "testdata/includes",
@@ -510,3 +541,37 @@ func TestIncludesDependencies(t *testing.T) {
 	}
 	tt.Run(t)
 }
+
+func TestIncludesCallingRoot(t *testing.T) {
+	tt := fileContentTest{
+		Dir:       "testdata/includes_call_root_task",
+		Target:    "included:call-root",
+		TrimSpace: true,
+		Files: map[string]string{
+			"root_task.txt": "root task",
+		},
+	}
+	tt.Run(t)
+}
+
+func TestSummary(t *testing.T) {
+	const dir = "testdata/summary"
+
+	var buff bytes.Buffer
+	e := task.Executor{
+		Dir:     dir,
+		Stdout:  &buff,
+		Stderr:  &buff,
+		Summary: true,
+		Silent:  true,
+	}
+	assert.NoError(t, e.Setup())
+	assert.NoError(t, e.Run(context.Background(), taskfile.Call{Task: "task-with-summary"}, taskfile.Call{Task: "other-task-with-summary"}))
+	assert.Equal(t, readTestFixture(t, dir, "task-with-summary.txt"), buff.String())
+}
+
+func readTestFixture(t *testing.T, dir string, file string) string {
+	b, err := ioutil.ReadFile(dir + "/" + file)
+	assert.NoError(t, err, "error reading text fixture")
+	return string(b)
+}

testdata/dry_checksum/Taskfile.yml

@@ -0,0 +1,9 @@
+version: '2'
+
+tasks:
+  default:
+    cmds:
+      - echo "Working..."
+    sources:
+      - source.txt
+    method: checksum

testdata/dry_checksum/source.txt

@@ -0,0 +1 @@
+Something...

testdata/env/.gitignore

@@ -1 +1 @@
-env.txt
+*.txt

testdata/env/Taskfile.yml

@@ -1,10 +1,33 @@
-default:
+version: '2'
-  vars:
+
-    AMD64: amd64
+vars:
-  env:
+  BAZ:
-    GOOS: linux
+    sh: echo baz
-    GOARCH: "{{.AMD64}}"
+
-    CGO_ENABLED:
+env:
-      sh: echo '0'
+  FOO: foo
-  cmds:
+  BAR: bar
-    - echo "GOOS='$GOOS' GOARCH='$GOARCH' CGO_ENABLED='$CGO_ENABLED'" > env.txt
+  BAZ: "{{.BAZ}}"
+
+tasks:
+  default:
+    cmds:
+      - task: local
+      - task: global
+
+  local:
+    vars:
+      AMD64: amd64
+    env:
+      GOOS: linux
+      GOARCH: "{{.AMD64}}"
+      CGO_ENABLED:
+        sh: echo '0'
+    cmds:
+      - echo "GOOS='$GOOS' GOARCH='$GOARCH' CGO_ENABLED='$CGO_ENABLED'" > local.txt
+
+  global:
+    env:
+      BAR: overriden
+    cmds:
+      - echo "FOO='$FOO' BAR='$BAR' BAZ='$BAZ'" > global.txt

testdata/generates/Taskfile.yml

@@ -29,3 +29,13 @@ sub/src.txt:
     - echo "hello world" > sub/src.txt
   status:
     - test -f sub/src.txt
+
+'my text file.txt':
+  desc: generate file with spaces in the name
+  deps: [sub/src.txt]
+  cmds:
+    - cat sub/src.txt > 'my text file.txt'
+  sources:
+    - sub/src.txt
+  generates:
+    - 'my text file.txt'

testdata/includes_call_root_task/.gitignore

@@ -0,0 +1 @@
+*.txt

testdata/includes_call_root_task/Taskfile.yml

@@ -0,0 +1,9 @@
+version: '2'
+
+includes:
+  included: Taskfile2.yml
+
+tasks:
+  root-task:
+    cmds:
+      - echo "root task" > root_task.txt

testdata/includes_call_root_task/Taskfile2.yml

@@ -0,0 +1,6 @@
+version: '2'
+
+tasks:
+  call-root:
+    cmds:
+      - task: :root-task

testdata/summary/Taskfile.yml

@@ -0,0 +1,26 @@
+version: 2
+
+tasks:
+  task-with-summary:
+    deps: [dependend-task-1, dependend-task-2]
+    summary: |
+      summary of task-with-summary - line 1
+      line 2
+      line 3
+    cmds:
+      - echo 'task-with-summary was executed'
+      - echo 'another command'
+      - exit 0
+
+  other-task-with-summary:
+    summary: summary of other-task-with-summary
+    cmds:
+      - echo 'other-task-with-summary was executed'
+
+  dependend-task-1:
+    cmds:
+      - echo 'dependend-task-1 was executed'
+
+  dependend-task-2:
+    cmds:
+      - echo 'dependend-task-2 was executed'

testdata/summary/task-with-summary.txt

@@ -0,0 +1,22 @@
+task: task-with-summary
+
+summary of task-with-summary - line 1
+line 2
+line 3
+
+dependencies:
+ - dependend-task-1
+ - dependend-task-2
+
+commands:
+ - echo 'task-with-summary was executed'
+ - echo 'another command'
+ - exit 0
+
+
+task: other-task-with-summary
+
+summary of other-task-with-summary
+
+commands:
+ - echo 'other-task-with-summary was executed'

variables.go

@@ -3,10 +3,9 @@ package task
 import (
 	"path/filepath"
 
+	"github.com/go-task/task/v2/internal/execext"
 	"github.com/go-task/task/v2/internal/taskfile"
 	"github.com/go-task/task/v2/internal/templater"
-
-	"mvdan.cc/sh/shell"
 )
 
 // CompiledTask returns a copy of a task, but replacing variables in almost all
@@ -31,13 +30,13 @@ func (e *Executor) CompiledTask(call taskfile.Call) (*taskfile.Task, error) {
 		Status:      r.ReplaceSlice(origTask.Status),
 		Dir:         r.Replace(origTask.Dir),
 		Vars:        nil,
-		Env:         r.ReplaceVars(origTask.Env),
+		Env:         nil,
 		Silent:      origTask.Silent,
 		Method:      r.Replace(origTask.Method),
 		Prefix:      r.Replace(origTask.Prefix),
 		IgnoreError: origTask.IgnoreError,
 	}
-	new.Dir, err = shell.Expand(new.Dir, nil)
+	new.Dir, err = execext.Expand(new.Dir)
 	if err != nil {
 		return nil, err
 	}
@@ -47,6 +46,14 @@ func (e *Executor) CompiledTask(call taskfile.Call) (*taskfile.Task, error) {
 	if new.Prefix == "" {
 		new.Prefix = new.Task
 	}
+
+	new.Env = make(taskfile.Vars, len(e.Taskfile.Env)+len(origTask.Env))
+	for k, v := range r.ReplaceVars(e.Taskfile.Env) {
+		new.Env[k] = v
+	}
+	for k, v := range r.ReplaceVars(origTask.Env) {
+		new.Env[k] = v
+	}
 	for k, v := range new.Env {
 		static, err := e.Compiler.HandleDynamicVar(v)
 		if err != nil {

vendor/github.com/radovskyb/watcher/README.md

@@ -15,7 +15,8 @@ Events contain the `os.FileInfo` of the file or directory that the event is base
 [Watcher Command](#command)  
 
 # Update
-Event.Path for Rename and Move events is now returned in the format of `fromPath -> toPath`
+- Added new file filter hooks (Including a built in regexp filtering hook) [Dec 12, 2018]
+- Event.Path for Rename and Move events is now returned in the format of `fromPath -> toPath`
 
 #### Chmod event is not supported under windows.
 
@@ -68,6 +69,11 @@ func main() {
 	// Only notify rename and move events.
 	w.FilterOps(watcher.Rename, watcher.Move)
 
+	// Only files that match the regular expression during file listings
+	// will be watched.
+	r := regexp.MustCompile("^abc$")
+	w.AddFilterHook(watcher.RegexFilterHook(r, false))
+
 	go func() {
 		for {
 			select {
@@ -128,6 +134,8 @@ Usage of watcher:
     	command to run when an event occurs
   -dotfiles
     	watch dot files (default true)
+  -ignore string
+        comma separated list of paths to ignore
   -interval string
     	watcher poll interval (default "100ms")
   -keepalive

vendor/github.com/radovskyb/watcher/ishidden.go

@@ -0,0 +1,12 @@
+// +build !windows
+
+package watcher
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+func isHiddenFile(path string) (bool, error) {
+	return strings.HasPrefix(filepath.Base(path), "."), nil
+}

vendor/github.com/radovskyb/watcher/ishidden_windows.go

@@ -0,0 +1,21 @@
+// +build windows
+
+package watcher
+
+import (
+	"syscall"
+)
+
+func isHiddenFile(path string) (bool, error) {
+	pointer, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return false, err
+	}
+
+	attributes, err := syscall.GetFileAttributes(pointer)
+	if err != nil {
+		return false, err
+	}
+
+	return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil
+}

vendor/github.com/radovskyb/watcher/samefile.go

@@ -4,6 +4,6 @@ package watcher
 
 import "os"
 
-func SameFile(fi1, fi2 os.FileInfo) bool {
+func sameFile(fi1, fi2 os.FileInfo) bool {
 	return os.SameFile(fi1, fi2)
 }

vendor/github.com/radovskyb/watcher/samefile_windows.go

@@ -4,7 +4,7 @@ package watcher
 
 import "os"
 
-func SameFile(fi1, fi2 os.FileInfo) bool {
+func sameFile(fi1, fi2 os.FileInfo) bool {
 	return fi1.ModTime() == fi2.ModTime() &&
 		fi1.Size() == fi2.Size() &&
 		fi1.Mode() == fi2.Mode() &&

vendor/github.com/radovskyb/watcher/watcher.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -24,6 +25,10 @@ var (
 	// ErrWatchedFileDeleted is an error that occurs when a file or folder that was
 	// being watched has been deleted.
 	ErrWatchedFileDeleted = errors.New("error: watched file or folder deleted")
+
+	// ErrSkip is less of an error, but more of a way for path hooks to skip a file or
+	// directory.
+	ErrSkip = errors.New("error: skipping file")
 )
 
 // An Op is a type that is used to describe what type
@@ -69,16 +74,43 @@ type Event struct {
 // String returns a string depending on what type of event occurred and the
 // file name associated with the event.
 func (e Event) String() string {
-	if e.FileInfo != nil {
+	if e.FileInfo == nil {
-		pathType := "FILE"
+		return "???"
-		if e.IsDir() {
-			pathType = "DIRECTORY"
-		}
-		return fmt.Sprintf("%s %q %s [%s]", pathType, e.Name(), e.Op, e.Path)
 	}
-	return "???"
+
+	pathType := "FILE"
+	if e.IsDir() {
+		pathType = "DIRECTORY"
+	}
+	return fmt.Sprintf("%s %q %s [%s]", pathType, e.Name(), e.Op, e.Path)
 }
 
+// FilterFileHookFunc is a function that is called to filter files during listings.
+// If a file is ok to be listed, nil is returned otherwise ErrSkip is returned.
+type FilterFileHookFunc func(info os.FileInfo, fullPath string) error
+
+// RegexFilterHook is a function that accepts or rejects a file
+// for listing based on whether it's filename or full path matches
+// a regular expression.
+func RegexFilterHook(r *regexp.Regexp, useFullPath bool) FilterFileHookFunc {
+	return func(info os.FileInfo, fullPath string) error {
+		str := info.Name()
+
+		if useFullPath {
+			str = fullPath
+		}
+
+		// Match
+		if r.MatchString(str) {
+			return nil
+		}
+
+		// No match.
+		return ErrSkip
+	}
+}
+
+// Watcher describes a process that watches files for changes.
 type Watcher struct {
 	Event  chan Event
 	Error  chan error
@@ -88,6 +120,7 @@ type Watcher struct {
 
 	// mu protects the following.
 	mu           *sync.Mutex
+	ffh          []FilterFileHookFunc
 	running      bool
 	names        map[string]bool        // bool for recursive or not.
 	files        map[string]os.FileInfo // map of files.
@@ -125,6 +158,13 @@ func (w *Watcher) SetMaxEvents(delta int) {
 	w.mu.Unlock()
 }
 
+// AddFilterHook
+func (w *Watcher) AddFilterHook(f FilterFileHookFunc) {
+	w.mu.Lock()
+	w.ffh = append(w.ffh, f)
+	w.mu.Unlock()
+}
+
 // IgnoreHiddenFiles sets the watcher to ignore any file or directory
 // that starts with a dot.
 func (w *Watcher) IgnoreHiddenFiles(ignore bool) {
@@ -157,7 +197,13 @@ func (w *Watcher) Add(name string) (err error) {
 	// If name is on the ignored list or if hidden files are
 	// ignored and name is a hidden file or directory, simply return.
 	_, ignored := w.ignored[name]
-	if ignored || (w.ignoreHidden && strings.HasPrefix(name, ".")) {
+
+	isHidden, err := isHiddenFile(name)
+	if err != nil {
+		return err
+	}
+
+	if ignored || (w.ignoreHidden && isHidden) {
 		return nil
 	}
 
@@ -200,18 +246,36 @@ func (w *Watcher) list(name string) (map[string]os.FileInfo, error) {
 	// Add all of the files in the directory to the file list as long
 	// as they aren't on the ignored list or are hidden files if ignoreHidden
 	// is set to true.
+outer:
 	for _, fInfo := range fInfoList {
 		path := filepath.Join(name, fInfo.Name())
 		_, ignored := w.ignored[path]
-		if ignored || (w.ignoreHidden && strings.HasPrefix(fInfo.Name(), ".")) {
+
+		isHidden, err := isHiddenFile(path)
+		if err != nil {
+			return nil, err
+		}
+
+		if ignored || (w.ignoreHidden && isHidden) {
 			continue
 		}
+
+		for _, f := range w.ffh {
+			err := f(fInfo, path)
+			if err == ErrSkip {
+				continue outer
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		fileList[path] = fInfo
 	}
 	return fileList, nil
 }
 
-// Add adds either a single file or directory recursively to the file list.
+// AddRecursive adds either a single file or directory recursively to the file list.
 func (w *Watcher) AddRecursive(name string) (err error) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
@@ -242,10 +306,27 @@ func (w *Watcher) listRecursive(name string) (map[string]os.FileInfo, error) {
 		if err != nil {
 			return err
 		}
+
+		for _, f := range w.ffh {
+			err := f(info, path)
+			if err == ErrSkip {
+				return nil
+			}
+			if err != nil {
+				return err
+			}
+		}
+
 		// If path is ignored and it's a directory, skip the directory. If it's
 		// ignored and it's a single file, skip the file.
 		_, ignored := w.ignored[path]
-		if ignored || (w.ignoreHidden && strings.HasPrefix(info.Name(), ".")) {
+
+		isHidden, err := isHiddenFile(path)
+		if err != nil {
+			return err
+		}
+
+		if ignored || (w.ignoreHidden && isHidden) {
 			if info.IsDir() {
 				return filepath.SkipDir
 			}
@@ -292,7 +373,7 @@ func (w *Watcher) Remove(name string) (err error) {
 	return nil
 }
 
-// Remove removes either a single file or a directory recursively from
+// RemoveRecursive removes either a single file or a directory recursively from
 // the file's list.
 func (w *Watcher) RemoveRecursive(name string) (err error) {
 	w.mu.Lock()
@@ -346,11 +427,17 @@ func (w *Watcher) Ignore(paths ...string) (err error) {
 	return nil
 }
 
+// WatchedFiles returns a map of files added to a Watcher.
 func (w *Watcher) WatchedFiles() map[string]os.FileInfo {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 
-	return w.files
+	files := make(map[string]os.FileInfo)
+	for k, v := range w.files {
+		files[k] = v
+	}
+
+	return files
 }
 
 // fileInfo is an implementation of os.FileInfo that can be used
@@ -560,7 +647,7 @@ func (w *Watcher) pollEvents(files map[string]os.FileInfo, evt chan Event,
 	// Check for renames and moves.
 	for path1, info1 := range removes {
 		for path2, info2 := range creates {
-			if SameFile(info1, info2) {
+			if sameFile(info1, info2) {
 				e := Event{
 					Op:       Move,
 					Path:     fmt.Sprintf("%s -> %s", path1, path2),
@@ -606,6 +693,7 @@ func (w *Watcher) Wait() {
 	w.wg.Wait()
 }
 
+// Close stops a Watcher and unlocks its mutex, then sends a close signal.
 func (w *Watcher) Close() {
 	w.mu.Lock()
 	if !w.running {

vendor/github.com/stretchr/testify/LICENSE

@@ -1,22 +1,21 @@
-Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell
+MIT License
 
-Please consider promoting this project if you find it useful.
+Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell
 
-Permission is hereby granted, free of charge, to any person 
+Permission is hereby granted, free of charge, to any person obtaining a copy
-obtaining a copy of this software and associated documentation 
+of this software and associated documentation files (the "Software"), to deal
-files (the "Software"), to deal in the Software without restriction, 
+in the Software without restriction, including without limitation the rights
-including without limitation the rights to use, copy, modify, merge, 
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-publish, distribute, sublicense, and/or sell copies of the Software, 
+copies of the Software, and to permit persons to whom the Software is
-and to permit persons to whom the Software is furnished to do so, 
+furnished to do so, subject to the following conditions:
-subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included
+The above copyright notice and this permission notice shall be included in all
-in all copies or substantial portions of the Software.
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+SOFTWARE.

vendor/github.com/stretchr/testify/assert/assertions.go

@@ -39,7 +39,7 @@ type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool
 // for table driven tests.
 type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool
 
-// ValuesAssertionFunc is a common function prototype when validating an error value.  Can be useful
+// ErrorAssertionFunc is a common function prototype when validating an error value.  Can be useful
 // for table driven tests.
 type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool
 
@@ -179,7 +179,11 @@ func messageFromMsgAndArgs(msgAndArgs ...interface{}) string {
 		return ""
 	}
 	if len(msgAndArgs) == 1 {
-		return msgAndArgs[0].(string)
+		msg := msgAndArgs[0]
+		if msgAsStr, ok := msg.(string); ok {
+			return msgAsStr
+		}
+		return fmt.Sprintf("%+v", msg)
 	}
 	if len(msgAndArgs) > 1 {
 		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
@@ -415,6 +419,17 @@ func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
 	return Fail(t, "Expected value not to be nil.", msgAndArgs...)
 }
 
+// containsKind checks if a specified kind in the slice of kinds.
+func containsKind(kinds []reflect.Kind, kind reflect.Kind) bool {
+	for i := 0; i < len(kinds); i++ {
+		if kind == kinds[i] {
+			return true
+		}
+	}
+
+	return false
+}
+
 // isNil checks if a specified object is nil or not, without Failing.
 func isNil(object interface{}) bool {
 	if object == nil {
@@ -423,7 +438,14 @@ func isNil(object interface{}) bool {
 
 	value := reflect.ValueOf(object)
 	kind := value.Kind()
-	if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {
+	isNilableKind := containsKind(
+		[]reflect.Kind{
+			reflect.Chan, reflect.Func,
+			reflect.Interface, reflect.Map,
+			reflect.Ptr, reflect.Slice},
+		kind)
+
+	if isNilableKind && value.IsNil() {
 		return true
 	}
 
@@ -1327,7 +1349,7 @@ func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) {
 }
 
 // diff returns a diff of both values as long as both are of the same type and
-// are a struct, map, slice or array. Otherwise it returns an empty string.
+// are a struct, map, slice, array or string. Otherwise it returns an empty string.
 func diff(expected interface{}, actual interface{}) string {
 	if expected == nil || actual == nil {
 		return ""
@@ -1345,7 +1367,7 @@ func diff(expected interface{}, actual interface{}) string {
 	}
 
 	var e, a string
-	if ek != reflect.String {
+	if et != reflect.TypeOf("") {
 		e = spewConfig.Sdump(expected)
 		a = spewConfig.Sdump(actual)
 	} else {

vendor/gopkg.in/yaml.v2/.travis.yml

@@ -1,12 +0,0 @@
-language: go
-
-go:
-    - 1.4
-    - 1.5
-    - 1.6
-    - 1.7
-    - 1.8
-    - 1.9
-    - tip
-
-go_import_path: gopkg.in/yaml.v2

vendor/gopkg.in/yaml.v2/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

vendor/gopkg.in/yaml.v2/writerc.go

@@ -1,26 +0,0 @@
-package yaml
-
-// Set the writer error and return false.
-func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool {
-	emitter.error = yaml_WRITER_ERROR
-	emitter.problem = problem
-	return false
-}
-
-// Flush the output buffer.
-func yaml_emitter_flush(emitter *yaml_emitter_t) bool {
-	if emitter.write_handler == nil {
-		panic("write handler not set")
-	}
-
-	// Check if the buffer is empty.
-	if emitter.buffer_pos == 0 {
-		return true
-	}
-
-	if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil {
-		return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error())
-	}
-	emitter.buffer_pos = 0
-	return true
-}

vendor/gopkg.in/yaml.v3/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+
+go:
+    - "1.4"
+    - "1.5"
+    - "1.6"
+    - "1.7"
+    - "1.8"
+    - "1.9"
+    - "1.10"
+    - "1.11"
+    - "1.12"
+    - tip
+
+go_import_path: gopkg.in/yaml.v2

vendor/gopkg.in/yaml.v3/LICENSE

@@ -1,16 +1,17 @@
+
+This project is covered by two different licenses: MIT and Apache.
+
+#### MIT License ####
+
 The following files were ported to Go from C files of libyaml, and thus
-are still covered by their original copyright and license:
+are still covered by their original MIT license, with the additional
+copyright staring in 2011 when the project was ported over:
 
-    apic.go
+    apic.go emitterc.go parserc.go readerc.go scannerc.go
-    emitterc.go
+    writerc.go yamlh.go yamlprivateh.go
-    parserc.go
-    readerc.go
-    scannerc.go
-    writerc.go
-    yamlh.go
-    yamlprivateh.go
 
-Copyright (c) 2006 Kirill Simonov
+Copyright (c) 2006-2010 Kirill Simonov
+Copyright (c) 2006-2011 Kirill Simonov
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -29,3 +30,21 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+### Apache License ###
+
+All the remaining project files are covered by the Apache license:
+
+Copyright (c) 2011-2019 Canonical Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

vendor/gopkg.in/yaml.v3/README.md

@@ -12,7 +12,23 @@ C library to parse and generate YAML data quickly and reliably.
 Compatibility
 -------------
 
-The yaml package supports most of YAML 1.1 and 1.2, including support for
+The yaml package supports most of YAML 1.2, but preserves some behavior
+from 1.1 for backwards compatibility.
+
+Specifically, as of v3 of the yaml package:
+
+ - YAML 1.1 bools (_yes/no, on/off_) are supported as long as they are being
+   decoded into a typed bool value. Otherwise they behave as a string. Booleans
+   in YAML 1.2 are _true/false_ only.
+ - Octals encode and decode as _0777_ per YAML 1.1, rather than _0o777_
+   as specified in YAML 1.2, because most parsers still use the old format.
+   Octals in the  _0o777_ format are supported though, so new files work.
+ - Does not support base-60 floats. These are gone from YAML 1.2, and were
+   actually never supported by this package as it's clearly a poor choice.
+
+and offers backwards
+compatibility with YAML 1.1 in some cases.
+1.2, including support for
 anchors, tags, map merging, etc. Multi-document unmarshalling is not yet
 implemented, and base-60 floats from YAML 1.1 are purposefully not
 supported since they're a poor design and are gone in YAML 1.2.
@@ -20,29 +36,30 @@ supported since they're a poor design and are gone in YAML 1.2.
 Installation and usage
 ----------------------
 
-The import path for the package is *gopkg.in/yaml.v2*.
+The import path for the package is *gopkg.in/yaml.v3*.
 
 To install it, run:
 
-    go get gopkg.in/yaml.v2
+    go get gopkg.in/yaml.v3
 
 API documentation
 -----------------
 
 If opened in a browser, the import path itself leads to the API documentation:
 
-  * [https://gopkg.in/yaml.v2](https://gopkg.in/yaml.v2)
+  - [https://gopkg.in/yaml.v3](https://gopkg.in/yaml.v3)
 
 API stability
 -------------
 
-The package API for yaml v2 will remain stable as described in [gopkg.in](https://gopkg.in).
+The package API for yaml v3 will remain stable as described in [gopkg.in](https://gopkg.in).
 
 
 License
 -------
 
-The yaml package is licensed under the Apache License 2.0. Please see the LICENSE file for details.
+The yaml package is licensed under the MIT and Apache License 2.0 licenses.
+Please see the LICENSE file for details.
 
 
 Example
@@ -55,7 +72,7 @@ import (
         "fmt"
         "log"
 
-        "gopkg.in/yaml.v2"
+        "gopkg.in/yaml.v3"
 )
 
 var data = `

vendor/gopkg.in/yaml.v3/apic.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (
@@ -138,7 +160,7 @@ func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) {
 	emitter.canonical = canonical
 }
 
-//// Set the indentation increment.
+// Set the indentation increment.
 func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) {
 	if indent < 2 || indent > 9 {
 		indent = 2
@@ -288,29 +310,14 @@ func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) {
 	}
 }
 
-///*
+// Create ALIAS.
-// * Create ALIAS.
+func yaml_alias_event_initialize(event *yaml_event_t, anchor []byte) bool {
-// */
+	*event = yaml_event_t{
-//
+		typ:    yaml_ALIAS_EVENT,
-//YAML_DECLARE(int)
+		anchor: anchor,
-//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t)
+	}
-//{
+	return true
-//    mark yaml_mark_t = { 0, 0, 0 }
+}
-//    anchor_copy *yaml_char_t = NULL
-//
-//    assert(event) // Non-NULL event object is expected.
-//    assert(anchor) // Non-NULL anchor is expected.
-//
-//    if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0
-//
-//    anchor_copy = yaml_strdup(anchor)
-//    if (!anchor_copy)
-//        return 0
-//
-//    ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark)
-//
-//    return 1
-//}
 
 // Create SCALAR.
 func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool {

vendor/gopkg.in/yaml.v3/decode.go

@@ -1,3 +1,18 @@
+//
+// Copyright (c) 2011-2019 Canonical Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package yaml
 
 import (
@@ -11,33 +26,14 @@ import (
 	"time"
 )
 
-const (
-	documentNode = 1 << iota
-	mappingNode
-	sequenceNode
-	scalarNode
-	aliasNode
-)
-
-type node struct {
-	kind         int
-	line, column int
-	tag          string
-	// For an alias node, alias holds the resolved alias.
-	alias    *node
-	value    string
-	implicit bool
-	children []*node
-	anchors  map[string]*node
-}
-
 // ----------------------------------------------------------------------------
 // Parser, produces a node tree out of a libyaml event stream.
 
 type parser struct {
 	parser   yaml_parser_t
 	event    yaml_event_t
-	doc      *node
+	doc      *Node
+	anchors  map[string]*Node
 	doneInit bool
 }
 
@@ -66,6 +62,7 @@ func (p *parser) init() {
 	if p.doneInit {
 		return
 	}
+	p.anchors = make(map[string]*Node)
 	p.expect(yaml_STREAM_START_EVENT)
 	p.doneInit = true
 }
@@ -132,13 +129,14 @@ func (p *parser) fail() {
 	failf("%s%s", where, msg)
 }
 
-func (p *parser) anchor(n *node, anchor []byte) {
+func (p *parser) anchor(n *Node, anchor []byte) {
 	if anchor != nil {
-		p.doc.anchors[string(anchor)] = n
+		n.Anchor = string(anchor)
+		p.anchors[n.Anchor] = n
 	}
 }
 
-func (p *parser) parse() *node {
+func (p *parser) parse() *Node {
 	p.init()
 	switch p.peek() {
 	case yaml_SCALAR_EVENT:
@@ -159,63 +157,120 @@ func (p *parser) parse() *node {
 	}
 }
 
-func (p *parser) node(kind int) *node {
+func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node {
-	return &node{
+	var style Style
-		kind:   kind,
+	if tag != "" && tag != "!" {
-		line:   p.event.start_mark.line,
+		tag = shortTag(tag)
-		column: p.event.start_mark.column,
+		style = TaggedStyle
+	} else if defaultTag != "" {
+		tag = defaultTag
+	} else if kind == ScalarNode {
+		tag, _ = resolve("", value)
+	}
+	return &Node{
+		Kind:        kind,
+		Tag:         tag,
+		Value:       value,
+		Style:       style,
+		Line:        p.event.start_mark.line + 1,
+		Column:      p.event.start_mark.column + 1,
+		HeadComment: string(p.event.head_comment),
+		LineComment: string(p.event.line_comment),
+		FootComment: string(p.event.foot_comment),
 	}
 }
 
-func (p *parser) document() *node {
+func (p *parser) parseChild(parent *Node) *Node {
-	n := p.node(documentNode)
+	child := p.parse()
-	n.anchors = make(map[string]*node)
+	parent.Content = append(parent.Content, child)
+	return child
+}
+
+func (p *parser) document() *Node {
+	n := p.node(DocumentNode, "", "", "")
 	p.doc = n
 	p.expect(yaml_DOCUMENT_START_EVENT)
-	n.children = append(n.children, p.parse())
+	p.parseChild(n)
+	if p.peek() == yaml_DOCUMENT_END_EVENT {
+		n.FootComment = string(p.event.foot_comment)
+	}
 	p.expect(yaml_DOCUMENT_END_EVENT)
 	return n
 }
 
-func (p *parser) alias() *node {
+func (p *parser) alias() *Node {
-	n := p.node(aliasNode)
+	n := p.node(AliasNode, "", "", string(p.event.anchor))
-	n.value = string(p.event.anchor)
+	n.Alias = p.anchors[n.Value]
-	n.alias = p.doc.anchors[n.value]
+	if n.Alias == nil {
-	if n.alias == nil {
+		failf("unknown anchor '%s' referenced", n.Value)
-		failf("unknown anchor '%s' referenced", n.value)
 	}
 	p.expect(yaml_ALIAS_EVENT)
 	return n
 }
 
-func (p *parser) scalar() *node {
+func (p *parser) scalar() *Node {
-	n := p.node(scalarNode)
+	var parsedStyle = p.event.scalar_style()
-	n.value = string(p.event.value)
+	var nodeStyle Style
-	n.tag = string(p.event.tag)
+	switch {
-	n.implicit = p.event.implicit
+	case parsedStyle&yaml_DOUBLE_QUOTED_SCALAR_STYLE != 0:
+		nodeStyle = DoubleQuotedStyle
+	case parsedStyle&yaml_SINGLE_QUOTED_SCALAR_STYLE != 0:
+		nodeStyle = SingleQuotedStyle
+	case parsedStyle&yaml_LITERAL_SCALAR_STYLE != 0:
+		nodeStyle = LiteralStyle
+	case parsedStyle&yaml_FOLDED_SCALAR_STYLE != 0:
+		nodeStyle = FoldedStyle
+	}
+	var nodeValue = string(p.event.value)
+	var nodeTag = string(p.event.tag)
+	var defaultTag string
+	if nodeStyle == 0 {
+		if nodeValue == "<<" {
+			defaultTag = mergeTag
+		}
+	} else {
+		defaultTag = strTag
+	}
+	n := p.node(ScalarNode, defaultTag, nodeTag, nodeValue)
+	n.Style |= nodeStyle
 	p.anchor(n, p.event.anchor)
 	p.expect(yaml_SCALAR_EVENT)
 	return n
 }
 
-func (p *parser) sequence() *node {
+func (p *parser) sequence() *Node {
-	n := p.node(sequenceNode)
+	n := p.node(SequenceNode, seqTag, string(p.event.tag), "")
+	if p.event.sequence_style()&yaml_FLOW_SEQUENCE_STYLE != 0 {
+		n.Style |= FlowStyle
+	}
 	p.anchor(n, p.event.anchor)
 	p.expect(yaml_SEQUENCE_START_EVENT)
 	for p.peek() != yaml_SEQUENCE_END_EVENT {
-		n.children = append(n.children, p.parse())
+		p.parseChild(n)
 	}
+	n.LineComment = string(p.event.line_comment)
+	n.FootComment = string(p.event.foot_comment)
 	p.expect(yaml_SEQUENCE_END_EVENT)
 	return n
 }
 
-func (p *parser) mapping() *node {
+func (p *parser) mapping() *Node {
-	n := p.node(mappingNode)
+	n := p.node(MappingNode, mapTag, string(p.event.tag), "")
+	if p.event.mapping_style()&yaml_FLOW_MAPPING_STYLE != 0 {
+		n.Style |= FlowStyle
+	}
 	p.anchor(n, p.event.anchor)
 	p.expect(yaml_MAPPING_START_EVENT)
 	for p.peek() != yaml_MAPPING_END_EVENT {
-		n.children = append(n.children, p.parse(), p.parse())
+		k := p.parseChild(n)
+		v := p.parseChild(n)
+		if v.FootComment != "" {
+			k.FootComment = v.FootComment
+			v.FootComment = ""
+		}
 	}
+	n.LineComment = string(p.event.line_comment)
+	n.FootComment = string(p.event.foot_comment)
 	p.expect(yaml_MAPPING_END_EVENT)
 	return n
 }
@@ -224,44 +279,60 @@ func (p *parser) mapping() *node {
 // Decoder, unmarshals a node into a provided value.
 
 type decoder struct {
-	doc     *node
+	doc     *Node
-	aliases map[*node]bool
+	aliases map[*Node]bool
-	mapType reflect.Type
 	terrors []string
-	strict  bool
+
+	stringMapType  reflect.Type
+	generalMapType reflect.Type
+
+	knownFields bool
+	uniqueKeys  bool
 }
 
 var (
-	mapItemType    = reflect.TypeOf(MapItem{})
+	nodeType       = reflect.TypeOf(Node{})
 	durationType   = reflect.TypeOf(time.Duration(0))
-	defaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
+	stringMapType  = reflect.TypeOf(map[string]interface{}{})
-	ifaceType      = defaultMapType.Elem()
+	generalMapType = reflect.TypeOf(map[interface{}]interface{}{})
+	ifaceType      = generalMapType.Elem()
 	timeType       = reflect.TypeOf(time.Time{})
 	ptrTimeType    = reflect.TypeOf(&time.Time{})
 )
 
-func newDecoder(strict bool) *decoder {
+func newDecoder() *decoder {
-	d := &decoder{mapType: defaultMapType, strict: strict}
+	d := &decoder{
-	d.aliases = make(map[*node]bool)
+		stringMapType:  stringMapType,
+		generalMapType: generalMapType,
+		uniqueKeys:     true,
+	}
+	d.aliases = make(map[*Node]bool)
 	return d
 }
 
-func (d *decoder) terror(n *node, tag string, out reflect.Value) {
+func (d *decoder) terror(n *Node, tag string, out reflect.Value) {
-	if n.tag != "" {
+	if n.Tag != "" {
-		tag = n.tag
+		tag = n.Tag
 	}
-	value := n.value
+	value := n.Value
-	if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG {
+	if tag != seqTag && tag != mapTag {
 		if len(value) > 10 {
 			value = " `" + value[:7] + "...`"
 		} else {
 			value = " `" + value + "`"
 		}
 	}
-	d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type()))
+	d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.Line, shortTag(tag), value, out.Type()))
 }
 
-func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) {
+func (d *decoder) callUnmarshaler(n *Node, u Unmarshaler) (good bool) {
+	if err := u.UnmarshalYAML(n); err != nil {
+		fail(err)
+	}
+	return true
+}
+
+func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good bool) {
 	terrlen := len(d.terrors)
 	err := u.UnmarshalYAML(func(v interface{}) (err error) {
 		defer handleErr(&err)
@@ -290,8 +361,8 @@ func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) {
 // its types unmarshalled appropriately.
 //
 // If n holds a null value, prepare returns before doing anything.
-func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) {
+func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) {
-	if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) {
+	if n.ShortTag() == nullTag {
 		return out, false, false
 	}
 	again := true
@@ -305,55 +376,84 @@ func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unm
 			again = true
 		}
 		if out.CanAddr() {
-			if u, ok := out.Addr().Interface().(Unmarshaler); ok {
+			outi := out.Addr().Interface()
+			if u, ok := outi.(Unmarshaler); ok {
 				good = d.callUnmarshaler(n, u)
 				return out, true, good
 			}
+			if u, ok := outi.(obsoleteUnmarshaler); ok {
+				good = d.callObsoleteUnmarshaler(n, u)
+				return out, true, good
+			}
 		}
 	}
 	return out, false, false
 }
 
-func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) {
+func (d *decoder) fieldByIndex(n *Node, v reflect.Value, index []int) (field reflect.Value) {
-	switch n.kind {
+	if n.ShortTag() == nullTag {
-	case documentNode:
+		return reflect.Value{}
+	}
+	for _, num := range index {
+		for {
+			if v.Kind() == reflect.Ptr {
+				if v.IsNil() {
+					v.Set(reflect.New(v.Type().Elem()))
+				}
+				v = v.Elem()
+				continue
+			}
+			break
+		}
+		v = v.Field(num)
+	}
+	return v
+}
+
+func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) {
+	if out.Type() == nodeType {
+		out.Set(reflect.ValueOf(n).Elem())
+		return true
+	}
+	switch n.Kind {
+	case DocumentNode:
 		return d.document(n, out)
-	case aliasNode:
+	case AliasNode:
 		return d.alias(n, out)
 	}
 	out, unmarshaled, good := d.prepare(n, out)
 	if unmarshaled {
 		return good
 	}
-	switch n.kind {
+	switch n.Kind {
-	case scalarNode:
+	case ScalarNode:
 		good = d.scalar(n, out)
-	case mappingNode:
+	case MappingNode:
 		good = d.mapping(n, out)
-	case sequenceNode:
+	case SequenceNode:
 		good = d.sequence(n, out)
 	default:
-		panic("internal error: unknown node kind: " + strconv.Itoa(n.kind))
+		panic("internal error: unknown node kind: " + strconv.Itoa(int(n.Kind)))
 	}
 	return good
 }
 
-func (d *decoder) document(n *node, out reflect.Value) (good bool) {
+func (d *decoder) document(n *Node, out reflect.Value) (good bool) {
-	if len(n.children) == 1 {
+	if len(n.Content) == 1 {
 		d.doc = n
-		d.unmarshal(n.children[0], out)
+		d.unmarshal(n.Content[0], out)
 		return true
 	}
 	return false
 }
 
-func (d *decoder) alias(n *node, out reflect.Value) (good bool) {
+func (d *decoder) alias(n *Node, out reflect.Value) (good bool) {
 	if d.aliases[n] {
 		// TODO this could actually be allowed in some circumstances.
-		failf("anchor '%s' value contains itself", n.value)
+		failf("anchor '%s' value contains itself", n.Value)
 	}
 	d.aliases[n] = true
-	good = d.unmarshal(n.alias, out)
+	good = d.unmarshal(n.Alias, out)
 	delete(d.aliases, n)
 	return good
 }
@@ -366,15 +466,15 @@ func resetMap(out reflect.Value) {
 	}
 }
 
-func (d *decoder) scalar(n *node, out reflect.Value) bool {
+func (d *decoder) scalar(n *Node, out reflect.Value) bool {
 	var tag string
 	var resolved interface{}
-	if n.tag == "" && !n.implicit {
+	if n.indicatedString() {
-		tag = yaml_STR_TAG
+		tag = strTag
-		resolved = n.value
+		resolved = n.Value
 	} else {
-		tag, resolved = resolve(n.tag, n.value)
+		tag, resolved = resolve(n.Tag, n.Value)
-		if tag == yaml_BINARY_TAG {
+		if tag == binaryTag {
 			data, err := base64.StdEncoding.DecodeString(resolved.(string))
 			if err != nil {
 				failf("!!binary value contains invalid base64 data")
@@ -383,12 +483,14 @@ func (d *decoder) scalar(n *node, out reflect.Value) bool {
 		}
 	}
 	if resolved == nil {
-		if out.Kind() == reflect.Map && !out.CanAddr() {
+		if out.CanAddr() {
-			resetMap(out)
+			switch out.Kind() {
-		} else {
+			case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice:
-			out.Set(reflect.Zero(out.Type()))
+				out.Set(reflect.Zero(out.Type()))
+				return true
+			}
 		}
-		return true
+		return false
 	}
 	if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() {
 		// We've resolved to exactly the type we want, so use that.
@@ -401,13 +503,13 @@ func (d *decoder) scalar(n *node, out reflect.Value) bool {
 		u, ok := out.Addr().Interface().(encoding.TextUnmarshaler)
 		if ok {
 			var text []byte
-			if tag == yaml_BINARY_TAG {
+			if tag == binaryTag {
 				text = []byte(resolved.(string))
 			} else {
 				// We let any value be unmarshaled into TextUnmarshaler.
 				// That might be more lax than we'd like, but the
 				// TextUnmarshaler itself should bowl out any dubious values.
-				text = []byte(n.value)
+				text = []byte(n.Value)
 			}
 			err := u.UnmarshalText(text)
 			if err != nil {
@@ -418,47 +520,37 @@ func (d *decoder) scalar(n *node, out reflect.Value) bool {
 	}
 	switch out.Kind() {
 	case reflect.String:
-		if tag == yaml_BINARY_TAG {
+		if tag == binaryTag {
 			out.SetString(resolved.(string))
 			return true
 		}
-		if resolved != nil {
+		out.SetString(n.Value)
-			out.SetString(n.value)
+		return true
-			return true
-		}
 	case reflect.Interface:
-		if resolved == nil {
+		out.Set(reflect.ValueOf(resolved))
-			out.Set(reflect.Zero(out.Type()))
-		} else if tag == yaml_TIMESTAMP_TAG {
-			// It looks like a timestamp but for backward compatibility
-			// reasons we set it as a string, so that code that unmarshals
-			// timestamp-like values into interface{} will continue to
-			// see a string and not a time.Time.
-			// TODO(v3) Drop this.
-			out.Set(reflect.ValueOf(n.value))
-		} else {
-			out.Set(reflect.ValueOf(resolved))
-		}
 		return true
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		// This used to work in v2, but it's very unfriendly.
+		isDuration := out.Type() == durationType
+
 		switch resolved := resolved.(type) {
 		case int:
-			if !out.OverflowInt(int64(resolved)) {
+			if !isDuration && !out.OverflowInt(int64(resolved)) {
 				out.SetInt(int64(resolved))
 				return true
 			}
 		case int64:
-			if !out.OverflowInt(resolved) {
+			if !isDuration && !out.OverflowInt(resolved) {
 				out.SetInt(resolved)
 				return true
 			}
 		case uint64:
-			if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
+			if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
 				out.SetInt(int64(resolved))
 				return true
 			}
 		case float64:
-			if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
+			if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) {
 				out.SetInt(int64(resolved))
 				return true
 			}
@@ -499,6 +591,17 @@ func (d *decoder) scalar(n *node, out reflect.Value) bool {
 		case bool:
 			out.SetBool(resolved)
 			return true
+		case string:
+			// This offers some compatibility with the 1.1 spec (https://yaml.org/type/bool.html).
+			// It only works if explicitly attempting to unmarshal into a typed bool value.
+			switch resolved {
+			case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON":
+				out.SetBool(true)
+				return true
+			case "n", "N", "no", "No", "NO", "off", "Off", "OFF":
+				out.SetBool(false)
+				return true
+			}
 		}
 	case reflect.Float32, reflect.Float64:
 		switch resolved := resolved.(type) {
@@ -521,13 +624,7 @@ func (d *decoder) scalar(n *node, out reflect.Value) bool {
 			return true
 		}
 	case reflect.Ptr:
-		if out.Type().Elem() == reflect.TypeOf(resolved) {
+		panic("yaml internal error: please report the issue")
-			// TODO DOes this make sense? When is out a Ptr except when decoding a nil value?
-			elem := reflect.New(out.Type().Elem())
-			elem.Elem().Set(reflect.ValueOf(resolved))
-			out.Set(elem)
-			return true
-		}
 	}
 	d.terror(n, tag, out)
 	return false
@@ -540,8 +637,8 @@ func settableValueOf(i interface{}) reflect.Value {
 	return sv
 }
 
-func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
+func (d *decoder) sequence(n *Node, out reflect.Value) (good bool) {
-	l := len(n.children)
+	l := len(n.Content)
 
 	var iface reflect.Value
 	switch out.Kind() {
@@ -556,7 +653,7 @@ func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
 		iface = out
 		out = settableValueOf(make([]interface{}, l))
 	default:
-		d.terror(n, yaml_SEQ_TAG, out)
+		d.terror(n, seqTag, out)
 		return false
 	}
 	et := out.Type().Elem()
@@ -564,7 +661,7 @@ func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
 	j := 0
 	for i := 0; i < l; i++ {
 		e := reflect.New(et).Elem()
-		if ok := d.unmarshal(n.children[i], e); ok {
+		if ok := d.unmarshal(n.Content[i], e); ok {
 			out.Index(j).Set(e)
 			j++
 		}
@@ -578,51 +675,65 @@ func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
 	return true
 }
 
-func (d *decoder) mapping(n *node, out reflect.Value) (good bool) {
+func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
+	l := len(n.Content)
+	if d.uniqueKeys {
+		nerrs := len(d.terrors)
+		for i := 0; i < l; i += 2 {
+			ni := n.Content[i]
+			for j := i + 2; j < l; j += 2 {
+				nj := n.Content[j]
+				if ni.Kind == nj.Kind && ni.Value == nj.Value {
+					d.terrors = append(d.terrors, fmt.Sprintf("line %d: mapping key %#v already defined at line %d", nj.Line, nj.Value, ni.Line))
+				}
+			}
+		}
+		if len(d.terrors) > nerrs {
+			return false
+		}
+	}
 	switch out.Kind() {
 	case reflect.Struct:
 		return d.mappingStruct(n, out)
-	case reflect.Slice:
-		return d.mappingSlice(n, out)
 	case reflect.Map:
 		// okay
 	case reflect.Interface:
-		if d.mapType.Kind() == reflect.Map {
+		iface := out
-			iface := out
+		if isStringMap(n) {
-			out = reflect.MakeMap(d.mapType)
+			out = reflect.MakeMap(d.stringMapType)
-			iface.Set(out)
 		} else {
-			slicev := reflect.New(d.mapType).Elem()
+			out = reflect.MakeMap(d.generalMapType)
-			if !d.mappingSlice(n, slicev) {
-				return false
-			}
-			out.Set(slicev)
-			return true
 		}
+		iface.Set(out)
 	default:
-		d.terror(n, yaml_MAP_TAG, out)
+		d.terror(n, mapTag, out)
 		return false
 	}
+
 	outt := out.Type()
 	kt := outt.Key()
 	et := outt.Elem()
 
-	mapType := d.mapType
+	stringMapType := d.stringMapType
-	if outt.Key() == ifaceType && outt.Elem() == ifaceType {
+	generalMapType := d.generalMapType
-		d.mapType = outt
+	if outt.Elem() == ifaceType {
+		if outt.Key().Kind() == reflect.String {
+			d.stringMapType = outt
+		} else if outt.Key() == ifaceType {
+			d.generalMapType = outt
+		}
 	}
 
 	if out.IsNil() {
 		out.Set(reflect.MakeMap(outt))
 	}
-	l := len(n.children)
 	for i := 0; i < l; i += 2 {
-		if isMerge(n.children[i]) {
+		if isMerge(n.Content[i]) {
-			d.merge(n.children[i+1], out)
+			d.merge(n.Content[i+1], out)
 			continue
 		}
 		k := reflect.New(kt).Elem()
-		if d.unmarshal(n.children[i], k) {
+		if d.unmarshal(n.Content[i], k) {
 			kkind := k.Kind()
 			if kkind == reflect.Interface {
 				kkind = k.Elem().Kind()
@@ -631,61 +742,34 @@ func (d *decoder) mapping(n *node, out reflect.Value) (good bool) {
 				failf("invalid map key: %#v", k.Interface())
 			}
 			e := reflect.New(et).Elem()
-			if d.unmarshal(n.children[i+1], e) {
+			if d.unmarshal(n.Content[i+1], e) {
-				d.setMapIndex(n.children[i+1], out, k, e)
+				out.SetMapIndex(k, e)
 			}
 		}
 	}
-	d.mapType = mapType
+	d.stringMapType = stringMapType
+	d.generalMapType = generalMapType
 	return true
 }
 
-func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) {
+func isStringMap(n *Node) bool {
-	if d.strict && out.MapIndex(k) != zeroValue {
+	if n.Kind != MappingNode {
-		d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface()))
-		return
-	}
-	out.SetMapIndex(k, v)
-}
-
-func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) {
-	outt := out.Type()
-	if outt.Elem() != mapItemType {
-		d.terror(n, yaml_MAP_TAG, out)
 		return false
 	}
-
+	l := len(n.Content)
-	mapType := d.mapType
+	for i := 0; i < l; i++ {
-	d.mapType = outt
+		if n.Content[i].ShortTag() != strTag {
-
+			return false
-	var slice []MapItem
-	var l = len(n.children)
-	for i := 0; i < l; i += 2 {
-		if isMerge(n.children[i]) {
-			d.merge(n.children[i+1], out)
-			continue
-		}
-		item := MapItem{}
-		k := reflect.ValueOf(&item.Key).Elem()
-		if d.unmarshal(n.children[i], k) {
-			v := reflect.ValueOf(&item.Value).Elem()
-			if d.unmarshal(n.children[i+1], v) {
-				slice = append(slice, item)
-			}
 		}
 	}
-	out.Set(reflect.ValueOf(slice))
-	d.mapType = mapType
 	return true
 }
 
-func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) {
+func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
 	sinfo, err := getStructInfo(out.Type())
 	if err != nil {
 		panic(err)
 	}
-	name := settableValueOf("")
-	l := len(n.children)
 
 	var inlineMap reflect.Value
 	var elemType reflect.Type
@@ -695,23 +779,30 @@ func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) {
 		elemType = inlineMap.Type().Elem()
 	}
 
+	for _, index := range sinfo.InlineUnmarshalers {
+		field := d.fieldByIndex(n, out, index)
+		d.prepare(n, field)
+	}
+
 	var doneFields []bool
-	if d.strict {
+	if d.uniqueKeys {
 		doneFields = make([]bool, len(sinfo.FieldsList))
 	}
+	name := settableValueOf("")
+	l := len(n.Content)
 	for i := 0; i < l; i += 2 {
-		ni := n.children[i]
+		ni := n.Content[i]
 		if isMerge(ni) {
-			d.merge(n.children[i+1], out)
+			d.merge(n.Content[i+1], out)
 			continue
 		}
 		if !d.unmarshal(ni, name) {
 			continue
 		}
 		if info, ok := sinfo.FieldsMap[name.String()]; ok {
-			if d.strict {
+			if d.uniqueKeys {
 				if doneFields[info.Id] {
-					d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type()))
+					d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type()))
 					continue
 				}
 				doneFields[info.Id] = true
@@ -720,18 +811,18 @@ func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) {
 			if info.Inline == nil {
 				field = out.Field(info.Num)
 			} else {
-				field = out.FieldByIndex(info.Inline)
+				field = d.fieldByIndex(n, out, info.Inline)
 			}
-			d.unmarshal(n.children[i+1], field)
+			d.unmarshal(n.Content[i+1], field)
 		} else if sinfo.InlineMap != -1 {
 			if inlineMap.IsNil() {
 				inlineMap.Set(reflect.MakeMap(inlineMap.Type()))
 			}
 			value := reflect.New(elemType).Elem()
-			d.unmarshal(n.children[i+1], value)
+			d.unmarshal(n.Content[i+1], value)
-			d.setMapIndex(n.children[i+1], inlineMap, name, value)
+			inlineMap.SetMapIndex(name, value)
-		} else if d.strict {
+		} else if d.knownFields {
-			d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type()))
+			d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type()))
 		}
 	}
 	return true
@@ -741,26 +832,24 @@ func failWantMap() {
 	failf("map merge requires map or sequence of maps as the value")
 }
 
-func (d *decoder) merge(n *node, out reflect.Value) {
+func (d *decoder) merge(n *Node, out reflect.Value) {
-	switch n.kind {
+	switch n.Kind {
-	case mappingNode:
+	case MappingNode:
 		d.unmarshal(n, out)
-	case aliasNode:
+	case AliasNode:
-		an, ok := d.doc.anchors[n.value]
+		if n.Alias != nil && n.Alias.Kind != MappingNode {
-		if ok && an.kind != mappingNode {
 			failWantMap()
 		}
 		d.unmarshal(n, out)
-	case sequenceNode:
+	case SequenceNode:
 		// Step backwards as earlier nodes take precedence.
-		for i := len(n.children) - 1; i >= 0; i-- {
+		for i := len(n.Content) - 1; i >= 0; i-- {
-			ni := n.children[i]
+			ni := n.Content[i]
-			if ni.kind == aliasNode {
+			if ni.Kind == AliasNode {
-				an, ok := d.doc.anchors[ni.value]
+				if ni.Alias != nil && ni.Alias.Kind != MappingNode {
-				if ok && an.kind != mappingNode {
 					failWantMap()
 				}
-			} else if ni.kind != mappingNode {
+			} else if ni.Kind != MappingNode {
 				failWantMap()
 			}
 			d.unmarshal(ni, out)
@@ -770,6 +859,6 @@ func (d *decoder) merge(n *node, out reflect.Value) {
 	}
 }
 
-func isMerge(n *node) bool {
+func isMerge(n *Node) bool {
-	return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG)
+	return n.Kind == ScalarNode && n.Value == "<<" && (n.Tag == "" || n.Tag == "!" || shortTag(n.Tag) == mergeTag)
 }

vendor/gopkg.in/yaml.v3/emitterc.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (
@@ -43,8 +65,13 @@ func put_break(emitter *yaml_emitter_t) bool {
 	default:
 		panic("unknown line break setting")
 	}
+	if emitter.column == 0 {
+		emitter.space_above = true
+	}
 	emitter.column = 0
 	emitter.line++
+	// [Go] Do this here and below and drop from everywhere else (see commented lines).
+	emitter.indention = true
 	return true
 }
 
@@ -97,8 +124,13 @@ func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool {
 		if !write(emitter, s, i) {
 			return false
 		}
+		if emitter.column == 0 {
+			emitter.space_above = true
+		}
 		emitter.column = 0
 		emitter.line++
+		// [Go] Do this here and above and drop from everywhere else (see commented lines).
+		emitter.indention = true
 	}
 	return true
 }
@@ -228,16 +260,22 @@ func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bo
 		return yaml_emitter_emit_document_end(emitter, event)
 
 	case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE:
-		return yaml_emitter_emit_flow_sequence_item(emitter, event, true)
+		return yaml_emitter_emit_flow_sequence_item(emitter, event, true, false)
+
+	case yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE:
+		return yaml_emitter_emit_flow_sequence_item(emitter, event, false, true)
 
 	case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE:
-		return yaml_emitter_emit_flow_sequence_item(emitter, event, false)
+		return yaml_emitter_emit_flow_sequence_item(emitter, event, false, false)
 
 	case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE:
-		return yaml_emitter_emit_flow_mapping_key(emitter, event, true)
+		return yaml_emitter_emit_flow_mapping_key(emitter, event, true, false)
+
+	case yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE:
+		return yaml_emitter_emit_flow_mapping_key(emitter, event, false, true)
 
 	case yaml_EMIT_FLOW_MAPPING_KEY_STATE:
-		return yaml_emitter_emit_flow_mapping_key(emitter, event, false)
+		return yaml_emitter_emit_flow_mapping_key(emitter, event, false, false)
 
 	case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE:
 		return yaml_emitter_emit_flow_mapping_value(emitter, event, true)
@@ -298,6 +336,7 @@ func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t
 	emitter.column = 0
 	emitter.whitespace = true
 	emitter.indention = true
+	emitter.space_above = true
 
 	if emitter.encoding != yaml_UTF8_ENCODING {
 		if !yaml_emitter_write_bom(emitter) {
@@ -392,13 +431,22 @@ func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event
 			if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) {
 				return false
 			}
-			if emitter.canonical {
+			if emitter.canonical || true {
 				if !yaml_emitter_write_indent(emitter) {
 					return false
 				}
 			}
 		}
 
+		if len(emitter.head_comment) > 0 {
+			if !yaml_emitter_process_head_comment(emitter) {
+				return false
+			}
+			if !put_break(emitter) {
+				return false
+			}
+		}
+
 		emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE
 		return true
 	}
@@ -425,7 +473,20 @@ func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event
 // Expect the root node.
 func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool {
 	emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE)
-	return yaml_emitter_emit_node(emitter, event, true, false, false, false)
+
+	if !yaml_emitter_process_head_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_emit_node(emitter, event, true, false, false, false) {
+		return false
+	}
+	if !yaml_emitter_process_line_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_process_foot_comment(emitter) {
+		return false
+	}
+	return true
 }
 
 // Expect DOCUMENT-END.
@@ -436,6 +497,14 @@ func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t
 	if !yaml_emitter_write_indent(emitter) {
 		return false
 	}
+	if len(emitter.foot_comment) > 0 {
+		if !put_break(emitter) {
+			return false
+		}
+		if !yaml_emitter_process_foot_comment(emitter) {
+			return false
+		}
+	}
 	if !event.implicit {
 		// [Go] Allocate the slice elsewhere.
 		if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) {
@@ -454,7 +523,7 @@ func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t
 }
 
 // Expect a flow item node.
-func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool {
+func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool {
 	if first {
 		if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) {
 			return false
@@ -480,29 +549,62 @@ func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_e
 		if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) {
 			return false
 		}
+		if !yaml_emitter_process_line_comment(emitter) {
+			return false
+		}
+		if !yaml_emitter_process_foot_comment(emitter) {
+			return false
+		}
 		emitter.state = emitter.states[len(emitter.states)-1]
 		emitter.states = emitter.states[:len(emitter.states)-1]
 
 		return true
 	}
 
-	if !first {
+	if !first && !trail {
 		if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) {
 			return false
 		}
 	}
 
+	if !yaml_emitter_process_head_comment(emitter) {
+		return false
+	}
+	if emitter.column == 0 {
+		if !yaml_emitter_write_indent(emitter) {
+			return false
+		}
+	}
+
 	if emitter.canonical || emitter.column > emitter.best_width {
 		if !yaml_emitter_write_indent(emitter) {
 			return false
 		}
 	}
-	emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE)
+	if len(emitter.line_comment) > 0 || len(emitter.foot_comment) > 0 {
-	return yaml_emitter_emit_node(emitter, event, false, true, false, false)
+		emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE)
+	} else {
+		emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE)
+	}
+	if !yaml_emitter_emit_node(emitter, event, false, true, false, false) {
+		return false
+	}
+	if len(emitter.line_comment) > 0 || len(emitter.foot_comment) > 0 {
+		if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) {
+			return false
+		}
+	}
+	if !yaml_emitter_process_line_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_process_foot_comment(emitter) {
+		return false
+	}
+	return true
 }
 
 // Expect a flow key node.
-func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool {
+func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool {
 	if first {
 		if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) {
 			return false
@@ -528,16 +630,32 @@ func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_eve
 		if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) {
 			return false
 		}
+		if !yaml_emitter_process_line_comment(emitter) {
+			return false
+		}
+		if !yaml_emitter_process_foot_comment(emitter) {
+			return false
+		}
 		emitter.state = emitter.states[len(emitter.states)-1]
 		emitter.states = emitter.states[:len(emitter.states)-1]
 		return true
 	}
 
-	if !first {
+	if !first && !trail {
 		if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) {
 			return false
 		}
 	}
+
+	if !yaml_emitter_process_head_comment(emitter) {
+		return false
+	}
+	if emitter.column == 0 {
+		if !yaml_emitter_write_indent(emitter) {
+			return false
+		}
+	}
+
 	if emitter.canonical || emitter.column > emitter.best_width {
 		if !yaml_emitter_write_indent(emitter) {
 			return false
@@ -571,8 +689,26 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e
 			return false
 		}
 	}
-	emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE)
+	if len(emitter.line_comment) > 0 || len(emitter.foot_comment) > 0 {
-	return yaml_emitter_emit_node(emitter, event, false, false, true, false)
+		emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE)
+	} else {
+		emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE)
+	}
+	if !yaml_emitter_emit_node(emitter, event, false, false, true, false) {
+		return false
+	}
+	if len(emitter.line_comment) > 0 || len(emitter.foot_comment) > 0 {
+		if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) {
+			return false
+		}
+	}
+	if !yaml_emitter_process_line_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_process_foot_comment(emitter) {
+		return false
+	}
+	return true
 }
 
 // Expect a block item node.
@@ -589,6 +725,9 @@ func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_
 		emitter.states = emitter.states[:len(emitter.states)-1]
 		return true
 	}
+	if !yaml_emitter_process_head_comment(emitter) {
+		return false
+	}
 	if !yaml_emitter_write_indent(emitter) {
 		return false
 	}
@@ -596,7 +735,16 @@ func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_
 		return false
 	}
 	emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE)
-	return yaml_emitter_emit_node(emitter, event, false, true, false, false)
+	if !yaml_emitter_emit_node(emitter, event, false, true, false, false) {
+		return false
+	}
+	if !yaml_emitter_process_line_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_process_foot_comment(emitter) {
+		return false
+	}
+	return true
 }
 
 // Expect a block key node.
@@ -613,9 +761,14 @@ func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_ev
 		emitter.states = emitter.states[:len(emitter.states)-1]
 		return true
 	}
-	if !yaml_emitter_write_indent(emitter) {
+	if !yaml_emitter_process_head_comment(emitter) {
 		return false
 	}
+	if !first || emitter.states[len(emitter.states)-1] != yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE {
+		if !yaml_emitter_write_indent(emitter) {
+			return false
+		}
+	}
 	if yaml_emitter_check_simple_key(emitter) {
 		emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE)
 		return yaml_emitter_emit_node(emitter, event, false, false, true, true)
@@ -642,7 +795,16 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_
 		}
 	}
 	emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE)
-	return yaml_emitter_emit_node(emitter, event, false, false, true, false)
+	if !yaml_emitter_emit_node(emitter, event, false, false, true, false) {
+		return false
+	}
+	if !yaml_emitter_process_line_comment(emitter) {
+		return false
+	}
+	if !yaml_emitter_process_foot_comment(emitter) {
+		return false
+	}
+	return true
 }
 
 // Expect a node.
@@ -908,6 +1070,68 @@ func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool {
 	panic("unknown scalar style")
 }
 
+// Write a head comment.
+func yaml_emitter_process_head_comment(emitter *yaml_emitter_t) bool {
+	if len(emitter.head_comment) == 0 {
+		return true
+	}
+	space_above := emitter.space_above
+	if !emitter.indention {
+		if !put_break(emitter) {
+			return false
+		}
+	}
+	if !space_above &&
+		emitter.state != yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE &&
+		emitter.state != yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE &&
+		emitter.state != yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE &&
+		emitter.state != yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE {
+		if !put_break(emitter) {
+			return false
+		}
+	}
+	if !yaml_emitter_write_indent(emitter) {
+		return false
+	}
+	if !yaml_emitter_write_comment(emitter, emitter.head_comment) {
+		return false
+	}
+	emitter.head_comment = emitter.head_comment[:0]
+	return true
+}
+
+// Write an line comment.
+func yaml_emitter_process_line_comment(emitter *yaml_emitter_t) bool {
+	if len(emitter.line_comment) == 0 {
+		return true
+	}
+	if !emitter.whitespace {
+		if !put(emitter, ' ') {
+			return false
+		}
+	}
+	if !yaml_emitter_write_comment(emitter, emitter.line_comment) {
+		return false
+	}
+	emitter.line_comment = emitter.line_comment[:0]
+	return true
+}
+
+// Write a foot comment.
+func yaml_emitter_process_foot_comment(emitter *yaml_emitter_t) bool {
+	if len(emitter.foot_comment) == 0 {
+		return true
+	}
+	if !yaml_emitter_write_indent(emitter) {
+		return false
+	}
+	if !yaml_emitter_write_comment(emitter, emitter.foot_comment) {
+		return false
+	}
+	emitter.foot_comment = emitter.foot_comment[:0]
+	return true
+}
+
 // Check if a %YAML directive is valid.
 func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool {
 	if version_directive.major != 1 || version_directive.minor != 1 {
@@ -1137,6 +1361,16 @@ func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bo
 	emitter.tag_data.suffix = nil
 	emitter.scalar_data.value = nil
 
+	if len(event.head_comment) > 0 {
+		emitter.head_comment = event.head_comment
+	}
+	if len(event.line_comment) > 0 {
+		emitter.line_comment = event.line_comment
+	}
+	if len(event.foot_comment) > 0 {
+		emitter.foot_comment = event.foot_comment
+	}
+
 	switch event.typ {
 	case yaml_ALIAS_EVENT:
 		if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) {
@@ -1214,7 +1448,8 @@ func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool {
 		}
 	}
 	emitter.whitespace = true
-	emitter.indention = true
+	//emitter.indention = true
+	emitter.space_above = false
 	return true
 }
 
@@ -1341,7 +1576,7 @@ func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allo
 			if !write_break(emitter, value, &i) {
 				return false
 			}
-			emitter.indention = true
+			//emitter.indention = true
 			breaks = true
 		} else {
 			if breaks {
@@ -1397,7 +1632,7 @@ func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []by
 			if !write_break(emitter, value, &i) {
 				return false
 			}
-			emitter.indention = true
+			//emitter.indention = true
 			breaks = true
 		} else {
 			if breaks {
@@ -1599,7 +1834,7 @@ func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bo
 	if !put_break(emitter) {
 		return false
 	}
-	emitter.indention = true
+	//emitter.indention = true
 	emitter.whitespace = true
 	breaks := true
 	for i := 0; i < len(value); {
@@ -1607,7 +1842,7 @@ func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bo
 			if !write_break(emitter, value, &i) {
 				return false
 			}
-			emitter.indention = true
+			//emitter.indention = true
 			breaks = true
 		} else {
 			if breaks {
@@ -1637,7 +1872,7 @@ func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) boo
 	if !put_break(emitter) {
 		return false
 	}
-	emitter.indention = true
+	//emitter.indention = true
 	emitter.whitespace = true
 
 	breaks := true
@@ -1658,7 +1893,7 @@ func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) boo
 			if !write_break(emitter, value, &i) {
 				return false
 			}
-			emitter.indention = true
+			//emitter.indention = true
 			breaks = true
 		} else {
 			if breaks {
@@ -1683,3 +1918,40 @@ func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) boo
 	}
 	return true
 }
+
+func yaml_emitter_write_comment(emitter *yaml_emitter_t, comment []byte) bool {
+	breaks := false
+	pound := false
+	for i := 0; i < len(comment); {
+		if is_break(comment, i) {
+			if !write_break(emitter, comment, &i) {
+				return false
+			}
+			//emitter.indention = true
+			breaks = true
+			pound = false
+		} else {
+			if breaks && !yaml_emitter_write_indent(emitter) {
+				return false
+			}
+			if !pound {
+				if comment[i] != '#' && (!put(emitter, '#') || !put(emitter, ' ')) {
+					return false
+				}
+				pound = true
+			}
+			if !write(emitter, comment, &i) {
+				return false
+			}
+			emitter.indention = false
+			breaks = false
+		}
+	}
+	if !breaks && !put_break(emitter) {
+		return false
+	}
+
+	emitter.whitespace = true
+	//emitter.indention = true
+	return true
+}

vendor/gopkg.in/yaml.v3/encode.go

@@ -1,3 +1,18 @@
+//
+// Copyright (c) 2011-2019 Canonical Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package yaml
 
 import (
@@ -14,12 +29,11 @@ import (
 )
 
 type encoder struct {
-	emitter yaml_emitter_t
+	emitter  yaml_emitter_t
-	event   yaml_event_t
+	event    yaml_event_t
-	out     []byte
+	out      []byte
-	flow    bool
+	flow     bool
-	// doneInit holds whether the initial stream_start_event has been
+	indent   int
-	// emitted.
 	doneInit bool
 }
 
@@ -43,6 +57,10 @@ func (e *encoder) init() {
 	if e.doneInit {
 		return
 	}
+	if e.indent == 0 {
+		e.indent = 4
+	}
+	e.emitter.best_indent = e.indent
 	yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING)
 	e.emit()
 	e.doneInit = true
@@ -75,27 +93,43 @@ func (e *encoder) must(ok bool) {
 
 func (e *encoder) marshalDoc(tag string, in reflect.Value) {
 	e.init()
-	yaml_document_start_event_initialize(&e.event, nil, nil, true)
+	var node *Node
-	e.emit()
+	if in.IsValid() {
-	e.marshal(tag, in)
+		node, _ = in.Interface().(*Node)
-	yaml_document_end_event_initialize(&e.event, true)
+	}
-	e.emit()
+	if node != nil && node.Kind == DocumentNode {
+		e.nodev(in)
+	} else {
+		yaml_document_start_event_initialize(&e.event, nil, nil, true)
+		e.emit()
+		e.marshal(tag, in)
+		yaml_document_end_event_initialize(&e.event, true)
+		e.emit()
+	}
 }
 
 func (e *encoder) marshal(tag string, in reflect.Value) {
+	tag = shortTag(tag)
 	if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() {
 		e.nilv()
 		return
 	}
 	iface := in.Interface()
-	switch m := iface.(type) {
+	switch value := iface.(type) {
-	case time.Time, *time.Time:
+	case *Node:
-		// Although time.Time implements TextMarshaler,
+		e.nodev(in)
-		// we don't want to treat it as a string for YAML
+		return
-		// purposes because YAML has special support for
+	case time.Time:
-		// timestamps.
+		e.timev(tag, in)
+		return
+	case *time.Time:
+		e.timev(tag, in.Elem())
+		return
+	case time.Duration:
+		e.stringv(tag, reflect.ValueOf(value.String()))
+		return
 	case Marshaler:
-		v, err := m.MarshalYAML()
+		v, err := value.MarshalYAML()
 		if err != nil {
 			fail(err)
 		}
@@ -103,9 +137,10 @@ func (e *encoder) marshal(tag string, in reflect.Value) {
 			e.nilv()
 			return
 		}
-		in = reflect.ValueOf(v)
+		e.marshal(tag, reflect.ValueOf(v))
+		return
 	case encoding.TextMarshaler:
-		text, err := m.MarshalText()
+		text, err := value.MarshalText()
 		if err != nil {
 			fail(err)
 		}
@@ -120,31 +155,15 @@ func (e *encoder) marshal(tag string, in reflect.Value) {
 	case reflect.Map:
 		e.mapv(tag, in)
 	case reflect.Ptr:
-		if in.Type() == ptrTimeType {
+		e.marshal(tag, in.Elem())
-			e.timev(tag, in.Elem())
-		} else {
-			e.marshal(tag, in.Elem())
-		}
 	case reflect.Struct:
-		if in.Type() == timeType {
+		e.structv(tag, in)
-			e.timev(tag, in)
-		} else {
-			e.structv(tag, in)
-		}
 	case reflect.Slice, reflect.Array:
-		if in.Type().Elem() == mapItemType {
+		e.slicev(tag, in)
-			e.itemsv(tag, in)
-		} else {
-			e.slicev(tag, in)
-		}
 	case reflect.String:
 		e.stringv(tag, in)
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		if in.Type() == durationType {
+		e.intv(tag, in)
-			e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String()))
-		} else {
-			e.intv(tag, in)
-		}
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
 		e.uintv(tag, in)
 	case reflect.Float32, reflect.Float64:
@@ -167,14 +186,21 @@ func (e *encoder) mapv(tag string, in reflect.Value) {
 	})
 }
 
-func (e *encoder) itemsv(tag string, in reflect.Value) {
+func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Value) {
-	e.mappingv(tag, func() {
+	for _, num := range index {
-		slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem)
+		for {
-		for _, item := range slice {
+			if v.Kind() == reflect.Ptr {
-			e.marshal("", reflect.ValueOf(item.Key))
+				if v.IsNil() {
-			e.marshal("", reflect.ValueOf(item.Value))
+					return reflect.Value{}
+				}
+				v = v.Elem()
+				continue
+			}
+			break
 		}
-	})
+		v = v.Field(num)
+	}
+	return v
 }
 
 func (e *encoder) structv(tag string, in reflect.Value) {
@@ -188,7 +214,10 @@ func (e *encoder) structv(tag string, in reflect.Value) {
 			if info.Inline == nil {
 				value = in.Field(info.Num)
 			} else {
-				value = in.FieldByIndex(info.Inline)
+				value = e.fieldByIndex(in, info.Inline)
+				if !value.IsValid() {
+					continue
+				}
 			}
 			if info.OmitEmpty && isZero(value) {
 				continue
@@ -205,7 +234,7 @@ func (e *encoder) structv(tag string, in reflect.Value) {
 				sort.Sort(keys)
 				for _, k := range keys {
 					if _, found := sinfo.FieldsMap[k.String()]; found {
-						panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String()))
+						panic(fmt.Sprintf("cannot have key %q in inlined map: conflicts with struct field", k.String()))
 					}
 					e.marshal("", k)
 					e.flow = false
@@ -275,7 +304,7 @@ func (e *encoder) stringv(tag string, in reflect.Value) {
 	canUsePlain := true
 	switch {
 	case !utf8.ValidString(s):
-		if tag == yaml_BINARY_TAG {
+		if tag == binaryTag {
 			failf("explicitly tagged !!binary data must be base64-encoded")
 		}
 		if tag != "" {
@@ -283,14 +312,14 @@ func (e *encoder) stringv(tag string, in reflect.Value) {
 		}
 		// It can't be encoded directly as YAML so use a binary tag
 		// and encode it as base64.
-		tag = yaml_BINARY_TAG
+		tag = binaryTag
 		s = encodeBase64(s)
 	case tag == "":
 		// Check to see if it would resolve to a specific
 		// tag when encoded unquoted. If it doesn't,
 		// there's no need to quote it.
 		rtag, _ := resolve("", s)
-		canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s)
+		canUsePlain = rtag == strTag && !isBase60Float(s)
 	}
 	// Note: it's possible for user code to emit invalid YAML
 	// if they explicitly specify a tag and a string containing
@@ -303,7 +332,7 @@ func (e *encoder) stringv(tag string, in reflect.Value) {
 	default:
 		style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
 	}
-	e.emitScalar(s, "", tag, style)
+	e.emitScalar(s, "", tag, style, nil, nil, nil)
 }
 
 func (e *encoder) boolv(tag string, in reflect.Value) {
@@ -313,23 +342,23 @@ func (e *encoder) boolv(tag string, in reflect.Value) {
 	} else {
 		s = "false"
 	}
-	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
 func (e *encoder) intv(tag string, in reflect.Value) {
 	s := strconv.FormatInt(in.Int(), 10)
-	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
 func (e *encoder) uintv(tag string, in reflect.Value) {
 	s := strconv.FormatUint(in.Uint(), 10)
-	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
 func (e *encoder) timev(tag string, in reflect.Value) {
 	t := in.Interface().(time.Time)
 	s := t.Format(time.RFC3339Nano)
-	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
 func (e *encoder) floatv(tag string, in reflect.Value) {
@@ -348,15 +377,148 @@ func (e *encoder) floatv(tag string, in reflect.Value) {
 	case "NaN":
 		s = ".nan"
 	}
-	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
 func (e *encoder) nilv() {
-	e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE)
+	e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE, nil, nil, nil)
 }
 
-func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) {
+func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t, head, line, foot []byte) {
+	// TODO Kill this function. Replace all initialize calls by their underlining Go literals.
 	implicit := tag == ""
+	if !implicit {
+		tag = longTag(tag)
+	}
 	e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style))
+	e.event.head_comment = head
+	e.event.line_comment = line
+	e.event.foot_comment = foot
 	e.emit()
 }
+
+func (e *encoder) nodev(in reflect.Value) {
+	e.node(in.Interface().(*Node))
+}
+
+func (e *encoder) node(node *Node) {
+	// If the tag was not explicitly requested, and dropping it won't change the
+	// implicit tag of the value, don't include it in the presentation.
+	var tag = node.Tag
+	var stag = shortTag(tag)
+	var rtag string
+	var forceQuoting bool
+	if tag != "" && node.Style&TaggedStyle == 0 {
+		if node.Kind == ScalarNode {
+			if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 {
+				tag = ""
+			} else {
+				rtag, _ = resolve("", node.Value)
+				if rtag == stag {
+					tag = ""
+				} else if stag == strTag {
+					tag = ""
+					forceQuoting = true
+				}
+			}
+		} else {
+			switch node.Kind {
+			case MappingNode:
+				rtag = mapTag
+			case SequenceNode:
+				rtag = seqTag
+			}
+			if rtag == stag {
+				tag = ""
+			}
+		}
+	}
+
+	switch node.Kind {
+	case DocumentNode:
+		yaml_document_start_event_initialize(&e.event, nil, nil, true)
+		e.event.head_comment = []byte(node.HeadComment)
+		e.emit()
+		for _, node := range node.Content {
+			e.node(node)
+		}
+		yaml_document_end_event_initialize(&e.event, true)
+		e.event.foot_comment = []byte(node.FootComment)
+		e.emit()
+
+	case SequenceNode:
+		style := yaml_BLOCK_SEQUENCE_STYLE
+		if node.Style&FlowStyle != 0 {
+			style = yaml_FLOW_SEQUENCE_STYLE
+		}
+		e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style))
+		e.event.head_comment = []byte(node.HeadComment)
+		e.emit()
+		for _, node := range node.Content {
+			e.node(node)
+		}
+		e.must(yaml_sequence_end_event_initialize(&e.event))
+		e.event.line_comment = []byte(node.LineComment)
+		e.event.foot_comment = []byte(node.FootComment)
+		e.emit()
+
+	case MappingNode:
+		style := yaml_BLOCK_MAPPING_STYLE
+		if node.Style&FlowStyle != 0 {
+			style = yaml_FLOW_MAPPING_STYLE
+		}
+		yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style)
+		e.event.head_comment = []byte(node.HeadComment)
+		e.emit()
+
+		for i := 0; i+1 < len(node.Content); i += 2 {
+			e.node(node.Content[i])
+			e.node(node.Content[i+1])
+		}
+
+		yaml_mapping_end_event_initialize(&e.event)
+		e.event.line_comment = []byte(node.LineComment)
+		e.event.foot_comment = []byte(node.FootComment)
+		e.emit()
+
+	case AliasNode:
+		yaml_alias_event_initialize(&e.event, []byte(node.Value))
+		e.event.head_comment = []byte(node.HeadComment)
+		e.event.line_comment = []byte(node.LineComment)
+		e.event.foot_comment = []byte(node.FootComment)
+		e.emit()
+
+	case ScalarNode:
+		value := node.Value
+		if !utf8.ValidString(value) {
+			if tag == binaryTag {
+				failf("explicitly tagged !!binary data must be base64-encoded")
+			}
+			if tag != "" {
+				failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag))
+			}
+			// It can't be encoded directly as YAML so use a binary tag
+			// and encode it as base64.
+			tag = binaryTag
+			value = encodeBase64(value)
+		}
+
+		style := yaml_PLAIN_SCALAR_STYLE
+		switch {
+		case node.Style&DoubleQuotedStyle != 0:
+			style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
+		case node.Style&SingleQuotedStyle != 0:
+			style = yaml_SINGLE_QUOTED_SCALAR_STYLE
+		case node.Style&LiteralStyle != 0:
+			style = yaml_LITERAL_SCALAR_STYLE
+		case node.Style&FoldedStyle != 0:
+			style = yaml_FOLDED_SCALAR_STYLE
+		case strings.Contains(value, "\n"):
+			style = yaml_LITERAL_SCALAR_STYLE
+		case forceQuoting:
+			style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
+		}
+
+		e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment))
+	}
+}

vendor/gopkg.in/yaml.v3/go.mod

@@ -1,4 +1,4 @@
-module "gopkg.in/yaml.v2"
+module "gopkg.in/yaml.v3"
 
 require (
 	"gopkg.in/check.v1" v0.0.0-20161208181325-20d25e280405

vendor/gopkg.in/yaml.v3/parserc.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (
@@ -45,11 +67,42 @@ import (
 // Peek the next token in the token queue.
 func peek_token(parser *yaml_parser_t) *yaml_token_t {
 	if parser.token_available || yaml_parser_fetch_more_tokens(parser) {
-		return &parser.tokens[parser.tokens_head]
+		token := &parser.tokens[parser.tokens_head]
+		yaml_parser_unfold_comments(parser, token)
+		return token
 	}
 	return nil
 }
 
+// yaml_parser_unfold_comments walks through the comments queue and joins all
+// comments behind the position of the provided token into the respective
+// top-level comment slices in the parser.
+func yaml_parser_unfold_comments(parser *yaml_parser_t, token *yaml_token_t) {
+	for parser.comments_head < len(parser.comments) && token.start_mark.index >= parser.comments[parser.comments_head].after.index {
+		comment := &parser.comments[parser.comments_head]
+		if len(comment.head) > 0 {
+			if len(parser.head_comment) > 0 {
+				parser.head_comment = append(parser.head_comment, '\n')
+			}
+			parser.head_comment = append(parser.head_comment, comment.head...)
+		}
+		if len(comment.foot) > 0 {
+			if len(parser.foot_comment) > 0 {
+				parser.foot_comment = append(parser.foot_comment, '\n')
+			}
+			parser.foot_comment = append(parser.foot_comment, comment.foot...)
+		}
+		if len(comment.line) > 0 {
+			if len(parser.line_comment) > 0 {
+				parser.line_comment = append(parser.line_comment, '\n')
+			}
+			parser.line_comment = append(parser.line_comment, comment.line...)
+		}
+		*comment = yaml_comment_t{}
+		parser.comments_head++
+	}
+}
+
 // Remove the next token from the queue (must be called after peek_token).
 func skip_token(parser *yaml_parser_t) {
 	parser.token_available = false
@@ -224,10 +277,32 @@ func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t
 		parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE)
 		parser.state = yaml_PARSE_BLOCK_NODE_STATE
 
+		var head_comment []byte
+		if len(parser.head_comment) > 0 {
+			// [Go] Scan the header comment backwards, and if an empty line is found, break
+			//      the header so the part before the last empty line goes into the
+			//      document header, while the bottom of it goes into a follow up event.
+			for i := len(parser.head_comment) - 1; i > 0; i-- {
+				if parser.head_comment[i] == '\n' {
+					if i == len(parser.head_comment)-1 {
+						head_comment = parser.head_comment[:i]
+						parser.head_comment = parser.head_comment[i+1:]
+						break
+					} else if parser.head_comment[i-1] == '\n' {
+						head_comment = parser.head_comment[:i-1]
+						parser.head_comment = parser.head_comment[i+1:]
+						break
+					}
+				}
+			}
+		}
+
 		*event = yaml_event_t{
 			typ:        yaml_DOCUMENT_START_EVENT,
 			start_mark: token.start_mark,
 			end_mark:   token.end_mark,
+
+			head_comment: head_comment,
 		}
 
 	} else if token.typ != yaml_STREAM_END_TOKEN {
@@ -326,10 +401,22 @@ func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t)
 		start_mark: start_mark,
 		end_mark:   end_mark,
 		implicit:   implicit,
+
+		foot_comment: parser.head_comment,
 	}
+	parser.head_comment = nil
 	return true
 }
 
+func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) {
+	event.head_comment = parser.head_comment
+	event.line_comment = parser.line_comment
+	event.foot_comment = parser.foot_comment
+	parser.head_comment = nil
+	parser.line_comment = nil
+	parser.foot_comment = nil
+}
+
 // Parse the productions:
 // block_node_or_indentless_sequence    ::=
 //                          ALIAS
@@ -373,6 +460,7 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i
 			end_mark:   token.end_mark,
 			anchor:     token.value,
 		}
+		yaml_parser_set_event_comments(parser, event)
 		skip_token(parser)
 		return true
 	}
@@ -486,6 +574,7 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i
 			quoted_implicit: quoted_implicit,
 			style:           yaml_style_t(token.style),
 		}
+		yaml_parser_set_event_comments(parser, event)
 		skip_token(parser)
 		return true
 	}
@@ -502,6 +591,7 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i
 			implicit:   implicit,
 			style:      yaml_style_t(yaml_FLOW_SEQUENCE_STYLE),
 		}
+		yaml_parser_set_event_comments(parser, event)
 		return true
 	}
 	if token.typ == yaml_FLOW_MAPPING_START_TOKEN {
@@ -516,6 +606,7 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i
 			implicit:   implicit,
 			style:      yaml_style_t(yaml_FLOW_MAPPING_STYLE),
 		}
+		yaml_parser_set_event_comments(parser, event)
 		return true
 	}
 	if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN {
@@ -820,6 +911,7 @@ func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_ev
 		start_mark: token.start_mark,
 		end_mark:   token.end_mark,
 	}
+	yaml_parser_set_event_comments(parser, event)
 
 	skip_token(parser)
 	return true
@@ -959,6 +1051,7 @@ func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event
 		start_mark: token.start_mark,
 		end_mark:   token.end_mark,
 	}
+	yaml_parser_set_event_comments(parser, event)
 	skip_token(parser)
 	return true
 }

vendor/gopkg.in/yaml.v3/readerc.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (

vendor/gopkg.in/yaml.v3/resolve.go

@@ -1,3 +1,18 @@
+//
+// Copyright (c) 2011-2019 Canonical Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package yaml
 
 import (
@@ -34,18 +49,14 @@ func init() {
 		tag string
 		l   []string
 	}{
-		{true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}},
+		{true, boolTag, []string{"true", "True", "TRUE"}},
-		{true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}},
+		{false, boolTag, []string{"false", "False", "FALSE"}},
-		{true, yaml_BOOL_TAG, []string{"on", "On", "ON"}},
+		{nil, nullTag, []string{"", "~", "null", "Null", "NULL"}},
-		{false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}},
+		{math.NaN(), floatTag, []string{".nan", ".NaN", ".NAN"}},
-		{false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}},
+		{math.Inf(+1), floatTag, []string{".inf", ".Inf", ".INF"}},
-		{false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}},
+		{math.Inf(+1), floatTag, []string{"+.inf", "+.Inf", "+.INF"}},
-		{nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}},
+		{math.Inf(-1), floatTag, []string{"-.inf", "-.Inf", "-.INF"}},
-		{math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}},
+		{"<<", mergeTag, []string{"<<"}},
-		{math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}},
-		{math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}},
-		{math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}},
-		{"<<", yaml_MERGE_TAG, []string{"<<"}},
 	}
 
 	m := resolveMap
@@ -56,11 +67,37 @@ func init() {
 	}
 }
 
+const (
+	nullTag      = "!!null"
+	boolTag      = "!!bool"
+	strTag       = "!!str"
+	intTag       = "!!int"
+	floatTag     = "!!float"
+	timestampTag = "!!timestamp"
+	seqTag       = "!!seq"
+	mapTag       = "!!map"
+	binaryTag    = "!!binary"
+	mergeTag     = "!!merge"
+)
+
+var longTags = make(map[string]string)
+var shortTags = make(map[string]string)
+
+func init() {
+	for _, stag := range []string{nullTag, boolTag, strTag, intTag, floatTag, timestampTag, seqTag, mapTag, binaryTag, mergeTag} {
+		ltag := longTag(stag)
+		longTags[stag] = ltag
+		shortTags[ltag] = stag
+	}
+}
+
 const longTagPrefix = "tag:yaml.org,2002:"
 
 func shortTag(tag string) string {
-	// TODO This can easily be made faster and produce less garbage.
 	if strings.HasPrefix(tag, longTagPrefix) {
+		if stag, ok := shortTags[tag]; ok {
+			return stag
+		}
 		return "!!" + tag[len(longTagPrefix):]
 	}
 	return tag
@@ -68,6 +105,9 @@ func shortTag(tag string) string {
 
 func longTag(tag string) string {
 	if strings.HasPrefix(tag, "!!") {
+		if ltag, ok := longTags[tag]; ok {
+			return ltag
+		}
 		return longTagPrefix + tag[2:]
 	}
 	return tag
@@ -75,32 +115,33 @@ func longTag(tag string) string {
 
 func resolvableTag(tag string) bool {
 	switch tag {
-	case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG:
+	case "", strTag, boolTag, intTag, floatTag, nullTag, timestampTag:
 		return true
 	}
 	return false
 }
 
-var yamlStyleFloat = regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+][0-9]+)?$`)
+var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`)
 
 func resolve(tag string, in string) (rtag string, out interface{}) {
+	tag = shortTag(tag)
 	if !resolvableTag(tag) {
 		return tag, in
 	}
 
 	defer func() {
 		switch tag {
-		case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG:
+		case "", rtag, strTag, binaryTag:
 			return
-		case yaml_FLOAT_TAG:
+		case floatTag:
-			if rtag == yaml_INT_TAG {
+			if rtag == intTag {
 				switch v := out.(type) {
 				case int64:
-					rtag = yaml_FLOAT_TAG
+					rtag = floatTag
 					out = float64(v)
 					return
 				case int:
-					rtag = yaml_FLOAT_TAG
+					rtag = floatTag
 					out = float64(v)
 					return
 				}
@@ -115,7 +156,7 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
 	if in != "" {
 		hint = resolveTable[in[0]]
 	}
-	if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG {
+	if hint != 0 && tag != strTag && tag != binaryTag {
 		// Handle things we can lookup in a map.
 		if item, ok := resolveMap[in]; ok {
 			return item.tag, item.value
@@ -133,17 +174,17 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
 			// Not in the map, so maybe a normal float.
 			floatv, err := strconv.ParseFloat(in, 64)
 			if err == nil {
-				return yaml_FLOAT_TAG, floatv
+				return floatTag, floatv
 			}
 
 		case 'D', 'S':
 			// Int, float, or timestamp.
 			// Only try values as a timestamp if the value is unquoted or there's an explicit
 			// !!timestamp tag.
-			if tag == "" || tag == yaml_TIMESTAMP_TAG {
+			if tag == "" || tag == timestampTag {
 				t, ok := parseTimestamp(in)
 				if ok {
-					return yaml_TIMESTAMP_TAG, t
+					return timestampTag, t
 				}
 			}
 
@@ -151,49 +192,76 @@ func resolve(tag string, in string) (rtag string, out interface{}) {
 			intv, err := strconv.ParseInt(plain, 0, 64)
 			if err == nil {
 				if intv == int64(int(intv)) {
-					return yaml_INT_TAG, int(intv)
+					return intTag, int(intv)
 				} else {
-					return yaml_INT_TAG, intv
+					return intTag, intv
 				}
 			}
 			uintv, err := strconv.ParseUint(plain, 0, 64)
 			if err == nil {
-				return yaml_INT_TAG, uintv
+				return intTag, uintv
 			}
 			if yamlStyleFloat.MatchString(plain) {
 				floatv, err := strconv.ParseFloat(plain, 64)
 				if err == nil {
-					return yaml_FLOAT_TAG, floatv
+					return floatTag, floatv
 				}
 			}
 			if strings.HasPrefix(plain, "0b") {
 				intv, err := strconv.ParseInt(plain[2:], 2, 64)
 				if err == nil {
 					if intv == int64(int(intv)) {
-						return yaml_INT_TAG, int(intv)
+						return intTag, int(intv)
 					} else {
-						return yaml_INT_TAG, intv
+						return intTag, intv
 					}
 				}
 				uintv, err := strconv.ParseUint(plain[2:], 2, 64)
 				if err == nil {
-					return yaml_INT_TAG, uintv
+					return intTag, uintv
 				}
 			} else if strings.HasPrefix(plain, "-0b") {
-				intv, err := strconv.ParseInt("-" + plain[3:], 2, 64)
+				intv, err := strconv.ParseInt("-"+plain[3:], 2, 64)
 				if err == nil {
 					if true || intv == int64(int(intv)) {
-						return yaml_INT_TAG, int(intv)
+						return intTag, int(intv)
 					} else {
-						return yaml_INT_TAG, intv
+						return intTag, intv
+					}
+				}
+			}
+			// Octals as introduced in version 1.2 of the spec.
+			// Octals from the 1.1 spec, spelled as 0777, are still
+			// decoded by default in v3 as well for compatibility.
+			// May be dropped in v4 depending on how usage evolves.
+			if strings.HasPrefix(plain, "0o") {
+				intv, err := strconv.ParseInt(plain[2:], 8, 64)
+				if err == nil {
+					if intv == int64(int(intv)) {
+						return intTag, int(intv)
+					} else {
+						return intTag, intv
+					}
+				}
+				uintv, err := strconv.ParseUint(plain[2:], 8, 64)
+				if err == nil {
+					return intTag, uintv
+				}
+			} else if strings.HasPrefix(plain, "-0o") {
+				intv, err := strconv.ParseInt("-"+plain[3:], 8, 64)
+				if err == nil {
+					if true || intv == int64(int(intv)) {
+						return intTag, int(intv)
+					} else {
+						return intTag, intv
 					}
 				}
 			}
 		default:
-			panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")")
+			panic("internal error: missing handler for resolver table: " + string(rune(hint)) + " (with " + in + ")")
 		}
 	}
-	return yaml_STR_TAG, in
+	return strTag, in
 }
 
 // encodeBase64 encodes s as base64 that is broken up into multiple lines

vendor/gopkg.in/yaml.v3/scannerc.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (
@@ -629,8 +651,11 @@ func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool {
 		// Check if we really need to fetch more tokens.
 		need_more_tokens := false
 
-		if parser.tokens_head == len(parser.tokens) {
+		// [Go] When parsing flow items, force the queue to have at least
-			// Queue is empty.
+		// two items so that comments after commas may be associated
+		// with the value being parsed before them.
+		if parser.tokens_head == len(parser.tokens) || parser.flow_level > 0 && parser.tokens_head >= len(parser.tokens)-1 {
+			// Queue is empty or has just one element inside a flow context.
 			need_more_tokens = true
 		} else {
 			// Check if any potential simple key may occupy the head position.
@@ -662,7 +687,7 @@ func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool {
 }
 
 // The dispatcher for token fetchers.
-func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
+func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) {
 	// Ensure that the buffer is initialized.
 	if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
 		return false
@@ -717,6 +742,25 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
 		return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN)
 	}
 
+	comment_mark := parser.mark
+	if parser.flow_level > 0 && buf[pos] == ',' && len(parser.tokens) > 0 {
+		// Associate any following comments with the prior token.
+		comment_mark = parser.tokens[len(parser.tokens)-1].start_mark
+	}
+	defer func() {
+		if !ok {
+			return
+		}
+		if !yaml_parser_scan_line_comment(parser, comment_mark) {
+			ok = false
+			return
+		}
+		if !yaml_parser_scan_foot_comment(parser, comment_mark) {
+			ok = false
+			return
+		}
+	}()
+
 	// Is it the flow sequence start indicator?
 	if buf[pos] == '[' {
 		return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN)
@@ -810,7 +854,7 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
 	// if it is followed by a non-space character.
 	//
 	// The last rule is more restrictive than the specification requires.
-	// [Go] Make this logic more reasonable.
+	// [Go] TODO Make this logic more reasonable.
 	//switch parser.buffer[parser.buffer_pos] {
 	//case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`':
 	//}
@@ -1097,6 +1141,7 @@ func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_
 
 // Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
 func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool {
+
 	// The indicators '[' and '{' may start a simple key.
 	if !yaml_parser_save_simple_key(parser) {
 		return false
@@ -1455,11 +1500,8 @@ func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool {
 
 		// Eat a comment until a line break.
 		if parser.buffer[parser.buffer_pos] == '#' {
-			for !is_breakz(parser.buffer, parser.buffer_pos) {
+			if !yaml_parser_scan_head_comment(parser, parser.mark) {
-				skip(parser)
+				return false
-				if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
-					return false
-				}
 			}
 		}
 
@@ -1557,6 +1599,10 @@ func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool
 	}
 
 	if parser.buffer[parser.buffer_pos] == '#' {
+		// [Go] Discard this inline comment for the time being.
+		//if !yaml_parser_scan_line_comment(parser, start_mark) {
+		//	return false
+		//}
 		for !is_breakz(parser.buffer, parser.buffer_pos) {
 			skip(parser)
 			if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
@@ -1972,7 +2018,7 @@ func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte
 	//      '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&',
 	//      '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']',
 	//      '%'.
-	// [Go] Convert this into more reasonable logic.
+	// [Go] TODO Convert this into more reasonable logic.
 	for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' ||
 		parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' ||
 		parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' ||
@@ -2127,11 +2173,8 @@ func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, l
 		}
 	}
 	if parser.buffer[parser.buffer_pos] == '#' {
-		for !is_breakz(parser.buffer, parser.buffer_pos) {
+		if !yaml_parser_scan_line_comment(parser, start_mark) {
-			skip(parser)
+			return false
-			if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
-				return false
-			}
 		}
 	}
 
@@ -2694,3 +2737,159 @@ func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) b
 	}
 	return true
 }
+
+func yaml_parser_scan_line_comment(parser *yaml_parser_t, after yaml_mark_t) bool {
+	if parser.mark.column == 0 {
+		return true
+	}
+
+	parser.comments = append(parser.comments, yaml_comment_t{after: after})
+	comment := &parser.comments[len(parser.comments)-1].line
+
+	for peek := 0; peek < 512; peek++ {
+		if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+			break
+		}
+		if is_blank(parser.buffer, parser.buffer_pos+peek) {
+			continue
+		}
+		if parser.buffer[parser.buffer_pos+peek] == '#' {
+			if len(*comment) > 0 {
+				*comment = append(*comment, '\n')
+			}
+			for !is_breakz(parser.buffer, parser.buffer_pos+peek) {
+				*comment = append(*comment, parser.buffer[parser.buffer_pos+peek])
+				peek++
+				if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+					return false
+				}
+			}
+
+			// Skip until after the consumed comment line.
+			until := parser.buffer_pos + peek
+			for parser.buffer_pos < until {
+				if is_break(parser.buffer, parser.buffer_pos) {
+					//break // Leave the break in the buffer so calling this function twice is safe.
+					if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
+						return false
+					}
+					skip_line(parser)
+				} else {
+					skip(parser)
+				}
+			}
+		}
+		break
+	}
+	return true
+}
+
+func yaml_parser_scan_head_comment(parser *yaml_parser_t, after yaml_mark_t) bool {
+	parser.comments = append(parser.comments, yaml_comment_t{after: after})
+	comment := &parser.comments[len(parser.comments)-1].head
+	breaks := false
+	for peek := 0; peek < 512; peek++ {
+		if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+			break
+		}
+		if parser.buffer[parser.buffer_pos+peek] == 0 {
+			break
+		}
+		if is_blank(parser.buffer, parser.buffer_pos+peek) {
+			continue
+		}
+		if is_break(parser.buffer, parser.buffer_pos+peek) {
+			if !breaks {
+				*comment = append(*comment, '\n')
+			}
+			breaks = true
+		} else if parser.buffer[parser.buffer_pos+peek] == '#' {
+			if len(*comment) > 0 {
+				*comment = append(*comment, '\n')
+			}
+			breaks = false
+			for !is_breakz(parser.buffer, parser.buffer_pos+peek) {
+				*comment = append(*comment, parser.buffer[parser.buffer_pos+peek])
+				peek++
+				if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+					return false
+				}
+			}
+
+			// Skip until after the consumed comment line.
+			until := parser.buffer_pos + peek
+			for parser.buffer_pos < until {
+				if is_break(parser.buffer, parser.buffer_pos) {
+					if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
+						return false
+					}
+					skip_line(parser)
+				} else {
+					skip(parser)
+				}
+			}
+			peek = 0
+		} else {
+			break
+		}
+	}
+	return true
+}
+
+func yaml_parser_scan_foot_comment(parser *yaml_parser_t, after yaml_mark_t) bool {
+	parser.comments = append(parser.comments, yaml_comment_t{after: after})
+	comment := &parser.comments[len(parser.comments)-1].foot
+	original := *comment
+	breaks := false
+	peek := 0
+	for ; peek < 32768; peek++ {
+		if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+			break
+		}
+		c := parser.buffer[parser.buffer_pos+peek]
+		if c == 0 {
+			break
+		}
+		if is_blank(parser.buffer, parser.buffer_pos+peek) {
+			continue
+		}
+		if is_break(parser.buffer, parser.buffer_pos+peek) {
+			if breaks {
+				break
+			}
+			breaks = true
+		} else if c == '#' {
+			if len(*comment) > 0 {
+				*comment = append(*comment, '\n')
+			}
+			for !is_breakz(parser.buffer, parser.buffer_pos+peek) {
+				*comment = append(*comment, parser.buffer[parser.buffer_pos+peek])
+				peek++
+				if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) {
+					return false
+				}
+			}
+			breaks = true
+		} else if c == ']' || c == '}' {
+			break
+		} else {
+			// Abort and allow that next line to have the comment as its header.
+			*comment = original
+			return true
+		}
+	}
+
+	// Skip until after the consumed comment lines.
+	until := parser.buffer_pos + peek
+	for parser.buffer_pos < until {
+		if is_break(parser.buffer, parser.buffer_pos) {
+			if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
+				return false
+			}
+			skip_line(parser)
+		} else {
+			skip(parser)
+		}
+	}
+	return true
+}

vendor/gopkg.in/yaml.v3/sorter.go

@@ -1,3 +1,18 @@
+//
+// Copyright (c) 2011-2019 Canonical Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package yaml
 
 import (
@@ -37,8 +52,10 @@ func (l keyList) Less(i, j int) bool {
 		return ak < bk
 	}
 	ar, br := []rune(a.String()), []rune(b.String())
+	digits := false
 	for i := 0; i < len(ar) && i < len(br); i++ {
 		if ar[i] == br[i] {
+			digits = unicode.IsDigit(ar[i])
 			continue
 		}
 		al := unicode.IsLetter(ar[i])
@@ -47,12 +64,16 @@ func (l keyList) Less(i, j int) bool {
 			return ar[i] < br[i]
 		}
 		if al || bl {
-			return bl
+			if digits {
+				return al
+			} else {
+				return bl
+			}
 		}
 		var ai, bi int
 		var an, bn int64
 		if ar[i] == '0' || br[i] == '0' {
-			for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- {
+			for j := i - 1; j >= 0 && unicode.IsDigit(ar[j]); j-- {
 				if ar[j] != '0' {
 					an = 1
 					bn = 1

vendor/gopkg.in/yaml.v3/writerc.go

@@ -0,0 +1,48 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package yaml
+
+// Set the writer error and return false.
+func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool {
+	emitter.error = yaml_WRITER_ERROR
+	emitter.problem = problem
+	return false
+}
+
+// Flush the output buffer.
+func yaml_emitter_flush(emitter *yaml_emitter_t) bool {
+	if emitter.write_handler == nil {
+		panic("write handler not set")
+	}
+
+	// Check if the buffer is empty.
+	if emitter.buffer_pos == 0 {
+		return true
+	}
+
+	if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil {
+		return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error())
+	}
+	emitter.buffer_pos = 0
+	return true
+}

vendor/gopkg.in/yaml.v3/yaml.go

@@ -1,3 +1,18 @@
+//
+// Copyright (c) 2011-2019 Canonical Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Package yaml implements YAML support for the Go language.
 //
 // Source code and other details for the project are available at GitHub:
@@ -13,23 +28,16 @@ import (
 	"reflect"
 	"strings"
 	"sync"
+	"unicode/utf8"
 )
 
-// MapSlice encodes and decodes as a YAML map.
+// The Unmarshaler interface may be implemented by types to customize their
-// The order of keys is preserved when encoding and decoding.
+// behavior when being unmarshaled from a YAML document.
-type MapSlice []MapItem
+type Unmarshaler interface {
-
+	UnmarshalYAML(value *Node) error
-// MapItem is an item in a MapSlice.
-type MapItem struct {
-	Key, Value interface{}
 }
 
-// The Unmarshaler interface may be implemented by types to customize their
+type obsoleteUnmarshaler interface {
-// behavior when being unmarshaled from a YAML document. The UnmarshalYAML
-// method receives a function that may be called to unmarshal the original
-// YAML value into a field or variable. It is safe to call the unmarshal
-// function parameter more than once if necessary.
-type Unmarshaler interface {
 	UnmarshalYAML(unmarshal func(interface{}) error) error
 }
 
@@ -81,18 +89,10 @@ func Unmarshal(in []byte, out interface{}) (err error) {
 	return unmarshal(in, out, false)
 }
 
-// UnmarshalStrict is like Unmarshal except that any fields that are found
-// in the data that do not have corresponding struct members, or mapping
-// keys that are duplicates, will result in
-// an error.
-func UnmarshalStrict(in []byte, out interface{}) (err error) {
-	return unmarshal(in, out, true)
-}
-
 // A Decorder reads and decodes YAML values from an input stream.
 type Decoder struct {
-	strict bool
+	parser      *parser
-	parser *parser
+	knownFields bool
 }
 
 // NewDecoder returns a new decoder that reads from r.
@@ -105,10 +105,10 @@ func NewDecoder(r io.Reader) *Decoder {
 	}
 }
 
-// SetStrict sets whether strict decoding behaviour is enabled when
+// KnownFields ensures that the keys in decoded mappings to
-// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict.
+// exist as fields in the struct being decoded into.
-func (dec *Decoder) SetStrict(strict bool) {
+func (dec *Decoder) KnownFields(enable bool) {
-	dec.strict = strict
+	dec.knownFields = enable
 }
 
 // Decode reads the next YAML-encoded value from its input
@@ -117,7 +117,8 @@ func (dec *Decoder) SetStrict(strict bool) {
 // See the documentation for Unmarshal for details about the
 // conversion of YAML into a Go value.
 func (dec *Decoder) Decode(v interface{}) (err error) {
-	d := newDecoder(dec.strict)
+	d := newDecoder()
+	d.knownFields = dec.knownFields
 	defer handleErr(&err)
 	node := dec.parser.parse()
 	if node == nil {
@@ -134,9 +135,27 @@ func (dec *Decoder) Decode(v interface{}) (err error) {
 	return nil
 }
 
+// Decode decodes the node and stores its data into the value pointed to by v.
+//
+// See the documentation for Unmarshal for details about the
+// conversion of YAML into a Go value.
+func (n *Node) Decode(v interface{}) (err error) {
+	d := newDecoder()
+	defer handleErr(&err)
+	out := reflect.ValueOf(v)
+	if out.Kind() == reflect.Ptr && !out.IsNil() {
+		out = out.Elem()
+	}
+	d.unmarshal(n, out)
+	if len(d.terrors) > 0 {
+		return &TypeError{d.terrors}
+	}
+	return nil
+}
+
 func unmarshal(in []byte, out interface{}, strict bool) (err error) {
 	defer handleErr(&err)
-	d := newDecoder(strict)
+	d := newDecoder()
 	p := newParser(in)
 	defer p.destroy()
 	node := p.parse()
@@ -233,6 +252,14 @@ func (e *Encoder) Encode(v interface{}) (err error) {
 	return nil
 }
 
+// SetIndent changes the used indentation used when encoding.
+func (e *Encoder) SetIndent(spaces int) {
+	if spaces < 0 {
+		panic("yaml: cannot indent to a negative number of spaces")
+	}
+	e.encoder.indent = spaces
+}
+
 // Close closes the encoder by writing any remaining data.
 // It does not write a stream terminating string "...".
 func (e *Encoder) Close() (err error) {
@@ -275,6 +302,150 @@ func (e *TypeError) Error() string {
 	return fmt.Sprintf("yaml: unmarshal errors:\n  %s", strings.Join(e.Errors, "\n  "))
 }
 
+type Kind uint32
+
+const (
+	DocumentNode Kind = 1 << iota
+	SequenceNode
+	MappingNode
+	ScalarNode
+	AliasNode
+)
+
+type Style uint32
+
+const (
+	TaggedStyle Style = 1 << iota
+	DoubleQuotedStyle
+	SingleQuotedStyle
+	LiteralStyle
+	FoldedStyle
+	FlowStyle
+)
+
+// Node represents an element in the YAML document hierarchy. While documents
+// are typically encoded and decoded into higher level types, such as structs
+// and maps, Node is an intermediate representation that allows detailed
+// control over the content being decoded or encoded.
+//
+// Values that make use of the Node type interact with the yaml package in the
+// same way any other type would do, by encoding and decoding yaml data
+// directly or indirectly into them.
+//
+// For example:
+//
+//     var person struct {
+//             Name    string
+//             Address yaml.Node
+//     }
+//     err := yaml.Unmarshal(data, &person)
+// 
+// Or by itself:
+//
+//     var person Node
+//     err := yaml.Unmarshal(data, &person)
+//
+type Node struct {
+	// Kind defines whether the node is a document, a mapping, a sequence,
+	// a scalar value, or an alias to another node. The specific data type of
+	// scalar nodes may be obtained via the ShortTag and LongTag methods.
+	Kind  Kind
+
+	// Style allows customizing the apperance of the node in the tree.
+	Style Style
+
+	// Tag holds the YAML tag defining the data type for the value.
+	// When decoding, this field will always be set to the resolved tag,
+	// even when it wasn't explicitly provided in the YAML content.
+	// When encoding, if this field is unset the value type will be
+	// implied from the node properties, and if it is set, it will only
+	// be serialized into the representation if TaggedStyle is used or
+	// the implicit tag diverges from the provided one.
+	Tag string
+
+	// Value holds the unescaped and unquoted represenation of the value.
+	Value string
+
+	// Anchor holds the anchor name for this node, which allows aliases to point to it.
+	Anchor string
+
+	// Alias holds the node that this alias points to. Only valid when Kind is AliasNode.
+	Alias *Node
+
+	// Content holds contained nodes for documents, mappings, and sequences.
+	Content []*Node
+
+	// HeadComment holds any comments in the lines preceding the node and
+	// not separated by an empty line.
+	HeadComment string
+
+	// LineComment holds any comments at the end of the line where the node is in.
+	LineComment string
+
+	// FootComment holds any comments following the node and before empty lines.
+	FootComment string
+
+	// Line and Column hold the node position in the decoded YAML text.
+	// These fields are not respected when encoding the node.
+	Line   int
+	Column int
+}
+
+// LongTag returns the long form of the tag that indicates the data type for
+// the node. If the Tag field isn't explicitly defined, one will be computed
+// based on the node properties.
+func (n *Node) LongTag() string {
+	return longTag(n.ShortTag())
+}
+
+// ShortTag returns the short form of the YAML tag that indicates data type for
+// the node. If the Tag field isn't explicitly defined, one will be computed
+// based on the node properties.
+func (n *Node) ShortTag() string {
+	if n.indicatedString() {
+		return strTag
+	}
+	if n.Tag == "" || n.Tag == "!" {
+		switch n.Kind {
+		case MappingNode:
+			return mapTag
+		case SequenceNode:
+			return seqTag
+		case AliasNode:
+			if n.Alias != nil {
+				return n.Alias.ShortTag()
+			}
+		case ScalarNode:
+			tag, _ := resolve("", n.Value)
+			return tag
+		}
+		return ""
+	}
+	return shortTag(n.Tag)
+}
+
+func (n *Node) indicatedString() bool {
+	return n.Kind == ScalarNode &&
+		(shortTag(n.Tag) == strTag ||
+			(n.Tag == "" || n.Tag == "!") && n.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0)
+}
+
+// SetString is a convenience function that sets the node to a string value
+// and defines its style in a pleasant way depending on its content.
+func (n *Node) SetString(s string) {
+	n.Kind = ScalarNode
+	if utf8.ValidString(s) {
+		n.Value = s
+		n.Tag = strTag
+	} else {
+		n.Value = encodeBase64(s)
+		n.Tag = binaryTag
+	}
+	if strings.Contains(n.Value, "\n") {
+		n.Style = LiteralStyle
+	}
+}
+
 // --------------------------------------------------------------------------
 // Maintain a mapping of keys to structure field indexes
 
@@ -289,6 +460,10 @@ type structInfo struct {
 	// InlineMap is the number of the field in the struct that
 	// contains an ,inline map, or -1 if there's none.
 	InlineMap int
+
+	// InlineUnmarshalers holds indexes to inlined fields that
+	// contain unmarshaler values.
+	InlineUnmarshalers [][]int
 }
 
 type fieldInfo struct {
@@ -306,6 +481,12 @@ type fieldInfo struct {
 
 var structMap = make(map[reflect.Type]*structInfo)
 var fieldMapMutex sync.RWMutex
+var unmarshalerType reflect.Type
+
+func init() {
+	var v Unmarshaler
+	unmarshalerType = reflect.ValueOf(&v).Elem().Type()
+}
 
 func getStructInfo(st reflect.Type) (*structInfo, error) {
 	fieldMapMutex.RLock()
@@ -319,6 +500,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
 	fieldsMap := make(map[string]fieldInfo)
 	fieldsList := make([]fieldInfo, 0, n)
 	inlineMap := -1
+	inlineUnmarshalers := [][]int(nil)
 	for i := 0; i != n; i++ {
 		field := st.Field(i)
 		if field.PkgPath != "" && !field.Anonymous {
@@ -347,7 +529,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
 				case "inline":
 					inline = true
 				default:
-					return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st))
+					return nil, errors.New(fmt.Sprintf("unsupported flag %q in tag %q of type %s", flag, tag, st))
 				}
 			}
 			tag = fields[0]
@@ -357,34 +539,47 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
 			switch field.Type.Kind() {
 			case reflect.Map:
 				if inlineMap >= 0 {
-					return nil, errors.New("Multiple ,inline maps in struct " + st.String())
+					return nil, errors.New("multiple ,inline maps in struct " + st.String())
 				}
 				if field.Type.Key() != reflect.TypeOf("") {
-					return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String())
+					return nil, errors.New("option ,inline needs a map with string keys in struct " + st.String())
 				}
 				inlineMap = info.Num
-			case reflect.Struct:
+			case reflect.Struct, reflect.Ptr:
-				sinfo, err := getStructInfo(field.Type)
+				ftype := field.Type
-				if err != nil {
+				for ftype.Kind() == reflect.Ptr {
-					return nil, err
+					ftype = ftype.Elem()
 				}
-				for _, finfo := range sinfo.FieldsList {
+				if ftype.Kind() != reflect.Struct {
-					if _, found := fieldsMap[finfo.Key]; found {
+					return nil, errors.New("option ,inline may only be used on a struct or map field")
-						msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String()
+				}
-						return nil, errors.New(msg)
+				if reflect.PtrTo(ftype).Implements(unmarshalerType) {
+					inlineUnmarshalers = append(inlineUnmarshalers, []int{i})
+				} else {
+					sinfo, err := getStructInfo(ftype)
+					if err != nil {
+						return nil, err
 					}
-					if finfo.Inline == nil {
+					for _, index := range sinfo.InlineUnmarshalers {
-						finfo.Inline = []int{i, finfo.Num}
+						inlineUnmarshalers = append(inlineUnmarshalers, append([]int{i}, index...))
-					} else {
+					}
-						finfo.Inline = append([]int{i}, finfo.Inline...)
+					for _, finfo := range sinfo.FieldsList {
+						if _, found := fieldsMap[finfo.Key]; found {
+							msg := "duplicated key '" + finfo.Key + "' in struct " + st.String()
+							return nil, errors.New(msg)
+						}
+						if finfo.Inline == nil {
+							finfo.Inline = []int{i, finfo.Num}
+						} else {
+							finfo.Inline = append([]int{i}, finfo.Inline...)
+						}
+						finfo.Id = len(fieldsList)
+						fieldsMap[finfo.Key] = finfo
+						fieldsList = append(fieldsList, finfo)
 					}
-					finfo.Id = len(fieldsList)
-					fieldsMap[finfo.Key] = finfo
-					fieldsList = append(fieldsList, finfo)
 				}
 			default:
-				//return nil, errors.New("Option ,inline needs a struct value or map field")
+				return nil, errors.New("option ,inline may only be used on a struct or map field")
-				return nil, errors.New("Option ,inline needs a struct value field")
 			}
 			continue
 		}
@@ -396,7 +591,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
 		}
 
 		if _, found = fieldsMap[info.Key]; found {
-			msg := "Duplicated key '" + info.Key + "' in struct " + st.String()
+			msg := "duplicated key '" + info.Key + "' in struct " + st.String()
 			return nil, errors.New(msg)
 		}
 
@@ -406,9 +601,10 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
 	}
 
 	sinfo = &structInfo{
-		FieldsMap:  fieldsMap,
+		FieldsMap:          fieldsMap,
-		FieldsList: fieldsList,
+		FieldsList:         fieldsList,
-		InlineMap:  inlineMap,
+		InlineMap:          inlineMap,
+		InlineUnmarshalers: inlineUnmarshalers,
 	}
 
 	fieldMapMutex.Lock()

vendor/gopkg.in/yaml.v3/yamlh.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 import (
@@ -73,13 +95,13 @@ type yaml_scalar_style_t yaml_style_t
 // Scalar styles.
 const (
 	// Let the emitter choose the style.
-	yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota
+	yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = 0
 
-	yaml_PLAIN_SCALAR_STYLE         // The plain scalar style.
+	yaml_PLAIN_SCALAR_STYLE         yaml_scalar_style_t = 1 << iota // The plain scalar style.
-	yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style.
+	yaml_SINGLE_QUOTED_SCALAR_STYLE                                 // The single-quoted scalar style.
-	yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style.
+	yaml_DOUBLE_QUOTED_SCALAR_STYLE                                 // The double-quoted scalar style.
-	yaml_LITERAL_SCALAR_STYLE       // The literal scalar style.
+	yaml_LITERAL_SCALAR_STYLE                                       // The literal scalar style.
-	yaml_FOLDED_SCALAR_STYLE        // The folded scalar style.
+	yaml_FOLDED_SCALAR_STYLE                                        // The folded scalar style.
 )
 
 type yaml_sequence_style_t yaml_style_t
@@ -279,6 +301,11 @@ type yaml_event_t struct {
 	// The list of tag directives (for yaml_DOCUMENT_START_EVENT).
 	tag_directives []yaml_tag_directive_t
 
+	// The comments
+	head_comment []byte
+	line_comment []byte
+	foot_comment []byte
+
 	// The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT).
 	anchor []byte
 
@@ -562,6 +589,15 @@ type yaml_parser_t struct {
 	offset int         // The offset of the current position (in bytes).
 	mark   yaml_mark_t // The mark of the current position.
 
+	// Comments
+
+	head_comment []byte // The current head comments
+	line_comment []byte // The current line comments
+	foot_comment []byte // The current foot comments
+
+	comments      []yaml_comment_t // The folded comments for all parsed tokens
+	comments_head int
+
 	// Scanner stuff
 
 	stream_start_produced bool // Have we started to scan the input stream?
@@ -594,6 +630,13 @@ type yaml_parser_t struct {
 	document *yaml_document_t // The currently parsed document.
 }
 
+type yaml_comment_t struct {
+	after yaml_mark_t
+	head  []byte
+	line  []byte
+	foot  []byte
+}
+
 // Emitter Definitions
 
 // The prototype of a write handler.
@@ -624,8 +667,10 @@ const (
 	yaml_EMIT_DOCUMENT_CONTENT_STATE           // Expect the content of a document.
 	yaml_EMIT_DOCUMENT_END_STATE               // Expect DOCUMENT-END.
 	yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE   // Expect the first item of a flow sequence.
+	yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE   // Expect the next item of a flow sequence, with the comma already written out
 	yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE         // Expect an item of a flow sequence.
 	yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE     // Expect the first key of a flow mapping.
+	yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE     // Expect the next key of a flow mapping, with the comma already written out
 	yaml_EMIT_FLOW_MAPPING_KEY_STATE           // Expect a key of a flow mapping.
 	yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE  // Expect a value for a simple key of a flow mapping.
 	yaml_EMIT_FLOW_MAPPING_VALUE_STATE         // Expect a value of a flow mapping.
@@ -697,6 +742,8 @@ type yaml_emitter_t struct {
 	indention  bool // If the last character was an indentation character (' ', '-', '?', ':')?
 	open_ended bool // If an explicit document end is required?
 
+	space_above bool // If there's an empty line right above?
+
 	// Anchor analysis.
 	anchor_data struct {
 		anchor []byte // The anchor value.
@@ -720,6 +767,11 @@ type yaml_emitter_t struct {
 		style                 yaml_scalar_style_t // The output style.
 	}
 
+	// Comments
+	head_comment []byte
+	line_comment []byte
+	foot_comment []byte
+
 	// Dumper stuff
 
 	opened bool // If the stream was already opened?

vendor/gopkg.in/yaml.v3/yamlprivateh.go

@@ -1,3 +1,25 @@
+// 
+// Copyright (c) 2011-2019 Canonical Ltd
+// Copyright (c) 2006-2010 Kirill Simonov
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
 package yaml
 
 const (

vendor/modules.txt

@@ -12,18 +12,18 @@ github.com/google/uuid
 github.com/huandu/xstrings
 # github.com/imdario/mergo v0.3.6
 github.com/imdario/mergo
-# github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53
+# github.com/mattn/go-zglob v0.0.1
 github.com/mattn/go-zglob
 github.com/mattn/go-zglob/fastwalk
 # github.com/mitchellh/go-homedir v1.0.0
 github.com/mitchellh/go-homedir
 # github.com/pmezard/go-difflib v1.0.0
 github.com/pmezard/go-difflib/difflib
-# github.com/radovskyb/watcher v1.0.2
+# github.com/radovskyb/watcher v1.0.5
 github.com/radovskyb/watcher
 # github.com/spf13/pflag v1.0.3
 github.com/spf13/pflag
-# github.com/stretchr/testify v1.2.2
+# github.com/stretchr/testify v1.3.0
 github.com/stretchr/testify/assert
 # golang.org/x/crypto v0.0.0-20180830192347-182538f80094
 golang.org/x/crypto/ssh/terminal
@@ -36,9 +36,10 @@ golang.org/x/sync/errgroup
 # golang.org/x/sys v0.0.0-20180831094639-fa5fdf94c789
 golang.org/x/sys/unix
 golang.org/x/sys/windows
-# gopkg.in/yaml.v2 v2.2.1
+# gopkg.in/yaml.v3 v3.0.0-20190409140830-cdc409dda467
-gopkg.in/yaml.v2
+gopkg.in/yaml.v3
-# mvdan.cc/sh v0.0.0-20180829163519-3a244a89e2e5
+# mvdan.cc/sh v2.6.4+incompatible
-mvdan.cc/sh/shell
+mvdan.cc/sh/expand
 mvdan.cc/sh/interp
+mvdan.cc/sh/shell
 mvdan.cc/sh/syntax

vendor/mvdan.cc/sh/expand/arith.go

@@ -1,57 +1,66 @@
 // Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
 // See LICENSE for licensing information
 
-package interp
+package expand
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 
 	"mvdan.cc/sh/syntax"
 )
 
-func (r *Runner) arithm(ctx context.Context, expr syntax.ArithmExpr) int {
+func Arithm(cfg *Config, expr syntax.ArithmExpr) (int, error) {
 	switch x := expr.(type) {
 	case *syntax.Word:
-		str := r.loneWord(ctx, x)
+		str, err := Literal(cfg, x)
+		if err != nil {
+			return 0, err
+		}
 		// recursively fetch vars
-		for str != "" {
+		i := 0
-			val := r.getVar(str)
+		for str != "" && syntax.ValidName(str) {
+			val := cfg.envGet(str)
 			if val == "" {
 				break
 			}
+			if i++; i >= maxNameRefDepth {
+				break
+			}
 			str = val
 		}
 		// default to 0
-		return atoi(str)
+		return atoi(str), nil
 	case *syntax.ParenArithm:
-		return r.arithm(ctx, x.X)
+		return Arithm(cfg, x.X)
 	case *syntax.UnaryArithm:
 		switch x.Op {
 		case syntax.Inc, syntax.Dec:
-			name := x.X.(*syntax.Word).Parts[0].(*syntax.Lit).Value
+			name := x.X.(*syntax.Word).Lit()
-			old := atoi(r.getVar(name))
+			old := atoi(cfg.envGet(name))
 			val := old
 			if x.Op == syntax.Inc {
 				val++
 			} else {
 				val--
 			}
-			r.setVarString(ctx, name, strconv.Itoa(val))
+			cfg.envSet(name, strconv.Itoa(val))
 			if x.Post {
-				return old
+				return old, nil
 			}
-			return val
+			return val, nil
+		}
+		val, err := Arithm(cfg, x.X)
+		if err != nil {
+			return 0, err
 		}
-		val := r.arithm(ctx, x.X)
 		switch x.Op {
 		case syntax.Not:
-			return oneIf(val == 0)
+			return oneIf(val == 0), nil
 		case syntax.Plus:
-			return val
+			return val, nil
 		default: // syntax.Minus
-			return -val
+			return -val, nil
 		}
 	case *syntax.BinaryArithm:
 		switch x.Op {
@@ -59,16 +68,27 @@ func (r *Runner) arithm(ctx context.Context, expr syntax.ArithmExpr) int {
 			syntax.MulAssgn, syntax.QuoAssgn, syntax.RemAssgn,
 			syntax.AndAssgn, syntax.OrAssgn, syntax.XorAssgn,
 			syntax.ShlAssgn, syntax.ShrAssgn:
-			return r.assgnArit(ctx, x)
+			return cfg.assgnArit(x)
 		case syntax.Quest: // Colon can't happen here
-			cond := r.arithm(ctx, x.X)
+			cond, err := Arithm(cfg, x.X)
+			if err != nil {
+				return 0, err
+			}
 			b2 := x.Y.(*syntax.BinaryArithm) // must have Op==Colon
 			if cond == 1 {
-				return r.arithm(ctx, b2.X)
+				return Arithm(cfg, b2.X)
 			}
-			return r.arithm(ctx, b2.Y)
+			return Arithm(cfg, b2.Y)
 		}
-		return binArit(x.Op, r.arithm(ctx, x.X), r.arithm(ctx, x.Y))
+		left, err := Arithm(cfg, x.X)
+		if err != nil {
+			return 0, err
+		}
+		right, err := Arithm(cfg, x.Y)
+		if err != nil {
+			return 0, err
+		}
+		return binArit(x.Op, left, right), nil
 	default:
 		panic(fmt.Sprintf("unexpected arithm expr: %T", x))
 	}
@@ -88,10 +108,13 @@ func atoi(s string) int {
 	return n
 }
 
-func (r *Runner) assgnArit(ctx context.Context, b *syntax.BinaryArithm) int {
+func (cfg *Config) assgnArit(b *syntax.BinaryArithm) (int, error) {
-	name := b.X.(*syntax.Word).Parts[0].(*syntax.Lit).Value
+	name := b.X.(*syntax.Word).Lit()
-	val := atoi(r.getVar(name))
+	val := atoi(cfg.envGet(name))
-	arg := r.arithm(ctx, b.Y)
+	arg, err := Arithm(cfg, b.Y)
+	if err != nil {
+		return 0, err
+	}
 	switch b.Op {
 	case syntax.Assgn:
 		val = arg
@@ -116,8 +139,8 @@ func (r *Runner) assgnArit(ctx context.Context, b *syntax.BinaryArithm) int {
 	case syntax.ShrAssgn:
 		val >>= uint(arg)
 	}
-	r.setVarString(ctx, name, strconv.Itoa(val))
+	cfg.envSet(name, strconv.Itoa(val))
-	return val
+	return val, nil
 }
 
 func intPow(a, b int) int {

vendor/mvdan.cc/sh/expand/braces.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2018, Daniel Martí <mvdan@mvdan.cc>
+// See LICENSE for licensing information
+
+package expand
+
+import "mvdan.cc/sh/syntax"
+
+// Braces performs Bash brace expansion on words. For example, passing it a
+// literal word "foo{bar,baz}" will return two literal words, "foobar" and
+// "foobaz".
+//
+// It does not return an error; malformed brace expansions are simply skipped.
+// For example, "a{b{c,d}" results in the words "a{bc" and "a{bd".
+//
+// Note that the resulting words may have more word parts than necessary, such
+// as contiguous *syntax.Lit nodes, and that these parts may be shared between
+// words.
+func Braces(words ...*syntax.Word) []*syntax.Word {
+	var res []*syntax.Word
+	for _, word := range words {
+		res = append(res, syntax.ExpandBraces(word)...)
+	}
+	return res
+}

vendor/mvdan.cc/sh/expand/doc.go

@@ -0,0 +1,5 @@
+// Copyright (c) 2018, Daniel Martí <mvdan@mvdan.cc>
+// See LICENSE for licensing information
+
+// Package expand contains code to perform various shell expansions.
+package expand

vendor/mvdan.cc/sh/expand/environ.go

@@ -0,0 +1,195 @@
+// Copyright (c) 2018, Daniel Martí <mvdan@mvdan.cc>
+// See LICENSE for licensing information
+
+package expand
+
+import (
+	"runtime"
+	"sort"
+	"strings"
+)
+
+// Environ is the base interface for a shell's environment, allowing it to fetch
+// variables by name and to iterate over all the currently set variables.
+type Environ interface {
+	// Get retrieves a variable by its name. To check if the variable is
+	// set, use Variable.IsSet.
+	Get(name string) Variable
+
+	// Each iterates over all the currently set variables, calling the
+	// supplied function on each variable. Iteration is stopped if the
+	// function returns false.
+	//
+	// The names used in the calls aren't required to be unique or sorted.
+	// If a variable name appears twice, the latest occurrence takes
+	// priority.
+	//
+	// Each is required to forward exported variables when executing
+	// programs.
+	Each(func(name string, vr Variable) bool)
+}
+
+// WriteEnviron is an extension on Environ that supports modifying and deleting
+// variables.
+type WriteEnviron interface {
+	Environ
+	// Set sets a variable by name. If !vr.IsSet(), the variable is being
+	// unset; otherwise, the variable is being replaced.
+	//
+	// It is the implementation's responsibility to handle variable
+	// attributes correctly. For example, changing an exported variable's
+	// value does not unexport it, and overwriting a name reference variable
+	// should modify its target.
+	Set(name string, vr Variable)
+}
+
+// Variable describes a shell variable, which can have a number of attributes
+// and a value.
+//
+// A Variable is unset if its Value field is untyped nil, which can be checked
+// via Variable.IsSet. The zero value of a Variable is thus a valid unset
+// variable.
+//
+// If a variable is set, its Value field will be a []string if it is an indexed
+// array, a map[string]string if it's an associative array, or a string
+// otherwise.
+type Variable struct {
+	Local    bool
+	Exported bool
+	ReadOnly bool
+	NameRef  bool        // if true, Value must be string
+	Value    interface{} // string, []string, or map[string]string
+}
+
+// IsSet returns whether the variable is set. An empty variable is set, but an
+// undeclared variable is not.
+func (v Variable) IsSet() bool {
+	return v.Value != nil
+}
+
+// String returns the variable's value as a string. In general, this only makes
+// sense if the variable has a string value or no value at all.
+func (v Variable) String() string {
+	switch x := v.Value.(type) {
+	case string:
+		return x
+	case []string:
+		if len(x) > 0 {
+			return x[0]
+		}
+	case map[string]string:
+		// nothing to do
+	}
+	return ""
+}
+
+// maxNameRefDepth defines the maximum number of times to follow references when
+// resolving a variable. Otherwise, simple name reference loops could crash a
+// program quite easily.
+const maxNameRefDepth = 100
+
+// Resolve follows a number of nameref variables, returning the last reference
+// name that was followed and the variable that it points to.
+func (v Variable) Resolve(env Environ) (string, Variable) {
+	name := ""
+	for i := 0; i < maxNameRefDepth; i++ {
+		if !v.NameRef {
+			return name, v
+		}
+		name = v.Value.(string)
+		v = env.Get(name)
+	}
+	return name, Variable{}
+}
+
+// FuncEnviron wraps a function mapping variable names to their string values,
+// and implements Environ. Empty strings returned by the function will be
+// treated as unset variables. All variables will be exported.
+//
+// Note that the returned Environ's Each method will be a no-op.
+func FuncEnviron(fn func(string) string) Environ {
+	return funcEnviron(fn)
+}
+
+type funcEnviron func(string) string
+
+func (f funcEnviron) Get(name string) Variable {
+	value := f(name)
+	if value == "" {
+		return Variable{}
+	}
+	return Variable{Exported: true, Value: value}
+}
+
+func (f funcEnviron) Each(func(name string, vr Variable) bool) {}
+
+// ListEnviron returns an Environ with the supplied variables, in the form
+// "key=value". All variables will be exported.
+//
+// On Windows, where environment variable names are case-insensitive, the
+// resulting variable names will all be uppercase.
+func ListEnviron(pairs ...string) Environ {
+	return listEnvironWithUpper(runtime.GOOS == "windows", pairs...)
+}
+
+// listEnvironWithUpper implements ListEnviron, but letting the tests specify
+// whether to uppercase all names or not.
+func listEnvironWithUpper(upper bool, pairs ...string) Environ {
+	list := append([]string{}, pairs...)
+	if upper {
+		// Uppercase before sorting, so that we can remove duplicates
+		// without the need for linear search nor a map.
+		for i, s := range list {
+			if sep := strings.IndexByte(s, '='); sep > 0 {
+				list[i] = strings.ToUpper(s[:sep]) + s[sep:]
+			}
+		}
+	}
+	sort.Strings(list)
+	last := ""
+	for i := 0; i < len(list); {
+		s := list[i]
+		sep := strings.IndexByte(s, '=')
+		if sep <= 0 {
+			// invalid element; remove it
+			list = append(list[:i], list[i+1:]...)
+			continue
+		}
+		name := s[:sep]
+		if last == name {
+			// duplicate; the last one wins
+			list = append(list[:i-1], list[i:]...)
+			continue
+		}
+		last = name
+		i++
+	}
+	return listEnviron(list)
+}
+
+type listEnviron []string
+
+func (l listEnviron) Get(name string) Variable {
+	// TODO: binary search
+	prefix := name + "="
+	for _, pair := range l {
+		if val := strings.TrimPrefix(pair, prefix); val != pair {
+			return Variable{Exported: true, Value: val}
+		}
+	}
+	return Variable{}
+}
+
+func (l listEnviron) Each(fn func(name string, vr Variable) bool) {
+	for _, pair := range l {
+		i := strings.IndexByte(pair, '=')
+		if i < 0 {
+			// can't happen; see above
+			panic("expand.listEnviron: did not expect malformed name-value pair: " + pair)
+		}
+		name, value := pair[:i], pair[i+1:]
+		if !fn(name, Variable{Exported: true, Value: value}) {
+			return
+		}
+	}
+}

vendor/mvdan.cc/sh/expand/expand.go

@@ -0,0 +1,799 @@
+// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
+// See LICENSE for licensing information
+
+package expand
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/user"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"mvdan.cc/sh/syntax"
+)
+
+// A Config specifies details about how shell expansion should be performed. The
+// zero value is a valid configuration.
+type Config struct {
+	// Env is used to get and set environment variables when performing
+	// shell expansions. Some special parameters are also expanded via this
+	// interface, such as:
+	//
+	//   * "#", "@", "*", "0"-"9" for the shell's parameters
+	//   * "?", "$", "PPID" for the shell's status and process
+	//   * "HOME foo" to retrieve user foo's home directory (if unset,
+	//     os/user.Lookup will be used)
+	//
+	// If nil, there are no environment variables set. Use
+	// ListEnviron(os.Environ()...) to use the system's environment
+	// variables.
+	Env Environ
+
+	// TODO(mvdan): consider replacing NoGlob==true with ReadDir==nil.
+
+	// NoGlob corresponds to the shell option that disables globbing.
+	NoGlob bool
+	// GlobStar corresponds to the shell option that allows globbing with
+	// "**".
+	GlobStar bool
+
+	// CmdSubst expands a command substitution node, writing its standard
+	// output to the provided io.Writer.
+	//
+	// If nil, encountering a command substitution will result in an
+	// UnexpectedCommandError.
+	CmdSubst func(io.Writer, *syntax.CmdSubst) error
+
+	// ReadDir is used for file path globbing. If nil, globbing is disabled.
+	// Use ioutil.ReadDir to use the filesystem directly.
+	ReadDir func(string) ([]os.FileInfo, error)
+
+	bufferAlloc bytes.Buffer
+	fieldAlloc  [4]fieldPart
+	fieldsAlloc [4][]fieldPart
+
+	ifs string
+	// A pointer to a parameter expansion node, if we're inside one.
+	// Necessary for ${LINENO}.
+	curParam *syntax.ParamExp
+}
+
+// UnexpectedCommandError is returned if a command substitution is encountered
+// when Config.CmdSubst is nil.
+type UnexpectedCommandError struct {
+	Node *syntax.CmdSubst
+}
+
+func (u UnexpectedCommandError) Error() string {
+	return fmt.Sprintf("unexpected command substitution at %s", u.Node.Pos())
+}
+
+var zeroConfig = &Config{}
+
+func prepareConfig(cfg *Config) *Config {
+	if cfg == nil {
+		cfg = zeroConfig
+	}
+	if cfg.Env == nil {
+		cfg.Env = FuncEnviron(func(string) string { return "" })
+	}
+
+	cfg.ifs = " \t\n"
+	if vr := cfg.Env.Get("IFS"); vr.IsSet() {
+		cfg.ifs = vr.String()
+	}
+	return cfg
+}
+
+func (cfg *Config) ifsRune(r rune) bool {
+	for _, r2 := range cfg.ifs {
+		if r == r2 {
+			return true
+		}
+	}
+	return false
+}
+
+func (cfg *Config) ifsJoin(strs []string) string {
+	sep := ""
+	if cfg.ifs != "" {
+		sep = cfg.ifs[:1]
+	}
+	return strings.Join(strs, sep)
+}
+
+func (cfg *Config) strBuilder() *bytes.Buffer {
+	b := &cfg.bufferAlloc
+	b.Reset()
+	return b
+}
+
+func (cfg *Config) envGet(name string) string {
+	return cfg.Env.Get(name).String()
+}
+
+func (cfg *Config) envSet(name, value string) {
+	wenv, ok := cfg.Env.(WriteEnviron)
+	if !ok {
+		// TODO: we should probably error here
+		return
+	}
+	wenv.Set(name, Variable{Value: value})
+}
+
+// Literal expands a single shell word. It is similar to Fields, but the result
+// is a single string. This is the behavior when a word is used as the value in
+// a shell variable assignment, for example.
+//
+// The config specifies shell expansion options; nil behaves the same as an
+// empty config.
+func Literal(cfg *Config, word *syntax.Word) (string, error) {
+	if word == nil {
+		return "", nil
+	}
+	cfg = prepareConfig(cfg)
+	field, err := cfg.wordField(word.Parts, quoteNone)
+	if err != nil {
+		return "", err
+	}
+	return cfg.fieldJoin(field), nil
+}
+
+// Document expands a single shell word as if it were within double quotes. It
+// is simlar to Literal, but without brace expansion, tilde expansion, and
+// globbing.
+//
+// The config specifies shell expansion options; nil behaves the same as an
+// empty config.
+func Document(cfg *Config, word *syntax.Word) (string, error) {
+	if word == nil {
+		return "", nil
+	}
+	cfg = prepareConfig(cfg)
+	field, err := cfg.wordField(word.Parts, quoteDouble)
+	if err != nil {
+		return "", err
+	}
+	return cfg.fieldJoin(field), nil
+}
+
+// Pattern expands a single shell word as a pattern, using syntax.QuotePattern
+// on any non-quoted parts of the input word. The result can be used on
+// syntax.TranslatePattern directly.
+//
+// The config specifies shell expansion options; nil behaves the same as an
+// empty config.
+func Pattern(cfg *Config, word *syntax.Word) (string, error) {
+	cfg = prepareConfig(cfg)
+	field, err := cfg.wordField(word.Parts, quoteNone)
+	if err != nil {
+		return "", err
+	}
+	buf := cfg.strBuilder()
+	for _, part := range field {
+		if part.quote > quoteNone {
+			buf.WriteString(syntax.QuotePattern(part.val))
+		} else {
+			buf.WriteString(part.val)
+		}
+	}
+	return buf.String(), nil
+}
+
+// Format expands a format string with a number of arguments, following the
+// shell's format specifications. These include printf(1), among others.
+//
+// The resulting string is returned, along with the number of arguments used.
+//
+// The config specifies shell expansion options; nil behaves the same as an
+// empty config.
+func Format(cfg *Config, format string, args []string) (string, int, error) {
+	cfg = prepareConfig(cfg)
+	buf := cfg.strBuilder()
+	esc := false
+	var fmts []rune
+	initialArgs := len(args)
+
+	for _, c := range format {
+		switch {
+		case esc:
+			esc = false
+			switch c {
+			case 'n':
+				buf.WriteRune('\n')
+			case 'r':
+				buf.WriteRune('\r')
+			case 't':
+				buf.WriteRune('\t')
+			case '\\':
+				buf.WriteRune('\\')
+			default:
+				buf.WriteRune('\\')
+				buf.WriteRune(c)
+			}
+
+		case len(fmts) > 0:
+			switch c {
+			case '%':
+				buf.WriteByte('%')
+				fmts = nil
+			case 'c':
+				var b byte
+				if len(args) > 0 {
+					arg := ""
+					arg, args = args[0], args[1:]
+					if len(arg) > 0 {
+						b = arg[0]
+					}
+				}
+				buf.WriteByte(b)
+				fmts = nil
+			case '+', '-', ' ':
+				if len(fmts) > 1 {
+					return "", 0, fmt.Errorf("invalid format char: %c", c)
+				}
+				fmts = append(fmts, c)
+			case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+				fmts = append(fmts, c)
+			case 's', 'd', 'i', 'u', 'o', 'x':
+				arg := ""
+				if len(args) > 0 {
+					arg, args = args[0], args[1:]
+				}
+				var farg interface{} = arg
+				if c != 's' {
+					n, _ := strconv.ParseInt(arg, 0, 0)
+					if c == 'i' || c == 'd' {
+						farg = int(n)
+					} else {
+						farg = uint(n)
+					}
+					if c == 'i' || c == 'u' {
+						c = 'd'
+					}
+				}
+				fmts = append(fmts, c)
+				fmt.Fprintf(buf, string(fmts), farg)
+				fmts = nil
+			default:
+				return "", 0, fmt.Errorf("invalid format char: %c", c)
+			}
+		case c == '\\':
+			esc = true
+		case args != nil && c == '%':
+			// if args == nil, we are not doing format
+			// arguments
+			fmts = []rune{c}
+		default:
+			buf.WriteRune(c)
+		}
+	}
+	if len(fmts) > 0 {
+		return "", 0, fmt.Errorf("missing format char")
+	}
+	return buf.String(), initialArgs - len(args), nil
+}
+
+func (cfg *Config) fieldJoin(parts []fieldPart) string {
+	switch len(parts) {
+	case 0:
+		return ""
+	case 1: // short-cut without a string copy
+		return parts[0].val
+	}
+	buf := cfg.strBuilder()
+	for _, part := range parts {
+		buf.WriteString(part.val)
+	}
+	return buf.String()
+}
+
+func (cfg *Config) escapedGlobField(parts []fieldPart) (escaped string, glob bool) {
+	buf := cfg.strBuilder()
+	for _, part := range parts {
+		if part.quote > quoteNone {
+			buf.WriteString(syntax.QuotePattern(part.val))
+			continue
+		}
+		buf.WriteString(part.val)
+		if syntax.HasPattern(part.val) {
+			glob = true
+		}
+	}
+	if glob { // only copy the string if it will be used
+		escaped = buf.String()
+	}
+	return escaped, glob
+}
+
+// Fields expands a number of words as if they were arguments in a shell
+// command. This includes brace expansion, tilde expansion, parameter expansion,
+// command substitution, arithmetic expansion, and quote removal.
+func Fields(cfg *Config, words ...*syntax.Word) ([]string, error) {
+	cfg = prepareConfig(cfg)
+	fields := make([]string, 0, len(words))
+	dir := cfg.envGet("PWD")
+	for _, expWord := range Braces(words...) {
+		wfields, err := cfg.wordFields(expWord.Parts)
+		if err != nil {
+			return nil, err
+		}
+		for _, field := range wfields {
+			path, doGlob := cfg.escapedGlobField(field)
+			var matches []string
+			abs := filepath.IsAbs(path)
+			if doGlob && !cfg.NoGlob {
+				base := ""
+				if !abs {
+					base = dir
+				}
+				matches, err = cfg.glob(base, path)
+				if err != nil {
+					return nil, err
+				}
+			}
+			if len(matches) == 0 {
+				fields = append(fields, cfg.fieldJoin(field))
+				continue
+			}
+			for _, match := range matches {
+				if !abs {
+					match = strings.TrimPrefix(match, dir)
+				}
+				fields = append(fields, match)
+			}
+		}
+	}
+	return fields, nil
+}
+
+type fieldPart struct {
+	val   string
+	quote quoteLevel
+}
+
+type quoteLevel uint
+
+const (
+	quoteNone quoteLevel = iota
+	quoteDouble
+	quoteSingle
+)
+
+func (cfg *Config) wordField(wps []syntax.WordPart, ql quoteLevel) ([]fieldPart, error) {
+	var field []fieldPart
+	for i, wp := range wps {
+		switch x := wp.(type) {
+		case *syntax.Lit:
+			s := x.Value
+			if i == 0 && ql == quoteNone {
+				if prefix, rest := cfg.expandUser(s); prefix != "" {
+					// TODO: return two separate fieldParts,
+					// like in wordFields?
+					s = prefix + rest
+				}
+			}
+			if ql == quoteDouble && strings.Contains(s, "\\") {
+				buf := cfg.strBuilder()
+				for i := 0; i < len(s); i++ {
+					b := s[i]
+					if b == '\\' && i+1 < len(s) {
+						switch s[i+1] {
+						case '\n': // remove \\\n
+							i++
+							continue
+						case '"', '\\', '$', '`': // special chars
+							continue
+						}
+					}
+					buf.WriteByte(b)
+				}
+				s = buf.String()
+			}
+			field = append(field, fieldPart{val: s})
+		case *syntax.SglQuoted:
+			fp := fieldPart{quote: quoteSingle, val: x.Value}
+			if x.Dollar {
+				fp.val, _, _ = Format(cfg, fp.val, nil)
+			}
+			field = append(field, fp)
+		case *syntax.DblQuoted:
+			wfield, err := cfg.wordField(x.Parts, quoteDouble)
+			if err != nil {
+				return nil, err
+			}
+			for _, part := range wfield {
+				part.quote = quoteDouble
+				field = append(field, part)
+			}
+		case *syntax.ParamExp:
+			val, err := cfg.paramExp(x)
+			if err != nil {
+				return nil, err
+			}
+			field = append(field, fieldPart{val: val})
+		case *syntax.CmdSubst:
+			val, err := cfg.cmdSubst(x)
+			if err != nil {
+				return nil, err
+			}
+			field = append(field, fieldPart{val: val})
+		case *syntax.ArithmExp:
+			n, err := Arithm(cfg, x.X)
+			if err != nil {
+				return nil, err
+			}
+			field = append(field, fieldPart{val: strconv.Itoa(n)})
+		default:
+			panic(fmt.Sprintf("unhandled word part: %T", x))
+		}
+	}
+	return field, nil
+}
+
+func (cfg *Config) cmdSubst(cs *syntax.CmdSubst) (string, error) {
+	if cfg.CmdSubst == nil {
+		return "", UnexpectedCommandError{Node: cs}
+	}
+	buf := cfg.strBuilder()
+	if err := cfg.CmdSubst(buf, cs); err != nil {
+		return "", err
+	}
+	return strings.TrimRight(buf.String(), "\n"), nil
+}
+
+func (cfg *Config) wordFields(wps []syntax.WordPart) ([][]fieldPart, error) {
+	fields := cfg.fieldsAlloc[:0]
+	curField := cfg.fieldAlloc[:0]
+	allowEmpty := false
+	flush := func() {
+		if len(curField) == 0 {
+			return
+		}
+		fields = append(fields, curField)
+		curField = nil
+	}
+	splitAdd := func(val string) {
+		for i, field := range strings.FieldsFunc(val, cfg.ifsRune) {
+			if i > 0 {
+				flush()
+			}
+			curField = append(curField, fieldPart{val: field})
+		}
+	}
+	for i, wp := range wps {
+		switch x := wp.(type) {
+		case *syntax.Lit:
+			s := x.Value
+			if i == 0 {
+				prefix, rest := cfg.expandUser(s)
+				curField = append(curField, fieldPart{
+					quote: quoteSingle,
+					val:   prefix,
+				})
+				s = rest
+			}
+			if strings.Contains(s, "\\") {
+				buf := cfg.strBuilder()
+				for i := 0; i < len(s); i++ {
+					b := s[i]
+					if b == '\\' {
+						i++
+						b = s[i]
+					}
+					buf.WriteByte(b)
+				}
+				s = buf.String()
+			}
+			curField = append(curField, fieldPart{val: s})
+		case *syntax.SglQuoted:
+			allowEmpty = true
+			fp := fieldPart{quote: quoteSingle, val: x.Value}
+			if x.Dollar {
+				fp.val, _, _ = Format(cfg, fp.val, nil)
+			}
+			curField = append(curField, fp)
+		case *syntax.DblQuoted:
+			allowEmpty = true
+			if len(x.Parts) == 1 {
+				pe, _ := x.Parts[0].(*syntax.ParamExp)
+				if elems := cfg.quotedElems(pe); elems != nil {
+					for i, elem := range elems {
+						if i > 0 {
+							flush()
+						}
+						curField = append(curField, fieldPart{
+							quote: quoteDouble,
+							val:   elem,
+						})
+					}
+					continue
+				}
+			}
+			wfield, err := cfg.wordField(x.Parts, quoteDouble)
+			if err != nil {
+				return nil, err
+			}
+			for _, part := range wfield {
+				part.quote = quoteDouble
+				curField = append(curField, part)
+			}
+		case *syntax.ParamExp:
+			val, err := cfg.paramExp(x)
+			if err != nil {
+				return nil, err
+			}
+			splitAdd(val)
+		case *syntax.CmdSubst:
+			val, err := cfg.cmdSubst(x)
+			if err != nil {
+				return nil, err
+			}
+			splitAdd(val)
+		case *syntax.ArithmExp:
+			n, err := Arithm(cfg, x.X)
+			if err != nil {
+				return nil, err
+			}
+			curField = append(curField, fieldPart{val: strconv.Itoa(n)})
+		default:
+			panic(fmt.Sprintf("unhandled word part: %T", x))
+		}
+	}
+	flush()
+	if allowEmpty && len(fields) == 0 {
+		fields = append(fields, curField)
+	}
+	return fields, nil
+}
+
+// quotedElems checks if a parameter expansion is exactly ${@} or ${foo[@]}
+func (cfg *Config) quotedElems(pe *syntax.ParamExp) []string {
+	if pe == nil || pe.Excl || pe.Length || pe.Width {
+		return nil
+	}
+	if pe.Param.Value == "@" {
+		return cfg.Env.Get("@").Value.([]string)
+	}
+	if nodeLit(pe.Index) != "@" {
+		return nil
+	}
+	val := cfg.Env.Get(pe.Param.Value).Value
+	if x, ok := val.([]string); ok {
+		return x
+	}
+	return nil
+}
+
+func (cfg *Config) expandUser(field string) (prefix, rest string) {
+	if len(field) == 0 || field[0] != '~' {
+		return "", field
+	}
+	name := field[1:]
+	if i := strings.Index(name, "/"); i >= 0 {
+		rest = name[i:]
+		name = name[:i]
+	}
+	if name == "" {
+		return cfg.Env.Get("HOME").String(), rest
+	}
+	if vr := cfg.Env.Get("HOME " + name); vr.IsSet() {
+		return vr.String(), rest
+	}
+
+	u, err := user.Lookup(name)
+	if err != nil {
+		return "", field
+	}
+	return u.HomeDir, rest
+}
+
+func findAllIndex(pattern, name string, n int) [][]int {
+	expr, err := syntax.TranslatePattern(pattern, true)
+	if err != nil {
+		return nil
+	}
+	rx := regexp.MustCompile(expr)
+	return rx.FindAllStringIndex(name, n)
+}
+
+// TODO: use this again to optimize globbing; see
+// https://github.com/mvdan/sh/issues/213
+func hasGlob(path string) bool {
+	magicChars := `*?[`
+	if runtime.GOOS != "windows" {
+		magicChars = `*?[\`
+	}
+	return strings.ContainsAny(path, magicChars)
+}
+
+var rxGlobStar = regexp.MustCompile(".*")
+
+// pathJoin2 is a simpler version of filepath.Join without cleaning the result,
+// since that's needed for globbing.
+func pathJoin2(elem1, elem2 string) string {
+	if elem1 == "" {
+		return elem2
+	}
+	if strings.HasSuffix(elem1, string(filepath.Separator)) {
+		return elem1 + elem2
+	}
+	return elem1 + string(filepath.Separator) + elem2
+}
+
+// pathSplit splits a file path into its elements, retaining empty ones. Before
+// splitting, slashes are replaced with filepath.Separator, so that splitting
+// Unix paths on Windows works as well.
+func pathSplit(path string) []string {
+	path = filepath.FromSlash(path)
+	return strings.Split(path, string(filepath.Separator))
+}
+
+func (cfg *Config) glob(base, pattern string) ([]string, error) {
+	parts := pathSplit(pattern)
+	matches := []string{""}
+	if filepath.IsAbs(pattern) {
+		if parts[0] == "" {
+			// unix-like
+			matches[0] = string(filepath.Separator)
+		} else {
+			// windows (for some reason it won't work without the
+			// trailing separator)
+			matches[0] = parts[0] + string(filepath.Separator)
+		}
+		parts = parts[1:]
+	}
+	for _, part := range parts {
+		switch {
+		case part == "", part == ".", part == "..":
+			var newMatches []string
+			for _, dir := range matches {
+				// TODO(mvdan): reuse the previous ReadDir call
+				if cfg.ReadDir == nil {
+					continue // no globbing
+				} else if _, err := cfg.ReadDir(filepath.Join(base, dir)); err != nil {
+					continue // not actually a dir
+				}
+				newMatches = append(newMatches, pathJoin2(dir, part))
+			}
+			matches = newMatches
+			continue
+		case part == "**" && cfg.GlobStar:
+			for i, match := range matches {
+				// "a/**" should match "a/ a/b a/b/cfg ..."; note
+				// how the zero-match case has a trailing
+				// separator.
+				matches[i] = pathJoin2(match, "")
+			}
+			// expand all the possible levels of **
+			latest := matches
+			for {
+				var newMatches []string
+				for _, dir := range latest {
+					var err error
+					newMatches, err = cfg.globDir(base, dir, rxGlobStar, newMatches)
+					if err != nil {
+						return nil, err
+					}
+				}
+				if len(newMatches) == 0 {
+					// not another level of directories to
+					// try; stop
+					break
+				}
+				matches = append(matches, newMatches...)
+				latest = newMatches
+			}
+			continue
+		}
+		expr, err := syntax.TranslatePattern(part, true)
+		if err != nil {
+			// If any glob part is not a valid pattern, don't glob.
+			return nil, nil
+		}
+		rx := regexp.MustCompile("^" + expr + "$")
+		var newMatches []string
+		for _, dir := range matches {
+			newMatches, err = cfg.globDir(base, dir, rx, newMatches)
+			if err != nil {
+				return nil, err
+			}
+		}
+		matches = newMatches
+	}
+	return matches, nil
+}
+
+func (cfg *Config) globDir(base, dir string, rx *regexp.Regexp, matches []string) ([]string, error) {
+	if cfg.ReadDir == nil {
+		// TODO(mvdan): check this at the beginning of a glob?
+		return nil, nil
+	}
+	infos, err := cfg.ReadDir(filepath.Join(base, dir))
+	if err != nil {
+		// Ignore the error, as this might be a file instead of a
+		// directory. v3 refactored globbing to only use one ReadDir
+		// call per directory instead of two, so it knows to skip this
+		// kind of path at the ReadDir call of its parent.
+		// Instead of backporting that complex rewrite into v2, just
+		// work around the edge case here. We might ignore other kinds
+		// of errors, but at least we don't fail on a correct glob.
+		return matches, nil
+	}
+	for _, info := range infos {
+		name := info.Name()
+		if !strings.HasPrefix(rx.String(), `^\.`) && name[0] == '.' {
+			continue
+		}
+		if rx.MatchString(name) {
+			matches = append(matches, pathJoin2(dir, name))
+		}
+	}
+	return matches, nil
+}
+
+//
+// The config specifies shell expansion options; nil behaves the same as an
+// empty config.
+func ReadFields(cfg *Config, s string, n int, raw bool) []string {
+	cfg = prepareConfig(cfg)
+	type pos struct {
+		start, end int
+	}
+	var fpos []pos
+
+	runes := make([]rune, 0, len(s))
+	infield := false
+	esc := false
+	for _, r := range s {
+		if infield {
+			if cfg.ifsRune(r) && (raw || !esc) {
+				fpos[len(fpos)-1].end = len(runes)
+				infield = false
+			}
+		} else {
+			if !cfg.ifsRune(r) && (raw || !esc) {
+				fpos = append(fpos, pos{start: len(runes), end: -1})
+				infield = true
+			}
+		}
+		if r == '\\' {
+			if raw || esc {
+				runes = append(runes, r)
+			}
+			esc = !esc
+			continue
+		}
+		runes = append(runes, r)
+		esc = false
+	}
+	if len(fpos) == 0 {
+		return nil
+	}
+	if infield {
+		fpos[len(fpos)-1].end = len(runes)
+	}
+
+	switch {
+	case n == 1:
+		// include heading/trailing IFSs
+		fpos[0].start, fpos[0].end = 0, len(runes)
+		fpos = fpos[:1]
+	case n != -1 && n < len(fpos):
+		// combine to max n fields
+		fpos[n-1].end = fpos[len(fpos)-1].end
+		fpos = fpos[:n]
+	}
+
+	var fields = make([]string, len(fpos))
+	for i, p := range fpos {
+		fields[i] = string(runes[p.start:p.end])
+	}
+	return fields
+}

vendor/mvdan.cc/sh/expand/param.go

@@ -1,12 +1,10 @@
 // Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
 // See LICENSE for licensing information
 
-package interp
+package expand
 
 import (
-	"context"
 	"fmt"
-	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -17,146 +15,133 @@ import (
 	"mvdan.cc/sh/syntax"
 )
 
-func anyOfLit(v interface{}, vals ...string) string {
+func nodeLit(node syntax.Node) string {
-	word, _ := v.(*syntax.Word)
+	if word, ok := node.(*syntax.Word); ok {
-	if word == nil || len(word.Parts) != 1 {
+		return word.Lit()
-		return ""
-	}
-	lit, ok := word.Parts[0].(*syntax.Lit)
-	if !ok {
-		return ""
-	}
-	for _, val := range vals {
-		if lit.Value == val {
-			return val
-		}
 	}
 	return ""
 }
 
-// quotedElems checks if a parameter expansion is exactly ${@} or ${foo[@]}
+type UnsetParameterError struct {
-func (r *Runner) quotedElems(pe *syntax.ParamExp) []string {
+	Node    *syntax.ParamExp
-	if pe == nil || pe.Excl || pe.Length || pe.Width {
+	Message string
-		return nil
-	}
-	if pe.Param.Value == "@" {
-		return r.Params
-	}
-	if anyOfLit(pe.Index, "@") == "" {
-		return nil
-	}
-	val, _ := r.lookupVar(pe.Param.Value)
-	if x, ok := val.Value.(IndexArray); ok {
-		return x
-	}
-	return nil
 }
 
-func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
+func (u UnsetParameterError) Error() string {
+	return u.Message
+}
+
+func (cfg *Config) paramExp(pe *syntax.ParamExp) (string, error) {
+	oldParam := cfg.curParam
+	cfg.curParam = pe
+	defer func() { cfg.curParam = oldParam }()
+
 	name := pe.Param.Value
-	var vr Variable
-	set := false
 	index := pe.Index
 	switch name {
-	case "#":
-		vr.Value = StringVal(strconv.Itoa(len(r.Params)))
 	case "@", "*":
-		vr.Value = IndexArray(r.Params)
 		index = &syntax.Word{Parts: []syntax.WordPart{
 			&syntax.Lit{Value: name},
 		}}
-	case "?":
+	}
-		vr.Value = StringVal(strconv.Itoa(r.exit))
+	var vr Variable
-	case "$":
+	switch name {
-		vr.Value = StringVal(strconv.Itoa(os.Getpid()))
-	case "PPID":
-		vr.Value = StringVal(strconv.Itoa(os.Getppid()))
 	case "LINENO":
-		line := uint64(pe.Pos().Line())
+		// This is the only parameter expansion that the environment
-		vr.Value = StringVal(strconv.FormatUint(line, 10))
+		// interface cannot satisfy.
-	case "DIRSTACK":
+		line := uint64(cfg.curParam.Pos().Line())
-		vr.Value = IndexArray(r.dirStack)
+		vr.Value = strconv.FormatUint(line, 10)
 	default:
-		if n, err := strconv.Atoi(name); err == nil {
+		vr = cfg.Env.Get(name)
-			if i := n - 1; i < len(r.Params) {
-				vr.Value, set = StringVal(r.Params[i]), true
-			}
-		} else {
-			vr, set = r.lookupVar(name)
-		}
 	}
-	str := r.varStr(vr, 0)
+	orig := vr
-	if index != nil {
+	_, vr = vr.Resolve(cfg.Env)
-		str = r.varInd(ctx, vr, index, 0)
+	str, err := cfg.varInd(vr, index)
+	if err != nil {
+		return "", err
 	}
-	slicePos := func(expr syntax.ArithmExpr) int {
+	slicePos := func(n int) int {
-		p := r.arithm(ctx, expr)
+		if n < 0 {
-		if p < 0 {
+			n = len(str) + n
-			p = len(str) + p
+			if n < 0 {
-			if p < 0 {
+				n = len(str)
-				p = len(str)
 			}
-		} else if p > len(str) {
+		} else if n > len(str) {
-			p = len(str)
+			n = len(str)
 		}
-		return p
+		return n
 	}
 	elems := []string{str}
-	if anyOfLit(index, "@", "*") != "" {
+	switch nodeLit(index) {
+	case "@", "*":
 		switch x := vr.Value.(type) {
 		case nil:
 			elems = nil
-		case IndexArray:
+		case []string:
 			elems = x
 		}
 	}
 	switch {
 	case pe.Length:
 		n := len(elems)
-		if anyOfLit(index, "@", "*") == "" {
+		switch nodeLit(index) {
+		case "@", "*":
+		default:
 			n = utf8.RuneCountInString(str)
 		}
 		str = strconv.Itoa(n)
 	case pe.Excl:
 		var strs []string
 		if pe.Names != 0 {
-			strs = r.namesByPrefix(pe.Param.Value)
+			strs = cfg.namesByPrefix(pe.Param.Value)
-		} else if vr.NameRef {
+		} else if orig.NameRef {
-			strs = append(strs, string(vr.Value.(StringVal)))
+			strs = append(strs, orig.Value.(string))
-		} else if x, ok := vr.Value.(IndexArray); ok {
+		} else if x, ok := vr.Value.([]string); ok {
 			for i, e := range x {
 				if e != "" {
 					strs = append(strs, strconv.Itoa(i))
 				}
 			}
-		} else if x, ok := vr.Value.(AssocArray); ok {
+		} else if x, ok := vr.Value.(map[string]string); ok {
 			for k := range x {
 				strs = append(strs, k)
 			}
 		} else if str != "" {
-			vr, _ = r.lookupVar(str)
+			vr = cfg.Env.Get(str)
-			strs = append(strs, r.varStr(vr, 0))
+			strs = append(strs, vr.String())
 		}
 		sort.Strings(strs)
 		str = strings.Join(strs, " ")
 	case pe.Slice != nil:
 		if pe.Slice.Offset != nil {
-			offset := slicePos(pe.Slice.Offset)
+			n, err := Arithm(cfg, pe.Slice.Offset)
-			str = str[offset:]
+			if err != nil {
+				return "", err
+			}
+			str = str[slicePos(n):]
 		}
 		if pe.Slice.Length != nil {
-			length := slicePos(pe.Slice.Length)
+			n, err := Arithm(cfg, pe.Slice.Length)
-			str = str[:length]
+			if err != nil {
+				return "", err
+			}
+			str = str[:slicePos(n)]
 		}
 	case pe.Repl != nil:
-		orig := r.lonePattern(ctx, pe.Repl.Orig)
+		orig, err := Pattern(cfg, pe.Repl.Orig)
-		with := r.loneWord(ctx, pe.Repl.With)
+		if err != nil {
+			return "", err
+		}
+		with, err := Literal(cfg, pe.Repl.With)
+		if err != nil {
+			return "", err
+		}
 		n := 1
 		if pe.Repl.All {
 			n = -1
 		}
 		locs := findAllIndex(orig, str, n)
-		buf := r.strBuilder()
+		buf := cfg.strBuilder()
 		last := 0
 		for _, loc := range locs {
 			buf.WriteString(str[last:loc[0]])
@@ -166,7 +151,10 @@ func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
 		buf.WriteString(str[last:])
 		str = buf.String()
 	case pe.Exp != nil:
-		arg := r.loneWord(ctx, pe.Exp.Word)
+		arg, err := Literal(cfg, pe.Exp.Word)
+		if err != nil {
+			return "", err
+		}
 		switch op := pe.Exp.Op; op {
 		case syntax.SubstColPlus:
 			if str == "" {
@@ -174,11 +162,11 @@ func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
 			}
 			fallthrough
 		case syntax.SubstPlus:
-			if set {
+			if vr.IsSet() {
 				str = arg
 			}
 		case syntax.SubstMinus:
-			if set {
+			if vr.IsSet() {
 				break
 			}
 			fallthrough
@@ -187,24 +175,25 @@ func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
 				str = arg
 			}
 		case syntax.SubstQuest:
-			if set {
+			if vr.IsSet() {
 				break
 			}
 			fallthrough
 		case syntax.SubstColQuest:
 			if str == "" {
-				r.errf("%s\n", arg)
+				return "", UnsetParameterError{
-				r.exit = 1
+					Node:    pe,
-				r.setErr(ShellExitStatus(r.exit))
+					Message: arg,
+				}
 			}
 		case syntax.SubstAssgn:
-			if set {
+			if vr.IsSet() {
 				break
 			}
 			fallthrough
 		case syntax.SubstColAssgn:
 			if str == "" {
-				r.setVarString(ctx, name, arg)
+				cfg.envSet(name, arg)
 				str = arg
 			}
 		case syntax.RemSmallPrefix, syntax.RemLargePrefix,
@@ -229,7 +218,7 @@ func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
 			// empty string means '?'; nothing to do there
 			expr, err := syntax.TranslatePattern(arg, false)
 			if err != nil {
-				return str
+				return str, nil
 			}
 			rx := regexp.MustCompile(expr)
 
@@ -266,7 +255,7 @@ func (r *Runner) paramExp(ctx context.Context, pe *syntax.ParamExp) string {
 			}
 		}
 	}
-	return str
+	return str, nil
 }
 
 func removePattern(str, pattern string, fromEnd, greedy bool) string {
@@ -293,3 +282,67 @@ func removePattern(str, pattern string, fromEnd, greedy bool) string {
 	}
 	return str
 }
+
+func (cfg *Config) varInd(vr Variable, idx syntax.ArithmExpr) (string, error) {
+	if idx == nil {
+		return vr.String(), nil
+	}
+	switch x := vr.Value.(type) {
+	case string:
+		n, err := Arithm(cfg, idx)
+		if err != nil {
+			return "", err
+		}
+		if n == 0 {
+			return x, nil
+		}
+	case []string:
+		switch nodeLit(idx) {
+		case "@":
+			return strings.Join(x, " "), nil
+		case "*":
+			return cfg.ifsJoin(x), nil
+		}
+		i, err := Arithm(cfg, idx)
+		if err != nil {
+			return "", err
+		}
+		if len(x) > 0 {
+			return x[i], nil
+		}
+	case map[string]string:
+		switch lit := nodeLit(idx); lit {
+		case "@", "*":
+			var strs []string
+			keys := make([]string, 0, len(x))
+			for k := range x {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+			for _, k := range keys {
+				strs = append(strs, x[k])
+			}
+			if lit == "*" {
+				return cfg.ifsJoin(strs), nil
+			}
+			return strings.Join(strs, " "), nil
+		}
+		val, err := Literal(cfg, idx.(*syntax.Word))
+		if err != nil {
+			return "", err
+		}
+		return x[val], nil
+	}
+	return "", nil
+}
+
+func (cfg *Config) namesByPrefix(prefix string) []string {
+	var names []string
+	cfg.Env.Each(func(name string, vr Variable) bool {
+		if strings.HasPrefix(name, prefix) {
+			names = append(names, name)
+		}
+		return true
+	})
+	return names
+}

vendor/mvdan.cc/sh/interp/builtin.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/syntax"
 )
 
@@ -29,6 +30,20 @@ func isBuiltin(name string) bool {
 	return false
 }
 
+func oneIf(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// atoi is just a shorthand for strconv.Atoi that ignores the error,
+// just like shells do.
+func atoi(s string) int {
+	n, _ := strconv.Atoi(s)
+	return n
+}
+
 func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, args []string) int {
 	switch name {
 	case "true", ":":
@@ -55,6 +70,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			r.errf("set: %v\n", err)
 			return 2
 		}
+		r.updateExpandOpts()
 	case "shift":
 		n := 1
 		switch len(args) {
@@ -91,7 +107,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 		}
 
 		for _, arg := range args {
-			if _, ok := r.lookupVar(arg); ok && vars {
+			if vr := r.lookupVar(arg); vr.IsSet() && vars {
 				r.delVar(arg)
 				continue
 			}
@@ -100,14 +116,14 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			}
 		}
 	case "echo":
-		newline, expand := true, false
+		newline, doExpand := true, false
 	echoOpts:
 		for len(args) > 0 {
 			switch args[0] {
 			case "-n":
 				newline = false
 			case "-e":
-				expand = true
+				doExpand = true
 			case "-E": // default
 			default:
 				break echoOpts
@@ -118,8 +134,8 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			if i > 0 {
 				r.out(" ")
 			}
-			if expand {
+			if doExpand {
-				_, arg, _ = r.expandFormat(arg, nil)
+				arg, _, _ = expand.Format(r.ecfg, arg, nil)
 			}
 			r.out(arg)
 		}
@@ -133,7 +149,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 		}
 		format, args := args[0], args[1:]
 		for {
-			n, s, err := r.expandFormat(format, args)
+			s, n, err := expand.Format(r.ecfg, format, args)
 			if err != nil {
 				r.errf("%v\n", err)
 				return 1
@@ -144,49 +160,35 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 				break
 			}
 		}
-	case "break":
+	case "break", "continue":
 		if !r.inLoop {
-			r.errf("break is only useful in a loop")
+			r.errf("%s is only useful in a loop", name)
 			break
 		}
+		enclosing := &r.breakEnclosing
+		if name == "continue" {
+			enclosing = &r.contnEnclosing
+		}
 		switch len(args) {
 		case 0:
-			r.breakEnclosing = 1
+			*enclosing = 1
 		case 1:
 			if n, err := strconv.Atoi(args[0]); err == nil {
-				r.breakEnclosing = n
+				*enclosing = n
 				break
 			}
 			fallthrough
 		default:
-			r.errf("usage: break [n]\n")
+			r.errf("usage: %s [n]\n", name)
-			return 2
-		}
-	case "continue":
-		if !r.inLoop {
-			r.errf("continue is only useful in a loop")
-			break
-		}
-		switch len(args) {
-		case 0:
-			r.contnEnclosing = 1
-		case 1:
-			if n, err := strconv.Atoi(args[0]); err == nil {
-				r.contnEnclosing = n
-				break
-			}
-			fallthrough
-		default:
-			r.errf("usage: continue [n]\n")
 			return 2
 		}
 	case "pwd":
-		r.outf("%s\n", r.getVar("PWD"))
+		r.outf("%s\n", r.envGet("PWD"))
 	case "cd":
 		var path string
 		switch len(args) {
 		case 0:
-			path = r.getVar("HOME")
+			path = r.envGet("HOME")
 		case 1:
 			path = args[0]
 		default:
@@ -462,13 +464,13 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			args = append(args, "REPLY")
 		}
 
-		values := r.ifsFields(string(line), len(args), raw)
+		values := expand.ReadFields(r.ecfg, string(line), len(args), raw)
 		for i, name := range args {
 			val := ""
 			if i < len(values) {
 				val = values[i]
 			}
-			r.setVar(ctx, name, nil, Variable{Value: StringVal(val)})
+			r.setVar(name, nil, expand.Variable{Value: val})
 		}
 
 		return 0
@@ -478,7 +480,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			r.errf("getopts: usage: getopts optstring name [arg]\n")
 			return 2
 		}
-		optind, _ := strconv.Atoi(r.getVar("OPTIND"))
+		optind, _ := strconv.Atoi(r.envGet("OPTIND"))
 		if optind-1 != r.optState.argidx {
 			if optind < 1 {
 				optind = 1
@@ -499,7 +501,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 
 		opt, optarg, done := r.optState.Next(optstr, args)
 
-		r.setVarString(ctx, name, string(opt))
+		r.setVarString(name, string(opt))
 		r.delVar("OPTARG")
 		switch {
 		case opt == '?' && diagnostics && !done:
@@ -508,11 +510,11 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 			r.errf("getopts: option requires an argument -- %q\n", optarg)
 		default:
 			if optarg != "" {
-				r.setVarString(ctx, "OPTARG", optarg)
+				r.setVarString("OPTARG", optarg)
 			}
 		}
 		if optind-1 != r.optState.argidx {
-			r.setVarString(ctx, "OPTIND", strconv.FormatInt(int64(r.optState.argidx+1), 10))
+			r.setVarString("OPTIND", strconv.FormatInt(int64(r.optState.argidx+1), 10))
 		}
 
 		return oneIf(done)
@@ -559,6 +561,7 @@ func (r *Runner) builtinCode(ctx context.Context, pos syntax.Pos, name string, a
 				r.printOptLine(arg, *opt)
 			}
 		}
+		r.updateExpandOpts()
 
 	default:
 		// "trap", "umask", "alias", "unalias", "fg", "bg",
@@ -575,62 +578,6 @@ func (r *Runner) printOptLine(name string, enabled bool) {
 	r.outf("%s\t%s\n", name, status)
 }
 
-func (r *Runner) ifsFields(s string, n int, raw bool) []string {
-	type pos struct {
-		start, end int
-	}
-	var fpos []pos
-
-	runes := make([]rune, 0, len(s))
-	infield := false
-	esc := false
-	for _, c := range s {
-		if infield {
-			if r.ifsRune(c) && (raw || !esc) {
-				fpos[len(fpos)-1].end = len(runes)
-				infield = false
-			}
-		} else {
-			if !r.ifsRune(c) && (raw || !esc) {
-				fpos = append(fpos, pos{start: len(runes), end: -1})
-				infield = true
-			}
-		}
-		if c == '\\' {
-			if raw || esc {
-				runes = append(runes, c)
-			}
-			esc = !esc
-			continue
-		}
-		runes = append(runes, c)
-		esc = false
-	}
-	if len(fpos) == 0 {
-		return nil
-	}
-	if infield {
-		fpos[len(fpos)-1].end = len(runes)
-	}
-
-	switch {
-	case n == 1:
-		// include heading/trailing IFSs
-		fpos[0].start, fpos[0].end = 0, len(runes)
-		fpos = fpos[:1]
-	case n != -1 && n < len(fpos):
-		// combine to max n fields
-		fpos[n-1].end = fpos[len(fpos)-1].end
-		fpos = fpos[:n]
-	}
-
-	var fields = make([]string, len(fpos))
-	for i, p := range fpos {
-		fields[i] = string(runes[p.start:p.end])
-	}
-	return fields
-}
-
 func (r *Runner) readLine(raw bool) ([]byte, error) {
 	var line []byte
 	esc := false
@@ -675,7 +622,7 @@ func (r *Runner) changeDir(path string) int {
 	}
 	r.Dir = path
 	r.Vars["OLDPWD"] = r.Vars["PWD"]
-	r.Vars["PWD"] = Variable{Value: StringVal(path)}
+	r.Vars["PWD"] = expand.Variable{Value: path}
 	return 0
 }
 

vendor/mvdan.cc/sh/interp/doc.go

@@ -4,7 +4,4 @@
 // Package interp implements an interpreter that executes shell
 // programs. It aims to support POSIX, but its support is not complete
 // yet. It also supports some Bash features.
-//
-// This package is a work in progress and EXPERIMENTAL; its API is not
-// subject to the 1.x backwards compatibility guarantee.
 package interp

vendor/mvdan.cc/sh/interp/expand.go

@@ -1,508 +0,0 @@
-// Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
-// See LICENSE for licensing information
-
-package interp
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/user"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"sort"
-	"strconv"
-	"strings"
-
-	"mvdan.cc/sh/syntax"
-)
-
-func (r *Runner) expandFormat(format string, args []string) (int, string, error) {
-	buf := r.strBuilder()
-	esc := false
-	var fmts []rune
-	initialArgs := len(args)
-
-	for _, c := range format {
-		switch {
-		case esc:
-			esc = false
-			switch c {
-			case 'n':
-				buf.WriteRune('\n')
-			case 'r':
-				buf.WriteRune('\r')
-			case 't':
-				buf.WriteRune('\t')
-			case '\\':
-				buf.WriteRune('\\')
-			default:
-				buf.WriteRune('\\')
-				buf.WriteRune(c)
-			}
-
-		case len(fmts) > 0:
-			switch c {
-			case '%':
-				buf.WriteByte('%')
-				fmts = nil
-			case 'c':
-				var b byte
-				if len(args) > 0 {
-					arg := ""
-					arg, args = args[0], args[1:]
-					if len(arg) > 0 {
-						b = arg[0]
-					}
-				}
-				buf.WriteByte(b)
-				fmts = nil
-			case '+', '-', ' ':
-				if len(fmts) > 1 {
-					return 0, "", fmt.Errorf("invalid format char: %c", c)
-				}
-				fmts = append(fmts, c)
-			case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-				fmts = append(fmts, c)
-			case 's', 'd', 'i', 'u', 'o', 'x':
-				arg := ""
-				if len(args) > 0 {
-					arg, args = args[0], args[1:]
-				}
-				var farg interface{} = arg
-				if c != 's' {
-					n, _ := strconv.ParseInt(arg, 0, 0)
-					if c == 'i' || c == 'd' {
-						farg = int(n)
-					} else {
-						farg = uint(n)
-					}
-					if c == 'i' || c == 'u' {
-						c = 'd'
-					}
-				}
-				fmts = append(fmts, c)
-				fmt.Fprintf(buf, string(fmts), farg)
-				fmts = nil
-			default:
-				return 0, "", fmt.Errorf("invalid format char: %c", c)
-			}
-		case c == '\\':
-			esc = true
-		case args != nil && c == '%':
-			// if args == nil, we are not doing format
-			// arguments
-			fmts = []rune{c}
-		default:
-			buf.WriteRune(c)
-		}
-	}
-	if len(fmts) > 0 {
-		return 0, "", fmt.Errorf("missing format char")
-	}
-	return initialArgs - len(args), buf.String(), nil
-}
-
-func (r *Runner) fieldJoin(parts []fieldPart) string {
-	switch len(parts) {
-	case 0:
-		return ""
-	case 1: // short-cut without a string copy
-		return parts[0].val
-	}
-	buf := r.strBuilder()
-	for _, part := range parts {
-		buf.WriteString(part.val)
-	}
-	return buf.String()
-}
-
-func (r *Runner) escapedGlobField(parts []fieldPart) (escaped string, glob bool) {
-	buf := r.strBuilder()
-	for _, part := range parts {
-		quoted := syntax.QuotePattern(part.val)
-		if quoted != part.val {
-			if part.quote > quoteNone {
-				buf.WriteString(quoted)
-			} else {
-				buf.WriteString(part.val)
-				glob = true
-			}
-		}
-	}
-	if glob { // only copy the string if it will be used
-		escaped = buf.String()
-	}
-	return escaped, glob
-}
-
-func (r *Runner) Fields(ctx context.Context, words ...*syntax.Word) ([]string, error) {
-	if !r.didReset {
-		r.Reset()
-	}
-	return r.fields(ctx, words...), r.err
-}
-
-func (r *Runner) fields(ctx context.Context, words ...*syntax.Word) []string {
-	fields := make([]string, 0, len(words))
-	baseDir := syntax.QuotePattern(r.Dir)
-	for _, word := range words {
-		for _, expWord := range syntax.ExpandBraces(word) {
-			for _, field := range r.wordFields(ctx, expWord.Parts) {
-				path, doGlob := r.escapedGlobField(field)
-				var matches []string
-				abs := filepath.IsAbs(path)
-				if doGlob && !r.opts[optNoGlob] {
-					if !abs {
-						path = filepath.Join(baseDir, path)
-					}
-					matches = glob(path, r.opts[optGlobStar])
-				}
-				if len(matches) == 0 {
-					fields = append(fields, r.fieldJoin(field))
-					continue
-				}
-				for _, match := range matches {
-					if !abs {
-						endSeparator := strings.HasSuffix(match, string(filepath.Separator))
-						match, _ = filepath.Rel(r.Dir, match)
-						if endSeparator {
-							match += string(filepath.Separator)
-						}
-					}
-					fields = append(fields, match)
-				}
-			}
-		}
-	}
-	return fields
-}
-
-func (r *Runner) loneWord(ctx context.Context, word *syntax.Word) string {
-	if word == nil {
-		return ""
-	}
-	field := r.wordField(ctx, word.Parts, quoteDouble)
-	return r.fieldJoin(field)
-}
-
-func (r *Runner) lonePattern(ctx context.Context, word *syntax.Word) string {
-	field := r.wordField(ctx, word.Parts, quoteSingle)
-	buf := r.strBuilder()
-	for _, part := range field {
-		if part.quote > quoteNone {
-			buf.WriteString(syntax.QuotePattern(part.val))
-		} else {
-			buf.WriteString(part.val)
-		}
-	}
-	return buf.String()
-}
-
-func (r *Runner) expandAssigns(ctx context.Context, as *syntax.Assign) []*syntax.Assign {
-	// Convert "declare $x" into "declare value".
-	// Don't use syntax.Parser here, as we only want the basic
-	// splitting by '='.
-	if as.Name != nil {
-		return []*syntax.Assign{as} // nothing to do
-	}
-	var asgns []*syntax.Assign
-	for _, field := range r.fields(ctx, as.Value) {
-		as := &syntax.Assign{}
-		parts := strings.SplitN(field, "=", 2)
-		as.Name = &syntax.Lit{Value: parts[0]}
-		if len(parts) == 1 {
-			as.Naked = true
-		} else {
-			as.Value = &syntax.Word{Parts: []syntax.WordPart{
-				&syntax.Lit{Value: parts[1]},
-			}}
-		}
-		asgns = append(asgns, as)
-	}
-	return asgns
-}
-
-type fieldPart struct {
-	val   string
-	quote quoteLevel
-}
-
-type quoteLevel uint
-
-const (
-	quoteNone quoteLevel = iota
-	quoteDouble
-	quoteSingle
-)
-
-func (r *Runner) wordField(ctx context.Context, wps []syntax.WordPart, ql quoteLevel) []fieldPart {
-	var field []fieldPart
-	for i, wp := range wps {
-		switch x := wp.(type) {
-		case *syntax.Lit:
-			s := x.Value
-			if i == 0 {
-				s = r.expandUser(s)
-			}
-			if ql == quoteDouble && strings.Contains(s, "\\") {
-				buf := r.strBuilder()
-				for i := 0; i < len(s); i++ {
-					b := s[i]
-					if b == '\\' && i+1 < len(s) {
-						switch s[i+1] {
-						case '\n': // remove \\\n
-							i++
-							continue
-						case '"', '\\', '$', '`': // special chars
-							continue
-						}
-					}
-					buf.WriteByte(b)
-				}
-				s = buf.String()
-			}
-			field = append(field, fieldPart{val: s})
-		case *syntax.SglQuoted:
-			fp := fieldPart{quote: quoteSingle, val: x.Value}
-			if x.Dollar {
-				_, fp.val, _ = r.expandFormat(fp.val, nil)
-			}
-			field = append(field, fp)
-		case *syntax.DblQuoted:
-			for _, part := range r.wordField(ctx, x.Parts, quoteDouble) {
-				part.quote = quoteDouble
-				field = append(field, part)
-			}
-		case *syntax.ParamExp:
-			field = append(field, fieldPart{val: r.paramExp(ctx, x)})
-		case *syntax.CmdSubst:
-			field = append(field, fieldPart{val: r.cmdSubst(ctx, x)})
-		case *syntax.ArithmExp:
-			field = append(field, fieldPart{
-				val: strconv.Itoa(r.arithm(ctx, x.X)),
-			})
-		default:
-			panic(fmt.Sprintf("unhandled word part: %T", x))
-		}
-	}
-	return field
-}
-
-func (r *Runner) cmdSubst(ctx context.Context, cs *syntax.CmdSubst) string {
-	r2 := r.sub()
-	buf := r.strBuilder()
-	r2.Stdout = buf
-	r2.stmts(ctx, cs.StmtList)
-	r.setErr(r2.err)
-	return strings.TrimRight(buf.String(), "\n")
-}
-
-func (r *Runner) wordFields(ctx context.Context, wps []syntax.WordPart) [][]fieldPart {
-	fields := r.fieldsAlloc[:0]
-	curField := r.fieldAlloc[:0]
-	allowEmpty := false
-	flush := func() {
-		if len(curField) == 0 {
-			return
-		}
-		fields = append(fields, curField)
-		curField = nil
-	}
-	splitAdd := func(val string) {
-		for i, field := range strings.FieldsFunc(val, r.ifsRune) {
-			if i > 0 {
-				flush()
-			}
-			curField = append(curField, fieldPart{val: field})
-		}
-	}
-	for i, wp := range wps {
-		switch x := wp.(type) {
-		case *syntax.Lit:
-			s := x.Value
-			if i == 0 {
-				s = r.expandUser(s)
-			}
-			if strings.Contains(s, "\\") {
-				buf := r.strBuilder()
-				for i := 0; i < len(s); i++ {
-					b := s[i]
-					if b == '\\' {
-						i++
-						b = s[i]
-					}
-					buf.WriteByte(b)
-				}
-				s = buf.String()
-			}
-			curField = append(curField, fieldPart{val: s})
-		case *syntax.SglQuoted:
-			allowEmpty = true
-			fp := fieldPart{quote: quoteSingle, val: x.Value}
-			if x.Dollar {
-				_, fp.val, _ = r.expandFormat(fp.val, nil)
-			}
-			curField = append(curField, fp)
-		case *syntax.DblQuoted:
-			allowEmpty = true
-			if len(x.Parts) == 1 {
-				pe, _ := x.Parts[0].(*syntax.ParamExp)
-				if elems := r.quotedElems(pe); elems != nil {
-					for i, elem := range elems {
-						if i > 0 {
-							flush()
-						}
-						curField = append(curField, fieldPart{
-							quote: quoteDouble,
-							val:   elem,
-						})
-					}
-					continue
-				}
-			}
-			for _, part := range r.wordField(ctx, x.Parts, quoteDouble) {
-				part.quote = quoteDouble
-				curField = append(curField, part)
-			}
-		case *syntax.ParamExp:
-			splitAdd(r.paramExp(ctx, x))
-		case *syntax.CmdSubst:
-			splitAdd(r.cmdSubst(ctx, x))
-		case *syntax.ArithmExp:
-			curField = append(curField, fieldPart{
-				val: strconv.Itoa(r.arithm(ctx, x.X)),
-			})
-		default:
-			panic(fmt.Sprintf("unhandled word part: %T", x))
-		}
-	}
-	flush()
-	if allowEmpty && len(fields) == 0 {
-		fields = append(fields, curField)
-	}
-	return fields
-}
-
-func (r *Runner) expandUser(field string) string {
-	if len(field) == 0 || field[0] != '~' {
-		return field
-	}
-	name := field[1:]
-	rest := ""
-	if i := strings.Index(name, "/"); i >= 0 {
-		rest = name[i:]
-		name = name[:i]
-	}
-	if name == "" {
-		return r.getVar("HOME") + rest
-	}
-	u, err := user.Lookup(name)
-	if err != nil {
-		return field
-	}
-	return u.HomeDir + rest
-}
-
-func match(pattern, name string) bool {
-	expr, err := syntax.TranslatePattern(pattern, true)
-	if err != nil {
-		return false
-	}
-	rx := regexp.MustCompile("^" + expr + "$")
-	return rx.MatchString(name)
-}
-
-func findAllIndex(pattern, name string, n int) [][]int {
-	expr, err := syntax.TranslatePattern(pattern, true)
-	if err != nil {
-		return nil
-	}
-	rx := regexp.MustCompile(expr)
-	return rx.FindAllStringIndex(name, n)
-}
-
-func hasGlob(path string) bool {
-	magicChars := `*?[`
-	if runtime.GOOS != "windows" {
-		magicChars = `*?[\`
-	}
-	return strings.ContainsAny(path, magicChars)
-}
-
-var rxGlobStar = regexp.MustCompile(".*")
-
-func glob(pattern string, globStar bool) []string {
-	parts := strings.Split(pattern, string(filepath.Separator))
-	matches := []string{"."}
-	if filepath.IsAbs(pattern) {
-		if parts[0] == "" {
-			// unix-like
-			matches[0] = string(filepath.Separator)
-		} else {
-			// windows (for some reason it won't work without the
-			// trailing separator)
-			matches[0] = parts[0] + string(filepath.Separator)
-		}
-		parts = parts[1:]
-	}
-	for _, part := range parts {
-		if part == "**" && globStar {
-			for i := range matches {
-				// "a/**" should match "a/ a/b a/b/c ..."; note
-				// how the zero-match case has a trailing
-				// separator.
-				matches[i] += string(filepath.Separator)
-			}
-			// expand all the possible levels of **
-			latest := matches
-			for {
-				var newMatches []string
-				for _, dir := range latest {
-					newMatches = globDir(dir, rxGlobStar, newMatches)
-				}
-				if len(newMatches) == 0 {
-					// not another level of directories to
-					// try; stop
-					break
-				}
-				matches = append(matches, newMatches...)
-				latest = newMatches
-			}
-			continue
-		}
-		expr, err := syntax.TranslatePattern(part, true)
-		if err != nil {
-			return nil
-		}
-		rx := regexp.MustCompile("^" + expr + "$")
-		var newMatches []string
-		for _, dir := range matches {
-			newMatches = globDir(dir, rx, newMatches)
-		}
-		matches = newMatches
-	}
-	return matches
-}
-
-func globDir(dir string, rx *regexp.Regexp, matches []string) []string {
-	d, err := os.Open(dir)
-	if err != nil {
-		return nil
-	}
-	defer d.Close()
-
-	names, _ := d.Readdirnames(-1)
-	sort.Strings(names)
-
-	for _, name := range names {
-		if !strings.HasPrefix(rx.String(), `^\.`) && name[0] == '.' {
-			continue
-		}
-		if rx.MatchString(name) {
-			matches = append(matches, filepath.Join(dir, name))
-		}
-	}
-	return matches
-}

vendor/mvdan.cc/sh/interp/interp.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -20,6 +21,7 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/syntax"
 )
 
@@ -46,10 +48,10 @@ func New(opts ...func(*Runner) error) (*Runner, error) {
 		}
 	}
 	if r.Exec == nil {
-		Module(nil)(r)
+		Module(ModuleExec(nil))(r)
 	}
 	if r.Open == nil {
-		Module(nil)(r)
+		Module(ModuleOpen(nil))(r)
 	}
 	if r.Stdout == nil || r.Stderr == nil {
 		StdIO(r.Stdin, r.Stdout, r.Stderr)(r)
@@ -57,12 +59,127 @@ func New(opts ...func(*Runner) error) (*Runner, error) {
 	return r, nil
 }
 
-// Env sets the interpreter's environment. If nil, the current process's
+func (r *Runner) fillExpandConfig(ctx context.Context) {
-// environment is used.
+	r.ectx = ctx
-func Env(env Environ) func(*Runner) error {
+	r.ecfg = &expand.Config{
+		Env: expandEnv{r},
+		CmdSubst: func(w io.Writer, cs *syntax.CmdSubst) error {
+			switch len(cs.Stmts) {
+			case 0: // nothing to do
+				return nil
+			case 1: // $(<file)
+				word := catShortcutArg(cs.Stmts[0])
+				if word == nil {
+					break
+				}
+				path := r.literal(word)
+				f, err := r.open(ctx, r.relPath(path), os.O_RDONLY, 0, true)
+				if err != nil {
+					return err
+				}
+				_, err = io.Copy(w, f)
+				return err
+			}
+			r2 := r.sub()
+			r2.Stdout = w
+			r2.stmts(ctx, cs.StmtList)
+			return r2.err
+		},
+		ReadDir: ioutil.ReadDir,
+	}
+	r.updateExpandOpts()
+}
+
+// catShortcutArg checks if a statement is of the form "$(<file)". The redirect
+// word is returned if there's a match, and nil otherwise.
+func catShortcutArg(stmt *syntax.Stmt) *syntax.Word {
+	if stmt.Cmd != nil || stmt.Negated || stmt.Background || stmt.Coprocess {
+		return nil
+	}
+	if len(stmt.Redirs) != 1 {
+		return nil
+	}
+	redir := stmt.Redirs[0]
+	if redir.Op != syntax.RdrIn {
+		return nil
+	}
+	return redir.Word
+}
+
+func (r *Runner) updateExpandOpts() {
+	r.ecfg.NoGlob = r.opts[optNoGlob]
+	r.ecfg.GlobStar = r.opts[optGlobStar]
+}
+
+func (r *Runner) expandErr(err error) {
+	switch err := err.(type) {
+	case nil:
+	case expand.UnsetParameterError:
+		r.errf("%s\n", err.Message)
+		r.exit = 1
+		r.setErr(ShellExitStatus(r.exit))
+	default:
+		r.setErr(err)
+		r.exit = 1
+	}
+}
+
+func (r *Runner) arithm(expr syntax.ArithmExpr) int {
+	n, err := expand.Arithm(r.ecfg, expr)
+	r.expandErr(err)
+	return n
+}
+
+func (r *Runner) fields(words ...*syntax.Word) []string {
+	strs, err := expand.Fields(r.ecfg, words...)
+	r.expandErr(err)
+	return strs
+}
+
+func (r *Runner) literal(word *syntax.Word) string {
+	str, err := expand.Literal(r.ecfg, word)
+	r.expandErr(err)
+	return str
+}
+
+func (r *Runner) document(word *syntax.Word) string {
+	str, err := expand.Document(r.ecfg, word)
+	r.expandErr(err)
+	return str
+}
+
+func (r *Runner) pattern(word *syntax.Word) string {
+	str, err := expand.Pattern(r.ecfg, word)
+	r.expandErr(err)
+	return str
+}
+
+// expandEnv exposes Runner's variables to the expand package.
+type expandEnv struct {
+	r *Runner
+}
+
+func (e expandEnv) Get(name string) expand.Variable {
+	return e.r.lookupVar(name)
+}
+func (e expandEnv) Set(name string, vr expand.Variable) {
+	e.r.setVarInternal(name, vr)
+}
+func (e expandEnv) Each(fn func(name string, vr expand.Variable) bool) {
+	e.r.Env.Each(fn)
+	for name, vr := range e.r.Vars {
+		if !fn(name, vr) {
+			return
+		}
+	}
+}
+
+// Env sets the interpreter's environment. If nil, a copy of the current
+// process's environment is used.
+func Env(env expand.Environ) func(*Runner) error {
 	return func(r *Runner) error {
 		if env == nil {
-			env, _ = EnvFromList(os.Environ())
+			env = expand.ListEnviron(os.Environ()...)
 		}
 		r.Env = env
 		return nil
@@ -203,7 +320,7 @@ func StdIO(in io.Reader, out, err io.Writer) func(*Runner) error {
 type Runner struct {
 	// Env specifies the environment of the interpreter, which must be
 	// non-nil.
-	Env Environ
+	Env expand.Environ
 
 	// Dir specifies the working directory of the command, which must be an
 	// absolute path.
@@ -223,12 +340,15 @@ type Runner struct {
 	Stdout io.Writer
 	Stderr io.Writer
 
-	// Separate maps, note that bash allows a name to be both a var
+	// Separate maps - note that bash allows a name to be both a var and a
-	// and a func simultaneously
+	// func simultaneously
-	// TODO: merge into Env?
+
-	Vars  map[string]Variable
+	Vars  map[string]expand.Variable
 	Funcs map[string]*syntax.Stmt
 
+	ecfg *expand.Config
+	ectx context.Context // just so that Runner.Sub can use it again
+
 	// didReset remembers whether the runner has ever been reset. This is
 	// used so that Reset is automatically called when running any program
 	// or node for the first time on a Runner.
@@ -239,7 +359,7 @@ type Runner struct {
 	filename string // only if Node was a File
 
 	// like Vars, but local to a func i.e. "local foo=bar"
-	funcVars map[string]Variable
+	funcVars map[string]expand.Variable
 
 	// like Vars, but local to a cmd i.e. "foo=bar prog args..."
 	cmdVars map[string]string
@@ -262,9 +382,6 @@ type Runner struct {
 
 	optState getopts
 
-	ifsJoin string
-	ifsRune func(rune) bool
-
 	// keepRedirs is used so that "exec" can make any redirections
 	// apply to the current shell, and not just the command.
 	keepRedirs bool
@@ -281,17 +398,6 @@ type Runner struct {
 	// On Windows, the kill signal is always sent immediately,
 	// because Go doesn't currently support sending Interrupt on Windows.
 	KillTimeout time.Duration
-
-	fieldAlloc  [4]fieldPart
-	fieldsAlloc [4][]fieldPart
-	bufferAlloc bytes.Buffer
-	oneWord     [1]*syntax.Word
-}
-
-func (r *Runner) strBuilder() *bytes.Buffer {
-	b := &r.bufferAlloc
-	b.Reset()
-	return b
 }
 
 func (r *Runner) optByFlag(flag string) *bool {
@@ -372,6 +478,7 @@ func (r *Runner) Reset() {
 		Exec:        r.Exec,
 		Open:        r.Open,
 		KillTimeout: r.KillTimeout,
+		opts:        r.opts,
 
 		// emptied below, to reuse the space
 		Vars:     r.Vars,
@@ -380,7 +487,7 @@ func (r *Runner) Reset() {
 		usedNew:  r.usedNew,
 	}
 	if r.Vars == nil {
-		r.Vars = make(map[string]Variable)
+		r.Vars = make(map[string]expand.Variable)
 	} else {
 		for k := range r.Vars {
 			delete(r.Vars, k)
@@ -393,29 +500,22 @@ func (r *Runner) Reset() {
 			delete(r.cmdVars, k)
 		}
 	}
-	if _, ok := r.Env.Get("HOME"); !ok {
+	if vr := r.Env.Get("HOME"); !vr.IsSet() {
 		u, _ := user.Current()
-		r.Vars["HOME"] = Variable{Value: StringVal(u.HomeDir)}
+		r.Vars["HOME"] = expand.Variable{Value: u.HomeDir}
 	}
-	r.Vars["PWD"] = Variable{Value: StringVal(r.Dir)}
+	r.Vars["PWD"] = expand.Variable{Value: r.Dir}
-	r.Vars["IFS"] = Variable{Value: StringVal(" \t\n")}
+	r.Vars["IFS"] = expand.Variable{Value: " \t\n"}
-	r.ifsUpdated()
+	r.Vars["OPTIND"] = expand.Variable{Value: "1"}
-	r.Vars["OPTIND"] = Variable{Value: StringVal("1")}
 
 	if runtime.GOOS == "windows" {
 		// convert $PATH to a unix path list
-		path, _ := r.Env.Get("PATH")
+		path := r.Env.Get("PATH").String()
 		path = strings.Join(filepath.SplitList(path), ":")
-		r.Vars["PATH"] = Variable{Value: StringVal(path)}
+		r.Vars["PATH"] = expand.Variable{Value: path}
 	}
 
 	r.dirStack = append(r.dirStack, r.Dir)
-	if r.Exec == nil {
-		r.Exec = DefaultExec
-	}
-	if r.Open == nil {
-		r.Open = DefaultOpen
-	}
 	if r.KillTimeout == 0 {
 		r.KillTimeout = 2 * time.Second
 	}
@@ -424,23 +524,26 @@ func (r *Runner) Reset() {
 
 func (r *Runner) modCtx(ctx context.Context) context.Context {
 	mc := ModuleCtx{
-		Env:         r.Env,
 		Dir:         r.Dir,
 		Stdin:       r.Stdin,
 		Stdout:      r.Stdout,
 		Stderr:      r.Stderr,
 		KillTimeout: r.KillTimeout,
 	}
-	mc.Env = r.Env.Copy()
+	oenv := overlayEnviron{
+		parent: r.Env,
+		values: make(map[string]expand.Variable),
+	}
 	for name, vr := range r.Vars {
-		if !vr.Exported {
+		oenv.Set(name, vr)
-			continue
-		}
-		mc.Env.Set(name, r.varStr(vr, 0))
 	}
-	for name, val := range r.cmdVars {
+	for name, vr := range r.funcVars {
-		mc.Env.Set(name, val)
+		oenv.Set(name, vr)
 	}
+	for name, value := range r.cmdVars {
+		oenv.Set(name, expand.Variable{Exported: true, Value: value})
+	}
+	mc.Env = oenv
 	return context.WithValue(ctx, moduleCtxKey{}, mc)
 }
 
@@ -471,6 +574,7 @@ func (r *Runner) Run(ctx context.Context, node syntax.Node) error {
 	if !r.didReset {
 		r.Reset()
 	}
+	r.fillExpandConfig(ctx)
 	r.err = nil
 	r.filename = ""
 	switch x := node.(type) {
@@ -482,7 +586,7 @@ func (r *Runner) Run(ctx context.Context, node syntax.Node) error {
 	case syntax.Command:
 		r.cmd(ctx, x)
 	default:
-		return fmt.Errorf("Node can only be File, Stmt, or Command: %T", x)
+		return fmt.Errorf("node can only be File, Stmt, or Command: %T", x)
 	}
 	if r.exit > 0 {
 		r.setErr(ExitStatus(r.exit))
@@ -564,6 +668,7 @@ func (r *Runner) sub() *Runner {
 	// Keep in sync with the Runner type. Manually copy fields, to not copy
 	// sensitive ones like errgroup.Group, and to do deep copies of slices.
 	r2 := &Runner{
+		Env:         r.Env,
 		Dir:         r.Dir,
 		Params:      r.Params,
 		Exec:        r.Exec,
@@ -576,19 +681,20 @@ func (r *Runner) sub() *Runner {
 		filename:    r.filename,
 		opts:        r.opts,
 	}
-	// TODO: perhaps we could do a lazy copy here, or some sort of
+	r2.Vars = make(map[string]expand.Variable, len(r.Vars))
-	// overlay to avoid copying all the time
-	r2.Env = r.Env.Copy()
-	r2.Vars = make(map[string]Variable, len(r.Vars))
 	for k, v := range r.Vars {
 		r2.Vars[k] = v
 	}
+	r2.funcVars = make(map[string]expand.Variable, len(r.funcVars))
+	for k, v := range r.funcVars {
+		r2.funcVars[k] = v
+	}
 	r2.cmdVars = make(map[string]string, len(r.cmdVars))
 	for k, v := range r.cmdVars {
 		r2.cmdVars[k] = v
 	}
 	r2.dirStack = append([]string(nil), r.dirStack...)
-	r2.ifsUpdated()
+	r2.fillExpandConfig(r.ectx)
 	r2.didReset = true
 	return r2
 }
@@ -606,23 +712,19 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 		r.exit = r2.exit
 		r.setErr(r2.err)
 	case *syntax.CallExpr:
-		fields := r.fields(ctx, x.Args...)
+		fields := r.fields(x.Args...)
 		if len(fields) == 0 {
 			for _, as := range x.Assigns {
-				vr, _ := r.lookupVar(as.Name.Value)
+				vr := r.lookupVar(as.Name.Value)
-				vr.Value = r.assignVal(ctx, as, "")
+				vr.Value = r.assignVal(as, "")
-				r.setVar(ctx, as.Name.Value, as.Index, vr)
+				r.setVar(as.Name.Value, as.Index, vr)
 			}
 			break
 		}
 		for _, as := range x.Assigns {
-			val := r.assignVal(ctx, as, "")
+			val := r.assignVal(as, "")
 			// we know that inline vars must be strings
-			r.cmdVars[as.Name.Value] = string(val.(StringVal))
+			r.cmdVars[as.Name.Value] = val.(string)
-			if as.Name.Value == "IFS" {
-				r.ifsUpdated()
-				defer r.ifsUpdated()
-			}
 		}
 		r.call(ctx, x.Args[0].Pos(), fields)
 		// cmdVars can be nuked here, as they are never useful
@@ -689,37 +791,41 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 		switch y := x.Loop.(type) {
 		case *syntax.WordIter:
 			name := y.Name.Value
-			for _, field := range r.fields(ctx, y.Items...) {
+			items := r.Params // for i; do ...
-				r.setVarString(ctx, name, field)
+			if y.InPos.IsValid() {
+				items = r.fields(y.Items...) // for i in ...; do ...
+			}
+			for _, field := range items {
+				r.setVarString(name, field)
 				if r.loopStmtsBroken(ctx, x.Do) {
 					break
 				}
 			}
 		case *syntax.CStyleLoop:
-			r.arithm(ctx, y.Init)
+			r.arithm(y.Init)
-			for r.arithm(ctx, y.Cond) != 0 {
+			for r.arithm(y.Cond) != 0 {
 				if r.loopStmtsBroken(ctx, x.Do) {
 					break
 				}
-				r.arithm(ctx, y.Post)
+				r.arithm(y.Post)
 			}
 		}
 	case *syntax.FuncDecl:
 		r.setFunc(x.Name.Value, x.Body)
 	case *syntax.ArithmCmd:
-		r.exit = oneIf(r.arithm(ctx, x.X) == 0)
+		r.exit = oneIf(r.arithm(x.X) == 0)
 	case *syntax.LetClause:
 		var val int
 		for _, expr := range x.Exprs {
-			val = r.arithm(ctx, expr)
+			val = r.arithm(expr)
 		}
 		r.exit = oneIf(val == 0)
 	case *syntax.CaseClause:
-		str := r.loneWord(ctx, x.Word)
+		str := r.literal(x.Word)
 		for _, ci := range x.Items {
 			for _, word := range ci.Patterns {
-				pat := r.lonePattern(ctx, word)
+				pattern := r.pattern(word)
-				if match(pat, str) {
+				if match(pattern, str) {
 					r.stmts(ctx, ci.StmtList)
 					return
 				}
@@ -732,13 +838,13 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 			r.exit = 1
 		}
 	case *syntax.DeclClause:
-		local := false
+		local, global := false, false
 		var modes []string
 		valType := ""
 		switch x.Variant.Value {
 		case "declare":
-			// When used in a function, "declare" acts as
+			// When used in a function, "declare" acts as "local"
-			// "local" unless the "-g" option is used.
+			// unless the "-g" option is used.
 			local = r.inFunc
 		case "local":
 			if !r.inFunc {
@@ -755,13 +861,13 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 			modes = append(modes, "-n")
 		}
 		for _, opt := range x.Opts {
-			switch s := r.loneWord(ctx, opt); s {
+			switch s := r.literal(opt); s {
 			case "-x", "-r", "-n":
 				modes = append(modes, s)
 			case "-a", "-A":
 				valType = s
 			case "-g":
-				local = false
+				global = true
 			default:
 				r.errf("declare: invalid option %q\n", s)
 				r.exit = 2
@@ -769,11 +875,20 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 			}
 		}
 		for _, as := range x.Assigns {
-			for _, as := range r.expandAssigns(ctx, as) {
+			for _, as := range r.flattenAssign(as) {
 				name := as.Name.Value
-				vr, _ := r.lookupVar(as.Name.Value)
+				if !syntax.ValidName(name) {
-				vr.Value = r.assignVal(ctx, as, valType)
+					r.errf("declare: invalid name %q\n", name)
-				vr.Local = local
+					r.exit = 1
+					return
+				}
+				vr := r.lookupVar(as.Name.Value)
+				vr.Value = r.assignVal(as, valType)
+				if global {
+					vr.Local = false
+				} else if local {
+					vr.Local = true
+				}
 				for _, mode := range modes {
 					switch mode {
 					case "-x":
@@ -784,7 +899,7 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 						vr.NameRef = true
 					}
 				}
-				r.setVar(ctx, name, as.Index, vr)
+				r.setVar(name, as.Index, vr)
 			}
 		}
 	case *syntax.TimeClause:
@@ -808,6 +923,39 @@ func (r *Runner) cmd(ctx context.Context, cm syntax.Command) {
 	}
 }
 
+func (r *Runner) flattenAssign(as *syntax.Assign) []*syntax.Assign {
+	// Convert "declare $x" into "declare value".
+	// Don't use syntax.Parser here, as we only want the basic
+	// splitting by '='.
+	if as.Name != nil {
+		return []*syntax.Assign{as} // nothing to do
+	}
+	var asgns []*syntax.Assign
+	for _, field := range r.fields(as.Value) {
+		as := &syntax.Assign{}
+		parts := strings.SplitN(field, "=", 2)
+		as.Name = &syntax.Lit{Value: parts[0]}
+		if len(parts) == 1 {
+			as.Naked = true
+		} else {
+			as.Value = &syntax.Word{Parts: []syntax.WordPart{
+				&syntax.Lit{Value: parts[1]},
+			}}
+		}
+		asgns = append(asgns, as)
+	}
+	return asgns
+}
+
+func match(pattern, name string) bool {
+	expr, err := syntax.TranslatePattern(pattern, true)
+	if err != nil {
+		return false
+	}
+	rx := regexp.MustCompile("^" + expr + "$")
+	return rx.MatchString(name)
+}
+
 func elapsedString(d time.Duration, posix bool) string {
 	if posix {
 		return fmt.Sprintf("%.2f", d.Seconds())
@@ -823,10 +971,42 @@ func (r *Runner) stmts(ctx context.Context, sl syntax.StmtList) {
 	}
 }
 
+func (r *Runner) hdocReader(rd *syntax.Redirect) io.Reader {
+	if rd.Op != syntax.DashHdoc {
+		hdoc := r.document(rd.Hdoc)
+		return strings.NewReader(hdoc)
+	}
+	var buf bytes.Buffer
+	var cur []syntax.WordPart
+	flushLine := func() {
+		if buf.Len() > 0 {
+			buf.WriteByte('\n')
+		}
+		buf.WriteString(r.document(&syntax.Word{Parts: cur}))
+		cur = cur[:0]
+	}
+	for _, wp := range rd.Hdoc.Parts {
+		lit, ok := wp.(*syntax.Lit)
+		if !ok {
+			cur = append(cur, wp)
+			continue
+		}
+		for i, part := range strings.Split(lit.Value, "\n") {
+			if i > 0 {
+				flushLine()
+				cur = cur[:0]
+			}
+			part = strings.TrimLeft(part, "\t")
+			cur = append(cur, &syntax.Lit{Value: part})
+		}
+	}
+	flushLine()
+	return &buf
+}
+
 func (r *Runner) redir(ctx context.Context, rd *syntax.Redirect) (io.Closer, error) {
 	if rd.Hdoc != nil {
-		hdoc := r.loneWord(ctx, rd.Hdoc)
+		r.Stdin = r.hdocReader(rd)
-		r.Stdin = strings.NewReader(hdoc)
 		return nil, nil
 	}
 	orig := &r.Stdout
@@ -837,7 +1017,7 @@ func (r *Runner) redir(ctx context.Context, rd *syntax.Redirect) (io.Closer, err
 			orig = &r.Stderr
 		}
 	}
-	arg := r.loneWord(ctx, rd.Word)
+	arg := r.literal(rd.Word)
 	switch rd.Op {
 	case syntax.WordHdoc:
 		r.Stdin = strings.NewReader(arg + "\n")
@@ -860,9 +1040,9 @@ func (r *Runner) redir(ctx context.Context, rd *syntax.Redirect) (io.Closer, err
 	mode := os.O_RDONLY
 	switch rd.Op {
 	case syntax.AppOut, syntax.AppAll:
-		mode = os.O_RDWR | os.O_CREATE | os.O_APPEND
+		mode = os.O_WRONLY | os.O_CREATE | os.O_APPEND
 	case syntax.RdrOut, syntax.RdrAll:
-		mode = os.O_RDWR | os.O_CREATE | os.O_TRUNC
+		mode = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
 	}
 	f, err := r.open(ctx, r.relPath(arg), mode, 0644, true)
 	if err != nil {
@@ -1006,6 +1186,10 @@ func (r *Runner) findExecutable(file string, exts []string) string {
 	return ""
 }
 
+func driveLetter(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
 // splitList is like filepath.SplitList, but always using the unix path
 // list separator ':'. On Windows, it also makes sure not to split
 // [A-Z]:[/\].
@@ -1022,8 +1206,7 @@ func splitList(path string) []string {
 	for i := 0; i < len(list); i++ {
 		s := list[i]
 		switch {
-		case len(s) != 1, s[0] < 'A', s[0] > 'Z':
+		case len(s) != 1, !driveLetter(s[0]):
-			// not a disk name
 		case i+1 >= len(list):
 			// last element
 		case strings.IndexAny(list[i+1], `/\`) != 0:
@@ -1039,7 +1222,7 @@ func splitList(path string) []string {
 }
 
 func (r *Runner) lookPath(file string) string {
-	pathList := splitList(r.getVar("PATH"))
+	pathList := splitList(r.envGet("PATH"))
 	chars := `/`
 	if runtime.GOOS == "windows" {
 		chars = `:\/`
@@ -1070,7 +1253,7 @@ func (r *Runner) pathExts() []string {
 	if runtime.GOOS != "windows" {
 		return nil
 	}
-	pathext := r.getVar("PATHEXT")
+	pathext := r.envGet("PATHEXT")
 	if pathext == "" {
 		return []string{".com", ".exe", ".bat", ".cmd"}
 	}

vendor/mvdan.cc/sh/interp/module.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"mvdan.cc/sh/expand"
 )
 
 // FromModuleContext returns the ModuleCtx value stored in ctx, if any.
@@ -27,7 +29,7 @@ type moduleCtxKey struct{}
 // It contains some of the current state of the Runner, as well as some fields
 // necessary to implement some of the modules.
 type ModuleCtx struct {
-	Env         Environ
+	Env         expand.Environ
 	Dir         string
 	Stdin       io.Reader
 	Stdout      io.Writer

vendor/mvdan.cc/sh/interp/test.go

@@ -19,22 +19,22 @@ import (
 func (r *Runner) bashTest(ctx context.Context, expr syntax.TestExpr, classic bool) string {
 	switch x := expr.(type) {
 	case *syntax.Word:
-		return r.loneWord(ctx, x)
+		return r.document(x)
 	case *syntax.ParenTest:
 		return r.bashTest(ctx, x.X, classic)
 	case *syntax.BinaryTest:
 		switch x.Op {
 		case syntax.TsMatch, syntax.TsNoMatch:
-			str := r.loneWord(ctx, x.X.(*syntax.Word))
+			str := r.literal(x.X.(*syntax.Word))
 			yw := x.Y.(*syntax.Word)
 			if classic { // test, [
-				lit := r.loneWord(ctx, yw)
+				lit := r.literal(yw)
 				if (str == lit) == (x.Op == syntax.TsMatch) {
 					return "1"
 				}
 			} else { // [[
-				pat := r.lonePattern(ctx, yw)
+				pattern := r.pattern(yw)
-				if match(pat, str) == (x.Op == syntax.TsMatch) {
+				if match(pattern, str) == (x.Op == syntax.TsMatch) {
 					return "1"
 				}
 			}
@@ -173,11 +173,9 @@ func (r *Runner) unTest(ctx context.Context, op syntax.UnTestOperator, x string)
 		}
 		return false
 	case syntax.TsVarSet:
-		_, e := r.lookupVar(x)
+		return r.lookupVar(x).IsSet()
-		return e
 	case syntax.TsRefVar:
-		v, _ := r.lookupVar(x)
+		return r.lookupVar(x).NameRef
-		return v.NameRef
 	case syntax.TsNot:
 		return x == ""
 	default:

vendor/mvdan.cc/sh/interp/vars.go

@@ -4,259 +4,137 @@
 package interp
 
 import (
-	"context"
+	"os"
-	"fmt"
 	"runtime"
-	"sort"
+	"strconv"
 	"strings"
 
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/syntax"
 )
 
-type Environ interface {
+type overlayEnviron struct {
-	Get(name string) (value string, exists bool)
+	parent expand.Environ
-	Set(name, value string)
+	values map[string]expand.Variable
-	Delete(name string)
-	Names() []string
-	Copy() Environ
 }
 
-type mapEnviron struct {
+func (o overlayEnviron) Get(name string) expand.Variable {
-	names  []string
+	if vr, ok := o.values[name]; ok {
-	values map[string]string
+		return vr
-}
-
-func (m *mapEnviron) Get(name string) (string, bool) {
-	val, ok := m.values[name]
-	return val, ok
-}
-
-func (m *mapEnviron) Set(name, value string) {
-	_, ok := m.values[name]
-	if !ok {
-		m.names = append(m.names, name)
-		sort.Strings(m.names)
 	}
-	m.values[name] = value
+	return o.parent.Get(name)
 }
 
-func (m *mapEnviron) Delete(name string) {
+func (o overlayEnviron) Set(name string, vr expand.Variable) {
-	if _, ok := m.values[name]; !ok {
+	o.values[name] = vr
-		return
+}
-	}
+
-	delete(m.values, name)
+func (o overlayEnviron) Each(f func(name string, vr expand.Variable) bool) {
-	for i, iname := range m.names {
+	o.parent.Each(f)
-		if iname == name {
+	for name, vr := range o.values {
-			m.names = append(m.names[:i], m.names[i+1:]...)
+		if !f(name, vr) {
 			return
 		}
 	}
 }
 
-func (m *mapEnviron) Names() []string {
+func execEnv(env expand.Environ) []string {
-	return m.names
+	list := make([]string, 0, 32)
-}
+	env.Each(func(name string, vr expand.Variable) bool {
-
+		if vr.Exported {
-func (m *mapEnviron) Copy() Environ {
+			list = append(list, name+"="+vr.String())
-	m2 := &mapEnviron{
+		}
-		names:  make([]string, len(m.names)),
+		return true
-		values: make(map[string]string, len(m.values)),
+	})
-	}
-	copy(m2.names, m.names)
-	for name, val := range m.values {
-		m2.values[name] = val
-	}
-	return m2
-}
-
-func execEnv(env Environ) []string {
-	names := env.Names()
-	list := make([]string, len(names))
-	for i, name := range names {
-		val, _ := env.Get(name)
-		list[i] = name + "=" + val
-	}
 	return list
 }
 
-func EnvFromList(list []string) (Environ, error) {
+func (r *Runner) lookupVar(name string) expand.Variable {
-	m := mapEnviron{
-		names:  make([]string, 0, len(list)),
-		values: make(map[string]string, len(list)),
-	}
-	for _, kv := range list {
-		i := strings.IndexByte(kv, '=')
-		if i < 0 {
-			return nil, fmt.Errorf("env not in the form key=value: %q", kv)
-		}
-		name, val := kv[:i], kv[i+1:]
-		if runtime.GOOS == "windows" {
-			name = strings.ToUpper(name)
-		}
-		m.names = append(m.names, name)
-		m.values[name] = val
-	}
-	sort.Strings(m.names)
-	return &m, nil
-}
-
-type FuncEnviron func(string) string
-
-func (f FuncEnviron) Get(name string) (string, bool) {
-	val := f(name)
-	return val, val != ""
-}
-
-func (f FuncEnviron) Set(name, value string) {}
-func (f FuncEnviron) Delete(name string)     {}
-func (f FuncEnviron) Names() []string        { return nil }
-func (f FuncEnviron) Copy() Environ          { return f }
-
-type Variable struct {
-	Local    bool
-	Exported bool
-	ReadOnly bool
-	NameRef  bool
-	Value    VarValue
-}
-
-// VarValue is one of:
-//
-//     StringVal
-//     IndexArray
-//     AssocArray
-type VarValue interface{}
-
-type StringVal string
-
-type IndexArray []string
-
-type AssocArray map[string]string
-
-func (r *Runner) lookupVar(name string) (Variable, bool) {
 	if name == "" {
 		panic("variable name must not be empty")
 	}
-	if val, e := r.cmdVars[name]; e {
+	var value interface{}
-		return Variable{Value: StringVal(val)}, true
+	switch name {
+	case "#":
+		value = strconv.Itoa(len(r.Params))
+	case "@", "*":
+		value = r.Params
+	case "?":
+		value = strconv.Itoa(r.exit)
+	case "$":
+		value = strconv.Itoa(os.Getpid())
+	case "PPID":
+		value = strconv.Itoa(os.Getppid())
+	case "DIRSTACK":
+		value = r.dirStack
+	case "0":
+		if r.filename != "" {
+			value = r.filename
+		} else {
+			value = "gosh"
+		}
+	case "1", "2", "3", "4", "5", "6", "7", "8", "9":
+		i := int(name[0] - '1')
+		if i < len(r.Params) {
+			value = r.Params[i]
+		} else {
+			value = ""
+		}
+	}
+	if value != nil {
+		return expand.Variable{Value: value}
+	}
+	if value, e := r.cmdVars[name]; e {
+		return expand.Variable{Value: value}
 	}
 	if vr, e := r.funcVars[name]; e {
-		return vr, true
+		vr.Local = true
+		return vr
 	}
 	if vr, e := r.Vars[name]; e {
-		return vr, true
+		return vr
 	}
-	if str, e := r.Env.Get(name); e {
+	if vr := r.Env.Get(name); vr.IsSet() {
-		return Variable{Value: StringVal(str)}, true
+		return vr
 	}
 	if runtime.GOOS == "windows" {
 		upper := strings.ToUpper(name)
-		if str, e := r.Env.Get(upper); e {
+		if vr := r.Env.Get(upper); vr.IsSet() {
-			return Variable{Value: StringVal(str)}, true
+			return vr
 		}
 	}
 	if r.opts[optNoUnset] {
 		r.errf("%s: unbound variable\n", name)
 		r.setErr(ShellExitStatus(1))
 	}
-	return Variable{}, false
+	return expand.Variable{}
 }
 
-func (r *Runner) getVar(name string) string {
+func (r *Runner) envGet(name string) string {
-	val, _ := r.lookupVar(name)
+	return r.lookupVar(name).String()
-	return r.varStr(val, 0)
 }
 
 func (r *Runner) delVar(name string) {
-	val, _ := r.lookupVar(name)
+	vr := r.lookupVar(name)
-	if val.ReadOnly {
+	if vr.ReadOnly {
 		r.errf("%s: readonly variable\n", name)
 		r.exit = 1
 		return
 	}
-	delete(r.Vars, name)
+	if vr.Local {
-	delete(r.funcVars, name)
+		// don't overwrite a non-local var with the same name
-	delete(r.cmdVars, name)
+		r.funcVars[name] = expand.Variable{}
-	r.Env.Delete(name)
+	} else {
+		r.Vars[name] = expand.Variable{} // to not query r.Env
+	}
 }
 
-// maxNameRefDepth defines the maximum number of times to follow
+func (r *Runner) setVarString(name, value string) {
-// references when expanding a variable. Otherwise, simple name
+	r.setVar(name, nil, expand.Variable{Value: value})
-// reference loops could crash the interpreter quite easily.
-const maxNameRefDepth = 100
-
-func (r *Runner) varStr(vr Variable, depth int) string {
-	if depth > maxNameRefDepth {
-		return ""
-	}
-	switch x := vr.Value.(type) {
-	case StringVal:
-		if vr.NameRef {
-			vr, _ = r.lookupVar(string(x))
-			return r.varStr(vr, depth+1)
-		}
-		return string(x)
-	case IndexArray:
-		if len(x) > 0 {
-			return x[0]
-		}
-	case AssocArray:
-		// nothing to do
-	}
-	return ""
 }
 
-func (r *Runner) varInd(ctx context.Context, vr Variable, e syntax.ArithmExpr, depth int) string {
+func (r *Runner) setVarInternal(name string, vr expand.Variable) {
-	if depth > maxNameRefDepth {
+	if _, ok := vr.Value.(string); ok {
-		return ""
-	}
-	switch x := vr.Value.(type) {
-	case StringVal:
-		if vr.NameRef {
-			vr, _ = r.lookupVar(string(x))
-			return r.varInd(ctx, vr, e, depth+1)
-		}
-		if r.arithm(ctx, e) == 0 {
-			return string(x)
-		}
-	case IndexArray:
-		switch anyOfLit(e, "@", "*") {
-		case "@":
-			return strings.Join(x, " ")
-		case "*":
-			return strings.Join(x, r.ifsJoin)
-		}
-		i := r.arithm(ctx, e)
-		if len(x) > 0 {
-			return x[i]
-		}
-	case AssocArray:
-		if lit := anyOfLit(e, "@", "*"); lit != "" {
-			var strs IndexArray
-			keys := make([]string, 0, len(x))
-			for k := range x {
-				keys = append(keys, k)
-			}
-			sort.Strings(keys)
-			for _, k := range keys {
-				strs = append(strs, x[k])
-			}
-			if lit == "*" {
-				return strings.Join(strs, r.ifsJoin)
-			}
-			return strings.Join(strs, " ")
-		}
-		return x[r.loneWord(ctx, e.(*syntax.Word))]
-	}
-	return ""
-}
-
-func (r *Runner) setVarString(ctx context.Context, name, val string) {
-	r.setVar(ctx, name, nil, Variable{Value: StringVal(val)})
-}
-
-func (r *Runner) setVarInternal(name string, vr Variable) {
-	if _, ok := vr.Value.(StringVal); ok {
 		if r.opts[optAllExport] {
 			vr.Exported = true
 		}
@@ -265,28 +143,31 @@ func (r *Runner) setVarInternal(name string, vr Variable) {
 	}
 	if vr.Local {
 		if r.funcVars == nil {
-			r.funcVars = make(map[string]Variable)
+			r.funcVars = make(map[string]expand.Variable)
 		}
 		r.funcVars[name] = vr
 	} else {
 		r.Vars[name] = vr
 	}
-	if name == "IFS" {
-		r.ifsUpdated()
-	}
 }
 
-func (r *Runner) setVar(ctx context.Context, name string, index syntax.ArithmExpr, vr Variable) {
+func (r *Runner) setVar(name string, index syntax.ArithmExpr, vr expand.Variable) {
-	cur, _ := r.lookupVar(name)
+	cur := r.lookupVar(name)
 	if cur.ReadOnly {
 		r.errf("%s: readonly variable\n", name)
 		r.exit = 1
 		return
 	}
-	_, isIndexArray := cur.Value.(IndexArray)
+	if name2, var2 := cur.Resolve(r.Env); name2 != "" {
-	_, isAssocArray := cur.Value.(AssocArray)
+		name = name2
+		cur = var2
+		vr.NameRef = false
+		cur.NameRef = false
+	}
+	_, isIndexArray := cur.Value.([]string)
+	_, isAssocArray := cur.Value.(map[string]string)
 
-	if _, ok := vr.Value.(StringVal); ok && index == nil {
+	if _, ok := vr.Value.(string); ok && index == nil {
 		// When assigning a string to an array, fall back to the
 		// zero value for the index.
 		if isIndexArray {
@@ -304,33 +185,33 @@ func (r *Runner) setVar(ctx context.Context, name string, index syntax.ArithmExp
 		return
 	}
 
-	// from the syntax package, we know that val must be a string if
+	// from the syntax package, we know that value must be a string if index
-	// index is non-nil; nested arrays are forbidden.
+	// is non-nil; nested arrays are forbidden.
-	valStr := string(vr.Value.(StringVal))
+	valStr := vr.Value.(string)
 
 	// if the existing variable is already an AssocArray, try our best
 	// to convert the key to a string
 	if isAssocArray {
-		amap := cur.Value.(AssocArray)
+		amap := cur.Value.(map[string]string)
 		w, ok := index.(*syntax.Word)
 		if !ok {
 			return
 		}
-		k := r.loneWord(ctx, w)
+		k := r.literal(w)
 		amap[k] = valStr
 		cur.Value = amap
 		r.setVarInternal(name, cur)
 		return
 	}
-	var list IndexArray
+	var list []string
 	switch x := cur.Value.(type) {
-	case StringVal:
+	case string:
-		list = append(list, string(x))
+		list = append(list, x)
-	case IndexArray:
+	case []string:
 		list = x
-	case AssocArray: // done above
+	case map[string]string: // done above
 	}
-	k := r.arithm(ctx, index)
+	k := r.arithm(index)
 	for len(list) < k+1 {
 		list = append(list, "")
 	}
@@ -358,32 +239,33 @@ func stringIndex(index syntax.ArithmExpr) bool {
 	return false
 }
 
-func (r *Runner) assignVal(ctx context.Context, as *syntax.Assign, valType string) VarValue {
+func (r *Runner) assignVal(as *syntax.Assign, valType string) interface{} {
-	prev, prevOk := r.lookupVar(as.Name.Value)
+	prev := r.lookupVar(as.Name.Value)
 	if as.Naked {
 		return prev.Value
 	}
 	if as.Value != nil {
-		s := r.loneWord(ctx, as.Value)
+		s := r.literal(as.Value)
-		if !as.Append || !prevOk {
+		if !as.Append || !prev.IsSet() {
-			return StringVal(s)
+			return s
 		}
 		switch x := prev.Value.(type) {
-		case StringVal:
+		case string:
-			return x + StringVal(s)
+			return x + s
-		case IndexArray:
+		case []string:
 			if len(x) == 0 {
 				x = append(x, "")
 			}
 			x[0] += s
 			return x
-		case AssocArray:
+		case map[string]string:
 			// TODO
 		}
-		return StringVal(s)
+		return s
 	}
 	if as.Array == nil {
-		return nil
+		// don't return nil, as that's an unset variable
+		return ""
 	}
 	elems := as.Array.Elems
 	if valType == "" {
@@ -395,12 +277,12 @@ func (r *Runner) assignVal(ctx context.Context, as *syntax.Assign, valType strin
 	}
 	if valType == "-A" {
 		// associative array
-		amap := AssocArray(make(map[string]string, len(elems)))
+		amap := make(map[string]string, len(elems))
 		for _, elem := range elems {
-			k := r.loneWord(ctx, elem.Index.(*syntax.Word))
+			k := r.literal(elem.Index.(*syntax.Word))
-			amap[k] = r.loneWord(ctx, elem.Value)
+			amap[k] = r.literal(elem.Value)
 		}
-		if !as.Append || !prevOk {
+		if !as.Append || !prev.IsSet() {
 			return amap
 		}
 		// TODO
@@ -414,7 +296,7 @@ func (r *Runner) assignVal(ctx context.Context, as *syntax.Assign, valType strin
 			indexes[i] = i
 			continue
 		}
-		k := r.arithm(ctx, elem.Index)
+		k := r.arithm(elem.Index)
 		indexes[i] = k
 		if k > maxIndex {
 			maxIndex = k
@@ -422,50 +304,18 @@ func (r *Runner) assignVal(ctx context.Context, as *syntax.Assign, valType strin
 	}
 	strs := make([]string, maxIndex+1)
 	for i, elem := range elems {
-		strs[indexes[i]] = r.loneWord(ctx, elem.Value)
+		strs[indexes[i]] = r.literal(elem.Value)
 	}
-	if !as.Append || !prevOk {
+	if !as.Append || !prev.IsSet() {
-		return IndexArray(strs)
+		return strs
 	}
 	switch x := prev.Value.(type) {
-	case StringVal:
+	case string:
-		prevList := IndexArray([]string{string(x)})
+		return append([]string{x}, strs...)
-		return append(prevList, strs...)
+	case []string:
-	case IndexArray:
 		return append(x, strs...)
-	case AssocArray:
+	case map[string]string:
 		// TODO
 	}
-	return IndexArray(strs)
+	return strs
-}
-
-func (r *Runner) ifsUpdated() {
-	runes := r.getVar("IFS")
-	r.ifsJoin = ""
-	if len(runes) > 0 {
-		r.ifsJoin = runes[:1]
-	}
-	r.ifsRune = func(r rune) bool {
-		for _, r2 := range runes {
-			if r == r2 {
-				return true
-			}
-		}
-		return false
-	}
-}
-
-func (r *Runner) namesByPrefix(prefix string) []string {
-	var names []string
-	for _, name := range r.Env.Names() {
-		if strings.HasPrefix(name, prefix) {
-			names = append(names, name)
-		}
-	}
-	for name := range r.Vars {
-		if strings.HasPrefix(name, prefix) {
-			names = append(names, name)
-		}
-	}
-	return names
 }

vendor/mvdan.cc/sh/shell/doc.go

@@ -1,9 +1,14 @@
 // Copyright (c) 2017, Daniel Martí <mvdan@mvdan.cc>
 // See LICENSE for licensing information
 
-// Package shell contains high-level features that use the syntax and
+// Package shell contains high-level features that use the syntax, expand, and
 // interp packages under the hood.
 //
-// This package is a work in progress and EXPERIMENTAL; its API is not
+// Please note that this package uses POSIX Shell syntax. As such, path names on
-// subject to the 1.x backwards compatibility guarantee.
+// Windows need to use double backslashes or be within single quotes when given
+// to functions like Fields. For example:
+//
+//     shell.Fields("echo /foo/bar")     // on Unix-like
+//     shell.Fields("echo C:\\foo\\bar") // on Windows
+//     shell.Fields("echo 'C:\foo\bar'") // on Windows, with quotes
 package shell

vendor/mvdan.cc/sh/shell/expand.go

@@ -4,41 +4,60 @@
 package shell
 
 import (
-	"context"
+	"os"
 	"strings"
 
-	"mvdan.cc/sh/interp"
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/syntax"
 )
 
-// Expand performs shell expansion on s, using env to resolve variables.
+// Expand performs shell expansion on s as if it were within double quotes,
-// The expansion will apply to parameter expansions like $var and
+// using env to resolve variables. This includes parameter expansion, arithmetic
-// ${#var}, but also to arithmetic expansions like $((var + 3)), and
+// expansion, and quote removal.
-// command substitutions like $(echo foo).
 //
-// If env is nil, the current environment variables are used.
+// If env is nil, the current environment variables are used. Empty variables
+// are treated as unset; to support variables which are set but empty, use the
+// expand package directly.
 //
-// Any side effects or modifications to the system are forbidden when
+// Command subsitutions like $(echo foo) aren't supported to avoid running
-// interpreting the program. This is enforced via whitelists when
+// arbitrary code. To support those, use an interpreter with the expand package.
-// executing programs and opening paths. The interpreter also has a timeout of
+//
-// two seconds.
+// An error will be reported if the input string had invalid syntax.
 func Expand(s string, env func(string) string) (string, error) {
 	p := syntax.NewParser()
-	src := "<<EXPAND_EOF\n" + s + "\nEXPAND_EOF"
+	word, err := p.Document(strings.NewReader(s))
-	f, err := p.Parse(strings.NewReader(src), "")
 	if err != nil {
 		return "", err
 	}
-	word := f.Stmts[0].Redirs[0].Hdoc
+	if env == nil {
-	last := word.Parts[len(word.Parts)-1].(*syntax.Lit)
+		env = os.Getenv
-	// since the heredoc implies a trailing newline
-	last.Value = strings.TrimSuffix(last.Value, "\n")
-	r := pureRunner()
-	if env != nil {
-		r.Env = interp.FuncEnviron(env)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), pureRunnerTimeout)
+	cfg := &expand.Config{Env: expand.FuncEnviron(env)}
-	defer cancel()
+	return expand.Document(cfg, word)
-	fields, err := r.Fields(ctx, word)
+}
-	return strings.Join(fields, ""), err
+
+// Fields performs shell expansion on s as if it were a command's arguments,
+// using env to resolve variables. It is similar to Expand, but includes brace
+// expansion, tilde expansion, and globbing.
+//
+// If env is nil, the current environment variables are used. Empty variables
+// are treated as unset; to support variables which are set but empty, use the
+// expand package directly.
+//
+// An error will be reported if the input string had invalid syntax.
+func Fields(s string, env func(string) string) ([]string, error) {
+	p := syntax.NewParser()
+	var words []*syntax.Word
+	err := p.Words(strings.NewReader(s), func(w *syntax.Word) bool {
+		words = append(words, w)
+		return true
+	})
+	if err != nil {
+		return nil, err
+	}
+	if env == nil {
+		env = os.Getenv
+	}
+	cfg := &expand.Config{Env: expand.FuncEnviron(env)}
+	return expand.Fields(cfg, words...)
 }

vendor/mvdan.cc/sh/shell/source.go

@@ -6,81 +6,42 @@ package shell
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
-	"time"
 
+	"mvdan.cc/sh/expand"
 	"mvdan.cc/sh/interp"
 	"mvdan.cc/sh/syntax"
 )
 
 // SourceFile sources a shell file from disk and returns the variables
-// declared in it.
+// declared in it. It is a convenience function that uses a default shell
+// parser, parses a file from disk, and calls SourceNode.
 //
-// A default parser is used; to set custom options, use SourceNode
+// This function should be used with caution, as it can interpret arbitrary
-// instead.
+// code. Untrusted shell programs shoudn't be sourced outside of a sandbox
-func SourceFile(path string) (map[string]interp.Variable, error) {
+// environment.
+func SourceFile(ctx context.Context, path string) (map[string]expand.Variable, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("could not open: %v", err)
 	}
 	defer f.Close()
-	p := syntax.NewParser()
+	file, err := syntax.NewParser().Parse(f, path)
-	file, err := p.Parse(f, path)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse: %v", err)
 	}
-	return SourceNode(file)
+	return SourceNode(ctx, file)
-}
-
-// purePrograms holds a list of common programs that do not have side
-// effects, or otherwise cannot modify or harm the system that runs
-// them.
-var purePrograms = []string{
-	// string handling
-	"sed", "grep", "tr", "cut", "cat", "head", "tail", "seq", "yes",
-	"wc",
-	// paths
-	"ls", "pwd", "basename", "realpath",
-	// others
-	"env", "sleep", "uniq", "sort",
-}
-
-var pureRunnerTimeout = 2 * time.Second
-
-func pureRunner() *interp.Runner {
-	// forbid executing programs that might cause trouble
-	exec := interp.ModuleExec(func(ctx context.Context, path string, args []string) error {
-		for _, name := range purePrograms {
-			if args[0] == name {
-				return interp.DefaultExec(ctx, path, args)
-			}
-		}
-		return fmt.Errorf("program not in whitelist: %s", args[0])
-	})
-	// forbid opening any real files
-	open := interp.OpenDevImpls(func(ctx context.Context, path string, flags int, mode os.FileMode) (io.ReadWriteCloser, error) {
-		mc, _ := interp.FromModuleContext(ctx)
-		return nil, fmt.Errorf("cannot open path: %s", mc.UnixPath(path))
-	})
-	r, err := interp.New(interp.Module(exec), interp.Module(open))
-	if err != nil {
-		panic(err)
-	}
-	return r
 }
 
 // SourceNode sources a shell program from a node and returns the
-// variables declared in it.
+// variables declared in it. It accepts the same set of node types that
+// interp/Runner.Run does.
 //
-// Any side effects or modifications to the system are forbidden when
+// This function should be used with caution, as it can interpret arbitrary
-// interpreting the program. This is enforced via whitelists when
+// code. Untrusted shell programs shoudn't be sourced outside of a sandbox
-// executing programs and opening paths. The interpreter also has a timeout of
+// environment.
-// two seconds.
+func SourceNode(ctx context.Context, node syntax.Node) (map[string]expand.Variable, error) {
-func SourceNode(node syntax.Node) (map[string]interp.Variable, error) {
+	r, _ := interp.New()
-	r := pureRunner()
-	ctx, cancel := context.WithTimeout(context.Background(), pureRunnerTimeout)
-	defer cancel()
 	if err := r.Run(ctx, node); err != nil {
 		return nil, fmt.Errorf("could not run: %v", err)
 	}

vendor/mvdan.cc/sh/syntax/expand.go

@@ -5,7 +5,8 @@ package syntax
 
 import "strconv"
 
-// TODO: consider making these special syntax nodes
+// TODO(v3): Consider making these special syntax nodes.
+// Among other things, we can make use of Word.Lit.
 
 type brace struct {
 	seq   bool // {x..y[..incr]} instead of {x,y[,...]}
@@ -265,17 +266,13 @@ func expandRec(bw *braceWord) []*Word {
 	return []*Word{{Parts: left}}
 }
 
+// TODO(v3): remove
+
 // ExpandBraces performs Bash brace expansion on a word. For example,
 // passing it a single-literal word "foo{bar,baz}" will return two
 // single-literal words, "foobar" and "foobaz".
 //
-// It does not return an error; malformed brace expansions are simply
+// Deprecated: use mvdan.cc/sh/expand.Braces instead.
-// skipped. For example, "a{b{c,d}" results in the words "a{bc" and
-// "a{bd".
-//
-// Note that the resulting words may have more word parts than
-// necessary, such as contiguous *Lit nodes, and that these parts may be
-// shared between words.
 func ExpandBraces(word *Word) []*Word {
 	topBrace, any := splitBraces(word)
 	if !any {

vendor/mvdan.cc/sh/syntax/lexer.go

@@ -60,10 +60,9 @@ func (p *Parser) rune() rune {
 		// p.r instead of b so that newline
 		// character positions don't have col 0.
 		p.npos.line++
-		p.npos.col = 1
+		p.npos.col = 0
-	} else {
-		p.npos.col += p.w
 	}
+	p.npos.col += p.w
 	bquotes := 0
 retry:
 	if p.bsp < len(p.bs) {
@@ -87,9 +86,8 @@ retry:
 			p.w, p.r = 1, rune(b)
 			return p.r
 		}
-		if p.bsp+utf8.UTFMax >= len(p.bs) {
+		if !utf8.FullRune(p.bs[p.bsp:]) {
-			// we might need up to 4 bytes to read a full
+			// we need more bytes to read a full non-ascii rune
-			// non-ascii rune
 			p.fill()
 		}
 		var w int
@@ -122,14 +120,18 @@ func (p *Parser) fill() {
 	p.offs += p.bsp
 	left := len(p.bs) - p.bsp
 	copy(p.readBuf[:left], p.readBuf[p.bsp:])
+readAgain:
 	n, err := 0, p.readErr
 	if err == nil {
 		n, err = p.src.Read(p.readBuf[left:])
 		p.readErr = err
 	}
 	if n == 0 {
+		if err == nil {
+			goto readAgain
+		}
 		// don't use p.errPass as we don't want to overwrite p.tok
-		if err != nil && err != io.EOF {
+		if err != io.EOF {
 			p.err = err
 		}
 		if left > 0 {
@@ -238,6 +240,7 @@ skipSpace:
 			return
 		}
 	}
+changedState:
 	p.pos = p.getPos()
 	switch {
 	case p.quote&allRegTokens != 0:
@@ -292,15 +295,21 @@ skipSpace:
 	case p.quote&allParamExp != 0 && paramOps(r):
 		p.tok = p.paramToken(r)
 	case p.quote == testRegexp:
+		if !p.rxFirstPart && p.spaced {
+			p.quote = noState
+			goto changedState
+		}
+		p.rxFirstPart = false
 		switch r {
 		case ';', '"', '\'', '$', '&', '>', '<', '`':
 			p.tok = p.regToken(r)
 		case ')':
-			if p.reOpenParens > 0 {
+			if p.rxOpenParens > 0 {
 				// continuation of open paren
 				p.advanceLitRe(r)
 			} else {
 				p.tok = rightParen
+				p.quote = noState
 			}
 		default: // including '(', '|'
 			p.advanceLitRe(r)
@@ -900,7 +909,6 @@ func (p *Parser) advanceLitHdoc(r rune) {
 	p.newLit(r)
 	if p.quote == hdocBodyTabs {
 		for r == '\t' {
-			p.discardLit(1)
 			r = p.rune()
 		}
 	}
@@ -916,7 +924,12 @@ func (p *Parser) advanceLitHdoc(r rune) {
 		case '\\': // escaped byte follows
 			p.rune()
 		case '\n', utf8.RuneSelf:
-			if bytes.HasPrefix(p.litBs[lStart:], p.hdocStop) {
+			if p.parsingDoc {
+				if r == utf8.RuneSelf {
+					p.val = p.endLit()
+					return
+				}
+			} else if bytes.HasPrefix(p.litBs[lStart:], p.hdocStop) {
 				p.val = p.endLit()[:lStart]
 				if p.val == "" {
 					p.tok = _Newl
@@ -930,7 +943,6 @@ func (p *Parser) advanceLitHdoc(r rune) {
 			if p.quote == hdocBodyTabs {
 				for p.peekByte('\t') {
 					p.rune()
-					p.discardLit(1)
 				}
 			}
 			lStart = len(p.litBs)
@@ -938,7 +950,7 @@ func (p *Parser) advanceLitHdoc(r rune) {
 	}
 }
 
-func (p *Parser) hdocLitWord() *Word {
+func (p *Parser) quotedHdocWord() *Word {
 	r := p.r
 	p.newLit(r)
 	pos := p.getPos()
@@ -948,7 +960,6 @@ func (p *Parser) hdocLitWord() *Word {
 		}
 		if p.quote == hdocBodyTabs {
 			for r == '\t' {
-				p.discardLit(1)
 				r = p.rune()
 			}
 		}
@@ -976,19 +987,25 @@ func (p *Parser) advanceLitRe(r rune) {
 		case '\\':
 			p.rune()
 		case '(':
-			p.reOpenParens++
+			p.rxOpenParens++
 		case ')':
-			if p.reOpenParens--; p.reOpenParens < 0 {
+			if p.rxOpenParens--; p.rxOpenParens < 0 {
 				p.tok, p.val = _LitWord, p.endLit()
+				p.quote = noState
 				return
 			}
 		case ' ', '\t', '\r', '\n':
-			if p.reOpenParens <= 0 {
+			if p.rxOpenParens <= 0 {
 				p.tok, p.val = _LitWord, p.endLit()
+				p.quote = noState
 				return
 			}
-		case utf8.RuneSelf, ';', '"', '\'', '$', '&', '>', '<', '`':
+		case '"', '\'', '$', '`':
+			p.tok, p.val = _Lit, p.endLit()
+			return
+		case utf8.RuneSelf, ';', '&', '>', '<':
 			p.tok, p.val = _LitWord, p.endLit()
+			p.quote = noState
 			return
 		}
 	}

vendor/mvdan.cc/sh/syntax/nodes.go

@@ -3,7 +3,10 @@
 
 package syntax
 
-import "fmt"
+import (
+	"fmt"
+	"strings"
+)
 
 // Node represents a syntax tree node.
 type Node interface {
@@ -243,7 +246,12 @@ func (r *Redirect) Pos() Pos {
 	}
 	return r.OpPos
 }
-func (r *Redirect) End() Pos { return r.Word.End() }
+func (r *Redirect) End() Pos {
+	if r.Hdoc != nil {
+		return r.Hdoc.End()
+	}
+	return r.Word.End()
+}
 
 // CallExpr represents a command execution or function call, otherwise known as
 // a "simple command".
@@ -289,6 +297,10 @@ type Block struct {
 func (b *Block) Pos() Pos { return b.Lbrace }
 func (b *Block) End() Pos { return posAddCol(b.Rbrace, 1) }
 
+// TODO(v3): Refactor and simplify elif/else. For example, we could likely make
+// Else an *IfClause, remove ElsePos, make IfPos also do opening "else"
+// positions, and join the comment slices as Last []Comment.
+
 // IfClause represents an if statement.
 type IfClause struct {
 	Elif    bool // whether this IfClause begins with "elif"
@@ -302,6 +314,7 @@ type IfClause struct {
 	Else StmtList
 
 	ElseComments []Comment // comments on the "else"
+	FiComments   []Comment // comments on the "fi"
 }
 
 func (c *IfClause) Pos() Pos { return c.IfPos }
@@ -363,14 +376,21 @@ func (*WordIter) loopNode()   {}
 func (*CStyleLoop) loopNode() {}
 
 // WordIter represents the iteration of a variable over a series of words in a
-// for clause.
+// for clause. If InPos is an invalid position, the "in" token was missing, so
+// the iteration is over the shell's positional parameters.
 type WordIter struct {
 	Name  *Lit
+	InPos Pos // position of "in"
 	Items []*Word
 }
 
 func (w *WordIter) Pos() Pos { return w.Name.Pos() }
-func (w *WordIter) End() Pos { return posMax(w.Name.End(), wordLastEnd(w.Items)) }
+func (w *WordIter) End() Pos {
+	if len(w.Items) > 0 {
+		return wordLastEnd(w.Items)
+	}
+	return posMax(w.Name.End(), posAddCol(w.InPos, 2))
+}
 
 // CStyleLoop represents the behaviour of a for clause similar to the C
 // language.
@@ -415,6 +435,28 @@ type Word struct {
 func (w *Word) Pos() Pos { return w.Parts[0].Pos() }
 func (w *Word) End() Pos { return w.Parts[len(w.Parts)-1].End() }
 
+// Lit returns the word as a literal value, if the word consists of *syntax.Lit
+// nodes only. An empty string is returned otherwise. Words with multiple
+// literals, which can appear in some edge cases, are handled properly.
+//
+// For example, the word "foo" will return "foo", but the word "foo${bar}" will
+// return "".
+func (w *Word) Lit() string {
+	// In the usual case, we'll have either a single part that's a literal,
+	// or one of the parts being a non-literal. Using strings.Join instead
+	// of a strings.Builder avoids extra work in these cases, since a single
+	// part is a shortcut, and many parts don't incur string copies.
+	lits := make([]string, 0, 1)
+	for _, part := range w.Parts {
+		lit, ok := part.(*Lit)
+		if !ok {
+			return ""
+		}
+		lits = append(lits, lit.Value)
+	}
+	return strings.Join(lits, "")
+}
+
 // WordPart represents all nodes that can form part of a word.
 //
 // These are *Lit, *SglQuoted, *DblQuoted, *ParamExp, *CmdSubst, *ArithmExp,
@@ -746,8 +788,12 @@ func (a *ArrayExpr) Pos() Pos { return a.Lparen }
 func (a *ArrayExpr) End() Pos { return posAddCol(a.Rparen, 1) }
 
 // ArrayElem represents a Bash array element.
+//
+// Index can be nil; for example, declare -a x=(value).
+// Value can be nil; for example, declare -A x=([index]=).
+// Finally, neither can be nil; for example, declare -A x=([index]=value)
 type ArrayElem struct {
-	Index    ArithmExpr // [i]=, ["k"]=
+	Index    ArithmExpr
 	Value    *Word
 	Comments []Comment
 }
@@ -758,7 +804,12 @@ func (a *ArrayElem) Pos() Pos {
 	}
 	return a.Value.Pos()
 }
-func (a *ArrayElem) End() Pos { return a.Value.End() }
+func (a *ArrayElem) End() Pos {
+	if a.Value != nil {
+		return a.Value.End()
+	}
+	return posAddCol(a.Index.Pos(), 1)
+}
 
 // ExtGlob represents a Bash extended globbing expression. Note that these are
 // parsed independently of whether shopt has been called or not.

vendor/mvdan.cc/sh/syntax/parser.go

@@ -113,6 +113,135 @@ func (p *Parser) Stmts(r io.Reader, fn func(*Stmt) bool) error {
 	return p.err
 }
 
+type wrappedReader struct {
+	*Parser
+	io.Reader
+
+	lastLine    uint16
+	accumulated []*Stmt
+	fn          func([]*Stmt) bool
+}
+
+func (w *wrappedReader) Read(p []byte) (n int, err error) {
+	// If we lexed a newline for the first time, we just finished a line, so
+	// we may need to give a callback for the edge cases below not covered
+	// by Parser.Stmts.
+	if w.r == '\n' && w.npos.line > w.lastLine {
+		if w.Incomplete() {
+			// Incomplete statement; call back to print "> ".
+			if !w.fn(w.accumulated) {
+				return 0, io.EOF
+			}
+		} else if len(w.accumulated) == 0 {
+			// Nothing was parsed; call back to print another "$ ".
+			if !w.fn(nil) {
+				return 0, io.EOF
+			}
+		}
+		w.lastLine = w.npos.line
+	}
+	return w.Reader.Read(p)
+}
+
+// Interactive implements what is necessary to parse statements in an
+// interactive shell. The parser will call the given function under two
+// circumstances outlined below.
+//
+// If a line containing any number of statements is parsed, the function will be
+// called with said statements.
+//
+// If a line ending in an incomplete statement is parsed, the function will be
+// called with any fully parsed statents, and Parser.Incomplete will return
+// true.
+//
+// One can imagine a simple interactive shell implementation as follows:
+//
+//         fmt.Fprintf(os.Stdout, "$ ")
+//         parser.Interactive(os.Stdin, func(stmts []*syntax.Stmt) bool {
+//                 if parser.Incomplete() {
+//                         fmt.Fprintf(os.Stdout, "> ")
+//                         return true
+//                 }
+//                 run(stmts)
+//                 fmt.Fprintf(os.Stdout, "$ ")
+//                 return true
+//         }
+//
+// If the callback function returns false, parsing is stopped and the function
+// is not called again.
+func (p *Parser) Interactive(r io.Reader, fn func([]*Stmt) bool) error {
+	w := wrappedReader{Parser: p, Reader: r, fn: fn}
+	return p.Stmts(&w, func(stmt *Stmt) bool {
+		w.accumulated = append(w.accumulated, stmt)
+		// We finished parsing a statement and we're at a newline token,
+		// so we finished fully parsing a number of statements. Call
+		// back to run the statements and print "$ ".
+		if p.tok == _Newl {
+			if !fn(w.accumulated) {
+				return false
+			}
+			w.accumulated = w.accumulated[:0]
+			// The callback above would already print "$ ", so we
+			// don't want the subsequent wrappedReader.Read to cause
+			// another "$ " print thinking that nothing was parsed.
+			w.lastLine = w.npos.line + 1
+		}
+		return true
+	})
+}
+
+// Words reads and parses words one at a time, calling a function each time one
+// is parsed. If the function returns false, parsing is stopped and the function
+// is not called again.
+//
+// Newlines are skipped, meaning that multi-line input will work fine. If the
+// parser encounters a token that isn't a word, such as a semicolon, an error
+// will be returned.
+//
+// Note that the lexer doesn't currently tokenize spaces, so it may need to read
+// a non-space byte such as a newline or a letter before finishing the parsing
+// of a word. This will be fixed in the future.
+func (p *Parser) Words(r io.Reader, fn func(*Word) bool) error {
+	p.reset()
+	p.f = &File{}
+	p.src = r
+	p.rune()
+	p.next()
+	for {
+		p.got(_Newl)
+		w := p.getWord()
+		if w == nil {
+			if p.tok != _EOF {
+				p.curErr("%s is not a valid word", p.tok)
+			}
+			return p.err
+		}
+		if !fn(w) {
+			return nil
+		}
+	}
+}
+
+// Document parses a single here-document word. That is, it parses the input as
+// if they were lines following a <<EOF redirection.
+//
+// In practice, this is the same as parsing the input as if it were within
+// double quotes, but without having to escape all double quote characters.
+// Similarly, the here-document word parsed here cannot be ended by any
+// delimiter other than reaching the end of the input.
+func (p *Parser) Document(r io.Reader) (*Word, error) {
+	p.reset()
+	p.f = &File{}
+	p.src = r
+	p.rune()
+	p.quote = hdocBody
+	p.hdocStop = []byte("MVDAN_CC_SH_SYNTAX_EOF")
+	p.parsingDoc = true
+	p.next()
+	w := p.getWord()
+	return w, p.err
+}
+
 // Parser holds the internal state of the parsing mechanism of a
 // program.
 type Parser struct {
@@ -150,18 +279,23 @@ type Parser struct {
 	buriedHdocs int
 	heredocs    []*Redirect
 	hdocStop    []byte
+	parsingDoc  bool
 
-	// openBquotes is how many levels of backquotes are open at the
+	// openStmts is how many entire statements we're currently parsing. A
-	// moment
+	// non-zero number means that we require certain tokens or words before
+	// reaching EOF.
+	openStmts int
+	// openBquotes is how many levels of backquotes are open at the moment.
 	openBquotes int
-	// lastBquoteEsc is how many times the last backquote token was
+
-	// escaped
+	// lastBquoteEsc is how many times the last backquote token was escaped
 	lastBquoteEsc int
-	// buriedBquotes is like openBquotes, but saved for when the
+	// buriedBquotes is like openBquotes, but saved for when the parser
-	// parser comes out of single quotes
+	// comes out of single quotes
 	buriedBquotes int
 
-	reOpenParens int
+	rxOpenParens int
+	rxFirstPart  bool
 
 	accComs []Comment
 	curComs *[]Comment
@@ -180,6 +314,14 @@ type Parser struct {
 	litBs   []byte
 }
 
+func (p *Parser) Incomplete() bool {
+	// If we're in a quote state other than noState, we're parsing a node
+	// such as a double-quoted string.
+	// If there are any open statements, we need to finish them.
+	// If we're constructing a literal, we need to finish it.
+	return p.quote != noState || p.openStmts > 0 || p.litBs != nil
+}
+
 const bufSize = 1 << 10
 
 func (p *Parser) reset() {
@@ -191,9 +333,10 @@ func (p *Parser) reset() {
 	p.r, p.w = 0, 0
 	p.err, p.readErr = nil, nil
 	p.quote, p.forbidNested = noState, false
+	p.openStmts = 0
 	p.heredocs, p.buriedHdocs = p.heredocs[:0], 0
+	p.parsingDoc = false
 	p.openBquotes, p.buriedBquotes = 0, 0
-	p.reOpenParens = 0
 	p.accComs, p.curComs = nil, &p.accComs
 }
 
@@ -270,6 +413,8 @@ func (p *Parser) call(w *Word) *CallExpr {
 	return ce
 }
 
+//go:generate stringer -type=quoteState
+
 type quoteState uint32
 
 const (
@@ -372,7 +517,7 @@ func (p *Parser) doHeredocs() {
 			p.rune()
 		}
 		if quoted {
-			r.Hdoc = p.hdocLitWord()
+			r.Hdoc = p.quotedHdocWord()
 		} else {
 			p.next()
 			r.Hdoc = p.getWord()
@@ -597,7 +742,9 @@ loop:
 		if p.tok == _EOF {
 			break
 		}
+		p.openStmts++
 		s := p.getStmt(true, false, false)
+		p.openStmts--
 		if s == nil {
 			p.invalidStmtStart()
 			break
@@ -619,7 +766,7 @@ func (p *Parser) stmtList(stops ...string) (sl StmtList) {
 	}
 	p.stmts(fn, stops...)
 	split := len(p.accComs)
-	if p.tok == _LitWord && (p.val == "elif" || p.val == "else") {
+	if p.tok == _LitWord && (p.val == "elif" || p.val == "else" || p.val == "fi") {
 		// Split the comments, so that any aligned with an opening token
 		// get attached to it. For example:
 		//
@@ -630,7 +777,7 @@ func (p *Parser) stmtList(stops ...string) (sl StmtList) {
 		//     fi
 		// TODO(mvdan): look into deduplicating this with similar logic
 		// in caseItems.
-		for i := len(p.accComs)-1; i >= 0; i-- {
+		for i := len(p.accComs) - 1; i >= 0; i-- {
 			c := p.accComs[i]
 			if c.Pos().Col() != p.pos.Col() {
 				break
@@ -856,6 +1003,11 @@ func (p *Parser) wordPart() WordPart {
 
 		p.next()
 		cs.StmtList = p.stmtList()
+		if p.tok == bckQuote && p.lastBquoteEsc < p.openBquotes-1 {
+			// e.g. found ` before the nested backquote \` was closed.
+			p.tok = _EOF
+			p.quoteErr(cs.Pos(), bckQuote)
+		}
 		p.postNested(old)
 		p.openBquotes--
 		cs.Right = p.pos
@@ -1250,15 +1402,8 @@ func (p *Parser) paramExp() *ParamExp {
 		default:
 			pe.Exp = p.paramExpExp()
 		}
-	case plus, colPlus, minus, colMinus, quest, colQuest, assgn, colAssgn:
+	case plus, colPlus, minus, colMinus, quest, colQuest, assgn, colAssgn,
-		// if unset/null actions
+		perc, dblPerc, hash, dblHash:
-		switch pe.Param.Value {
-		case "#", "$", "?", "!":
-			p.curErr("$%s can never be unset or null", pe.Param.Value)
-		}
-		pe.Exp = p.paramExpExp()
-	case perc, dblPerc, hash, dblHash:
-		// pattern string manipulation
 		pe.Exp = p.paramExpExp()
 	case _EOF:
 	default:
@@ -1333,6 +1478,9 @@ func (p *Parser) backquoteEnd() bool {
 
 // ValidName returns whether val is a valid name as per the POSIX spec.
 func ValidName(val string) bool {
+	if val == "" {
+		return false
+	}
 	for i, r := range val {
 		switch {
 		case 'a' <= r && r <= 'z':
@@ -1444,11 +1592,16 @@ func (p *Parser) getAssign(needEqual bool) *Assign {
 				p.follow(left, `"[x]"`, assgn)
 			}
 			if ae.Value = p.getWord(); ae.Value == nil {
-				if p.tok == leftParen {
+				switch p.tok {
+				case leftParen:
 					p.curErr("arrays cannot be nested")
+					return nil
+				case _Newl, rightParen, leftBrack:
+					// TODO: support [index]=[
+				default:
+					p.curErr("array element values must be words")
+					break
 				}
-				p.curErr("array element values must be words")
-				break
 			}
 			if len(p.accComs) > 0 {
 				c := p.accComs[0]
@@ -1797,6 +1950,8 @@ func (p *Parser) ifClause(s *Stmt) {
 		curIf.ElsePos = elsePos
 		curIf.Else = p.followStmts("else", curIf.ElsePos, "fi")
 	}
+	curIf.FiComments = p.accComs
+	p.accComs = nil
 	rif.FiPos = p.stmtEnd(rif, "if", "fi")
 	curIf.FiPos = rif.FiPos
 	s.Cmd = rif
@@ -1867,7 +2022,8 @@ func (p *Parser) wordIter(ftok string, fpos Pos) *WordIter {
 		return wi
 	}
 	p.got(_Newl)
-	if _, ok := p.gotRsrv("in"); ok {
+	if pos, ok := p.gotRsrv("in"); ok {
+		wi.InPos = pos
 		for !stopToken(p.tok) {
 			if w := p.getWord(); w == nil {
 				p.curErr("word list can only contain words")
@@ -1952,7 +2108,7 @@ func (p *Parser) caseItems(stop string) (items []*CaseItem) {
 		p.got(_Newl)
 		split := len(p.accComs)
 		if p.tok == _LitWord && p.val != stop {
-			for i := len(p.accComs)-1; i >= 0; i-- {
+			for i := len(p.accComs) - 1; i >= 0; i-- {
 				c := p.accComs[i]
 				if c.Pos().Col() != p.pos.Col() {
 					break
@@ -1982,6 +2138,7 @@ func (p *Parser) testClause(s *Stmt) {
 }
 
 func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
+	p.got(_Newl)
 	var left TestExpr
 	if pastAndOr {
 		left = p.testExprBase(ftok, fpos)
@@ -1991,6 +2148,7 @@ func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
 	if left == nil {
 		return left
 	}
+	p.got(_Newl)
 	switch p.tok {
 	case andAnd, orOr:
 	case _LitWord:
@@ -2015,10 +2173,12 @@ func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
 		Op:    BinTestOperator(p.tok),
 		X:     left,
 	}
+	// Save the previous quoteState, since we change it in TsReMatch.
+	oldQuote := p.quote
+
 	switch b.Op {
 	case AndTest, OrTest:
 		p.next()
-		p.got(_Newl)
 		if b.Y = p.testExpr(token(b.Op), b.OpPos, false); b.Y == nil {
 			p.followErrExp(b.OpPos, b.Op.String())
 		}
@@ -2026,12 +2186,12 @@ func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
 		if p.lang != LangBash {
 			p.langErr(p.pos, "regex tests", LangBash)
 		}
-		oldReOpenParens := p.reOpenParens
+		p.rxOpenParens = 0
-		old := p.preNested(testRegexp)
+		p.rxFirstPart = true
-		defer func() {
+		// TODO(mvdan): Using nested states within a regex will break in
-			p.postNested(old)
+		// all sorts of ways. The better fix is likely to use a stop
-			p.reOpenParens = oldReOpenParens
+		// token, like we do with heredocs.
-		}()
+		p.quote = testRegexp
 		fallthrough
 	default:
 		if _, ok := b.X.(*Word); !ok {
@@ -2041,6 +2201,7 @@ func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
 		p.next()
 		b.Y = p.followWordTok(token(b.Op), b.OpPos)
 	}
+	p.quote = oldQuote
 	return b
 }
 
@@ -2079,14 +2240,12 @@ func (p *Parser) testExprBase(ftok token, fpos Pos) TestExpr {
 	case leftParen:
 		pe := &ParenTest{Lparen: p.pos}
 		p.next()
-		p.got(_Newl)
 		if pe.X = p.testExpr(leftParen, pe.Lparen, false); pe.X == nil {
 			p.followErrExp(pe.Lparen, "(")
 		}
 		pe.Rparen = p.matched(pe.Lparen, leftParen, rightParen)
 		return pe
 	default:
-		p.got(_Newl)
 		return p.followWordTok(ftok, fpos)
 	}
 }