From 4b93866d0fc3f7edc5ae8d8e0c4ea849aaa04b57 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 07:39:11 -0400 Subject: [PATCH 01/16] wip: Initial docker container works --- .circleci/{pack.sh => pack} | 0 .dockerignore | 3 +++ .gitignore | 1 + Dockerfile.dev | 27 +++++++++++++++++++++++++++ Makefile | 6 +++--- docker-compose.yml | 13 +++++++++++++ git/git_test.go | 1 + 7 files changed, 48 insertions(+), 3 deletions(-) rename .circleci/{pack.sh => pack} (100%) create mode 100644 .dockerignore create mode 100644 Dockerfile.dev create mode 100644 docker-compose.yml diff --git a/.circleci/pack.sh b/.circleci/pack similarity index 100% rename from .circleci/pack.sh rename to .circleci/pack diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..642d2e2e9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +Dockerfile* +docker-compose*.yml diff --git a/.gitignore b/.gitignore index fe774dfed..9eb2a154d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ packrd/ bin/golangci-lint integration_tests/tmp *.exe +/issues diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..ece7c73e2 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,27 @@ +FROM golang:1.12.17-buster@sha256:d9d06cb747d49db5d5703610ecdef837c0432efcaccc4a5959155dd61fe42f4a AS deps + +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /project + +COPY go.mod go.sum ./ + +RUN go mod download + +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends make bash && \ + rm -rf /var/lib/apt/lists/* + +COPY . ./ + +RUN make + +ENTRYPOINT ["make"] + +FROM golang:1.12.17-buster@sha256:d9d06cb747d49db5d5703610ecdef837c0432efcaccc4a5959155dd61fe42f4a + +COPY --from=deps /project/build/linux/amd64/circleci /usr/local/bin/circleci + +ENTRYPOINT ["circleci"] diff --git a/Makefile b/Makefile index f50603470..1bbbe12e9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) build: always - GO111MODULE=on .circleci/pack.sh + .circleci/pack go build -o build/$(GOOS)/$(GOARCH)/circleci build-all: build/linux/amd64/circleci build/darwin/amd64/circleci @@ -16,7 +16,7 @@ build/%/amd64/circleci: always clean: GO111MODULE=off go clean -i rm -rf build out docs dist - .circleci/pack.sh clean + .circleci/pack clean .PHONY: test test: @@ -40,7 +40,7 @@ install-packr: .PHONY: pack pack: - bash .circleci/pack.sh + bash .circleci/pack .PHONY: install-lint install-lint: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..23204fc15 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.8' +services: + dev: &dev + build: + context: . + dockerfile: Dockerfile.dev + interactive: + <<: *dev + volumes: + - .:/project + test: + <<: *dev + command: ['test'] diff --git a/git/git_test.go b/git/git_test.go index 334bfbd18..62ef2b75e 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" ) +// TODO These are broken when not run from a git repo var _ = Describe("Dealing with git", func() { Context("running commands", func() { From 3859d35b99008f82f3c4ba51b64c0f3e3624badf Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 09:34:29 -0400 Subject: [PATCH 02/16] wip: Further dockerization --- Dockerfile.dev | 30 +++++++++++++++++++++++++++++- bin/dev-entrypoint | 5 +++++ docker-compose.yml | 8 ++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 bin/dev-entrypoint diff --git a/Dockerfile.dev b/Dockerfile.dev index ece7c73e2..5faacba09 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -22,6 +22,34 @@ ENTRYPOINT ["make"] FROM golang:1.12.17-buster@sha256:d9d06cb747d49db5d5703610ecdef837c0432efcaccc4a5959155dd61fe42f4a +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common \ + lxc \ + iptables && \ + curl -fsSL https://download.docker.com/linux/debian/gpg | \ + APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + apt-get -qq update && \ + apt-get -qq install --no-install-recommends docker-ce && \ + add-apt-repository --remove "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + apt-get -qq purge apt-transport-https \ + curl \ + gnupg-agent \ + software-properties-common && \ + rm -rf /var/lib/apt/list/* + +VOLUME /var/lib/docker + COPY --from=deps /project/build/linux/amd64/circleci /usr/local/bin/circleci -ENTRYPOINT ["circleci"] +COPY bin/dev-entrypoint /usr/local/bin/ + +RUN chmod +x /usr/local/bin/dev-entrypoint + +ENV CIRCLECI_CLI_SKIP_UPDATE_CHECK true + +ENTRYPOINT ["dev-entrypoint"] diff --git a/bin/dev-entrypoint b/bin/dev-entrypoint new file mode 100644 index 000000000..5d2b8a4e6 --- /dev/null +++ b/bin/dev-entrypoint @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +dockerd & + +circleci "$@" diff --git a/docker-compose.yml b/docker-compose.yml index 23204fc15..7377962b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,14 @@ services: build: context: . dockerfile: Dockerfile.dev + # HERE docker-ce is installed on the image and daemon appears to start + # but picard fails to be found. + # + # Downloading latest CircleCI build agent... + # INFO[2020-10-07T13:27:18.978990600Z] Attempting next endpoint for pull after error: failed to register layer: ApplyLayer exit status 1 stdout: stderr: Error creating mount namespace before pivot: operation not permitted + # Error: Could not find picard image: failed to pull latest docker image: exit status 1 + cap_add: + - NET_ADMIN interactive: <<: *dev volumes: From febd2153ad523bf12364424daf151e4ab02a3cb8 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 09:37:00 -0400 Subject: [PATCH 03/16] [revert me] Trying to emit environment. --- local/local.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/local/local.go b/local/local.go index 2141400d4..6c6365466 100644 --- a/local/local.go +++ b/local/local.go @@ -10,6 +10,8 @@ import ( "path" "regexp" "syscall" + "log" + "strings" "github.com/CircleCI-Public/circleci-cli/api" "github.com/CircleCI-Public/circleci-cli/api/graphql" @@ -97,6 +99,13 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { return errors.Wrap(err, "Could not find a `docker` executable on $PATH; please ensure that docker installed") } + for _, e := range os.Environ() { + pair := strings.SplitN(e, "=", 2) + fmt.Fprintf(os.Stderr, "%s=%s", pair[0], pair[1]) + } + + log.Fatal("Boom") + err = syscall.Exec(dockerPath, arguments, os.Environ()) // #nosec return errors.Wrap(err, "failed to execute docker") } From 3c50f3371e2e2f02d231f81997147be1f9807fac Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 17:01:30 -0400 Subject: [PATCH 04/16] Install docker-ce-cli in runtime container --- Dockerfile.dev | 14 +++----------- docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 5faacba09..527876bd4 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -27,14 +27,12 @@ RUN apt-get -qq update && \ ca-certificates \ curl \ gnupg-agent \ - software-properties-common \ - lxc \ - iptables && \ + software-properties-common && \ curl -fsSL https://download.docker.com/linux/debian/gpg | \ APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true apt-key add - && \ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -qq update && \ - apt-get -qq install --no-install-recommends docker-ce && \ + apt-get -qq install --no-install-recommends docker-ce-cli && \ add-apt-repository --remove "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ apt-get -qq purge apt-transport-https \ curl \ @@ -42,14 +40,8 @@ RUN apt-get -qq update && \ software-properties-common && \ rm -rf /var/lib/apt/list/* -VOLUME /var/lib/docker - COPY --from=deps /project/build/linux/amd64/circleci /usr/local/bin/circleci -COPY bin/dev-entrypoint /usr/local/bin/ - -RUN chmod +x /usr/local/bin/dev-entrypoint - ENV CIRCLECI_CLI_SKIP_UPDATE_CHECK true -ENTRYPOINT ["dev-entrypoint"] +ENTRYPOINT ["circleci"] diff --git a/docker-compose.yml b/docker-compose.yml index 7377962b2..579c9b9da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: # Downloading latest CircleCI build agent... # INFO[2020-10-07T13:27:18.978990600Z] Attempting next endpoint for pull after error: failed to register layer: ApplyLayer exit status 1 stdout: stderr: Error creating mount namespace before pivot: operation not permitted # Error: Could not find picard image: failed to pull latest docker image: exit status 1 - cap_add: - - NET_ADMIN + volumes: + - /var/run/docker.sock:/var/run/docker.sock interactive: <<: *dev volumes: From 90a9545a1c75c572723337541da274560cc87496 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 17:23:40 -0400 Subject: [PATCH 05/16] wip: Got all the way to trying to run docker but failed with not being able to share in /root/.circleci because Docker for Mac. Just use Vagrant. --- circleci.tmp.yml | 7 +++++++ docker-compose.yml | 17 ++++++++++------- local/local.go | 7 ++----- 3 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 circleci.tmp.yml diff --git a/circleci.tmp.yml b/circleci.tmp.yml new file mode 100644 index 000000000..ce8606cbb --- /dev/null +++ b/circleci.tmp.yml @@ -0,0 +1,7 @@ +version: '2.1' +jobs: + build: + docker: + - image: bash + steps: + - run: echo "$CHARNOCK" diff --git a/docker-compose.yml b/docker-compose.yml index 579c9b9da..863f04801 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,18 +4,21 @@ services: build: context: . dockerfile: Dockerfile.dev - # HERE docker-ce is installed on the image and daemon appears to start - # but picard fails to be found. - # - # Downloading latest CircleCI build agent... - # INFO[2020-10-07T13:27:18.978990600Z] Attempting next endpoint for pull after error: failed to register layer: ApplyLayer exit status 1 stdout: stderr: Error creating mount namespace before pivot: operation not permitted - # Error: Could not find picard image: failed to pull latest docker image: exit status 1 + # HERE Most of the way there but now we're in trouble because we're + # running Docker for Mac. I think we should just forget it and go with + # a Vagrantfile that sets up docker. volumes: + - ~/.circleci:/root/.circleci - /var/run/docker.sock:/var/run/docker.sock interactive: <<: *dev + working_dir: /tmp/project + entrypoint: '' + command: bash volumes: - - .:/project + - ~/.circleci:/root/.circleci + - /var/run/docker.sock:/var/run/docker.sock + - .:/tmp/project test: <<: *dev command: ['test'] diff --git a/local/local.go b/local/local.go index 6c6365466..8f7afdeb0 100644 --- a/local/local.go +++ b/local/local.go @@ -10,7 +10,6 @@ import ( "path" "regexp" "syscall" - "log" "strings" "github.com/CircleCI-Public/circleci-cli/api" @@ -89,7 +88,7 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { arguments := generateDockerCommand(processedConfigPath, image, pwd, processedArgs...) if cfg.Debug { - _, err = fmt.Fprintf(os.Stderr, "Starting docker with args: %s", arguments) + _, err = fmt.Fprintf(os.Stderr, "Starting docker with args: %s\n", arguments) if err != nil { return err } @@ -101,11 +100,9 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { for _, e := range os.Environ() { pair := strings.SplitN(e, "=", 2) - fmt.Fprintf(os.Stderr, "%s=%s", pair[0], pair[1]) + fmt.Fprintf(os.Stderr, "%s=%s\n", pair[0], pair[1]) } - log.Fatal("Boom") - err = syscall.Exec(dockerPath, arguments, os.Environ()) // #nosec return errors.Wrap(err, "failed to execute docker") } From 3410e659cb0dd9ef1748f7ba4c5f937e58c3a138 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Wed, 7 Oct 2020 19:19:23 -0400 Subject: [PATCH 06/16] Still broken. :( Error: failed to start event processor: failed to compute task config: failed to read config file: read /tmp/local_build_config.yml: is a directory --- .dockerignore | 1 + .gitignore | 2 ++ Vagrantfile | 18 ++++++++++++++++++ bin/provision | 44 ++++++++++++++++++++++++++++++++++++++++++++ local/local.go | 6 ------ 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 Vagrantfile create mode 100755 bin/provision diff --git a/.dockerignore b/.dockerignore index 642d2e2e9..abea34be9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ .git Dockerfile* docker-compose*.yml +.vagrant diff --git a/.gitignore b/.gitignore index 9eb2a154d..450328771 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ bin/golangci-lint integration_tests/tmp *.exe /issues +/.vagrant +/bash_history diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..5a1d86399 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,18 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'etc' + +Vagrant.require_version ">= 2.0.0", "< 3.0.0" + +Vagrant.configure("2") do |config| + config.vm.box = "bento/ubuntu-20.04" + + config.vm.provider "virtualbox" do |vb| + vb.memory = "2000" + vb.cpus = Etc.nprocessors + end + + config.vm.synced_folder "~/.circleci", '/home/vagrant/.circleci' + config.vm.provision "shell", path: "bin/provision" +end diff --git a/bin/provision b/bin/provision new file mode 100755 index 000000000..4047bb505 --- /dev/null +++ b/bin/provision @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +apt_update() { + if [[ -z $apt_updated ]] + then + apt-get -qq update && + apt_updated=true || exit + fi +} + +if ! sudo -u vagrant docker run --rm hello-world +then + apt_update && + apt-get -qq install apt-transport-https ca-certificates curl software-properties-common && + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" && + apt-get -qq update && + apt-get -qq install docker-ce && + usermod -aG docker vagrant && + sudo -u vagrant docker run --rm hello-world || + exit +fi + +if ! docker-compose --version +then + curl --silent --show-error --fail --location \ + "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" \ + -o /usr/local/bin/docker-compose && + chmod +x /usr/local/bin/docker-compose && + docker-compose --version || + exit +fi + +if ! grep '^cd /vagrant$' ~vagrant/.bashrc +then + sudo -u vagrant <<<'cd /vagrant' tee -a ~vagrant/.bashrc || + exit +fi + +if ! grep '^HISTFILE=/vagrant/bash_history$' ~vagrant/.bashrc +then + sudo -u vagrant tee -a <<<'HISTFILE=/vagrant/bash_history' ~vagrant/.bashrc || + exit +fi diff --git a/local/local.go b/local/local.go index 8f7afdeb0..a511de324 100644 --- a/local/local.go +++ b/local/local.go @@ -10,7 +10,6 @@ import ( "path" "regexp" "syscall" - "strings" "github.com/CircleCI-Public/circleci-cli/api" "github.com/CircleCI-Public/circleci-cli/api/graphql" @@ -98,11 +97,6 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { return errors.Wrap(err, "Could not find a `docker` executable on $PATH; please ensure that docker installed") } - for _, e := range os.Environ() { - pair := strings.SplitN(e, "=", 2) - fmt.Fprintf(os.Stderr, "%s=%s\n", pair[0], pair[1]) - } - err = syscall.Exec(dockerPath, arguments, os.Environ()) // #nosec return errors.Wrap(err, "failed to execute docker") } From 372d1579e733c4e0a246f51cb27e00ddfd215419 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 06:49:35 -0400 Subject: [PATCH 07/16] Install golang on VM --- bin/provision | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/provision b/bin/provision index 4047bb505..dc32a60da 100755 --- a/bin/provision +++ b/bin/provision @@ -42,3 +42,13 @@ then sudo -u vagrant tee -a <<<'HISTFILE=/vagrant/bash_history' ~vagrant/.bashrc || exit fi + +if ! sudo -u vagrant -i go version +then + curl --silent --show-error --fail --location \ + "https://golang.org/dl/go1.15.2.linux-amd64.tar.gz" | + tar -C /usr/local -xz && + sudo -u vagrant tee -a <<<'export PATH=$PATH:/usr/local/go/bin' ~vagrant/.bash_profile && + sudo -u vagrant -i go version || + exit 1 +fi From c1794469d64b3f6b16b919a5c5f8c233e790ab85 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 07:14:58 -0400 Subject: [PATCH 08/16] wip --- circleci.tmp.yml | 2 ++ local/local.go | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/circleci.tmp.yml b/circleci.tmp.yml index ce8606cbb..58e6cab6c 100644 --- a/circleci.tmp.yml +++ b/circleci.tmp.yml @@ -5,3 +5,5 @@ jobs: - image: bash steps: - run: echo "$CHARNOCK" + - run: >- + while date; do sleep 60; done diff --git a/local/local.go b/local/local.go index a511de324..e4c42e386 100644 --- a/local/local.go +++ b/local/local.go @@ -10,6 +10,7 @@ import ( "path" "regexp" "syscall" + "strings" "github.com/CircleCI-Public/circleci-cli/api" "github.com/CircleCI-Public/circleci-cli/api/graphql" @@ -97,6 +98,11 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { return errors.Wrap(err, "Could not find a `docker` executable on $PATH; please ensure that docker installed") } + for _, e := range os.Environ() { + pair := strings.SplitN(e, "=", 2) + fmt.Fprintf(os.Stderr, "%s=%s\n", pair[0], pair[1]) + } + err = syscall.Exec(dockerPath, arguments, os.Environ()) // #nosec return errors.Wrap(err, "failed to execute docker") } @@ -118,7 +124,7 @@ func AddFlagsForDocumentation(flags *pflag.FlagSet) { flags.String("revision", "", "Git Revision") flags.String("branch", "", "Git branch") flags.String("repo-url", "", "Git Url") - flags.StringArrayP("env", "e", nil, "Set environment variables, e.g. `-e VAR=VAL`") + flags.StringArrayP("env", "e", nil, "Set environment variables, e.g. `-e VAR=VAL` or `-e VAR`") } // Given the full set of flags that were passed to this command, return the path @@ -295,6 +301,8 @@ func generateDockerCommand(configPath, image, pwd string, arguments ...string) [ "--volume", fmt.Sprintf("%s:%s", pwd, pwd), "--volume", fmt.Sprintf("%s:/root/.circleci", settings.SettingsPath()), "--workdir", pwd, + // HERE env args needs to go here _before_ image … + // HERE volumes appear not to be being passed? image, "circleci", "build", "--config", configPathInsideContainer} return append(core, arguments...) } From a571303191350d5eb2f737d7af3d26266d89c40f Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:13:12 -0400 Subject: [PATCH 09/16] Add resolution of environment variables to local execute --- Makefile | 2 +- bin/provision | 8 ++++++++ local/local.go | 40 ++++++++++++++++++++++++++++++++++------ local/local_test.go | 16 +++++++++++----- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 1bbbe12e9..eb84095b4 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ clean: .PHONY: test test: - go test -v ./... + test_env_var=test_env_var_value go test -v ./... .PHONY: cover cover: diff --git a/bin/provision b/bin/provision index dc32a60da..e95b452f2 100755 --- a/bin/provision +++ b/bin/provision @@ -52,3 +52,11 @@ then sudo -u vagrant -i go version || exit 1 fi + +if ! dpkg-query -s build-essential +then + apt_update && + apt-get -qq install build-essential && + dpkg-query -s build-essential || + exit +fi diff --git a/local/local.go b/local/local.go index e4c42e386..d61b401ba 100644 --- a/local/local.go +++ b/local/local.go @@ -42,7 +42,11 @@ func UpdateBuildAgent() error { func Execute(flags *pflag.FlagSet, cfg *settings.Config) error { - processedArgs, configPath := buildAgentArguments(flags) + processedArgs, configPath, err := buildAgentArguments(flags) + if err != nil { + return err + } + cl := graphql.NewClient(cfg.Host, cfg.Endpoint, cfg.Token, cfg.Debug) configResponse, err := api.ConfigQuery(cl, configPath, pipeline.FabricatedValues()) @@ -135,21 +139,33 @@ func AddFlagsForDocumentation(flags *pflag.FlagSet) { // GraphQL API, and feed the result of that into `build-agent`. The first step of // that process is to find the local path to the config file. This is supplied with // the `config` flag. -func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { +func buildAgentArguments(flags *pflag.FlagSet) ([]string, string, error) { var result []string = []string{} + var outerErr error // build a list of all supplied flags, that we will pass on to build-agent flags.Visit(func(flag *pflag.Flag) { if flag.Name != "config" && flag.Name != "debug" { - result = append(result, unparseFlag(flags, flag)...) + unparsedFlag, err := unparseFlag(flags, flag) + + if err != nil { + outerErr = errors.Wrap(err, "Failed to build agent arguments") + } + + result = append(result, unparsedFlag...) } }) + + if outerErr != nil { + return nil, "", outerErr + } + result = append(result, flags.Args()...) configPath, _ := flags.GetString("config") - return result, configPath + return result, configPath, nil } func picardImage(output io.Writer) (string, error) { @@ -310,7 +326,7 @@ func generateDockerCommand(configPath, image, pwd string, arguments ...string) [ // Convert the given flag back into a list of strings suitable to be passed on // the command line to run docker. // https://github.com/CircleCI-Public/circleci-cli/issues/391 -func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) []string { +func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) ([]string, error) { flagName := "--" + flag.Name result := []string{} switch flag.Value.Type() { @@ -318,10 +334,22 @@ func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) []string { // `--foo 1 --foo 2` will result in a single `foo` flag with an array of values. case "stringArray": for _, value := range flag.Value.(pflag.SliceValue).GetSlice() { + if flag.Name == "env" && ! strings.Contains(value, "=") { + // Bare env arg passed. Resolve from environment. + variableValue, ok := os.LookupEnv(value) + if !ok { + return nil, errors.New(fmt.Sprintf( + "Failed to resolve environment variable ‘%s’ in the environment.\n", + value, + )) + } else { + value = fmt.Sprintf("%s=%s", value, variableValue) + } + } result = append(result, flagName, value) } default: result = append(result, flagName, flag.Value.String()) } - return result + return result, nil } diff --git a/local/local_test.go b/local/local_test.go index 7dc4b5e42..73b208462 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -67,7 +67,7 @@ var _ = Describe("build", func() { if testCase.expectedError != "" { Expect(err).To(MatchError(testCase.expectedError)) } - args, configPath := buildAgentArguments(flags) + args, configPath, err := buildAgentArguments(flags) Expect(args).To(Equal(testCase.expectedArgs)) Expect(configPath).To(Equal(testCase.expectedConfigPath)) @@ -98,15 +98,21 @@ var _ = Describe("build", func() { }), Entry("many args, multiple envs", TestCase{ - input: []string{"--env", "foo", "--env", "bar", "--env", "baz"}, + input: []string{"--env", "foo=foo", "--env", "bar=bar", "--env", "baz=baz"}, expectedConfigPath: ".circleci/config.yml", - expectedArgs: []string{"--env", "foo", "--env", "bar", "--env", "baz"}, + expectedArgs: []string{"--env", "foo=foo", "--env", "bar=bar", "--env", "baz=baz"}, }), Entry("comma in env value (issue #440)", TestCase{ - input: []string{"--env", "{\"json\":[\"like\",\"value\"]}"}, + input: []string{"--env", "JSON={\"json\":[\"like\",\"value\"]}"}, expectedConfigPath: ".circleci/config.yml", - expectedArgs: []string{"--env", "{\"json\":[\"like\",\"value\"]}"}, + expectedArgs: []string{"--env", "JSON={\"json\":[\"like\",\"value\"]}"}, + }), + + Entry("bare env value (issue #440)", TestCase{ + input: []string{"--env", "test_env_var"}, + expectedConfigPath: ".circleci/config.yml", + expectedArgs: []string{"--env", "test_env_var=test_env_var_value"}, }), Entry("args that are not flags", TestCase{ From a7de3d9e64bc05d3c3589e299b0e05487fea48b8 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:13:48 -0400 Subject: [PATCH 10/16] wip --- circleci.tmp.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 circleci.tmp.yml diff --git a/circleci.tmp.yml b/circleci.tmp.yml deleted file mode 100644 index 58e6cab6c..000000000 --- a/circleci.tmp.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2.1' -jobs: - build: - docker: - - image: bash - steps: - - run: echo "$CHARNOCK" - - run: >- - while date; do sleep 60; done From 76983635dba61bfd3349be493dbaef4982845ad4 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:20:32 -0400 Subject: [PATCH 11/16] wip --- Dockerfile.dev | 47 ---------------------------------------------- docker-compose.yml | 24 ----------------------- local/local.go | 2 -- 3 files changed, 73 deletions(-) delete mode 100644 Dockerfile.dev delete mode 100644 docker-compose.yml diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 527876bd4..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,47 +0,0 @@ -FROM golang:1.12.17-buster@sha256:d9d06cb747d49db5d5703610ecdef837c0432efcaccc4a5959155dd61fe42f4a AS deps - -RUN apt-get -qq update && \ - apt-get -qq install --no-install-recommends git && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /project - -COPY go.mod go.sum ./ - -RUN go mod download - -RUN apt-get -qq update && \ - apt-get -qq install --no-install-recommends make bash && \ - rm -rf /var/lib/apt/lists/* - -COPY . ./ - -RUN make - -ENTRYPOINT ["make"] - -FROM golang:1.12.17-buster@sha256:d9d06cb747d49db5d5703610ecdef837c0432efcaccc4a5959155dd61fe42f4a - -RUN apt-get -qq update && \ - apt-get -qq install --no-install-recommends apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common && \ - curl -fsSL https://download.docker.com/linux/debian/gpg | \ - APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true apt-key add - && \ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ - apt-get -qq update && \ - apt-get -qq install --no-install-recommends docker-ce-cli && \ - add-apt-repository --remove "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ - apt-get -qq purge apt-transport-https \ - curl \ - gnupg-agent \ - software-properties-common && \ - rm -rf /var/lib/apt/list/* - -COPY --from=deps /project/build/linux/amd64/circleci /usr/local/bin/circleci - -ENV CIRCLECI_CLI_SKIP_UPDATE_CHECK true - -ENTRYPOINT ["circleci"] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 863f04801..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: '3.8' -services: - dev: &dev - build: - context: . - dockerfile: Dockerfile.dev - # HERE Most of the way there but now we're in trouble because we're - # running Docker for Mac. I think we should just forget it and go with - # a Vagrantfile that sets up docker. - volumes: - - ~/.circleci:/root/.circleci - - /var/run/docker.sock:/var/run/docker.sock - interactive: - <<: *dev - working_dir: /tmp/project - entrypoint: '' - command: bash - volumes: - - ~/.circleci:/root/.circleci - - /var/run/docker.sock:/var/run/docker.sock - - .:/tmp/project - test: - <<: *dev - command: ['test'] diff --git a/local/local.go b/local/local.go index d61b401ba..e21136b7f 100644 --- a/local/local.go +++ b/local/local.go @@ -317,8 +317,6 @@ func generateDockerCommand(configPath, image, pwd string, arguments ...string) [ "--volume", fmt.Sprintf("%s:%s", pwd, pwd), "--volume", fmt.Sprintf("%s:/root/.circleci", settings.SettingsPath()), "--workdir", pwd, - // HERE env args needs to go here _before_ image … - // HERE volumes appear not to be being passed? image, "circleci", "build", "--config", configPathInsideContainer} return append(core, arguments...) } From c1282e7efb1a6a11761a3ac1d531b27cc9e9966b Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:22:05 -0400 Subject: [PATCH 12/16] wip --- git/git_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/git/git_test.go b/git/git_test.go index 62ef2b75e..334bfbd18 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -8,7 +8,6 @@ import ( . "github.com/onsi/gomega" ) -// TODO These are broken when not run from a git repo var _ = Describe("Dealing with git", func() { Context("running commands", func() { From 1ac147b9e78f356bc4411e4340045d438571c9f0 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:30:39 -0400 Subject: [PATCH 13/16] wip --- bin/dev-entrypoint | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 bin/dev-entrypoint diff --git a/bin/dev-entrypoint b/bin/dev-entrypoint deleted file mode 100644 index 5d2b8a4e6..000000000 --- a/bin/dev-entrypoint +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env sh - -dockerd & - -circleci "$@" From 779497202c649f4693daf6439be2b8839e168781 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:47:15 -0400 Subject: [PATCH 14/16] Fix windows tests --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a1e2c702..43fe490ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,6 +100,8 @@ jobs: - run: mkdir test_results - run: name: Run tests + environment: + test_env_var: test_env_var_value command: | C:\Users\circleci\go\bin\gotestsum.exe --junitfile test_results/windows.xml - store_test_results: From 4384100f6a304bcace7047b928841372e2489049 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:50:01 -0400 Subject: [PATCH 15/16] Add test_env_var to cover target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eb84095b4..5a17e2da9 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test: .PHONY: cover cover: - go test -race -coverprofile=coverage.txt ./... + test_env_var=test_env_var_value go test -race -coverprofile=coverage.txt ./... .PHONY: lint lint: From 10f8e3e5db9b495764c79ee0642fbdc1f02999c8 Mon Sep 17 00:00:00 2001 From: Tim Visher Date: Thu, 8 Oct 2020 10:56:38 -0400 Subject: [PATCH 16/16] Fix lint --- local/local_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/local_test.go b/local/local_test.go index 73b208462..522f1ab3a 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -67,7 +67,7 @@ var _ = Describe("build", func() { if testCase.expectedError != "" { Expect(err).To(MatchError(testCase.expectedError)) } - args, configPath, err := buildAgentArguments(flags) + args, configPath, _ := buildAgentArguments(flags) Expect(args).To(Equal(testCase.expectedArgs)) Expect(configPath).To(Equal(testCase.expectedConfigPath))