From 01876438c2f2c5493375019b7b033b8a67876a43 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 23 Feb 2020 15:01:25 -0800 Subject: [PATCH] shared container for job --- go.sum | 1 + pkg/common/executor.go | 9 + pkg/container/docker_pull.go | 2 +- pkg/container/docker_run.go | 228 ++++++++++++++++++---- pkg/model/workflow.go | 78 +++++++- pkg/runner/expression.go | 14 +- pkg/runner/run_context.go | 292 ++++++++++++++--------------- pkg/runner/runner.go | 64 +------ pkg/runner/step.go | 276 --------------------------- pkg/runner/testdata/basic/push.yml | 11 +- pkg/runner/testdata/node/push.yml | 1 - 11 files changed, 445 insertions(+), 531 deletions(-) delete mode 100644 pkg/runner/step.go diff --git a/go.sum b/go.sum index c747a1dc..1a0a56fd 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb h1:PyjxRdW1mqCmSoxy/6uP01P7CGbsD+woX+oOWbaUPwQ= github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= +github.com/docker/engine v1.13.1 h1:Cks33UT9YBW5Xyc3MtGDq2IPgqfJtJ+qkFaxc2b0Euc= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= diff --git a/pkg/common/executor.go b/pkg/common/executor.go index f16de41d..197cd5b3 100644 --- a/pkg/common/executor.go +++ b/pkg/common/executor.go @@ -40,6 +40,15 @@ func NewInfoExecutor(format string, args ...interface{}) Executor { } } +// NewDebugExecutor is an executor that logs messages +func NewDebugExecutor(format string, args ...interface{}) Executor { + return func(ctx context.Context) error { + logger := Logger(ctx) + logger.Debugf(format, args...) + return nil + } +} + // NewPipelineExecutor creates a new executor from a series of other executors func NewPipelineExecutor(executors ...Executor) Executor { if len(executors) == 0 { diff --git a/pkg/container/docker_pull.go b/pkg/container/docker_pull.go index 7aa9b893..69a2e2f4 100644 --- a/pkg/container/docker_pull.go +++ b/pkg/container/docker_pull.go @@ -21,7 +21,7 @@ type NewDockerPullExecutorInput struct { func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - logger.Infof("%sdocker pull %v", logPrefix, input.Image) + logger.Debugf("%sdocker pull %v", logPrefix, input.Image) if common.Dryrun(ctx) { return nil diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index b54ab875..63d83923 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -1,6 +1,8 @@ package container import ( + "archive/tar" + "bytes" "context" "fmt" "io" @@ -8,60 +10,119 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" "github.com/nektos/act/pkg/common" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh/terminal" ) -// NewDockerRunExecutorInput the input for the NewDockerRunExecutor function -type NewDockerRunExecutorInput struct { - Image string - Entrypoint []string - Cmd []string - WorkingDir string - Env []string - Binds []string - Content map[string]io.Reader - Volumes []string - Name string - ReuseContainers bool - Stdout io.Writer - Stderr io.Writer +// NewContainerInput the input for the New function +type NewContainerInput struct { + Image string + Entrypoint []string + Cmd []string + WorkingDir string + Env []string + Binds []string + Mounts map[string]string + Name string + Stdout io.Writer + Stderr io.Writer } -// NewDockerRunExecutor function to create a run executor for the container -func NewDockerRunExecutor(input NewDockerRunExecutorInput) common.Executor { +// FileEntry is a file to copy to a container +type FileEntry struct { + Name string + Mode int64 + Body string +} + +// Container for managing docker run containers +type Container interface { + Create() common.Executor + Copy(destPath string, files ...*FileEntry) common.Executor + Pull(forcePull bool) common.Executor + Start(attach bool) common.Executor + Exec(command []string, env map[string]string) common.Executor + Remove() common.Executor +} + +// NewContainer creates a reference to a container +func NewContainer(input *NewContainerInput) Container { cr := new(containerReference) cr.input = input + return cr +} +func (cr *containerReference) Create() common.Executor { return common. - NewInfoExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, input.Image, input.Entrypoint, input.Cmd). + NewDebugExecutor("%sdocker create image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). Then( common.NewPipelineExecutor( cr.connect(), cr.find(), - cr.remove().IfBool(!input.ReuseContainers), cr.create(), - cr.copyContent(), - cr.attach(), - cr.start(), - cr.wait(), - ).Finally( - cr.remove().IfBool(!input.ReuseContainers), ).IfNot(common.Dryrun), ) } +func (cr *containerReference) Start(attach bool) common.Executor { + return common. + NewDebugExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd). + Then( + common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.attach().IfBool(attach), + cr.start(), + cr.wait().IfBool(attach), + ).IfNot(common.Dryrun), + ) +} +func (cr *containerReference) Pull(forcePull bool) common.Executor { + return NewDockerPullExecutor(NewDockerPullExecutorInput{ + Image: cr.input.Image, + ForcePull: forcePull, + }) +} +func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.copyContent(destPath, files...), + ).IfNot(common.Dryrun) +} + +func (cr *containerReference) Exec(command []string, env map[string]string) common.Executor { + + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.exec(command, env), + ).IfNot(common.Dryrun) +} +func (cr *containerReference) Remove() common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + ).Finally( + cr.remove(), + ).IfNot(common.Dryrun) +} type containerReference struct { - input NewDockerRunExecutorInput cli *client.Client id string + input *NewContainerInput } func (cr *containerReference) connect() common.Executor { return func(ctx context.Context) error { + if cr.cli != nil { + return nil + } cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return errors.WithStack(err) @@ -74,6 +135,9 @@ func (cr *containerReference) connect() common.Executor { func (cr *containerReference) find() common.Executor { return func(ctx context.Context) error { + if cr.id != "" { + return nil + } containers, err := cr.cli.ContainerList(ctx, types.ContainerListOptions{ All: true, }) @@ -134,15 +198,18 @@ func (cr *containerReference) create() common.Executor { Tty: isTerminal, } - if len(input.Volumes) > 0 { - config.Volumes = make(map[string]struct{}) - for _, vol := range input.Volumes { - config.Volumes[vol] = struct{}{} - } + mounts := make([]mount.Mount, 0) + for mountSource, mountTarget := range input.Mounts { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: mountSource, + Target: mountTarget, + }) } resp, err := cr.cli.ContainerCreate(ctx, config, &container.HostConfig{ - Binds: input.Binds, + Binds: input.Binds, + Mounts: mounts, }, nil, input.Name) if err != nil { return errors.WithStack(err) @@ -155,15 +222,100 @@ func (cr *containerReference) create() common.Executor { } } -func (cr *containerReference) copyContent() common.Executor { +func (cr *containerReference) exec(cmd []string, env map[string]string) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - for dstPath, srcReader := range cr.input.Content { - logger.Debugf("Extracting content to '%s'", dstPath) - err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, srcReader, types.CopyToContainerOptions{}) - if err != nil { - return errors.WithStack(err) + logger.Debugf("Exec command '%s'", cmd) + isTerminal := terminal.IsTerminal(int(os.Stdout.Fd())) + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + + idResp, err := cr.cli.ContainerExecCreate(ctx, cr.id, types.ExecConfig{ + Cmd: cmd, + WorkingDir: cr.input.WorkingDir, + Env: envList, + Tty: isTerminal, + AttachStderr: true, + AttachStdout: true, + }) + if err != nil { + return errors.WithStack(err) + } + + resp, err := cr.cli.ContainerExecAttach(ctx, idResp.ID, types.ExecStartCheck{ + Tty: isTerminal, + }) + if err != nil { + return errors.WithStack(err) + } + var outWriter io.Writer + outWriter = cr.input.Stdout + if outWriter == nil { + outWriter = os.Stdout + } + errWriter := cr.input.Stderr + if errWriter == nil { + errWriter = os.Stderr + } + + err = cr.cli.ContainerExecStart(ctx, idResp.ID, types.ExecStartCheck{ + Tty: isTerminal, + }) + if err != nil { + return errors.WithStack(err) + } + + if !isTerminal || os.Getenv("NORAW") != "" { + _, err = stdcopy.StdCopy(outWriter, errWriter, resp.Reader) + } else { + _, err = io.Copy(outWriter, resp.Reader) + } + if err != nil { + logger.Error(err) + } + + inspectResp, err := cr.cli.ContainerExecInspect(ctx, idResp.ID) + if err != nil { + return errors.WithStack(err) + } + + if inspectResp.ExitCode == 0 { + return nil + } + + return fmt.Errorf("exit with `FAILURE`: %v", inspectResp.ExitCode) + } +} + +func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, file := range files { + log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body)) + hdr := &tar.Header{ + Name: file.Name, + Mode: file.Mode, + Size: int64(len(file.Body)), } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + + logger.Debugf("Extracting content to '%s'", dstPath) + err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, &buf, types.CopyToContainerOptions{}) + if err != nil { + return errors.WithStack(err) } return nil } @@ -207,7 +359,7 @@ func (cr *containerReference) attach() common.Executor { func (cr *containerReference) start() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) - logger.Debugf("STARTING image=%s entrypoint=%s cmd=%v", cr.input.Image, cr.input.Entrypoint, cr.input.Cmd) + logger.Debugf("Starting container: %v", cr.id) if err := cr.cli.ContainerStart(ctx, cr.id, types.ContainerStartOptions{}); err != nil { return errors.WithStack(err) diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index f0c2a462..33cf6e7f 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -3,10 +3,11 @@ package model import ( "fmt" "io" - "log" "regexp" "strings" + "github.com/nektos/act/pkg/common" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -94,6 +95,58 @@ func (j *Job) Needs() []string { return nil } +// GetMatrixes returns the matrix cross product +func (j *Job) GetMatrixes() []map[string]interface{} { + matrixes := make([]map[string]interface{}, 0) + if j.Strategy != nil { + includes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["include"] { + includes = append(includes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "include") + + excludes := make([]map[string]interface{}, 0) + for _, v := range j.Strategy.Matrix["exclude"] { + excludes = append(excludes, v.(map[string]interface{})) + } + delete(j.Strategy.Matrix, "exclude") + + matrixProduct := common.CartesianProduct(j.Strategy.Matrix) + + MATRIX: + for _, matrix := range matrixProduct { + for _, exclude := range excludes { + if commonKeysMatch(matrix, exclude) { + log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) + continue MATRIX + } + } + for _, include := range includes { + if commonKeysMatch(matrix, include) { + log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include) + for k, v := range include { + matrix[k] = v + } + } + } + matrixes = append(matrixes, matrix) + } + + } else { + matrixes = append(matrixes, make(map[string]interface{})) + } + return matrixes +} + +func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { + for aKey, aVal := range a { + if bVal, ok := b[aKey]; ok && aVal != bVal { + return false + } + } + return true +} + // ContainerSpec is the specification of the container to use for the job type ContainerSpec struct { Image string `yaml:"image"` @@ -148,6 +201,29 @@ func (s *Step) GetEnv() map[string]string { return rtnEnv } +// ShellCommand returns the command for the shell +func (s *Step) ShellCommand() string { + shellCommand := "" + + switch s.Shell { + case "", "bash": + shellCommand = "bash --noprofile --norc -eo pipefail {0}" + case "pwsh": + shellCommand = "pwsh -command \"& '{0}'\"" + case "python": + shellCommand = "python {0}" + case "sh": + shellCommand = "sh -e -c {0}" + case "cmd": + shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" + case "powershell": + shellCommand = "powershell -command \"& '{0}'\"" + default: + shellCommand = s.Shell + } + return shellCommand +} + // StepType describes what type of step we are about to run type StepType int diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 9226faf1..9de525c5 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -11,7 +11,6 @@ import ( "regexp" "strings" - "github.com/nektos/act/pkg/model" "github.com/robertkrimen/otto" "github.com/sirupsen/logrus" "gopkg.in/godo.v2/glob" @@ -34,11 +33,11 @@ func (rc *RunContext) NewExpressionEvaluator() ExpressionEvaluator { } } -// NewStepExpressionEvaluator creates a new evaluator -func (rc *RunContext) NewStepExpressionEvaluator(step *model.Step) ExpressionEvaluator { - vm := rc.newVM() +// NewExpressionEvaluator creates a new evaluator +func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator { + vm := sc.RunContext.newVM() configers := []func(*otto.Otto){ - rc.vmEnv(step), + sc.vmEnv(), } for _, configer := range configers { configer(vm) @@ -236,10 +235,9 @@ func (rc *RunContext) vmGithub() func(*otto.Otto) { } } -func (rc *RunContext) vmEnv(step *model.Step) func(*otto.Otto) { +func (sc *StepContext) vmEnv() func(*otto.Otto) { return func(vm *otto.Otto) { - env := rc.StepEnv(step) - _ = vm.Set("env", env) + _ = vm.Set("env", sc.Env) } } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 5a662d65..39b3737b 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -1,16 +1,12 @@ package runner import ( - "archive/tar" - "bytes" "context" "encoding/json" "fmt" - "io" - "io/ioutil" "os" + "path/filepath" "regexp" - "runtime" "strings" "github.com/nektos/act/pkg/container" @@ -23,16 +19,16 @@ import ( // RunContext contains info about current job type RunContext struct { - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - Tempdir string - ExtraPath []string - CurrentStep string - StepResults map[string]*stepResult - ExprEval ExpressionEvaluator + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*stepResult + ExprEval ExpressionEvaluator + JobContainer container.Container } type stepResult struct { @@ -48,80 +44,141 @@ func (rc *RunContext) GetEnv() map[string]string { return rc.Env } -// Close cleans up temp dir -func (rc *RunContext) Close(ctx context.Context) error { - return os.RemoveAll(rc.Tempdir) +func (rc *RunContext) jobContainerName() string { + return createContainerName(filepath.Base(rc.Config.Workdir), rc.Run.String()) +} + +func (rc *RunContext) startJobContainer() common.Executor { + job := rc.Run.Job() + + var image string + if job.Container != nil { + image = job.Container.Image + } else { + platformName := rc.ExprEval.Interpolate(job.RunsOn) + image = rc.Config.Platforms[strings.ToLower(platformName)] + } + + return func(ctx context.Context) error { + rawLogger := common.Logger(ctx).WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) { + if rc.Config.LogOutput { + rawLogger.Infof(s) + } else { + rawLogger.Debugf(s) + } + }) + + common.Logger(ctx).Infof("\U0001f680 Start image=%s", image) + name := rc.jobContainerName() + + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ + Cmd: nil, + Entrypoint: []string{"/bin/cat"}, + WorkingDir: "/github/workspace", + Image: image, + Name: name, + Mounts: map[string]string{ + name: "/github", + }, + Binds: []string{ + fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"), + fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), + }, + Stdout: logWriter, + Stderr: logWriter, + }) + + return common.NewPipelineExecutor( + rc.JobContainer.Pull(rc.Config.ForcePull), + rc.JobContainer.Remove().IfBool(!rc.Config.ReuseContainers), + rc.JobContainer.Create(), + rc.JobContainer.Start(false), + rc.JobContainer.Copy("/github/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 644, + Body: rc.EventJSON, + }), + )(ctx) + } +} +func (rc *RunContext) execJobContainer(cmd []string, env map[string]string) common.Executor { + return func(ctx context.Context) error { + return rc.JobContainer.Exec(cmd, env)(ctx) + } +} +func (rc *RunContext) stopJobContainer() common.Executor { + return func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false))(ctx) + } + return nil + } } // Executor returns a pipeline executor for all the steps in the job func (rc *RunContext) Executor() common.Executor { - - err := rc.setupTempDir() - if err != nil { - return common.NewErrorExecutor(err) - } steps := make([]common.Executor, 0) + steps = append(steps, rc.startJobContainer()) for i, step := range rc.Run.Job().Steps { if step.ID == "" { step.ID = fmt.Sprintf("%d", i) } - s := step - steps = append(steps, func(ctx context.Context) error { - rc.CurrentStep = s.ID - rc.StepResults[rc.CurrentStep] = &stepResult{ - Success: true, - Outputs: make(map[string]string), - } - rc.ExprEval = rc.NewStepExpressionEvaluator(s) + steps = append(steps, rc.newStepExecutor(step)) + } + steps = append(steps, rc.stopJobContainer()) - if !rc.EvalBool(s.If) { - log.Debugf("Skipping step '%s' due to '%s'", s.String(), s.If) - return nil - } + return common.NewPipelineExecutor(steps...).If(rc.isEnabled) +} - common.Logger(ctx).Infof("\u2B50 Run %s", s) - err := rc.newStepExecutor(s)(ctx) - if err == nil { - common.Logger(ctx).Infof(" \u2705 Success - %s", s) - } else { - common.Logger(ctx).Errorf(" \u274C Failure - %s", s) - rc.StepResults[rc.CurrentStep].Success = false - } - return err - }) +func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { + sc := &StepContext{ + RunContext: rc, + Step: step, } return func(ctx context.Context) error { - defer rc.Close(ctx) - job := rc.Run.Job() - log := common.Logger(ctx) - if !rc.EvalBool(job.If) { - log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If) + rc.CurrentStep = sc.Step.ID + rc.StepResults[rc.CurrentStep] = &stepResult{ + Success: true, + Outputs: make(map[string]string), + } + rc.ExprEval = sc.NewExpressionEvaluator() + + if !rc.EvalBool(sc.Step.If) { + log.Debugf("Skipping step '%s' due to '%s'", sc.Step.String(), sc.Step.If) return nil } - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" { - log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName) - return nil + common.Logger(ctx).Infof("\u2B50 Run %s", sc.Step) + err := sc.Executor()(ctx) + if err == nil { + common.Logger(ctx).Infof(" \u2705 Success - %s", sc.Step) + } else { + common.Logger(ctx).Errorf(" \u274C Failure - %s", sc.Step) + rc.StepResults[rc.CurrentStep].Success = false } - - nullLogger := logrus.New() - nullLogger.Out = ioutil.Discard - if !rc.Config.ReuseContainers { - _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) - } - - err := common.NewPipelineExecutor(steps...)(ctx) - - if !rc.Config.ReuseContainers { - _ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger)) - } - return err } } +func (rc *RunContext) isEnabled(ctx context.Context) bool { + job := rc.Run.Job() + log := common.Logger(ctx) + if !rc.EvalBool(job.If) { + log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If) + return false + } + + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" { + log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName) + return false + } + return true +} + // EvalBool evaluates an expression against current run context func (rc *RunContext) EvalBool(expr string) bool { if expr != "" { @@ -145,33 +202,7 @@ func mergeMaps(maps ...map[string]string) map[string]string { return rtnMap } -func (rc *RunContext) setupTempDir() error { - var err error - tempBase := "" - if runtime.GOOS == "darwin" { - tempBase = "/tmp" - } - rc.Tempdir, err = ioutil.TempDir(tempBase, "act-") - if err != nil { - return err - } - err = os.Chmod(rc.Tempdir, 0755) - if err != nil { - return err - } - log.Debugf("Setup tempdir %s", rc.Tempdir) - return err -} - -func (rc *RunContext) pullImage(containerSpec *model.ContainerSpec) common.Executor { - return func(ctx context.Context) error { - return container.NewDockerPullExecutor(container.NewDockerPullExecutorInput{ - Image: containerSpec.Image, - ForcePull: rc.Config.ForcePull, - })(ctx) - } -} - +/* func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Executor { return func(ctx context.Context) error { ghReader, err := rc.createGithubTarball() @@ -200,7 +231,7 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex } }) - return container.NewDockerRunExecutor(container.NewDockerRunExecutorInput{ + c := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, Image: containerSpec.Image, @@ -212,64 +243,27 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex fmt.Sprintf("%s:%s", rc.Tempdir, "/github/home"), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, - Content: map[string]io.Reader{"/github": ghReader}, - ReuseContainers: containerSpec.Reuse, - Stdout: logWriter, - Stderr: logWriter, - })(ctx) + Stdout: logWriter, + Stderr: logWriter, + }) + + return c.Create(). + Then(c.Copy("/github", ghReader)). + Then(c.Start()). + Finally(c.Remove().IfBool(!rc.Config.ReuseContainers))(ctx) + } } -func (rc *RunContext) createGithubTarball() (io.Reader, error) { - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - var files = []struct { - Name string - Mode int64 - Body string - }{ - {"workflow/event.json", 0644, rc.EventJSON}, +*/ + +func createContainerName(parts ...string) string { + name := make([]string, 0) + pattern := regexp.MustCompile("[^a-zA-Z0-9]") + for _, part := range parts { + name = append(name, pattern.ReplaceAllString(part, "-")) } - for _, file := range files { - log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(rc.EventJSON)) - hdr := &tar.Header{ - Name: file.Name, - Mode: file.Mode, - Size: int64(len(rc.EventJSON)), - } - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := tw.Write([]byte(rc.EventJSON)); err != nil { - return nil, err - } - } - if err := tw.Close(); err != nil { - return nil, err - } - - return &buf, nil - -} - -func (rc *RunContext) createContainerName() string { - containerName := rc.Run.String() - containerName = regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(containerName, "-") - - prefix := "" - suffix := "" - containerName = trimToLen(containerName, 30-(len(prefix)+len(suffix))) - return fmt.Sprintf("%s%s%s", prefix, containerName, suffix) - -} - -func (rc *RunContext) createStepContainerName(stepID string) string { - - prefix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(rc.createContainerName(), "-") - suffix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(stepID, "-") - prefix = trimToLen(prefix, 30-(1+len(suffix))) - name := strings.Trim(fmt.Sprintf("%s-%s", prefix, suffix), "-") - return name + return trimToLen(strings.Join(name, "-"), 30) } func trimToLen(s string, l int) string { diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 2c1888a4..6aa1eaf7 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -13,7 +13,6 @@ import ( // Runner provides capabilities to run GitHub actions type Runner interface { NewPlanExecutor(plan *model.Plan) common.Executor - NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor } // Config contains the config for a new runner @@ -59,49 +58,12 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { stageExecutor := make([]common.Executor, 0) for _, run := range stage.Runs { job := run.Job() - matrixes := make([]map[string]interface{}, 0) - if job.Strategy != nil { - includes := make([]map[string]interface{}, 0) - for _, v := range job.Strategy.Matrix["include"] { - includes = append(includes, v.(map[string]interface{})) - } - delete(job.Strategy.Matrix, "include") - - excludes := make([]map[string]interface{}, 0) - for _, v := range job.Strategy.Matrix["exclude"] { - excludes = append(excludes, v.(map[string]interface{})) - } - delete(job.Strategy.Matrix, "exclude") - - matrixProduct := common.CartesianProduct(job.Strategy.Matrix) - - MATRIX: - for _, matrix := range matrixProduct { - for _, exclude := range excludes { - if commonKeysMatch(matrix, exclude) { - log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) - continue MATRIX - } - } - for _, include := range includes { - if commonKeysMatch(matrix, include) { - log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include) - for k, v := range include { - matrix[k] = v - } - } - } - matrixes = append(matrixes, matrix) - } - - } else { - matrixes = append(matrixes, make(map[string]interface{})) - } + matrixes := job.GetMatrixes() jobName := fmt.Sprintf("%-*s", maxJobNameLen, run.String()) for _, matrix := range matrixes { m := matrix - runExecutor := runner.NewRunExecutor(run, matrix) + runExecutor := runner.newRunExecutor(run, matrix) stageExecutor = append(stageExecutor, func(ctx context.Context) error { ctx = WithJobLogger(ctx, jobName) if len(m) > 0 { @@ -117,22 +79,14 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { return common.NewPipelineExecutor(pipeline...) } -func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { - for aKey, aVal := range a { - if bVal, ok := b[aKey]; ok && aVal != bVal { - return false - } +func (runner *runnerImpl) newRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor { + rc := &RunContext{ + Config: runner.config, + Run: run, + EventJSON: runner.eventJSON, + StepResults: make(map[string]*stepResult), + Matrix: matrix, } - return true -} - -func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor { - rc := new(RunContext) - rc.Config = runner.config - rc.Run = run - rc.EventJSON = runner.eventJSON - rc.StepResults = make(map[string]*stepResult) - rc.Matrix = matrix rc.ExprEval = rc.NewExpressionEvaluator() return rc.Executor() } diff --git a/pkg/runner/step.go b/pkg/runner/step.go deleted file mode 100644 index e5cf61c4..00000000 --- a/pkg/runner/step.go +++ /dev/null @@ -1,276 +0,0 @@ -package runner - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/container" - "github.com/nektos/act/pkg/model" - log "github.com/sirupsen/logrus" -) - -func (rc *RunContext) StepEnv(step *model.Step) map[string]string { - var env map[string]string - job := rc.Run.Job() - if job.Container != nil { - env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv()) - } else { - env = mergeMaps(rc.GetEnv(), step.GetEnv()) - } - - for k, v := range env { - env[k] = rc.ExprEval.Interpolate(v) - } - return env -} - -func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { - return func(ctx context.Context) error { - containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step)) - return nil - } -} - -func (rc *RunContext) newContainerCleaner() common.Executor { - job := rc.Run.Job() - containerSpec := new(model.ContainerSpec) - containerSpec.Name = rc.createContainerName() - containerSpec.Reuse = false - - if job.Container != nil { - containerSpec.Image = job.Container.Image - } else { - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] - } - containerSpec.Entrypoint = "bash --noprofile --norc -o pipefail -c echo 'cleaning up'" - return common.NewPipelineExecutor( - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) -} - -func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { - job := rc.Run.Job() - containerSpec := new(model.ContainerSpec) - containerSpec.Name = rc.createContainerName() - containerSpec.Reuse = true - - if job.Container != nil { - containerSpec.Image = job.Container.Image - } else { - platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) - containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)] - } - - switch step.Type() { - case model.StepTypeRun: - if job.Container != nil { - containerSpec.Ports = job.Container.Ports - containerSpec.Volumes = job.Container.Volumes - containerSpec.Options = job.Container.Options - } - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.setupShellCommand(containerSpec, step.Shell, step.Run), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - - case model.StepTypeUsesDockerURL: - containerSpec.Image = strings.TrimPrefix(step.Uses, "docker://") - containerSpec.Name = rc.createStepContainerName(step.ID) - containerSpec.Entrypoint = step.With["entrypoint"] - containerSpec.Args = step.With["args"] - containerSpec.Reuse = rc.Config.ReuseContainers - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - - case model.StepTypeUsesActionLocal: - return common.NewPipelineExecutor( - rc.setupEnv(containerSpec, step), - rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - case model.StepTypeUsesActionRemote: - remoteAction := newRemoteAction(step.Uses) - if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" { - return func(ctx context.Context) error { - common.Logger(ctx).Debugf("Skipping actions/checkout") - return nil - } - } - cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo) - if err != nil { - return common.NewErrorExecutor(err) - } - return common.NewPipelineExecutor( - common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ - URL: remoteAction.CloneURL(), - Ref: remoteAction.Ref, - Dir: cloneDir, - }), - rc.setupEnv(containerSpec, step), - rc.setupAction(containerSpec, filepath.Join(cloneDir, remoteAction.Path)), - applyWith(containerSpec, step), - rc.pullImage(containerSpec), - rc.runContainer(containerSpec), - ) - } - - return common.NewErrorExecutor(fmt.Errorf("Unable to determine how to run job:%s step:%+v", rc.Run, step)) -} - -func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { - return func(ctx context.Context) error { - if entrypoint, ok := step.With["entrypoint"]; ok { - containerSpec.Entrypoint = entrypoint - } - if args, ok := step.With["args"]; ok { - containerSpec.Args = args - } - return nil - } -} - -func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shell string, run string) common.Executor { - return func(ctx context.Context) error { - shellCommand := "" - - switch shell { - case "", "bash": - shellCommand = "bash --noprofile --norc -eo pipefail {0}" - case "pwsh": - shellCommand = "pwsh -command \"& '{0}'\"" - case "python": - shellCommand = "python {0}" - case "sh": - shellCommand = "sh -e -c {0}" - case "cmd": - shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" - case "powershell": - shellCommand = "powershell -command \"& '{0}'\"" - default: - shellCommand = shell - } - - tempScript, err := ioutil.TempFile(rc.Tempdir, ".temp-script-") - if err != nil { - return err - } - - _, err = tempScript.WriteString(fmt.Sprintf("PATH=\"%s:${PATH}\"\n", strings.Join(rc.ExtraPath, ":"))) - if err != nil { - return err - } - - run = rc.ExprEval.Interpolate(run) - - if _, err := tempScript.WriteString(run); err != nil { - return err - } - log.Debugf("Wrote command '%s' to '%s'", run, tempScript.Name()) - if err := tempScript.Close(); err != nil { - return err - } - containerPath := fmt.Sprintf("/github/home/%s", filepath.Base(tempScript.Name())) - containerSpec.Entrypoint = strings.Replace(shellCommand, "{0}", containerPath, 1) - return nil - } -} - -func (rc *RunContext) setupAction(containerSpec *model.ContainerSpec, actionDir string) common.Executor { - return func(ctx context.Context) error { - f, err := os.Open(filepath.Join(actionDir, "action.yml")) - if os.IsNotExist(err) { - f, err = os.Open(filepath.Join(actionDir, "action.yaml")) - if err != nil { - return err - } - } else if err != nil { - return err - } - - action, err := model.ReadAction(f) - if err != nil { - return err - } - - for inputID, input := range action.Inputs { - envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_") - envKey = fmt.Sprintf("INPUT_%s", envKey) - if _, ok := containerSpec.Env[envKey]; !ok { - containerSpec.Env[envKey] = input.Default - } - } - - switch action.Runs.Using { - case model.ActionRunsUsingNode12: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) - } - case model.ActionRunsUsingDocker: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) - } - containerSpec.Reuse = rc.Config.ReuseContainers - if strings.HasPrefix(action.Runs.Image, "docker://") { - containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") - containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") - containerSpec.Args = strings.Join(action.Runs.Args, " ") - } else { - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") - contextDir := filepath.Join(actionDir, action.Runs.Main) - return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ - ContextDir: contextDir, - ImageTag: containerSpec.Image, - })(ctx) - } - } - return nil - } -} - -type remoteAction struct { - Org string - Repo string - Path string - Ref string -} - -func (ra *remoteAction) CloneURL() string { - return fmt.Sprintf("https://github.com/%s/%s", ra.Org, ra.Repo) -} - -func newRemoteAction(action string) *remoteAction { - r := regexp.MustCompile(`^([^/@]+)/([^/@]+)(/([^@]*))?(@(.*))?$`) - matches := r.FindStringSubmatch(action) - - ra := new(remoteAction) - ra.Org = matches[1] - ra.Repo = matches[2] - ra.Path = "" - ra.Ref = "master" - if len(matches) >= 5 { - ra.Path = matches[4] - } - if len(matches) >= 7 { - ra.Ref = matches[6] - } - return ra -} diff --git a/pkg/runner/testdata/basic/push.yml b/pkg/runner/testdata/basic/push.yml index 720ca259..50c6553a 100644 --- a/pkg/runner/testdata/basic/push.yml +++ b/pkg/runner/testdata/basic/push.yml @@ -5,9 +5,12 @@ jobs: check: runs-on: ubuntu-latest steps: - - run: echo 'hello world' + - run: echo 'hello world' + - run: echo ${GITHUB_SHA} >> /github/sha.txt + - run: cat /github/sha.txt | grep ${GITHUB_SHA} build: + if: false runs-on: ubuntu-latest needs: [check] steps: @@ -20,4 +23,8 @@ jobs: steps: - uses: docker://ubuntu:18.04 with: - args: echo ${GITHUB_REF} | grep nektos/act + args: env + - uses: docker://ubuntu:18.04 + with: + entrypoint: /bin/echo + args: ${{github.event_name}} diff --git a/pkg/runner/testdata/node/push.yml b/pkg/runner/testdata/node/push.yml index 0eebcf42..e8a284d3 100644 --- a/pkg/runner/testdata/node/push.yml +++ b/pkg/runner/testdata/node/push.yml @@ -6,7 +6,6 @@ jobs: test: runs-on: ubuntu-latest steps: - - run: which node - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: