p79813206
/
act
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

shared container for job

This commit is contained in:
Casey Lee 2020-02-23 15:01:25 -08:00
parent 9179d8924d
commit 01876438c2
No known key found for this signature in database
GPG Key ID: 1899120ECD0A1784
11 changed files with 445 additions and 531 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.sum
View File

@ -23,6 +23,7 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb h1:PyjxRdW1mqCmSoxy/6uP01P7CGbsD+woX+oOWbaUPwQ=
github.com/docker/engine v0.0.0-20181106193140-f5749085e9cb/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
github.com/docker/engine v1.13.1 h1:Cks33UT9YBW5Xyc3MtGDq2IPgqfJtJ+qkFaxc2b0Euc=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=

9
pkg/common/executor.go
View File

@ -40,6 +40,15 @@ func NewInfoExecutor(format string, args ...interface{}) Executor {
}
}
// NewDebugExecutor is an executor that logs messages
func NewDebugExecutor(format string, args ...interface{}) Executor {
return func(ctx context.Context) error {
logger := Logger(ctx)
logger.Debugf(format, args...)
return nil
}
}
// NewPipelineExecutor creates a new executor from a series of other executors
func NewPipelineExecutor(executors ...Executor) Executor {
if len(executors) == 0 {

2
pkg/container/docker_pull.go
View File

@ -21,7 +21,7 @@ type NewDockerPullExecutorInput struct {
func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
logger.Infof("%sdocker pull %v", logPrefix, input.Image)
logger.Debugf("%sdocker pull %v", logPrefix, input.Image)
if common.Dryrun(ctx) {
return nil

228
pkg/container/docker_run.go
View File

@ -1,6 +1,8 @@
package container
import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
@ -8,60 +10,119 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/stdcopy"
"github.com/nektos/act/pkg/common"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh/terminal"
)
// NewDockerRunExecutorInput the input for the NewDockerRunExecutor function
type NewDockerRunExecutorInput struct {
Image string
Entrypoint []string
Cmd []string
WorkingDir string
Env []string
Binds []string
Content map[string]io.Reader
Volumes []string
Name string
ReuseContainers bool
Stdout io.Writer
Stderr io.Writer
// NewContainerInput the input for the New function
type NewContainerInput struct {
Image string
Entrypoint []string
Cmd []string
WorkingDir string
Env []string
Binds []string
Mounts map[string]string
Name string
Stdout io.Writer
Stderr io.Writer
}
// NewDockerRunExecutor function to create a run executor for the container
func NewDockerRunExecutor(input NewDockerRunExecutorInput) common.Executor {
// FileEntry is a file to copy to a container
type FileEntry struct {
Name string
Mode int64
Body string
}
// Container for managing docker run containers
type Container interface {
Create() common.Executor
Copy(destPath string, files ...*FileEntry) common.Executor
Pull(forcePull bool) common.Executor
Start(attach bool) common.Executor
Exec(command []string, env map[string]string) common.Executor
Remove() common.Executor
}
// NewContainer creates a reference to a container
func NewContainer(input *NewContainerInput) Container {
cr := new(containerReference)
cr.input = input
return cr
}
func (cr *containerReference) Create() common.Executor {
return common.
NewInfoExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, input.Image, input.Entrypoint, input.Cmd).
NewDebugExecutor("%sdocker create image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd).
Then(
common.NewPipelineExecutor(
cr.connect(),
cr.find(),
cr.remove().IfBool(!input.ReuseContainers),
cr.create(),
cr.copyContent(),
cr.attach(),
cr.start(),
cr.wait(),
).Finally(
cr.remove().IfBool(!input.ReuseContainers),
).IfNot(common.Dryrun),
)
}
func (cr *containerReference) Start(attach bool) common.Executor {
return common.
NewDebugExecutor("%sdocker run image=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Entrypoint, cr.input.Cmd).
Then(
common.NewPipelineExecutor(
cr.connect(),
cr.find(),
cr.attach().IfBool(attach),
cr.start(),
cr.wait().IfBool(attach),
).IfNot(common.Dryrun),
)
}
func (cr *containerReference) Pull(forcePull bool) common.Executor {
return NewDockerPullExecutor(NewDockerPullExecutorInput{
Image: cr.input.Image,
ForcePull: forcePull,
})
}
func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common.Executor {
return common.NewPipelineExecutor(
cr.connect(),
cr.find(),
cr.copyContent(destPath, files...),
).IfNot(common.Dryrun)
}
func (cr *containerReference) Exec(command []string, env map[string]string) common.Executor {
return common.NewPipelineExecutor(
cr.connect(),
cr.find(),
cr.exec(command, env),
).IfNot(common.Dryrun)
}
func (cr *containerReference) Remove() common.Executor {
return common.NewPipelineExecutor(
cr.connect(),
cr.find(),
).Finally(
cr.remove(),
).IfNot(common.Dryrun)
}
type containerReference struct {
input NewDockerRunExecutorInput
cli *client.Client
id string
input *NewContainerInput
}
func (cr *containerReference) connect() common.Executor {
return func(ctx context.Context) error {
if cr.cli != nil {
return nil
}
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
return errors.WithStack(err)
@ -74,6 +135,9 @@ func (cr *containerReference) connect() common.Executor {
func (cr *containerReference) find() common.Executor {
return func(ctx context.Context) error {
if cr.id != "" {
return nil
}
containers, err := cr.cli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
@ -134,15 +198,18 @@ func (cr *containerReference) create() common.Executor {
Tty: isTerminal,
}
if len(input.Volumes) > 0 {
config.Volumes = make(map[string]struct{})
for _, vol := range input.Volumes {
config.Volumes[vol] = struct{}{}
}
mounts := make([]mount.Mount, 0)
for mountSource, mountTarget := range input.Mounts {
mounts = append(mounts, mount.Mount{
Type: mount.TypeVolume,
Source: mountSource,
Target: mountTarget,
})
}
resp, err := cr.cli.ContainerCreate(ctx, config, &container.HostConfig{
Binds: input.Binds,
Binds: input.Binds,
Mounts: mounts,
}, nil, input.Name)
if err != nil {
return errors.WithStack(err)
@ -155,15 +222,100 @@ func (cr *containerReference) create() common.Executor {
}
}
func (cr *containerReference) copyContent() common.Executor {
func (cr *containerReference) exec(cmd []string, env map[string]string) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
for dstPath, srcReader := range cr.input.Content {
logger.Debugf("Extracting content to '%s'", dstPath)
err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, srcReader, types.CopyToContainerOptions{})
if err != nil {
return errors.WithStack(err)
logger.Debugf("Exec command '%s'", cmd)
isTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
envList := make([]string, 0)
for k, v := range env {
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
}
idResp, err := cr.cli.ContainerExecCreate(ctx, cr.id, types.ExecConfig{
Cmd: cmd,
WorkingDir: cr.input.WorkingDir,
Env: envList,
Tty: isTerminal,
AttachStderr: true,
AttachStdout: true,
})
if err != nil {
return errors.WithStack(err)
}
resp, err := cr.cli.ContainerExecAttach(ctx, idResp.ID, types.ExecStartCheck{
Tty: isTerminal,
})
if err != nil {
return errors.WithStack(err)
}
var outWriter io.Writer
outWriter = cr.input.Stdout
if outWriter == nil {
outWriter = os.Stdout
}
errWriter := cr.input.Stderr
if errWriter == nil {
errWriter = os.Stderr
}
err = cr.cli.ContainerExecStart(ctx, idResp.ID, types.ExecStartCheck{
Tty: isTerminal,
})
if err != nil {
return errors.WithStack(err)
}
if !isTerminal || os.Getenv("NORAW") != "" {
_, err = stdcopy.StdCopy(outWriter, errWriter, resp.Reader)
} else {
_, err = io.Copy(outWriter, resp.Reader)
}
if err != nil {
logger.Error(err)
}
inspectResp, err := cr.cli.ContainerExecInspect(ctx, idResp.ID)
if err != nil {
return errors.WithStack(err)
}
if inspectResp.ExitCode == 0 {
return nil
}
return fmt.Errorf("exit with `FAILURE`: %v", inspectResp.ExitCode)
}
}
func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for _, file := range files {
log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
hdr := &tar.Header{
Name: file.Name,
Mode: file.Mode,
Size: int64(len(file.Body)),
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if _, err := tw.Write([]byte(file.Body)); err != nil {
return err
}
}
if err := tw.Close(); err != nil {
return err
}
logger.Debugf("Extracting content to '%s'", dstPath)
err := cr.cli.CopyToContainer(ctx, cr.id, dstPath, &buf, types.CopyToContainerOptions{})
if err != nil {
return errors.WithStack(err)
}
return nil
}
@ -207,7 +359,7 @@ func (cr *containerReference) attach() common.Executor {
func (cr *containerReference) start() common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
logger.Debugf("STARTING image=%s entrypoint=%s cmd=%v", cr.input.Image, cr.input.Entrypoint, cr.input.Cmd)
logger.Debugf("Starting container: %v", cr.id)
if err := cr.cli.ContainerStart(ctx, cr.id, types.ContainerStartOptions{}); err != nil {
return errors.WithStack(err)

78
pkg/model/workflow.go
View File

@ -3,10 +3,11 @@ package model
import (
"fmt"
"io"
"log"
"regexp"
"strings"
"github.com/nektos/act/pkg/common"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
@ -94,6 +95,58 @@ func (j *Job) Needs() []string {
return nil
}
// GetMatrixes returns the matrix cross product
func (j *Job) GetMatrixes() []map[string]interface{} {
matrixes := make([]map[string]interface{}, 0)
if j.Strategy != nil {
includes := make([]map[string]interface{}, 0)
for _, v := range j.Strategy.Matrix["include"] {
includes = append(includes, v.(map[string]interface{}))
}
delete(j.Strategy.Matrix, "include")
excludes := make([]map[string]interface{}, 0)
for _, v := range j.Strategy.Matrix["exclude"] {
excludes = append(excludes, v.(map[string]interface{}))
}
delete(j.Strategy.Matrix, "exclude")
matrixProduct := common.CartesianProduct(j.Strategy.Matrix)
MATRIX:
for _, matrix := range matrixProduct {
for _, exclude := range excludes {
if commonKeysMatch(matrix, exclude) {
log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude)
continue MATRIX
}
}
for _, include := range includes {
if commonKeysMatch(matrix, include) {
log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include)
for k, v := range include {
matrix[k] = v
}
}
}
matrixes = append(matrixes, matrix)
}
} else {
matrixes = append(matrixes, make(map[string]interface{}))
}
return matrixes
}
func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool {
for aKey, aVal := range a {
if bVal, ok := b[aKey]; ok && aVal != bVal {
return false
}
}
return true
}
// ContainerSpec is the specification of the container to use for the job
type ContainerSpec struct {
Image string `yaml:"image"`
@ -148,6 +201,29 @@ func (s *Step) GetEnv() map[string]string {
return rtnEnv
}
// ShellCommand returns the command for the shell
func (s *Step) ShellCommand() string {
shellCommand := ""
switch s.Shell {
case "", "bash":
shellCommand = "bash --noprofile --norc -eo pipefail {0}"
case "pwsh":
shellCommand = "pwsh -command \"& '{0}'\""
case "python":
shellCommand = "python {0}"
case "sh":
shellCommand = "sh -e -c {0}"
case "cmd":
shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\""
case "powershell":
shellCommand = "powershell -command \"& '{0}'\""
default:
shellCommand = s.Shell
}
return shellCommand
}
// StepType describes what type of step we are about to run
type StepType int

14
pkg/runner/expression.go
View File

@ -11,7 +11,6 @@ import (
"regexp"
"strings"
"github.com/nektos/act/pkg/model"
"github.com/robertkrimen/otto"
"github.com/sirupsen/logrus"
"gopkg.in/godo.v2/glob"
@ -34,11 +33,11 @@ func (rc *RunContext) NewExpressionEvaluator() ExpressionEvaluator {
}
}
// NewStepExpressionEvaluator creates a new evaluator
func (rc *RunContext) NewStepExpressionEvaluator(step *model.Step) ExpressionEvaluator {
vm := rc.newVM()
// NewExpressionEvaluator creates a new evaluator
func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator {
vm := sc.RunContext.newVM()
configers := []func(*otto.Otto){
rc.vmEnv(step),
sc.vmEnv(),
}
for _, configer := range configers {
configer(vm)
@ -236,10 +235,9 @@ func (rc *RunContext) vmGithub() func(*otto.Otto) {
}
}
func (rc *RunContext) vmEnv(step *model.Step) func(*otto.Otto) {
func (sc *StepContext) vmEnv() func(*otto.Otto) {
return func(vm *otto.Otto) {
env := rc.StepEnv(step)
_ = vm.Set("env", env)
_ = vm.Set("env", sc.Env)
}
}

292
pkg/runner/run_context.go
View File

@ -1,16 +1,12 @@
package runner
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"github.com/nektos/act/pkg/container"
@ -23,16 +19,16 @@ import (
// RunContext contains info about current job
type RunContext struct {
Config *Config
Matrix map[string]interface{}
Run *model.Run
EventJSON string
Env map[string]string
Tempdir string
ExtraPath []string
CurrentStep string
StepResults map[string]*stepResult
ExprEval ExpressionEvaluator
Config *Config
Matrix map[string]interface{}
Run *model.Run
EventJSON string
Env map[string]string
ExtraPath []string
CurrentStep string
StepResults map[string]*stepResult
ExprEval ExpressionEvaluator
JobContainer container.Container
}
type stepResult struct {
@ -48,80 +44,141 @@ func (rc *RunContext) GetEnv() map[string]string {
return rc.Env
}
// Close cleans up temp dir
func (rc *RunContext) Close(ctx context.Context) error {
return os.RemoveAll(rc.Tempdir)
func (rc *RunContext) jobContainerName() string {
return createContainerName(filepath.Base(rc.Config.Workdir), rc.Run.String())
}
func (rc *RunContext) startJobContainer() common.Executor {
job := rc.Run.Job()
var image string
if job.Container != nil {
image = job.Container.Image
} else {
platformName := rc.ExprEval.Interpolate(job.RunsOn)
image = rc.Config.Platforms[strings.ToLower(platformName)]
}
return func(ctx context.Context) error {
rawLogger := common.Logger(ctx).WithField("raw_output", true)
logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) {
if rc.Config.LogOutput {
rawLogger.Infof(s)
} else {
rawLogger.Debugf(s)
}
})
common.Logger(ctx).Infof("\U0001f680 Start image=%s", image)
name := rc.jobContainerName()
rc.JobContainer = container.NewContainer(&container.NewContainerInput{
Cmd: nil,
Entrypoint: []string{"/bin/cat"},
WorkingDir: "/github/workspace",
Image: image,
Name: name,
Mounts: map[string]string{
name: "/github",
},
Binds: []string{
fmt.Sprintf("%s:%s", rc.Config.Workdir, "/github/workspace"),
fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"),
},
Stdout: logWriter,
Stderr: logWriter,
})
return common.NewPipelineExecutor(
rc.JobContainer.Pull(rc.Config.ForcePull),
rc.JobContainer.Remove().IfBool(!rc.Config.ReuseContainers),
rc.JobContainer.Create(),
rc.JobContainer.Start(false),
rc.JobContainer.Copy("/github/", &container.FileEntry{
Name: "workflow/event.json",
Mode: 644,
Body: rc.EventJSON,
}),
)(ctx)
}
}
func (rc *RunContext) execJobContainer(cmd []string, env map[string]string) common.Executor {
return func(ctx context.Context) error {
return rc.JobContainer.Exec(cmd, env)(ctx)
}
}
func (rc *RunContext) stopJobContainer() common.Executor {
return func(ctx context.Context) error {
if rc.JobContainer != nil && !rc.Config.ReuseContainers {
return rc.JobContainer.Remove().
Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false))(ctx)
}
return nil
}
}
// Executor returns a pipeline executor for all the steps in the job
func (rc *RunContext) Executor() common.Executor {
err := rc.setupTempDir()
if err != nil {
return common.NewErrorExecutor(err)
}
steps := make([]common.Executor, 0)
steps = append(steps, rc.startJobContainer())
for i, step := range rc.Run.Job().Steps {
if step.ID == "" {
step.ID = fmt.Sprintf("%d", i)
}
s := step
steps = append(steps, func(ctx context.Context) error {
rc.CurrentStep = s.ID
rc.StepResults[rc.CurrentStep] = &stepResult{
Success: true,
Outputs: make(map[string]string),
}
rc.ExprEval = rc.NewStepExpressionEvaluator(s)
steps = append(steps, rc.newStepExecutor(step))
}
steps = append(steps, rc.stopJobContainer())
if !rc.EvalBool(s.If) {
log.Debugf("Skipping step '%s' due to '%s'", s.String(), s.If)
return nil
}
return common.NewPipelineExecutor(steps...).If(rc.isEnabled)
}
common.Logger(ctx).Infof("\u2B50 Run %s", s)
err := rc.newStepExecutor(s)(ctx)
if err == nil {
common.Logger(ctx).Infof(" \u2705 Success - %s", s)
} else {
common.Logger(ctx).Errorf(" \u274C Failure - %s", s)
rc.StepResults[rc.CurrentStep].Success = false
}
return err
})
func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
sc := &StepContext{
RunContext: rc,
Step: step,
}
return func(ctx context.Context) error {
defer rc.Close(ctx)
job := rc.Run.Job()
log := common.Logger(ctx)
if !rc.EvalBool(job.If) {
log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If)
rc.CurrentStep = sc.Step.ID
rc.StepResults[rc.CurrentStep] = &stepResult{
Success: true,
Outputs: make(map[string]string),
}
rc.ExprEval = sc.NewExpressionEvaluator()
if !rc.EvalBool(sc.Step.If) {
log.Debugf("Skipping step '%s' due to '%s'", sc.Step.String(), sc.Step.If)
return nil
}
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" {
log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName)
return nil
common.Logger(ctx).Infof("\u2B50 Run %s", sc.Step)
err := sc.Executor()(ctx)
if err == nil {
common.Logger(ctx).Infof(" \u2705 Success - %s", sc.Step)
} else {
common.Logger(ctx).Errorf(" \u274C Failure - %s", sc.Step)
rc.StepResults[rc.CurrentStep].Success = false
}
nullLogger := logrus.New()
nullLogger.Out = ioutil.Discard
if !rc.Config.ReuseContainers {
_ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger))
}
err := common.NewPipelineExecutor(steps...)(ctx)
if !rc.Config.ReuseContainers {
_ = rc.newContainerCleaner()(common.WithLogger(ctx, nullLogger))
}
return err
}
}
func (rc *RunContext) isEnabled(ctx context.Context) bool {
job := rc.Run.Job()
log := common.Logger(ctx)
if !rc.EvalBool(job.If) {
log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If)
return false
}
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
if img, ok := rc.Config.Platforms[strings.ToLower(platformName)]; !ok || img == "" {
log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName)
return false
}
return true
}
// EvalBool evaluates an expression against current run context
func (rc *RunContext) EvalBool(expr string) bool {
if expr != "" {
@ -145,33 +202,7 @@ func mergeMaps(maps ...map[string]string) map[string]string {
return rtnMap
}
func (rc *RunContext) setupTempDir() error {
var err error
tempBase := ""
if runtime.GOOS == "darwin" {
tempBase = "/tmp"
}
rc.Tempdir, err = ioutil.TempDir(tempBase, "act-")
if err != nil {
return err
}
err = os.Chmod(rc.Tempdir, 0755)
if err != nil {
return err
}
log.Debugf("Setup tempdir %s", rc.Tempdir)
return err
}
func (rc *RunContext) pullImage(containerSpec *model.ContainerSpec) common.Executor {
return func(ctx context.Context) error {
return container.NewDockerPullExecutor(container.NewDockerPullExecutorInput{
Image: containerSpec.Image,
ForcePull: rc.Config.ForcePull,
})(ctx)
}
}
/*
func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Executor {
return func(ctx context.Context) error {
ghReader, err := rc.createGithubTarball()
@ -200,7 +231,7 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex
}
})
return container.NewDockerRunExecutor(container.NewDockerRunExecutorInput{
c := container.NewContainer(&container.NewContainerInput{
Cmd: cmd,
Entrypoint: entrypoint,
Image: containerSpec.Image,
@ -212,64 +243,27 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex
fmt.Sprintf("%s:%s", rc.Tempdir, "/github/home"),
fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"),
},
Content: map[string]io.Reader{"/github": ghReader},
ReuseContainers: containerSpec.Reuse,
Stdout: logWriter,
Stderr: logWriter,
})(ctx)
Stdout: logWriter,
Stderr: logWriter,
})
return c.Create().
Then(c.Copy("/github", ghReader)).
Then(c.Start()).
Finally(c.Remove().IfBool(!rc.Config.ReuseContainers))(ctx)
}
}
func (rc *RunContext) createGithubTarball() (io.Reader, error) {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
var files = []struct {
Name string
Mode int64
Body string
}{
{"workflow/event.json", 0644, rc.EventJSON},
*/
func createContainerName(parts ...string) string {
name := make([]string, 0)
pattern := regexp.MustCompile("[^a-zA-Z0-9]")
for _, part := range parts {
name = append(name, pattern.ReplaceAllString(part, "-"))
}
for _, file := range files {
log.Debugf("Writing entry to tarball %s len:%d", file.Name, len(rc.EventJSON))
hdr := &tar.Header{
Name: file.Name,
Mode: file.Mode,
Size: int64(len(rc.EventJSON)),
}
if err := tw.WriteHeader(hdr); err != nil {
return nil, err
}
if _, err := tw.Write([]byte(rc.EventJSON)); err != nil {
return nil, err
}
}
if err := tw.Close(); err != nil {
return nil, err
}
return &buf, nil
}
func (rc *RunContext) createContainerName() string {
containerName := rc.Run.String()
containerName = regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(containerName, "-")
prefix := ""
suffix := ""
containerName = trimToLen(containerName, 30-(len(prefix)+len(suffix)))
return fmt.Sprintf("%s%s%s", prefix, containerName, suffix)
}
func (rc *RunContext) createStepContainerName(stepID string) string {
prefix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(rc.createContainerName(), "-")
suffix := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(stepID, "-")
prefix = trimToLen(prefix, 30-(1+len(suffix)))
name := strings.Trim(fmt.Sprintf("%s-%s", prefix, suffix), "-")
return name
return trimToLen(strings.Join(name, "-"), 30)
}
func trimToLen(s string, l int) string {

64
pkg/runner/runner.go
View File

@ -13,7 +13,6 @@ import (
// Runner provides capabilities to run GitHub actions
type Runner interface {
NewPlanExecutor(plan *model.Plan) common.Executor
NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor
}
// Config contains the config for a new runner
@ -59,49 +58,12 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
stageExecutor := make([]common.Executor, 0)
for _, run := range stage.Runs {
job := run.Job()
matrixes := make([]map[string]interface{}, 0)
if job.Strategy != nil {
includes := make([]map[string]interface{}, 0)
for _, v := range job.Strategy.Matrix["include"] {
includes = append(includes, v.(map[string]interface{}))
}
delete(job.Strategy.Matrix, "include")
excludes := make([]map[string]interface{}, 0)
for _, v := range job.Strategy.Matrix["exclude"] {
excludes = append(excludes, v.(map[string]interface{}))
}
delete(job.Strategy.Matrix, "exclude")
matrixProduct := common.CartesianProduct(job.Strategy.Matrix)
MATRIX:
for _, matrix := range matrixProduct {
for _, exclude := range excludes {
if commonKeysMatch(matrix, exclude) {
log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude)
continue MATRIX
}
}
for _, include := range includes {
if commonKeysMatch(matrix, include) {
log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include)
for k, v := range include {
matrix[k] = v
}
}
}
matrixes = append(matrixes, matrix)
}
} else {
matrixes = append(matrixes, make(map[string]interface{}))
}
matrixes := job.GetMatrixes()
jobName := fmt.Sprintf("%-*s", maxJobNameLen, run.String())
for _, matrix := range matrixes {
m := matrix
runExecutor := runner.NewRunExecutor(run, matrix)
runExecutor := runner.newRunExecutor(run, matrix)
stageExecutor = append(stageExecutor, func(ctx context.Context) error {
ctx = WithJobLogger(ctx, jobName)
if len(m) > 0 {
@ -117,22 +79,14 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
return common.NewPipelineExecutor(pipeline...)
}
func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool {
for aKey, aVal := range a {
if bVal, ok := b[aKey]; ok && aVal != bVal {
return false
}
func (runner *runnerImpl) newRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor {
rc := &RunContext{
Config: runner.config,
Run: run,
EventJSON: runner.eventJSON,
StepResults: make(map[string]*stepResult),
Matrix: matrix,
}
return true
}
func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor {
rc := new(RunContext)
rc.Config = runner.config
rc.Run = run
rc.EventJSON = runner.eventJSON
rc.StepResults = make(map[string]*stepResult)
rc.Matrix = matrix
rc.ExprEval = rc.NewExpressionEvaluator()
return rc.Executor()
}

276
pkg/runner/step.go
View File

@ -1,276 +0,0 @@
package runner
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/nektos/act/pkg/common"
"github.com/nektos/act/pkg/container"
"github.com/nektos/act/pkg/model"
log "github.com/sirupsen/logrus"
)
func (rc *RunContext) StepEnv(step *model.Step) map[string]string {
var env map[string]string
job := rc.Run.Job()
if job.Container != nil {
env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv())
} else {
env = mergeMaps(rc.GetEnv(), step.GetEnv())
}
for k, v := range env {
env[k] = rc.ExprEval.Interpolate(v)
}
return env
}
func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.Step) common.Executor {
return func(ctx context.Context) error {
containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step))
return nil
}
}
func (rc *RunContext) newContainerCleaner() common.Executor {
job := rc.Run.Job()
containerSpec := new(model.ContainerSpec)
containerSpec.Name = rc.createContainerName()
containerSpec.Reuse = false
if job.Container != nil {
containerSpec.Image = job.Container.Image
} else {
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)]
}
containerSpec.Entrypoint = "bash --noprofile --norc -o pipefail -c echo 'cleaning up'"
return common.NewPipelineExecutor(
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
}
func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
job := rc.Run.Job()
containerSpec := new(model.ContainerSpec)
containerSpec.Name = rc.createContainerName()
containerSpec.Reuse = true
if job.Container != nil {
containerSpec.Image = job.Container.Image
} else {
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
containerSpec.Image = rc.Config.Platforms[strings.ToLower(platformName)]
}
switch step.Type() {
case model.StepTypeRun:
if job.Container != nil {
containerSpec.Ports = job.Container.Ports
containerSpec.Volumes = job.Container.Volumes
containerSpec.Options = job.Container.Options
}
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.setupShellCommand(containerSpec, step.Shell, step.Run),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
case model.StepTypeUsesDockerURL:
containerSpec.Image = strings.TrimPrefix(step.Uses, "docker://")
containerSpec.Name = rc.createStepContainerName(step.ID)
containerSpec.Entrypoint = step.With["entrypoint"]
containerSpec.Args = step.With["args"]
containerSpec.Reuse = rc.Config.ReuseContainers
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
case model.StepTypeUsesActionLocal:
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)),
applyWith(containerSpec, step),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
case model.StepTypeUsesActionRemote:
remoteAction := newRemoteAction(step.Uses)
if remoteAction.Org == "actions" && remoteAction.Repo == "checkout" {
return func(ctx context.Context) error {
common.Logger(ctx).Debugf("Skipping actions/checkout")
return nil
}
}
cloneDir, err := ioutil.TempDir(rc.Tempdir, remoteAction.Repo)
if err != nil {
return common.NewErrorExecutor(err)
}
return common.NewPipelineExecutor(
common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{
URL: remoteAction.CloneURL(),
Ref: remoteAction.Ref,
Dir: cloneDir,
}),
rc.setupEnv(containerSpec, step),
rc.setupAction(containerSpec, filepath.Join(cloneDir, remoteAction.Path)),
applyWith(containerSpec, step),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
}
return common.NewErrorExecutor(fmt.Errorf("Unable to determine how to run job:%s step:%+v", rc.Run, step))
}
func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Executor {
return func(ctx context.Context) error {
if entrypoint, ok := step.With["entrypoint"]; ok {
containerSpec.Entrypoint = entrypoint
}
if args, ok := step.With["args"]; ok {
containerSpec.Args = args
}
return nil
}
}
func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shell string, run string) common.Executor {
return func(ctx context.Context) error {
shellCommand := ""
switch shell {
case "", "bash":
shellCommand = "bash --noprofile --norc -eo pipefail {0}"
case "pwsh":
shellCommand = "pwsh -command \"& '{0}'\""
case "python":
shellCommand = "python {0}"
case "sh":
shellCommand = "sh -e -c {0}"
case "cmd":
shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\""
case "powershell":
shellCommand = "powershell -command \"& '{0}'\""
default:
shellCommand = shell
}
tempScript, err := ioutil.TempFile(rc.Tempdir, ".temp-script-")
if err != nil {
return err
}
_, err = tempScript.WriteString(fmt.Sprintf("PATH=\"%s:${PATH}\"\n", strings.Join(rc.ExtraPath, ":")))
if err != nil {
return err
}
run = rc.ExprEval.Interpolate(run)
if _, err := tempScript.WriteString(run); err != nil {
return err
}
log.Debugf("Wrote command '%s' to '%s'", run, tempScript.Name())
if err := tempScript.Close(); err != nil {
return err
}
containerPath := fmt.Sprintf("/github/home/%s", filepath.Base(tempScript.Name()))
containerSpec.Entrypoint = strings.Replace(shellCommand, "{0}", containerPath, 1)
return nil
}
}
func (rc *RunContext) setupAction(containerSpec *model.ContainerSpec, actionDir string) common.Executor {
return func(ctx context.Context) error {
f, err := os.Open(filepath.Join(actionDir, "action.yml"))
if os.IsNotExist(err) {
f, err = os.Open(filepath.Join(actionDir, "action.yaml"))
if err != nil {
return err
}
} else if err != nil {
return err
}
action, err := model.ReadAction(f)
if err != nil {
return err
}
for inputID, input := range action.Inputs {
envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_")
envKey = fmt.Sprintf("INPUT_%s", envKey)
if _, ok := containerSpec.Env[envKey]; !ok {
containerSpec.Env[envKey] = input.Default
}
}
switch action.Runs.Using {
case model.ActionRunsUsingNode12:
if strings.HasPrefix(actionDir, rc.Config.Workdir) {
containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main)
} else if strings.HasPrefix(actionDir, rc.Tempdir) {
containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main)
}
case model.ActionRunsUsingDocker:
if strings.HasPrefix(actionDir, rc.Config.Workdir) {
containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir))
} else if strings.HasPrefix(actionDir, rc.Tempdir) {
containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir))
}
containerSpec.Reuse = rc.Config.ReuseContainers
if strings.HasPrefix(action.Runs.Image, "docker://") {
containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://")
containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ")
containerSpec.Args = strings.Join(action.Runs.Args, " ")
} else {
containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest")
contextDir := filepath.Join(actionDir, action.Runs.Main)
return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
ContextDir: contextDir,
ImageTag: containerSpec.Image,
})(ctx)
}
}
return nil
}
}
type remoteAction struct {
Org string
Repo string
Path string
Ref string
}
func (ra *remoteAction) CloneURL() string {
return fmt.Sprintf("https://github.com/%s/%s", ra.Org, ra.Repo)
}
func newRemoteAction(action string) *remoteAction {
r := regexp.MustCompile(`^([^/@]+)/([^/@]+)(/([^@]*))?(@(.*))?$`)
matches := r.FindStringSubmatch(action)
ra := new(remoteAction)
ra.Org = matches[1]
ra.Repo = matches[2]
ra.Path = ""
ra.Ref = "master"
if len(matches) >= 5 {
ra.Path = matches[4]
}
if len(matches) >= 7 {
ra.Ref = matches[6]
}
return ra
}

11
pkg/runner/testdata/basic/push.yml vendored
View File

@ -5,9 +5,12 @@ jobs:
check:
runs-on: ubuntu-latest
steps:
- run: echo 'hello world'
- run: echo 'hello world'
- run: echo ${GITHUB_SHA} >> /github/sha.txt
- run: cat /github/sha.txt | grep ${GITHUB_SHA}
build:
if: false
runs-on: ubuntu-latest
needs: [check]
steps:
@ -20,4 +23,8 @@ jobs:
steps:
- uses: docker://ubuntu:18.04
with:
args: echo ${GITHUB_REF} | grep nektos/act
args: env
- uses: docker://ubuntu:18.04
with:
entrypoint: /bin/echo
args: ${{github.event_name}}

1
pkg/runner/testdata/node/push.yml vendored
View File

@ -6,7 +6,6 @@ jobs:
test:
runs-on: ubuntu-latest
steps:
- run: which node
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with: