From f2b98ed301290f53e96c64814ab9949640e10307 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 16 Nov 2022 22:29:45 +0100 Subject: [PATCH] feat: Host environment (#1293) --- .github/workflows/checks.yml | 19 + go.mod | 1 + pkg/container/docker_run.go | 3 +- pkg/container/docker_run_test.go | 3 + pkg/container/executions_environment.go | 13 + pkg/container/file_collector.go | 23 + pkg/container/host_environment.go | 470 ++++++++++++++++++ pkg/container/host_environment_test.go | 4 + .../linux_container_environment_extensions.go | 73 +++ ...x_container_environment_extensions_test.go | 71 +++ pkg/container/util.go | 26 + pkg/container/util_openbsd_mips64.go | 17 + pkg/container/util_plan9.go | 17 + pkg/container/util_windows.go | 15 + pkg/lookpath/LICENSE | 27 + pkg/lookpath/env.go | 18 + pkg/lookpath/error.go | 10 + pkg/lookpath/lp_js.go | 23 + pkg/lookpath/lp_plan9.go | 56 +++ pkg/lookpath/lp_unix.go | 59 +++ pkg/lookpath/lp_windows.go | 94 ++++ pkg/runner/action.go | 6 +- pkg/runner/container_mock_test.go | 1 + pkg/runner/expression.go | 57 +-- pkg/runner/expression_test.go | 1 - pkg/runner/job_executor.go | 14 + pkg/runner/job_executor_test.go | 1 + pkg/runner/run_context.go | 165 ++++-- pkg/runner/run_context_test.go | 2 - pkg/runner/runner.go | 49 -- pkg/runner/runner_test.go | 146 +++--- pkg/runner/step.go | 9 +- pkg/runner/step_action_remote.go | 2 +- pkg/runner/step_docker.go | 2 +- pkg/runner/step_docker_test.go | 2 +- pkg/runner/step_run.go | 6 +- pkg/runner/testdata/nix-prepend-path/push.yml | 26 + pkg/runner/testdata/windows-add-env/push.yml | 27 + .../testdata/windows-prepend-path/push.yml | 25 + 39 files changed, 1396 insertions(+), 187 deletions(-) create mode 100644 pkg/container/executions_environment.go create mode 100644 pkg/container/host_environment.go create mode 100644 pkg/container/host_environment_test.go create mode 100644 pkg/container/linux_container_environment_extensions.go create mode 100644 pkg/container/linux_container_environment_extensions_test.go create mode 100644 pkg/container/util.go create mode 100644 pkg/container/util_openbsd_mips64.go create mode 100644 pkg/container/util_plan9.go create mode 100644 pkg/container/util_windows.go create mode 100644 pkg/lookpath/LICENSE create mode 100644 pkg/lookpath/env.go create mode 100644 pkg/lookpath/error.go create mode 100644 pkg/lookpath/lp_js.go create mode 100644 pkg/lookpath/lp_plan9.go create mode 100644 pkg/lookpath/lp_unix.go create mode 100644 pkg/lookpath/lp_windows.go create mode 100644 pkg/runner/testdata/nix-prepend-path/push.yml create mode 100644 pkg/runner/testdata/windows-add-env/push.yml create mode 100644 pkg/runner/testdata/windows-prepend-path/push.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d69a32d9..4a321285 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,6 +57,25 @@ jobs: files: coverage.txt fail_ci_if_error: true # optional (default = false) + test-host: + strategy: + matrix: + os: + - windows-latest + - macos-latest + name: test-${{matrix.os}} + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + - run: go test -v -run ^TestRunEventHostEnvironment$ ./... + # TODO merge coverage with test-linux + snapshot: name: snapshot runs-on: ubuntu-latest diff --git a/go.mod b/go.mod index dcae80b6..27407546 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Masterminds/semver v1.5.0 github.com/andreaskoch/go-fswatch v1.0.0 + github.com/creack/pty v1.1.17 github.com/docker/cli v20.10.21+incompatible github.com/docker/distribution v2.8.1+incompatible github.com/docker/docker v20.10.21+incompatible diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index 896a269c..2740ad4d 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -84,7 +84,7 @@ type Container interface { } // NewContainer creates a reference to a container -func NewContainer(input *NewContainerInput) Container { +func NewContainer(input *NewContainerInput) ExecutionsEnvironment { cr := new(containerReference) cr.input = input return cr @@ -233,6 +233,7 @@ type containerReference struct { input *NewContainerInput UID int GID int + LinuxContainerEnvironmentExtensions } func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) { diff --git a/pkg/container/docker_run_test.go b/pkg/container/docker_run_test.go index 2e12fcfc..bc3ab4bd 100644 --- a/pkg/container/docker_run_test.go +++ b/pkg/container/docker_run_test.go @@ -163,3 +163,6 @@ func TestDockerExecFailure(t *testing.T) { conn.AssertExpectations(t) client.AssertExpectations(t) } + +// Type assert containerReference implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &containerReference{} diff --git a/pkg/container/executions_environment.go b/pkg/container/executions_environment.go new file mode 100644 index 00000000..1c21f943 --- /dev/null +++ b/pkg/container/executions_environment.go @@ -0,0 +1,13 @@ +package container + +import "context" + +type ExecutionsEnvironment interface { + Container + ToContainerPath(string) string + GetActPath() string + GetPathVariableName() string + DefaultPathVariable() string + JoinPathVariable(...string) string + GetRunnerContext(ctx context.Context) map[string]interface{} +} diff --git a/pkg/container/file_collector.go b/pkg/container/file_collector.go index 164bfe7d..a4143edc 100644 --- a/pkg/container/file_collector.go +++ b/pkg/container/file_collector.go @@ -59,6 +59,29 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } +type copyCollector struct { + DstDir string +} + +func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { + fdestpath := filepath.Join(cc.DstDir, fpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + if f == nil { + return os.Symlink(linkName, fdestpath) + } + df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, f); err != nil { + return err + } + return nil +} + type fileCollector struct { Ignorer gitignore.Matcher SrcPath string diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go new file mode 100644 index 00000000..b404e86d --- /dev/null +++ b/pkg/container/host_environment.go @@ -0,0 +1,470 @@ +package container + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "errors" + + "github.com/go-git/go-billy/v5/helper/polyfill" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/lookpath" + "golang.org/x/term" +) + +type HostEnvironment struct { + Path string + TmpDir string + ToolCache string + Workdir string + ActPath string + CleanUp func() + StdOut io.Writer +} + +func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Close() common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + for _, f := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil { + return err + } + } + return nil + } +} + +func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath) + var ignorer gitignore.Matcher + if useGitIgnore { + ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil) + if err != nil { + logger.Debugf("Error loading .gitignore: %v", err) + } + + ignorer = gitignore.NewMatcher(ps) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + Ignorer: ignorer, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: ©Collector{ + DstDir: destPath, + }, + } + return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + } +} + +func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + tc := &tarCollector{ + TarWriter: tw, + } + if fi.IsDir() { + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: tc, + } + err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + if err != nil { + return nil, err + } + } else { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, f) + if err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil +} + +func (e *HostEnvironment) Pull(forcePull bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Start(attach bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +type ptyWriter struct { + Out io.Writer + AutoStop bool + dirtyLine bool +} + +func (w *ptyWriter) Write(buf []byte) (int, error) { + if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 { + n, err := w.Out.Write(buf[:len(buf)-1]) + if err != nil { + return n, err + } + if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' { + _, _ = w.Out.Write([]byte("\n")) + return n, io.EOF + } + return n, io.EOF + } + w.dirtyLine = strings.LastIndex(string(buf), "\n") < len(buf)-1 + return w.Out.Write(buf) +} + +type localEnv struct { + env map[string]string +} + +func (l *localEnv) Getenv(name string) string { + if runtime.GOOS == "windows" { + for k, v := range l.env { + if strings.EqualFold(name, k) { + return v + } + } + return "" + } + return l.env[name] +} + +func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) { + f, err := lookpath.LookPath2(cmd, &localEnv{env: env}) + if err != nil { + err := "Cannot find: " + fmt.Sprint(cmd) + " in PATH" + if _, _err := writer.Write([]byte(err + "\n")); _err != nil { + return "", fmt.Errorf("%v: %w", err, _err) + } + return "", errors.New(err) + } + return f, nil +} + +func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) { + ppty, tty, err := openPty() + if err != nil { + return nil, nil, err + } + if term.IsTerminal(int(tty.Fd())) { + _, err := term.MakeRaw(int(tty.Fd())) + if err != nil { + ppty.Close() + tty.Close() + return nil, nil, err + } + } + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = getSysProcAttr(cmdline, true) + return ppty, tty, nil +} + +func writeKeepAlive(ppty io.Writer) { + c := 1 + var err error + for c == 1 && err == nil { + c, err = ppty.Write([]byte{4}) + <-time.After(time.Second) + } +} + +func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) { + defer func() { + finishLog() + }() + if _, err := io.Copy(writer, ppty); err != nil { + return + } +} + +func (e *HostEnvironment) UpdateFromImageEnv(env *map[string]string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func getEnvListFromMap(env map[string]string) []string { + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + return envList +} + +func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error { + envList := getEnvListFromMap(env) + var wd string + if workdir != "" { + if filepath.IsAbs(workdir) { + wd = workdir + } else { + wd = filepath.Join(e.Path, workdir) + } + } else { + wd = e.Path + } + f, err := lookupPathHost(command[0], env, e.StdOut) + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, f) + cmd.Path = f + cmd.Args = command + cmd.Stdin = nil + cmd.Stdout = e.StdOut + cmd.Env = envList + cmd.Stderr = e.StdOut + cmd.Dir = wd + cmd.SysProcAttr = getSysProcAttr(cmdline, false) + var ppty *os.File + var tty *os.File + defer func() { + if ppty != nil { + ppty.Close() + } + if tty != nil { + tty.Close() + } + }() + if true /* allocate Terminal */ { + var err error + ppty, tty, err = setupPty(cmd, cmdline) + if err != nil { + common.Logger(ctx).Debugf("Failed to setup Pty %v\n", err.Error()) + } + } + writer := &ptyWriter{Out: e.StdOut} + logctx, finishLog := context.WithCancel(context.Background()) + if ppty != nil { + go copyPtyOutput(writer, ppty, finishLog) + } else { + finishLog() + } + if ppty != nil { + go writeKeepAlive(ppty) + } + err = cmd.Run() + if err != nil { + return err + } + if tty != nil { + writer.AutoStop = true + if _, err := tty.Write([]byte("\x04")); err != nil { + common.Logger(ctx).Debug("Failed to write EOT") + } + } + <-logctx.Done() + + if ppty != nil { + ppty.Close() + ppty = nil + } + return err +} + +func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, "" /*cmdline*/, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + envTar, err := e.GetContainerArchive(ctx, srcPath) + if err != nil { + return nil + } + defer envTar.Close() + reader := tar.NewReader(envTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + singleLineEnv := strings.Index(line, "=") + multiLineEnv := strings.Index(line, "<<") + if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { + localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] + } else if multiLineEnv != -1 { + multiLineEnvContent := "" + multiLineEnvDelimiter := line[multiLineEnv+2:] + delimiterFound := false + for s.Scan() { + content := s.Text() + if content == multiLineEnvDelimiter { + delimiterFound = true + break + } + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += content + } + if !delimiterFound { + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) + } + localEnv[line[:multiLineEnv]] = multiLineEnvContent + } else { + return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) + } + } + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"]) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + pathSep := string(filepath.ListSeparator) + localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()]) + } + + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) Remove() common.Executor { + return func(ctx context.Context) error { + if e.CleanUp != nil { + e.CleanUp() + } + return os.RemoveAll(e.Path) + } +} + +func (e *HostEnvironment) ToContainerPath(path string) string { + if bp, err := filepath.Rel(e.Workdir, path); err != nil { + return filepath.Join(e.Path, bp) + } else if filepath.Clean(e.Workdir) == filepath.Clean(path) { + return e.Path + } + return path +} + +func (e *HostEnvironment) GetActPath() string { + return e.ActPath +} + +func (*HostEnvironment) GetPathVariableName() string { + if runtime.GOOS == "plan9" { + return "path" + } else if runtime.GOOS == "windows" { + return "Path" // Actually we need a case insensitive map + } + return "PATH" +} + +func (e *HostEnvironment) DefaultPathVariable() string { + v, _ := os.LookupEnv(e.GetPathVariableName()) + return v +} + +func (*HostEnvironment) JoinPathVariable(paths ...string) string { + return strings.Join(paths, string(filepath.ListSeparator)) +} + +func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "temp": e.TmpDir, + "tool_cache": e.ToolCache, + } +} + +func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { + org := e.StdOut + e.StdOut = stdout + return org, org +} diff --git a/pkg/container/host_environment_test.go b/pkg/container/host_environment_test.go new file mode 100644 index 00000000..67787d95 --- /dev/null +++ b/pkg/container/host_environment_test.go @@ -0,0 +1,4 @@ +package container + +// Type assert HostEnvironment implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &HostEnvironment{} diff --git a/pkg/container/linux_container_environment_extensions.go b/pkg/container/linux_container_environment_extensions.go new file mode 100644 index 00000000..c369055d --- /dev/null +++ b/pkg/container/linux_container_environment_extensions.go @@ -0,0 +1,73 @@ +package container + +import ( + "context" + "path/filepath" + "regexp" + "runtime" + "strings" + + log "github.com/sirupsen/logrus" +) + +type LinuxContainerEnvironmentExtensions struct { +} + +// Resolves the equivalent host path inside the container +// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject +// For use in docker volumes and binds +func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string { + if runtime.GOOS == "windows" && strings.Contains(path, "/") { + log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") + return "" + } + + abspath, err := filepath.Abs(path) + if err != nil { + log.Error(err) + return "" + } + + // Test if the path is a windows path + windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) + windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) + + // Return as-is if no match + if windowsPathComponents == nil { + return abspath + } + + // Convert to WSL2-compatible path if it is a windows path + // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows + // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. + driveLetter := strings.ToLower(windowsPathComponents[1]) + translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) + // Should make something like /mnt/c/Users/person/My Folder/MyActProject + result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) + return result +} + +func (*LinuxContainerEnvironmentExtensions) GetActPath() string { + return "/var/run/act" +} + +func (*LinuxContainerEnvironmentExtensions) GetPathVariableName() string { + return "PATH" +} + +func (*LinuxContainerEnvironmentExtensions) DefaultPathVariable() string { + return "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +} + +func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) string { + return strings.Join(paths, ":") +} + +func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": "Linux", + "arch": RunnerArch(ctx), + "temp": "/tmp", + "tool_cache": "/opt/hostedtoolcache", + } +} diff --git a/pkg/container/linux_container_environment_extensions_test.go b/pkg/container/linux_container_environment_extensions_test.go new file mode 100644 index 00000000..38111714 --- /dev/null +++ b/pkg/container/linux_container_environment_extensions_test.go @@ -0,0 +1,71 @@ +package container + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestContainerPath(t *testing.T) { + type containerPathJob struct { + destinationPath string + sourcePath string + workDir string + } + + linuxcontainerext := &LinuxContainerEnvironmentExtensions{} + + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + + rootDrive := os.Getenv("SystemDrive") + rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + for _, v := range []containerPathJob{ + {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, + {"/mnt/f/work/dir", `F:\work\dir`, ""}, + {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, + {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, + } { + if v.workDir != "" { + if err := os.Chdir(v.workDir); err != nil { + log.Error(err) + t.Fail() + } + } + + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + + if err := os.Chdir(cwd); err != nil { + log.Error(err) + } + } else { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + for _, v := range []containerPathJob{ + {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, + {"/home/act", `/home/act/`, ""}, + {cwd, ".", ""}, + } { + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + } +} + +type typeAssertMockContainer struct { + Container + LinuxContainerEnvironmentExtensions +} + +// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &typeAssertMockContainer{} diff --git a/pkg/container/util.go b/pkg/container/util.go new file mode 100644 index 00000000..eb7f46c6 --- /dev/null +++ b/pkg/container/util.go @@ -0,0 +1,26 @@ +//go:build (!windows && !plan9 && !openbsd) || (!windows && !plan9 && !mips64) + +package container + +import ( + "os" + "syscall" + + "github.com/creack/pty" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + if tty { + return &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + } + } + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return pty.Open() +} diff --git a/pkg/container/util_openbsd_mips64.go b/pkg/container/util_openbsd_mips64.go new file mode 100644 index 00000000..b991d694 --- /dev/null +++ b/pkg/container/util_openbsd_mips64.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/container/util_plan9.go b/pkg/container/util_plan9.go new file mode 100644 index 00000000..cd64b4ab --- /dev/null +++ b/pkg/container/util_plan9.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Rfork: syscall.RFNOTEG, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/container/util_windows.go b/pkg/container/util_windows.go new file mode 100644 index 00000000..0a94c83a --- /dev/null +++ b/pkg/container/util_windows.go @@ -0,0 +1,15 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{CmdLine: cmdLine, CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP} +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/pkg/lookpath/LICENSE b/pkg/lookpath/LICENSE new file mode 100644 index 00000000..83403eff --- /dev/null +++ b/pkg/lookpath/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/lookpath/env.go b/pkg/lookpath/env.go new file mode 100644 index 00000000..dc376e7b --- /dev/null +++ b/pkg/lookpath/env.go @@ -0,0 +1,18 @@ +package lookpath + +import "os" + +type Env interface { + Getenv(name string) string +} + +type defaultEnv struct { +} + +func (*defaultEnv) Getenv(name string) string { + return os.Getenv(name) +} + +func LookPath(file string) (string, error) { + return LookPath2(file, &defaultEnv{}) +} diff --git a/pkg/lookpath/error.go b/pkg/lookpath/error.go new file mode 100644 index 00000000..0e3a3735 --- /dev/null +++ b/pkg/lookpath/error.go @@ -0,0 +1,10 @@ +package lookpath + +type Error struct { + Name string + Err error +} + +func (e *Error) Error() string { + return e.Err.Error() +} diff --git a/pkg/lookpath/lp_js.go b/pkg/lookpath/lp_js.go new file mode 100644 index 00000000..a967b861 --- /dev/null +++ b/pkg/lookpath/lp_js.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm + +package lookpath + +import ( + "errors" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // Wasm can not execute processes, so act as if there are no executables at all. + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_plan9.go b/pkg/lookpath/lp_plan9.go new file mode 100644 index 00000000..a201b715 --- /dev/null +++ b/pkg/lookpath/lp_plan9.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $path") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_unix.go b/pkg/lookpath/lp_unix.go new file mode 100644 index 00000000..233e21f9 --- /dev/null +++ b/pkg/lookpath/lp_unix.go @@ -0,0 +1,59 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + path := lenv.Getenv("PATH") + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/lookpath/lp_windows.go b/pkg/lookpath/lp_windows.go new file mode 100644 index 00000000..48204a79 --- /dev/null +++ b/pkg/lookpath/lp_windows.go @@ -0,0 +1,94 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in %PATH%") + +func chkStat(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsDir() { + return fs.ErrPermission + } + return nil +} + +func hasExt(file string) bool { + i := strings.LastIndex(file, ".") + if i < 0 { + return false + } + return strings.LastIndexAny(file, `:\/`) < i +} + +func findExecutable(file string, exts []string) (string, error) { + if len(exts) == 0 { + return file, chkStat(file) + } + if hasExt(file) { + if chkStat(file) == nil { + return file, nil + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return "", fs.ErrNotExist +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// LookPath also uses PATHEXT environment variable to match +// a suitable candidate. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + var exts []string + x := lenv.Getenv(`PATHEXT`) + if x != "" { + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + } else { + exts = []string{".com", ".exe", ".bat", ".cmd"} + } + + if strings.ContainsAny(file, `:\/`) { + if f, err := findExecutable(file, exts); err == nil { + return f, nil + } else { + return "", &Error{file, err} + } + } + if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { + return f, nil + } + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil { + return f, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/pkg/runner/action.go b/pkg/runner/action.go index 2dee988c..d3b012db 100644 --- a/pkg/runner/action.go +++ b/pkg/runner/action.go @@ -352,7 +352,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], @@ -396,11 +396,11 @@ func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) containerActionDir := "." if step.Type() != model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir) - containerActionDir = rc.Config.ContainerWorkdir() + "/" + actionName + containerActionDir = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName actionName = "./" + actionName } else if step.Type() == model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir()) - containerActionDir = ActPath + "/actions/" + actionName + containerActionDir = rc.JobContainer.GetActPath() + "/actions/" + actionName } if actionName == "" { diff --git a/pkg/runner/container_mock_test.go b/pkg/runner/container_mock_test.go index a336d404..0de07815 100644 --- a/pkg/runner/container_mock_test.go +++ b/pkg/runner/container_mock_test.go @@ -11,6 +11,7 @@ import ( type containerMock struct { mock.Mock container.Container + container.LinuxContainerEnvironmentExtensions } func (cm *containerMock) Create(capAdd []string, capDrop []string) common.Executor { diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 43699800..c2257b12 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" "gopkg.in/yaml.v3" @@ -23,20 +22,22 @@ type ExpressionEvaluator interface { // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation - job := rc.Run.Job() - strategy := make(map[string]interface{}) - if job.Strategy != nil { - strategy["fail-fast"] = job.Strategy.FailFast - strategy["max-parallel"] = job.Strategy.MaxParallel - } - - jobs := rc.Run.Workflow.Jobs - jobNeeds := rc.Run.Job().Needs() - using := make(map[string]map[string]map[string]string) - for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + strategy := make(map[string]interface{}) + if rc.Run != nil { + job := rc.Run.Job() + if job != nil && job.Strategy != nil { + strategy["fail-fast"] = job.Strategy.FailFast + strategy["max-parallel"] = job.Strategy.MaxParallel + } + + jobs := rc.Run.Workflow.Jobs + jobNeeds := rc.Run.Job().Needs() + + for _, needs := range jobNeeds { + using[needs] = map[string]map[string]string{ + "outputs": jobs[needs].Outputs, + } } } @@ -49,19 +50,16 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval Job: rc.getJobContext(), // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, Needs: using, Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, @@ -95,16 +93,10 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) inputs := getEvaluatorInputs(ctx, rc, step, ghc) ee := &exprparser.EvaluationEnvironment{ - Github: step.getGithubContext(ctx), - Env: *step.getEnv(), - Job: rc.getJobContext(), - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Github: step.getGithubContext(ctx), + Env: *step.getEnv(), + Job: rc.getJobContext(), + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, @@ -113,6 +105,9 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) // but required to interpolate/evaluate the inputs in actions/composite Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, diff --git a/pkg/runner/expression_test.go b/pkg/runner/expression_test.go index b784a9a7..283b6cf9 100644 --- a/pkg/runner/expression_test.go +++ b/pkg/runner/expression_test.go @@ -117,7 +117,6 @@ func TestEvaluateRunContext(t *testing.T) { {"github.run_id", "1", ""}, {"github.run_number", "1", ""}, {"job.status", "success", ""}, - {"runner.os", "Linux", ""}, {"matrix.os", "Linux", ""}, {"matrix.foo", "bar", ""}, {"env.key", "value", ""}, diff --git a/pkg/runner/job_executor.go b/pkg/runner/job_executor.go index 5c3e3b1d..547e234c 100644 --- a/pkg/runner/job_executor.go +++ b/pkg/runner/job_executor.go @@ -38,6 +38,20 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo return common.NewDebugExecutor("No steps found") } + preSteps = append(preSteps, func(ctx context.Context) error { + // Have to be skipped for some Tests + if rc.Run == nil { + return nil + } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) + // evaluate environment variables since they can contain + // GitHub's special environment variables. + for k, v := range rc.GetEnv() { + rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) + } + return nil + }) + for i, stepModel := range infoSteps { stepModel := stepModel if stepModel == nil { diff --git a/pkg/runner/job_executor_test.go b/pkg/runner/job_executor_test.go index 8299f63f..edcbfc1b 100644 --- a/pkg/runner/job_executor_test.go +++ b/pkg/runner/job_executor_test.go @@ -79,6 +79,7 @@ func (jim *jobInfoMock) result(result string) { type jobContainerMock struct { container.Container + container.LinuxContainerEnvironmentExtensions } func (jcm *jobContainerMock) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) { diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 2e945ad8..ce2532fa 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -2,7 +2,10 @@ package runner import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -21,26 +24,25 @@ import ( "github.com/nektos/act/pkg/model" ) -const ActPath string = "/var/run/act" - // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*model.StepResult - ExprEval ExpressionEvaluator - JobContainer container.Container - OutputMappings map[MappableOutput]MappableOutput - JobName string - ActionPath string - Parent *RunContext - Masks []string + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*model.StepResult + ExprEval ExpressionEvaluator + JobContainer container.ExecutionsEnvironment + OutputMappings map[MappableOutput]MappableOutput + JobName string + ActionPath string + Parent *RunContext + Masks []string + cleanUpJobContainer common.Executor } func (rc *RunContext) AddMask(mask string) { @@ -59,7 +61,13 @@ func (rc *RunContext) String() string { // GetEnv returns the env for the context func (rc *RunContext) GetEnv() map[string]string { if rc.Env == nil { - rc.Env = mergeMaps(rc.Run.Workflow.Env, rc.Run.Job().Environment(), rc.Config.Env) + rc.Env = map[string]string{} + if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { + job := rc.Run.Job() + if job != nil { + rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + } + } } rc.Env["ACT"] = "true" return rc.Env @@ -81,9 +89,11 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { fmt.Sprintf("%s:%s", rc.Config.ContainerDaemonSocket, "/var/run/docker.sock"), } + ext := container.LinuxContainerEnvironmentExtensions{} + mounts := map[string]string{ "act-toolcache": "/toolcache", - name + "-env": ActPath, + name + "-env": ext.GetActPath(), } if job := rc.Run.Job(); job != nil { @@ -109,14 +119,84 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { if selinux.GetEnabled() { bindModifiers = ":z" } - binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.ContainerWorkdir(), bindModifiers)) + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers)) } else { - mounts[name] = rc.Config.ContainerWorkdir() + mounts[name] = ext.ToContainerPath(rc.Config.Workdir) } return binds, mounts } +func (rc *RunContext) startHostEnvironment() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + rc.Env["RUNNER_TOOL_CACHE"] = toolCache + rc.Env["RUNNER_OS"] = runtime.GOOS + rc.Env["RUNNER_ARCH"] = runtime.GOARCH + rc.Env["RUNNER_TEMP"] = runnerTmp + for _, env := range os.Environ() { + i := strings.Index(env, "=") + if i > 0 { + rc.Env[env[0:i]] = env[i+1:] + } + } + + return common.NewPipelineExecutor( + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0666, + Body: "", + }, &container.FileEntry{ + Name: "workflow/paths.txt", + Mode: 0666, + Body: "", + }), + )(ctx) + } +} + func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -146,12 +226,22 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) + ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() + rc.cleanUpJobContainer = func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + } + return nil + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, Password: password, @@ -167,6 +257,9 @@ func (rc *RunContext) startJobContainer() common.Executor { Platform: rc.Config.ContainerArchitecture, Options: rc.options(ctx), }) + if rc.JobContainer == nil { + return errors.New("Failed to create job container") + } return common.NewPipelineExecutor( rc.JobContainer.Pull(rc.Config.ForcePull), @@ -175,7 +268,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Start(false), rc.JobContainer.UpdateFromImageEnv(&rc.Env), rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), - rc.JobContainer.Copy(ActPath+"/", &container.FileEntry{ + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", Mode: 0644, Body: rc.EventJSON, @@ -201,10 +294,8 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user // stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + return rc.cleanUpJobContainer(ctx) } return nil } @@ -241,7 +332,13 @@ func (rc *RunContext) interpolateOutputs() common.Executor { } func (rc *RunContext) startContainer() common.Executor { - return rc.startJobContainer() + return func(ctx context.Context) error { + image := rc.platformImage(ctx) + if strings.EqualFold(image, "-self-hosted") { + return rc.startHostEnvironment()(ctx) + } + return rc.startJobContainer()(ctx) + } } func (rc *RunContext) stopContainer() common.Executor { @@ -409,13 +506,11 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext logger := common.Logger(ctx) ghc := &model.GithubContext{ Event: make(map[string]interface{}), - EventPath: ActPath + "/workflow/event.json", Workflow: rc.Run.Workflow.Name, RunID: rc.Config.Env["GITHUB_RUN_ID"], RunNumber: rc.Config.Env["GITHUB_RUN_NUMBER"], Actor: rc.Config.Actor, EventName: rc.Config.EventName, - Workspace: rc.Config.ContainerWorkdir(), Action: rc.CurrentStep, Token: rc.Config.Token, ActionPath: rc.ActionPath, @@ -424,6 +519,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext RunnerPerflog: rc.Config.Env["RUNNER_PERFLOG"], RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"], } + if rc.JobContainer != nil { + ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json" + ghc.Workspace = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + } if ghc.RunID == "" { ghc.RunID = "1" @@ -538,8 +637,8 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) map[string]string { env["CI"] = "true" - env["GITHUB_ENV"] = ActPath + "/workflow/envs.txt" - env["GITHUB_PATH"] = ActPath + "/workflow/paths.txt" + env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" + env["GITHUB_PATH"] = rc.JobContainer.GetActPath() + "/workflow/paths.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/pkg/runner/run_context_test.go b/pkg/runner/run_context_test.go index cdafb8ee..de8f177a 100644 --- a/pkg/runner/run_context_test.go +++ b/pkg/runner/run_context_test.go @@ -385,14 +385,12 @@ func TestGetGitHubContext(t *testing.T) { } assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.Workspace, rc.Config.containerPath(cwd)) assert.Equal(t, ghc.RunNumber, "1") assert.Equal(t, ghc.RetentionDays, "0") assert.Equal(t, ghc.Actor, actor) assert.Equal(t, ghc.Repository, repo) assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.EventPath, ActPath+"/workflow/event.json") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 9c10fbe4..564814bc 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -4,10 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" - "regexp" - "runtime" - "strings" "time" log "github.com/sirupsen/logrus" @@ -57,46 +53,6 @@ type Config struct { ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. } -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -// For use in docker volumes and binds -func (config *Config) containerPath(path string) string { - if runtime.GOOS == "windows" && strings.Contains(path, "/") { - log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") - return "" - } - - abspath, err := filepath.Abs(path) - if err != nil { - log.Error(err) - return "" - } - - // Test if the path is a windows path - windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) - windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) - - // Return as-is if no match - if windowsPathComponents == nil { - return abspath - } - - // Convert to WSL2-compatible path if it is a windows path - // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows - // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. - driveLetter := strings.ToLower(windowsPathComponents[1]) - translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) - // Should make something like /mnt/c/Users/person/My Folder/MyActProject - result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) - return result -} - -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -func (config *Config) ContainerWorkdir() string { - return config.containerPath(config.Workdir) -} - type runnerImpl struct { config *Config eventJSON string @@ -163,11 +119,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { if len(matrixes) > 1 { rc.Name = fmt.Sprintf("%s-%d", rc.Name, i+1) } - // evaluate environment variables since they can contain - // GitHub's special environment variables. - for k, v := range rc.GetEnv() { - rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) - } if len(rc.String()) > maxJobNameLen { maxJobNameLen = len(rc.String()) } diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index 232fc163..9cb4ff40 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -205,6 +205,95 @@ func TestRunEvent(t *testing.T) { } } +func TestRunEventHostEnvironment(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + tables := []TestJobFileInfo{} + + if runtime.GOOS == "linux" { + platforms := map[string]string{ + "ubuntu-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + // Shells + {workdir, "shells/defaults", "push", "", platforms}, + {workdir, "shells/pwsh", "push", "", platforms}, + {workdir, "shells/bash", "push", "", platforms}, + {workdir, "shells/python", "push", "", platforms}, + {workdir, "shells/sh", "push", "", platforms}, + + // Local action + {workdir, "local-action-js", "push", "", platforms}, + + // Uses + {workdir, "uses-composite", "push", "", platforms}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, + {workdir, "uses-nested-composite", "push", "", platforms}, + {workdir, "act-composite-env-test", "push", "", platforms}, + + // Eval + {workdir, "evalmatrix", "push", "", platforms}, + {workdir, "evalmatrixneeds", "push", "", platforms}, + {workdir, "evalmatrixneeds2", "push", "", platforms}, + {workdir, "evalmatrix-merge-map", "push", "", platforms}, + {workdir, "evalmatrix-merge-array", "push", "", platforms}, + {workdir, "issue-1195", "push", "", platforms}, + + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, + {workdir, "runs-on", "push", "", platforms}, + {workdir, "checkout", "push", "", platforms}, + {workdir, "remote-action-js", "push", "", platforms}, + {workdir, "matrix", "push", "", platforms}, + {workdir, "matrix-include-exclude", "push", "", platforms}, + {workdir, "commands", "push", "", platforms}, + {workdir, "defaults-run", "push", "", platforms}, + {workdir, "composite-fail-with-output", "push", "", platforms}, + {workdir, "issue-597", "push", "", platforms}, + {workdir, "issue-598", "push", "", platforms}, + {workdir, "if-env-act", "push", "", platforms}, + {workdir, "env-and-path", "push", "", platforms}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, + {workdir, "outputs", "push", "", platforms}, + {workdir, "steps-context/conclusion", "push", "", platforms}, + {workdir, "steps-context/outcome", "push", "", platforms}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, + {workdir, "evalenv", "push", "", platforms}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, + }...) + } + if runtime.GOOS == "windows" { + platforms := map[string]string{ + "windows-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "windows-prepend-path", "push", "", platforms}, + {workdir, "windows-add-env", "push", "", platforms}, + }...) + } else { + platforms := map[string]string{ + "self-hosted": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "nix-prepend-path", "push", "", platforms}, + }...) + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, &Config{}) + }) + } +} + func TestDryrunEvent(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -320,60 +409,3 @@ func TestRunEventPullRequest(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } - -func TestContainerPath(t *testing.T) { - type containerPathJob struct { - destinationPath string - sourcePath string - workDir string - } - - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - - rootDrive := os.Getenv("SystemDrive") - rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") - for _, v := range []containerPathJob{ - {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, - {"/mnt/f/work/dir", `F:\work\dir`, ""}, - {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, - {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, - } { - if v.workDir != "" { - if err := os.Chdir(v.workDir); err != nil { - log.Error(err) - t.Fail() - } - } - - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - - if err := os.Chdir(cwd); err != nil { - log.Error(err) - } - } else { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - for _, v := range []containerPathJob{ - {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, - {"/home/act", `/home/act/`, ""}, - {cwd, ".", ""}, - } { - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - } -} diff --git a/pkg/runner/step.go b/pkg/runner/step.go index 8682d582..4f7803e1 100644 --- a/pkg/runner/step.go +++ b/pkg/runner/step.go @@ -162,13 +162,12 @@ func mergeEnv(ctx context.Context, step step) { mergeIntoMap(env, rc.GetEnv()) } - if (*env)["PATH"] == "" { - (*env)["PATH"] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` + path := rc.JobContainer.GetPathVariableName() + if (*env)[path] == "" { + (*env)[path] = rc.JobContainer.DefaultPathVariable() } if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { - p := (*env)["PATH"] - (*env)["PATH"] = strings.Join(rc.ExtraPath, `:`) - (*env)["PATH"] += `:` + p + (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) } rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) diff --git a/pkg/runner/step_action_remote.go b/pkg/runner/step_action_remote.go index 8afbe31b..00d9502a 100644 --- a/pkg/runner/step_action_remote.go +++ b/pkg/runner/step_action_remote.go @@ -118,7 +118,7 @@ func (sar *stepActionRemote) main() common.Executor { return nil } eval := sar.RunContext.NewExpressionEvaluator(ctx) - copyToPath := path.Join(sar.RunContext.Config.ContainerWorkdir(), eval.Interpolate(ctx, sar.Step.With["path"])) + copyToPath := path.Join(sar.RunContext.JobContainer.ToContainerPath(sar.RunContext.Config.Workdir), eval.Interpolate(ctx, sar.Step.With["path"])) return sar.RunContext.JobContainer.CopyDir(copyToPath, sar.RunContext.Config.Workdir+string(filepath.Separator)+".", sar.RunContext.Config.UseGitIgnore)(ctx) } diff --git a/pkg/runner/step_docker.go b/pkg/runner/step_docker.go index 4ac3b27f..c955b162 100644 --- a/pkg/runner/step_docker.go +++ b/pkg/runner/step_docker.go @@ -116,7 +116,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd [] stepContainer := ContainerNewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], diff --git a/pkg/runner/step_docker_test.go b/pkg/runner/step_docker_test.go index c7482718..c6794b47 100644 --- a/pkg/runner/step_docker_test.go +++ b/pkg/runner/step_docker_test.go @@ -17,7 +17,7 @@ func TestStepDockerMain(t *testing.T) { // mock the new container call origContainerNewContainer := ContainerNewContainer - ContainerNewContainer = func(containerInput *container.NewContainerInput) container.Container { + ContainerNewContainer = func(containerInput *container.NewContainerInput) container.ExecutionsEnvironment { input = containerInput return cm } diff --git a/pkg/runner/step_run.go b/pkg/runner/step_run.go index 85dfed64..a74f781b 100644 --- a/pkg/runner/step_run.go +++ b/pkg/runner/step_run.go @@ -68,7 +68,8 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { return err } - return sr.RunContext.JobContainer.Copy(ActPath, &container.FileEntry{ + rc := sr.getRunContext() + return rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, Mode: 0755, Body: script, @@ -128,7 +129,8 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, logger.Debugf("Wrote add-mask command to '%s'", name) } - scriptPath := fmt.Sprintf("%s/%s", ActPath, name) + rc := sr.getRunContext() + scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name) sr.cmd, err = shellquote.Split(strings.Replace(scCmd, `{0}`, scriptPath, 1)) return name, script, err diff --git a/pkg/runner/testdata/nix-prepend-path/push.yml b/pkg/runner/testdata/nix-prepend-path/push.yml new file mode 100644 index 00000000..71fd5bc7 --- /dev/null +++ b/pkg/runner/testdata/nix-prepend-path/push.yml @@ -0,0 +1,26 @@ +on: + push: +defaults: + run: + shell: sh +jobs: + test: + runs-on: self-hosted + steps: + - run: | + mkdir build + echo '#!/usr/bin/env sh' > build/testtool + echo 'echo Hi' >> build/testtool + chmod +x build/testtool + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}/build + - run: | + echo "$GITHUB_PATH" + echo '${{ github.workspace }}/build' > "$GITHUB_PATH" + cat "$GITHUB_PATH" + - run: | + echo "$PATH" + testtool diff --git a/pkg/runner/testdata/windows-add-env/push.yml b/pkg/runner/testdata/windows-add-env/push.yml new file mode 100644 index 00000000..275c5f1f --- /dev/null +++ b/pkg/runner/testdata/windows-add-env/push.yml @@ -0,0 +1,27 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + echo $env:GITHUB_ENV + echo "key=val" > $env:GITHUB_ENV + echo "key2<> $env:GITHUB_ENV + echo "line1" >> $env:GITHUB_ENV + echo "line2" >> $env:GITHUB_ENV + echo "EOF" >> $env:GITHUB_ENV + cat $env:GITHUB_ENV + - run: | + ls env: + if($env:key -ne 'val') { + echo "Unexpected value for `$env:key: $env:key" + exit 1 + } + if($env:key2 -ne "line1`nline2") { + echo "Unexpected value for `$env:key2: $env:key2" + exit 1 + } diff --git a/pkg/runner/testdata/windows-prepend-path/push.yml b/pkg/runner/testdata/windows-prepend-path/push.yml new file mode 100644 index 00000000..176de691 --- /dev/null +++ b/pkg/runner/testdata/windows-prepend-path/push.yml @@ -0,0 +1,25 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + mkdir build + echo '@echo off' > build/test.cmd + echo 'echo Hi' >> build/test.cmd + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}\build + - run: | + echo $env:GITHUB_PATH + echo '${{ github.workspace }}\build' > $env:GITHUB_PATH + cat $env:GITHUB_PATH + - run: | + echo $env:PATH + test