From ac6a465e27dccae89e3fb98de355ae79f00f2d6e Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 23 Feb 2024 08:40:14 +0100 Subject: [PATCH] feat: Test writing remote Actions locally (#2155) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/input.go | 1 + cmd/root.go | 15 ++- pkg/runner/local_repository_cache.go | 91 +++++++++++++++++++ pkg/runner/runner_test.go | 26 ++++++ .../local-remote-action-overrides/config.yml | 3 + .../local-remote-action-overrides/push.yml | 9 ++ 6 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 pkg/runner/local_repository_cache.go create mode 100644 pkg/runner/testdata/local-remote-action-overrides/config.yml create mode 100644 pkg/runner/testdata/local-remote-action-overrides/push.yml diff --git a/cmd/input.go b/cmd/input.go index 36af6d86..59c14002 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -59,6 +59,7 @@ type Input struct { logPrefixJobID bool networkName string useNewActionCache bool + localRepository []string } func (i *Input) resolve(path string) string { diff --git a/cmd/root.go b/cmd/root.go index 9a4b5ffb..3eca0406 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -100,6 +100,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull") rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally") + rootCmd.PersistentFlags().StringArrayVarP(&input.localRepository, "local-repository", "", []string{}, "Replaces the specified repository and ref with a local folder (e.g. https://github.com/test/test@v0=/home/act/test or test/test@v0=/home/act/test, the latter matches any hosts or protocols)") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -561,7 +562,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Matrix: matrixes, ContainerNetworkMode: docker_container.NetworkMode(input.networkName), } - if input.useNewActionCache { + if input.useNewActionCache || len(input.localRepository) > 0 { if input.actionOfflineMode { config.ActionCache = &runner.GoGitActionCacheOfflineMode{ Parent: runner.GoGitActionCache{ @@ -573,6 +574,18 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Path: config.ActionCacheDir, } } + if len(input.localRepository) > 0 { + localRepositories := map[string]string{} + for _, l := range input.localRepository { + k, v, _ := strings.Cut(l, "=") + localRepositories[k] = v + } + config.ActionCache = &runner.LocalRepositoryCache{ + Parent: config.ActionCache, + LocalRepositories: localRepositories, + CacheDirCache: map[string]string{}, + } + } } r, err := runner.New(config) if err != nil { diff --git a/pkg/runner/local_repository_cache.go b/pkg/runner/local_repository_cache.go new file mode 100644 index 00000000..b59ad34e --- /dev/null +++ b/pkg/runner/local_repository_cache.go @@ -0,0 +1,91 @@ +package runner + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "io/fs" + goURL "net/url" + "os" + "path/filepath" + "strings" + + "github.com/nektos/act/pkg/filecollector" +) + +type LocalRepositoryCache struct { + Parent ActionCache + LocalRepositories map[string]string + CacheDirCache map[string]string +} + +func (l *LocalRepositoryCache) Fetch(ctx context.Context, cacheDir, url, ref, token string) (string, error) { + if dest, ok := l.LocalRepositories[fmt.Sprintf("%s@%s", url, ref)]; ok { + l.CacheDirCache[fmt.Sprintf("%s@%s", cacheDir, ref)] = dest + return ref, nil + } + if purl, err := goURL.Parse(url); err == nil { + if dest, ok := l.LocalRepositories[fmt.Sprintf("%s@%s", strings.TrimPrefix(purl.Path, "/"), ref)]; ok { + l.CacheDirCache[fmt.Sprintf("%s@%s", cacheDir, ref)] = dest + return ref, nil + } + } + return l.Parent.Fetch(ctx, cacheDir, url, ref, token) +} + +func (l *LocalRepositoryCache) GetTarArchive(ctx context.Context, cacheDir, sha, includePrefix string) (io.ReadCloser, error) { + // sha is mapped to ref in fetch if there is a local override + if dest, ok := l.CacheDirCache[fmt.Sprintf("%s@%s", cacheDir, sha)]; ok { + srcPath := filepath.Join(dest, includePrefix) + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + tc := &filecollector.TarCollector{ + TarWriter: tw, + } + if fi.IsDir() { + srcPrefix := srcPath + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: tc, + } + err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) + if err != nil { + return nil, err + } + } else { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, f) + if err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil + } + return l.Parent.GetTarArchive(ctx, cacheDir, sha, includePrefix) +} diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index 96738a88..4dc01e15 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "runtime" "strings" @@ -14,6 +15,7 @@ import ( "github.com/joho/godotenv" log "github.com/sirupsen/logrus" assert "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/model" @@ -187,6 +189,7 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config GitHubInstance: "github.com", ContainerArchitecture: cfg.ContainerArchitecture, Matrix: cfg.Matrix, + ActionCache: cfg.ActionCache, } runner, err := New(runnerConfig) @@ -209,6 +212,10 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config fmt.Println("::endgroup::") } +type TestConfig struct { + LocalRepositories map[string]string `yaml:"local-repositories"` +} + func TestRunEvent(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -307,6 +314,9 @@ func TestRunEvent(t *testing.T) { {workdir, "services", "push", "", platforms, secrets}, {workdir, "services-host-network", "push", "", platforms, secrets}, {workdir, "services-with-container", "push", "", platforms, secrets}, + + // local remote action overrides + {workdir, "local-remote-action-overrides", "push", "", platforms, secrets}, } for _, table := range tables { @@ -320,6 +330,22 @@ func TestRunEvent(t *testing.T) { config.EventPath = eventFile } + testConfigFile := filepath.Join(workdir, table.workflowPath, "config.yml") + if file, err := os.ReadFile(testConfigFile); err == nil { + testConfig := &TestConfig{} + if yaml.Unmarshal(file, testConfig) == nil { + if testConfig.LocalRepositories != nil { + config.ActionCache = &LocalRepositoryCache{ + Parent: GoGitActionCache{ + path.Clean(path.Join(workdir, "cache")), + }, + LocalRepositories: testConfig.LocalRepositories, + CacheDirCache: map[string]string{}, + } + } + } + } + table.runTest(ctx, t, config) }) } diff --git a/pkg/runner/testdata/local-remote-action-overrides/config.yml b/pkg/runner/testdata/local-remote-action-overrides/config.yml new file mode 100644 index 00000000..ecabeb5d --- /dev/null +++ b/pkg/runner/testdata/local-remote-action-overrides/config.yml @@ -0,0 +1,3 @@ +local-repositories: + https://github.com/nektos/test-override@a: testdata/actions/node20 + nektos/test-override@b: testdata/actions/node16 \ No newline at end of file diff --git a/pkg/runner/testdata/local-remote-action-overrides/push.yml b/pkg/runner/testdata/local-remote-action-overrides/push.yml new file mode 100644 index 00000000..9482438f --- /dev/null +++ b/pkg/runner/testdata/local-remote-action-overrides/push.yml @@ -0,0 +1,9 @@ +name: basic +on: push + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: nektos/test-override@a + - uses: nektos/test-override@b \ No newline at end of file