p79813206
/
act
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added support for chunked uploads. (#1208) 

* Added tests for mid-size and big artifacts, reproducing a problem with chunked uploads.

* Added support for chunked uploads.

* Enforced overwriting uploaded artifacts on receiving the first chunk.

Co-authored-by: Casey Lee <cplee@nektos.com>
This commit is contained in:
Grigory Entin 2022-06-21 00:06:55 +02:00 committed by GitHub
parent 1d4c2aaa3f
commit 7105919f0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 126 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/artifacts/server.go
View File

@ -49,6 +49,7 @@ type MkdirFS interface {
fs.FS
MkdirAll(path string, perm fs.FileMode) error
Open(name string) (fs.File, error)
OpenAtEnd(name string) (fs.File, error)
}
type MkdirFsImpl struct {
@ -61,7 +62,22 @@ func (fsys MkdirFsImpl) MkdirAll(path string, perm fs.FileMode) error {
}
func (fsys MkdirFsImpl) Open(name string) (fs.File, error) {
return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644)
return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
}
func (fsys MkdirFsImpl) OpenAtEnd(name string) (fs.File, error) {
file, err := os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return nil, err
}
_, err = file.Seek(0, os.SEEK_END)
if err != nil {
return nil, err
}
return file, nil
}
var gzipExtension = ".gz__"
@ -98,7 +114,14 @@ func uploads(router *httprouter.Router, fsys MkdirFS) {
panic(err)
}
file, err := fsys.Open(filePath)
file, err := func() (fs.File, error) {
contentRange := req.Header.Get("Content-Range")
if contentRange != "" && !strings.HasPrefix(contentRange, "bytes 0-") {
return fsys.OpenAtEnd(filePath)
}
return fsys.Open(filePath)
}()
if err != nil {
panic(err)
}

10
pkg/artifacts/server_test.go
View File

@ -51,6 +51,16 @@ func (fsys MapFsImpl) Open(path string) (fs.File, error) {
return WritableFile{result, fsys.MapFS, path}, err
}
func (fsys MapFsImpl) OpenAtEnd(path string) (fs.File, error) {
var file = fstest.MapFile{
Data: []byte("content2"),
}
fsys.MapFS[path] = &file
result, err := fsys.MapFS.Open(path)
return WritableFile{result, fsys.MapFS, path}, err
}
func TestNewArtifactUploadPrepare(t *testing.T) {
assert := assert.New(t)

92
pkg/artifacts/testdata/upload-and-download/artifacts.yml vendored
View File

@ -26,10 +26,17 @@ jobs:
run: |
mkdir -p path/to/dir-1
mkdir -p path/to/dir-2
mkdir -p path/to/dir-3
mkdir -p path/to/dir-3
mkdir -p path/to/dir-5
mkdir -p path/to/dir-6
mkdir -p path/to/dir-7
echo "Lorem ipsum dolor sit amet" > path/to/dir-1/file1.txt
echo "Hello world from file #2" > path/to/dir-2/file2.txt
echo "This is a going to be a test for a large enough file that should get compressed with GZip. The @actions/artifact package uses GZip to upload files. This text should have a compression ratio greater than 100% so it should get uploaded using GZip" > path/to/dir-3/gzip.txt
dd if=/dev/random of=path/to/dir-5/file5.rnd bs=1024 count=1024
dd if=/dev/random of=path/to/dir-6/file6.rnd bs=1024 count=$((10*1024))
dd if=/dev/random of=path/to/dir-7/file7.rnd bs=1024 count=$((10*1024))
# Upload a single file artifact
- name: 'Upload artifact #1'
uses: actions/upload-artifact@v2
@ -59,6 +66,35 @@ jobs:
path/to/dir-1/*
path/to/dir-[23]/*
!path/to/dir-3/*.txt
# Upload a mid-size file artifact
- name: 'Upload artifact #5'
uses: actions/upload-artifact@v2
with:
name: 'Mid-Size-Artifact'
path: path/to/dir-5/file5.rnd
# Upload a big file artifact
- name: 'Upload artifact #6'
uses: actions/upload-artifact@v2
with:
name: 'Big-Artifact'
path: path/to/dir-6/file6.rnd
# Upload a big file artifact twice
- name: 'Upload artifact #7 (First)'
uses: actions/upload-artifact@v2
with:
name: 'Big-Uploaded-Twice'
path: path/to/dir-7/file7.rnd
# Upload a big file artifact twice
- name: 'Upload artifact #7 (Second)'
uses: actions/upload-artifact@v2
with:
name: 'Big-Uploaded-Twice'
path: path/to/dir-7/file7.rnd
# Verify artifacts. Switch to download-artifact@v2 once it's out of preview
# Download Artifact #1 and verify the correctness of the content
@ -138,3 +174,57 @@ jobs:
echo "File contents of downloaded artifacts are incorrect"
exit 1
fi
- name: 'Download artifact #5'
uses: actions/download-artifact@v2
with:
name: 'Mid-Size-Artifact'
path: mid-size/artifact/path
- name: 'Verify Artifact #5'
run: |
file="mid-size/artifact/path/file5.rnd"
if [ ! -f $file ] ; then
echo "Expected file does not exist"
exit 1
fi
if ! diff $file path/to/dir-5/file5.rnd ; then
echo "File contents of downloaded artifact are incorrect"
exit 1
fi
- name: 'Download artifact #6'
uses: actions/download-artifact@v2
with:
name: 'Big-Artifact'
path: big/artifact/path
- name: 'Verify Artifact #6'
run: |
file="big/artifact/path/file6.rnd"
if [ ! -f $file ] ; then
echo "Expected file does not exist"
exit 1
fi
if ! diff $file path/to/dir-6/file6.rnd ; then
echo "File contents of downloaded artifact are incorrect"
exit 1
fi
- name: 'Download artifact #7'
uses: actions/download-artifact@v2
with:
name: 'Big-Uploaded-Twice'
path: big-uploaded-twice/artifact/path
- name: 'Verify Artifact #7'
run: |
file="big-uploaded-twice/artifact/path/file7.rnd"
if [ ! -f $file ] ; then
echo "Expected file does not exist"
exit 1
fi
if ! diff $file path/to/dir-7/file7.rnd ; then
echo "File contents of downloaded artifact are incorrect"
exit 1
fi