p85947160
/
gitea
1
0
Fork 1
Code Issues Pull Requests 1 Projects Releases Wiki Activity

[Backport] CI optimisation & add github token env var (#9875) 

* ci: use docker image for golangci-lint (#9737)

* ci: re-ordering Drone CI for optimizing time (#9719)

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try offloading mysql8 to arm64

* Revert "ci: try offloading mysql8 to arm64"

This reverts commit c60de5db1cf8b5984c3014a57da6490f06c8d980.

* ci: try offloading pgsql to arm64

* ci: activate ldap on arm64

* ci: test mysql8 in place pgsql arm64

* chore: clean un-needed move

* typo

* ci: revert runnning mysql on arm64

* ci: run compliance on arm

* chore: limit change

* chore: readd maybe need for release fetch-tags

* ci: remove docker-linux-amd64-dry-run

* ci: remove docker-linux-amd64-dry-run

* Revert "ci: remove docker-linux-amd64-dry-run"

This reverts commit 0715f65b11c37869359aaaa5d22901da512e8184.

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

* ci: use new mssql image (#9720)

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

* ci: run notify on arm64 (#9762)

* ci: run notify on arm64

Free one jobs on amrd64

* Update .drone.yml

* Update .drone.yml

* Update .drone.yml

Based on: https://github.com/appleboy/drone-discord/blob/master/.drone.yml#L339

* improve trigger

Co-authored-by: techknowlogick <matti@mdranta.net>

* ci: move some integration tests on arm64 (#9747)

* tests: configure github remaining limit + read token (#9800)

* ci: configure remaining github limmit

* prepend with github since package is common to all migrations

* add RefreshRate

* Update github.go

* add missing space

* go fmt

* Read env variable GITHUB_READ_TOKEN for token

* Update .drone.yml

* ci: simplify tag/release by always running coverage (#9774)

* ci: simplify tag/release by always running coverage

* use mod and vendor for unit test coverage

* remove not needed lfs for unit test

* use arm drone agent for docs (#9776)

* run translations pipeline on arm server (#9865)

* add git-check to Makefile

Co-authored-by: Antoine GIRARD <sapk@users.noreply.github.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: techknowlogick <matti@mdranta.net>
This commit is contained in:
6543 2020-01-19 23:21:54 +01:00 committed by Antoine GIRARD
parent bcb722daec
commit 3fa14d89a2
4 changed files with 226 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

402
.drone.yml
View File

@ -1,6 +1,61 @@
---
kind: pipeline
name: testing
name: compliance
platform:
os: linux
arch: arm64
workspace:
base: /go
path: src/code.gitea.io/gitea
steps:
- name: pre-build
pull: always
image: node:10 # this step is kept at the lowest version of node that we support
commands:
- make css
- make js
- name: build-without-gcc
pull: always
image: golang:1.11 # this step is kept as the lowest version of golang that we support
environment:
GO111MODULE: on
GOPROXY: off
commands:
- go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag
- name: build-linux-386
pull: always
image: golang:1.13
environment:
GO111MODULE: on
GOPROXY: off
GOOS: linux
GOARCH: 386
commands:
- go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit
- name: check
pull: always
image: golang:1.13
commands:
- make clean
- make golangci-lint
- make revive
- make swagger-check
- make swagger-validate
- make test-vendor
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
---
kind: pipeline
name: testing-amd64
platform:
os: linux
@ -25,15 +80,9 @@ services:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: testgitea
- name: pgsql
pull: default
image: postgres:9.5
environment:
POSTGRES_DB: test
- name: mssql
pull: default
image: microsoft/mssql-server-linux:latest
image: mcr.microsoft.com/mssql/server:latest
environment:
ACCEPT_EULA: Y
MSSQL_PID: Standard
@ -54,46 +103,11 @@ steps:
exclude:
- pull_request
- name: pre-build
pull: always
image: node:10 # this step is kept at the lowest version of node that we support
commands:
- make css
- make js
- name: build-without-gcc
pull: always
image: golang:1.11 # this step is kept as the lowest version of golang that we support
environment:
GO111MODULE: on
GOPROXY: off
commands:
- curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
- go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag
- name: build-linux-386
pull: always
image: golang:1.13
environment:
GO111MODULE: on
GOPROXY: off
GOOS: linux
GOARCH: 386
commands:
- curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
- go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit
- name: build
pull: always
image: golang:1.13
commands:
- curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
- make clean
- make golangci-lint
- make revive
- make swagger-check
- make swagger-validate
- make test-vendor
- make build
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
@ -108,70 +122,8 @@ steps:
environment:
GOPROXY: off
TAGS: bindata sqlite sqlite_unlock_notify
depends_on:
- build
when:
branch:
- master
event:
- push
- pull_request
- name: release-test
pull: always
image: golang:1.13
commands:
- make test
environment:
GOPROXY: off
TAGS: bindata sqlite sqlite_unlock_notify
depends_on:
- build
when:
branch:
- "release/*"
event:
- push
- pull_request
- name: tag-pre-condition
pull: always
image: alpine/git
commands:
- git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
depends_on:
- build
when:
event:
- tag
- name: tag-test
pull: always
image: golang:1.13
commands:
- make test
environment:
GOPROXY: off
TAGS: bindata
depends_on:
- tag-pre-condition
when:
event:
- tag
- name: test-sqlite
pull: always
image: golang:1.13
commands:
- "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
- apt-get install -y git-lfs
- timeout -s ABRT 20m make test-sqlite-migration
- timeout -s ABRT 20m make test-sqlite
environment:
GOPROXY: off
TAGS: bindata
depends_on:
- build
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: test-mysql
pull: always
@ -187,30 +139,6 @@ steps:
TEST_LDAP: 1
depends_on:
- build
when:
branch:
- master
event:
- push
- pull_request
- name: tag-test-mysql
pull: always
image: golang:1.13
commands:
- "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
- apt-get install -y git-lfs
- timeout -s ABRT 20m make test-mysql-migration
- timeout -s ABRT 20m make test-mysql
environment:
GOPROXY: off
TAGS: bindata
TEST_LDAP: 1
depends_on:
- build
when:
event:
- tag
- name: test-mysql8
pull: always
@ -227,21 +155,6 @@ steps:
depends_on:
- build
- name: test-pgsql
pull: always
image: golang:1.13
commands:
- "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
- apt-get install -y git-lfs
- timeout -s ABRT 20m make test-pgsql-migration
- timeout -s ABRT 20m make test-pgsql
environment:
GOPROXY: off
TAGS: bindata
TEST_LDAP: 1
depends_on:
- build
- name: test-mssql
pull: always
image: golang:1.13
@ -293,13 +206,89 @@ steps:
- push
- pull_request
---
kind: pipeline
name: testing-arm64
platform:
os: linux
arch: arm64
workspace:
base: /go
path: src/code.gitea.io/gitea
services:
- name: pgsql
pull: default
image: postgres:9.5
environment:
POSTGRES_DB: test
- name: ldap
pull: default
image: gitea/test-openldap:latest
steps:
- name: fetch-tags
pull: default
image: docker:git
commands:
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: build
pull: always
image: golang:1.13
commands:
- curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
- make build
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: test-sqlite
pull: always
image: golang:1.13
commands:
- "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
- apt-get install -y git-lfs
- timeout -s ABRT 20m make test-sqlite-migration
- timeout -s ABRT 20m make test-sqlite
environment:
GOPROXY: off
TAGS: bindata
depends_on:
- build
- name: test-pgsql
pull: always
image: golang:1.13
commands:
- "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
- apt-get install -y git-lfs
- timeout -s ABRT 20m make test-pgsql-migration
- timeout -s ABRT 20m make test-pgsql
environment:
GOPROXY: off
TAGS: bindata
TEST_LDAP: 1
depends_on:
- build
---
kind: pipeline
name: translations
platform:
os: linux
arch: amd64
arch: arm64
workspace:
base: /go
@ -378,7 +367,8 @@ trigger:
- push
depends_on:
- testing
- testing-amd64
- testing-arm64
- translations
steps:
@ -476,7 +466,8 @@ trigger:
- tag
depends_on:
- testing
- testing-arm64
- testing-amd64
steps:
- name: fetch-tags
@ -545,17 +536,14 @@ name: docs
platform:
os: linux
arch: amd64
workspace:
base: /go
path: src/code.gitea.io/gitea
arch: arm64
steps:
- name: build-docs
pull: always
image: webhippie/hugo:latest
image: plugins/hugo:latest
commands:
- apk add --no-cache make bash curl
- cd docs
- make trans-copy
- make clean
@ -563,7 +551,7 @@ steps:
- name: publish-docs
pull: always
image: lucap/drone-netlify:latest
image: techknowlogick/drone-netlify:latest
settings:
path: docs/public/
site_id: d2260bae-7861-4c02-8646-8f6440b12672
@ -578,7 +566,7 @@ steps:
---
kind: pipeline
name: docker-linux-amd64
name: docker-linux-amd64-release
platform:
os: linux
@ -589,13 +577,13 @@ workspace:
path: src/code.gitea.io/gitea
depends_on:
- testing
- testing-amd64
- testing-arm64
trigger:
ref:
- refs/heads/master
- "refs/tags/**"
- "refs/pull/**"
steps:
- name: fetch-tags
@ -603,23 +591,6 @@ steps:
image: docker:git
commands:
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: dryrun
pull: always
image: plugins/docker:linux-amd64
settings:
dry_run: true
repo: gitea/gitea
tags: linux-amd64
build_args:
- GOPROXY=off
when:
event:
- pull_request
- name: publish
pull: always
@ -641,7 +612,7 @@ steps:
---
kind: pipeline
name: docker-linux-arm64
name: docker-linux-arm64-dry-run
platform:
os: linux
@ -652,25 +623,13 @@ workspace:
path: src/code.gitea.io/gitea
depends_on:
- testing
- compliance
trigger:
ref:
- refs/heads/master
- "refs/tags/**"
- "refs/pull/**"
steps:
- name: fetch-tags
pull: default
image: docker:git
commands:
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: dryrun
pull: always
image: plugins/docker:linux-arm64
@ -684,6 +643,33 @@ steps:
event:
- pull_request
---
kind: pipeline
name: docker-linux-arm64-release
platform:
os: linux
arch: arm64
workspace:
base: /go
path: src/code.gitea.io/gitea
depends_on:
- testing-amd64
- testing-arm64
trigger:
ref:
- refs/heads/master
- "refs/tags/**"
steps:
- name: fetch-tags
pull: default
image: docker:git
commands:
- git fetch --tags --force
- name: publish
pull: always
image: plugins/docker:linux-arm64
@ -729,45 +715,49 @@ trigger:
- "refs/tags/**"
depends_on:
- docker-linux-amd64
- docker-linux-arm64
- docker-linux-amd64-release
- docker-linux-arm64-release
---
kind: pipeline
name: notify
name: notifications
platform:
os: linux
arch: amd64
workspace:
base: /go
path: src/code.gitea.io/gitea
arch: arm64
clone:
disable: true
when:
trigger:
branch:
- master
- "release/*"
event:
- push
- tag
status:
- success
- failure
depends_on:
- testing
- testing-amd64
- testing-arm64
- translations
- release-version
- release-master
- docker-linux-amd64
- docker-linux-arm64
- docker-linux-amd64-release
- docker-linux-arm64-release
- docker-manifest
- docs
steps:
- name: discord
pull: always
image: appleboy/drone-discord:1.0.0
environment:
DISCORD_WEBHOOK_ID:
image: appleboy/drone-discord:1.2.4
settings:
message: "{{#success build.status}} ✅ Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌ Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
webhook_id:
from_secret: discord_webhook_id
DISCORD_WEBHOOK_TOKEN:
webhook_token:
from_secret: discord_webhook_token

11
Makefile
View File

@ -119,6 +119,13 @@ go-check:
exit 1; \
fi
.PHONY: git-check
git-check:
@if git lfs >/dev/null 2>&1 ; then : ; else \
echo "Gitea requires git with lfs support to run tests." ; \
exit 1; \
fi
.PHONY: node-check
node-check:
$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?' | tr '.' ' ');))
@ -233,7 +240,7 @@ coverage:
.PHONY: unit-test-coverage
unit-test-coverage:
$(GO) test -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
GO111MODULE=on $(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: vendor
vendor:
@ -376,7 +383,7 @@ integrations.mssql.test: $(GO_SOURCES)
integrations.sqlite.test: $(GO_SOURCES)
GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
integrations.cover.test: $(GO_SOURCES)
integrations.cover.test: git-check $(GO_SOURCES)
GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test
.PHONY: migrations.mysql.test

19
modules/migrations/github.go
View File

@ -24,6 +24,8 @@ import (
var (
_ base.Downloader = &GithubDownloaderV3{}
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
// GithubLimitRateRemaining limit to wait for new rate to apply
GithubLimitRateRemaining = 0
)
func init() {
@ -115,7 +117,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
}
func (g *GithubDownloaderV3) sleep() {
for g.rate != nil && g.rate.Remaining <= 0 {
for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining {
timer := time.NewTimer(time.Until(g.rate.Reset.Time))
select {
case <-g.ctx.Done():
@ -124,15 +126,24 @@ func (g *GithubDownloaderV3) sleep() {
case <-timer.C:
}
rates, _, err := g.client.RateLimits(g.ctx)
err := g.RefreshRate()
if err != nil {
log.Error("g.client.RateLimits: %s", err)
}
g.rate = rates.GetCore()
}
}
// RefreshRate update the current rate (doesn't count in rate limit)
func (g *GithubDownloaderV3) RefreshRate() error {
rates, _, err := g.client.RateLimits(g.ctx)
if err != nil {
return err
}
g.rate = rates.GetCore()
return nil
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
g.sleep()

7
modules/migrations/github_test.go
View File

@ -6,6 +6,7 @@
package migrations
import (
"os"
"testing"
"time"
@ -62,7 +63,11 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
}
func TestGitHubDownloadRepo(t *testing.T) {
downloader := NewGithubDownloaderV3("", "", "go-gitea", "test_repo")
GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
err := downloader.RefreshRate()
assert.NoError(t, err)
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assert.EqualValues(t, &base.Repository{