From 3a03725b57b2873a615c0f5601a8a1d0a900448e Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 16:31:26 +1100 Subject: [PATCH 01/12] Get deps in a separate step --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d253efa0..db12aff0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: - checkout - run: sudo apt-get update - run: sudo apt install fuse pv - # - run: go get -t -d -v -race $PROJECT_GO_PACKAGE/... + - run: go get ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... # - run: CGO_ENABLED=0 go get -t -d -v $PROJECT_GO_PACKAGE/... From 1105d6582c72561a1da3a572ae534e40c3927ee3 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 16:37:40 +1100 Subject: [PATCH 02/12] Add extra go debug steps --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index db12aff0..b3a38458 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,9 +12,11 @@ jobs: # - run: echo $PROJECT_GO_PACKAGE - run: echo $CIRCLE_WORKING_DIRECTORY - run: echo $PWD + - run: echo $GOPATH - run: echo 'export GOPATH=$HOME/go' >> $BASH_ENV - run: echo 'export PATH="$GOPATH/bin:$PATH"' >> $BASH_ENV - run: echo $GOPATH + - run: which go - run: go version - run: cd /usr/local && sudo rm -r go && sudo mkdir go go1.4 && sudo chown `whoami` go go1.4 && git clone git://github.com/golang/go && cd go && git branch -f -t go1.4 origin/release-branch.go1.4 && cd .. && git clone --single-branch -b go1.4 go go1.4 && cd go1.4/src && ./make.bash && cd ../../go/src && GOROOT_BOOTSTRAP=/usr/local/go1.4 ./make.bash - run: go version From 53358d32be33402a9c16ba29b6e4dd10292456b0 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 16:57:33 +1100 Subject: [PATCH 03/12] Add pkg/mod caching --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3a38458..118ed967 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,15 +23,21 @@ jobs: - checkout - run: sudo apt-get update - run: sudo apt install fuse pv + - restore_cache: + keys: + - go-mod - run: go get ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... - # - run: CGO_ENABLED=0 go get -t -d -v $PROJECT_GO_PACKAGE/... - run: set +e; CGO_ENABLED=0 go test -v ./...; true - run: go install golang.org/x/mobile/cmd/gomobile - run: gomobile init - run: gomobile build -target=android $PROJECT_GO_PACKAGE # - run: GO111MODULE=off go install github.com/anacrolix/godo - run: go install ./cmd/torrentfs + - save_cache: + key: go-mod + paths: + - $GOPATH/pkg/mod - run: sudo modprobe fuse - run: fs/test.sh From 26983bb031b934f919d5d152dafd7089cb59319b Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 16:59:49 +1100 Subject: [PATCH 04/12] Fix yaml formatting What a stupid format. --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 118ed967..097dba06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,8 +24,8 @@ jobs: - run: sudo apt-get update - run: sudo apt install fuse pv - restore_cache: - keys: - - go-mod + keys: + - go-mod - run: go get ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... @@ -36,8 +36,8 @@ jobs: # - run: GO111MODULE=off go install github.com/anacrolix/godo - run: go install ./cmd/torrentfs - save_cache: - key: go-mod - paths: - - $GOPATH/pkg/mod + key: go-mod + paths: + - $GOPATH/pkg/mod - run: sudo modprobe fuse - run: fs/test.sh From 40d1fbc685f893a961f9a234be341b1d08abc9aa Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 17:07:11 +1100 Subject: [PATCH 05/12] Break up compiling go master into different steps --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 097dba06..006469b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,11 @@ jobs: - run: echo $GOPATH - run: which go - run: go version - - run: cd /usr/local && sudo rm -r go && sudo mkdir go go1.4 && sudo chown `whoami` go go1.4 && git clone git://github.com/golang/go && cd go && git branch -f -t go1.4 origin/release-branch.go1.4 && cd .. && git clone --single-branch -b go1.4 go go1.4 && cd go1.4/src && ./make.bash && cd ../../go/src && GOROOT_BOOTSTRAP=/usr/local/go1.4 ./make.bash + - run: cd /usr/local && sudo rm -r go && sudo mkdir go go1.4 && sudo chown `whoami` go go1.4 + - run: cd /usr/local && git clone git://github.com/golang/go + - run: cd /usr/local/go && git branch -f -t go1.4 origin/release-branch.go1.4 && cd .. && git clone --single-branch -b go1.4 go go1.4 + - run: cd /usr/local/go1.4/src && ./make.bash + - run: cd /usr/local/go/src && GOROOT_BOOTSTRAP=/usr/local/go1.4 ./make.bash - run: go version - checkout - run: sudo apt-get update From 885af9d29dd9647b9840697f6e718a762552dacf Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 18:57:15 +1100 Subject: [PATCH 06/12] Change the go-mod cache key and path --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 006469b1..742e763f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,8 +40,8 @@ jobs: # - run: GO111MODULE=off go install github.com/anacrolix/godo - run: go install ./cmd/torrentfs - save_cache: - key: go-mod + key: go-mod-{{ epoch }} paths: - - $GOPATH/pkg/mod + - ~/go/pkg/mod - run: sudo modprobe fuse - run: fs/test.sh From 0047eefef4ea3d9b6ff597442fac22c3f5f4e817 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 8 Jan 2019 19:20:53 +1100 Subject: [PATCH 07/12] tracker: Fix network error handling in unit test --- tracker/udp_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracker/udp_test.go b/tracker/udp_test.go index 5b12bc6b..3a22aee2 100644 --- a/tracker/udp_test.go +++ b/tracker/udp_test.go @@ -15,6 +15,7 @@ import ( "github.com/anacrolix/dht/krpc" _ "github.com/anacrolix/envpprof" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -141,7 +142,7 @@ func TestUDPTracker(t *testing.T) { Request: req, }.Do() // Skip any net errors as we don't control the server. - if _, ok := err.(net.Error); ok { + if _, ok := errors.Cause(err).(net.Error); ok { t.Skip(err) } require.NoError(t, err) From beeec3c475e9d0d7f32427789515d1be5b655d92 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 9 Jan 2019 10:47:19 +1100 Subject: [PATCH 08/12] Manually add modules that refuse to be added automatically --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index b3ea8d89..9361a9a2 100644 --- a/go.mod +++ b/go.mod @@ -17,12 +17,14 @@ require ( github.com/elgatito/upnp v0.0.0-20180711183757-2f244d205f9a github.com/fsnotify/fsnotify v1.4.7 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c + github.com/gosuri/uilive v0.0.0-20170323041506-ac356e6e42cd github.com/gosuri/uiprogress v0.0.0-20170224063937-d0567a9d84a1 github.com/ipfs/go-ipfs v0.4.18 // indirect github.com/jessevdk/go-flags v1.4.0 github.com/mattn/go-sqlite3 v1.10.0 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect github.com/pkg/errors v0.8.0 + github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 github.com/stretchr/testify v1.2.2 github.com/willf/bitset v1.1.9 // indirect golang.org/x/net v0.0.0-20181220203305-927f97764cc3 From 84132ba2423c7538343f34e4ba5dbc03fbc880df Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 9 Jan 2019 11:21:49 +1100 Subject: [PATCH 09/12] Time the module downloads --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 742e763f..bd193344 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ jobs: - restore_cache: keys: - go-mod - - run: go get ./... + - run: go get -d ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... - run: set +e; CGO_ENABLED=0 go test -v ./...; true From 8dc4f1c81fbd12fb7da6c566ce8d1bc1c744ffc6 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 9 Jan 2019 11:23:07 +1100 Subject: [PATCH 10/12] Cache the entire go pkg folder --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd193344..3cfa7c47 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: - run: sudo apt install fuse pv - restore_cache: keys: - - go-mod + - go-pkg - run: go get -d ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... @@ -40,8 +40,8 @@ jobs: # - run: GO111MODULE=off go install github.com/anacrolix/godo - run: go install ./cmd/torrentfs - save_cache: - key: go-mod-{{ epoch }} + key: go-pkg-{{ epoch }} paths: - - ~/go/pkg/mod + - ~/go/pkg - run: sudo modprobe fuse - run: fs/test.sh From fa4dbacd799db7fe72cdebafcf23efb3b238cd6e Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 9 Jan 2019 12:25:06 +1100 Subject: [PATCH 11/12] Cache the GOCACHE Go mod doesn't store build artifacts in $GOPATH/pkg anymore. --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cfa7c47..34d8e801 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,9 @@ jobs: - restore_cache: keys: - go-pkg + - restore_cache: + keys: + - go-cache - run: go get -d ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... @@ -45,3 +48,8 @@ jobs: - ~/go/pkg - run: sudo modprobe fuse - run: fs/test.sh + - save_cache: + key: go-cache-{{ epoch }} + paths: + - ~/.cache/go-build + when: always From c9f4078224dea7d5ff8edfbc7e07bb34986d3522 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 9 Jan 2019 19:55:55 +1100 Subject: [PATCH 12/12] Cache go master, use existing go for bootstrap, and tighten some cache keys --- .circleci/config.yml | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34d8e801..aa61f2f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,21 +18,35 @@ jobs: - run: echo $GOPATH - run: which go - run: go version - - run: cd /usr/local && sudo rm -r go && sudo mkdir go go1.4 && sudo chown `whoami` go go1.4 - - run: cd /usr/local && git clone git://github.com/golang/go - - run: cd /usr/local/go && git branch -f -t go1.4 origin/release-branch.go1.4 && cd .. && git clone --single-branch -b go1.4 go go1.4 - - run: cd /usr/local/go1.4/src && ./make.bash - - run: cd /usr/local/go/src && GOROOT_BOOTSTRAP=/usr/local/go1.4 ./make.bash + - run: | + cd /usr/local + sudo mkdir go.master + sudo chown `whoami` go.master + - restore_cache: + key: go-root- + - run: | + cd /usr/local + git clone git://github.com/golang/go go.master || true + cd go.master + git pull + [[ -x bin/go && `git rev-parse HEAD` == `cat anacrolix.built` ]] && exit + cd src + ./make.bash || exit + git rev-parse HEAD > ../anacrolix.built + - save_cache: + paths: /usr/local/go.master + key: go-root-{{ checksum "/usr/local/go.master/anacrolix.built" }} + - run: echo 'export PATH="/usr/local/go.master/bin:$PATH"' >> $BASH_ENV - run: go version - checkout - run: sudo apt-get update - run: sudo apt install fuse pv - restore_cache: keys: - - go-pkg + - go-pkg- - restore_cache: keys: - - go-cache + - go-cache- - run: go get -d ./... - run: go test -v -race ./... -count 2 - run: go test -bench . ./... @@ -43,13 +57,13 @@ jobs: # - run: GO111MODULE=off go install github.com/anacrolix/godo - run: go install ./cmd/torrentfs - save_cache: - key: go-pkg-{{ epoch }} + key: go-pkg-{{ checksum "go.mod" }} paths: - ~/go/pkg - run: sudo modprobe fuse - run: fs/test.sh - save_cache: - key: go-cache-{{ epoch }} + key: go-cache-{{ .Revision }} paths: - ~/.cache/go-build when: always