From 62256f908a516f9367f6b57526de9f404186af3b Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 26 Nov 2025 10:00:44 +0000 Subject: [PATCH] WIP Change-Id: I0374b8409d09dc69d5a6dbeeda05d95bf3a716b2 Signed-off-by: Tom Proctor --- .github/actions/go-cache/action.sh | 8 +++---- .github/actions/go-cache/action.yml | 1 + .github/workflows/test.yml | 36 ++++++++++++++++++----------- cmd/cigocacher/cigocacher.go | 33 ++++++++++++++++---------- cmd/cigocacher/disk.go | 1 + 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.github/actions/go-cache/action.sh b/.github/actions/go-cache/action.sh index 784f1064a..c0825d8d7 100755 --- a/.github/actions/go-cache/action.sh +++ b/.github/actions/go-cache/action.sh @@ -43,8 +43,8 @@ BIN_NAME="cigocacher" if [[ "${RUNNER_OS:-}" == "Windows" ]]; then BIN_NAME="cigocacher.exe" fi -BIN_PATH="${RUNNER_TEMP:-/tmp}/$BIN_NAME" +BIN_PATH="${RUNNER_TEMP:-/tmp}/${BIN_NAME}" -go build -o "$BIN_PATH" ./cmd/cigocacher -echo "GOCACHEPROG=$BIN_PATH --cigocached-url $URL --token ${CIGOCACHER_TOKEN}" >> "$GITHUB_ENV" -echo "success=true" >> "$GITHUB_OUTPUT" +go build -o "${BIN_PATH}" ./cmd/cigocacher +echo "GOCACHEPROG=${BIN_PATH} --cache-dir ${CACHE_DIR} --cigocached-url ${URL} --token ${CIGOCACHER_TOKEN}" >> "${GITHUB_ENV}" +echo "success=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/go-cache/action.yml b/.github/actions/go-cache/action.yml index 75a40164d..bc37965d3 100644 --- a/.github/actions/go-cache/action.yml +++ b/.github/actions/go-cache/action.yml @@ -22,5 +22,6 @@ runs: shell: bash env: URL: ${{ inputs.cigocached-url }} + CACHE_DIR: ${{ inputs.cache-dir }} working-directory: ${{ inputs.checkout-path }} run: .github/actions/go-cache/action.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95b95ceee..4803d34ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -213,7 +213,7 @@ jobs: # org with 8 cores and 32 GB of RAM: # https://github.com/organizations/tailscale/settings/actions/github-hosted-runners/1 runs-on: ci-windows-github-1 - needs: gomod-cache + # needs: gomod-cache name: Windows (${{ matrix.name || matrix.shard}}) strategy: fail-fast: false # don't abort the entire matrix if one element fails @@ -230,45 +230,55 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: src + # - name: Restore Go module cache + # uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + # with: + # path: + - uses: samypr100/setup-dev-drive@30f0f98ae5636b2b6501e181dfb3631b9974818d # v4.0.0 + with: + drive-size: 50GB # 300GB available: https://docs.github.com/en/enterprise-cloud@latest/actions/reference/runners/larger-runners#machine-sizes-for-larger-runners + trusted-dev-drive: true + workspace-copy: true - name: Install Go uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version-file: src/go.mod + go-version-file: ${{ env.DEV_DRIVE_WORKSPACE }}/src/go.mod cache: false - - name: Restore Go module cache - uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true + # - name: Restore Go module cache + # uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + # with: + # path: gomodcache + # key: ${{ needs.gomod-cache.outputs.cache-key }} + # enableCrossOsArchive: true - name: Setup cigocacher id: cigocacher-setup uses: ./src/.github/actions/go-cache with: - checkout-path: src + checkout-path: ${{ env.DEV_DRIVE_WORKSPACE }}/src + cache-dir: ${{ env.DEV_DRIVE_WORKSPACE }}/cigocacher cigocached-url: ${{ vars.CIGOCACHED_AZURE_URL }} - name: test if: matrix.key != 'win-bench' # skip on bench builder - working-directory: src + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}/src run: go run ./cmd/testwrapper sharded:${{ matrix.shard }} - name: bench all if: matrix.key == 'win-bench' - working-directory: src + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}/src # Don't use -bench=. -benchtime=1x. # Somewhere in the layers (powershell?) # the equals signs cause great confusion. run: go test ./... -bench . -benchtime 1x -run "^$" - name: Tidy cache - working-directory: src + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}/src shell: bash run: | - find $(go env GOCACHE) -type f -mmin +90 -delete + find ${{ env.DEV_DRIVE_WORKSPACE }}/cigocacher -type f -mmin +90 -delete win-tool-go: # windows-8vpu is a 2022 GitHub-managed runner in our diff --git a/cmd/cigocacher/cigocacher.go b/cmd/cigocacher/cigocacher.go index ffe964b52..8d75dcbe2 100644 --- a/cmd/cigocacher/cigocacher.go +++ b/cmd/cigocacher/cigocacher.go @@ -36,6 +36,7 @@ func main() { auth = flag.Bool("auth", false, "auth with cigocached and exit, printing the access token as output") token = flag.String("token", "", "the cigocached access token to use, as created using --auth") cigocachedURL = flag.String("cigocached-url", "", "optional cigocached URL (scheme, host, and port). empty means to not use one.") + dir = flag.String("cache-dir", "", "cache directory; empty means automatic") verbose = flag.Bool("verbose", false, "enable verbose logging") ) flag.Parse() @@ -54,18 +55,23 @@ func main() { return } - d, err := os.UserCacheDir() - if err != nil { - log.Fatal(err) + if *dir == "" { + d, err := os.UserCacheDir() + if err != nil { + log.Fatal(err) + } + *dir = filepath.Join(d, "go-cacher") + log.Printf("Defaulting to cache dir %v ...", *dir) } - d = filepath.Join(d, "go-cacher") - log.Printf("Defaulting to cache dir %v ...", d) - if err := os.MkdirAll(d, 0750); err != nil { + if err := os.MkdirAll(*dir, 0750); err != nil { log.Fatal(err) } c := &cigocacher{ - disk: &DiskCache{Dir: d}, + disk: &DiskCache{ + Dir: *dir, + Verbose: *verbose, + }, verbose: *verbose, } if *cigocachedURL != "" { @@ -80,8 +86,10 @@ func main() { var p *cacheproc.Process p = &cacheproc.Process{ Close: func() error { - log.Printf("gocacheprog: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)", - p.Gets.Load(), p.GetHits.Load(), p.GetMisses.Load(), p.GetErrors.Load(), p.Puts.Load(), p.PutErrors.Load()) + if c.verbose { + log.Printf("gocacheprog: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)", + p.Gets.Load(), p.GetHits.Load(), p.GetMisses.Load(), p.GetErrors.Load(), p.Puts.Load(), p.PutErrors.Load()) + } return c.close() }, Get: c.get, @@ -235,13 +243,14 @@ func (c *cigocacher) put(ctx context.Context, actionID, outputID string, size in } func (c *cigocacher) close() error { - log.Printf("cigocacher HTTP stats: %d gets (%.1fMiB, %.2fs, %d hits, %d misses, %d errors ignored); %d puts (%.1fMiB, %.2fs, %d errors ignored)", - c.getHTTP.Load(), float64(c.getHTTPBytes.Load())/float64(1<<20), float64(c.getHTTPNanos.Load())/float64(time.Second), c.getHTTPHits.Load(), c.getHTTPMisses.Load(), c.getHTTPErrors.Load(), - c.putHTTP.Load(), float64(c.putHTTPBytes.Load())/float64(1<<20), float64(c.putHTTPNanos.Load())/float64(time.Second), c.putHTTPErrors.Load()) if !c.verbose || c.gocached == nil { return nil } + log.Printf("cigocacher HTTP stats: %d gets (%.1fMiB, %.2fs, %d hits, %d misses, %d errors ignored); %d puts (%.1fMiB, %.2fs, %d errors ignored)", + c.getHTTP.Load(), float64(c.getHTTPBytes.Load())/float64(1<<20), float64(c.getHTTPNanos.Load())/float64(time.Second), c.getHTTPHits.Load(), c.getHTTPMisses.Load(), c.getHTTPErrors.Load(), + c.putHTTP.Load(), float64(c.putHTTPBytes.Load())/float64(1<<20), float64(c.putHTTPNanos.Load())/float64(time.Second), c.putHTTPErrors.Load()) + stats, err := c.gocached.fetchStats() if err != nil { log.Printf("error fetching gocached stats: %v", err) diff --git a/cmd/cigocacher/disk.go b/cmd/cigocacher/disk.go index 68235af6a..e5a8d04f8 100644 --- a/cmd/cigocacher/disk.go +++ b/cmd/cigocacher/disk.go @@ -170,6 +170,7 @@ func writeAtomic(dest string, r io.Reader) (int64, error) { os.Remove(tf.Name()) if runtime.GOOS == "windows" { if st, statErr := os.Stat(dest); statErr == nil && st.Size() == size { + log.Printf("DEBUG: WE DID THE WINTHING") return size, nil } else { log.Printf("DEBUG: %v", statErr)