mirror of https://github.com/tailscale/tailscale/
taildrop: implement asynchronous file deletion (#9844)
File resumption requires keeping partial files around for some time, but we must still eventually delete them if never resumed. Thus, we implement asynchronous file deletion, which could spawn a background goroutine to delete the files. We also use the same mechanism for deleting files on Windows, where a file can't be deleted if there is still an open file handle. We can enqueue those with the asynchronous file deleter as well. Updates tailscale/corp#14772 Signed-off-by: Joe Tsai <joetsai@digital-static.net>pull/9859/head
Joe Tsai
8 months ago
committed by
GitHub
parent
33bb2bbfe9
commit
c2a551469c
@ -0,0 +1,182 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package taildrop
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"tailscale.com/syncs"
|
||||
"tailscale.com/tstime"
|
||||
"tailscale.com/types/logger"
|
||||
)
|
||||
|
||||
// deleteDelay is the amount of time to wait before we delete a file.
|
||||
// A shorter value ensures timely deletion of deleted and partial files, while
|
||||
// a longer value provides more opportunity for partial files to be resumed.
|
||||
const deleteDelay = time.Hour
|
||||
|
||||
// fileDeleter manages asynchronous deletion of files after deleteDelay.
|
||||
type fileDeleter struct {
|
||||
logf logger.Logf
|
||||
clock tstime.DefaultClock
|
||||
event func(string) // called for certain events; for testing only
|
||||
dir string
|
||||
|
||||
mu sync.Mutex
|
||||
queue list.List
|
||||
byName map[string]*list.Element
|
||||
|
||||
emptySignal chan struct{} // signal that the queue is empty
|
||||
group syncs.WaitGroup
|
||||
shutdownCtx context.Context
|
||||
shutdown context.CancelFunc
|
||||
}
|
||||
|
||||
// deleteFile is a specific file to delete after deleteDelay.
|
||||
type deleteFile struct {
|
||||
name string
|
||||
inserted time.Time
|
||||
}
|
||||
|
||||
func (d *fileDeleter) Init(logf logger.Logf, clock tstime.DefaultClock, event func(string), dir string) {
|
||||
d.logf = logf
|
||||
d.clock = clock
|
||||
d.dir = dir
|
||||
d.event = event
|
||||
|
||||
// From a cold-start, load the list of partial and deleted files.
|
||||
d.byName = make(map[string]*list.Element)
|
||||
d.emptySignal = make(chan struct{})
|
||||
d.shutdownCtx, d.shutdown = context.WithCancel(context.Background())
|
||||
d.group.Go(func() {
|
||||
d.event("start init")
|
||||
defer d.event("end init")
|
||||
rangeDir(dir, func(de fs.DirEntry) bool {
|
||||
switch {
|
||||
case d.shutdownCtx.Err() != nil:
|
||||
return false // terminate early
|
||||
case !de.Type().IsRegular():
|
||||
return true
|
||||
case strings.Contains(de.Name(), partialSuffix):
|
||||
d.Insert(de.Name())
|
||||
case strings.Contains(de.Name(), deletedSuffix):
|
||||
// Best-effort immediate deletion of deleted files.
|
||||
name := strings.TrimSuffix(de.Name(), deletedSuffix)
|
||||
if os.Remove(filepath.Join(dir, name)) == nil {
|
||||
if os.Remove(filepath.Join(dir, de.Name())) == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Otherwise, enqueue the file for later deletion.
|
||||
d.Insert(de.Name())
|
||||
}
|
||||
return true
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Insert enqueues baseName for eventual deletion.
|
||||
func (d *fileDeleter) Insert(baseName string) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.shutdownCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := d.byName[baseName]; ok {
|
||||
return // already queued for deletion
|
||||
}
|
||||
d.byName[baseName] = d.queue.PushBack(&deleteFile{baseName, d.clock.Now()})
|
||||
if d.queue.Len() == 1 {
|
||||
d.group.Go(func() { d.waitAndDelete(deleteDelay) })
|
||||
}
|
||||
}
|
||||
|
||||
// waitAndDelete is an asynchronous deletion goroutine.
|
||||
// At most one waitAndDelete routine is ever running at a time.
|
||||
// It is not started unless there is at least one file in the queue.
|
||||
func (d *fileDeleter) waitAndDelete(wait time.Duration) {
|
||||
tc, ch := d.clock.NewTimer(wait)
|
||||
defer tc.Stop() // cleanup the timer resource if we stop early
|
||||
d.event("start waitAndDelete")
|
||||
defer d.event("end waitAndDelete")
|
||||
select {
|
||||
case <-d.shutdownCtx.Done():
|
||||
case <-d.emptySignal:
|
||||
case now := <-ch:
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
// Iterate over all files to delete, and delete anything old enough.
|
||||
var next *list.Element
|
||||
var failed []*list.Element
|
||||
for elem := d.queue.Front(); elem != nil; elem = next {
|
||||
next = elem.Next()
|
||||
file := elem.Value.(*deleteFile)
|
||||
if now.Sub(file.inserted) < deleteDelay {
|
||||
break // everything after this is recently inserted
|
||||
}
|
||||
|
||||
// Delete the expired file.
|
||||
if name, ok := strings.CutSuffix(file.name, deletedSuffix); ok {
|
||||
if err := os.Remove(filepath.Join(d.dir, name)); err != nil && !os.IsNotExist(err) {
|
||||
d.logf("could not delete: %v", redactError(err))
|
||||
failed = append(failed, elem)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := os.Remove(filepath.Join(d.dir, file.name)); err != nil && !os.IsNotExist(err) {
|
||||
d.logf("could not delete: %v", redactError(err))
|
||||
failed = append(failed, elem)
|
||||
continue
|
||||
}
|
||||
d.queue.Remove(elem)
|
||||
delete(d.byName, file.name)
|
||||
d.event("deleted " + file.name)
|
||||
}
|
||||
for _, elem := range failed {
|
||||
elem.Value.(*deleteFile).inserted = now // retry after deleteDelay
|
||||
d.queue.MoveToBack(elem)
|
||||
}
|
||||
|
||||
// If there are still some files to delete, retry again later.
|
||||
if d.queue.Len() > 0 {
|
||||
file := d.queue.Front().Value.(*deleteFile)
|
||||
retryAfter := deleteDelay - now.Sub(file.inserted)
|
||||
d.group.Go(func() { d.waitAndDelete(retryAfter) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove dequeues baseName from eventual deletion.
|
||||
func (d *fileDeleter) Remove(baseName string) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if elem := d.byName[baseName]; elem != nil {
|
||||
d.queue.Remove(elem)
|
||||
delete(d.byName, baseName)
|
||||
// Signal to terminate any waitAndDelete goroutines.
|
||||
if d.queue.Len() == 0 {
|
||||
select {
|
||||
case <-d.shutdownCtx.Done():
|
||||
case d.emptySignal <- struct{}{}:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown shuts down the deleter.
|
||||
// It blocks until all goroutines are stopped.
|
||||
func (d *fileDeleter) Shutdown() {
|
||||
d.mu.Lock() // acquire lock to ensure no new goroutines start after shutdown
|
||||
d.shutdown()
|
||||
d.mu.Unlock()
|
||||
d.group.Wait()
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package taildrop
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"tailscale.com/tstest"
|
||||
"tailscale.com/tstime"
|
||||
"tailscale.com/util/must"
|
||||
)
|
||||
|
||||
func TestDeleter(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
must.Do(touchFile(filepath.Join(dir, "foo.partial")))
|
||||
must.Do(touchFile(filepath.Join(dir, "bar.partial")))
|
||||
must.Do(touchFile(filepath.Join(dir, "fizz")))
|
||||
must.Do(touchFile(filepath.Join(dir, "fizz.deleted")))
|
||||
must.Do(touchFile(filepath.Join(dir, "buzz.deleted"))) // lacks a matching "buzz" file
|
||||
|
||||
checkDirectory := func(want ...string) {
|
||||
t.Helper()
|
||||
var got []string
|
||||
for _, de := range must.Get(os.ReadDir(dir)) {
|
||||
got = append(got, de.Name())
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if diff := cmp.Diff(got, want); diff != "" {
|
||||
t.Fatalf("directory mismatch (-got +want):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
clock := tstest.NewClock(tstest.ClockOpts{Start: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)})
|
||||
advance := func(d time.Duration) {
|
||||
t.Helper()
|
||||
t.Logf("advance: %v", d)
|
||||
clock.Advance(d)
|
||||
}
|
||||
|
||||
eventsChan := make(chan string, 1000)
|
||||
checkEvents := func(want ...string) {
|
||||
t.Helper()
|
||||
tm := time.NewTimer(10 * time.Second)
|
||||
defer tm.Stop()
|
||||
var got []string
|
||||
for range want {
|
||||
select {
|
||||
case event := <-eventsChan:
|
||||
t.Logf("event: %s", event)
|
||||
got = append(got, event)
|
||||
case <-tm.C:
|
||||
t.Fatalf("timed out waiting for event: got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(want)
|
||||
if diff := cmp.Diff(got, want); diff != "" {
|
||||
t.Fatalf("events mismatch (-got +want):\n%s", diff)
|
||||
}
|
||||
}
|
||||
eventHook := func(event string) { eventsChan <- event }
|
||||
|
||||
var fd fileDeleter
|
||||
fd.Init(t.Logf, tstime.DefaultClock{Clock: clock}, eventHook, dir)
|
||||
defer fd.Shutdown()
|
||||
insert := func(name string) {
|
||||
t.Helper()
|
||||
t.Logf("insert: %v", name)
|
||||
fd.Insert(name)
|
||||
}
|
||||
remove := func(name string) {
|
||||
t.Helper()
|
||||
t.Logf("remove: %v", name)
|
||||
fd.Remove(name)
|
||||
}
|
||||
|
||||
checkEvents("start init")
|
||||
checkEvents("end init", "start waitAndDelete")
|
||||
checkDirectory("foo.partial", "bar.partial", "buzz.deleted")
|
||||
|
||||
advance(deleteDelay / 2)
|
||||
checkDirectory("foo.partial", "bar.partial", "buzz.deleted")
|
||||
advance(deleteDelay / 2)
|
||||
checkEvents("deleted foo.partial", "deleted bar.partial", "deleted buzz.deleted")
|
||||
checkEvents("end waitAndDelete")
|
||||
checkDirectory()
|
||||
|
||||
must.Do(touchFile(filepath.Join(dir, "one.partial")))
|
||||
insert("one.partial")
|
||||
checkEvents("start waitAndDelete")
|
||||
advance(deleteDelay / 4)
|
||||
must.Do(touchFile(filepath.Join(dir, "two.partial")))
|
||||
insert("two.partial")
|
||||
advance(deleteDelay / 4)
|
||||
must.Do(touchFile(filepath.Join(dir, "three.partial")))
|
||||
insert("three.partial")
|
||||
advance(deleteDelay / 4)
|
||||
must.Do(touchFile(filepath.Join(dir, "four.partial")))
|
||||
insert("four.partial")
|
||||
|
||||
advance(deleteDelay / 4)
|
||||
checkEvents("deleted one.partial")
|
||||
checkDirectory("two.partial", "three.partial", "four.partial")
|
||||
checkEvents("end waitAndDelete", "start waitAndDelete")
|
||||
|
||||
advance(deleteDelay / 4)
|
||||
checkEvents("deleted two.partial")
|
||||
checkDirectory("three.partial", "four.partial")
|
||||
checkEvents("end waitAndDelete", "start waitAndDelete")
|
||||
|
||||
advance(deleteDelay / 4)
|
||||
checkEvents("deleted three.partial")
|
||||
checkDirectory("four.partial")
|
||||
checkEvents("end waitAndDelete", "start waitAndDelete")
|
||||
|
||||
advance(deleteDelay / 4)
|
||||
checkEvents("deleted four.partial")
|
||||
checkDirectory()
|
||||
checkEvents("end waitAndDelete")
|
||||
|
||||
insert("wuzz.partial")
|
||||
checkEvents("start waitAndDelete")
|
||||
remove("wuzz.partial")
|
||||
checkEvents("end waitAndDelete")
|
||||
}
|
||||
Loading…
Reference in New Issue