@ -4,11 +4,10 @@
package taildrop package taildrop
import ( import (
"crypto/sha256"
"errors"
"io"
"io"
"net/http"
"net/url"
"os"
"os"
"strings"
"sync"
"sync"
"time"
"time"
@ -17,10 +16,14 @@ import (
"tailscale.com/version/distro"
"tailscale.com/version/distro"
) )
type incomingFileKey struct {
id ClientID
name string // e.g., "foo.jpeg"
}
type incomingFile struct { type incomingFile struct {
clock tstime . Clock
clock tstime . Clock
name string // "foo.jpg"
started time . Time
started time . Time
size int64 // or -1 if unknown; never 0
size int64 // or -1 if unknown; never 0
w io . Writer // underlying writer
w io . Writer // underlying writer
@ -33,13 +36,6 @@ type incomingFile struct {
lastNotify time . Time
lastNotify time . Time
} }
func ( f * incomingFile ) markAndNotifyDone ( ) {
f . mu . Lock ( )
f . done = true
f . mu . Unlock ( )
f . sendFileNotify ( )
}
func ( f * incomingFile ) Write ( p [ ] byte ) ( n int , err error ) { func ( f * incomingFile ) Write ( p [ ] byte ) ( n int , err error ) {
n , err = f . w . Write ( p )
n , err = f . w . Write ( p )
@ -62,123 +58,197 @@ func (f *incomingFile) Write(p []byte) (n int, err error) {
return n , err
return n , err
} }
// HandlePut receives a file.
// PutFile stores a file into [Manager.Dir] from a given client id.
// It handles an HTTP PUT request to the "/v0/put/{filename}" endpoint,
// The baseName must be a base filename without any slashes.
// where {filename} is a base filename.
// The length is the expected length of content to read from r,
// It returns the number of bytes received and whether it was received successfully.
// it may be negative to indicate that it is unknown.
func ( h * Handler ) HandlePut ( w http . ResponseWriter , r * http . Request ) ( finalSize int64 , success bool ) { //
if ! envknob . CanTaildrop ( ) {
// If there is a failure reading from r, then the partial file is not deleted
http . Error ( w , "Taildrop disabled on device" , http . StatusForbidden )
// for some period of time. The [Manager.PartialFiles] and [Manager.HashPartialFile]
return finalSize , success
// methods may be used to list all partial files and to compute the hash for a
// specific partial file. This allows the client to determine whether to resume
// a partial file. While resuming, PutFile may be called again with a non-zero
// offset to specify where to resume receiving data at.
func ( m * Manager ) PutFile ( id ClientID , baseName string , r io . Reader , offset , length int64 ) ( int64 , error ) {
switch {
case m == nil || m . Dir == "" :
return 0 , ErrNoTaildrop
case ! envknob . CanTaildrop ( ) :
return 0 , ErrNoTaildrop
case distro . Get ( ) == distro . Unraid && ! m . DirectFileMode :
return 0 , ErrNotAccessible
}
dstPath , ok := m . joinDir ( baseName )
if ! ok {
return 0 , ErrInvalidFileName
}
}
if r . Method != "PUT" {
http . Error ( w , "expected method PUT" , http . StatusMethodNotAllowed )
redactAndLogError := func ( action string , err error ) error {
return finalSize , success
err = redactErr ( err )
m . Logf ( "put %v error: %v" , action , err )
return err
}
}
if h == nil || h . Dir == "" {
http . Error ( w , errNoTaildrop . Error ( ) , http . StatusInternalServerError )
avoidPartialRename := m . DirectFileMode && m . AvoidFinalRename
return finalSize , success
if avoidPartialRename {
// Users using AvoidFinalRename are depending on the exact filename
// of the partial files. So avoid injecting the id into it.
id = ""
}
}
if distro . Get ( ) == distro . Unraid && ! h . DirectFileMode {
http . Error ( w , "Taildrop folder not configured or accessible" , http . StatusInternalServerError )
// Check whether there is an in-progress transfer for the file.
return finalSize , success
sendFileNotify := m . SendFileNotify
if sendFileNotify == nil {
sendFileNotify = func ( ) { } // avoid nil panics below
}
}
rawPath := r . URL . EscapedPath ( )
partialPath := dstPath + id . partialSuffix ( )
suffix , ok := strings . CutPrefix ( rawPath , "/v0/put/" )
inFileKey := incomingFileKey { id , baseName }
if ! ok {
inFile , loaded := m . incomingFiles . LoadOrInit ( inFileKey , func ( ) * incomingFile {
http . Error ( w , "misconfigured internals" , http . StatusInternalServerError )
inFile := & incomingFile {
return finalSize , success
clock : m . Clock ,
started : m . Clock . Now ( ) ,
size : length ,
sendFileNotify : sendFileNotify ,
}
}
if suffix == "" {
if m . DirectFileMode {
http . Error ( w , "empty filename" , http . StatusBadRequest )
inFile . partialPath = partialPath
return finalSize , success
}
}
if strings . Contains ( suffix , "/" ) {
return inFile
http . Error ( w , "directories not supported" , http . StatusBadRequest )
} )
return finalSize , success
if loaded {
return 0 , ErrFileExists
}
}
baseName , err := url . PathUnescape ( suffix )
defer m . incomingFiles . Delete ( inFileKey )
// Create (if not already) the partial file with read-write permissions.
f , err := os . OpenFile ( partialPath , os . O_CREATE | os . O_RDWR , 0666 )
if err != nil {
if err != nil {
http . Error ( w , "bad path encoding" , http . StatusBadRequest )
return 0 , redactAndLogError ( "Create" , err )
return finalSize , success
}
}
dstFile , ok := h . diskPath ( baseName )
defer func ( ) {
if ! ok {
f . Close ( ) // best-effort to cleanup dangling file handles
http . Error ( w , "bad filename" , http . StatusBadRequest )
if err != nil {
return finalSize , success
if avoidPartialRename {
os . Remove ( partialPath ) // best-effort
return
}
}
// TODO(bradfitz): prevent same filename being sent by two peers at once
// prevent same filename being sent twice
// TODO: We need to delete partialPath eventually.
if _ , err := os . Stat ( dstFile ) ; err == nil {
// However, this must be done after some period of time.
http . Error ( w , "file exists" , http . StatusConflict )
return finalSize , success
}
}
} ( )
inFile . w = f
partialFile := dstFile + partialSuffix
// A positive offset implies that we are resuming an existing file.
f , err := os . Create ( partialFile )
// Seek to the appropriate offset and truncate the file.
if offset != 0 {
currLength , err := f . Seek ( 0 , io . SeekEnd )
if err != nil {
if err != nil {
h . Logf ( "put Create error: %v" , redactErr ( err ) )
return 0 , redactAndLogError ( "Seek" , err )
http . Error ( w , err . Error ( ) , http . StatusInternalServerError )
return finalSize , success
}
}
defer func ( ) {
if offset < 0 || offset > currLength {
if ! success {
return 0 , redactAndLogError ( "Seek" , err )
os . Remove ( partialFile )
}
}
} ( )
if _ , err := f . Seek ( offset , io . SeekStart ) ; err != nil {
var inFile * incomingFile
return 0 , redactAndLogError ( "Seek" , err )
sendFileNotify := h . SendFileNotify
if sendFileNotify == nil {
sendFileNotify = func ( ) { } // avoid nil panics below
}
}
if r . ContentLength != 0 {
if err := f . Truncate ( offset ) ; err != nil {
inFile = & incomingFile {
return 0 , redactAndLogError ( "Truncate" , err )
clock : h . Clock ,
name : baseName ,
started : h . Clock . Now ( ) ,
size : r . ContentLength ,
w : f ,
sendFileNotify : sendFileNotify ,
}
}
if h . DirectFileMode {
inFile . partialPath = partialFile
}
}
h . incomingFiles . Store ( inFile , struct { } { } )
defer h . incomingFiles . Delete ( inFile )
// Copy the contents of the file.
, err := io . Copy ( inFile , r . Body ) copyLength , err := io . Copy ( inFile , r )
if err != nil {
if err != nil {
err = redactErr ( err )
return 0 , redactAndLogError ( "Copy" , err )
f . Close ( )
h . Logf ( "put Copy error: %v" , err )
http . Error ( w , err . Error ( ) , http . StatusInternalServerError )
return finalSize , success
}
}
finalSize = n
if length >= 0 && copyLength != length {
return 0 , redactAndLogError ( "Copy" , errors . New ( "copied an unexpected number of bytes" ) )
}
}
if err := redactErr ( f . Close ( ) ) ; err != nil {
if err := f . Close ( ) ; err != nil {
h . Logf ( "put Close error: %v" , err )
return 0 , redactAndLogError ( "Close" , err )
http . Error ( w , err . Error ( ) , http . StatusInternalServerError )
return finalSize , success
}
}
if h . DirectFileMode && h . AvoidFinalRename {
fileLength := offset + copyLength
if inFile != nil { // non-zero length; TODO: notify even for zero length
inFile . markAndNotifyDone ( )
// Return early for avoidPartialRename since users of AvoidFinalRename
// are depending on the exact naming of partial files.
if avoidPartialRename {
inFile . mu . Lock ( )
inFile . done = true
inFile . mu . Unlock ( )
m . knownEmpty . Store ( false )
sendFileNotify ( )
return fileLength , nil
}
}
} else {
if err := os . Rename ( partialFile , dstFile ) ; err != nil {
// File has been successfully received, rename the partial file
err = redactErr ( err )
// to the final destination filename. If a file of that name already exists,
h . Logf ( "put final rename: %v" , err )
// then try multiple times with variations of the filename.
http . Error ( w , err . Error ( ) , http . StatusInternalServerError )
computePartialSum := sync . OnceValues ( func ( ) ( [ sha256 . Size ] byte , error ) {
return finalSize , success
return sha256File ( partialPath )
} )
maxRetries := 10
for ; maxRetries > 0 ; maxRetries -- {
// Atomically rename the partial file as the destination file if it doesn't exist.
// Otherwise, it returns the length of the current destination file.
// The operation is atomic.
dstLength , err := func ( ) ( int64 , error ) {
m . renameMu . Lock ( )
defer m . renameMu . Unlock ( )
switch fi , err := os . Stat ( dstPath ) ; {
case os . IsNotExist ( err ) :
return - 1 , os . Rename ( partialPath , dstPath )
case err != nil :
return - 1 , err
default :
return fi . Size ( ) , nil
}
} ( )
if err != nil {
return 0 , redactAndLogError ( "Rename" , err )
}
if dstLength < 0 {
break // we successfully renamed; so stop
}
// Avoid the final rename if a destination file has the same contents.
if dstLength == fileLength {
partialSum , err := computePartialSum ( )
if err != nil {
return 0 , redactAndLogError ( "Rename" , err )
}
dstSum , err := sha256File ( dstPath )
if err != nil {
return 0 , redactAndLogError ( "Rename" , err )
}
if dstSum == partialSum {
if err := os . Remove ( partialPath ) ; err != nil {
return 0 , redactAndLogError ( "Remove" , err )
}
break // we successfully found a content match; so stop
}
}
}
}
// TODO: set modtime
// Choose a new destination filename and try again.
// TODO: some real response
dstPath = NextFilename ( dstPath )
success = true
}
io . WriteString ( w , "{}\n" )
if maxRetries <= 0 {
h . knownEmpty . Store ( false )
return 0 , errors . New ( "too many retries trying to rename partial file" )
}
m . knownEmpty . Store ( false )
sendFileNotify ( )
sendFileNotify ( )
return finalSize , success
return fileLength , nil
}
func sha256File ( file string ) ( out [ sha256 . Size ] byte , err error ) {
h := sha256 . New ( )
f , err := os . Open ( file )
if err != nil {
return out , err
}
defer f . Close ( )
if _ , err := io . Copy ( h , f ) ; err != nil {
return out , err
}
return [ sha256 . Size ] byte ( h . Sum ( nil ) ) , nil
} }
Joe Tsai
1 year ago
committed by