ipn/{ipnlocal/peerapi, localapi} initial taildrop resume api plumbing (#9798)

This change:
* adds a partial files peerAPI endpoint to get a list of partial files
* adds a helper function to extract the basename of a file
* updates the peer put peerAPI endpoint
* updates the file put localapi endpoint to allow resume functionality

Updates #14772

Signed-off-by: Rhea Ghosh <rhea@tailscale.com>

cmd/tailscaled/depaware.txt

@@ -301,7 +301,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
   LD 💣 tailscale.com/ssh/tailssh                                    from tailscale.com/cmd/tailscaled
         tailscale.com/syncs                                          from tailscale.com/net/netcheck+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale/apitype+
-        tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal
+        tailscale.com/taildrop                                       from tailscale.com/ipn/ipnlocal+
      💣 tailscale.com/tempfork/device                                from tailscale.com/net/tstun/table
   LD    tailscale.com/tempfork/gliderlabs/ssh                        from tailscale.com/ssh/tailssh
         tailscale.com/tempfork/heap                                  from tailscale.com/wgengine/magicsock
@@ -340,6 +340,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/goroutines                                from tailscale.com/ipn/ipnlocal
         tailscale.com/util/groupmember                               from tailscale.com/ipn/ipnauth
      💣 tailscale.com/util/hashx                                     from tailscale.com/util/deephash
+        tailscale.com/util/httphdr                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/httpm                                     from tailscale.com/client/tailscale+
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
    L    tailscale.com/util/linuxfw                                   from tailscale.com/net/netns+

ipn/ipnlocal/peerapi.go

@@ -41,6 +41,7 @@ import (
 	"tailscale.com/taildrop"
 	"tailscale.com/types/views"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/httphdr"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -304,6 +305,10 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Frame-Options", "DENY")
 		w.Header().Set("X-Content-Type-Options", "nosniff")
 	}
+	if strings.HasPrefix(r.URL.Path, "/v0/partial-files/") {
+		h.handlePartialFileGet(w, r)
+		return
+	}
 	if strings.HasPrefix(r.URL.Path, "/v0/put/") {
 		metricPutCalls.Add(1)
 		h.handlePeerPut(w, r)
@@ -626,41 +631,99 @@ func (h *peerAPIHandler) peerHasCap(wantCap tailcfg.PeerCapability) bool {
 	return h.ps.b.PeerCaps(h.remoteAddr.Addr()).HasCapability(wantCap)
 }
 
-func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
-	if !h.canPutFile() {
-		http.Error(w, "Taildrop access denied", http.StatusForbidden)
-		return
+var errMisconfiguredInternals = errors.New("misconfigured internals")
+
+func (h *peerAPIHandler) extractBaseName(rawPath, prefix string) (ret string, err error) {
+	prefix, ok := strings.CutPrefix(rawPath, prefix)
+	if !ok {
+		return "", errMisconfiguredInternals
+	}
+	if prefix == "" {
+		return "", taildrop.ErrInvalidFileName
+	}
+	if strings.Contains(prefix, "/") {
+		return "", taildrop.ErrInvalidFileName
+	}
+	baseName, err := url.PathUnescape(prefix)
+	if err == errMisconfiguredInternals {
+		return "", errMisconfiguredInternals
+	} else if err != nil {
+		return "", taildrop.ErrInvalidFileName
 	}
+	return baseName, nil
+}
+
+func (h *peerAPIHandler) handlePartialFileGet(w http.ResponseWriter, r *http.Request) {
 	if !h.ps.b.hasCapFileSharing() {
-		http.Error(w, "file sharing not enabled by Tailscale admin", http.StatusForbidden)
+		http.Error(w, taildrop.ErrNoTaildrop.Error(), http.StatusForbidden)
 		return
 	}
-	if r.Method != "PUT" {
-		http.Error(w, "expected method PUT", http.StatusMethodNotAllowed)
+	if r.Method != "GET" {
+		http.Error(w, "expected method GET", http.StatusMethodNotAllowed)
 		return
 	}
-	rawPath := r.URL.EscapedPath()
-	suffix, ok := strings.CutPrefix(rawPath, "/v0/put/")
-	if !ok {
-		http.Error(w, "misconfigured internals", http.StatusInternalServerError)
+	var resp any
+	var err error
+	id := taildrop.ClientID(h.peerNode.StableID())
+
+	if r.URL.Path == "/v0/partial-files/" {
+		resp, err = h.ps.taildrop.PartialFiles(id)
+	} else {
+		baseName, _ := h.extractBaseName(r.URL.EscapedPath(), "/v0/partial-files/")
+		ranges, ok := httphdr.ParseRange(r.Header.Get("Range"))
+		if !ok || len(ranges) != 1 || ranges[0].Length < 0 {
+			http.Error(w, "invalid Range header", http.StatusBadRequest)
+			return
+		}
+		offset := ranges[0].Start
+		length := ranges[0].Length
+		if length == 0 {
+			length = -1 // httphdr.Range.Length == 0 implies reading the rest of file
+		}
+		resp, err = h.ps.taildrop.HashPartialFile(id, baseName, offset, length)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if err := json.NewEncoder(w).Encode(resp); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+}
+
+func (h *peerAPIHandler) handlePeerPut(w http.ResponseWriter, r *http.Request) {
+	if !h.canPutFile() {
+		http.Error(w, taildrop.ErrNoTaildrop.Error(), http.StatusForbidden)
 		return
 	}
-	if suffix == "" {
-		http.Error(w, "empty filename", http.StatusBadRequest)
+	if !h.ps.b.hasCapFileSharing() {
+		http.Error(w, "file sharing not enabled by Tailscale admin", http.StatusForbidden)
 		return
 	}
-	if strings.Contains(suffix, "/") {
-		http.Error(w, "directories not supported", http.StatusBadRequest)
+	if r.Method != "PUT" {
+		http.Error(w, "expected method PUT", http.StatusMethodNotAllowed)
 		return
 	}
-	baseName, err := url.PathUnescape(suffix)
+	baseName, err := h.extractBaseName(r.URL.EscapedPath(), "/v0/put/")
 	if err != nil {
-		http.Error(w, "bad path encoding", http.StatusBadRequest)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 	t0 := h.ps.b.clock.Now()
-	// TODO(rhea,joetsai): Set the client ID and starting offset.
-	n, err := h.ps.taildrop.PutFile("", baseName, r.Body, 0, r.ContentLength)
+	id := taildrop.ClientID(h.peerNode.StableID())
+
+	var offset int64
+	if rangeHdr := r.Header.Get("Range"); rangeHdr != "" {
+		ranges, ok := httphdr.ParseRange(rangeHdr)
+		if !ok || len(ranges) != 1 || ranges[0].Length != 0 {
+			http.Error(w, "invalid Range header", http.StatusBadRequest)
+			return
+		}
+		offset = ranges[0].Start
+	}
+	n, err := h.ps.taildrop.PutFile(taildrop.ClientID(fmt.Sprint(id)), baseName, r.Body, offset, r.ContentLength)
 	switch err {
 	case nil:
 		d := h.ps.b.clock.Since(t0).Round(time.Second / 10)

ipn/ipnlocal/peerapi_test.go

@@ -173,7 +173,7 @@ func TestHandlePeerAPI(t *testing.T) {
 			reqs:       []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo", nil)},
 			checks: checks(
 				httpStatus(http.StatusForbidden),
-				bodyContains("Taildrop access denied"),
+				bodyContains("Taildrop disabled"),
 			),
 		},
 		{
@@ -280,7 +280,7 @@ func TestHandlePeerAPI(t *testing.T) {
 			reqs:       []*http.Request{httptest.NewRequest("PUT", "/v0/put/", nil)},
 			checks: checks(
 				httpStatus(400),
-				bodyContains("empty filename"),
+				bodyContains("invalid filename"),
 			),
 		},
 		{
@@ -290,7 +290,7 @@ func TestHandlePeerAPI(t *testing.T) {
 			reqs:       []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo/bar", nil)},
 			checks: checks(
 				httpStatus(400),
-				bodyContains("directories not supported"),
+				bodyContains("invalid filename"),
 			),
 		},
 		{

ipn/localapi/localapi.go

@@ -37,6 +37,7 @@ import (
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/portmapper"
 	"tailscale.com/tailcfg"
+	"tailscale.com/taildrop"
 	"tailscale.com/tka"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
@@ -45,6 +46,7 @@ import (
 	"tailscale.com/types/ptr"
 	"tailscale.com/types/tkatype"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/httphdr"
 	"tailscale.com/util/httpm"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/osdiag"
@@ -1293,12 +1295,52 @@ func (h *Handler) serveFilePut(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus peer URL", http.StatusInternalServerError)
 		return
 	}
-	outReq, err := http.NewRequestWithContext(r.Context(), "PUT", "http://peer/v0/put/"+filenameEscaped, r.Body)
+
+	// Before we PUT a file we check to see if there are any existing partial file and if so,
+	// we resume the upload from where we left off by sending the remaining file instead of
+	// the full file.
+	offset, remainingBody, err := taildrop.ResumeReader(r.Body, func(offset, length int64) (taildrop.FileChecksums, error) {
+		client := &http.Client{
+			Transport: h.b.Dialer().PeerAPITransport(),
+			Timeout:   10 * time.Second,
+		}
+		req, err := http.NewRequestWithContext(r.Context(), "GET", "http://peer/v0/partial-files/"+filenameEscaped, nil)
+		if err != nil {
+			return taildrop.FileChecksums{}, err
+		}
+
+		rangeHdr, ok := httphdr.FormatRange([]httphdr.Range{{Start: offset, Length: length}})
+		if !ok {
+			return taildrop.FileChecksums{}, fmt.Errorf("invalid offset and length")
+		}
+		req.Header.Set("Range", rangeHdr)
+		resp, err := client.Do(req)
+		if err != nil {
+			return taildrop.FileChecksums{}, err
+		}
+
+		var checksums taildrop.FileChecksums
+		err = json.NewDecoder(resp.Body).Decode(&checksums)
+		return checksums, err
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	outReq, err := http.NewRequestWithContext(r.Context(), "PUT", "http://peer/v0/put/"+filenameEscaped, remainingBody)
 	if err != nil {
 		http.Error(w, "bogus outreq", http.StatusInternalServerError)
 		return
 	}
 	outReq.ContentLength = r.ContentLength
+	if offset > 0 {
+		rangeHdr, _ := httphdr.FormatRange([]httphdr.Range{{offset, 0}})
+		outReq.Header.Set("Range", rangeHdr)
+		if outReq.ContentLength >= 0 {
+			outReq.ContentLength -= offset
+		}
+	}
 
 	rp := httputil.NewSingleHostReverseProxy(dstURL)
 	rp.Transport = h.b.Dialer().PeerAPITransport()