@ -9,7 +9,6 @@ import (
"context"
"context"
"crypto/rand"
"crypto/rand"
"encoding/binary"
"encoding/binary"
"encoding/json"
"fmt"
"fmt"
"io"
"io"
"log"
"log"
@ -19,11 +18,13 @@ import (
"os"
"os"
"regexp"
"regexp"
"runtime"
"runtime"
"slices"
"strconv"
"strconv"
"sync"
"sync"
"sync/atomic"
"sync/atomic"
"time"
"time"
"github.com/go-json-experiment/json/jsontext"
"tailscale.com/envknob"
"tailscale.com/envknob"
"tailscale.com/net/netmon"
"tailscale.com/net/netmon"
"tailscale.com/net/sockstats"
"tailscale.com/net/sockstats"
@ -32,9 +33,26 @@ import (
tslogger "tailscale.com/types/logger"
tslogger "tailscale.com/types/logger"
"tailscale.com/types/logid"
"tailscale.com/types/logid"
"tailscale.com/util/set"
"tailscale.com/util/set"
"tailscale.com/util/truncate"
"tailscale.com/util/zstdframe"
"tailscale.com/util/zstdframe"
) )
// maxSize is the maximum size that a single log entry can be.
// It is also the maximum body size that may be uploaded at a time.
const maxSize = 256 << 10
// maxTextSize is the maximum size for a text log message.
// Note that JSON log messages can be as large as maxSize.
const maxTextSize = 16 << 10
// lowMemRatio reduces maxSize and maxTextSize by this ratio in lowMem mode.
const lowMemRatio = 4
// bufferSize is the typical buffer size to retain.
// It is large enough to handle most log messages,
// but not too large to be a notable waste of memory if retained forever.
const bufferSize = 4 << 10
// DefaultHost is the default host name to upload logs to when
// DefaultHost is the default host name to upload logs to when
// Config.BaseURL isn't provided.
// Config.BaseURL isn't provided.
const DefaultHost = "log.tailscale.io" const DefaultHost = "log.tailscale.io"
@ -175,7 +193,7 @@ type Logger struct {
netMonitor * netmon . Monitor
netMonitor * netmon . Monitor
buffer Buffer
buffer Buffer
drainWake chan struct { } // signal to speed up drain
drainWake chan struct { } // signal to speed up drain
drainBuf bytes . Buffer // owned by drainPending for reuse
drainBuf [ ] byte // owned by drainPending for reuse
flushDelayFn func ( ) time . Duration // negative or zero return value to upload aggressively, or >0 to batch at this delay
flushDelayFn func ( ) time . Duration // negative or zero return value to upload aggressively, or >0 to batch at this delay
flushPending atomic . Bool
flushPending atomic . Bool
sentinel chan int32
sentinel chan int32
@ -194,6 +212,8 @@ type Logger struct {
writeLock sync . Mutex // guards procSequence, flushTimer, buffer.Write calls
writeLock sync . Mutex // guards procSequence, flushTimer, buffer.Write calls
procSequence uint64
procSequence uint64
flushTimer tstime . TimerController // used when flushDelay is >0
flushTimer tstime . TimerController // used when flushDelay is >0
writeBuf [ bufferSize ] byte // owned by Write for reuse
jsonDec jsontext . Decoder // owned by appendTextOrJSONLocked for reuse
shutdownStartMu sync . Mutex // guards the closing of shutdownStart
shutdownStartMu sync . Mutex // guards the closing of shutdownStart
shutdownStart chan struct { } // closed when shutdown begins
shutdownStart chan struct { } // closed when shutdown begins
@ -290,42 +310,66 @@ func (l *Logger) drainBlock() (shuttingDown bool) {
// drainPending drains and encodes a batch of logs from the buffer for upload.
// drainPending drains and encodes a batch of logs from the buffer for upload.
// If no logs are available, drainPending blocks until logs are available.
// If no logs are available, drainPending blocks until logs are available.
func ( l * Logger ) drainPending ( ) ( res [ ] byte ) { // The returned buffer is only valid until the next call to drainPending.
buf := & l . drainBuf
func ( l * Logger ) drainPending ( ) ( b [ ] byte ) {
buf . Reset ( )
b = l . drainBuf [ : 0 ]
buf . WriteByte ( '[' )
b = append ( b , '[' )
entries := 0
defer func ( ) {
b = bytes . TrimRight ( b , "," )
var batchDone bool
b = append ( b , ']' )
const maxLen = 256 << 10
l . drainBuf = b
for buf . Len ( ) < maxLen && ! batchDone {
if len ( b ) <= len ( "[]" ) {
b , err := l . buffer . TryReadLine ( )
b = nil
if err == io . EOF {
}
break
} ( )
} else if err != nil {
b = fmt . Appendf ( nil , "reading ringbuffer: %v" , err )
maxLen := maxSize
batchDone = true
if l . lowMem {
} else if b == nil {
// When operating in a low memory environment, it is better to upload
if entries > 0 {
// in multiple operations than it is to allocate a large body and OOM.
break
// Even if maxLen is less than maxSize, we can still upload an entry
// that is up to maxSize if we happen to encounter one.
maxLen /= lowMemRatio
}
for len ( b ) < maxLen {
line , err := l . buffer . TryReadLine ( )
switch {
case err == io . EOF :
return b
case err != nil :
b = append ( b , '{' )
b = l . appendMetadata ( b , false , true , 0 , 0 , "reading ringbuffer: " + err . Error ( ) , nil , 0 )
b = bytes . TrimRight ( b , "," )
b = append ( b , '}' )
return b
case line == nil :
// If we read at least some log entries, return immediately.
if len ( b ) > len ( "[" ) {
return b
}
}
// We're about to block. If we're holding on to too much memory
// We're about to block. If we're holding on to too much memory
// in our buffer from a previous large write, let it go.
// in our buffer from a previous large write, let it go.
if buf . Available ( ) > 4 << 10 {
if cap ( b ) > bufferSize {
cur := buf . Bytes ( )
b = bytes . Clone ( b )
l . drainBuf = bytes . Buffer { }
l . drainBuf = b
buf . Write ( cur )
}
}
batchDone = l . drainBlock ( )
if shuttingDown := l . drainBlock ( ) ; shuttingDown {
return b
}
continue
continue
}
}
if len ( b ) == 0 {
switch {
case len ( line ) == 0 :
continue
continue
}
case line [ 0 ] == '{' && jsontext . Value ( line ) . IsValid ( ) :
if b [ 0 ] != '{' || ! json . Valid ( b ) {
// This is already a valid JSON object, so just append it.
// This may exceed maxLen, but should be no larger than maxSize
// so long as logic writing into the buffer enforces the limit.
b = append ( b , line ... )
default :
// This is probably a log added to stderr by filch
// This is probably a log added to stderr by filch
// outside of the logtail logger. Encode it.
// outside of the logtail logger. Encode it.
if ! l . explainedRaw {
if ! l . explainedRaw {
@ -336,24 +380,14 @@ func (l *Logger) drainPending() (res []byte) {
l . explainedRaw = true
l . explainedRaw = true
}
}
fmt . Fprintf ( l . stderr , "RAW-STDERR: %s" , b )
fmt . Fprintf ( l . stderr , "RAW-STDERR: %s" , b )
// Do not add a client time, as it could have been
// Do not add a client time, as it could be really old.
// been written a long time ago. Don't include instance key or ID
// Do not include instance key or ID either,
// either, since this came from a different instance.
// since this came from a different instance.
b = l . encodeText ( b , true , 0 , 0 , 0 )
b = l . appendText ( b , line , true , 0 , 0 , 0 )
}
if entries > 0 {
buf . WriteByte ( ',' )
}
buf . Write ( b )
entries ++
}
}
b = append ( b , ',' )
buf . WriteByte ( ']' )
if buf . Len ( ) <= len ( "[]" ) {
return nil
}
}
return b uf. Bytes ( )
return b
} }
// This is the goroutine that repeatedly uploads logs in the background.
// This is the goroutine that repeatedly uploads logs in the background.
@ -573,169 +607,211 @@ func (l *Logger) sendLocked(jsonBlob []byte) (int, error) {
return n , err
return n , err
} }
// TODO: instead of allocating, this should probably just append
// appendMetadata appends optional "logtail", "metrics", and "v" JSON members.
// directly into the output log buffer.
// This assumes dst is already within a JSON object.
func ( l * Logger ) encodeText ( buf [ ] byte , skipClientTime bool , procID uint32 , procSequence uint64 , level int ) [ ] byte { // Each member is comma-terminated.
now := l . clock . Now ( )
func ( l * Logger ) appendMetadata ( dst [ ] byte , skipClientTime , skipMetrics bool , procID uint32 , procSequence uint64 , errDetail string , errData jsontext . Value , level int ) [ ] byte {
// Append optional logtail metadata.
// Factor in JSON encoding overhead to try to only do one alloc
if ! skipClientTime || procID != 0 || procSequence != 0 || errDetail != "" || errData != nil {
// in the make below (so appends don't resize the buffer).
dst = append ( dst , ` "logtail": { ` ... )
overhead := len ( ` { "text": ""}\n ` )
includeLogtail := ! skipClientTime || procID != 0 || procSequence != 0
if includeLogtail {
overhead += len ( ` "logtail": { }, ` )
}
if ! skipClientTime {
if ! skipClientTime {
overhead += len ( ` "client_time": "2006-01-02T15:04:05.999999999Z07:00", ` )
dst = append ( dst , ` "client_time":" ` ... )
dst = l . clock . Now ( ) . UTC ( ) . AppendFormat ( dst , time . RFC3339Nano )
dst = append ( dst , '"' , ',' )
}
}
if procID != 0 {
if procID != 0 {
overhead += len ( ` "proc_id": 4294967296, ` )
dst = append ( dst , ` "proc_id": ` ... )
dst = strconv . AppendUint ( dst , uint64 ( procID ) , 10 )
dst = append ( dst , ',' )
}
}
if procSequence != 0 {
if procSequence != 0 {
overhead += len ( ` "proc_seq": 9007199254740992, ` )
dst = append ( dst , ` "proc_seq": ` ... )
}
dst = strconv . AppendUint ( dst , procSequence , 10 )
// TODO: do a pass over buf and count how many backslashes will be needed?
dst = append ( dst , ',' )
// For now just factor in a dozen.
overhead += 12
// Put a sanity cap on buf's size.
max := 16 << 10
if l . lowMem {
max = 4 << 10
}
var nTruncated int
if len ( buf ) > max {
nTruncated = len ( buf ) - max
// TODO: this can break a UTF-8 character
// mid-encoding. We don't tend to log
// non-ASCII stuff ourselves, but e.g. client
// names might be.
buf = buf [ : max ]
}
}
if errDetail != "" || errData != nil {
b := make ( [ ] byte , 0 , len ( buf ) + overhead )
dst = append ( dst , ` "error": { ` ... )
b = append ( b , '{' )
if errDetail != "" {
dst = append ( dst , ` "detail": ` ... )
if includeLogtail {
dst , _ = jsontext . AppendQuote ( dst , errDetail )
b = append ( b , ` "logtail": { ` ... )
dst = append ( dst , ',' )
if ! skipClientTime {
b = append ( b , ` "client_time": " ` ... )
b = now . UTC ( ) . AppendFormat ( b , time . RFC3339Nano )
b = append ( b , ` ", ` ... )
}
}
if procID != 0 {
if errData != nil {
b = append ( b , ` "proc_id": ` ... )
dst = append ( dst , ` "bad_data": ` ... )
b = strconv . AppendUint ( b , uint64 ( procID ) , 10 )
dst = append ( dst , errData ... )
b = append ( b , ',' )
dst = append ( dst , ',' )
}
}
if procSequence != 0 {
dst = bytes . TrimRight ( dst , "," )
b = append ( b , ` "proc_seq": ` ... )
dst = append ( dst , '}' , ',' )
b = strconv . AppendUint ( b , procSequence , 10 )
b = append ( b , ',' )
}
}
b = bytes . TrimRight ( b , "," )
dst = bytes . TrimRight ( dst , "," )
b = append ( b , "}, " ... )
dst = append ( dst , '}' , ',' )
}
}
if l . metricsDelta != nil {
// Append optional metrics metadata.
if ! skipMetrics && l . metricsDelta != nil {
if d := l . metricsDelta ( ) ; d != "" {
if d := l . metricsDelta ( ) ; d != "" {
b = append ( b , ` "metrics": "` ... )
dst = append ( dst , ` "metrics": "` ... )
b = append ( b , d ... )
dst = append ( dst , d ... )
b = append ( b , ` ", ` ... )
dst = append ( dst , '"' , ',' )
}
}
}
}
// Add the log level, if non-zero. Note that we only use log
// Add the optional log level, if non-zero.
// levels 1 and 2 currently. It's unlikely we'll ever make it
// Note that we only use log levels 1 and 2 currently.
// past 9.
// It's unlikely we'll ever make it past 9.
if level > 0 && level < 10 {
if level > 0 && level < 10 {
b = append ( b , ` "v": ` ... )
dst = append ( dst , ` "v": ` ... )
b = append ( b , '0' + byte ( level ) )
dst = append ( dst , '0' + byte ( level ) )
b = append ( b , ',' )
dst = append ( dst , ',' )
}
}
b = append ( b , "\"text\": \"" ... )
for _ , c := range buf {
return dst
switch c {
case '\b' :
b = append ( b , '\\' , 'b' )
case '\f' :
b = append ( b , '\\' , 'f' )
case '\n' :
b = append ( b , '\\' , 'n' )
case '\r' :
b = append ( b , '\\' , 'r' )
case '\t' :
b = append ( b , '\\' , 't' )
case '"' :
b = append ( b , '\\' , '"' )
case '\\' :
b = append ( b , '\\' , '\\' )
default :
// TODO: what about binary gibberish or non UTF-8?
b = append ( b , c )
} }
// appendText appends a raw text message in the Tailscale JSON log entry format.
func ( l * Logger ) appendText ( dst , src [ ] byte , skipClientTime bool , procID uint32 , procSequence uint64 , level int ) [ ] byte {
dst = slices . Grow ( dst , len ( src ) )
dst = append ( dst , '{' )
dst = l . appendMetadata ( dst , skipClientTime , false , procID , procSequence , "" , nil , level )
if len ( src ) == 0 {
dst = bytes . TrimRight ( dst , "," )
return append ( dst , "}\n" ... )
}
}
if nTruncated > 0 {
b = append ( b , "…+" ... )
// Append the text string, which may be truncated.
b = strconv . AppendInt ( b , int64 ( nTruncated ) , 10 )
// Invalid UTF-8 will be mangled with the Unicode replacement character.
max := maxTextSize
if l . lowMem {
max /= lowMemRatio
}
}
b = append ( b , "\"}\n" ... )
dst = append ( dst , ` "text": ` ... )
return b
dst = appendTruncatedString ( dst , src , max )
return append ( dst , "}\n" ... )
} }
func ( l * Logger ) encodeLocked ( buf [ ] byte , level int ) [ ] byte { // appendTruncatedString appends a JSON string for src,
// truncating the src to be no larger than n.
func appendTruncatedString ( dst , src [ ] byte , n int ) [ ] byte {
srcLen := len ( src )
src = truncate . String ( src , n )
dst , _ = jsontext . AppendQuote ( dst , src ) // ignore error; only occurs for invalid UTF-8
if srcLen > len ( src ) {
dst = dst [ : len ( dst ) - len ( ` " ` ) ] // trim off preceding double-quote
dst = append ( dst , "…+" ... )
dst = strconv . AppendInt ( dst , int64 ( srcLen - len ( src ) ) , 10 )
dst = append ( dst , '"' ) // re-append succeeding double-quote
}
return dst
}
func ( l * Logger ) AppendTextOrJSONLocked ( dst , src [ ] byte ) [ ] byte {
l . clock = tstime . StdClock { }
return l . appendTextOrJSONLocked ( dst , src , 0 )
}
// appendTextOrJSONLocked appends a raw text message or a raw JSON object
// in the Tailscale JSON log format.
func ( l * Logger ) appendTextOrJSONLocked ( dst , src [ ] byte , level int ) [ ] byte {
if l . includeProcSequence {
if l . includeProcSequence {
l . procSequence ++
l . procSequence ++
}
}
if buf [ 0 ] != '{' {
if len ( src ) == 0 || src [ 0 ] != '{' {
return l . encodeText ( buf , l . skipClientTime , l . procID , l . procSequence , level ) // text fast-path
return l . appendText( dst , src , l . skipClientTime , l . procID , l . procSequence , level )
}
}
now := l . clock . Now ( )
// Check whether the input is a valid JSON object and
// whether it contains the reserved "logtail" name at the top-level.
obj := make ( map [ string ] any )
var logtailKeyOffset , logtailValOffset , logtailValLength int
if err := json . Unmarshal ( buf , & obj ) ; err != nil {
validJSON := func ( ) bool {
for k := range obj {
// TODO(dsnet): Avoid allocation of bytes.Buffer struct.
delete ( obj , k )
dec := & l . jsonDec
dec . Reset ( bytes . NewBuffer ( src ) )
if tok , err := dec . ReadToken ( ) ; tok . Kind ( ) != '{' || err != nil {
return false
}
}
obj [ "text" ] = string ( buf )
for dec . PeekKind ( ) != '}' {
keyOffset := dec . InputOffset ( )
tok , err := dec . ReadToken ( )
if err != nil {
return false
}
}
if txt , isStr := obj [ "text" ] . ( string ) ; l . lowMem && isStr && len ( txt ) > 254 {
isLogtail := tok . String ( ) == "logtail"
// TODO(crawshaw): trim to unicode code point
valOffset := dec . InputOffset ( )
obj [ "text" ] = txt [ : 254 ] + "…"
if dec . SkipValue ( ) != nil {
return false
}
}
if isLogtail {
hasLogtail := obj [ "logtail" ] != nil
logtailKeyOffset = int ( keyOffset )
if hasLogtail {
logtailValOffset = int ( valOffset )
obj [ "error_has_logtail" ] = obj [ "logtail" ]
logtailValLength = int ( dec . InputOffset ( ) ) - logtailValOffset
obj [ "logtail" ] = nil
}
}
if ! l . skipClientTime || l . procID != 0 || l . procSequence != 0 {
logtail := map [ string ] any { }
if ! l . skipClientTime {
logtail [ "client_time" ] = now . UTC ( ) . Format ( time . RFC3339Nano )
}
}
if l. procID != 0 {
if tok , err := dec . ReadToken ( ) ; tok . Kind ( ) != '}' || err != nil {
logtail [ "proc_id" ] = l . procID
return false
}
}
if l. procSequence != 0 {
if _, err := dec . ReadToken ( ) ; err != io . EOF {
logtail [ "proc_seq" ] = l . procSequence
return false // trailing junk after JSON object
}
}
obj [ "logtail" ] = logtail
return true
} ( )
// Treat invalid JSON as a raw text message.
if ! validJSON {
return l . appendText ( dst , src , l . skipClientTime , l . procID , l . procSequence , level )
}
// Check whether the JSON payload is too large.
// Due to logtail metadata, the formatted log entry could exceed maxSize.
// That's okay as the Tailscale log service limit is actually 2*maxSize.
// However, so long as logging applications aim to target the maxSize limit,
// there should be no trouble eventually uploading logs.
if len ( src ) > maxSize {
errDetail := fmt . Sprintf ( "entry too large: %d bytes" , len ( src ) )
errData := appendTruncatedString ( nil , src , maxSize / len ( ` \uffff ` ) ) // escaping could increase size
dst = append ( dst , '{' )
dst = l . appendMetadata ( dst , l . skipClientTime , true , l . procID , l . procSequence , errDetail , errData , level )
dst = bytes . TrimRight ( dst , "," )
return append ( dst , "}\n" ... )
}
// Check whether the reserved logtail member occurs in the log data.
// If so, it is moved to the the logtail/error member.
const jsonSeperators = ",:" // per RFC 8259, section 2
const jsonWhitespace = " \n\r\t" // per RFC 8259, section 2
var errDetail string
var errData jsontext . Value
if logtailValLength > 0 {
errDetail = "duplicate logtail member"
errData = bytes . Trim ( src [ logtailValOffset : ] [ : logtailValLength ] , jsonSeperators + jsonWhitespace )
}
dst = slices . Grow ( dst , len ( src ) )
dst = append ( dst , '{' )
dst = l . appendMetadata ( dst , l . skipClientTime , true , l . procID , l . procSequence , errDetail , errData , level )
if logtailValLength > 0 {
// Exclude original logtail member from the message.
dst = appendWithoutNewline ( dst , src [ len ( "{" ) : logtailKeyOffset ] )
dst = bytes . TrimRight ( dst , jsonSeperators + jsonWhitespace )
dst = appendWithoutNewline ( dst , src [ logtailValOffset + logtailValLength : ] )
} else {
dst = appendWithoutNewline ( dst , src [ len ( "{" ) : ] )
}
}
if level > 0 {
dst = bytes . TrimRight ( dst , jsonWhitespace )
obj [ "v" ] = level
dst = dst [ : len ( dst ) - len ( "}" ) ]
dst = bytes . TrimRight ( dst , jsonSeperators + jsonWhitespace )
return append ( dst , "}\n" ... )
} }
b , err := json . Marshal ( obj )
// appendWithoutNewline appends src to dst except that it ignores newlines
if err != nil {
// since newlines are used to frame individual log entries.
fmt . Fprintf ( l . stderr , "logtail: re-encoding JSON failed: %v\n" , err )
func appendWithoutNewline ( dst , src [ ] byte ) [ ] byte {
// I know of no conditions under which this could fail.
for _ , c := range src {
// Report it very loudly.
if c != '\n' {
panic ( "logtail: re-encoding JSON failed: " + err . Error ( ) )
dst = append ( dst , c )
}
}
b = append ( b , '\n' )
}
return b
return dst
} }
// Logf logs to l using the provided fmt-style format and optional arguments.
// Logf logs to l using the provided fmt-style format and optional arguments.
@ -776,7 +852,7 @@ func (l *Logger) Write(buf []byte) (int, error) {
l . writeLock . Lock ( )
l . writeLock . Lock ( )
defer l . writeLock . Unlock ( )
defer l . writeLock . Unlock ( )
b := l . co deLocked( buf , level )
b := l . app endT extOrJSON Locked( l . writeBuf [ : 0 ] , buf , level )
_ , err := l . sendLocked ( b )
_ , err := l . sendLocked ( b )
return inLen , err
return inLen , err
} }
Joe Tsai
8 months ago
committed by