mirror of https://github.com/tailscale/tailscale/
cmd/jsonimports: add static analyzer for consistent "json" imports (#17669)
This migrates an internal tool to open source so that we can run it on the tailscale.com module as well. We add the "util/safediff" also as a dependency of the tool. This PR does not yet set up a CI to run this analyzer. Updates tailscale/corp#791 Signed-off-by: Joe Tsai <joetsai@digital-static.net>pull/17699/head
Joe Tsai
1 month ago
committed by
GitHub
parent
09a2a1048d
commit
fcb614a53e
@ -0,0 +1,175 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"go/ast"
|
||||||
|
"go/format"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"go/types"
|
||||||
|
"path"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"tailscale.com/util/must"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mustFormatFile formats a Go source file and adjust "json" imports.
|
||||||
|
// It panics if there are any parsing errors.
|
||||||
|
//
|
||||||
|
// - "encoding/json" is imported under the name "jsonv1" or "jsonv1std"
|
||||||
|
// - "encoding/json/v2" is rewritten to import "github.com/go-json-experiment/json" instead
|
||||||
|
// - "encoding/json/jsontext" is rewritten to import "github.com/go-json-experiment/json/jsontext" instead
|
||||||
|
// - "github.com/go-json-experiment/json" is imported under the name "jsonv2"
|
||||||
|
// - "github.com/go-json-experiment/json/v1" is imported under the name "jsonv1"
|
||||||
|
//
|
||||||
|
// If no changes to the file is made, it returns input.
|
||||||
|
func mustFormatFile(in []byte) (out []byte) {
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
f := must.Get(parser.ParseFile(fset, "", in, parser.ParseComments))
|
||||||
|
|
||||||
|
// Check for the existence of "json" imports.
|
||||||
|
jsonImports := make(map[string][]*ast.ImportSpec)
|
||||||
|
for _, imp := range f.Imports {
|
||||||
|
switch pkgPath := must.Get(strconv.Unquote(imp.Path.Value)); pkgPath {
|
||||||
|
case
|
||||||
|
"encoding/json",
|
||||||
|
"encoding/json/v2",
|
||||||
|
"encoding/json/jsontext",
|
||||||
|
"github.com/go-json-experiment/json",
|
||||||
|
"github.com/go-json-experiment/json/v1",
|
||||||
|
"github.com/go-json-experiment/json/jsontext":
|
||||||
|
jsonImports[pkgPath] = append(jsonImports[pkgPath], imp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(jsonImports) == 0 {
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
|
||||||
|
// Best-effort local type-check of the file
|
||||||
|
// to resolve local declarations to detect shadowed variables.
|
||||||
|
typeInfo := &types.Info{Uses: make(map[*ast.Ident]types.Object)}
|
||||||
|
(&types.Config{
|
||||||
|
Error: func(err error) {},
|
||||||
|
}).Check("", fset, []*ast.File{f}, typeInfo)
|
||||||
|
|
||||||
|
// Rewrite imports to instead use "github.com/go-json-experiment/json".
|
||||||
|
// This ensures that code continues to build even if
|
||||||
|
// goexperiment.jsonv2 is *not* specified.
|
||||||
|
// As of https://github.com/go-json-experiment/json/pull/186,
|
||||||
|
// imports to "github.com/go-json-experiment/json" are identical
|
||||||
|
// to the standard library if built with goexperiment.jsonv2.
|
||||||
|
for fromPath, toPath := range map[string]string{
|
||||||
|
"encoding/json/v2": "github.com/go-json-experiment/json",
|
||||||
|
"encoding/json/jsontext": "github.com/go-json-experiment/json/jsontext",
|
||||||
|
} {
|
||||||
|
for _, imp := range jsonImports[fromPath] {
|
||||||
|
imp.Path.Value = strconv.Quote(toPath)
|
||||||
|
jsonImports[toPath] = append(jsonImports[toPath], imp)
|
||||||
|
}
|
||||||
|
delete(jsonImports, fromPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// While in a transitory state, where both v1 and v2 json imports
|
||||||
|
// may exist in our codebase, always explicitly import with
|
||||||
|
// either jsonv1 or jsonv2 in the package name to avoid ambiguities
|
||||||
|
// when looking at a particular Marshal or Unmarshal call site.
|
||||||
|
renames := make(map[string]string) // mapping of old names to new names
|
||||||
|
deletes := make(map[*ast.ImportSpec]bool) // set of imports to delete
|
||||||
|
for pkgPath, imps := range jsonImports {
|
||||||
|
var newName string
|
||||||
|
switch pkgPath {
|
||||||
|
case "encoding/json":
|
||||||
|
newName = "jsonv1"
|
||||||
|
// If "github.com/go-json-experiment/json/v1" is also imported,
|
||||||
|
// then use jsonv1std for "encoding/json" to avoid a conflict.
|
||||||
|
if len(jsonImports["github.com/go-json-experiment/json/v1"]) > 0 {
|
||||||
|
newName += "std"
|
||||||
|
}
|
||||||
|
case "github.com/go-json-experiment/json":
|
||||||
|
newName = "jsonv2"
|
||||||
|
case "github.com/go-json-experiment/json/v1":
|
||||||
|
newName = "jsonv1"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rename the import if different than expected.
|
||||||
|
if oldName := importName(imps[0]); oldName != newName && newName != "" {
|
||||||
|
renames[oldName] = newName
|
||||||
|
pos := imps[0].Pos() // preserve original positioning
|
||||||
|
imps[0].Name = ast.NewIdent(newName)
|
||||||
|
imps[0].Name.NamePos = pos
|
||||||
|
}
|
||||||
|
|
||||||
|
// For all redundant imports, use the first imported name.
|
||||||
|
for _, imp := range imps[1:] {
|
||||||
|
renames[importName(imp)] = importName(imps[0])
|
||||||
|
deletes[imp] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(deletes) > 0 {
|
||||||
|
f.Imports = slices.DeleteFunc(f.Imports, func(imp *ast.ImportSpec) bool {
|
||||||
|
return deletes[imp]
|
||||||
|
})
|
||||||
|
for _, decl := range f.Decls {
|
||||||
|
if genDecl, ok := decl.(*ast.GenDecl); ok && genDecl.Tok == token.IMPORT {
|
||||||
|
genDecl.Specs = slices.DeleteFunc(genDecl.Specs, func(spec ast.Spec) bool {
|
||||||
|
return deletes[spec.(*ast.ImportSpec)]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(renames) > 0 {
|
||||||
|
ast.Walk(astVisitor(func(n ast.Node) bool {
|
||||||
|
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||||
|
if id, ok := sel.X.(*ast.Ident); ok {
|
||||||
|
// Just because the selector looks like "json.Marshal"
|
||||||
|
// does not mean that it is referencing the "json" package.
|
||||||
|
// There could be a local "json" declaration that shadows
|
||||||
|
// the package import. Check partial type information
|
||||||
|
// to see if there was a local declaration.
|
||||||
|
if obj, ok := typeInfo.Uses[id]; ok {
|
||||||
|
if _, ok := obj.(*types.PkgName); !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if newName, ok := renames[id.String()]; ok {
|
||||||
|
id.Name = newName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}), f)
|
||||||
|
}
|
||||||
|
|
||||||
|
bb := new(bytes.Buffer)
|
||||||
|
must.Do(format.Node(bb, fset, f))
|
||||||
|
return must.Get(format.Source(bb.Bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// importName is the local package name used for an import.
|
||||||
|
// If no explicit local name is used, then it uses string parsing
|
||||||
|
// to derive the package name from the path, relying on the convention
|
||||||
|
// that the package name is the base name of the package path.
|
||||||
|
func importName(imp *ast.ImportSpec) string {
|
||||||
|
if imp.Name != nil {
|
||||||
|
return imp.Name.String()
|
||||||
|
}
|
||||||
|
pkgPath, _ := strconv.Unquote(imp.Path.Value)
|
||||||
|
pkgPath = strings.TrimRight(pkgPath, "/v0123456789") // exclude version directories
|
||||||
|
return path.Base(pkgPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// astVisitor is a function that implements [ast.Visitor].
|
||||||
|
type astVisitor func(ast.Node) bool
|
||||||
|
|
||||||
|
func (f astVisitor) Visit(node ast.Node) ast.Visitor {
|
||||||
|
if !f(node) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
@ -0,0 +1,162 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/format"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"tailscale.com/util/must"
|
||||||
|
"tailscale.com/util/safediff"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFormatFile(t *testing.T) {
|
||||||
|
tests := []struct{ in, want string }{{
|
||||||
|
in: `package foobar
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
jsonv2exp "github.com/go-json-experiment/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
jsonv2exp.Marshal()
|
||||||
|
{
|
||||||
|
var json T // deliberately shadow "json" package name
|
||||||
|
json.Marshal() // should not be re-written
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
|
||||||
|
import (
|
||||||
|
jsonv1 "encoding/json"
|
||||||
|
jsonv2 "github.com/go-json-experiment/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
jsonv1.Marshal()
|
||||||
|
jsonv2.Marshal()
|
||||||
|
{
|
||||||
|
var json T // deliberately shadow "json" package name
|
||||||
|
json.Marshal() // should not be re-written
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}, {
|
||||||
|
in: `package foobar
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/go-json-experiment/json"
|
||||||
|
jsonv2exp "github.com/go-json-experiment/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
jsonv2exp.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
import (
|
||||||
|
jsonv2 "github.com/go-json-experiment/json"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
jsonv2.Marshal()
|
||||||
|
jsonv2.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}, {
|
||||||
|
in: `package foobar
|
||||||
|
import "github.com/go-json-experiment/json/v1"
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
import jsonv1 "github.com/go-json-experiment/json/v1"
|
||||||
|
func main() {
|
||||||
|
jsonv1.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}, {
|
||||||
|
in: `package foobar
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
jsonv1in2 "github.com/go-json-experiment/json/v1"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
jsonv1in2.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
import (
|
||||||
|
jsonv1std "encoding/json"
|
||||||
|
jsonv1 "github.com/go-json-experiment/json/v1"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
jsonv1std.Marshal()
|
||||||
|
jsonv1.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}, {
|
||||||
|
in: `package foobar
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
jsonv1in2 "github.com/go-json-experiment/json/v1"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
jsonv1in2.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
import (
|
||||||
|
jsonv1std "encoding/json"
|
||||||
|
jsonv1 "github.com/go-json-experiment/json/v1"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
jsonv1std.Marshal()
|
||||||
|
jsonv1.Marshal()
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}, {
|
||||||
|
in: `package foobar
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
j2 "encoding/json/v2"
|
||||||
|
"encoding/json/jsontext"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
json.Marshal()
|
||||||
|
j2.Marshal()
|
||||||
|
jsontext.NewEncoder
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
want: `package foobar
|
||||||
|
import (
|
||||||
|
jsonv1 "encoding/json"
|
||||||
|
jsonv2 "github.com/go-json-experiment/json"
|
||||||
|
"github.com/go-json-experiment/json/jsontext"
|
||||||
|
)
|
||||||
|
func main() {
|
||||||
|
jsonv1.Marshal()
|
||||||
|
jsonv2.Marshal()
|
||||||
|
jsontext.NewEncoder
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
}}
|
||||||
|
for _, tt := range tests {
|
||||||
|
got := string(must.Get(format.Source([]byte(tt.in))))
|
||||||
|
got = string(mustFormatFile([]byte(got)))
|
||||||
|
want := string(must.Get(format.Source([]byte(tt.want))))
|
||||||
|
if got != want {
|
||||||
|
diff, _ := safediff.Lines(got, want, -1)
|
||||||
|
t.Errorf("mismatch (-got +want)\n%s", diff)
|
||||||
|
t.Error(got)
|
||||||
|
t.Error(want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,124 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
// The jsonimports tool formats all Go source files in the repository
|
||||||
|
// to enforce that "json" imports are consistent.
|
||||||
|
//
|
||||||
|
// With Go 1.25, the "encoding/json/v2" and "encoding/json/jsontext"
|
||||||
|
// packages are now available under goexperiment.jsonv2.
|
||||||
|
// This leads to possible confusion over the following:
|
||||||
|
//
|
||||||
|
// - "encoding/json"
|
||||||
|
// - "encoding/json/v2"
|
||||||
|
// - "encoding/json/jsontext"
|
||||||
|
// - "github.com/go-json-experiment/json/v1"
|
||||||
|
// - "github.com/go-json-experiment/json"
|
||||||
|
// - "github.com/go-json-experiment/json/jsontext"
|
||||||
|
//
|
||||||
|
// In order to enforce consistent usage, we apply the following rules:
|
||||||
|
//
|
||||||
|
// - Until the Go standard library formally accepts "encoding/json/v2"
|
||||||
|
// and "encoding/json/jsontext" into the standard library
|
||||||
|
// (i.e., they are no longer considered experimental),
|
||||||
|
// we forbid any code from directly importing those packages.
|
||||||
|
// Go code should instead import "github.com/go-json-experiment/json"
|
||||||
|
// and "github.com/go-json-experiment/json/jsontext".
|
||||||
|
// The latter packages contain aliases to the standard library
|
||||||
|
// if built on Go 1.25 with the goexperiment.jsonv2 tag specified.
|
||||||
|
//
|
||||||
|
// - Imports of "encoding/json" or "github.com/go-json-experiment/json/v1"
|
||||||
|
// must be explicitly imported under the package name "jsonv1".
|
||||||
|
// If both packages need to be imported, then the former should
|
||||||
|
// be imported under the package name "jsonv1std".
|
||||||
|
//
|
||||||
|
// - Imports of "github.com/go-json-experiment/json"
|
||||||
|
// must be explicitly imported under the package name "jsonv2".
|
||||||
|
//
|
||||||
|
// The latter two rules exist to provide clarity when reading code.
|
||||||
|
// Without them, it is unclear whether "json.Marshal" refers to v1 or v2.
|
||||||
|
// With them, however, it is clear that "jsonv1.Marshal" is calling v1 and
|
||||||
|
// that "jsonv2.Marshal" is calling v2.
|
||||||
|
//
|
||||||
|
// TODO(@joetsai): At this present moment, there is no guidance given on
|
||||||
|
// whether to use v1 or v2 for newly written Go source code.
|
||||||
|
// I will write a document in the near future providing more guidance.
|
||||||
|
// Feel free to continue using v1 "encoding/json" as you are accustomed to.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"tailscale.com/syncs"
|
||||||
|
"tailscale.com/util/must"
|
||||||
|
"tailscale.com/util/safediff"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
update := flag.Bool("update", false, "update all Go source files")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// Change working directory to Git repository root.
|
||||||
|
repoRoot := strings.TrimSuffix(string(must.Get(exec.Command(
|
||||||
|
"git", "rev-parse", "--show-toplevel",
|
||||||
|
).Output())), "\n")
|
||||||
|
must.Do(os.Chdir(repoRoot))
|
||||||
|
|
||||||
|
// Iterate over all indexed files in the Git repository.
|
||||||
|
var printMu sync.Mutex
|
||||||
|
var group sync.WaitGroup
|
||||||
|
sema := syncs.NewSemaphore(runtime.NumCPU())
|
||||||
|
var numDiffs int
|
||||||
|
files := string(must.Get(exec.Command("git", "ls-files").Output()))
|
||||||
|
for file := range strings.Lines(files) {
|
||||||
|
sema.Acquire()
|
||||||
|
group.Go(func() {
|
||||||
|
defer sema.Release()
|
||||||
|
|
||||||
|
// Ignore non-Go source files.
|
||||||
|
file = strings.TrimSuffix(file, "\n")
|
||||||
|
if !strings.HasSuffix(file, ".go") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format all "json" imports in the Go source file.
|
||||||
|
srcIn := must.Get(os.ReadFile(file))
|
||||||
|
srcOut := mustFormatFile(srcIn)
|
||||||
|
|
||||||
|
// Print differences with each formatted file.
|
||||||
|
if !bytes.Equal(srcIn, srcOut) {
|
||||||
|
numDiffs++
|
||||||
|
|
||||||
|
printMu.Lock()
|
||||||
|
fmt.Println(file)
|
||||||
|
lines, _ := safediff.Lines(string(srcIn), string(srcOut), -1)
|
||||||
|
for line := range strings.Lines(lines) {
|
||||||
|
fmt.Print("\t", line)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
printMu.Unlock()
|
||||||
|
|
||||||
|
// If -update is specified, write out the changes.
|
||||||
|
if *update {
|
||||||
|
mode := must.Get(os.Stat(file)).Mode()
|
||||||
|
must.Do(os.WriteFile(file, srcOut, mode))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
|
||||||
|
// Report whether any differences were detected.
|
||||||
|
if numDiffs > 0 && !*update {
|
||||||
|
fmt.Printf(`%d files with "json" imports that need formatting`+"\n", numDiffs)
|
||||||
|
fmt.Println("Please run:")
|
||||||
|
fmt.Println("\t./tool/go run tailscale.com/cmd/jsonimports -update")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,280 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
// Package safediff computes the difference between two lists.
|
||||||
|
//
|
||||||
|
// It is guaranteed to run in O(n), but may not produce an optimal diff.
|
||||||
|
// Most diffing algorithms produce optimal diffs but run in O(n²).
|
||||||
|
// It is safe to pass in untrusted input.
|
||||||
|
package safediff
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"github.com/google/go-cmp/cmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
var diffTest = false
|
||||||
|
|
||||||
|
// Lines constructs a humanly readable line-by-line diff from x to y.
|
||||||
|
// The output (if multiple lines) is guaranteed to be no larger than maxSize,
|
||||||
|
// by truncating the output if necessary. A negative maxSize enforces no limit.
|
||||||
|
//
|
||||||
|
// Example diff:
|
||||||
|
//
|
||||||
|
// … 440 identical lines
|
||||||
|
// "ssh": [
|
||||||
|
// … 35 identical lines
|
||||||
|
// {
|
||||||
|
// - "src": ["maisem@tailscale.com"],
|
||||||
|
// - "dst": ["tag:maisem-test"],
|
||||||
|
// - "users": ["maisem", "root"],
|
||||||
|
// - "action": "check",
|
||||||
|
// - // "recorder": ["100.12.34.56:80"],
|
||||||
|
// + "src": ["maisem@tailscale.com"],
|
||||||
|
// + "dst": ["tag:maisem-test"],
|
||||||
|
// + "users": ["maisem", "root"],
|
||||||
|
// + "action": "check",
|
||||||
|
// + "recorder": ["node:recorder-2"],
|
||||||
|
// },
|
||||||
|
// … 77 identical lines
|
||||||
|
// ],
|
||||||
|
// … 345 identical lines
|
||||||
|
//
|
||||||
|
// Meaning of each line prefix:
|
||||||
|
//
|
||||||
|
// - '…' precedes a summary statement
|
||||||
|
// - ' ' precedes an identical line printed for context
|
||||||
|
// - '-' precedes a line removed from x
|
||||||
|
// - '+' precedes a line inserted from y
|
||||||
|
//
|
||||||
|
// The diffing algorithm runs in O(n) and is safe to use with untrusted inputs.
|
||||||
|
func Lines(x, y string, maxSize int) (out string, truncated bool) {
|
||||||
|
// Convert x and y into a slice of lines and compute the edit-script.
|
||||||
|
xs := strings.Split(x, "\n")
|
||||||
|
ys := strings.Split(y, "\n")
|
||||||
|
es := diffStrings(xs, ys)
|
||||||
|
|
||||||
|
// Modify the edit-script to support printing identical lines of context.
|
||||||
|
const identicalContext edit = '*' // special edit code to indicate printed line
|
||||||
|
var xi, yi int // index into xs or ys
|
||||||
|
isIdentical := func(e edit) bool { return e == identical || e == identicalContext }
|
||||||
|
indentOf := func(s string) string { return s[:len(s)-len(strings.TrimLeftFunc(s, unicode.IsSpace))] }
|
||||||
|
for i, e := range es {
|
||||||
|
if isIdentical(e) {
|
||||||
|
// Print current line if adjacent symbols are non-identical.
|
||||||
|
switch {
|
||||||
|
case i-1 >= 0 && !isIdentical(es[i-1]):
|
||||||
|
es[i] = identicalContext
|
||||||
|
case i+1 < len(es) && !isIdentical(es[i+1]):
|
||||||
|
es[i] = identicalContext
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Print any preceding or succeeding lines,
|
||||||
|
// where the leading indent is a prefix of the current indent.
|
||||||
|
// Indentation often indicates a parent-child relationship
|
||||||
|
// in structured source code.
|
||||||
|
addParents := func(ss []string, si, direction int) {
|
||||||
|
childIndent := indentOf(ss[si])
|
||||||
|
for j := direction; i+j >= 0 && i+j < len(es) && isIdentical(es[i+j]); j += direction {
|
||||||
|
parentIndent := indentOf(ss[si+j])
|
||||||
|
if strings.HasPrefix(childIndent, parentIndent) && len(parentIndent) < len(childIndent) && parentIndent != "" {
|
||||||
|
es[i+j] = identicalContext
|
||||||
|
childIndent = parentIndent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch e {
|
||||||
|
case removed, modified: // arbitrarily use the x value for modified values
|
||||||
|
addParents(xs, xi, -1)
|
||||||
|
addParents(xs, xi, +1)
|
||||||
|
case inserted:
|
||||||
|
addParents(ys, yi, -1)
|
||||||
|
addParents(ys, yi, +1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if e != inserted {
|
||||||
|
xi++
|
||||||
|
}
|
||||||
|
if e != removed {
|
||||||
|
yi++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show the line for a single hidden identical line,
|
||||||
|
// since it occupies the same vertical height.
|
||||||
|
for i, e := range es {
|
||||||
|
if e == identical {
|
||||||
|
prevNotIdentical := i-1 < 0 || es[i-1] != identical
|
||||||
|
nextNotIdentical := i+1 >= len(es) || es[i+1] != identical
|
||||||
|
if prevNotIdentical && nextNotIdentical {
|
||||||
|
es[i] = identicalContext
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adjust the maxSize, reserving space for the final summary.
|
||||||
|
if maxSize < 0 {
|
||||||
|
maxSize = math.MaxInt
|
||||||
|
}
|
||||||
|
maxSize -= len(stats{len(xs) + len(ys), len(xs), len(ys)}.appendText(nil))
|
||||||
|
|
||||||
|
// mayAppendLine appends a line if it does not exceed maxSize.
|
||||||
|
// Otherwise, it just updates prevStats.
|
||||||
|
var buf []byte
|
||||||
|
var prevStats stats
|
||||||
|
mayAppendLine := func(edit edit, line string) {
|
||||||
|
// Append the stats (if non-zero) and the line text.
|
||||||
|
// The stats reports the number of preceding identical lines.
|
||||||
|
if !truncated {
|
||||||
|
bufLen := len(buf) // original length (in case we exceed maxSize)
|
||||||
|
if !prevStats.isZero() {
|
||||||
|
buf = prevStats.appendText(buf)
|
||||||
|
prevStats = stats{} // just printed, so clear the stats
|
||||||
|
}
|
||||||
|
buf = fmt.Appendf(buf, "%c %s\n", edit, line)
|
||||||
|
truncated = len(buf) > maxSize
|
||||||
|
if !truncated {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
buf = buf[:bufLen] // restore original buffer contents
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output is truncated, so just update the statistics.
|
||||||
|
switch edit {
|
||||||
|
case identical:
|
||||||
|
prevStats.numIdentical++
|
||||||
|
case removed:
|
||||||
|
prevStats.numRemoved++
|
||||||
|
case inserted:
|
||||||
|
prevStats.numInserted++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process the entire edit script.
|
||||||
|
for len(es) > 0 {
|
||||||
|
num := len(es) - len(bytes.TrimLeft(es, string(es[:1])))
|
||||||
|
switch es[0] {
|
||||||
|
case identical:
|
||||||
|
prevStats.numIdentical += num
|
||||||
|
xs, ys = xs[num:], ys[num:]
|
||||||
|
case identicalContext:
|
||||||
|
for n := len(xs) - num; len(xs) > n; xs, ys = xs[1:], ys[1:] {
|
||||||
|
mayAppendLine(identical, xs[0]) // implies xs[0] == ys[0]
|
||||||
|
}
|
||||||
|
case modified:
|
||||||
|
for n := len(xs) - num; len(xs) > n; xs = xs[1:] {
|
||||||
|
mayAppendLine(removed, xs[0])
|
||||||
|
}
|
||||||
|
for n := len(ys) - num; len(ys) > n; ys = ys[1:] {
|
||||||
|
mayAppendLine(inserted, ys[0])
|
||||||
|
}
|
||||||
|
case removed:
|
||||||
|
for n := len(xs) - num; len(xs) > n; xs = xs[1:] {
|
||||||
|
mayAppendLine(removed, xs[0])
|
||||||
|
}
|
||||||
|
case inserted:
|
||||||
|
for n := len(ys) - num; len(ys) > n; ys = ys[1:] {
|
||||||
|
mayAppendLine(inserted, ys[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
es = es[num:]
|
||||||
|
}
|
||||||
|
if len(xs)+len(ys)+len(es) > 0 {
|
||||||
|
panic("BUG: slices not fully consumed")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !prevStats.isZero() {
|
||||||
|
buf = prevStats.appendText(buf) // may exceed maxSize
|
||||||
|
}
|
||||||
|
return string(buf), truncated
|
||||||
|
}
|
||||||
|
|
||||||
|
type stats struct{ numIdentical, numRemoved, numInserted int }
|
||||||
|
|
||||||
|
func (s stats) isZero() bool { return s.numIdentical+s.numRemoved+s.numInserted == 0 }
|
||||||
|
|
||||||
|
func (s stats) appendText(b []byte) []byte {
|
||||||
|
switch {
|
||||||
|
case s.numIdentical > 0 && s.numRemoved > 0 && s.numInserted > 0:
|
||||||
|
return fmt.Appendf(b, "… %d identical, %d removed, and %d inserted lines\n", s.numIdentical, s.numRemoved, s.numInserted)
|
||||||
|
case s.numIdentical > 0 && s.numRemoved > 0:
|
||||||
|
return fmt.Appendf(b, "… %d identical and %d removed lines\n", s.numIdentical, s.numRemoved)
|
||||||
|
case s.numIdentical > 0 && s.numInserted > 0:
|
||||||
|
return fmt.Appendf(b, "… %d identical and %d inserted lines\n", s.numIdentical, s.numInserted)
|
||||||
|
case s.numRemoved > 0 && s.numInserted > 0:
|
||||||
|
return fmt.Appendf(b, "… %d removed and %d inserted lines\n", s.numRemoved, s.numInserted)
|
||||||
|
case s.numIdentical > 0:
|
||||||
|
return fmt.Appendf(b, "… %d identical lines\n", s.numIdentical)
|
||||||
|
case s.numRemoved > 0:
|
||||||
|
return fmt.Appendf(b, "… %d removed lines\n", s.numRemoved)
|
||||||
|
case s.numInserted > 0:
|
||||||
|
return fmt.Appendf(b, "… %d inserted lines\n", s.numInserted)
|
||||||
|
default:
|
||||||
|
return fmt.Appendf(b, "…\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// diffStrings computes an edit-script of two slices of strings.
|
||||||
|
//
|
||||||
|
// This calls cmp.Equal to access the "github.com/go-cmp/cmp/internal/diff"
|
||||||
|
// implementation, which has an O(N) diffing algorithm. It is not guaranteed
|
||||||
|
// to produce an optimal edit-script, but protects our runtime against
|
||||||
|
// adversarial inputs that would wreck the optimal O(N²) algorithm used by
|
||||||
|
// most diffing packages available in open-source.
|
||||||
|
//
|
||||||
|
// TODO(https://go.dev/issue/58893): Use "golang.org/x/tools/diff" instead?
|
||||||
|
func diffStrings(xs, ys []string) []edit {
|
||||||
|
d := new(diffRecorder)
|
||||||
|
cmp.Equal(xs, ys, cmp.Reporter(d))
|
||||||
|
if diffTest {
|
||||||
|
numRemoved := bytes.Count(d.script, []byte{removed})
|
||||||
|
numInserted := bytes.Count(d.script, []byte{inserted})
|
||||||
|
if len(xs) != len(d.script)-numInserted || len(ys) != len(d.script)-numRemoved {
|
||||||
|
panic("BUG: edit-script is inconsistent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return d.script
|
||||||
|
}
|
||||||
|
|
||||||
|
type edit = byte
|
||||||
|
|
||||||
|
const (
|
||||||
|
identical edit = ' ' // equal symbol in both x and y
|
||||||
|
modified edit = '~' // modified symbol in both x and y
|
||||||
|
removed edit = '-' // removed symbol from x
|
||||||
|
inserted edit = '+' // inserted symbol from y
|
||||||
|
)
|
||||||
|
|
||||||
|
// diffRecorder reproduces an edit-script, essentially recording
|
||||||
|
// the edit-script from "github.com/google/go-cmp/cmp/internal/diff".
|
||||||
|
// This implements the cmp.Reporter interface.
|
||||||
|
type diffRecorder struct {
|
||||||
|
last cmp.PathStep
|
||||||
|
script []edit
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *diffRecorder) PushStep(ps cmp.PathStep) { d.last = ps }
|
||||||
|
|
||||||
|
func (d *diffRecorder) Report(rs cmp.Result) {
|
||||||
|
if si, ok := d.last.(cmp.SliceIndex); ok {
|
||||||
|
if rs.Equal() {
|
||||||
|
d.script = append(d.script, identical)
|
||||||
|
} else {
|
||||||
|
switch xi, yi := si.SplitKeys(); {
|
||||||
|
case xi >= 0 && yi >= 0:
|
||||||
|
d.script = append(d.script, modified)
|
||||||
|
case xi >= 0:
|
||||||
|
d.script = append(d.script, removed)
|
||||||
|
case yi >= 0:
|
||||||
|
d.script = append(d.script, inserted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *diffRecorder) PopStep() { d.last = nil }
|
||||||
@ -0,0 +1,196 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
package safediff
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/go-cmp/cmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() { diffTest = true }
|
||||||
|
|
||||||
|
func TestLines(t *testing.T) {
|
||||||
|
// The diffs shown below technically depend on the stability of cmp,
|
||||||
|
// but that should be fine for sufficiently simple diffs like these.
|
||||||
|
// If the output does change, that would suggest a significant regression
|
||||||
|
// in the optimality of cmp's diffing algorithm.
|
||||||
|
|
||||||
|
x := `{
|
||||||
|
"firstName": "John",
|
||||||
|
"lastName": "Smith",
|
||||||
|
"isAlive": true,
|
||||||
|
"age": 27,
|
||||||
|
"address": {
|
||||||
|
"streetAddress": "21 2nd Street",
|
||||||
|
"city": "New York",
|
||||||
|
"state": "NY",
|
||||||
|
"postalCode": "10021-3100"
|
||||||
|
},
|
||||||
|
"phoneNumbers": [{
|
||||||
|
"type": "home",
|
||||||
|
"number": "212 555-1234"
|
||||||
|
}, {
|
||||||
|
"type": "office",
|
||||||
|
"number": "646 555-4567"
|
||||||
|
}],
|
||||||
|
"children": [
|
||||||
|
"Catherine",
|
||||||
|
"Thomas",
|
||||||
|
"Trevor"
|
||||||
|
],
|
||||||
|
"spouse": null
|
||||||
|
}`
|
||||||
|
y := x
|
||||||
|
y = strings.ReplaceAll(y, `"New York"`, `"Los Angeles"`)
|
||||||
|
y = strings.ReplaceAll(y, `"NY"`, `"CA"`)
|
||||||
|
y = strings.ReplaceAll(y, `"646 555-4567"`, `"315 252-8888"`)
|
||||||
|
|
||||||
|
wantDiff := `
|
||||||
|
… 5 identical lines
|
||||||
|
"address": {
|
||||||
|
"streetAddress": "21 2nd Street",
|
||||||
|
- "city": "New York",
|
||||||
|
- "state": "NY",
|
||||||
|
+ "city": "Los Angeles",
|
||||||
|
+ "state": "CA",
|
||||||
|
"postalCode": "10021-3100"
|
||||||
|
},
|
||||||
|
… 3 identical lines
|
||||||
|
}, {
|
||||||
|
"type": "office",
|
||||||
|
- "number": "646 555-4567"
|
||||||
|
+ "number": "315 252-8888"
|
||||||
|
}],
|
||||||
|
… 7 identical lines
|
||||||
|
`[1:]
|
||||||
|
gotDiff, gotTrunc := Lines(x, y, -1)
|
||||||
|
if d := cmp.Diff(gotDiff, wantDiff); d != "" {
|
||||||
|
t.Errorf("Lines mismatch (-got +want):\n%s\ngot:\n%s\nwant:\n%s", d, gotDiff, wantDiff)
|
||||||
|
} else if gotTrunc == true {
|
||||||
|
t.Errorf("Lines: output unexpectedly truncated")
|
||||||
|
}
|
||||||
|
|
||||||
|
wantDiff = `
|
||||||
|
… 5 identical lines
|
||||||
|
"address": {
|
||||||
|
"streetAddress": "21 2nd Street",
|
||||||
|
- "city": "New York",
|
||||||
|
- "state": "NY",
|
||||||
|
+ "city": "Los Angeles",
|
||||||
|
… 15 identical, 1 removed, and 2 inserted lines
|
||||||
|
`[1:]
|
||||||
|
gotDiff, gotTrunc = Lines(x, y, 200)
|
||||||
|
if d := cmp.Diff(gotDiff, wantDiff); d != "" {
|
||||||
|
t.Errorf("Lines mismatch (-got +want):\n%s\ngot:\n%s\nwant:\n%s", d, gotDiff, wantDiff)
|
||||||
|
} else if gotTrunc == false {
|
||||||
|
t.Errorf("Lines: output unexpectedly not truncated")
|
||||||
|
}
|
||||||
|
|
||||||
|
wantDiff = "… 17 identical, 3 removed, and 3 inserted lines\n"
|
||||||
|
gotDiff, gotTrunc = Lines(x, y, 0)
|
||||||
|
if d := cmp.Diff(gotDiff, wantDiff); d != "" {
|
||||||
|
t.Errorf("Lines mismatch (-got +want):\n%s\ngot:\n%s\nwant:\n%s", d, gotDiff, wantDiff)
|
||||||
|
} else if gotTrunc == false {
|
||||||
|
t.Errorf("Lines: output unexpectedly not truncated")
|
||||||
|
}
|
||||||
|
|
||||||
|
x = `{
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
"related": {
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
"related": {
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
"related": {
|
||||||
|
"related": "changed",
|
||||||
|
},
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"unrelated": [
|
||||||
|
"unrelated",
|
||||||
|
],
|
||||||
|
}`
|
||||||
|
y = strings.ReplaceAll(x, "changed", "CHANGED")
|
||||||
|
|
||||||
|
wantDiff = `
|
||||||
|
… 4 identical lines
|
||||||
|
"related": {
|
||||||
|
… 3 identical lines
|
||||||
|
"related": {
|
||||||
|
… 3 identical lines
|
||||||
|
"related": {
|
||||||
|
- "related": "changed",
|
||||||
|
+ "related": "CHANGED",
|
||||||
|
},
|
||||||
|
… 3 identical lines
|
||||||
|
},
|
||||||
|
… 3 identical lines
|
||||||
|
},
|
||||||
|
… 4 identical lines
|
||||||
|
`[1:]
|
||||||
|
gotDiff, gotTrunc = Lines(x, y, -1)
|
||||||
|
if d := cmp.Diff(gotDiff, wantDiff); d != "" {
|
||||||
|
t.Errorf("Lines mismatch (-got +want):\n%s\ngot:\n%s\nwant:\n%s", d, gotDiff, wantDiff)
|
||||||
|
} else if gotTrunc == true {
|
||||||
|
t.Errorf("Lines: output unexpectedly truncated")
|
||||||
|
}
|
||||||
|
|
||||||
|
x = `{
|
||||||
|
"ACLs": [
|
||||||
|
{
|
||||||
|
"Action": "accept",
|
||||||
|
"Users": ["group:all"],
|
||||||
|
"Ports": ["tag:tmemes:80"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}`
|
||||||
|
y = strings.ReplaceAll(x, "tag:tmemes:80", "tag:tmemes:80,8383")
|
||||||
|
wantDiff = `
|
||||||
|
{
|
||||||
|
"ACLs": [
|
||||||
|
{
|
||||||
|
"Action": "accept",
|
||||||
|
"Users": ["group:all"],
|
||||||
|
- "Ports": ["tag:tmemes:80"],
|
||||||
|
+ "Ports": ["tag:tmemes:80,8383"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
`[1:]
|
||||||
|
gotDiff, gotTrunc = Lines(x, y, -1)
|
||||||
|
if d := cmp.Diff(gotDiff, wantDiff); d != "" {
|
||||||
|
t.Errorf("Lines mismatch (-got +want):\n%s\ngot:\n%s\nwant:\n%s", d, gotDiff, wantDiff)
|
||||||
|
} else if gotTrunc == true {
|
||||||
|
t.Errorf("Lines: output unexpectedly truncated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func FuzzDiff(f *testing.F) {
|
||||||
|
f.Fuzz(func(t *testing.T, x, y string, maxSize int) {
|
||||||
|
const maxInput = 1e3
|
||||||
|
if len(x) > maxInput {
|
||||||
|
x = x[:maxInput]
|
||||||
|
}
|
||||||
|
if len(y) > maxInput {
|
||||||
|
y = y[:maxInput]
|
||||||
|
}
|
||||||
|
diff, _ := Lines(x, y, maxSize) // make sure this does not panic
|
||||||
|
if strings.Count(diff, "\n") > 1 && maxSize >= 0 && len(diff) > maxSize {
|
||||||
|
t.Fatal("maxSize exceeded")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue