Add support for compressed images when pre-loading images (#2165)

* Add support for compressed images when pre-loading images

Signed-off-by: Frederick F. Kautz IV <fkautz@alumni.cmu.edu>

* attempting to fix vendor source being dirty

Signed-off-by: Frederick F. Kautz IV <fkautz@alumni.cmu.edu>

* fixing file extension for .tar.lz4

Signed-off-by: Frederick F. Kautz IV <fkautz@alumni.cmu.edu>

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/docker/docker v1.4.2-0.20191205034852-d163fbba3c82
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
+	github.com/frankban/quicktest v1.10.2 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/googleapis v1.3.0 // indirect
@@ -93,6 +94,7 @@ require (
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/opencontainers/runc v1.0.0-rc10
 	github.com/opencontainers/selinux v1.5.3-0.20200613095409-bb88c45a3863
+	github.com/pierrec/lz4 v2.5.2+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/rancher/dynamiclistener v0.2.1
 	github.com/rancher/helm-controller v0.7.2

go.sum

@@ -215,6 +215,8 @@ github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
+github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -366,6 +368,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.0.0-20190617215043-876b8855d23c/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -475,6 +479,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJ
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -603,6 +609,8 @@ github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
+github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -995,6 +1003,8 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191017205301-920acffc3e65 h1:GwXwgmbrvlcHLDsENMqrQTTIC2C0kIPszsq929NruKI=
 golang.org/x/tools v0.0.0-20191017205301-920acffc3e65/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q=

pkg/agent/containerd/containerd.go

@@ -2,6 +2,8 @@ package containerd
 
 import (
 	"bufio"
+	"compress/bzip2"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -16,6 +18,7 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/natefinch/lumberjack"
 	"github.com/opencontainers/runc/libcontainer/system"
+	"github.com/pierrec/lz4"
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/agent/templates"
 	util2 "github.com/rancher/k3s/pkg/agent/util"
@@ -182,7 +185,24 @@ func preloadImages(ctx context.Context, cfg *config.Node) error {
 		}
 
 		logrus.Debugf("Import %s", filePath)
-		_, err = client.Import(ctxContainerD, file)
+		var imageReader io.Reader
+		imageReader = file
+		if strings.HasSuffix(fileInfo.Name(), ".tar.bz2") {
+			imageReader = bzip2.NewReader(file)
+		}
+		if strings.HasSuffix(fileInfo.Name(), ".tar.lz4") {
+			imageReader = lz4.NewReader(file)
+		}
+		if strings.HasSuffix(fileInfo.Name(), ".tar.gz") {
+			// WARNING: gzip reader close does not close the underlying image
+			imageReader, err = gzip.NewReader(file)
+			if err != nil {
+				logrus.Errorf("Unable to import %s: %v", filePath, err)
+				file.Close()
+				continue
+			}
+		}
+		_, err = client.Import(ctxContainerD, imageReader)
 		file.Close()
 		if err != nil {
 			logrus.Errorf("Unable to import %s: %v", filePath, err)

vendor/github.com/google/go-cmp/cmp/compare.go

@@ -6,6 +6,10 @@
 //
 // This package is intended to be a more powerful and safer alternative to
 // reflect.DeepEqual for comparing whether two values are semantically equal.
+// It is intended to only be used in tests, as performance is not a goal and
+// it may panic if it cannot compare the values. Its propensity towards
+// panicking means that its unsuitable for production environments where a
+// spurious panic may be fatal.
 //
 // The primary features of cmp are:
 //
@@ -22,8 +26,8 @@
 // equality is determined by recursively comparing the primitive kinds on both
 // values, much like reflect.DeepEqual. Unlike reflect.DeepEqual, unexported
 // fields are not compared by default; they result in panics unless suppressed
-// by using an Ignore option (see cmpopts.IgnoreUnexported) or explicitly compared
-// using the AllowUnexported option.
+// by using an Ignore option (see cmpopts.IgnoreUnexported) or explicitly
+// compared using the Exporter option.
 package cmp
 
 import (
@@ -62,8 +66,8 @@ import (
 //
 // Structs are equal if recursively calling Equal on all fields report equal.
 // If a struct contains unexported fields, Equal panics unless an Ignore option
-// (e.g., cmpopts.IgnoreUnexported) ignores that field or the AllowUnexported
-// option explicitly permits comparing the unexported field.
+// (e.g., cmpopts.IgnoreUnexported) ignores that field or the Exporter option
+// explicitly permits comparing the unexported field.
 //
 // Slices are equal if they are both nil or both non-nil, where recursively
 // calling Equal on all non-ignored slice or array elements report equal.
@@ -80,7 +84,58 @@ import (
 // Pointers and interfaces are equal if they are both nil or both non-nil,
 // where they have the same underlying concrete type and recursively
 // calling Equal on the underlying values reports equal.
+//
+// Before recursing into a pointer, slice element, or map, the current path
+// is checked to detect whether the address has already been visited.
+// If there is a cycle, then the pointed at values are considered equal
+// only if both addresses were previously visited in the same path step.
 func Equal(x, y interface{}, opts ...Option) bool {
+	s := newState(opts)
+	s.compareAny(rootStep(x, y))
+	return s.result.Equal()
+}
+
+// Diff returns a human-readable report of the differences between two values:
+// y - x. It returns an empty string if and only if Equal returns true for the
+// same input values and options.
+//
+// The output is displayed as a literal in pseudo-Go syntax.
+// At the start of each line, a "-" prefix indicates an element removed from y,
+// a "+" prefix to indicates an element added to y, and the lack of a prefix
+// indicates an element common to both x and y. If possible, the output
+// uses fmt.Stringer.String or error.Error methods to produce more humanly
+// readable outputs. In such cases, the string is prefixed with either an
+// 's' or 'e' character, respectively, to indicate that the method was called.
+//
+// Do not depend on this output being stable. If you need the ability to
+// programmatically interpret the difference, consider using a custom Reporter.
+func Diff(x, y interface{}, opts ...Option) string {
+	s := newState(opts)
+
+	// Optimization: If there are no other reporters, we can optimize for the
+	// common case where the result is equal (and thus no reported difference).
+	// This avoids the expensive construction of a difference tree.
+	if len(s.reporters) == 0 {
+		s.compareAny(rootStep(x, y))
+		if s.result.Equal() {
+			return ""
+		}
+		s.result = diff.Result{} // Reset results
+	}
+
+	r := new(defaultReporter)
+	s.reporters = append(s.reporters, reporter{r})
+	s.compareAny(rootStep(x, y))
+	d := r.String()
+	if (d == "") != s.result.Equal() {
+		panic("inconsistent difference and equality results")
+	}
+	return d
+}
+
+// rootStep constructs the first path step. If x and y have differing types,
+// then they are stored within an empty interface type.
+func rootStep(x, y interface{}) PathStep {
 	vx := reflect.ValueOf(x)
 	vy := reflect.ValueOf(y)
 
@@ -103,33 +158,7 @@ func Equal(x, y interface{}, opts ...Option) bool {
 		t = vx.Type()
 	}
 
-	s := newState(opts)
-	s.compareAny(&pathStep{t, vx, vy})
-	return s.result.Equal()
-}
-
-// Diff returns a human-readable report of the differences between two values.
-// It returns an empty string if and only if Equal returns true for the same
-// input values and options.
-//
-// The output is displayed as a literal in pseudo-Go syntax.
-// At the start of each line, a "-" prefix indicates an element removed from x,
-// a "+" prefix to indicates an element added to y, and the lack of a prefix
-// indicates an element common to both x and y. If possible, the output
-// uses fmt.Stringer.String or error.Error methods to produce more humanly
-// readable outputs. In such cases, the string is prefixed with either an
-// 's' or 'e' character, respectively, to indicate that the method was called.
-//
-// Do not depend on this output being stable. If you need the ability to
-// programmatically interpret the difference, consider using a custom Reporter.
-func Diff(x, y interface{}, opts ...Option) string {
-	r := new(defaultReporter)
-	eq := Equal(x, y, Options(opts), Reporter(r))
-	d := r.String()
-	if (d == "") != eq {
-		panic("inconsistent difference and equality results")
-	}
-	return d
+	return &pathStep{t, vx, vy}
 }
 
 type state struct {
@@ -137,6 +166,7 @@ type state struct {
 	// Calling statelessCompare must not result in observable changes to these.
 	result    diff.Result // The current result of comparison
 	curPath   Path        // The current path in the value tree
+	curPtrs   pointerPath // The current set of visited pointers
 	reporters []reporter  // Optional reporters
 
 	// recChecker checks for infinite cycles applying the same set of
@@ -148,13 +178,14 @@ type state struct {
 	dynChecker dynChecker
 
 	// These fields, once set by processOption, will not change.
-	exporters map[reflect.Type]bool // Set of structs with unexported field visibility
+	exporters []exporter // List of exporters for structs with unexported fields
 	opts      Options    // List of all fundamental and filter options
 }
 
 func newState(opts []Option) *state {
 	// Always ensure a validator option exists to validate the inputs.
 	s := &state{opts: Options{validator{}}}
+	s.curPtrs.Init()
 	s.processOption(Options(opts))
 	return s
 }
@@ -174,13 +205,8 @@ func (s *state) processOption(opt Option) {
 			panic(fmt.Sprintf("cannot use an unfiltered option: %v", opt))
 		}
 		s.opts = append(s.opts, opt)
-	case visibleStructs:
-		if s.exporters == nil {
-			s.exporters = make(map[reflect.Type]bool)
-		}
-		for t := range opt {
-			s.exporters[t] = true
-		}
+	case exporter:
+		s.exporters = append(s.exporters, opt)
 	case reporter:
 		s.reporters = append(s.reporters, opt)
 	default:
@@ -192,9 +218,9 @@ func (s *state) processOption(opt Option) {
 // This function is stateless in that it does not alter the current result,
 // or output to any registered reporters.
 func (s *state) statelessCompare(step PathStep) diff.Result {
-	// We do not save and restore the curPath because all of the compareX
-	// methods should properly push and pop from the path.
-	// It is an implementation bug if the contents of curPath differs from
+	// We do not save and restore curPath and curPtrs because all of the
+	// compareX methods should properly push and pop from them.
+	// It is an implementation bug if the contents of the paths differ from
 	// when calling this function to when returning from it.
 
 	oldResult, oldReporters := s.result, s.reporters
@@ -216,9 +242,17 @@ func (s *state) compareAny(step PathStep) {
 	}
 	s.recChecker.Check(s.curPath)
 
-	// Obtain the current type and values.
+	// Cycle-detection for slice elements (see NOTE in compareSlice).
 	t := step.Type()
 	vx, vy := step.Values()
+	if si, ok := step.(SliceIndex); ok && si.isSlice && vx.IsValid() && vy.IsValid() {
+		px, py := vx.Addr(), vy.Addr()
+		if eq, visited := s.curPtrs.Push(px, py); visited {
+			s.report(eq, reportByCycle)
+			return
+		}
+		defer s.curPtrs.Pop(px, py)
+	}
 
 	// Rule 1: Check whether an option applies on this node in the value tree.
 	if s.tryOptions(t, vx, vy) {
@@ -342,7 +376,7 @@ func detectRaces(c chan<- reflect.Value, f reflect.Value, vs ...reflect.Value) {
 // assuming that T is assignable to R.
 // Otherwise, it returns the input value as is.
 func sanitizeValue(v reflect.Value, t reflect.Type) reflect.Value {
-	// TODO(dsnet): Workaround for reflect bug (https://golang.org/issue/22143).
+	// TODO(≥go1.10): Workaround for reflect bug (https://golang.org/issue/22143).
 	if !flags.AtLeastGo110 {
 		if v.Kind() == reflect.Interface && v.IsNil() && v.Type() != t {
 			return reflect.New(t).Elem()
@@ -352,8 +386,10 @@ func sanitizeValue(v reflect.Value, t reflect.Type) reflect.Value {
 }
 
 func (s *state) compareStruct(t reflect.Type, vx, vy reflect.Value) {
+	var addr bool
 	var vax, vay reflect.Value // Addressable versions of vx and vy
 
+	var mayForce, mayForceInit bool
 	step := StructField{&structField{}}
 	for i := 0; i < t.NumField(); i++ {
 		step.typ = t.Field(i).Type
@@ -372,10 +408,18 @@ func (s *state) compareStruct(t reflect.Type, vx, vy reflect.Value) {
 				// For retrieveUnexportedField to work, the parent struct must
 				// be addressable. Create a new copy of the values if
 				// necessary to make them addressable.
+				addr = vx.CanAddr() || vy.CanAddr()
 				vax = makeAddressable(vx)
 				vay = makeAddressable(vy)
 			}
-			step.mayForce = s.exporters[t]
+			if !mayForceInit {
+				for _, xf := range s.exporters {
+					mayForce = mayForce || xf(t)
+				}
+				mayForceInit = true
+			}
+			step.mayForce = mayForce
+			step.paddr = addr
 			step.pvx = vax
 			step.pvy = vay
 			step.field = t.Field(i)
@@ -391,9 +435,21 @@ func (s *state) compareSlice(t reflect.Type, vx, vy reflect.Value) {
 		return
 	}
 
-	// TODO: Support cyclic data structures.
+	// NOTE: It is incorrect to call curPtrs.Push on the slice header pointer
+	// since slices represents a list of pointers, rather than a single pointer.
+	// The pointer checking logic must be handled on a per-element basis
+	// in compareAny.
+	//
+	// A slice header (see reflect.SliceHeader) in Go is a tuple of a starting
+	// pointer P, a length N, and a capacity C. Supposing each slice element has
+	// a memory size of M, then the slice is equivalent to the list of pointers:
+	//	[P+i*M for i in range(N)]
+	//
+	// For example, v[:0] and v[:1] are slices with the same starting pointer,
+	// but they are clearly different values. Using the slice pointer alone
+	// violates the assumption that equal pointers implies equal values.
 
-	step := SliceIndex{&sliceIndex{pathStep: pathStep{typ: t.Elem()}}}
+	step := SliceIndex{&sliceIndex{pathStep: pathStep{typ: t.Elem()}, isSlice: isSlice}}
 	withIndexes := func(ix, iy int) SliceIndex {
 		if ix >= 0 {
 			step.vx, step.xkey = vx.Index(ix), ix
@@ -470,7 +526,12 @@ func (s *state) compareMap(t reflect.Type, vx, vy reflect.Value) {
 		return
 	}
 
-	// TODO: Support cyclic data structures.
+	// Cycle-detection for maps.
+	if eq, visited := s.curPtrs.Push(vx, vy); visited {
+		s.report(eq, reportByCycle)
+		return
+	}
+	defer s.curPtrs.Pop(vx, vy)
 
 	// We combine and sort the two map keys so that we can perform the
 	// comparisons in a deterministic order.
@@ -507,7 +568,12 @@ func (s *state) comparePtr(t reflect.Type, vx, vy reflect.Value) {
 		return
 	}
 
-	// TODO: Support cyclic data structures.
+	// Cycle-detection for pointers.
+	if eq, visited := s.curPtrs.Push(vx, vy); visited {
+		s.report(eq, reportByCycle)
+		return
+	}
+	defer s.curPtrs.Pop(vx, vy)
 
 	vx, vy = vx.Elem(), vy.Elem()
 	s.compareAny(Indirect{&indirect{pathStep{t.Elem(), vx, vy}}})

vendor/github.com/google/go-cmp/cmp/export_panic.go

@@ -8,8 +8,8 @@ package cmp
 
 import "reflect"
 
-const supportAllowUnexported = false
+const supportExporters = false
 
-func retrieveUnexportedField(reflect.Value, reflect.StructField) reflect.Value {
-	panic("retrieveUnexportedField is not implemented")
+func retrieveUnexportedField(reflect.Value, reflect.StructField, bool) reflect.Value {
+	panic("no support for forcibly accessing unexported fields")
 }

vendor/github.com/google/go-cmp/cmp/export_unsafe.go

@@ -11,13 +11,25 @@ import (
 	"unsafe"
 )
 
-const supportAllowUnexported = true
+const supportExporters = true
 
 // retrieveUnexportedField uses unsafe to forcibly retrieve any field from
 // a struct such that the value has read-write permissions.
 //
 // The parent struct, v, must be addressable, while f must be a StructField
-// describing the field to retrieve.
-func retrieveUnexportedField(v reflect.Value, f reflect.StructField) reflect.Value {
-	return reflect.NewAt(f.Type, unsafe.Pointer(v.UnsafeAddr()+f.Offset)).Elem()
+// describing the field to retrieve. If addr is false,
+// then the returned value will be shallowed copied to be non-addressable.
+func retrieveUnexportedField(v reflect.Value, f reflect.StructField, addr bool) reflect.Value {
+	ve := reflect.NewAt(f.Type, unsafe.Pointer(uintptr(unsafe.Pointer(v.UnsafeAddr()))+f.Offset)).Elem()
+	if !addr {
+		// A field is addressable if and only if the struct is addressable.
+		// If the original parent value was not addressable, shallow copy the
+		// value to make it non-addressable to avoid leaking an implementation
+		// detail of how forcibly exporting a field works.
+		if ve.Kind() == reflect.Interface && ve.IsNil() {
+			return reflect.Zero(f.Type)
+		}
+		return reflect.ValueOf(ve.Interface()).Convert(f.Type)
+	}
+	return ve
 }

vendor/github.com/google/go-cmp/cmp/internal/diff/diff.go

@@ -12,6 +12,13 @@
 // is more important than obtaining a minimal Levenshtein distance.
 package diff
 
+import (
+	"math/rand"
+	"time"
+
+	"github.com/google/go-cmp/cmp/internal/flags"
+)
+
 // EditType represents a single operation within an edit-script.
 type EditType uint8
 
@@ -112,6 +119,8 @@ func (r Result) Similar() bool {
 	return r.NumSame+1 >= r.NumDiff
 }
 
+var randInt = rand.New(rand.NewSource(time.Now().Unix())).Intn(2)
+
 // Difference reports whether two lists of lengths nx and ny are equal
 // given the definition of equality provided as f.
 //
@@ -159,6 +168,17 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 	// A vertical edge is equivalent to inserting a symbol from list Y.
 	// A diagonal edge is equivalent to a matching symbol between both X and Y.
 
+	// To ensure flexibility in changing the algorithm in the future,
+	// introduce some degree of deliberate instability.
+	// This is achieved by fiddling the zigzag iterator to start searching
+	// the graph starting from the bottom-right versus than the top-left.
+	// The result may differ depending on the starting search location,
+	// but still produces a valid edit script.
+	zigzagInit := randInt // either 0 or 1
+	if flags.Deterministic {
+		zigzagInit = 0
+	}
+
 	// Invariants:
 	//	• 0 ≤ fwdPath.X ≤ (fwdFrontier.X, revFrontier.X) ≤ revPath.X ≤ nx
 	//	• 0 ≤ fwdPath.Y ≤ (fwdFrontier.Y, revFrontier.Y) ≤ revPath.Y ≤ ny
@@ -209,7 +229,7 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 		if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
 			break
 		}
-		for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
+		for stop1, stop2, i := false, false, zigzagInit; !(stop1 && stop2) && searchBudget > 0; i++ {
 			// Search in a diagonal pattern for a match.
 			z := zigzag(i)
 			p := point{fwdFrontier.X + z, fwdFrontier.Y - z}

vendor/github.com/google/go-cmp/cmp/internal/value/name.go

@@ -0,0 +1,157 @@
+// Copyright 2020, The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package value
+
+import (
+	"reflect"
+	"strconv"
+)
+
+// TypeString is nearly identical to reflect.Type.String,
+// but has an additional option to specify that full type names be used.
+func TypeString(t reflect.Type, qualified bool) string {
+	return string(appendTypeName(nil, t, qualified, false))
+}
+
+func appendTypeName(b []byte, t reflect.Type, qualified, elideFunc bool) []byte {
+	// BUG: Go reflection provides no way to disambiguate two named types
+	// of the same name and within the same package,
+	// but declared within the namespace of different functions.
+
+	// Named type.
+	if t.Name() != "" {
+		if qualified && t.PkgPath() != "" {
+			b = append(b, '"')
+			b = append(b, t.PkgPath()...)
+			b = append(b, '"')
+			b = append(b, '.')
+			b = append(b, t.Name()...)
+		} else {
+			b = append(b, t.String()...)
+		}
+		return b
+	}
+
+	// Unnamed type.
+	switch k := t.Kind(); k {
+	case reflect.Bool, reflect.String, reflect.UnsafePointer,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+		reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
+		b = append(b, k.String()...)
+	case reflect.Chan:
+		if t.ChanDir() == reflect.RecvDir {
+			b = append(b, "<-"...)
+		}
+		b = append(b, "chan"...)
+		if t.ChanDir() == reflect.SendDir {
+			b = append(b, "<-"...)
+		}
+		b = append(b, ' ')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Func:
+		if !elideFunc {
+			b = append(b, "func"...)
+		}
+		b = append(b, '(')
+		for i := 0; i < t.NumIn(); i++ {
+			if i > 0 {
+				b = append(b, ", "...)
+			}
+			if i == t.NumIn()-1 && t.IsVariadic() {
+				b = append(b, "..."...)
+				b = appendTypeName(b, t.In(i).Elem(), qualified, false)
+			} else {
+				b = appendTypeName(b, t.In(i), qualified, false)
+			}
+		}
+		b = append(b, ')')
+		switch t.NumOut() {
+		case 0:
+			// Do nothing
+		case 1:
+			b = append(b, ' ')
+			b = appendTypeName(b, t.Out(0), qualified, false)
+		default:
+			b = append(b, " ("...)
+			for i := 0; i < t.NumOut(); i++ {
+				if i > 0 {
+					b = append(b, ", "...)
+				}
+				b = appendTypeName(b, t.Out(i), qualified, false)
+			}
+			b = append(b, ')')
+		}
+	case reflect.Struct:
+		b = append(b, "struct{ "...)
+		for i := 0; i < t.NumField(); i++ {
+			if i > 0 {
+				b = append(b, "; "...)
+			}
+			sf := t.Field(i)
+			if !sf.Anonymous {
+				if qualified && sf.PkgPath != "" {
+					b = append(b, '"')
+					b = append(b, sf.PkgPath...)
+					b = append(b, '"')
+					b = append(b, '.')
+				}
+				b = append(b, sf.Name...)
+				b = append(b, ' ')
+			}
+			b = appendTypeName(b, sf.Type, qualified, false)
+			if sf.Tag != "" {
+				b = append(b, ' ')
+				b = strconv.AppendQuote(b, string(sf.Tag))
+			}
+		}
+		if b[len(b)-1] == ' ' {
+			b = b[:len(b)-1]
+		} else {
+			b = append(b, ' ')
+		}
+		b = append(b, '}')
+	case reflect.Slice, reflect.Array:
+		b = append(b, '[')
+		if k == reflect.Array {
+			b = strconv.AppendUint(b, uint64(t.Len()), 10)
+		}
+		b = append(b, ']')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Map:
+		b = append(b, "map["...)
+		b = appendTypeName(b, t.Key(), qualified, false)
+		b = append(b, ']')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Ptr:
+		b = append(b, '*')
+		b = appendTypeName(b, t.Elem(), qualified, false)
+	case reflect.Interface:
+		b = append(b, "interface{ "...)
+		for i := 0; i < t.NumMethod(); i++ {
+			if i > 0 {
+				b = append(b, "; "...)
+			}
+			m := t.Method(i)
+			if qualified && m.PkgPath != "" {
+				b = append(b, '"')
+				b = append(b, m.PkgPath...)
+				b = append(b, '"')
+				b = append(b, '.')
+			}
+			b = append(b, m.Name...)
+			b = appendTypeName(b, m.Type, qualified, true)
+		}
+		if b[len(b)-1] == ' ' {
+			b = b[:len(b)-1]
+		} else {
+			b = append(b, ' ')
+		}
+		b = append(b, '}')
+	default:
+		panic("invalid kind: " + k.String())
+	}
+	return b
+}

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_purego.go

@@ -21,3 +21,13 @@ func PointerOf(v reflect.Value) Pointer {
 	// assumes that the GC implementation does not use a moving collector.
 	return Pointer{v.Pointer(), v.Type()}
 }
+
+// IsNil reports whether the pointer is nil.
+func (p Pointer) IsNil() bool {
+	return p.p == 0
+}
+
+// Uintptr returns the pointer as a uintptr.
+func (p Pointer) Uintptr() uintptr {
+	return p.p
+}

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_unsafe.go

@@ -24,3 +24,13 @@ func PointerOf(v reflect.Value) Pointer {
 	// which is necessary if the GC ever uses a moving collector.
 	return Pointer{unsafe.Pointer(v.Pointer()), v.Type()}
 }
+
+// IsNil reports whether the pointer is nil.
+func (p Pointer) IsNil() bool {
+	return p.p == nil
+}
+
+// Uintptr returns the pointer as a uintptr.
+func (p Pointer) Uintptr() uintptr {
+	return uintptr(p.p)
+}

vendor/github.com/google/go-cmp/cmp/options.go

@@ -225,8 +225,23 @@ func (validator) apply(s *state, vx, vy reflect.Value) {
 
 	// Unable to Interface implies unexported field without visibility access.
 	if !vx.CanInterface() || !vy.CanInterface() {
-		const help = "consider using a custom Comparer; if you control the implementation of type, you can also consider AllowUnexported or cmpopts.IgnoreUnexported"
-		panic(fmt.Sprintf("cannot handle unexported field: %#v\n%s", s.curPath, help))
+		help := "consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported"
+		var name string
+		if t := s.curPath.Index(-2).Type(); t.Name() != "" {
+			// Named type with unexported fields.
+			name = fmt.Sprintf("%q.%v", t.PkgPath(), t.Name()) // e.g., "path/to/package".MyType
+			if _, ok := reflect.New(t).Interface().(error); ok {
+				help = "consider using cmpopts.EquateErrors to compare error values"
+			}
+		} else {
+			// Unnamed type with unexported fields. Derive PkgPath from field.
+			var pkgPath string
+			for i := 0; i < t.NumField() && pkgPath == ""; i++ {
+				pkgPath = t.Field(i).PkgPath
+			}
+			name = fmt.Sprintf("%q.(%v)", pkgPath, t.String()) // e.g., "path/to/package".(struct { a int })
+		}
+		panic(fmt.Sprintf("cannot handle unexported field at %#v:\n\t%v\n%s", s.curPath, name, help))
 	}
 
 	panic("not reachable")
@@ -360,9 +375,8 @@ func (cm comparer) String() string {
 	return fmt.Sprintf("Comparer(%s)", function.NameOf(cm.fnc))
 }
 
-// AllowUnexported returns an Option that forcibly allows operations on
-// unexported fields in certain structs, which are specified by passing in a
-// value of each struct type.
+// Exporter returns an Option that specifies whether Equal is allowed to
+// introspect into the unexported fields of certain struct types.
 //
 // Users of this option must understand that comparing on unexported fields
 // from external packages is not safe since changes in the internal
@@ -386,10 +400,24 @@ func (cm comparer) String() string {
 //
 // In other cases, the cmpopts.IgnoreUnexported option can be used to ignore
 // all unexported fields on specified struct types.
-func AllowUnexported(types ...interface{}) Option {
-	if !supportAllowUnexported {
-		panic("AllowUnexported is not supported on purego builds, Google App Engine Standard, or GopherJS")
+func Exporter(f func(reflect.Type) bool) Option {
+	if !supportExporters {
+		panic("Exporter is not supported on purego builds")
 	}
+	return exporter(f)
+}
+
+type exporter func(reflect.Type) bool
+
+func (exporter) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
+	panic("not implemented")
+}
+
+// AllowUnexported returns an Options that allows Equal to forcibly introspect
+// unexported fields of the specified struct types.
+//
+// See Exporter for the proper use of this option.
+func AllowUnexported(types ...interface{}) Option {
 	m := make(map[reflect.Type]bool)
 	for _, typ := range types {
 		t := reflect.TypeOf(typ)
@@ -398,13 +426,7 @@ func AllowUnexported(types ...interface{}) Option {
 		}
 		m[t] = true
 	}
-	return visibleStructs(m)
-}
-
-type visibleStructs map[reflect.Type]bool
-
-func (visibleStructs) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
-	panic("not implemented")
+	return exporter(func(t reflect.Type) bool { return m[t] })
 }
 
 // Result represents the comparison result for a single node and
@@ -436,6 +458,11 @@ func (r Result) ByFunc() bool {
 	return r.flags&reportByFunc != 0
 }
 
+// ByCycle reports whether a reference cycle was detected.
+func (r Result) ByCycle() bool {
+	return r.flags&reportByCycle != 0
+}
+
 type resultFlags uint
 
 const (
@@ -446,6 +473,7 @@ const (
 	reportByIgnore
 	reportByMethod
 	reportByFunc
+	reportByCycle
 )
 
 // Reporter is an Option that can be passed to Equal. When Equal traverses

vendor/github.com/google/go-cmp/cmp/path.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/google/go-cmp/cmp/internal/value"
 )
 
 // Path is a list of PathSteps describing the sequence of operations to get
@@ -41,7 +43,7 @@ type PathStep interface {
 	// In some cases, one or both may be invalid or have restrictions:
 	//	• For StructField, both are not interface-able if the current field
 	//	is unexported and the struct type is not explicitly permitted by
-	//	AllowUnexported to traverse unexported fields.
+	//	an Exporter to traverse unexported fields.
 	//	• For SliceIndex, one may be invalid if an element is missing from
 	//	either the x or y slice.
 	//	• For MapIndex, one may be invalid if an entry is missing from
@@ -175,7 +177,8 @@ type structField struct {
 	// pvx, pvy, and field are only valid if unexported is true.
 	unexported bool
 	mayForce   bool                // Forcibly allow visibility
-	pvx, pvy   reflect.Value       // Parent values
+	paddr      bool                // Was parent addressable?
+	pvx, pvy   reflect.Value       // Parent values (always addressible)
 	field      reflect.StructField // Field information
 }
 
@@ -187,8 +190,8 @@ func (sf StructField) Values() (vx, vy reflect.Value) {
 
 	// Forcibly obtain read-write access to an unexported struct field.
 	if sf.mayForce {
-		vx = retrieveUnexportedField(sf.pvx, sf.field)
-		vy = retrieveUnexportedField(sf.pvy, sf.field)
+		vx = retrieveUnexportedField(sf.pvx, sf.field, sf.paddr)
+		vy = retrieveUnexportedField(sf.pvy, sf.field, sf.paddr)
 		return vx, vy // CanInterface reports true
 	}
 	return sf.vx, sf.vy // CanInterface reports false
@@ -207,6 +210,7 @@ type SliceIndex struct{ *sliceIndex }
 type sliceIndex struct {
 	pathStep
 	xkey, ykey int
+	isSlice    bool // False for reflect.Array
 }
 
 func (si SliceIndex) Type() reflect.Type             { return si.typ }
@@ -301,6 +305,72 @@ func (tf Transform) Func() reflect.Value { return tf.trans.fnc }
 // The == operator can be used to detect the exact option used.
 func (tf Transform) Option() Option { return tf.trans }
 
+// pointerPath represents a dual-stack of pointers encountered when
+// recursively traversing the x and y values. This data structure supports
+// detection of cycles and determining whether the cycles are equal.
+// In Go, cycles can occur via pointers, slices, and maps.
+//
+// The pointerPath uses a map to represent a stack; where descension into a
+// pointer pushes the address onto the stack, and ascension from a pointer
+// pops the address from the stack. Thus, when traversing into a pointer from
+// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
+// by checking whether the pointer has already been visited. The cycle detection
+// uses a seperate stack for the x and y values.
+//
+// If a cycle is detected we need to determine whether the two pointers
+// should be considered equal. The definition of equality chosen by Equal
+// requires two graphs to have the same structure. To determine this, both the
+// x and y values must have a cycle where the previous pointers were also
+// encountered together as a pair.
+//
+// Semantically, this is equivalent to augmenting Indirect, SliceIndex, and
+// MapIndex with pointer information for the x and y values.
+// Suppose px and py are two pointers to compare, we then search the
+// Path for whether px was ever encountered in the Path history of x, and
+// similarly so with py. If either side has a cycle, the comparison is only
+// equal if both px and py have a cycle resulting from the same PathStep.
+//
+// Using a map as a stack is more performant as we can perform cycle detection
+// in O(1) instead of O(N) where N is len(Path).
+type pointerPath struct {
+	// mx is keyed by x pointers, where the value is the associated y pointer.
+	mx map[value.Pointer]value.Pointer
+	// my is keyed by y pointers, where the value is the associated x pointer.
+	my map[value.Pointer]value.Pointer
+}
+
+func (p *pointerPath) Init() {
+	p.mx = make(map[value.Pointer]value.Pointer)
+	p.my = make(map[value.Pointer]value.Pointer)
+}
+
+// Push indicates intent to descend into pointers vx and vy where
+// visited reports whether either has been seen before. If visited before,
+// equal reports whether both pointers were encountered together.
+// Pop must be called if and only if the pointers were never visited.
+//
+// The pointers vx and vy must be a reflect.Ptr, reflect.Slice, or reflect.Map
+// and be non-nil.
+func (p pointerPath) Push(vx, vy reflect.Value) (equal, visited bool) {
+	px := value.PointerOf(vx)
+	py := value.PointerOf(vy)
+	_, ok1 := p.mx[px]
+	_, ok2 := p.my[py]
+	if ok1 || ok2 {
+		equal = p.mx[px] == py && p.my[py] == px // Pointers paired together
+		return equal, true
+	}
+	p.mx[px] = py
+	p.my[py] = px
+	return false, false
+}
+
+// Pop ascends from pointers vx and vy.
+func (p pointerPath) Pop(vx, vy reflect.Value) {
+	delete(p.mx, value.PointerOf(vx))
+	delete(p.my, value.PointerOf(vy))
+}
+
 // isExported reports whether the identifier is exported.
 func isExported(id string) bool {
 	r, _ := utf8.DecodeRuneInString(id)

vendor/github.com/google/go-cmp/cmp/report.go

@@ -41,7 +41,10 @@ func (r *defaultReporter) String() string {
 	if r.root.NumDiff == 0 {
 		return ""
 	}
-	return formatOptions{}.FormatDiff(r.root).String()
+	ptrs := new(pointerReferences)
+	text := formatOptions{}.FormatDiff(r.root, ptrs)
+	resolveReferences(text)
+	return text.String()
 }
 
 func assert(ok bool) {

vendor/github.com/google/go-cmp/cmp/report_compare.go

@@ -11,14 +11,6 @@ import (
 	"github.com/google/go-cmp/cmp/internal/value"
 )
 
-// TODO: Enforce limits?
-//	* Enforce maximum number of records to print per node?
-//	* Enforce maximum size in bytes allowed?
-//	* As a heuristic, use less verbosity for equal nodes than unequal nodes.
-// TODO: Enforce unique outputs?
-//	* Avoid Stringer methods if it results in same output?
-//	* Print pointer address if outputs still equal?
-
 // numContextRecords is the number of surrounding equal records to print.
 const numContextRecords = 2
 
@@ -71,24 +63,66 @@ func (opts formatOptions) WithTypeMode(t typeMode) formatOptions {
 	opts.TypeMode = t
 	return opts
 }
+func (opts formatOptions) WithVerbosity(level int) formatOptions {
+	opts.VerbosityLevel = level
+	opts.LimitVerbosity = true
+	return opts
+}
+func (opts formatOptions) verbosity() uint {
+	switch {
+	case opts.VerbosityLevel < 0:
+		return 0
+	case opts.VerbosityLevel > 16:
+		return 16 // some reasonable maximum to avoid shift overflow
+	default:
+		return uint(opts.VerbosityLevel)
+	}
+}
+
+const maxVerbosityPreset = 3
+
+// verbosityPreset modifies the verbosity settings given an index
+// between 0 and maxVerbosityPreset, inclusive.
+func verbosityPreset(opts formatOptions, i int) formatOptions {
+	opts.VerbosityLevel = int(opts.verbosity()) + 2*i
+	if i > 0 {
+		opts.AvoidStringer = true
+	}
+	if i >= maxVerbosityPreset {
+		opts.PrintAddresses = true
+		opts.QualifiedNames = true
+	}
+	return opts
+}
 
 // FormatDiff converts a valueNode tree into a textNode tree, where the later
 // is a textual representation of the differences detected in the former.
-func (opts formatOptions) FormatDiff(v *valueNode) textNode {
+func (opts formatOptions) FormatDiff(v *valueNode, ptrs *pointerReferences) (out textNode) {
+	if opts.DiffMode == diffIdentical {
+		opts = opts.WithVerbosity(1)
+	} else {
+		opts = opts.WithVerbosity(3)
+	}
+
 	// Check whether we have specialized formatting for this node.
 	// This is not necessary, but helpful for producing more readable outputs.
 	if opts.CanFormatDiffSlice(v) {
 		return opts.FormatDiffSlice(v)
 	}
 
+	var parentKind reflect.Kind
+	if v.parent != nil && v.parent.TransformerName == "" {
+		parentKind = v.parent.Type.Kind()
+	}
+
 	// For leaf nodes, format the value based on the reflect.Values alone.
 	if v.MaxDepth == 0 {
 		switch opts.DiffMode {
 		case diffUnknown, diffIdentical:
 			// Format Equal.
 			if v.NumDiff == 0 {
-				outx := opts.FormatValue(v.ValueX, visitedPointers{})
-				outy := opts.FormatValue(v.ValueY, visitedPointers{})
+				outx := opts.FormatValue(v.ValueX, parentKind, ptrs)
+				outy := opts.FormatValue(v.ValueY, parentKind, ptrs)
 				if v.NumIgnored > 0 && v.NumSame == 0 {
 					return textEllipsis
 				} else if outx.Len() < outy.Len() {
@@ -101,8 +135,13 @@ func (opts formatOptions) FormatDiff(v *valueNode) textNode {
 			// Format unequal.
 			assert(opts.DiffMode == diffUnknown)
 			var list textList
-			outx := opts.WithTypeMode(elideType).FormatValue(v.ValueX, visitedPointers{})
-			outy := opts.WithTypeMode(elideType).FormatValue(v.ValueY, visitedPointers{})
+			outx := opts.WithTypeMode(elideType).FormatValue(v.ValueX, parentKind, ptrs)
+			outy := opts.WithTypeMode(elideType).FormatValue(v.ValueY, parentKind, ptrs)
+			for i := 0; i <= maxVerbosityPreset && outx != nil && outy != nil && outx.Equal(outy); i++ {
+				opts2 := verbosityPreset(opts, i).WithTypeMode(elideType)
+				outx = opts2.FormatValue(v.ValueX, parentKind, ptrs)
+				outy = opts2.FormatValue(v.ValueY, parentKind, ptrs)
+			}
 			if outx != nil {
 				list = append(list, textRecord{Diff: '-', Value: outx})
 			}
@@ -111,34 +150,57 @@ func (opts formatOptions) FormatDiff(v *valueNode) textNode {
 			}
 			return opts.WithTypeMode(emitType).FormatType(v.Type, list)
 		case diffRemoved:
-			return opts.FormatValue(v.ValueX, visitedPointers{})
+			return opts.FormatValue(v.ValueX, parentKind, ptrs)
 		case diffInserted:
-			return opts.FormatValue(v.ValueY, visitedPointers{})
+			return opts.FormatValue(v.ValueY, parentKind, ptrs)
 		default:
 			panic("invalid diff mode")
 		}
 	}
 
+	// Register slice element to support cycle detection.
+	if parentKind == reflect.Slice {
+		ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, true)
+		defer ptrs.Pop()
+		defer func() { out = wrapTrunkReferences(ptrRefs, out) }()
+	}
+
 	// Descend into the child value node.
 	if v.TransformerName != "" {
-		out := opts.WithTypeMode(emitType).FormatDiff(v.Value)
-		out = textWrap{"Inverse(" + v.TransformerName + ", ", out, ")"}
+		out := opts.WithTypeMode(emitType).FormatDiff(v.Value, ptrs)
+		out = &textWrap{Prefix: "Inverse(" + v.TransformerName + ", ", Value: out, Suffix: ")"}
 		return opts.FormatType(v.Type, out)
 	} else {
 		switch k := v.Type.Kind(); k {
-		case reflect.Struct, reflect.Array, reflect.Slice, reflect.Map:
-			return opts.FormatType(v.Type, opts.formatDiffList(v.Records, k))
+		case reflect.Struct, reflect.Array, reflect.Slice:
+			out = opts.formatDiffList(v.Records, k, ptrs)
+			out = opts.FormatType(v.Type, out)
+		case reflect.Map:
+			// Register map to support cycle detection.
+			ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, false)
+			defer ptrs.Pop()
+
+			out = opts.formatDiffList(v.Records, k, ptrs)
+			out = wrapTrunkReferences(ptrRefs, out)
+			out = opts.FormatType(v.Type, out)
 		case reflect.Ptr:
-			return textWrap{"&", opts.FormatDiff(v.Value), ""}
+			// Register pointer to support cycle detection.
+			ptrRefs := ptrs.PushPair(v.ValueX, v.ValueY, opts.DiffMode, false)
+			defer ptrs.Pop()
+
+			out = opts.FormatDiff(v.Value, ptrs)
+			out = wrapTrunkReferences(ptrRefs, out)
+			out = &textWrap{Prefix: "&", Value: out}
 		case reflect.Interface:
-			return opts.WithTypeMode(emitType).FormatDiff(v.Value)
+			out = opts.WithTypeMode(emitType).FormatDiff(v.Value, ptrs)
 		default:
 			panic(fmt.Sprintf("%v cannot have children", k))
 		}
+		return out
 	}
 }
 
-func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) textNode {
+func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind, ptrs *pointerReferences) textNode {
 	// Derive record name based on the data structure kind.
 	var name string
 	var formatKey func(reflect.Value) string
@@ -154,7 +216,17 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
 	case reflect.Map:
 		name = "entry"
 		opts = opts.WithTypeMode(elideType)
-		formatKey = formatMapKey
+		formatKey = func(v reflect.Value) string { return formatMapKey(v, false, ptrs) }
+	}
+
+	maxLen := -1
+	if opts.LimitVerbosity {
+		if opts.DiffMode == diffIdentical {
+			maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
+		} else {
+			maxLen = (1 << opts.verbosity()) << 1 // 2, 4, 8, 16, 32, 64, etc...
+		}
+		opts.VerbosityLevel--
 	}
 
 	// Handle unification.
@@ -163,6 +235,11 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
 		var list textList
 		var deferredEllipsis bool // Add final "..." to indicate records were dropped
 		for _, r := range recs {
+			if len(list) == maxLen {
+				deferredEllipsis = true
+				break
+			}
+
 			// Elide struct fields that are zero value.
 			if k == reflect.Struct {
 				var isZero bool
@@ -186,23 +263,31 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
 				}
 				continue
 			}
-			if out := opts.FormatDiff(r.Value); out != nil {
+			if out := opts.FormatDiff(r.Value, ptrs); out != nil {
 				list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
 			}
 		}
 		if deferredEllipsis {
 			list.AppendEllipsis(diffStats{})
 		}
-		return textWrap{"{", list, "}"}
+		return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
 	case diffUnknown:
 	default:
 		panic("invalid diff mode")
 	}
 
 	// Handle differencing.
+	var numDiffs int
 	var list textList
+	var keys []reflect.Value // invariant: len(list) == len(keys)
 	groups := coalesceAdjacentRecords(name, recs)
+	maxGroup := diffStats{Name: name}
 	for i, ds := range groups {
+		if maxLen >= 0 && numDiffs >= maxLen {
+			maxGroup = maxGroup.Append(ds)
+			continue
+		}
+
 		// Handle equal records.
 		if ds.NumDiff() == 0 {
 			// Compute the number of leading and trailing records to print.
@@ -226,16 +311,21 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
 
 			// Format the equal values.
 			for _, r := range recs[:numLo] {
-				out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value)
+				out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value, ptrs)
 				list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
+				keys = append(keys, r.Key)
 			}
 			if numEqual > numLo+numHi {
 				ds.NumIdentical -= numLo + numHi
 				list.AppendEllipsis(ds)
+				for len(keys) < len(list) {
+					keys = append(keys, reflect.Value{})
+				}
 			}
 			for _, r := range recs[numEqual-numHi : numEqual] {
-				out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value)
+				out := opts.WithDiffMode(diffIdentical).FormatDiff(r.Value, ptrs)
 				list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
+				keys = append(keys, r.Key)
 			}
 			recs = recs[numEqual:]
 			continue
@@ -247,24 +337,70 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
 			case opts.CanFormatDiffSlice(r.Value):
 				out := opts.FormatDiffSlice(r.Value)
 				list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
+				keys = append(keys, r.Key)
 			case r.Value.NumChildren == r.Value.MaxDepth:
-				outx := opts.WithDiffMode(diffRemoved).FormatDiff(r.Value)
-				outy := opts.WithDiffMode(diffInserted).FormatDiff(r.Value)
+				outx := opts.WithDiffMode(diffRemoved).FormatDiff(r.Value, ptrs)
+				outy := opts.WithDiffMode(diffInserted).FormatDiff(r.Value, ptrs)
+				for i := 0; i <= maxVerbosityPreset && outx != nil && outy != nil && outx.Equal(outy); i++ {
+					opts2 := verbosityPreset(opts, i)
+					outx = opts2.WithDiffMode(diffRemoved).FormatDiff(r.Value, ptrs)
+					outy = opts2.WithDiffMode(diffInserted).FormatDiff(r.Value, ptrs)
+				}
 				if outx != nil {
 					list = append(list, textRecord{Diff: diffRemoved, Key: formatKey(r.Key), Value: outx})
+					keys = append(keys, r.Key)
 				}
 				if outy != nil {
 					list = append(list, textRecord{Diff: diffInserted, Key: formatKey(r.Key), Value: outy})
+					keys = append(keys, r.Key)
 				}
 			default:
-				out := opts.FormatDiff(r.Value)
+				out := opts.FormatDiff(r.Value, ptrs)
 				list = append(list, textRecord{Key: formatKey(r.Key), Value: out})
+				keys = append(keys, r.Key)
 			}
 		}
 		recs = recs[ds.NumDiff():]
+		numDiffs += ds.NumDiff()
 	}
+	if maxGroup.IsZero() {
 		assert(len(recs) == 0)
-	return textWrap{"{", list, "}"}
+	} else {
+		list.AppendEllipsis(maxGroup)
+		for len(keys) < len(list) {
+			keys = append(keys, reflect.Value{})
+		}
+	}
+	assert(len(list) == len(keys))
+
+	// For maps, the default formatting logic uses fmt.Stringer which may
+	// produce ambiguous output. Avoid calling String to disambiguate.
+	if k == reflect.Map {
+		var ambiguous bool
+		seenKeys := map[string]reflect.Value{}
+		for i, currKey := range keys {
+			if currKey.IsValid() {
+				strKey := list[i].Key
+				prevKey, seen := seenKeys[strKey]
+				if seen && prevKey.CanInterface() && currKey.CanInterface() {
+					ambiguous = prevKey.Interface() != currKey.Interface()
+					if ambiguous {
+						break
+					}
+				}
+				seenKeys[strKey] = currKey
+			}
+		}
+		if ambiguous {
+			for i, k := range keys {
+				if k.IsValid() {
+					list[i].Key = formatMapKey(k, true, ptrs)
+				}
+			}
+		}
+	}
+
+	return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
 }
 
 // coalesceAdjacentRecords coalesces the list of records into groups of

vendor/github.com/google/go-cmp/cmp/report_references.go

@@ -0,0 +1,264 @@
+// Copyright 2020, The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package cmp
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/google/go-cmp/cmp/internal/flags"
+	"github.com/google/go-cmp/cmp/internal/value"
+)
+
+const (
+	pointerDelimPrefix = "⟪"
+	pointerDelimSuffix = "⟫"
+)
+
+// formatPointer prints the address of the pointer.
+func formatPointer(p value.Pointer, withDelims bool) string {
+	v := p.Uintptr()
+	if flags.Deterministic {
+		v = 0xdeadf00f // Only used for stable testing purposes
+	}
+	if withDelims {
+		return pointerDelimPrefix + formatHex(uint64(v)) + pointerDelimSuffix
+	}
+	return formatHex(uint64(v))
+}
+
+// pointerReferences is a stack of pointers visited so far.
+type pointerReferences [][2]value.Pointer
+
+func (ps *pointerReferences) PushPair(vx, vy reflect.Value, d diffMode, deref bool) (pp [2]value.Pointer) {
+	if deref && vx.IsValid() {
+		vx = vx.Addr()
+	}
+	if deref && vy.IsValid() {
+		vy = vy.Addr()
+	}
+	switch d {
+	case diffUnknown, diffIdentical:
+		pp = [2]value.Pointer{value.PointerOf(vx), value.PointerOf(vy)}
+	case diffRemoved:
+		pp = [2]value.Pointer{value.PointerOf(vx), value.Pointer{}}
+	case diffInserted:
+		pp = [2]value.Pointer{value.Pointer{}, value.PointerOf(vy)}
+	}
+	*ps = append(*ps, pp)
+	return pp
+}
+
+func (ps *pointerReferences) Push(v reflect.Value) (p value.Pointer, seen bool) {
+	p = value.PointerOf(v)
+	for _, pp := range *ps {
+		if p == pp[0] || p == pp[1] {
+			return p, true
+		}
+	}
+	*ps = append(*ps, [2]value.Pointer{p, p})
+	return p, false
+}
+
+func (ps *pointerReferences) Pop() {
+	*ps = (*ps)[:len(*ps)-1]
+}
+
+// trunkReferences is metadata for a textNode indicating that the sub-tree
+// represents the value for either pointer in a pair of references.
+type trunkReferences struct{ pp [2]value.Pointer }
+
+// trunkReference is metadata for a textNode indicating that the sub-tree
+// represents the value for the given pointer reference.
+type trunkReference struct{ p value.Pointer }
+
+// leafReference is metadata for a textNode indicating that the value is
+// truncated as it refers to another part of the tree (i.e., a trunk).
+type leafReference struct{ p value.Pointer }
+
+func wrapTrunkReferences(pp [2]value.Pointer, s textNode) textNode {
+	switch {
+	case pp[0].IsNil():
+		return &textWrap{Value: s, Metadata: trunkReference{pp[1]}}
+	case pp[1].IsNil():
+		return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
+	case pp[0] == pp[1]:
+		return &textWrap{Value: s, Metadata: trunkReference{pp[0]}}
+	default:
+		return &textWrap{Value: s, Metadata: trunkReferences{pp}}
+	}
+}
+func wrapTrunkReference(p value.Pointer, printAddress bool, s textNode) textNode {
+	var prefix string
+	if printAddress {
+		prefix = formatPointer(p, true)
+	}
+	return &textWrap{Prefix: prefix, Value: s, Metadata: trunkReference{p}}
+}
+func makeLeafReference(p value.Pointer, printAddress bool) textNode {
+	out := &textWrap{Prefix: "(", Value: textEllipsis, Suffix: ")"}
+	var prefix string
+	if printAddress {
+		prefix = formatPointer(p, true)
+	}
+	return &textWrap{Prefix: prefix, Value: out, Metadata: leafReference{p}}
+}
+
+// resolveReferences walks the textNode tree searching for any leaf reference
+// metadata and resolves each against the corresponding trunk references.
+// Since pointer addresses in memory are not particularly readable to the user,
+// it replaces each pointer value with an arbitrary and unique reference ID.
+func resolveReferences(s textNode) {
+	var walkNodes func(textNode, func(textNode))
+	walkNodes = func(s textNode, f func(textNode)) {
+		f(s)
+		switch s := s.(type) {
+		case *textWrap:
+			walkNodes(s.Value, f)
+		case textList:
+			for _, r := range s {
+				walkNodes(r.Value, f)
+			}
+		}
+	}
+
+	// Collect all trunks and leaves with reference metadata.
+	var trunks, leaves []*textWrap
+	walkNodes(s, func(s textNode) {
+		if s, ok := s.(*textWrap); ok {
+			switch s.Metadata.(type) {
+			case leafReference:
+				leaves = append(leaves, s)
+			case trunkReference, trunkReferences:
+				trunks = append(trunks, s)
+			}
+		}
+	})
+
+	// No leaf references to resolve.
+	if len(leaves) == 0 {
+		return
+	}
+
+	// Collect the set of all leaf references to resolve.
+	leafPtrs := make(map[value.Pointer]bool)
+	for _, leaf := range leaves {
+		leafPtrs[leaf.Metadata.(leafReference).p] = true
+	}
+
+	// Collect the set of trunk pointers that are always paired together.
+	// This allows us to assign a single ID to both pointers for brevity.
+	// If a pointer in a pair ever occurs by itself or as a different pair,
+	// then the pair is broken.
+	pairedTrunkPtrs := make(map[value.Pointer]value.Pointer)
+	unpair := func(p value.Pointer) {
+		if !pairedTrunkPtrs[p].IsNil() {
+			pairedTrunkPtrs[pairedTrunkPtrs[p]] = value.Pointer{} // invalidate other half
+		}
+		pairedTrunkPtrs[p] = value.Pointer{} // invalidate this half
+	}
+	for _, trunk := range trunks {
+		switch p := trunk.Metadata.(type) {
+		case trunkReference:
+			unpair(p.p) // standalone pointer cannot be part of a pair
+		case trunkReferences:
+			p0, ok0 := pairedTrunkPtrs[p.pp[0]]
+			p1, ok1 := pairedTrunkPtrs[p.pp[1]]
+			switch {
+			case !ok0 && !ok1:
+				// Register the newly seen pair.
+				pairedTrunkPtrs[p.pp[0]] = p.pp[1]
+				pairedTrunkPtrs[p.pp[1]] = p.pp[0]
+			case ok0 && ok1 && p0 == p.pp[1] && p1 == p.pp[0]:
+				// Exact pair already seen; do nothing.
+			default:
+				// Pair conflicts with some other pair; break all pairs.
+				unpair(p.pp[0])
+				unpair(p.pp[1])
+			}
+		}
+	}
+
+	// Correlate each pointer referenced by leaves to a unique identifier,
+	// and print the IDs for each trunk that matches those pointers.
+	var nextID uint
+	ptrIDs := make(map[value.Pointer]uint)
+	newID := func() uint {
+		id := nextID
+		nextID++
+		return id
+	}
+	for _, trunk := range trunks {
+		switch p := trunk.Metadata.(type) {
+		case trunkReference:
+			if print := leafPtrs[p.p]; print {
+				id, ok := ptrIDs[p.p]
+				if !ok {
+					id = newID()
+					ptrIDs[p.p] = id
+				}
+				trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
+			}
+		case trunkReferences:
+			print0 := leafPtrs[p.pp[0]]
+			print1 := leafPtrs[p.pp[1]]
+			if print0 || print1 {
+				id0, ok0 := ptrIDs[p.pp[0]]
+				id1, ok1 := ptrIDs[p.pp[1]]
+				isPair := pairedTrunkPtrs[p.pp[0]] == p.pp[1] && pairedTrunkPtrs[p.pp[1]] == p.pp[0]
+				if isPair {
+					var id uint
+					assert(ok0 == ok1) // must be seen together or not at all
+					if ok0 {
+						assert(id0 == id1) // must have the same ID
+						id = id0
+					} else {
+						id = newID()
+						ptrIDs[p.pp[0]] = id
+						ptrIDs[p.pp[1]] = id
+					}
+					trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id))
+				} else {
+					if print0 && !ok0 {
+						id0 = newID()
+						ptrIDs[p.pp[0]] = id0
+					}
+					if print1 && !ok1 {
+						id1 = newID()
+						ptrIDs[p.pp[1]] = id1
+					}
+					switch {
+					case print0 && print1:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0)+","+formatReference(id1))
+					case print0:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id0))
+					case print1:
+						trunk.Prefix = updateReferencePrefix(trunk.Prefix, formatReference(id1))
+					}
+				}
+			}
+		}
+	}
+
+	// Update all leaf references with the unique identifier.
+	for _, leaf := range leaves {
+		if id, ok := ptrIDs[leaf.Metadata.(leafReference).p]; ok {
+			leaf.Prefix = updateReferencePrefix(leaf.Prefix, formatReference(id))
+		}
+	}
+}
+
+func formatReference(id uint) string {
+	return fmt.Sprintf("ref#%d", id)
+}
+
+func updateReferencePrefix(prefix, ref string) string {
+	if prefix == "" {
+		return pointerDelimPrefix + ref + pointerDelimSuffix
+	}
+	suffix := strings.TrimPrefix(prefix, pointerDelimPrefix)
+	return pointerDelimPrefix + ref + ": " + suffix
+}

vendor/github.com/google/go-cmp/cmp/report_reflect.go

@@ -5,13 +5,14 @@
 package cmp
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 	"strconv"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 
-	"github.com/google/go-cmp/cmp/internal/flags"
 	"github.com/google/go-cmp/cmp/internal/value"
 )
 
@@ -20,14 +21,22 @@ type formatValueOptions struct {
 	// methods like error.Error or fmt.Stringer.String.
 	AvoidStringer bool
 
-	// ShallowPointers controls whether to avoid descending into pointers.
-	// Useful when printing map keys, where pointer comparison is performed
-	// on the pointer address rather than the pointed-at value.
-	ShallowPointers bool
-
 	// PrintAddresses controls whether to print the address of all pointers,
 	// slice elements, and maps.
 	PrintAddresses bool
+
+	// QualifiedNames controls whether FormatType uses the fully qualified name
+	// (including the full package path as opposed to just the package name).
+	QualifiedNames bool
+
+	// VerbosityLevel controls the amount of output to produce.
+	// A higher value produces more output. A value of zero or lower produces
+	// no output (represented using an ellipsis).
+	// If LimitVerbosity is false, then the level is treated as infinite.
+	VerbosityLevel int
+
+	// LimitVerbosity specifies that formatting should respect VerbosityLevel.
+	LimitVerbosity bool
 }
 
 // FormatType prints the type as if it were wrapping s.
@@ -44,12 +53,15 @@ func (opts formatOptions) FormatType(t reflect.Type, s textNode) textNode {
 		default:
 			return s
 		}
+		if opts.DiffMode == diffIdentical {
+			return s // elide type for identical nodes
+		}
 	case elideType:
 		return s
 	}
 
 	// Determine the type label, applying special handling for unnamed types.
-	typeName := t.String()
+	typeName := value.TypeString(t, opts.QualifiedNames)
 	if t.Name() == "" {
 		// According to Go grammar, certain type literals contain symbols that
 		// do not strongly bind to the next lexicographical token (e.g., *T).
@@ -57,39 +69,77 @@ func (opts formatOptions) FormatType(t reflect.Type, s textNode) textNode {
 		case reflect.Chan, reflect.Func, reflect.Ptr:
 			typeName = "(" + typeName + ")"
 		}
-		typeName = strings.Replace(typeName, "struct {", "struct{", -1)
-		typeName = strings.Replace(typeName, "interface {", "interface{", -1)
+	}
+	return &textWrap{Prefix: typeName, Value: wrapParens(s)}
+}
+
+// wrapParens wraps s with a set of parenthesis, but avoids it if the
+// wrapped node itself is already surrounded by a pair of parenthesis or braces.
+// It handles unwrapping one level of pointer-reference nodes.
+func wrapParens(s textNode) textNode {
+	var refNode *textWrap
+	if s2, ok := s.(*textWrap); ok {
+		// Unwrap a single pointer reference node.
+		switch s2.Metadata.(type) {
+		case leafReference, trunkReference, trunkReferences:
+			refNode = s2
+			if s3, ok := refNode.Value.(*textWrap); ok {
+				s2 = s3
+			}
 		}
 
-	// Avoid wrap the value in parenthesis if unnecessary.
-	if s, ok := s.(textWrap); ok {
-		hasParens := strings.HasPrefix(s.Prefix, "(") && strings.HasSuffix(s.Suffix, ")")
-		hasBraces := strings.HasPrefix(s.Prefix, "{") && strings.HasSuffix(s.Suffix, "}")
+		// Already has delimiters that make parenthesis unnecessary.
+		hasParens := strings.HasPrefix(s2.Prefix, "(") && strings.HasSuffix(s2.Suffix, ")")
+		hasBraces := strings.HasPrefix(s2.Prefix, "{") && strings.HasSuffix(s2.Suffix, "}")
 		if hasParens || hasBraces {
-			return textWrap{typeName, s, ""}
+			return s
 		}
 	}
-	return textWrap{typeName + "(", s, ")"}
+	if refNode != nil {
+		refNode.Value = &textWrap{Prefix: "(", Value: refNode.Value, Suffix: ")"}
+		return s
+	}
+	return &textWrap{Prefix: "(", Value: s, Suffix: ")"}
 }
 
 // FormatValue prints the reflect.Value, taking extra care to avoid descending
-// into pointers already in m. As pointers are visited, m is also updated.
-func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out textNode) {
+// into pointers already in ptrs. As pointers are visited, ptrs is also updated.
+func (opts formatOptions) FormatValue(v reflect.Value, parentKind reflect.Kind, ptrs *pointerReferences) (out textNode) {
 	if !v.IsValid() {
 		return nil
 	}
 	t := v.Type()
 
+	// Check slice element for cycles.
+	if parentKind == reflect.Slice {
+		ptrRef, visited := ptrs.Push(v.Addr())
+		if visited {
+			return makeLeafReference(ptrRef, false)
+		}
+		defer ptrs.Pop()
+		defer func() { out = wrapTrunkReference(ptrRef, false, out) }()
+	}
+
 	// Check whether there is an Error or String method to call.
 	if !opts.AvoidStringer && v.CanInterface() {
 		// Avoid calling Error or String methods on nil receivers since many
 		// implementations crash when doing so.
 		if (t.Kind() != reflect.Ptr && t.Kind() != reflect.Interface) || !v.IsNil() {
+			var prefix, strVal string
+			func() {
+				// Swallow and ignore any panics from String or Error.
+				defer func() { recover() }()
 				switch v := v.Interface().(type) {
 				case error:
-				return textLine("e" + formatString(v.Error()))
+					strVal = v.Error()
+					prefix = "e"
 				case fmt.Stringer:
-				return textLine("s" + formatString(v.String()))
+					strVal = v.String()
+					prefix = "s"
+				}
+			}()
+			if prefix != "" {
+				return opts.formatString(prefix, strVal)
 			}
 		}
 	}
@@ -102,94 +152,140 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
 		}
 	}()
 
-	var ptr string
 	switch t.Kind() {
 	case reflect.Bool:
 		return textLine(fmt.Sprint(v.Bool()))
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		return textLine(fmt.Sprint(v.Int()))
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		// Unnamed uints are usually bytes or words, so use hexadecimal.
-		if t.PkgPath() == "" || t.Kind() == reflect.Uintptr {
+	case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return textLine(fmt.Sprint(v.Uint()))
+	case reflect.Uint8:
+		if parentKind == reflect.Slice || parentKind == reflect.Array {
 			return textLine(formatHex(v.Uint()))
 		}
 		return textLine(fmt.Sprint(v.Uint()))
+	case reflect.Uintptr:
+		return textLine(formatHex(v.Uint()))
 	case reflect.Float32, reflect.Float64:
 		return textLine(fmt.Sprint(v.Float()))
 	case reflect.Complex64, reflect.Complex128:
 		return textLine(fmt.Sprint(v.Complex()))
 	case reflect.String:
-		return textLine(formatString(v.String()))
+		return opts.formatString("", v.String())
 	case reflect.UnsafePointer, reflect.Chan, reflect.Func:
-		return textLine(formatPointer(v))
+		return textLine(formatPointer(value.PointerOf(v), true))
 	case reflect.Struct:
 		var list textList
+		v := makeAddressable(v) // needed for retrieveUnexportedField
+		maxLen := v.NumField()
+		if opts.LimitVerbosity {
+			maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
+			opts.VerbosityLevel--
+		}
 		for i := 0; i < v.NumField(); i++ {
 			vv := v.Field(i)
 			if value.IsZero(vv) {
 				continue // Elide fields with zero values
 			}
-			s := opts.WithTypeMode(autoType).FormatValue(vv, m)
-			list = append(list, textRecord{Key: t.Field(i).Name, Value: s})
+			if len(list) == maxLen {
+				list.AppendEllipsis(diffStats{})
+				break
 			}
-		return textWrap{"{", list, "}"}
+			sf := t.Field(i)
+			if supportExporters && !isExported(sf.Name) {
+				vv = retrieveUnexportedField(v, sf, true)
+			}
+			s := opts.WithTypeMode(autoType).FormatValue(vv, t.Kind(), ptrs)
+			list = append(list, textRecord{Key: sf.Name, Value: s})
+		}
+		return &textWrap{Prefix: "{", Value: list, Suffix: "}"}
 	case reflect.Slice:
 		if v.IsNil() {
 			return textNil
 		}
-		if opts.PrintAddresses {
-			ptr = formatPointer(v)
+
+		// Check whether this is a []byte of text data.
+		if t.Elem() == reflect.TypeOf(byte(0)) {
+			b := v.Bytes()
+			isPrintSpace := func(r rune) bool { return unicode.IsPrint(r) && unicode.IsSpace(r) }
+			if len(b) > 0 && utf8.Valid(b) && len(bytes.TrimFunc(b, isPrintSpace)) == 0 {
+				out = opts.formatString("", string(b))
+				return opts.WithTypeMode(emitType).FormatType(t, out)
 			}
+		}
+
 		fallthrough
 	case reflect.Array:
+		maxLen := v.Len()
+		if opts.LimitVerbosity {
+			maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
+			opts.VerbosityLevel--
+		}
 		var list textList
 		for i := 0; i < v.Len(); i++ {
-			vi := v.Index(i)
-			if vi.CanAddr() { // Check for cyclic elements
-				p := vi.Addr()
-				if m.Visit(p) {
-					var out textNode
-					out = textLine(formatPointer(p))
-					out = opts.WithTypeMode(emitType).FormatType(p.Type(), out)
-					out = textWrap{"*", out, ""}
-					list = append(list, textRecord{Value: out})
-					continue
+			if len(list) == maxLen {
+				list.AppendEllipsis(diffStats{})
+				break
 			}
-			}
-			s := opts.WithTypeMode(elideType).FormatValue(vi, m)
+			s := opts.WithTypeMode(elideType).FormatValue(v.Index(i), t.Kind(), ptrs)
 			list = append(list, textRecord{Value: s})
 		}
-		return textWrap{ptr + "{", list, "}"}
+
+		out = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
+		if t.Kind() == reflect.Slice && opts.PrintAddresses {
+			header := fmt.Sprintf("ptr:%v, len:%d, cap:%d", formatPointer(value.PointerOf(v), false), v.Len(), v.Cap())
+			out = &textWrap{Prefix: pointerDelimPrefix + header + pointerDelimSuffix, Value: out}
+		}
+		return out
 	case reflect.Map:
 		if v.IsNil() {
 			return textNil
 		}
-		if m.Visit(v) {
-			return textLine(formatPointer(v))
-		}
 
+		// Check pointer for cycles.
+		ptrRef, visited := ptrs.Push(v)
+		if visited {
+			return makeLeafReference(ptrRef, opts.PrintAddresses)
+		}
+		defer ptrs.Pop()
+
+		maxLen := v.Len()
+		if opts.LimitVerbosity {
+			maxLen = ((1 << opts.verbosity()) >> 1) << 2 // 0, 4, 8, 16, 32, etc...
+			opts.VerbosityLevel--
+		}
 		var list textList
 		for _, k := range value.SortKeys(v.MapKeys()) {
-			sk := formatMapKey(k)
-			sv := opts.WithTypeMode(elideType).FormatValue(v.MapIndex(k), m)
+			if len(list) == maxLen {
+				list.AppendEllipsis(diffStats{})
+				break
+			}
+			sk := formatMapKey(k, false, ptrs)
+			sv := opts.WithTypeMode(elideType).FormatValue(v.MapIndex(k), t.Kind(), ptrs)
 			list = append(list, textRecord{Key: sk, Value: sv})
 		}
-		if opts.PrintAddresses {
-			ptr = formatPointer(v)
-		}
-		return textWrap{ptr + "{", list, "}"}
+
+		out = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
+		out = wrapTrunkReference(ptrRef, opts.PrintAddresses, out)
+		return out
 	case reflect.Ptr:
 		if v.IsNil() {
 			return textNil
 		}
-		if m.Visit(v) || opts.ShallowPointers {
-			return textLine(formatPointer(v))
-		}
-		if opts.PrintAddresses {
-			ptr = formatPointer(v)
+
+		// Check pointer for cycles.
+		ptrRef, visited := ptrs.Push(v)
+		if visited {
+			out = makeLeafReference(ptrRef, opts.PrintAddresses)
+			return &textWrap{Prefix: "&", Value: out}
 		}
+		defer ptrs.Pop()
+
 		skipType = true // Let the underlying value print the type instead
-		return textWrap{"&" + ptr, opts.FormatValue(v.Elem(), m), ""}
+		out = opts.FormatValue(v.Elem(), t.Kind(), ptrs)
+		out = wrapTrunkReference(ptrRef, opts.PrintAddresses, out)
+		out = &textWrap{Prefix: "&", Value: out}
+		return out
 	case reflect.Interface:
 		if v.IsNil() {
 			return textNil
@@ -197,19 +293,65 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
 		// Interfaces accept different concrete types,
 		// so configure the underlying value to explicitly print the type.
 		skipType = true // Print the concrete type instead
-		return opts.WithTypeMode(emitType).FormatValue(v.Elem(), m)
+		return opts.WithTypeMode(emitType).FormatValue(v.Elem(), t.Kind(), ptrs)
 	default:
 		panic(fmt.Sprintf("%v kind not handled", v.Kind()))
 	}
 }
 
+func (opts formatOptions) formatString(prefix, s string) textNode {
+	maxLen := len(s)
+	maxLines := strings.Count(s, "\n") + 1
+	if opts.LimitVerbosity {
+		maxLen = (1 << opts.verbosity()) << 5   // 32, 64, 128, 256, etc...
+		maxLines = (1 << opts.verbosity()) << 2 //  4, 8, 16, 32, 64, etc...
+	}
+
+	// For multiline strings, use the triple-quote syntax,
+	// but only use it when printing removed or inserted nodes since
+	// we only want the extra verbosity for those cases.
+	lines := strings.Split(strings.TrimSuffix(s, "\n"), "\n")
+	isTripleQuoted := len(lines) >= 4 && (opts.DiffMode == '-' || opts.DiffMode == '+')
+	for i := 0; i < len(lines) && isTripleQuoted; i++ {
+		lines[i] = strings.TrimPrefix(strings.TrimSuffix(lines[i], "\r"), "\r") // trim leading/trailing carriage returns for legacy Windows endline support
+		isPrintable := func(r rune) bool {
+			return unicode.IsPrint(r) || r == '\t' // specially treat tab as printable
+		}
+		line := lines[i]
+		isTripleQuoted = !strings.HasPrefix(strings.TrimPrefix(line, prefix), `"""`) && !strings.HasPrefix(line, "...") && strings.TrimFunc(line, isPrintable) == "" && len(line) <= maxLen
+	}
+	if isTripleQuoted {
+		var list textList
+		list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(prefix + `"""`), ElideComma: true})
+		for i, line := range lines {
+			if numElided := len(lines) - i; i == maxLines-1 && numElided > 1 {
+				comment := commentString(fmt.Sprintf("%d elided lines", numElided))
+				list = append(list, textRecord{Diff: opts.DiffMode, Value: textEllipsis, ElideComma: true, Comment: comment})
+				break
+			}
+			list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(line), ElideComma: true})
+		}
+		list = append(list, textRecord{Diff: opts.DiffMode, Value: textLine(prefix + `"""`), ElideComma: true})
+		return &textWrap{Prefix: "(", Value: list, Suffix: ")"}
+	}
+
+	// Format the string as a single-line quoted string.
+	if len(s) > maxLen+len(textEllipsis) {
+		return textLine(prefix + formatString(s[:maxLen]) + string(textEllipsis))
+	}
+	return textLine(prefix + formatString(s))
+}
+
 // formatMapKey formats v as if it were a map key.
 // The result is guaranteed to be a single line.
-func formatMapKey(v reflect.Value) string {
+func formatMapKey(v reflect.Value, disambiguate bool, ptrs *pointerReferences) string {
 	var opts formatOptions
+	opts.DiffMode = diffIdentical
 	opts.TypeMode = elideType
-	opts.ShallowPointers = true
-	s := opts.FormatValue(v, visitedPointers{}).String()
+	opts.PrintAddresses = disambiguate
+	opts.AvoidStringer = disambiguate
+	opts.QualifiedNames = disambiguate
+	s := opts.FormatValue(v, reflect.Map, ptrs).String()
 	return strings.TrimSpace(s)
 }
 
@@ -227,7 +369,7 @@ func formatString(s string) string {
 	rawInvalid := func(r rune) bool {
 		return r == '`' || r == '\n' || !(unicode.IsPrint(r) || r == '\t')
 	}
-	if strings.IndexFunc(s, rawInvalid) < 0 {
+	if utf8.ValidString(s) && strings.IndexFunc(s, rawInvalid) < 0 {
 		return "`" + s + "`"
 	}
 	return qs
@@ -256,23 +398,3 @@ func formatHex(u uint64) string {
 	}
 	return fmt.Sprintf(f, u)
 }
-
-// formatPointer prints the address of the pointer.
-func formatPointer(v reflect.Value) string {
-	p := v.Pointer()
-	if flags.Deterministic {
-		p = 0xdeadf00f // Only used for stable testing purposes
-	}
-	return fmt.Sprintf("⟪0x%x⟫", p)
-}
-
-type visitedPointers map[value.Pointer]struct{}
-
-// Visit inserts pointer v into the visited map and reports whether it had
-// already been visited before.
-func (m visitedPointers) Visit(v reflect.Value) bool {
-	p := value.PointerOf(v)
-	_, visited := m[p]
-	m[p] = struct{}{}
-	return visited
-}

vendor/github.com/google/go-cmp/cmp/report_slices.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -23,11 +24,25 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 		return false // Must be formatting in diff mode
 	case v.NumDiff == 0:
 		return false // No differences detected
-	case v.NumIgnored+v.NumCompared+v.NumTransformed > 0:
-		// TODO: Handle the case where someone uses bytes.Equal on a large slice.
-		return false // Some custom option was used to determined equality
 	case !v.ValueX.IsValid() || !v.ValueY.IsValid():
 		return false // Both values must be valid
+	case v.Type.Kind() == reflect.Slice && (v.ValueX.Len() == 0 || v.ValueY.Len() == 0):
+		return false // Both slice values have to be non-empty
+	case v.NumIgnored > 0:
+		return false // Some ignore option was used
+	case v.NumTransformed > 0:
+		return false // Some transform option was used
+	case v.NumCompared > 1:
+		return false // More than one comparison was used
+	case v.NumCompared == 1 && v.Type.Name() != "":
+		// The need for cmp to check applicability of options on every element
+		// in a slice is a significant performance detriment for large []byte.
+		// The workaround is to specify Comparer(bytes.Equal),
+		// which enables cmp to compare []byte more efficiently.
+		// If they differ, we still want to provide batched diffing.
+		// The logic disallows named types since they tend to have their own
+		// String method, with nicer formatting than what this provides.
+		return false
 	}
 
 	switch t := v.Type; t.Kind() {
@@ -82,7 +97,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 	if isText || isBinary {
 		var numLines, lastLineIdx, maxLineLen int
-		isBinary = false
+		isBinary = !utf8.ValidString(sx) || !utf8.ValidString(sy)
 		for i, r := range sx + sy {
 			if !(unicode.IsPrint(r) || unicode.IsSpace(r)) || r == utf8.RuneError {
 				isBinary = true
@@ -97,7 +112,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			}
 		}
 		isText = !isBinary
-		isLinedText = isText && numLines >= 4 && maxLineLen <= 256
+		isLinedText = isText && numLines >= 4 && maxLineLen <= 1024
 	}
 
 	// Format the string into printable records.
@@ -117,6 +132,83 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			},
 		)
 		delim = "\n"
+
+		// If possible, use a custom triple-quote (""") syntax for printing
+		// differences in a string literal. This format is more readable,
+		// but has edge-cases where differences are visually indistinguishable.
+		// This format is avoided under the following conditions:
+		//	• A line starts with `"""`
+		//	• A line starts with "..."
+		//	• A line contains non-printable characters
+		//	• Adjacent different lines differ only by whitespace
+		//
+		// For example:
+		//		"""
+		//		... // 3 identical lines
+		//		foo
+		//		bar
+		//	-	baz
+		//	+	BAZ
+		//		"""
+		isTripleQuoted := true
+		prevRemoveLines := map[string]bool{}
+		prevInsertLines := map[string]bool{}
+		var list2 textList
+		list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
+		for _, r := range list {
+			if !r.Value.Equal(textEllipsis) {
+				line, _ := strconv.Unquote(string(r.Value.(textLine)))
+				line = strings.TrimPrefix(strings.TrimSuffix(line, "\r"), "\r") // trim leading/trailing carriage returns for legacy Windows endline support
+				normLine := strings.Map(func(r rune) rune {
+					if unicode.IsSpace(r) {
+						return -1 // drop whitespace to avoid visually indistinguishable output
+					}
+					return r
+				}, line)
+				isPrintable := func(r rune) bool {
+					return unicode.IsPrint(r) || r == '\t' // specially treat tab as printable
+				}
+				isTripleQuoted = !strings.HasPrefix(line, `"""`) && !strings.HasPrefix(line, "...") && strings.TrimFunc(line, isPrintable) == ""
+				switch r.Diff {
+				case diffRemoved:
+					isTripleQuoted = isTripleQuoted && !prevInsertLines[normLine]
+					prevRemoveLines[normLine] = true
+				case diffInserted:
+					isTripleQuoted = isTripleQuoted && !prevRemoveLines[normLine]
+					prevInsertLines[normLine] = true
+				}
+				if !isTripleQuoted {
+					break
+				}
+				r.Value = textLine(line)
+				r.ElideComma = true
+			}
+			if !(r.Diff == diffRemoved || r.Diff == diffInserted) { // start a new non-adjacent difference group
+				prevRemoveLines = map[string]bool{}
+				prevInsertLines = map[string]bool{}
+			}
+			list2 = append(list2, r)
+		}
+		if r := list2[len(list2)-1]; r.Diff == diffIdentical && len(r.Value.(textLine)) == 0 {
+			list2 = list2[:len(list2)-1] // elide single empty line at the end
+		}
+		list2 = append(list2, textRecord{Value: textLine(`"""`), ElideComma: true})
+		if isTripleQuoted {
+			var out textNode = &textWrap{Prefix: "(", Value: list2, Suffix: ")"}
+			switch t.Kind() {
+			case reflect.String:
+				if t != reflect.TypeOf(string("")) {
+					out = opts.FormatType(t, out)
+				}
+			case reflect.Slice:
+				// Always emit type for slices since the triple-quote syntax
+				// looks like a string (not a slice).
+				opts = opts.WithTypeMode(emitType)
+				out = opts.FormatType(t, out)
+			}
+			return out
+		}
+
 	// If the text appears to be single-lined text,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The output is printed as quoted strings.
@@ -129,6 +221,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 			},
 		)
 		delim = ""
+
 	// If the text appears to be binary data,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The output is inspired by hexdump.
@@ -145,6 +238,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 				return textRecord{Diff: d, Value: textLine(s), Comment: comment}
 			},
 		)
+
 	// For all other slices of primitive types,
 	// then perform differencing in approximately fixed-sized chunks.
 	// The size of each chunk depends on the width of the element kind.
@@ -172,7 +266,9 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 					switch t.Elem().Kind() {
 					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 						ss = append(ss, fmt.Sprint(v.Index(i).Int()))
-					case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+					case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+						ss = append(ss, fmt.Sprint(v.Index(i).Uint()))
+					case reflect.Uint8, reflect.Uintptr:
 						ss = append(ss, formatHex(v.Index(i).Uint()))
 					case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
 						ss = append(ss, fmt.Sprint(v.Index(i).Interface()))
@@ -185,7 +281,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 
 	// Wrap the output with appropriate type information.
-	var out textNode = textWrap{"{", list, "}"}
+	var out textNode = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
 	if !isText {
 		// The "{...}" byte-sequence literal is not valid Go syntax for strings.
 		// Emit the type for extra clarity (e.g. "string{...}").
@@ -196,12 +292,12 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	}
 	switch t.Kind() {
 	case reflect.String:
-		out = textWrap{"strings.Join(", out, fmt.Sprintf(", %q)", delim)}
+		out = &textWrap{Prefix: "strings.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
 		if t != reflect.TypeOf(string("")) {
 			out = opts.FormatType(t, out)
 		}
 	case reflect.Slice:
-		out = textWrap{"bytes.Join(", out, fmt.Sprintf(", %q)", delim)}
+		out = &textWrap{Prefix: "bytes.Join(", Value: out, Suffix: fmt.Sprintf(", %q)", delim)}
 		if t != reflect.TypeOf([]byte(nil)) {
 			out = opts.FormatType(t, out)
 		}
@@ -242,9 +338,22 @@ func (opts formatOptions) formatDiffSlice(
 		return n0 - v.Len()
 	}
 
+	var numDiffs int
+	maxLen := -1
+	if opts.LimitVerbosity {
+		maxLen = (1 << opts.verbosity()) << 2 // 4, 8, 16, 32, 64, etc...
+		opts.VerbosityLevel--
+	}
+
 	groups := coalesceAdjacentEdits(name, es)
 	groups = coalesceInterveningIdentical(groups, chunkSize/4)
+	maxGroup := diffStats{Name: name}
 	for i, ds := range groups {
+		if maxLen >= 0 && numDiffs >= maxLen {
+			maxGroup = maxGroup.Append(ds)
+			continue
+		}
+
 		// Print equal.
 		if ds.NumDiff() == 0 {
 			// Compute the number of leading and trailing equal bytes to print.
@@ -273,12 +382,18 @@ func (opts formatOptions) formatDiffSlice(
 		}
 
 		// Print unequal.
+		len0 := len(list)
 		nx := appendChunks(vx.Slice(0, ds.NumIdentical+ds.NumRemoved+ds.NumModified), diffRemoved)
 		vx = vx.Slice(nx, vx.Len())
 		ny := appendChunks(vy.Slice(0, ds.NumIdentical+ds.NumInserted+ds.NumModified), diffInserted)
 		vy = vy.Slice(ny, vy.Len())
+		numDiffs += len(list) - len0
 	}
+	if maxGroup.IsZero() {
 		assert(vx.Len() == 0 && vy.Len() == 0)
+	} else {
+		list.AppendEllipsis(maxGroup)
+	}
 	return list
 }
 

vendor/github.com/google/go-cmp/cmp/report_text.go

@@ -10,12 +10,15 @@ import (
 	"math/rand"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/google/go-cmp/cmp/internal/flags"
 )
 
 var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
 
+const maxColumnLength = 80
+
 type indentMode int
 
 func (n indentMode) appendIndent(b []byte, d diffMode) []byte {
@@ -94,18 +97,19 @@ type textWrap struct {
 	Prefix   string      // e.g., "bytes.Buffer{"
 	Value    textNode    // textWrap | textList | textLine
 	Suffix   string      // e.g., "}"
+	Metadata interface{} // arbitrary metadata; has no effect on formatting
 }
 
-func (s textWrap) Len() int {
+func (s *textWrap) Len() int {
 	return len(s.Prefix) + s.Value.Len() + len(s.Suffix)
 }
-func (s1 textWrap) Equal(s2 textNode) bool {
-	if s2, ok := s2.(textWrap); ok {
+func (s1 *textWrap) Equal(s2 textNode) bool {
+	if s2, ok := s2.(*textWrap); ok {
 		return s1.Prefix == s2.Prefix && s1.Value.Equal(s2.Value) && s1.Suffix == s2.Suffix
 	}
 	return false
 }
-func (s textWrap) String() string {
+func (s *textWrap) String() string {
 	var d diffMode
 	var n indentMode
 	_, s2 := s.formatCompactTo(nil, d)
@@ -114,7 +118,7 @@ func (s textWrap) String() string {
 	b = append(b, '\n')              // Trailing newline
 	return string(b)
 }
-func (s textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
+func (s *textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	n0 := len(b) // Original buffer length
 	b = append(b, s.Prefix...)
 	b, s.Value = s.Value.formatCompactTo(b, d)
@@ -124,7 +128,7 @@ func (s textWrap) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	}
 	return b, s
 }
-func (s textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
+func (s *textWrap) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 	b = append(b, s.Prefix...)
 	b = s.Value.formatExpandedTo(b, d, n)
 	b = append(b, s.Suffix...)
@@ -139,6 +143,7 @@ type textRecord struct {
 	Diff       diffMode     // e.g., 0 or '-' or '+'
 	Key        string       // e.g., "MyField"
 	Value      textNode     // textWrap | textLine
+	ElideComma bool         // avoid trailing comma
 	Comment    fmt.Stringer // e.g., "6 identical fields"
 }
 
@@ -146,12 +151,12 @@ type textRecord struct {
 // exists at the end. If cs is non-zero it coalesces the statistics with the
 // previous diffStats.
 func (s *textList) AppendEllipsis(ds diffStats) {
-	hasStats := ds != diffStats{}
+	hasStats := !ds.IsZero()
 	if len(*s) == 0 || !(*s)[len(*s)-1].Value.Equal(textEllipsis) {
 		if hasStats {
-			*s = append(*s, textRecord{Value: textEllipsis, Comment: ds})
+			*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true, Comment: ds})
 		} else {
-			*s = append(*s, textRecord{Value: textEllipsis})
+			*s = append(*s, textRecord{Value: textEllipsis, ElideComma: true})
 		}
 		return
 	}
@@ -191,7 +196,7 @@ func (s1 textList) Equal(s2 textNode) bool {
 }
 
 func (s textList) String() string {
-	return textWrap{"{", s, "}"}.String()
+	return (&textWrap{Prefix: "{", Value: s, Suffix: "}"}).String()
 }
 
 func (s textList) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
@@ -221,7 +226,7 @@ func (s textList) formatCompactTo(b []byte, d diffMode) ([]byte, textNode) {
 	}
 	// Force multi-lined output when printing a removed/inserted node that
 	// is sufficiently long.
-	if (d == diffInserted || d == diffRemoved) && len(b[n0:]) > 80 {
+	if (d == diffInserted || d == diffRemoved) && len(b[n0:]) > maxColumnLength {
 		multiLine = true
 	}
 	if !multiLine {
@@ -236,16 +241,50 @@ func (s textList) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 			_, isLine := r.Value.(textLine)
 			return r.Key == "" || !isLine
 		},
-		func(r textRecord) int { return len(r.Key) },
+		func(r textRecord) int { return utf8.RuneCountInString(r.Key) },
 	)
 	alignValueLens := s.alignLens(
 		func(r textRecord) bool {
 			_, isLine := r.Value.(textLine)
 			return !isLine || r.Value.Equal(textEllipsis) || r.Comment == nil
 		},
-		func(r textRecord) int { return len(r.Value.(textLine)) },
+		func(r textRecord) int { return utf8.RuneCount(r.Value.(textLine)) },
 	)
 
+	// Format lists of simple lists in a batched form.
+	// If the list is sequence of only textLine values,
+	// then batch multiple values on a single line.
+	var isSimple bool
+	for _, r := range s {
+		_, isLine := r.Value.(textLine)
+		isSimple = r.Diff == 0 && r.Key == "" && isLine && r.Comment == nil
+		if !isSimple {
+			break
+		}
+	}
+	if isSimple {
+		n++
+		var batch []byte
+		emitBatch := func() {
+			if len(batch) > 0 {
+				b = n.appendIndent(append(b, '\n'), d)
+				b = append(b, bytes.TrimRight(batch, " ")...)
+				batch = batch[:0]
+			}
+		}
+		for _, r := range s {
+			line := r.Value.(textLine)
+			if len(batch)+len(line)+len(", ") > maxColumnLength {
+				emitBatch()
+			}
+			batch = append(batch, line...)
+			batch = append(batch, ", "...)
+		}
+		emitBatch()
+		n--
+		return n.appendIndent(append(b, '\n'), d)
+	}
+
 	// Format the list as a multi-lined output.
 	n++
 	for i, r := range s {
@@ -256,7 +295,7 @@ func (s textList) formatExpandedTo(b []byte, d diffMode, n indentMode) []byte {
 		b = alignKeyLens[i].appendChar(b, ' ')
 
 		b = r.Value.formatExpandedTo(b, d|r.Diff, n)
-		if !r.Value.Equal(textEllipsis) {
+		if !r.ElideComma {
 			b = append(b, ',')
 		}
 		b = alignValueLens[i].appendChar(b, ' ')
@@ -332,6 +371,11 @@ type diffStats struct {
 	NumModified  int
 }
 
+func (s diffStats) IsZero() bool {
+	s.Name = ""
+	return s == diffStats{}
+}
+
 func (s diffStats) NumDiff() int {
 	return s.NumRemoved + s.NumInserted + s.NumModified
 }

vendor/github.com/pierrec/lz4/.gitignore

@@ -0,0 +1,34 @@
+# Created by https://www.gitignore.io/api/macos
+
+### macOS ###
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# End of https://www.gitignore.io/api/macos
+
+cmd/*/*exe
+.idea

vendor/github.com/pierrec/lz4/.travis.yml

@@ -0,0 +1,24 @@
+language: go
+
+env:
+  - GO111MODULE=off
+
+go:
+  - 1.9.x
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  - master
+
+matrix:
+ fast_finish: true
+ allow_failures:
+   - go: master
+
+sudo: false
+
+script: 
+ - go test -v -cpu=2
+ - go test -v -cpu=2 -race
+ - go test -v -cpu=2 -tags noasm
+ - go test -v -cpu=2 -race -tags noasm

vendor/github.com/pierrec/lz4/LICENSE

@@ -0,0 +1,28 @@
+Copyright (c) 2015, Pierre Curto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of xxHash nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

vendor/github.com/pierrec/lz4/README.md

@@ -0,0 +1,90 @@
+# lz4 : LZ4 compression in pure Go
+
+[![GoDoc](https://godoc.org/github.com/pierrec/lz4?status.svg)](https://godoc.org/github.com/pierrec/lz4)
+[![Build Status](https://travis-ci.org/pierrec/lz4.svg?branch=master)](https://travis-ci.org/pierrec/lz4)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4)
+[![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags)
+
+## Overview
+
+This package provides a streaming interface to [LZ4 data streams](http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html) as well as low level compress and uncompress functions for LZ4 data blocks.
+The implementation is based on the reference C [one](https://github.com/lz4/lz4).
+
+## Install
+
+Assuming you have the go toolchain installed:
+
+```
+go get github.com/pierrec/lz4
+```
+
+There is a command line interface tool to compress and decompress LZ4 files.
+
+```
+go install github.com/pierrec/lz4/cmd/lz4c
+```
+
+Usage
+
+```
+Usage of lz4c:
+  -version
+        print the program version
+
+Subcommands:
+Compress the given files or from stdin to stdout.
+compress [arguments] [<file name> ...]
+  -bc
+        enable block checksum
+  -l int
+        compression level (0=fastest)
+  -sc
+        disable stream checksum
+  -size string
+        block max size [64K,256K,1M,4M] (default "4M")
+
+Uncompress the given files or from stdin to stdout.
+uncompress [arguments] [<file name> ...]
+
+```
+
+
+## Example
+
+```
+// Compress and uncompress an input string.
+s := "hello world"
+r := strings.NewReader(s)
+
+// The pipe will uncompress the data from the writer.
+pr, pw := io.Pipe()
+zw := lz4.NewWriter(pw)
+zr := lz4.NewReader(pr)
+
+go func() {
+	// Compress the input string.
+	_, _ = io.Copy(zw, r)
+	_ = zw.Close() // Make sure the writer is closed
+	_ = pw.Close() // Terminate the pipe
+}()
+
+_, _ = io.Copy(os.Stdout, zr)
+
+// Output:
+// hello world
+```
+
+## Contributing
+
+Contributions are very welcome for bug fixing, performance improvements...!
+
+- Open an issue with a proper description
+- Send a pull request with appropriate test case(s)
+
+## Contributors
+
+Thanks to all [contributors](https://github.com/pierrec/lz4/graphs/contributors)  so far!
+
+Special thanks to [@Zariel](https://github.com/Zariel) for his asm implementation of the decoder.
+
+Special thanks to [@klauspost](https://github.com/klauspost) for his work on optimizing the code.

vendor/github.com/pierrec/lz4/block.go

@@ -0,0 +1,413 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"math/bits"
+	"sync"
+)
+
+// blockHash hashes the lower 6 bytes into a value < htSize.
+func blockHash(x uint64) uint32 {
+	const prime6bytes = 227718039650203
+	return uint32(((x << (64 - 48)) * prime6bytes) >> (64 - hashLog))
+}
+
+// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
+func CompressBlockBound(n int) int {
+	return n + n/255 + 16
+}
+
+// UncompressBlock uncompresses the source buffer into the destination one,
+// and returns the uncompressed size.
+//
+// The destination buffer must be sized appropriately.
+//
+// An error is returned if the source data is invalid or the destination buffer is too small.
+func UncompressBlock(src, dst []byte) (int, error) {
+	if len(src) == 0 {
+		return 0, nil
+	}
+	if di := decodeBlock(dst, src); di >= 0 {
+		return di, nil
+	}
+	return 0, ErrInvalidSourceShortBuffer
+}
+
+// CompressBlock compresses the source buffer into the destination one.
+// This is the fast version of LZ4 compression and also the default one.
+//
+// The argument hashTable is scratch space for a hash table used by the
+// compressor. If provided, it should have length at least 1<<16. If it is
+// shorter (or nil), CompressBlock allocates its own hash table.
+//
+// The size of the compressed data is returned.
+//
+// If the destination buffer size is lower than CompressBlockBound and
+// the compressed size is 0 and no error, then the data is incompressible.
+//
+// An error is returned if the destination buffer is too small.
+func CompressBlock(src, dst []byte, hashTable []int) (_ int, err error) {
+	defer recoverBlock(&err)
+
+	// Return 0, nil only if the destination buffer size is < CompressBlockBound.
+	isNotCompressible := len(dst) < CompressBlockBound(len(src))
+
+	// adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible.
+	// This significantly speeds up incompressible data and usually has very small impact on compression.
+	// bytes to skip =  1 + (bytes since last match >> adaptSkipLog)
+	const adaptSkipLog = 7
+	if len(hashTable) < htSize {
+		htIface := htPool.Get()
+		defer htPool.Put(htIface)
+		hashTable = (*(htIface).(*[htSize]int))[:]
+	}
+	// Prove to the compiler the table has at least htSize elements.
+	// The compiler can see that "uint32() >> hashShift" cannot be out of bounds.
+	hashTable = hashTable[:htSize]
+
+	// si: Current position of the search.
+	// anchor: Position of the current literals.
+	var si, di, anchor int
+	sn := len(src) - mfLimit
+	if sn <= 0 {
+		goto lastLiterals
+	}
+
+	// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
+	for si < sn {
+		// Hash the next 6 bytes (sequence)...
+		match := binary.LittleEndian.Uint64(src[si:])
+		h := blockHash(match)
+		h2 := blockHash(match >> 8)
+
+		// We check a match at s, s+1 and s+2 and pick the first one we get.
+		// Checking 3 only requires us to load the source one.
+		ref := hashTable[h]
+		ref2 := hashTable[h2]
+		hashTable[h] = si
+		hashTable[h2] = si + 1
+		offset := si - ref
+
+		// If offset <= 0 we got an old entry in the hash table.
+		if offset <= 0 || offset >= winSize || // Out of window.
+			uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
+			// No match. Start calculating another hash.
+			// The processor can usually do this out-of-order.
+			h = blockHash(match >> 16)
+			ref = hashTable[h]
+
+			// Check the second match at si+1
+			si += 1
+			offset = si - ref2
+
+			if offset <= 0 || offset >= winSize ||
+				uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) {
+				// No match. Check the third match at si+2
+				si += 1
+				offset = si - ref
+				hashTable[h] = si
+
+				if offset <= 0 || offset >= winSize ||
+					uint32(match>>16) != binary.LittleEndian.Uint32(src[ref:]) {
+					// Skip one extra byte (at si+3) before we check 3 matches again.
+					si += 2 + (si-anchor)>>adaptSkipLog
+					continue
+				}
+			}
+		}
+
+		// Match found.
+		lLen := si - anchor // Literal length.
+		// We already matched 4 bytes.
+		mLen := 4
+
+		// Extend backwards if we can, reducing literals.
+		tOff := si - offset - 1
+		for lLen > 0 && tOff >= 0 && src[si-1] == src[tOff] {
+			si--
+			tOff--
+			lLen--
+			mLen++
+		}
+
+		// Add the match length, so we continue search at the end.
+		// Use mLen to store the offset base.
+		si, mLen = si+mLen, si+minMatch
+
+		// Find the longest match by looking by batches of 8 bytes.
+		for si+8 < sn {
+			x := binary.LittleEndian.Uint64(src[si:]) ^ binary.LittleEndian.Uint64(src[si-offset:])
+			if x == 0 {
+				si += 8
+			} else {
+				// Stop is first non-zero byte.
+				si += bits.TrailingZeros64(x) >> 3
+				break
+			}
+		}
+
+		mLen = si - mLen
+		if mLen < 0xF {
+			dst[di] = byte(mLen)
+		} else {
+			dst[di] = 0xF
+		}
+
+		// Encode literals length.
+		if lLen < 0xF {
+			dst[di] |= byte(lLen << 4)
+		} else {
+			dst[di] |= 0xF0
+			di++
+			l := lLen - 0xF
+			for ; l >= 0xFF; l -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(l)
+		}
+		di++
+
+		// Literals.
+		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
+		di += lLen + 2
+		anchor = si
+
+		// Encode offset.
+		_ = dst[di] // Bound check elimination.
+		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
+
+		// Encode match length part 2.
+		if mLen >= 0xF {
+			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(mLen)
+			di++
+		}
+		// Check if we can load next values.
+		if si >= sn {
+			break
+		}
+		// Hash match end-2
+		h = blockHash(binary.LittleEndian.Uint64(src[si-2:]))
+		hashTable[h] = si - 2
+	}
+
+lastLiterals:
+	if isNotCompressible && anchor == 0 {
+		// Incompressible.
+		return 0, nil
+	}
+
+	// Last literals.
+	lLen := len(src) - anchor
+	if lLen < 0xF {
+		dst[di] = byte(lLen << 4)
+	} else {
+		dst[di] = 0xF0
+		di++
+		for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
+			dst[di] = 0xFF
+			di++
+		}
+		dst[di] = byte(lLen)
+	}
+	di++
+
+	// Write the last literals.
+	if isNotCompressible && di >= anchor {
+		// Incompressible.
+		return 0, nil
+	}
+	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
+	return di, nil
+}
+
+// Pool of hash tables for CompressBlock.
+var htPool = sync.Pool{
+	New: func() interface{} {
+		return new([htSize]int)
+	},
+}
+
+// blockHash hashes 4 bytes into a value < winSize.
+func blockHashHC(x uint32) uint32 {
+	const hasher uint32 = 2654435761 // Knuth multiplicative hash.
+	return x * hasher >> (32 - winSizeLog)
+}
+
+// CompressBlockHC compresses the source buffer src into the destination dst
+// with max search depth (use 0 or negative value for no max).
+//
+// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
+//
+// The size of the compressed data is returned.
+//
+// If the destination buffer size is lower than CompressBlockBound and
+// the compressed size is 0 and no error, then the data is incompressible.
+//
+// An error is returned if the destination buffer is too small.
+func CompressBlockHC(src, dst []byte, depth int) (_ int, err error) {
+	defer recoverBlock(&err)
+
+	// Return 0, nil only if the destination buffer size is < CompressBlockBound.
+	isNotCompressible := len(dst) < CompressBlockBound(len(src))
+
+	// adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible.
+	// This significantly speeds up incompressible data and usually has very small impact on compression.
+	// bytes to skip =  1 + (bytes since last match >> adaptSkipLog)
+	const adaptSkipLog = 7
+
+	var si, di, anchor int
+
+	// hashTable: stores the last position found for a given hash
+	// chainTable: stores previous positions for a given hash
+	var hashTable, chainTable [winSize]int
+
+	if depth <= 0 {
+		depth = winSize
+	}
+
+	sn := len(src) - mfLimit
+	if sn <= 0 {
+		goto lastLiterals
+	}
+
+	for si < sn {
+		// Hash the next 4 bytes (sequence).
+		match := binary.LittleEndian.Uint32(src[si:])
+		h := blockHashHC(match)
+
+		// Follow the chain until out of window and give the longest match.
+		mLen := 0
+		offset := 0
+		for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
+			// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
+			// must match to improve on the match length.
+			if src[next+mLen] != src[si+mLen] {
+				continue
+			}
+			ml := 0
+			// Compare the current position with a previous with the same hash.
+			for ml < sn-si {
+				x := binary.LittleEndian.Uint64(src[next+ml:]) ^ binary.LittleEndian.Uint64(src[si+ml:])
+				if x == 0 {
+					ml += 8
+				} else {
+					// Stop is first non-zero byte.
+					ml += bits.TrailingZeros64(x) >> 3
+					break
+				}
+			}
+			if ml < minMatch || ml <= mLen {
+				// Match too small (<minMath) or smaller than the current match.
+				continue
+			}
+			// Found a longer match, keep its position and length.
+			mLen = ml
+			offset = si - next
+			// Try another previous position with the same hash.
+			try--
+		}
+		chainTable[si&winMask] = hashTable[h]
+		hashTable[h] = si
+
+		// No match found.
+		if mLen == 0 {
+			si += 1 + (si-anchor)>>adaptSkipLog
+			continue
+		}
+
+		// Match found.
+		// Update hash/chain tables with overlapping bytes:
+		// si already hashed, add everything from si+1 up to the match length.
+		winStart := si + 1
+		if ws := si + mLen - winSize; ws > winStart {
+			winStart = ws
+		}
+		for si, ml := winStart, si+mLen; si < ml; {
+			match >>= 8
+			match |= uint32(src[si+3]) << 24
+			h := blockHashHC(match)
+			chainTable[si&winMask] = hashTable[h]
+			hashTable[h] = si
+			si++
+		}
+
+		lLen := si - anchor
+		si += mLen
+		mLen -= minMatch // Match length does not include minMatch.
+
+		if mLen < 0xF {
+			dst[di] = byte(mLen)
+		} else {
+			dst[di] = 0xF
+		}
+
+		// Encode literals length.
+		if lLen < 0xF {
+			dst[di] |= byte(lLen << 4)
+		} else {
+			dst[di] |= 0xF0
+			di++
+			l := lLen - 0xF
+			for ; l >= 0xFF; l -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(l)
+		}
+		di++
+
+		// Literals.
+		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
+		di += lLen
+		anchor = si
+
+		// Encode offset.
+		di += 2
+		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
+
+		// Encode match length part 2.
+		if mLen >= 0xF {
+			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(mLen)
+			di++
+		}
+	}
+
+	if isNotCompressible && anchor == 0 {
+		// Incompressible.
+		return 0, nil
+	}
+
+	// Last literals.
+lastLiterals:
+	lLen := len(src) - anchor
+	if lLen < 0xF {
+		dst[di] = byte(lLen << 4)
+	} else {
+		dst[di] = 0xF0
+		di++
+		lLen -= 0xF
+		for ; lLen >= 0xFF; lLen -= 0xFF {
+			dst[di] = 0xFF
+			di++
+		}
+		dst[di] = byte(lLen)
+	}
+	di++
+
+	// Write the last literals.
+	if isNotCompressible && di >= anchor {
+		// Incompressible.
+		return 0, nil
+	}
+	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
+	return di, nil
+}

vendor/github.com/pierrec/lz4/debug.go

@@ -0,0 +1,23 @@
+// +build lz4debug
+
+package lz4
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+const debugFlag = true
+
+func debug(args ...interface{}) {
+	_, file, line, _ := runtime.Caller(1)
+	file = filepath.Base(file)
+
+	f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0])
+	if f[len(f)-1] != '\n' {
+		f += "\n"
+	}
+	fmt.Fprintf(os.Stderr, f, args[1:]...)
+}

vendor/github.com/pierrec/lz4/debug_stub.go

@@ -0,0 +1,7 @@
+// +build !lz4debug
+
+package lz4
+
+const debugFlag = false
+
+func debug(args ...interface{}) {}

vendor/github.com/pierrec/lz4/decode_amd64.go

@@ -0,0 +1,8 @@
+// +build !appengine
+// +build gc
+// +build !noasm
+
+package lz4
+
+//go:noescape
+func decodeBlock(dst, src []byte) int

vendor/github.com/pierrec/lz4/decode_amd64.s

@@ -0,0 +1,375 @@
+// +build !appengine
+// +build gc
+// +build !noasm
+
+#include "textflag.h"
+
+// AX scratch
+// BX scratch
+// CX scratch
+// DX token
+//
+// DI &dst
+// SI &src
+// R8 &dst + len(dst)
+// R9 &src + len(src)
+// R11 &dst
+// R12 short output end
+// R13 short input end
+// func decodeBlock(dst, src []byte) int
+// using 50 bytes of stack currently
+TEXT ·decodeBlock(SB), NOSPLIT, $64-56
+	MOVQ dst_base+0(FP), DI
+	MOVQ DI, R11
+	MOVQ dst_len+8(FP), R8
+	ADDQ DI, R8
+
+	MOVQ src_base+24(FP), SI
+	MOVQ src_len+32(FP), R9
+	ADDQ SI, R9
+
+	// shortcut ends
+	// short output end
+	MOVQ R8, R12
+	SUBQ $32, R12
+	// short input end
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+loop:
+	// for si < len(src)
+	CMPQ SI, R9
+	JGE end
+
+	// token := uint32(src[si])
+	MOVBQZX (SI), DX
+	INCQ SI
+
+	// lit_len = token >> 4
+	// if lit_len > 0
+	// CX = lit_len
+	MOVQ DX, CX
+	SHRQ $4, CX
+
+	// if lit_len != 0xF
+	CMPQ CX, $0xF
+	JEQ lit_len_loop_pre
+	CMPQ DI, R12
+	JGE lit_len_loop_pre
+	CMPQ SI, R13
+	JGE lit_len_loop_pre
+
+	// copy shortcut
+
+	// A two-stage shortcut for the most common case:
+	// 1) If the literal length is 0..14, and there is enough space,
+	// enter the shortcut and copy 16 bytes on behalf of the literals
+	// (in the fast mode, only 8 bytes can be safely copied this way).
+	// 2) Further if the match length is 4..18, copy 18 bytes in a similar
+	// manner; but we ensure that there's enough space in the output for
+	// those 18 bytes earlier, upon entering the shortcut (in other words,
+	// there is a combined check for both stages).
+
+	// copy literal
+	MOVOU (SI), X0
+	MOVOU X0, (DI)
+	ADDQ CX, DI
+	ADDQ CX, SI
+
+	MOVQ DX, CX
+	ANDQ $0xF, CX
+
+	// The second stage: prepare for match copying, decode full info.
+	// If it doesn't work out, the info won't be wasted.
+	// offset := uint16(data[:2])
+	MOVWQZX (SI), DX
+	ADDQ $2, SI
+
+	MOVQ DI, AX
+	SUBQ DX, AX
+	CMPQ AX, DI
+	JGT err_short_buf
+
+	// if we can't do the second stage then jump straight to read the
+	// match length, we already have the offset.
+	CMPQ CX, $0xF
+	JEQ match_len_loop_pre
+	CMPQ DX, $8
+	JLT match_len_loop_pre
+	CMPQ AX, R11
+	JLT err_short_buf
+
+	// memcpy(op + 0, match + 0, 8);
+	MOVQ (AX), BX
+	MOVQ BX, (DI)
+	// memcpy(op + 8, match + 8, 8);
+	MOVQ 8(AX), BX
+	MOVQ BX, 8(DI)
+	// memcpy(op +16, match +16, 2);
+	MOVW 16(AX), BX
+	MOVW BX, 16(DI)
+
+	ADDQ $4, DI // minmatch
+	ADDQ CX, DI
+
+	// shortcut complete, load next token
+	JMP loop
+
+lit_len_loop_pre:
+	// if lit_len > 0
+	CMPQ CX, $0
+	JEQ offset
+	CMPQ CX, $0xF
+	JNE copy_literal
+
+lit_len_loop:
+	// for src[si] == 0xFF
+	CMPB (SI), $0xFF
+	JNE lit_len_finalise
+
+	// bounds check src[si+1]
+	MOVQ SI, AX
+	ADDQ $1, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	// lit_len += 0xFF
+	ADDQ $0xFF, CX
+	INCQ SI
+	JMP lit_len_loop
+
+lit_len_finalise:
+	// lit_len += int(src[si])
+	// si++
+	MOVBQZX (SI), AX
+	ADDQ AX, CX
+	INCQ SI
+
+copy_literal:
+	// bounds check src and dst
+	MOVQ SI, AX
+	ADDQ CX, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	MOVQ DI, AX
+	ADDQ CX, AX
+	CMPQ AX, R8
+	JGT err_short_buf
+
+	// whats a good cut off to call memmove?
+	CMPQ CX, $16
+	JGT memmove_lit
+
+	// if len(dst[di:]) < 16
+	MOVQ R8, AX
+	SUBQ DI, AX
+	CMPQ AX, $16
+	JLT memmove_lit
+
+	// if len(src[si:]) < 16
+	MOVQ R9, AX
+	SUBQ SI, AX
+	CMPQ AX, $16
+	JLT memmove_lit
+
+	MOVOU (SI), X0
+	MOVOU X0, (DI)
+
+	JMP finish_lit_copy
+
+memmove_lit:
+	// memmove(to, from, len)
+	MOVQ DI, 0(SP)
+	MOVQ SI, 8(SP)
+	MOVQ CX, 16(SP)
+	// spill
+	MOVQ DI, 24(SP)
+	MOVQ SI, 32(SP)
+	MOVQ CX, 40(SP) // need len to inc SI, DI after
+	MOVB DX, 48(SP)
+	CALL runtime·memmove(SB)
+
+	// restore registers
+	MOVQ 24(SP), DI
+	MOVQ 32(SP), SI
+	MOVQ 40(SP), CX
+	MOVB 48(SP), DX
+
+	// recalc initial values
+	MOVQ dst_base+0(FP), R8
+	MOVQ R8, R11
+	ADDQ dst_len+8(FP), R8
+	MOVQ src_base+24(FP), R9
+	ADDQ src_len+32(FP), R9
+	MOVQ R8, R12
+	SUBQ $32, R12
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+finish_lit_copy:
+	ADDQ CX, SI
+	ADDQ CX, DI
+
+	CMPQ SI, R9
+	JGE end
+
+offset:
+	// CX := mLen
+	// free up DX to use for offset
+	MOVQ DX, CX
+
+	MOVQ SI, AX
+	ADDQ $2, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	// offset
+	// DX := int(src[si]) | int(src[si+1])<<8
+	MOVWQZX (SI), DX
+	ADDQ $2, SI
+
+	// 0 offset is invalid
+	CMPQ DX, $0
+	JEQ err_corrupt
+
+	ANDB $0xF, CX
+
+match_len_loop_pre:
+	// if mlen != 0xF
+	CMPB CX, $0xF
+	JNE copy_match
+
+match_len_loop:
+	// for src[si] == 0xFF
+	// lit_len += 0xFF
+	CMPB (SI), $0xFF
+	JNE match_len_finalise
+
+	// bounds check src[si+1]
+	MOVQ SI, AX
+	ADDQ $1, AX
+	CMPQ AX, R9
+	JGT err_short_buf
+
+	ADDQ $0xFF, CX
+	INCQ SI
+	JMP match_len_loop
+
+match_len_finalise:
+	// lit_len += int(src[si])
+	// si++
+	MOVBQZX (SI), AX
+	ADDQ AX, CX
+	INCQ SI
+
+copy_match:
+	// mLen += minMatch
+	ADDQ $4, CX
+
+	// check we have match_len bytes left in dst
+	// di+match_len < len(dst)
+	MOVQ DI, AX
+	ADDQ CX, AX
+	CMPQ AX, R8
+	JGT err_short_buf
+
+	// DX = offset
+	// CX = match_len
+	// BX = &dst + (di - offset)
+	MOVQ DI, BX
+	SUBQ DX, BX
+
+	// check BX is within dst
+	// if BX < &dst
+	CMPQ BX, R11
+	JLT err_short_buf
+
+	// if offset + match_len < di
+	MOVQ BX, AX
+	ADDQ CX, AX
+	CMPQ DI, AX
+	JGT copy_interior_match
+
+	// AX := len(dst[:di])
+	// MOVQ DI, AX
+	// SUBQ R11, AX
+
+	// copy 16 bytes at a time
+	// if di-offset < 16 copy 16-(di-offset) bytes to di
+	// then do the remaining
+
+copy_match_loop:
+	// for match_len >= 0
+	// dst[di] = dst[i]
+	// di++
+	// i++
+	MOVB (BX), AX
+	MOVB AX, (DI)
+	INCQ DI
+	INCQ BX
+	DECQ CX
+
+	CMPQ CX, $0
+	JGT copy_match_loop
+
+	JMP loop
+
+copy_interior_match:
+	CMPQ CX, $16
+	JGT memmove_match
+
+	// if len(dst[di:]) < 16
+	MOVQ R8, AX
+	SUBQ DI, AX
+	CMPQ AX, $16
+	JLT memmove_match
+
+	MOVOU (BX), X0
+	MOVOU X0, (DI)
+
+	ADDQ CX, DI
+	JMP loop
+
+memmove_match:
+	// memmove(to, from, len)
+	MOVQ DI, 0(SP)
+	MOVQ BX, 8(SP)
+	MOVQ CX, 16(SP)
+	// spill
+	MOVQ DI, 24(SP)
+	MOVQ SI, 32(SP)
+	MOVQ CX, 40(SP) // need len to inc SI, DI after
+	CALL runtime·memmove(SB)
+
+	// restore registers
+	MOVQ 24(SP), DI
+	MOVQ 32(SP), SI
+	MOVQ 40(SP), CX
+
+	// recalc initial values
+	MOVQ dst_base+0(FP), R8
+	MOVQ R8, R11 // TODO: make these sensible numbers
+	ADDQ dst_len+8(FP), R8
+	MOVQ src_base+24(FP), R9
+	ADDQ src_len+32(FP), R9
+	MOVQ R8, R12
+	SUBQ $32, R12
+	MOVQ R9, R13
+	SUBQ $16, R13
+
+	ADDQ CX, DI
+	JMP loop
+
+err_corrupt:
+	MOVQ $-1, ret+48(FP)
+	RET
+
+err_short_buf:
+	MOVQ $-2, ret+48(FP)
+	RET
+
+end:
+	SUBQ R11, DI
+	MOVQ DI, ret+48(FP)
+	RET

vendor/github.com/pierrec/lz4/decode_other.go

@@ -0,0 +1,98 @@
+// +build !amd64 appengine !gc noasm
+
+package lz4
+
+func decodeBlock(dst, src []byte) (ret int) {
+	const hasError = -2
+	defer func() {
+		if recover() != nil {
+			ret = hasError
+		}
+	}()
+
+	var si, di int
+	for {
+		// Literals and match lengths (token).
+		b := int(src[si])
+		si++
+
+		// Literals.
+		if lLen := b >> 4; lLen > 0 {
+			switch {
+			case lLen < 0xF && si+16 < len(src):
+				// Shortcut 1
+				// if we have enough room in src and dst, and the literals length
+				// is small enough (0..14) then copy all 16 bytes, even if not all
+				// are part of the literals.
+				copy(dst[di:], src[si:si+16])
+				si += lLen
+				di += lLen
+				if mLen := b & 0xF; mLen < 0xF {
+					// Shortcut 2
+					// if the match length (4..18) fits within the literals, then copy
+					// all 18 bytes, even if not all are part of the literals.
+					mLen += 4
+					if offset := int(src[si]) | int(src[si+1])<<8; mLen <= offset {
+						i := di - offset
+						end := i + 18
+						if end > len(dst) {
+							// The remaining buffer may not hold 18 bytes.
+							// See https://github.com/pierrec/lz4/issues/51.
+							end = len(dst)
+						}
+						copy(dst[di:], dst[i:end])
+						si += 2
+						di += mLen
+						continue
+					}
+				}
+			case lLen == 0xF:
+				for src[si] == 0xFF {
+					lLen += 0xFF
+					si++
+				}
+				lLen += int(src[si])
+				si++
+				fallthrough
+			default:
+				copy(dst[di:di+lLen], src[si:si+lLen])
+				si += lLen
+				di += lLen
+			}
+		}
+		if si >= len(src) {
+			return di
+		}
+
+		offset := int(src[si]) | int(src[si+1])<<8
+		if offset == 0 {
+			return hasError
+		}
+		si += 2
+
+		// Match.
+		mLen := b & 0xF
+		if mLen == 0xF {
+			for src[si] == 0xFF {
+				mLen += 0xFF
+				si++
+			}
+			mLen += int(src[si])
+			si++
+		}
+		mLen += minMatch
+
+		// Copy the match.
+		expanded := dst[di-offset:]
+		if mLen > offset {
+			// Efficiently copy the match dst[di-offset:di] into the dst slice.
+			bytesToCopy := offset * (mLen / offset)
+			for n := offset; n <= bytesToCopy+offset; n *= 2 {
+				copy(expanded[n:], expanded[:n])
+			}
+			di += bytesToCopy
+			mLen -= bytesToCopy
+		}
+		di += copy(dst[di:di+mLen], expanded[:mLen])
+	}
+}

vendor/github.com/pierrec/lz4/errors.go

@@ -0,0 +1,30 @@
+package lz4
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	rdebug "runtime/debug"
+)
+
+var (
+	// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
+	// block is corrupted or the destination buffer is not large enough for the uncompressed data.
+	ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
+	// ErrInvalid is returned when reading an invalid LZ4 archive.
+	ErrInvalid = errors.New("lz4: bad magic number")
+	// ErrBlockDependency is returned when attempting to decompress an archive created with block dependency.
+	ErrBlockDependency = errors.New("lz4: block dependency not supported")
+	// ErrUnsupportedSeek is returned when attempting to Seek any way but forward from the current position.
+	ErrUnsupportedSeek = errors.New("lz4: can only seek forward from io.SeekCurrent")
+)
+
+func recoverBlock(e *error) {
+	if r := recover(); r != nil && *e == nil {
+		if debugFlag {
+			fmt.Fprintln(os.Stderr, r)
+			rdebug.PrintStack()
+		}
+		*e = ErrInvalidSourceShortBuffer
+	}
+}

vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go

@@ -0,0 +1,223 @@
+// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version).
+// (https://github.com/Cyan4973/XXH/)
+package xxh32
+
+import (
+	"encoding/binary"
+)
+
+const (
+	prime1 uint32 = 2654435761
+	prime2 uint32 = 2246822519
+	prime3 uint32 = 3266489917
+	prime4 uint32 = 668265263
+	prime5 uint32 = 374761393
+
+	primeMask   = 0xFFFFFFFF
+	prime1plus2 = uint32((uint64(prime1) + uint64(prime2)) & primeMask) // 606290984
+	prime1minus = uint32((-int64(prime1)) & primeMask)                  // 1640531535
+)
+
+// XXHZero represents an xxhash32 object with seed 0.
+type XXHZero struct {
+	v1       uint32
+	v2       uint32
+	v3       uint32
+	v4       uint32
+	totalLen uint64
+	buf      [16]byte
+	bufused  int
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (xxh XXHZero) Sum(b []byte) []byte {
+	h32 := xxh.Sum32()
+	return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24))
+}
+
+// Reset resets the Hash to its initial state.
+func (xxh *XXHZero) Reset() {
+	xxh.v1 = prime1plus2
+	xxh.v2 = prime2
+	xxh.v3 = 0
+	xxh.v4 = prime1minus
+	xxh.totalLen = 0
+	xxh.bufused = 0
+}
+
+// Size returns the number of bytes returned by Sum().
+func (xxh *XXHZero) Size() int {
+	return 4
+}
+
+// BlockSize gives the minimum number of bytes accepted by Write().
+func (xxh *XXHZero) BlockSize() int {
+	return 1
+}
+
+// Write adds input bytes to the Hash.
+// It never returns an error.
+func (xxh *XXHZero) Write(input []byte) (int, error) {
+	if xxh.totalLen == 0 {
+		xxh.Reset()
+	}
+	n := len(input)
+	m := xxh.bufused
+
+	xxh.totalLen += uint64(n)
+
+	r := len(xxh.buf) - m
+	if n < r {
+		copy(xxh.buf[m:], input)
+		xxh.bufused += len(input)
+		return n, nil
+	}
+
+	p := 0
+	// Causes compiler to work directly from registers instead of stack:
+	v1, v2, v3, v4 := xxh.v1, xxh.v2, xxh.v3, xxh.v4
+	if m > 0 {
+		// some data left from previous update
+		copy(xxh.buf[xxh.bufused:], input[:r])
+		xxh.bufused += len(input) - r
+
+		// fast rotl(13)
+		buf := xxh.buf[:16] // BCE hint.
+		v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime2) * prime1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime2) * prime1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime2) * prime1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime2) * prime1
+		p = r
+		xxh.bufused = 0
+	}
+
+	for n := n - 16; p <= n; p += 16 {
+		sub := input[p:][:16] //BCE hint for compiler
+		v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1
+	}
+	xxh.v1, xxh.v2, xxh.v3, xxh.v4 = v1, v2, v3, v4
+
+	copy(xxh.buf[xxh.bufused:], input[p:])
+	xxh.bufused += len(input) - p
+
+	return n, nil
+}
+
+// Sum32 returns the 32 bits Hash value.
+func (xxh *XXHZero) Sum32() uint32 {
+	h32 := uint32(xxh.totalLen)
+	if h32 >= 16 {
+		h32 += rol1(xxh.v1) + rol7(xxh.v2) + rol12(xxh.v3) + rol18(xxh.v4)
+	} else {
+		h32 += prime5
+	}
+
+	p := 0
+	n := xxh.bufused
+	buf := xxh.buf
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime3
+		h32 = rol17(h32) * prime4
+	}
+	for ; p < n; p++ {
+		h32 += uint32(buf[p]) * prime5
+		h32 = rol11(h32) * prime1
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime2
+	h32 ^= h32 >> 13
+	h32 *= prime3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// ChecksumZero returns the 32bits Hash value.
+func ChecksumZero(input []byte) uint32 {
+	n := len(input)
+	h32 := uint32(n)
+
+	if n < 16 {
+		h32 += prime5
+	} else {
+		v1 := prime1plus2
+		v2 := prime2
+		v3 := uint32(0)
+		v4 := prime1minus
+		p := 0
+		for n := n - 16; p <= n; p += 16 {
+			sub := input[p:][:16] //BCE hint for compiler
+			v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1
+			v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1
+			v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1
+			v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1
+		}
+		input = input[p:]
+		n -= p
+		h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
+	}
+
+	p := 0
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime3
+		h32 = rol17(h32) * prime4
+	}
+	for p < n {
+		h32 += uint32(input[p]) * prime5
+		h32 = rol11(h32) * prime1
+		p++
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime2
+	h32 ^= h32 >> 13
+	h32 *= prime3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// Uint32Zero hashes x with seed 0.
+func Uint32Zero(x uint32) uint32 {
+	h := prime5 + 4 + x*prime3
+	h = rol17(h) * prime4
+	h ^= h >> 15
+	h *= prime2
+	h ^= h >> 13
+	h *= prime3
+	h ^= h >> 16
+	return h
+}
+
+func rol1(u uint32) uint32 {
+	return u<<1 | u>>31
+}
+
+func rol7(u uint32) uint32 {
+	return u<<7 | u>>25
+}
+
+func rol11(u uint32) uint32 {
+	return u<<11 | u>>21
+}
+
+func rol12(u uint32) uint32 {
+	return u<<12 | u>>20
+}
+
+func rol13(u uint32) uint32 {
+	return u<<13 | u>>19
+}
+
+func rol17(u uint32) uint32 {
+	return u<<17 | u>>15
+}
+
+func rol18(u uint32) uint32 {
+	return u<<18 | u>>14
+}

vendor/github.com/pierrec/lz4/lz4.go

@@ -0,0 +1,113 @@
+// Package lz4 implements reading and writing lz4 compressed data (a frame),
+// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
+//
+// Although the block level compression and decompression functions are exposed and are fully compatible
+// with the lz4 block format definition, they are low level and should not be used directly.
+// For a complete description of an lz4 compressed block, see:
+// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
+//
+// See https://github.com/Cyan4973/lz4 for the reference C implementation.
+//
+package lz4
+
+import "math/bits"
+
+import "sync"
+
+const (
+	// Extension is the LZ4 frame file name extension
+	Extension = ".lz4"
+	// Version is the LZ4 frame format version
+	Version = 1
+
+	frameMagic     uint32 = 0x184D2204
+	frameSkipMagic uint32 = 0x184D2A50
+
+	// The following constants are used to setup the compression algorithm.
+	minMatch            = 4  // the minimum size of the match sequence size (4 bytes)
+	winSizeLog          = 16 // LZ4 64Kb window size limit
+	winSize             = 1 << winSizeLog
+	winMask             = winSize - 1 // 64Kb window of previous data for dependent blocks
+	compressedBlockFlag = 1 << 31
+	compressedBlockMask = compressedBlockFlag - 1
+
+	// hashLog determines the size of the hash table used to quickly find a previous match position.
+	// Its value influences the compression speed and memory usage, the lower the faster,
+	// but at the expense of the compression ratio.
+	// 16 seems to be the best compromise for fast compression.
+	hashLog = 16
+	htSize  = 1 << hashLog
+
+	mfLimit = 10 + minMatch // The last match cannot start within the last 14 bytes.
+)
+
+// map the block max size id with its value in bytes: 64Kb, 256Kb, 1Mb and 4Mb.
+const (
+	blockSize64K = 1 << (16 + 2*iota)
+	blockSize256K
+	blockSize1M
+	blockSize4M
+)
+
+var (
+	// Keep a pool of buffers for each valid block sizes.
+	bsMapValue = [...]*sync.Pool{
+		newBufferPool(2 * blockSize64K),
+		newBufferPool(2 * blockSize256K),
+		newBufferPool(2 * blockSize1M),
+		newBufferPool(2 * blockSize4M),
+	}
+)
+
+// newBufferPool returns a pool for buffers of the given size.
+func newBufferPool(size int) *sync.Pool {
+	return &sync.Pool{
+		New: func() interface{} {
+			return make([]byte, size)
+		},
+	}
+}
+
+// getBuffer returns a buffer to its pool.
+func getBuffer(size int) []byte {
+	idx := blockSizeValueToIndex(size) - 4
+	return bsMapValue[idx].Get().([]byte)
+}
+
+// putBuffer returns a buffer to its pool.
+func putBuffer(size int, buf []byte) {
+	if cap(buf) > 0 {
+		idx := blockSizeValueToIndex(size) - 4
+		bsMapValue[idx].Put(buf[:cap(buf)])
+	}
+}
+func blockSizeIndexToValue(i byte) int {
+	return 1 << (16 + 2*uint(i))
+}
+func isValidBlockSize(size int) bool {
+	const blockSizeMask = blockSize64K | blockSize256K | blockSize1M | blockSize4M
+
+	return size&blockSizeMask > 0 && bits.OnesCount(uint(size)) == 1
+}
+func blockSizeValueToIndex(size int) byte {
+	return 4 + byte(bits.TrailingZeros(uint(size)>>16)/2)
+}
+
+// Header describes the various flags that can be set on a Writer or obtained from a Reader.
+// The default values match those of the LZ4 frame format definition
+// (http://fastcompression.blogspot.com/2013/04/lz4-streaming-format-final.html).
+//
+// NB. in a Reader, in case of concatenated frames, the Header values may change between Read() calls.
+// It is the caller's responsibility to check them if necessary.
+type Header struct {
+	BlockChecksum    bool   // Compressed blocks checksum flag.
+	NoChecksum       bool   // Frame checksum flag.
+	BlockMaxSize     int    // Size of the uncompressed data block (one of [64KB, 256KB, 1MB, 4MB]). Default=4MB.
+	Size             uint64 // Frame total size. It is _not_ computed by the Writer.
+	CompressionLevel int    // Compression level (higher is better, use 0 for fastest compression).
+	done             bool   // Header processed flag (Read or Write and checked).
+}
+
+func (h *Header) Reset() {
+	h.done = false
+}

vendor/github.com/pierrec/lz4/lz4_go1.10.go

@@ -0,0 +1,29 @@
+//+build go1.10
+
+package lz4
+
+import (
+	"fmt"
+	"strings"
+)
+
+func (h Header) String() string {
+	var s strings.Builder
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/lz4_notgo1.10.go

@@ -0,0 +1,29 @@
+//+build !go1.10
+
+package lz4
+
+import (
+	"bytes"
+	"fmt"
+)
+
+func (h Header) String() string {
+	var s bytes.Buffer
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/reader.go

@@ -0,0 +1,335 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/pierrec/lz4/internal/xxh32"
+)
+
+// Reader implements the LZ4 frame decoder.
+// The Header is set after the first call to Read().
+// The Header may change between Read() calls in case of concatenated frames.
+type Reader struct {
+	Header
+	// Handler called when a block has been successfully read.
+	// It provides the number of bytes read.
+	OnBlockDone func(size int)
+
+	buf      [8]byte       // Scrap buffer.
+	pos      int64         // Current position in src.
+	src      io.Reader     // Source.
+	zdata    []byte        // Compressed data.
+	data     []byte        // Uncompressed data.
+	idx      int           // Index of unread bytes into data.
+	checksum xxh32.XXHZero // Frame hash.
+	skip     int64         // Bytes to skip before next read.
+	dpos     int64         // Position in dest
+}
+
+// NewReader returns a new LZ4 frame decoder.
+// No access to the underlying io.Reader is performed.
+func NewReader(src io.Reader) *Reader {
+	r := &Reader{src: src}
+	return r
+}
+
+// readHeader checks the frame magic number and parses the frame descriptoz.
+// Skippable frames are supported even as a first frame although the LZ4
+// specifications recommends skippable frames not to be used as first frames.
+func (z *Reader) readHeader(first bool) error {
+	defer z.checksum.Reset()
+
+	buf := z.buf[:]
+	for {
+		magic, err := z.readUint32()
+		if err != nil {
+			z.pos += 4
+			if !first && err == io.ErrUnexpectedEOF {
+				return io.EOF
+			}
+			return err
+		}
+		if magic == frameMagic {
+			break
+		}
+		if magic>>8 != frameSkipMagic>>8 {
+			return ErrInvalid
+		}
+		skipSize, err := z.readUint32()
+		if err != nil {
+			return err
+		}
+		z.pos += 4
+		m, err := io.CopyN(ioutil.Discard, z.src, int64(skipSize))
+		if err != nil {
+			return err
+		}
+		z.pos += m
+	}
+
+	// Header.
+	if _, err := io.ReadFull(z.src, buf[:2]); err != nil {
+		return err
+	}
+	z.pos += 8
+
+	b := buf[0]
+	if v := b >> 6; v != Version {
+		return fmt.Errorf("lz4: invalid version: got %d; expected %d", v, Version)
+	}
+	if b>>5&1 == 0 {
+		return ErrBlockDependency
+	}
+	z.BlockChecksum = b>>4&1 > 0
+	frameSize := b>>3&1 > 0
+	z.NoChecksum = b>>2&1 == 0
+
+	bmsID := buf[1] >> 4 & 0x7
+	if bmsID < 4 || bmsID > 7 {
+		return fmt.Errorf("lz4: invalid block max size ID: %d", bmsID)
+	}
+	bSize := blockSizeIndexToValue(bmsID - 4)
+	z.BlockMaxSize = bSize
+
+	// Allocate the compressed/uncompressed buffers.
+	// The compressed buffer cannot exceed the uncompressed one.
+	if n := 2 * bSize; cap(z.zdata) < n {
+		z.zdata = make([]byte, n, n)
+	}
+	if debugFlag {
+		debug("header block max size id=%d size=%d", bmsID, bSize)
+	}
+	z.zdata = z.zdata[:bSize]
+	z.data = z.zdata[:cap(z.zdata)][bSize:]
+	z.idx = len(z.data)
+
+	_, _ = z.checksum.Write(buf[0:2])
+
+	if frameSize {
+		buf := buf[:8]
+		if _, err := io.ReadFull(z.src, buf); err != nil {
+			return err
+		}
+		z.Size = binary.LittleEndian.Uint64(buf)
+		z.pos += 8
+		_, _ = z.checksum.Write(buf)
+	}
+
+	// Header checksum.
+	if _, err := io.ReadFull(z.src, buf[:1]); err != nil {
+		return err
+	}
+	z.pos++
+	if h := byte(z.checksum.Sum32() >> 8 & 0xFF); h != buf[0] {
+		return fmt.Errorf("lz4: invalid header checksum: got %x; expected %x", buf[0], h)
+	}
+
+	z.Header.done = true
+	if debugFlag {
+		debug("header read: %v", z.Header)
+	}
+
+	return nil
+}
+
+// Read decompresses data from the underlying source into the supplied buffer.
+//
+// Since there can be multiple streams concatenated, Header values may
+// change between calls to Read(). If that is the case, no data is actually read from
+// the underlying io.Reader, to allow for potential input buffer resizing.
+func (z *Reader) Read(buf []byte) (int, error) {
+	if debugFlag {
+		debug("Read buf len=%d", len(buf))
+	}
+	if !z.Header.done {
+		if err := z.readHeader(true); err != nil {
+			return 0, err
+		}
+		if debugFlag {
+			debug("header read OK compressed buffer %d / %d uncompressed buffer %d : %d index=%d",
+				len(z.zdata), cap(z.zdata), len(z.data), cap(z.data), z.idx)
+		}
+	}
+
+	if len(buf) == 0 {
+		return 0, nil
+	}
+
+	if z.idx == len(z.data) {
+		// No data ready for reading, process the next block.
+		if debugFlag {
+			debug("reading block from writer")
+		}
+		// Reset uncompressed buffer
+		z.data = z.zdata[:cap(z.zdata)][len(z.zdata):]
+
+		// Block length: 0 = end of frame, highest bit set: uncompressed.
+		bLen, err := z.readUint32()
+		if err != nil {
+			return 0, err
+		}
+		z.pos += 4
+
+		if bLen == 0 {
+			// End of frame reached.
+			if !z.NoChecksum {
+				// Validate the frame checksum.
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				if debugFlag {
+					debug("frame checksum got=%x / want=%x", z.checksum.Sum32(), checksum)
+				}
+				z.pos += 4
+				if h := z.checksum.Sum32(); checksum != h {
+					return 0, fmt.Errorf("lz4: invalid frame checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			// Get ready for the next concatenated frame and keep the position.
+			pos := z.pos
+			z.Reset(z.src)
+			z.pos = pos
+
+			// Since multiple frames can be concatenated, check for more.
+			return 0, z.readHeader(false)
+		}
+
+		if debugFlag {
+			debug("raw block size %d", bLen)
+		}
+		if bLen&compressedBlockFlag > 0 {
+			// Uncompressed block.
+			bLen &= compressedBlockMask
+			if debugFlag {
+				debug("uncompressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			z.data = z.data[:bLen]
+			if _, err := io.ReadFull(z.src, z.data); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+			if z.OnBlockDone != nil {
+				z.OnBlockDone(int(bLen))
+			}
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(z.data); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+		} else {
+			// Compressed block.
+			if debugFlag {
+				debug("compressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			zdata := z.zdata[:bLen]
+			if _, err := io.ReadFull(z.src, zdata); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(zdata); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			n, err := UncompressBlock(zdata, z.data)
+			if err != nil {
+				return 0, err
+			}
+			z.data = z.data[:n]
+			if z.OnBlockDone != nil {
+				z.OnBlockDone(n)
+			}
+		}
+
+		if !z.NoChecksum {
+			_, _ = z.checksum.Write(z.data)
+			if debugFlag {
+				debug("current frame checksum %x", z.checksum.Sum32())
+			}
+		}
+		z.idx = 0
+	}
+
+	if z.skip > int64(len(z.data[z.idx:])) {
+		z.skip -= int64(len(z.data[z.idx:]))
+		z.dpos += int64(len(z.data[z.idx:]))
+		z.idx = len(z.data)
+		return 0, nil
+	}
+
+	z.idx += int(z.skip)
+	z.dpos += z.skip
+	z.skip = 0
+
+	n := copy(buf, z.data[z.idx:])
+	z.idx += n
+	z.dpos += int64(n)
+	if debugFlag {
+		debug("copied %d bytes to input", n)
+	}
+
+	return n, nil
+}
+
+// Seek implements io.Seeker, but supports seeking forward from the current
+// position only. Any other seek will return an error. Allows skipping output
+// bytes which aren't needed, which in some scenarios is faster than reading
+// and discarding them.
+// Note this may cause future calls to Read() to read 0 bytes if all of the
+// data they would have returned is skipped.
+func (z *Reader) Seek(offset int64, whence int) (int64, error) {
+	if offset < 0 || whence != io.SeekCurrent {
+		return z.dpos + z.skip, ErrUnsupportedSeek
+	}
+	z.skip += offset
+	return z.dpos + z.skip, nil
+}
+
+// Reset discards the Reader's state and makes it equivalent to the
+// result of its original state from NewReader, but reading from r instead.
+// This permits reusing a Reader rather than allocating a new one.
+func (z *Reader) Reset(r io.Reader) {
+	z.Header = Header{}
+	z.pos = 0
+	z.src = r
+	z.zdata = z.zdata[:0]
+	z.data = z.data[:0]
+	z.idx = 0
+	z.checksum.Reset()
+}
+
+// readUint32 reads an uint32 into the supplied buffer.
+// The idea is to make use of the already allocated buffers avoiding additional allocations.
+func (z *Reader) readUint32() (uint32, error) {
+	buf := z.buf[:4]
+	_, err := io.ReadFull(z.src, buf)
+	x := binary.LittleEndian.Uint32(buf)
+	return x, err
+}

vendor/github.com/pierrec/lz4/writer.go

@@ -0,0 +1,408 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"fmt"
+	"github.com/pierrec/lz4/internal/xxh32"
+	"io"
+	"runtime"
+)
+
+// zResult contains the results of compressing a block.
+type zResult struct {
+	size     uint32 // Block header
+	data     []byte // Compressed data
+	checksum uint32 // Data checksum
+}
+
+// Writer implements the LZ4 frame encoder.
+type Writer struct {
+	Header
+	// Handler called when a block has been successfully written out.
+	// It provides the number of bytes written.
+	OnBlockDone func(size int)
+
+	buf       [19]byte      // magic number(4) + header(flags(2)+[Size(8)+DictID(4)]+checksum(1)) does not exceed 19 bytes
+	dst       io.Writer     // Destination.
+	checksum  xxh32.XXHZero // Frame checksum.
+	data      []byte        // Data to be compressed + buffer for compressed data.
+	idx       int           // Index into data.
+	hashtable [winSize]int  // Hash table used in CompressBlock().
+
+	// For concurrency.
+	c   chan chan zResult // Channel for block compression goroutines and writer goroutine.
+	err error             // Any error encountered while writing to the underlying destination.
+}
+
+// NewWriter returns a new LZ4 frame encoder.
+// No access to the underlying io.Writer is performed.
+// The supplied Header is checked at the first Write.
+// It is ok to change it before the first Write but then not until a Reset() is performed.
+func NewWriter(dst io.Writer) *Writer {
+	z := new(Writer)
+	z.Reset(dst)
+	return z
+}
+
+// WithConcurrency sets the number of concurrent go routines used for compression.
+// A negative value sets the concurrency to GOMAXPROCS.
+func (z *Writer) WithConcurrency(n int) *Writer {
+	switch {
+	case n == 0 || n == 1:
+		z.c = nil
+		return z
+	case n < 0:
+		n = runtime.GOMAXPROCS(0)
+	}
+	z.c = make(chan chan zResult, n)
+	// Writer goroutine managing concurrent block compression goroutines.
+	go func() {
+		// Process next block compression item.
+		for c := range z.c {
+			// Read the next compressed block result.
+			// Waiting here ensures that the blocks are output in the order they were sent.
+			// The incoming channel is always closed as it indicates to the caller that
+			// the block has been processed.
+			res := <-c
+			n := len(res.data)
+			if n == 0 {
+				// Notify the block compression routine that we are done with its result.
+				// This is used when a sentinel block is sent to terminate the compression.
+				close(c)
+				return
+			}
+			// Write the block.
+			if err := z.writeUint32(res.size); err != nil && z.err == nil {
+				z.err = err
+			}
+			if _, err := z.dst.Write(res.data); err != nil && z.err == nil {
+				z.err = err
+			}
+			if z.BlockChecksum {
+				if err := z.writeUint32(res.checksum); err != nil && z.err == nil {
+					z.err = err
+				}
+			}
+			if isCompressed := res.size&compressedBlockFlag == 0; isCompressed {
+				// It is now safe to release the buffer as no longer in use by any goroutine.
+				putBuffer(cap(res.data), res.data)
+			}
+			if h := z.OnBlockDone; h != nil {
+				h(n)
+			}
+			close(c)
+		}
+	}()
+	return z
+}
+
+// newBuffers instantiates new buffers which size matches the one in Header.
+// The returned buffers are for decompression and compression respectively.
+func (z *Writer) newBuffers() {
+	bSize := z.Header.BlockMaxSize
+	buf := getBuffer(bSize)
+	z.data = buf[:bSize] // Uncompressed buffer is the first half.
+}
+
+// freeBuffers puts the writer's buffers back to the pool.
+func (z *Writer) freeBuffers() {
+	// Put the buffer back into the pool, if any.
+	putBuffer(z.Header.BlockMaxSize, z.data)
+	z.data = nil
+}
+
+// writeHeader builds and writes the header (magic+header) to the underlying io.Writer.
+func (z *Writer) writeHeader() error {
+	// Default to 4Mb if BlockMaxSize is not set.
+	if z.Header.BlockMaxSize == 0 {
+		z.Header.BlockMaxSize = blockSize4M
+	}
+	// The only option that needs to be validated.
+	bSize := z.Header.BlockMaxSize
+	if !isValidBlockSize(z.Header.BlockMaxSize) {
+		return fmt.Errorf("lz4: invalid block max size: %d", bSize)
+	}
+	// Allocate the compressed/uncompressed buffers.
+	// The compressed buffer cannot exceed the uncompressed one.
+	z.newBuffers()
+	z.idx = 0
+
+	// Size is optional.
+	buf := z.buf[:]
+
+	// Set the fixed size data: magic number, block max size and flags.
+	binary.LittleEndian.PutUint32(buf[0:], frameMagic)
+	flg := byte(Version << 6)
+	flg |= 1 << 5 // No block dependency.
+	if z.Header.BlockChecksum {
+		flg |= 1 << 4
+	}
+	if z.Header.Size > 0 {
+		flg |= 1 << 3
+	}
+	if !z.Header.NoChecksum {
+		flg |= 1 << 2
+	}
+	buf[4] = flg
+	buf[5] = blockSizeValueToIndex(z.Header.BlockMaxSize) << 4
+
+	// Current buffer size: magic(4) + flags(1) + block max size (1).
+	n := 6
+	// Optional items.
+	if z.Header.Size > 0 {
+		binary.LittleEndian.PutUint64(buf[n:], z.Header.Size)
+		n += 8
+	}
+
+	// The header checksum includes the flags, block max size and optional Size.
+	buf[n] = byte(xxh32.ChecksumZero(buf[4:n]) >> 8 & 0xFF)
+	z.checksum.Reset()
+
+	// Header ready, write it out.
+	if _, err := z.dst.Write(buf[0 : n+1]); err != nil {
+		return err
+	}
+	z.Header.done = true
+	if debugFlag {
+		debug("wrote header %v", z.Header)
+	}
+
+	return nil
+}
+
+// Write compresses data from the supplied buffer into the underlying io.Writer.
+// Write does not return until the data has been written.
+func (z *Writer) Write(buf []byte) (int, error) {
+	if !z.Header.done {
+		if err := z.writeHeader(); err != nil {
+			return 0, err
+		}
+	}
+	if debugFlag {
+		debug("input buffer len=%d index=%d", len(buf), z.idx)
+	}
+
+	zn := len(z.data)
+	var n int
+	for len(buf) > 0 {
+		if z.idx == 0 && len(buf) >= zn {
+			// Avoid a copy as there is enough data for a block.
+			if err := z.compressBlock(buf[:zn]); err != nil {
+				return n, err
+			}
+			n += zn
+			buf = buf[zn:]
+			continue
+		}
+		// Accumulate the data to be compressed.
+		m := copy(z.data[z.idx:], buf)
+		n += m
+		z.idx += m
+		buf = buf[m:]
+		if debugFlag {
+			debug("%d bytes copied to buf, current index %d", n, z.idx)
+		}
+
+		if z.idx < len(z.data) {
+			// Buffer not filled.
+			if debugFlag {
+				debug("need more data for compression")
+			}
+			return n, nil
+		}
+
+		// Buffer full.
+		if err := z.compressBlock(z.data); err != nil {
+			return n, err
+		}
+		z.idx = 0
+	}
+
+	return n, nil
+}
+
+// compressBlock compresses a block.
+func (z *Writer) compressBlock(data []byte) error {
+	if !z.NoChecksum {
+		_, _ = z.checksum.Write(data)
+	}
+
+	if z.c != nil {
+		c := make(chan zResult)
+		z.c <- c // Send now to guarantee order
+		go writerCompressBlock(c, z.Header, data)
+		return nil
+	}
+
+	zdata := z.data[z.Header.BlockMaxSize:cap(z.data)]
+	// The compressed block size cannot exceed the input's.
+	var zn int
+
+	if level := z.Header.CompressionLevel; level != 0 {
+		zn, _ = CompressBlockHC(data, zdata, level)
+	} else {
+		zn, _ = CompressBlock(data, zdata, z.hashtable[:])
+	}
+
+	var bLen uint32
+	if debugFlag {
+		debug("block compression %d => %d", len(data), zn)
+	}
+	if zn > 0 && zn < len(data) {
+		// Compressible and compressed size smaller than uncompressed: ok!
+		bLen = uint32(zn)
+		zdata = zdata[:zn]
+	} else {
+		// Uncompressed block.
+		bLen = uint32(len(data)) | compressedBlockFlag
+		zdata = data
+	}
+	if debugFlag {
+		debug("block compression to be written len=%d data len=%d", bLen, len(zdata))
+	}
+
+	// Write the block.
+	if err := z.writeUint32(bLen); err != nil {
+		return err
+	}
+	written, err := z.dst.Write(zdata)
+	if err != nil {
+		return err
+	}
+	if h := z.OnBlockDone; h != nil {
+		h(written)
+	}
+
+	if !z.BlockChecksum {
+		if debugFlag {
+			debug("current frame checksum %x", z.checksum.Sum32())
+		}
+		return nil
+	}
+	checksum := xxh32.ChecksumZero(zdata)
+	if debugFlag {
+		debug("block checksum %x", checksum)
+		defer func() { debug("current frame checksum %x", z.checksum.Sum32()) }()
+	}
+	return z.writeUint32(checksum)
+}
+
+// Flush flushes any pending compressed data to the underlying writer.
+// Flush does not return until the data has been written.
+// If the underlying writer returns an error, Flush returns that error.
+func (z *Writer) Flush() error {
+	if debugFlag {
+		debug("flush with index %d", z.idx)
+	}
+	if z.idx == 0 {
+		return nil
+	}
+
+	data := z.data[:z.idx]
+	z.idx = 0
+	if z.c == nil {
+		return z.compressBlock(data)
+	}
+	if !z.NoChecksum {
+		_, _ = z.checksum.Write(data)
+	}
+	c := make(chan zResult)
+	z.c <- c
+	writerCompressBlock(c, z.Header, data)
+	return nil
+}
+
+func (z *Writer) close() error {
+	if z.c == nil {
+		return nil
+	}
+	// Send a sentinel block (no data to compress) to terminate the writer main goroutine.
+	c := make(chan zResult)
+	z.c <- c
+	c <- zResult{}
+	// Wait for the main goroutine to complete.
+	<-c
+	// At this point the main goroutine has shut down or is about to return.
+	z.c = nil
+	return z.err
+}
+
+// Close closes the Writer, flushing any unwritten data to the underlying io.Writer, but does not close the underlying io.Writer.
+func (z *Writer) Close() error {
+	if !z.Header.done {
+		if err := z.writeHeader(); err != nil {
+			return err
+		}
+	}
+	if err := z.Flush(); err != nil {
+		return err
+	}
+	if err := z.close(); err != nil {
+		return err
+	}
+	z.freeBuffers()
+
+	if debugFlag {
+		debug("writing last empty block")
+	}
+	if err := z.writeUint32(0); err != nil {
+		return err
+	}
+	if z.NoChecksum {
+		return nil
+	}
+	checksum := z.checksum.Sum32()
+	if debugFlag {
+		debug("stream checksum %x", checksum)
+	}
+	return z.writeUint32(checksum)
+}
+
+// Reset clears the state of the Writer z such that it is equivalent to its
+// initial state from NewWriter, but instead writing to w.
+// No access to the underlying io.Writer is performed.
+func (z *Writer) Reset(w io.Writer) {
+	n := cap(z.c)
+	_ = z.close()
+	z.freeBuffers()
+	z.Header.Reset()
+	z.dst = w
+	z.checksum.Reset()
+	z.idx = 0
+	z.err = nil
+	z.WithConcurrency(n)
+}
+
+// writeUint32 writes a uint32 to the underlying writer.
+func (z *Writer) writeUint32(x uint32) error {
+	buf := z.buf[:4]
+	binary.LittleEndian.PutUint32(buf, x)
+	_, err := z.dst.Write(buf)
+	return err
+}
+
+// writerCompressBlock compresses data into a pooled buffer and writes its result
+// out to the input channel.
+func writerCompressBlock(c chan zResult, header Header, data []byte) {
+	zdata := getBuffer(header.BlockMaxSize)
+	// The compressed block size cannot exceed the input's.
+	var zn int
+	if level := header.CompressionLevel; level != 0 {
+		zn, _ = CompressBlockHC(data, zdata, level)
+	} else {
+		var hashTable [winSize]int
+		zn, _ = CompressBlock(data, zdata, hashTable[:])
+	}
+	var res zResult
+	if zn > 0 && zn < len(data) {
+		res.size = uint32(zn)
+		res.data = zdata[:zn]
+	} else {
+		res.size = uint32(len(data)) | compressedBlockFlag
+		res.data = data
+	}
+	if header.BlockChecksum {
+		res.checksum = xxh32.ChecksumZero(res.data)
+	}
+	c <- res
+}

vendor/modules.txt

@@ -532,7 +532,7 @@ github.com/google/cadvisor/utils/sysinfo
 github.com/google/cadvisor/version
 github.com/google/cadvisor/watcher
 github.com/google/cadvisor/zfs
-# github.com/google/go-cmp v0.3.1
+# github.com/google/go-cmp v0.5.2
 github.com/google/go-cmp/cmp
 github.com/google/go-cmp/cmp/internal/diff
 github.com/google/go-cmp/cmp/internal/flags
@@ -702,6 +702,9 @@ github.com/opencontainers/selinux/go-selinux/label
 github.com/opencontainers/selinux/pkg/pwalk
 # github.com/peterbourgon/diskv v2.0.1+incompatible
 github.com/peterbourgon/diskv
+# github.com/pierrec/lz4 v2.5.2+incompatible
+github.com/pierrec/lz4
+github.com/pierrec/lz4/internal/xxh32
 # github.com/pkg/errors v0.9.1
 github.com/pkg/errors
 # github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021