Integrate BACKBEAT SDK and resolve KACHING license validation

Major integrations and fixes:
- Added BACKBEAT SDK integration for P2P operation timing
- Implemented beat-aware status tracking for distributed operations
- Added Docker secrets support for secure license management
- Resolved KACHING license validation via HTTPS/TLS
- Updated docker-compose configuration for clean stack deployment
- Disabled rollback policies to prevent deployment failures
- Added license credential storage (CHORUS-DEV-MULTI-001)

Technical improvements:
- BACKBEAT P2P operation tracking with phase management
- Enhanced configuration system with file-based secrets
- Improved error handling for license validation
- Clean separation of KACHING and CHORUS deployment stacks

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

vendor/github.com/polydawn/refmt/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Eric Myhre
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/polydawn/refmt/cbor/cborCommon.go

@@ -0,0 +1,44 @@
+package cbor
+
+// "Major types" enum, as per https://tools.ietf.org/html/rfc7049#section-2.1 .
+//
+// These numbers are the bottom of the range for that major type when encoded;
+// that is, ints can be between `cborMajorUint` (inclusive) and `cborMajorNegInt` (exclusive).
+// Zero out the 0x1f bitrange of a byte to see which major type it is (those bits are
+// used for packing either length info or other specific enumerated meanings).
+const (
+	cborMajorUint   byte = 0x00
+	cborMajorNegInt      = 0x20
+	cborMajorBytes       = 0x40
+	cborMajorString      = 0x60
+	cborMajorArray       = 0x80
+	cborMajorMap         = 0xa0
+	cborMajorTag         = 0xc0
+	cborMajorSimple      = 0xe0 // Floating point, "simple" types like bool, etc, are above.
+)
+
+// Enumeration of some values with single fixed-byte representations.
+// All of these are in the "simple" space.
+// See https://tools.ietf.org/html/rfc7049#section-2.3 for tables.
+// The prefix indicating a float is also packed into the simple space.
+const (
+	cborSigilFalse     byte = 0xf4
+	cborSigilTrue           = 0xf5
+	cborSigilNil            = 0xf6
+	cborSigilUndefined      = 0xf7
+	cborSigilFloat16        = 0xf9
+	cborSigilFloat32        = 0xfA
+	cborSigilFloat64        = 0xfB
+)
+
+// The highest value in the range for bytes, text, arrays, and maps all indicate
+// an "indefinite length" / "streaming" entry coming up.  These have a different parse path.
+// The special 'break' value from the "simple" space (all bits on)
+// indicates termination of stream for all four kinds major types in this mode.
+const (
+	cborSigilIndefiniteBytes  byte = 0x5f
+	cborSigilIndefiniteString      = 0x7f
+	cborSigilIndefiniteArray       = 0x9f
+	cborSigilIndefiniteMap         = 0xbf
+	cborSigilBreak                 = 0xff
+)

vendor/github.com/polydawn/refmt/cbor/cborDecoder.go

@@ -0,0 +1,309 @@
+package cbor
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/polydawn/refmt/shared"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type Decoder struct {
+	cfg DecodeOptions
+	r   shared.SlickReader
+
+	stack []decoderPhase // When empty, and step returns done, all done.
+	phase decoderPhase   // Shortcut to end of stack.
+	left  []int          // Statekeeping space for definite-len map and array.
+}
+
+type decoderPhase uint8
+
+const (
+	decoderPhase_acceptValue decoderPhase = iota
+	decoderPhase_acceptArrValueOrBreak
+	decoderPhase_acceptMapIndefKey
+	decoderPhase_acceptMapIndefValueOrBreak
+	decoderPhase_acceptArrValue
+	decoderPhase_acceptMapKey
+	decoderPhase_acceptMapValue
+)
+
+func NewDecoder(cfg DecodeOptions, r io.Reader) (d *Decoder) {
+	d = &Decoder{
+		cfg:   cfg,
+		r:     shared.NewReader(r),
+		stack: make([]decoderPhase, 0, 10),
+		left:  make([]int, 0, 10),
+	}
+	d.phase = decoderPhase_acceptValue
+	return
+}
+
+func (d *Decoder) Reset() {
+	d.stack = d.stack[0:0]
+	d.phase = decoderPhase_acceptValue
+	d.left = d.left[0:0]
+}
+
+type decoderStep func(tokenSlot *Token) (done bool, err error)
+
+func (d *Decoder) Step(tokenSlot *Token) (done bool, err error) {
+	switch d.phase {
+	case decoderPhase_acceptValue:
+		done, err = d.step_acceptValue(tokenSlot)
+	case decoderPhase_acceptArrValueOrBreak:
+		done, err = d.step_acceptArrValueOrBreak(tokenSlot)
+	case decoderPhase_acceptMapIndefKey:
+		done, err = d.step_acceptMapIndefKey(tokenSlot)
+	case decoderPhase_acceptMapIndefValueOrBreak:
+		done, err = d.step_acceptMapIndefValueOrBreak(tokenSlot)
+	case decoderPhase_acceptArrValue:
+		done, err = d.step_acceptArrValue(tokenSlot)
+	case decoderPhase_acceptMapKey:
+		done, err = d.step_acceptMapKey(tokenSlot)
+	case decoderPhase_acceptMapValue:
+		done, err = d.step_acceptMapValue(tokenSlot)
+	}
+	// If the step errored: out, entirely.
+	if err != nil {
+		return true, err
+	}
+	// If the step wasn't done, return same status.
+	if !done {
+		return false, nil
+	}
+	// If it WAS done, pop next, or if stack empty, we're entirely done.
+	nSteps := len(d.stack) - 1
+	if nSteps <= 0 {
+		return true, nil // that's all folks
+	}
+	d.phase = d.stack[nSteps]
+	d.stack = d.stack[0:nSteps]
+	return false, nil
+}
+
+func (d *Decoder) pushPhase(newPhase decoderPhase) {
+	d.stack = append(d.stack, d.phase)
+	d.phase = newPhase
+}
+
+// The original step, where any value is accepted, and no terminators for composites are valid.
+// ONLY used in the original step; all other steps handle leaf nodes internally.
+func (d *Decoder) step_acceptValue(tokenSlot *Token) (done bool, err error) {
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	tokenSlot.Tagged = false
+	return d.stepHelper_acceptValue(majorByte, tokenSlot)
+}
+
+// Step in midst of decoding an indefinite-length array.
+func (d *Decoder) step_acceptArrValueOrBreak(tokenSlot *Token) (done bool, err error) {
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	tokenSlot.Tagged = false
+	switch majorByte {
+	case cborSigilBreak:
+		tokenSlot.Type = TArrClose
+		return true, nil
+	default:
+		_, err := d.stepHelper_acceptValue(majorByte, tokenSlot)
+		return false, err
+	}
+}
+
+// Step in midst of decoding an indefinite-length map, key expected up next, or end.
+func (d *Decoder) step_acceptMapIndefKey(tokenSlot *Token) (done bool, err error) {
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	tokenSlot.Tagged = false
+	switch majorByte {
+	case cborSigilBreak:
+		tokenSlot.Type = TMapClose
+		return true, nil
+	default:
+		d.phase = decoderPhase_acceptMapIndefValueOrBreak
+		_, err := d.stepHelper_acceptValue(majorByte, tokenSlot) // FIXME surely not *any* value?  not composites, at least?
+		return false, err
+	}
+}
+
+// Step in midst of decoding an indefinite-length map, value expected up next.
+func (d *Decoder) step_acceptMapIndefValueOrBreak(tokenSlot *Token) (done bool, err error) {
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	tokenSlot.Tagged = false
+	switch majorByte {
+	case cborSigilBreak:
+		return true, fmt.Errorf("unexpected break; expected value in indefinite-length map")
+	default:
+		d.phase = decoderPhase_acceptMapIndefKey
+		_, err = d.stepHelper_acceptValue(majorByte, tokenSlot)
+		return false, err
+	}
+}
+
+// Step in midst of decoding a definite-length array.
+func (d *Decoder) step_acceptArrValue(tokenSlot *Token) (done bool, err error) {
+	// Yield close token, pop state, and return done flag if expecting no more entries.
+	ll := len(d.left) - 1
+	if d.left[ll] == 0 {
+		d.left = d.left[0:ll]
+		tokenSlot.Type = TArrClose
+		return true, nil
+	}
+	d.left[ll]--
+	// Read next value.
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	tokenSlot.Tagged = false
+	_, err = d.stepHelper_acceptValue(majorByte, tokenSlot)
+	return false, err
+}
+
+// Step in midst of decoding an definite-length map, key expected up next.
+func (d *Decoder) step_acceptMapKey(tokenSlot *Token) (done bool, err error) {
+	// Yield close token, pop state, and return done flag if expecting no more entries.
+	ll := len(d.left) - 1
+	if d.left[ll] == 0 {
+		d.left = d.left[0:ll]
+		tokenSlot.Type = TMapClose
+		return true, nil
+	}
+	d.left[ll]--
+	// Read next key.
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	d.phase = decoderPhase_acceptMapValue
+	tokenSlot.Tagged = false
+	_, err = d.stepHelper_acceptValue(majorByte, tokenSlot) // FIXME surely not *any* value?  not composites, at least?
+	return false, err
+}
+
+// Step in midst of decoding an definite-length map, value expected up next.
+func (d *Decoder) step_acceptMapValue(tokenSlot *Token) (done bool, err error) {
+	// Read next value.
+	majorByte, err := d.r.Readn1()
+	if err != nil {
+		return true, err
+	}
+	d.phase = decoderPhase_acceptMapKey
+	tokenSlot.Tagged = false
+	_, err = d.stepHelper_acceptValue(majorByte, tokenSlot)
+	return false, err
+}
+
+func (d *Decoder) stepHelper_acceptValue(majorByte byte, tokenSlot *Token) (done bool, err error) {
+	switch majorByte {
+	case cborSigilNil:
+		tokenSlot.Type = TNull
+		return true, nil
+	case cborSigilUndefined:
+		if d.cfg.CoerceUndefToNull {
+			tokenSlot.Type = TNull
+			return true, nil
+		}
+		return true, fmt.Errorf("encountered cbor 'undefined' byte (%x) during decoding", cborSigilUndefined)
+	case cborSigilFalse:
+		tokenSlot.Type = TBool
+		tokenSlot.Bool = false
+		return true, nil
+	case cborSigilTrue:
+		tokenSlot.Type = TBool
+		tokenSlot.Bool = true
+		return true, nil
+	case cborSigilFloat16, cborSigilFloat32, cborSigilFloat64:
+		tokenSlot.Type = TFloat64
+		tokenSlot.Float64, err = d.decodeFloat(majorByte)
+		return true, err
+	case cborSigilIndefiniteBytes:
+		tokenSlot.Type = TBytes
+		tokenSlot.Bytes, err = d.decodeBytesIndefinite(nil)
+		return true, err
+	case cborSigilIndefiniteString:
+		tokenSlot.Type = TString
+		tokenSlot.Str, err = d.decodeStringIndefinite()
+		return true, err
+	case cborSigilIndefiniteArray:
+		tokenSlot.Type = TArrOpen
+		tokenSlot.Length = -1
+		d.pushPhase(decoderPhase_acceptArrValueOrBreak)
+		return false, nil
+	case cborSigilIndefiniteMap:
+		tokenSlot.Type = TMapOpen
+		tokenSlot.Length = -1
+		d.pushPhase(decoderPhase_acceptMapIndefKey)
+		return false, nil
+	default:
+		switch {
+		case majorByte >= cborMajorUint && majorByte < cborMajorNegInt:
+			tokenSlot.Type = TUint
+			tokenSlot.Uint, err = d.decodeUint(majorByte)
+			return true, err
+		case majorByte >= cborMajorNegInt && majorByte < cborMajorBytes:
+			tokenSlot.Type = TInt
+			tokenSlot.Int, err = d.decodeNegInt(majorByte)
+			return true, err
+		case majorByte >= cborMajorBytes && majorByte < cborMajorString:
+			tokenSlot.Type = TBytes
+			tokenSlot.Bytes, err = d.decodeBytes(majorByte)
+			return true, err
+		case majorByte >= cborMajorString && majorByte < cborMajorArray:
+			tokenSlot.Type = TString
+			tokenSlot.Str, err = d.decodeString(majorByte)
+			return true, err
+		case majorByte >= cborMajorArray && majorByte < cborMajorMap:
+			var n int
+			n, err = d.decodeLen(majorByte)
+			tokenSlot.Type = TArrOpen
+			tokenSlot.Length = n
+			d.left = append(d.left, n)
+			d.pushPhase(decoderPhase_acceptArrValue)
+			return false, err
+		case majorByte >= cborMajorMap && majorByte < cborMajorTag:
+			var n int
+			n, err = d.decodeLen(majorByte)
+			tokenSlot.Type = TMapOpen
+			tokenSlot.Length = n
+			d.left = append(d.left, n)
+			d.pushPhase(decoderPhase_acceptMapKey)
+			return false, err
+		case majorByte >= cborMajorTag && majorByte < cborMajorSimple:
+			// CBOR tags are, frankly, bonkers, and should not be used.
+			// They break isomorphism to basic standards like JSON.
+			// We'll parse basic integer tag values -- SINGLE layer only.
+			// We will NOT parse the full gamut of recursive tags: doing so
+			// would mean allowing an unbounded number of allocs *during
+			// *processing of a single token*, which is _not reasonable_.
+			if tokenSlot.Tagged {
+				return true, fmt.Errorf("unsupported multiple tags on a single data item")
+			}
+			tokenSlot.Tagged = true
+			tokenSlot.Tag, err = d.decodeLen(majorByte)
+			if err != nil {
+				return true, err
+			}
+			// Okay, we slurped a tag.
+			// Read next value.
+			majorByte, err := d.r.Readn1()
+			if err != nil {
+				return true, err
+			}
+			return d.stepHelper_acceptValue(majorByte, tokenSlot)
+		default:
+			return true, fmt.Errorf("Invalid majorByte: 0x%x", majorByte)
+		}
+	}
+}

vendor/github.com/polydawn/refmt/cbor/cborDecoderTerminals.go

@@ -0,0 +1,235 @@
+package cbor
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+)
+
+const (
+	maxUint = ^uint(0)
+	maxInt  = int(maxUint >> 1)
+)
+
+func (d *Decoder) decodeFloat(majorByte byte) (f float64, err error) {
+	var bs []byte
+	switch majorByte {
+	case cborSigilFloat16:
+		bs, err = d.r.Readnzc(2)
+		f = float64(math.Float32frombits(halfFloatToFloatBits(binary.BigEndian.Uint16(bs))))
+	case cborSigilFloat32:
+		bs, err = d.r.Readnzc(4)
+		f = float64(math.Float32frombits(binary.BigEndian.Uint32(bs)))
+	case cborSigilFloat64:
+		bs, err = d.r.Readnzc(8)
+		f = math.Float64frombits(binary.BigEndian.Uint64(bs))
+	}
+	return
+}
+
+// Decode an unsigned int.
+// Must continue to hand down the majorByte because some of its bits are either
+// packed with the value outright, or tell us how many more bytes the value fills.
+func (d *Decoder) decodeUint(majorByte byte) (ui uint64, err error) {
+	v := majorByte & 0x1f
+	if v <= 0x17 {
+		ui = uint64(v)
+	} else {
+		if v == 0x18 {
+			var b byte
+			b, err = d.r.Readn1()
+			ui = uint64(b)
+		} else if v == 0x19 {
+			var bs []byte
+			bs, err = d.r.Readnzc(2)
+			ui = uint64(binary.BigEndian.Uint16(bs))
+		} else if v == 0x1a {
+			var bs []byte
+			bs, err = d.r.Readnzc(4)
+			ui = uint64(binary.BigEndian.Uint32(bs))
+		} else if v == 0x1b {
+			var bs []byte
+			bs, err = d.r.Readnzc(8)
+			ui = uint64(binary.BigEndian.Uint64(bs))
+		} else {
+			err = fmt.Errorf("decodeUint: Invalid descriptor: %v", majorByte)
+			return
+		}
+	}
+	return
+}
+
+// Decode a *negative* integer.
+// Note that CBOR has a very funny-shaped hole here: there is unsigned positive int,
+// and there is explicitly negative signed int... and there is no signed, positive int.
+// *We have no 'decodeInt' function because that **doesn't exist** in CBOR.*
+// So!  Hopefully our consumer doesn't mind having to cast uints to ints fairly frequently.
+func (d *Decoder) decodeNegInt(majorByte byte) (i int64, err error) {
+	// The packed bits in the majorByte and the following bytes if any are layed out
+	// the exact same as a uint; only the major type bits are different.
+	ui, err := d.decodeUint(majorByte)
+	if err != nil {
+		return 0, err
+	}
+	pos := ui + 1
+	if pos > uint64(-math.MinInt64) {
+		return -1, errors.New("cbor: negative integer out of rage of int64 type")
+	}
+
+	return -int64(pos), nil
+}
+
+// Decode expecting a positive integer.
+// None of our token-yielding functions call this directly;
+// it's used inside the library when we expect to read e.g. a length header.
+// Does not check that your majorByte indicates an integer type at all;
+// in context, it often doesn't, e.g. when decoding the length of a string.
+func (d *Decoder) decodeLen(majorByte byte) (i int, err error) {
+	ui, err := d.decodeUint(majorByte)
+	if err != nil {
+		return 0, err
+	}
+	if ui > uint64(maxInt) {
+		return 0, errors.New("cbor: positive integer is out of length")
+	}
+	return int(ui), nil
+}
+
+// Decoding indefinite-length byte strings in cbor is actually decoding a sequence of
+// definite-length byte strings until you encounter a break.
+// Caller: use `bs[:0]` if you have something to reuse, or nil
+func (d *Decoder) decodeBytesIndefinite(bs []byte) (bsOut []byte, err error) {
+	return d.decodeBytesOrStringIndefinite(bs, cborMajorBytes)
+}
+
+func (d *Decoder) decodeStringIndefinite() (s string, err error) {
+	bs, err := d.decodeBytesOrStringIndefinite(nil, cborMajorString)
+	if err != nil {
+		return "", err
+	}
+	return string(bs), nil
+}
+
+func (d *Decoder) decodeBytesOrStringIndefinite(bs []byte, majorWanted byte) (bsOut []byte, err error) {
+	if bs == nil {
+		bs = make([]byte, 0, 16)
+	}
+	var n int
+	for {
+		// Read first byte; check for break, or hunk, or invalid.
+		// (It's not necessary to have the first majorByte as a param to this function, because
+		// indefinite length sequences have a separate sigil which doesn't pack any len info.)
+		majorByte, err := d.r.Readn1()
+		if err != nil {
+			return bs, err
+		}
+		if majorByte == cborSigilBreak {
+			return bs, nil
+		} else if major := majorByte | 0x1f - 0x1f; major != majorWanted {
+			return bs, fmt.Errorf("cbor: expect bytes or string major type in indefinite string/bytes; got: %v, byte: %v", major, majorByte)
+		}
+		// Read length header for this hunk, and ensure we have at least that much cap.
+		n, err = d.decodeLen(majorByte)
+		if err != nil {
+			return bs, err
+		}
+		oldLen := len(bs)
+		newLen := oldLen + n
+		if n > 33554432 {
+			return nil, fmt.Errorf("cbor: decoding rejected oversized indefinite string/bytes field: %d is too large", n)
+		}
+		if newLen > cap(bs) {
+			bs2 := make([]byte, newLen, 2*cap(bs)+n)
+			copy(bs2, bs)
+			bs = bs2
+		} else {
+			bs = bs[:newLen]
+		}
+		// Read that hunk.
+		d.r.Readb(bs[oldLen:newLen])
+	}
+}
+
+// Decode a single length-prefixed hunk of bytes.
+//
+// There are a number of ways this may try to conserve allocations:
+//
+// - If you say zerocopy=true, and the underlying reader system already has an
+//  appropriate byte slice available, then a slice from that will be returned.
+//
+// - If you provide a byte slice, we will attempt to use it.
+//  The byte slice is truncated and used for its capacity only -- not appended.
+//  The final returned slice may be a different one if the provided slice did not
+//  have sufficient capacity.
+//
+// - If you say zerocopy=true, and the underlying read system doesn't have an
+//  efficient way to yield a slice of its internal buffer, and you provided no
+//  destination slice, then we will use a recycleable piece of memory in the Decoder
+//  state and return a slice viewing into it.  For small values this will
+//  likely save an alloc.
+//
+// The above rules are resolved in this order; e.g. your byte slice is disregarded
+// if zerocopy=true and the underlying reader can do something even more efficient,
+// though there is also no harm to providing the slice argument.
+// Generally, set zerocopy if you know you're not going to publicly yield the results,
+// and the implementation will do its best to be as efficient as possible.
+//
+// Zerocopy is appropriate when planning to turn the bytes into a string, for example,
+// since in that path we know the slice will be treated immutably, not publicly
+// exposed, and also any other copy to another intermediate is definitely useless.
+func (d *Decoder) decodeBytes(majorByte byte) (bs []byte, err error) {
+	n, err := d.decodeLen(majorByte)
+	if err != nil {
+		return nil, err
+	}
+	if n > 33554432 {
+		return nil, fmt.Errorf("cbor: decoding rejected oversized byte field: %d is too large", n)
+	}
+	return d.r.Readn(n)
+}
+
+// Decode a single length-prefixed string.
+func (d *Decoder) decodeString(majorByte byte) (s string, err error) {
+	n, err := d.decodeLen(majorByte)
+	if err != nil {
+		return "", err
+	}
+	if n > 33554432 {
+		return "", fmt.Errorf("cbor: decoding rejected oversized string field: %d is too large", n)
+	}
+	bs, err := d.r.Readnzc(n)
+	return string(bs), err
+}
+
+// culled from OGRE (Object-Oriented Graphics Rendering Engine)
+// function: halfToFloatI (http://stderr.org/doc/ogre-doc/api/OgreBitwise_8h-source.html)
+func halfFloatToFloatBits(yy uint16) (d uint32) {
+	y := uint32(yy)
+	s := (y >> 15) & 0x01
+	e := (y >> 10) & 0x1f
+	m := y & 0x03ff
+
+	if e == 0 {
+		if m == 0 { // plu or minus 0
+			return s << 31
+		} else { // Denormalized number -- renormalize it
+			for (m & 0x00000400) == 0 {
+				m <<= 1
+				e -= 1
+			}
+			e += 1
+			const zz uint32 = 0x0400
+			m &= ^zz
+		}
+	} else if e == 31 {
+		if m == 0 { // Inf
+			return (s << 31) | 0x7f800000
+		} else { // NaN
+			return (s << 31) | 0x7f800000 | (m << 13)
+		}
+	}
+	e = e + (127 - 15)
+	m = m << 13
+	return (s << 31) | (e << 23) | m
+}

vendor/github.com/polydawn/refmt/cbor/cborEncoder.go

@@ -0,0 +1,276 @@
+package cbor
+
+import (
+	"io"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type Encoder struct {
+	w quickWriter
+
+	stack   []encoderPhase // When empty, and step returns done, all done.
+	current encoderPhase   // Shortcut to end of stack.
+	// Note unlike decoder, we need no statekeeping space for definite-len map and array.
+
+	spareBytes []byte
+}
+
+func NewEncoder(w io.Writer) (d *Encoder) {
+	d = &Encoder{
+		w:          newQuickWriterStream(w),
+		stack:      make([]encoderPhase, 0, 10),
+		current:    phase_anyExpectValue,
+		spareBytes: make([]byte, 8),
+	}
+	return
+}
+
+func (d *Encoder) Reset() {
+	d.stack = d.stack[0:0]
+	d.current = phase_anyExpectValue
+}
+
+type encoderPhase byte
+
+// There's about twice as many phases that the cbor encoder can be in compared to the json encoder
+// because the presense of indefinite vs definite length maps and arrays effectively adds a dimension to those.
+const (
+	phase_anyExpectValue           encoderPhase = iota
+	phase_mapDefExpectKeyOrEnd                  // must not yield break at end
+	phase_mapDefExpectValue                     // only necessary to flip back to DefExpectKey
+	phase_mapIndefExpectKeyOrEnd                // must yield break at end
+	phase_mapIndefExpectValue                   // only necessary to flip back to IndefExpectKey
+	phase_arrDefExpectValueOrEnd                // must not yield break at end
+	phase_arrIndefExpectValueOrEnd              // must yield break at end
+)
+
+func (d *Encoder) pushPhase(p encoderPhase) {
+	d.current = p
+	d.stack = append(d.stack, d.current)
+}
+
+// Pop a phase from the stack; return 'true' if stack now empty.
+func (d *Encoder) popPhase() bool {
+	n := len(d.stack) - 1
+	if n == 0 {
+		return true
+	}
+	if n < 0 { // the state machines are supposed to have already errored better
+		panic("cborEncoder stack overpopped")
+	}
+	d.current = d.stack[n-1]
+	d.stack = d.stack[0:n]
+	return false
+}
+
+func (d *Encoder) Step(tokenSlot *Token) (done bool, err error) {
+	/*
+		Though it reads somewhat backwards from how a human would probably intuit
+		cause and effect, switching on the token type we got first,
+		*then* switching for whether it is acceptable for our current phase... is by
+		far the shorter volume of code to write.
+	*/
+	phase := d.current
+	switch tokenSlot.Type {
+	case TMapOpen:
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			if tokenSlot.Length >= 0 {
+				d.pushPhase(phase_mapDefExpectKeyOrEnd)
+				d.emitMajorPlusLen(cborMajorMap, uint64(tokenSlot.Length))
+			} else {
+				d.pushPhase(phase_mapIndefExpectKeyOrEnd)
+				d.w.writen1(cborSigilIndefiniteMap)
+			}
+			return false, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TMapClose:
+		switch phase {
+		case phase_mapDefExpectKeyOrEnd:
+			return d.popPhase(), nil
+		case phase_mapIndefExpectKeyOrEnd:
+			d.w.writen1(cborSigilBreak)
+			return d.popPhase(), d.w.checkErr()
+		case phase_anyExpectValue, phase_mapDefExpectValue, phase_mapIndefExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForValue}
+		default:
+			panic("unreachable phase")
+		}
+	case TArrOpen:
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			if tokenSlot.Length >= 0 {
+				d.pushPhase(phase_arrDefExpectValueOrEnd)
+				d.emitMajorPlusLen(cborMajorArray, uint64(tokenSlot.Length))
+			} else {
+				d.pushPhase(phase_arrIndefExpectValueOrEnd)
+				d.w.writen1(cborSigilIndefiniteArray)
+			}
+			return false, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TArrClose:
+		switch phase {
+		case phase_arrDefExpectValueOrEnd:
+			return d.popPhase(), nil
+		case phase_arrIndefExpectValueOrEnd:
+			d.w.writen1(cborSigilBreak)
+			return d.popPhase(), d.w.checkErr()
+		case phase_anyExpectValue, phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForValue}
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TNull: // terminal value; not accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.w.writen1(cborSigilNil)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TString: // terminal value; YES, accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			goto emitStr
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			d.current += 1
+			goto emitStr
+		default:
+			panic("unreachable phase")
+		}
+	emitStr:
+		{
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeString(tokenSlot.Str)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		}
+	case TBytes: // terminal value; not accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeBytes(tokenSlot.Bytes)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TBool: // terminal value; not accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeBool(tokenSlot.Bool)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	case TInt: // terminal value; YES, accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			goto emitInt
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			d.current += 1
+			goto emitInt
+		default:
+			panic("unreachable phase")
+		}
+	emitInt:
+		{
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeInt64(tokenSlot.Int)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		}
+	case TUint: // terminal value; YES, accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			goto emitUint
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			d.current += 1
+			goto emitUint
+		default:
+			panic("unreachable phase")
+		}
+	emitUint:
+		{
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeUint64(tokenSlot.Uint)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		}
+	case TFloat64: // terminal value; not accepted as map key.
+		switch phase {
+		case phase_mapDefExpectValue, phase_mapIndefExpectValue:
+			d.current -= 1
+			fallthrough
+		case phase_anyExpectValue, phase_arrDefExpectValueOrEnd, phase_arrIndefExpectValueOrEnd:
+			if tokenSlot.Tagged {
+				d.emitMajorPlusLen(cborMajorTag, uint64(tokenSlot.Tag))
+			}
+			d.encodeFloat64(tokenSlot.Float64)
+			return phase == phase_anyExpectValue, d.w.checkErr()
+		case phase_mapDefExpectKeyOrEnd, phase_mapIndefExpectKeyOrEnd:
+			return true, &ErrInvalidTokenStream{Got: *tokenSlot, Acceptable: tokenTypesForKey}
+		default:
+			panic("unreachable phase")
+		}
+	default:
+		panic("unhandled token type")
+	}
+}

vendor/github.com/polydawn/refmt/cbor/cborEncoderTerminals.go

@@ -0,0 +1,77 @@
+package cbor
+
+import (
+	"encoding/binary"
+	"math"
+)
+
+func (d *Encoder) emitLen(majorByte byte, length int) {
+	d.emitMajorPlusLen(majorByte, uint64(length))
+}
+
+func (d *Encoder) emitMajorPlusLen(majorByte byte, v uint64) {
+	if v <= 0x17 {
+		d.w.writen1(majorByte + byte(v))
+	} else if v <= math.MaxUint8 {
+		d.w.writen2(majorByte+0x18, uint8(v))
+	} else if v <= math.MaxUint16 {
+		d.w.writen1(majorByte + 0x19)
+		d.spareBytes = d.spareBytes[:2]
+		binary.BigEndian.PutUint16(d.spareBytes, uint16(v))
+		d.w.writeb(d.spareBytes)
+	} else if v <= math.MaxUint32 {
+		d.w.writen1(majorByte + 0x1a)
+		d.spareBytes = d.spareBytes[:4]
+		binary.BigEndian.PutUint32(d.spareBytes, uint32(v))
+		d.w.writeb(d.spareBytes)
+	} else { // if v <= math.MaxUint64 {
+		d.w.writen1(majorByte + 0x1b)
+		d.spareBytes = d.spareBytes[:8]
+		binary.BigEndian.PutUint64(d.spareBytes, v)
+		d.w.writeb(d.spareBytes)
+	}
+}
+
+func (d *Encoder) encodeNull() {
+	d.w.writen1(cborSigilNil)
+}
+
+func (d *Encoder) encodeString(s string) {
+	d.emitMajorPlusLen(cborMajorString, uint64(len(s)))
+	d.w.writestr(s)
+}
+
+func (d *Encoder) encodeBytes(bs []byte) {
+	d.emitMajorPlusLen(cborMajorBytes, uint64(len(bs)))
+	d.w.writeb(bs)
+}
+
+func (d *Encoder) encodeBool(b bool) {
+	if b {
+		d.w.writen1(cborSigilTrue)
+	} else {
+		d.w.writen1(cborSigilFalse)
+	}
+}
+
+func (d *Encoder) encodeInt64(v int64) {
+	if v >= 0 {
+		d.emitMajorPlusLen(cborMajorUint, uint64(v))
+	} else {
+		d.emitMajorPlusLen(cborMajorNegInt, uint64(-1-v))
+	}
+}
+
+func (d *Encoder) encodeUint64(v uint64) {
+	d.emitMajorPlusLen(cborMajorUint, v)
+}
+
+func (d *Encoder) encodeFloat64(v float64) {
+	// Can we pack it into 32?  No idea: float precision is fraught with peril.
+	// See https://play.golang.org/p/u9sN6x0kk6
+	// So we *only* emit the full 64-bit style.  The CBOR spec permits this.
+	d.w.writen1(cborSigilFloat64)
+	d.spareBytes = d.spareBytes[:8]
+	binary.BigEndian.PutUint64(d.spareBytes, math.Float64bits(v))
+	d.w.writeb(d.spareBytes)
+}

vendor/github.com/polydawn/refmt/cbor/cborHelpers.go

@@ -0,0 +1,106 @@
+package cbor
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/polydawn/refmt/obj"
+	"github.com/polydawn/refmt/obj/atlas"
+	"github.com/polydawn/refmt/shared"
+)
+
+// All of the methods in this file are exported,
+// and their names and type declarations are intended to be
+// identical to the naming and types of the golang stdlib
+// 'encoding/json' packages, with ONE EXCEPTION:
+// what stdlib calls "NewEncoder", we call "NewMarshaller";
+// what stdlib calls "NewDecoder", we call "NewUnmarshaller";
+// and similarly the types and methods are "Marshaller.Marshal"
+// and "Unmarshaller.Unmarshal".
+// You should be able to migrate with a sed script!
+//
+// (In refmt, the encoder/decoder systems are for token streams;
+// if you're talking about object mapping, we consistently
+// refer to that as marshalling/unmarshalling.)
+//
+// Most methods also have an "Atlased" variant,
+// which lets you specify advanced type mapping instructions.
+
+func Marshal(v interface{}) ([]byte, error) {
+	var buf bytes.Buffer
+	if err := NewMarshaller(&buf).Marshal(v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func MarshalAtlased(v interface{}, atl atlas.Atlas) ([]byte, error) {
+	var buf bytes.Buffer
+	if err := NewMarshallerAtlased(&buf, atl).Marshal(v); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+type Marshaller struct {
+	marshaller *obj.Marshaller
+	encoder    *Encoder
+	pump       shared.TokenPump
+}
+
+func (x *Marshaller) Marshal(v interface{}) error {
+	x.marshaller.Bind(v)
+	x.encoder.Reset()
+	return x.pump.Run()
+}
+
+func NewMarshaller(wr io.Writer) *Marshaller {
+	return NewMarshallerAtlased(wr, atlas.MustBuild())
+}
+
+func NewMarshallerAtlased(wr io.Writer, atl atlas.Atlas) *Marshaller {
+	x := &Marshaller{
+		marshaller: obj.NewMarshaller(atl),
+		encoder:    NewEncoder(wr),
+	}
+	x.pump = shared.TokenPump{
+		x.marshaller,
+		x.encoder,
+	}
+	return x
+}
+
+func Unmarshal(cfg DecodeOptions, data []byte, v interface{}) error {
+	return NewUnmarshaller(cfg, bytes.NewBuffer(data)).Unmarshal(v)
+}
+
+func UnmarshalAtlased(cfg DecodeOptions, data []byte, v interface{}, atl atlas.Atlas) error {
+	return NewUnmarshallerAtlased(cfg, bytes.NewBuffer(data), atl).Unmarshal(v)
+}
+
+type Unmarshaller struct {
+	unmarshaller *obj.Unmarshaller
+	decoder      *Decoder
+	pump         shared.TokenPump
+}
+
+func (x *Unmarshaller) Unmarshal(v interface{}) error {
+	x.unmarshaller.Bind(v)
+	x.decoder.Reset()
+	return x.pump.Run()
+}
+
+func NewUnmarshaller(cfg DecodeOptions, r io.Reader) *Unmarshaller {
+	return NewUnmarshallerAtlased(cfg, r, atlas.MustBuild())
+}
+func NewUnmarshallerAtlased(cfg DecodeOptions, r io.Reader, atl atlas.Atlas) *Unmarshaller {
+	x := &Unmarshaller{
+		unmarshaller: obj.NewUnmarshaller(atl),
+		decoder:      NewDecoder(cfg, r),
+	}
+	x.pump = shared.TokenPump{
+		x.decoder,
+		x.unmarshaller,
+	}
+	return x
+}

vendor/github.com/polydawn/refmt/cbor/cborOptions.go

@@ -0,0 +1,20 @@
+package cbor
+
+type EncodeOptions struct {
+	// there aren't a ton of options for cbor, but we still need this
+	// for use as a sigil for the top-level refmt methods to demux on.
+}
+
+// marker method -- you may use this type to instruct `refmt.Marshal`
+// what kind of encoder to use.
+func (EncodeOptions) IsEncodeOptions() {}
+
+type DecodeOptions struct {
+	CoerceUndefToNull bool
+
+	// future: options to validate canonical serial order
+}
+
+// marker method -- you may use this type to instruct `refmt.Marshal`
+// what kind of encoder to use.
+func (DecodeOptions) IsDecodeOptions() {}

vendor/github.com/polydawn/refmt/cbor/doc.go

@@ -0,0 +1,27 @@
+/*
+	Package implementing the CBOR -- Concise Binary Object Notation
+	-- http://cbor.io/ -- spec.
+
+	CBOR is more or less freely interchangable with json: it's schemaless,
+	and composed of similar map and array types.
+	However, CBOR is binary, length delimited -- and thus very fast to parse,
+	and can store binary data without expansion problems -- and also distinguishes
+	types like integer, string, float, and bytes all clearly from each other.
+
+	The `cbor.Marshal` and `cbor.Unmarshal` functions are the quickest way
+	to convert your Go objects to and from serial CBOR.
+
+	The `cbor.NewMarshaller` and `cbor.NewUmarshaller` functions give a little
+	more control.  If performance is important, prefer these; recycling
+	the marshaller instances will significantly cut down on memory allocations
+	and improve performance.
+
+	The `*Atlased` variants of constructors allow you set up marshalling with
+	an `refmt/obj/atlas.Atlas`, unlocking all of refmt's advanced features
+	and custom object mapping powertools.
+
+	The `cbor.Encoder` and `cbor.Decoder` types implement the low-level functionality
+	of converting serial CBOR byte streams into refmt Token streams.
+	Users don't usually need to use these directly.
+*/
+package cbor

vendor/github.com/polydawn/refmt/cbor/encodeWriter.go

@@ -0,0 +1,107 @@
+package cbor
+
+import (
+	"fmt"
+	"io"
+)
+
+var (
+	_ quickWriter = &quickWriterStream{}
+)
+
+// quickWriter is implements several methods that are specificly useful to the performance
+// needs of our encoders, and abstracts writing to a byte array or to an io.Writer.
+type quickWriter interface {
+	writeb([]byte)
+	writestr(string)
+	writen1(byte)
+	writen2(byte, byte)
+	checkErr() error
+	clearErr()
+}
+
+// Interface used to detect if efficient string writing is supported.
+// Same as in stdlib 'io' pkg; also not exported there, so we declare it again ourselves.
+type stringWriter interface {
+	WriteString(s string) (n int, err error)
+}
+
+// quickWriterStream is a quickWriter that routes bytes to an io.Writer.
+// While this implementation does use some internal buffers, it's still advisable
+// to use a buffered writer to avoid small operations for any external IO like disk or network.
+type quickWriterStream struct {
+	w  io.Writer
+	ws stringWriter // nil if not available
+
+	scratch  [2]byte
+	scratch1 []byte
+	scratch2 []byte
+	err      error
+}
+
+func newQuickWriterStream(w io.Writer) *quickWriterStream {
+	z := &quickWriterStream{w: w}
+	if ws, ok := w.(stringWriter); ok {
+		z.ws = ws
+	}
+	z.scratch1 = z.scratch[:1]
+	z.scratch2 = z.scratch[:2]
+	return z
+}
+
+func (z *quickWriterStream) writeb(bs []byte) {
+	n, err := z.w.Write(bs)
+	if err != nil && z.err == nil {
+		z.err = err
+	}
+	if n < len(bs) && z.err == nil {
+		z.err = fmt.Errorf("underwrite")
+	}
+}
+
+func (z *quickWriterStream) writestr(s string) {
+	var n int
+	var err error
+	if z.ws != nil {
+		n, err = z.ws.WriteString(s)
+	} else {
+		n, err = z.w.Write([]byte(s)) // Notice: alloc!
+	}
+	if err != nil && z.err == nil {
+		z.err = err
+	}
+	if n < len(s) && z.err == nil {
+		z.err = fmt.Errorf("underwrite")
+	}
+}
+
+func (z *quickWriterStream) writen1(b byte) {
+	z.scratch1[0] = b
+	n, err := z.w.Write(z.scratch1)
+	if err != nil && z.err == nil {
+		z.err = err
+	}
+	if n < 1 && z.err == nil {
+		z.err = fmt.Errorf("underwrite")
+	}
+}
+
+func (z *quickWriterStream) writen2(b1 byte, b2 byte) {
+	z.scratch2[0] = b1
+	z.scratch2[1] = b2
+	n, err := z.w.Write(z.scratch2)
+	if err != nil && z.err == nil {
+		z.err = err
+	}
+	if n < 2 && z.err == nil {
+		z.err = fmt.Errorf("underwrite")
+	}
+}
+
+func (z *quickWriterStream) checkErr() error {
+	return z.err
+}
+
+func (z *quickWriterStream) clearErr() {
+	z.err = nil
+}

vendor/github.com/polydawn/refmt/cbor/errors.go

@@ -0,0 +1,22 @@
+package cbor
+
+import (
+	"fmt"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+// Error raised by Encoder when invalid tokens or invalid ordering, e.g. a MapClose with no matching open.
+// Should never be seen by the user in practice unless generating their own token streams.
+type ErrInvalidTokenStream struct {
+	Got        Token
+	Acceptable []TokenType
+}
+
+func (e *ErrInvalidTokenStream) Error() string {
+	return fmt.Sprintf("ErrInvalidTokenStream: unexpected %v, expected %v", e.Got, e.Acceptable)
+	// More comprehensible strings might include "start of value", "start of key or end of map", "start of value or end of array".
+}
+
+var tokenTypesForKey = []TokenType{TString, TInt, TUint}
+var tokenTypesForValue = []TokenType{TMapOpen, TArrOpen, TNull, TString, TBytes, TInt, TUint, TFloat64}

vendor/github.com/polydawn/refmt/obj/atlas/atlas.go

@@ -0,0 +1,79 @@
+package atlas
+
+import (
+	"fmt"
+	"reflect"
+)
+
+type Atlas struct {
+	// Map typeinfo to a static description of how that type should be handled.
+	// (The internal machinery that will wield this information, and has memory of
+	// progress as it does so, is configured using the AtlasEntry, but allocated separately.
+	// The machinery is stateful and mutable; the AtlasEntry is not.)
+	//
+	// We use 'var rtid uintptr = reflect.ValueOf(rt).Pointer()' -- pointer of the
+	// value of the reflect.Type info -- as an index.
+	// This is both unique and correctly converges when recomputed, and much
+	// faster to compare against than reflect.Type (which is an interface that
+	// tends to contain fairly large structures).
+	mappings map[uintptr]*AtlasEntry
+
+	// Mapping of tag ints to atlasEntry for quick lookups when the
+	// unmarshaller hits a tag.  Values are a subset of `mappings`.
+	tagMappings map[int]*AtlasEntry
+
+	// MapMorphism specifies the default map sorting scheme
+	defaultMapMorphism *MapMorphism
+}
+
+func Build(entries ...*AtlasEntry) (Atlas, error) {
+	atl := Atlas{
+		mappings:           make(map[uintptr]*AtlasEntry),
+		tagMappings:        make(map[int]*AtlasEntry),
+		defaultMapMorphism: &MapMorphism{KeySortMode_Default},
+	}
+	for _, entry := range entries {
+		rtid := reflect.ValueOf(entry.Type).Pointer()
+		if _, exists := atl.mappings[rtid]; exists {
+			return Atlas{}, fmt.Errorf("repeated entry for type %v", entry.Type)
+		}
+		atl.mappings[rtid] = entry
+
+		if entry.Tagged == true {
+			if prev, exists := atl.tagMappings[entry.Tag]; exists {
+				return Atlas{}, fmt.Errorf("repeated tag %v on type %v (already mapped to type %v)", entry.Tag, entry.Type, prev.Type)
+			}
+			atl.tagMappings[entry.Tag] = entry
+		}
+	}
+	return atl, nil
+}
+func MustBuild(entries ...*AtlasEntry) Atlas {
+	atl, err := Build(entries...)
+	if err != nil {
+		panic(err)
+	}
+	return atl
+}
+
+func (atl Atlas) WithMapMorphism(m MapMorphism) Atlas {
+	atl.defaultMapMorphism = &m
+	return atl
+}
+
+// Gets the AtlasEntry for a typeID.  Used by obj package, not meant for user facing.
+func (atl Atlas) Get(rtid uintptr) (*AtlasEntry, bool) {
+	ent, ok := atl.mappings[rtid]
+	return ent, ok
+}
+
+// Gets the AtlasEntry for a tag int.  Used by obj package, not meant for user facing.
+func (atl Atlas) GetEntryByTag(tag int) (*AtlasEntry, bool) {
+	ent, ok := atl.tagMappings[tag]
+	return ent, ok
+}
+
+// Gets the default map morphism config.  Used by obj package, not meant for user facing.
+func (atl Atlas) GetDefaultMapMorphism() *MapMorphism {
+	return atl.defaultMapMorphism
+}

vendor/github.com/polydawn/refmt/obj/atlas/atlasCommon.go

@@ -0,0 +1,10 @@
+package atlas
+
+// A type to enumerate key sorting modes.
+type KeySortMode string
+
+const (
+	KeySortMode_Default = KeySortMode("default") // the default mode -- for structs, this is the source-order of the fields; for maps, it's identify to "strings" sort mode.
+	KeySortMode_Strings = KeySortMode("strings") // lexical sort by strings.  this *is* the default for maps; it overrides source-order sorting for structs.
+	KeySortMode_RFC7049 = KeySortMode("rfc7049") // "Canonical" as proposed by rfc7049 § 3.9 (shorter byte sequences sort to top).
+)

vendor/github.com/polydawn/refmt/obj/atlas/atlasEntry.go

@@ -0,0 +1,150 @@
+package atlas
+
+import (
+	"reflect"
+)
+
+/*
+	The AtlasEntry is a declarative roadmap of what we should do for
+	marshal and unmarshal of a single object, keyed by type.
+
+	There are a lot of paths your mappings might want to take:
+
+	  - For a struct type, you may simply want to specify some alternate keys, or some to leave out, etc.
+	  - For an interface type, you probably want to specify one of our interface muxing strategies
+	     with a mapping between enumstr:typeinfo (and, what to do if we get a struct we don't recognize).
+	  - For a string, int, or other primitive, you don't need to say anything: defaults will DTRT.
+	  - For a typedef'd string, int, or other primitive, you *still* don't need to say anything: but,
+	     if you want custom behavior (say, transform the string to an int at the last second, and back again),
+		 you can specify transformer functions for that.
+	  - For a struct type that you want to turn into a whole different kind (like a string): use
+	     those same transform functions.  (You'll no longer need a FieldMap.)
+	  - For the most esoteric needs, you can fall all the way back to providing a custom MarshalMachine
+	     (but avoid that; it's a lot of work, and one of these other transform methods should suffice).
+*/
+type AtlasEntry struct {
+	// The reflect info of the type this morphism is regarding.
+	Type reflect.Type
+
+	// --------------------------------------------------------
+	// The big escape valves: wanna map to some other kind completely?
+	// --------------------------------------------------------
+
+	// Transforms the value we reached by walking (the 'live' value -- which
+	// must be of `this.Type`) into another value (the 'serialable' value --
+	// which will be of `this.MarshalTransformTargetType`).
+	//
+	// The target type may be anything, even of a completely different Kind!
+	//
+	// This transform func runs first, then the resulting value is
+	// serialized (by running through the path through Atlas again, so
+	// chaining of transform funcs is supported, though not recommended).
+	MarshalTransformFunc MarshalTransformFunc
+	// The type of value we expect after using the MarshalTransformFunc.
+	//
+	// The match between transform func and target type should be checked
+	// during construction of this AtlasEntry.
+	MarshalTransformTargetType reflect.Type
+
+	// Expects a different type (the 'serialable' value -- which will be of
+	// 'this.UnmarshalTransformTargetType') than the value we reached by
+	// walking (the 'live' value -- which must be of `this.Type`).
+	//
+	// The target type may be anything, even of a completely different Kind!
+	//
+	// The unmarshal of that target type will be run first, then the
+	// resulting value is fed through this function to produce the real value,
+	// which is then placed correctly into bigger mid-unmarshal object tree.
+	//
+	// For non-primitives, unmarshal of the target type will always target
+	// an empty pointer or empty slice, roughly as per if it was
+	// operating on a value produced by `TargetType.New()`.
+	UnmarshalTransformFunc UnmarshalTransformFunc
+	// The type of value we will manufacture an instance of and unmarshal
+	// into, then when done provide to the UnmarshalTransformFunc.
+	//
+	// The match between transform func and target type should be checked
+	// during construction of this AtlasEntry.
+	UnmarshalTransformTargetType reflect.Type
+
+	// --------------------------------------------------------
+	// Standard options for how to map (varies by Kind)
+	// --------------------------------------------------------
+
+	// A "tag" to emit when marshalling this type of value;
+	// and when unmarshalling, this tag will cause unmarshal to pick
+	// this atlas (and if there's conflicting type info, error).
+	Tag int
+	// Flag for whether the Tag feature should be used (zero is a valid tag).
+	Tagged bool
+
+	// A mapping of fields in a struct to serial keys.
+	// Only valid if `this.Type.Kind() == Struct`.
+	StructMap *StructMap
+
+	// Configuration for how to traverse a map kind.
+	// Only valid if `this.Type.Kind() == Map`.
+	MapMorphism *MapMorphism
+
+	// Configuration for how to pick concrete types to fill a union interface.
+	// Only valid if `this.Type.Kind() == Interface`.
+	UnionKeyedMorphism *UnionKeyedMorphism
+
+	// FUTURE: enum-ish primitives, multiplexers for interfaces,
+	//  lots of such things will belong here.
+
+	// --------------------------------------------------------
+	// Hooks, validate helpers
+	// --------------------------------------------------------
+
+	// A validation function which will be called for the whole value
+	// after unmarshalling reached the end of the object.
+	// If it returns an error, the entire unmarshal will error.
+	//
+	// Not used in marshalling.
+	// Not reachable if an UnmarshalTransform is set.
+	ValidateFn func(v interface{}) error
+}
+
+func BuildEntry(typeHintObj interface{}) *BuilderCore {
+	rt := reflect.TypeOf(typeHintObj)
+	if rt.Kind() == reflect.Ptr {
+		if rt.Elem().Kind() == reflect.Interface {
+			rt = rt.Elem()
+		} else {
+			panic("invalid atlas build: use the bare object, not a pointer (refmt will handle pointers automatically)")
+		}
+	}
+	return &BuilderCore{
+		&AtlasEntry{Type: rt},
+	}
+}
+
+/*
+	Intermediate step in building an AtlasEntry: use `BuildEntry` to
+	get one of these to start with, then call one of the methods
+	on this type to get a specialized builder which has the methods
+	relevant for setting up that specific kind of mapping.
+
+	One full example of using this builder may look like the following:
+
+		atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()
+
+	Some intermediate manipulations may be performed on this object,
+	for example setting the "tag" (if you want to use cbor tagging),
+	before calling the specializer method.
+	In this case, just keep chaining the configuration calls like so:
+
+		atlas.BuildEntry(Formula{}).UseTag(4000)
+			.StructMap().Autogenerate().Complete()
+
+*/
+type BuilderCore struct {
+	entry *AtlasEntry
+}
+
+func (x *BuilderCore) UseTag(tag int) *BuilderCore {
+	x.entry.Tagged = true
+	x.entry.Tag = tag
+	return x
+}

vendor/github.com/polydawn/refmt/obj/atlas/doc.go

@@ -0,0 +1,45 @@
+/*
+	Atlas types are used to define how to map Go values into refmt token streams.
+
+	Atlas information may be autogenerated based on struct tags automatically,
+	but you can also specify custom AtlasEntry info to use advanced features
+	and define custom transformations.
+
+	An Atlas is a collection of AtlasEntry (plus some internal indexing).
+	Typical usage is to declare an AtlasEntry for your structs (often near by the
+	struct definition), then
+
+	Building an AtlasEntry for some type called `Formula` looks like this:
+
+		atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()
+
+	Building an AtlasEntry always starts with `atlas.BuildEntry(x)` where `x` is
+	a dummy object used to convey type information.
+	The next function in the chain declares what kind of behavior we're going
+	to use to turn that type of object into its serial form.
+	(In the above example, we're declaring that we want refmt to see the `Formula`
+	type as a struct and traverse its fields.  There are many other options!)
+	Subsequent functions are specific to what kind of walking and mapping we've
+	chosen.  For struct walking, this may involve declaring fields and custom serial
+	names to map them to; for a "Transform" we'd instead have to provide callbacks
+	to do the transformation from the `Formula` type to some other type; etcetera.
+	The final function in the chain is always called `Complete`, and returns
+	a ready-to-use AtlasEntry.
+
+	Building a complete Atlas for a whole suite of serializable types is as
+	easy as putting a bunch of them together:
+
+		atlas.Build(
+			atlas.BuildEntry(Foo{}).StructMap().Autogenerate().Complete(),
+			atlas.BuildEntry(Bar{}).StructMap().Autogenerate().Complete(),
+			atlas.BuildEntry(Baz{}).StructMap().Autogenerate().Complete(),
+		)
+
+	You can put your entire protocol into one Atlas.
+	It's also possible to build several different Atlases each with different
+	sets of AtlasEntry.  This may be useful if you have a protocol where some
+	messages are not valid during some phases of communication, and you would
+	like to use the Atlas as a form of whitelisting for what can be
+	marshalled/unmarshalled.
+*/
+package atlas

vendor/github.com/polydawn/refmt/obj/atlas/errors.go

@@ -0,0 +1,14 @@
+package atlas
+
+// Error type raised when initializing an Atlas, and field entries do
+// not resolve against the type.
+// (If you recently refactored names of fields in your types, check
+// to make sure you updated any references to those fields by name to match!)
+type ErrStructureMismatch struct {
+	TypeName string
+	Reason   string
+}
+
+func (e ErrStructureMismatch) Error() string {
+	return "structure mismatch: " + e.TypeName + " " + e.Reason
+}

vendor/github.com/polydawn/refmt/obj/atlas/mapMorphism.go

@@ -0,0 +1,38 @@
+package atlas
+
+import (
+	"fmt"
+	"reflect"
+)
+
+type MapMorphism struct {
+	KeySortMode KeySortMode
+}
+
+func (x *BuilderCore) MapMorphism() *BuilderMapMorphism {
+	if x.entry.Type.Kind() != reflect.Map {
+		panic(fmt.Errorf("cannot use mapMorphism for type %q, which is kind %s", x.entry.Type, x.entry.Type.Kind()))
+	}
+	x.entry.MapMorphism = &MapMorphism{
+		KeySortMode_Default,
+	}
+	return &BuilderMapMorphism{x.entry}
+}
+
+type BuilderMapMorphism struct {
+	entry *AtlasEntry
+}
+
+func (x *BuilderMapMorphism) Complete() *AtlasEntry {
+	return x.entry
+}
+
+func (x *BuilderMapMorphism) SetKeySortMode(km KeySortMode) *BuilderMapMorphism {
+	switch km {
+	case KeySortMode_Default, KeySortMode_Strings, KeySortMode_RFC7049:
+		x.entry.MapMorphism.KeySortMode = km
+	default:
+		panic(fmt.Errorf("invalid key sort mode %q", km))
+	}
+	return x
+}

vendor/github.com/polydawn/refmt/obj/atlas/structMap.go

@@ -0,0 +1,46 @@
+package atlas
+
+import "reflect"
+
+type StructMap struct {
+	// A slice of descriptions of each field in the type.
+	// Each entry specifies the name by which each field should be referenced
+	// when serialized, and defines a way to get an address to the field.
+	Fields []StructMapEntry
+}
+
+type StructMapEntry struct {
+	// The field name; will be emitted as token during marshal, and used for
+	// lookup during unmarshal.  Required.
+	SerialName string
+
+	// If true, a key token with this SerialName will be ignored during unmarshal.
+	// (By default, if there's no StructMapEntry for a key token, it's an error.)
+	// If true, the ReflectRoute, Type, etc fields are irrelevant and may be nil.
+	Ignore bool
+
+	ReflectRoute ReflectRoute // reflection generates these.
+	Type         reflect.Type // type to expect on the far side of the ReflectRoute.
+	tagged       bool         // used during autogen.
+
+	// Theoretical feature which would be alternative to ReflectRoute.  Support dropped for the moment.
+	//addrFunc     func(interface{}) interface{} // custom user function.
+
+	// If true, marshalling will skip this field if it's the zero value.
+	OmitEmpty bool
+}
+
+type ReflectRoute []int
+
+func (rr ReflectRoute) TraverseToValue(v reflect.Value) reflect.Value {
+	for _, i := range rr {
+		if v.Kind() == reflect.Ptr {
+			if v.IsNil() {
+				return reflect.Value{}
+			}
+			v = v.Elem()
+		}
+		v = v.Field(i)
+	}
+	return v
+}

vendor/github.com/polydawn/refmt/obj/atlas/structMapAutogen.go

@@ -0,0 +1,320 @@
+package atlas
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"unicode"
+)
+
+func AutogenerateStructMapEntry(rt reflect.Type) *AtlasEntry {
+	return AutogenerateStructMapEntryUsingTags(rt, "refmt", KeySortMode_Default)
+}
+
+func AutogenerateStructMapEntryUsingTags(rt reflect.Type, tagName string, sorter KeySortMode) *AtlasEntry {
+	if rt.Kind() != reflect.Struct {
+		panic(fmt.Errorf("cannot use structMap for type %q, which is kind %s", rt, rt.Kind()))
+	}
+	entry := &AtlasEntry{
+		Type:      rt,
+		StructMap: &StructMap{Fields: exploreFields(rt, tagName, sorter)},
+	}
+	return entry
+}
+
+// exploreFields returns a list of fields that StructAtlas should recognize for the given type.
+// The algorithm is breadth-first search over the set of structs to include - the top struct
+// and then any reachable anonymous structs.
+func exploreFields(rt reflect.Type, tagName string, sorter KeySortMode) []StructMapEntry {
+	// Anonymous fields to explore at the current level and the next.
+	current := []StructMapEntry{}
+	next := []StructMapEntry{{Type: rt}}
+
+	// Count of queued names for current level and the next.
+	count := map[reflect.Type]int{}
+	nextCount := map[reflect.Type]int{}
+
+	// Types already visited at an earlier level.
+	visited := map[reflect.Type]bool{}
+
+	// Fields found.
+	var fields []StructMapEntry
+
+	for len(next) > 0 {
+		current, next = next, current[:0]
+		count, nextCount = nextCount, map[reflect.Type]int{}
+
+		for _, f := range current {
+			if visited[f.Type] {
+				continue
+			}
+			visited[f.Type] = true
+
+			// Scan f.Type for fields to include.
+			for i := 0; i < f.Type.NumField(); i++ {
+				sf := f.Type.Field(i)
+				if sf.PkgPath != "" && !sf.Anonymous { // unexported
+					continue
+				}
+				tag := sf.Tag.Get(tagName)
+				if tag == "-" {
+					continue
+				}
+				name, opts := parseTag(tag)
+				if !isValidTag(name) {
+					name = ""
+				}
+				route := make([]int, len(f.ReflectRoute)+1)
+				copy(route, f.ReflectRoute)
+				route[len(f.ReflectRoute)] = i
+
+				ft := sf.Type
+				if ft.Name() == "" && ft.Kind() == reflect.Ptr {
+					// Follow pointer.
+					ft = ft.Elem()
+				}
+
+				// Record found field and index sequence.
+				if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
+					tagged := name != ""
+					if name == "" {
+						name = downcaseFirstLetter(sf.Name)
+					}
+					fields = append(fields, StructMapEntry{
+						SerialName:   name,
+						ReflectRoute: route,
+						Type:         sf.Type,
+						tagged:       tagged,
+						OmitEmpty:    opts.Contains("omitempty"),
+					})
+					if count[f.Type] > 1 {
+						// If there were multiple instances, add a second,
+						// so that the annihilation code will see a duplicate.
+						// It only cares about the distinction between 1 or 2,
+						// so don't bother generating any more copies.
+						fields = append(fields, fields[len(fields)-1])
+					}
+					continue
+				}
+
+				// Record new anonymous struct to explore in next round.
+				nextCount[ft]++
+				if nextCount[ft] == 1 {
+					next = append(next, StructMapEntry{
+						ReflectRoute: route,
+						Type:         ft,
+					})
+				}
+			}
+		}
+	}
+
+	sort.Sort(StructMapEntry_byName(fields))
+
+	// Delete all fields that are hidden by the Go rules for embedded fields,
+	// except that fields with JSON tags are promoted.
+
+	// The fields are sorted in primary order of name, secondary order
+	// of field index length. Loop over names; for each name, delete
+	// hidden fields by choosing the one dominant field that survives.
+	out := fields[:0]
+	for advance, i := 0, 0; i < len(fields); i += advance {
+		// One iteration per name.
+		// Find the sequence of fields with the name of this first field.
+		fi := fields[i]
+		name := fi.SerialName
+		for advance = 1; i+advance < len(fields); advance++ {
+			fj := fields[i+advance]
+			if fj.SerialName != name {
+				break
+			}
+		}
+		if advance == 1 { // Only one field with this name
+			out = append(out, fi)
+			continue
+		}
+		dominant, ok := dominantField(fields[i : i+advance])
+		if ok {
+			out = append(out, dominant)
+		}
+	}
+
+	fields = out
+	switch sorter {
+	case KeySortMode_Default:
+		sort.Sort(StructMapEntry_byFieldRoute(fields))
+	case KeySortMode_Strings:
+		//sort.Sort(StructMapEntry_byName(fields))
+		// it's already in this order, though, so, pass
+	case KeySortMode_RFC7049:
+		sort.Sort(StructMapEntry_RFC7049(fields))
+	default:
+		panic("invalid struct sorter option")
+	}
+
+	return fields
+}
+
+// If the first character of the string is uppercase, return a string
+// where it is switched to lowercase.
+// We use this to make go field names look more like what everyone else
+// in the universe expects their json to look like by default: snakeCase.
+func downcaseFirstLetter(s string) string {
+	if s == "" {
+		return ""
+	}
+	r := rune(s[0]) // if multibyte chars: you're left alone.
+	if !unicode.IsUpper(r) {
+		return s
+	}
+	return string(unicode.ToLower(r)) + s[1:]
+}
+
+// dominantField looks through the fields, all of which are known to
+// have the same name, to find the single field that dominates the
+// others using Go's embedding rules, modified by the presence of
+// JSON tags. If there are multiple top-level fields, the boolean
+// will be false: This condition is an error in Go and we skip all
+// the fields.
+func dominantField(fields []StructMapEntry) (StructMapEntry, bool) {
+	// The fields are sorted in increasing index-length order. The winner
+	// must therefore be one with the shortest index length. Drop all
+	// longer entries, which is easy: just truncate the slice.
+	length := len(fields[0].ReflectRoute)
+	tagged := -1 // Index of first tagged field.
+	for i, f := range fields {
+		if len(f.ReflectRoute) > length {
+			fields = fields[:i]
+			break
+		}
+		if f.tagged {
+			if tagged >= 0 {
+				// Multiple tagged fields at the same level: conflict.
+				// Return no field.
+				return StructMapEntry{}, false
+			}
+			tagged = i
+		}
+	}
+	if tagged >= 0 {
+		return fields[tagged], true
+	}
+	// All remaining fields have the same length. If there's more than one,
+	// we have a conflict (two fields named "X" at the same level) and we
+	// return no field.
+	if len(fields) > 1 {
+		return StructMapEntry{}, false
+	}
+	return fields[0], true
+}
+
+// StructMapEntry_byName sorts field by name,
+// breaking ties with depth,
+// then breaking ties with "name came from tag",
+// then breaking ties with FieldRoute sequence.
+type StructMapEntry_byName []StructMapEntry
+
+func (x StructMapEntry_byName) Len() int      { return len(x) }
+func (x StructMapEntry_byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x StructMapEntry_byName) Less(i, j int) bool {
+	if x[i].SerialName != x[j].SerialName {
+		return x[i].SerialName < x[j].SerialName
+	}
+	if len(x[i].ReflectRoute) != len(x[j].ReflectRoute) {
+		return len(x[i].ReflectRoute) < len(x[j].ReflectRoute)
+	}
+	if x[i].tagged != x[j].tagged {
+		return x[i].tagged
+	}
+	return StructMapEntry_byFieldRoute(x).Less(i, j)
+}
+
+// StructMapEntry_RFC7049 sorts fields as specified in RFC7049,
+type StructMapEntry_RFC7049 []StructMapEntry
+
+func (x StructMapEntry_RFC7049) Len() int      { return len(x) }
+func (x StructMapEntry_RFC7049) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x StructMapEntry_RFC7049) Less(i, j int) bool {
+	il, jl := len(x[i].SerialName), len(x[j].SerialName)
+	switch {
+	case il < jl:
+		return true
+	case il > jl:
+		return false
+	default:
+		return x[i].SerialName < x[j].SerialName
+	}
+}
+
+// StructMapEntry_byFieldRoute sorts field by FieldRoute sequence
+// (e.g., roughly source declaration order within each type).
+type StructMapEntry_byFieldRoute []StructMapEntry
+
+func (x StructMapEntry_byFieldRoute) Len() int      { return len(x) }
+func (x StructMapEntry_byFieldRoute) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x StructMapEntry_byFieldRoute) Less(i, j int) bool {
+	for k, xik := range x[i].ReflectRoute {
+		if k >= len(x[j].ReflectRoute) {
+			return false
+		}
+		if xik != x[j].ReflectRoute[k] {
+			return xik < x[j].ReflectRoute[k]
+		}
+	}
+	return len(x[i].ReflectRoute) < len(x[j].ReflectRoute)
+}
+
+// tagOptions is the string following a comma in a struct field's
+// tag, or the empty string. It does not include the leading comma.
+type tagOptions string
+
+// parseTag splits a struct field's tag into its name and
+// comma-separated options.
+func parseTag(tag string) (string, tagOptions) {
+	if idx := strings.Index(tag, ","); idx != -1 {
+		return tag[:idx], tagOptions(tag[idx+1:])
+	}
+	return tag, tagOptions("")
+}
+
+// Contains reports whether a comma-separated list of options
+// contains a particular substr flag. substr must be surrounded by a
+// string boundary or commas.
+func (o tagOptions) Contains(optionName string) bool {
+	if len(o) == 0 {
+		return false
+	}
+	s := string(o)
+	for s != "" {
+		var next string
+		i := strings.Index(s, ",")
+		if i >= 0 {
+			s, next = s[:i], s[i+1:]
+		}
+		if s == optionName {
+			return true
+		}
+		s = next
+	}
+	return false
+}
+
+func isValidTag(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, c := range s {
+		switch {
+		case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c):
+			// Backslash and quote chars are reserved, but
+			// otherwise any punctuation chars are allowed
+			// in a tag name.
+		default:
+			if !unicode.IsLetter(c) && !unicode.IsDigit(c) {
+				return false
+			}
+		}
+	}
+	return true
+}

vendor/github.com/polydawn/refmt/obj/atlas/structMapBuilding.go

@@ -0,0 +1,96 @@
+package atlas
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+func (x *BuilderCore) StructMap() *BuilderStructMap {
+	if x.entry.Type.Kind() != reflect.Struct {
+		panic(fmt.Errorf("cannot use structMap for type %q, which is kind %s", x.entry.Type, x.entry.Type.Kind()))
+	}
+	x.entry.StructMap = &StructMap{}
+	return &BuilderStructMap{x.entry}
+}
+
+type BuilderStructMap struct {
+	entry *AtlasEntry
+}
+
+func (x *BuilderStructMap) Complete() *AtlasEntry {
+	return x.entry
+}
+
+/*
+	Add a field to the mapping based on its name.
+
+	Given a struct:
+
+		struct{
+			X int
+			Y struct{ Z int }
+		}
+
+	`AddField("X", {"x", ...}) will cause that field to be serialized as key "x";
+	`AddField("Y.Z", {"z", ...})` will cause that *nested* field to be serialized
+	as key "z" in the same object (e.g. "x" and "z" will be siblings).
+
+	Returns the mutated builder for convenient call chaining.
+
+	If the fieldName string doesn't map onto the structure type info,
+	a panic will be raised.
+*/
+func (x *BuilderStructMap) AddField(fieldName string, mapping StructMapEntry) *BuilderStructMap {
+	fieldNameSplit := strings.Split(fieldName, ".")
+	rr, rt, err := fieldNameToReflectRoute(x.entry.Type, fieldNameSplit)
+	if err != nil {
+		panic(err) // REVIEW: now that we have the builder obj, we could just curry these into it until 'Complete' is called (or, thus, 'MustComplete'!).
+	}
+	mapping.ReflectRoute = rr
+	mapping.Type = rt
+	x.entry.StructMap.Fields = append(x.entry.StructMap.Fields, mapping)
+	return x
+}
+
+func (x *BuilderStructMap) IgnoreKey(serialKeyName string) *BuilderStructMap {
+	x.entry.StructMap.Fields = append(x.entry.StructMap.Fields, StructMapEntry{
+		SerialName: serialKeyName,
+		Ignore:     true,
+	})
+	return x
+}
+
+func fieldNameToReflectRoute(rt reflect.Type, fieldNameSplit []string) (rr ReflectRoute, _ reflect.Type, _ error) {
+	for _, fn := range fieldNameSplit {
+		rf, ok := rt.FieldByName(fn)
+		if !ok {
+			return nil, nil, ErrStructureMismatch{rt.Name(), "does not have field named " + fn}
+		}
+		rr = append(rr, rf.Index...)
+		rt = rf.Type
+	}
+	return rr, rt, nil
+}
+
+/*
+	Automatically generate mappings by looking at the struct type info,
+	taking any hints from tags, and appending that to the builder.
+
+	You may use autogeneration in concert with manually adding field mappings,
+	though if doing so be mindful not to map the same fields twice.
+*/
+func (x *BuilderStructMap) Autogenerate() *BuilderStructMap {
+	autoEntry := AutogenerateStructMapEntry(x.entry.Type)
+	x.entry.StructMap.Fields = append(x.entry.StructMap.Fields, autoEntry.StructMap.Fields...)
+	return x
+}
+
+/*
+	Automatically generate mappings using a given struct field sorting scheme
+*/
+func (x *BuilderStructMap) AutogenerateWithSortingScheme(sorting KeySortMode) *BuilderStructMap {
+	autoEntry := AutogenerateStructMapEntryUsingTags(x.entry.Type, "refmt", sorting)
+	x.entry.StructMap.Fields = append(x.entry.StructMap.Fields, autoEntry.StructMap.Fields...)
+	return x
+}

vendor/github.com/polydawn/refmt/obj/atlas/transformBuilding.go

@@ -0,0 +1,30 @@
+package atlas
+
+import (
+	"reflect"
+)
+
+func (x *BuilderCore) Transform() *BuilderTransform {
+	// no checks on x.entry.Type.Kind() here -- transforms can be pretty much any<->any
+	return &BuilderTransform{x.entry}
+}
+
+type BuilderTransform struct {
+	entry *AtlasEntry
+}
+
+func (x *BuilderTransform) Complete() *AtlasEntry {
+	return x.entry
+}
+
+func (x *BuilderTransform) TransformMarshal(trFunc MarshalTransformFunc, toType reflect.Type) *BuilderTransform {
+	x.entry.MarshalTransformFunc = trFunc
+	x.entry.MarshalTransformTargetType = toType
+	return x
+}
+
+func (x *BuilderTransform) TransformUnmarshal(trFunc UnmarshalTransformFunc, toType reflect.Type) *BuilderTransform {
+	x.entry.UnmarshalTransformFunc = trFunc
+	x.entry.UnmarshalTransformTargetType = toType
+	return x
+}

vendor/github.com/polydawn/refmt/obj/atlas/transformFuncs.go

@@ -0,0 +1,68 @@
+package atlas
+
+import "reflect"
+
+type MarshalTransformFunc func(liveForm reflect.Value) (serialForm reflect.Value, err error)
+type UnmarshalTransformFunc func(serialForm reflect.Value) (liveForm reflect.Value, err error)
+
+var err_rt = reflect.TypeOf((*error)(nil)).Elem()
+
+/*
+	Takes a wildcard object which must be `func (live T1) (serialable T2, error)`
+	and returns a MarshalTransformFunc and the typeinfo of T2.
+*/
+func MakeMarshalTransformFunc(fn interface{}) (MarshalTransformFunc, reflect.Type) {
+	fn_rv := reflect.ValueOf(fn)
+	if fn_rv.Kind() != reflect.Func {
+		panic("no")
+	}
+	fn_rt := fn_rv.Type()
+	if fn_rt.NumIn() != 1 {
+		panic("no")
+	}
+	if fn_rt.NumOut() != 2 {
+		panic("no")
+	}
+	if !fn_rt.Out(1).AssignableTo(err_rt) {
+		panic("no")
+	}
+	// nothing to do for `fn_rt.In(0)` -- whatever type it is... TODO redesign to make less sketchy; we should most certainly be able to check this in the builder
+	out_rt := fn_rt.Out(0)
+	return func(liveForm reflect.Value) (serialForm reflect.Value, err error) {
+		results := fn_rv.Call([]reflect.Value{liveForm})
+		if results[1].IsNil() {
+			return results[0], nil
+		}
+		return results[0], results[1].Interface().(error)
+	}, out_rt
+}
+
+/*
+	Takes a wildcard object which must be `func (serialable T1) (live T2, error)`
+	and returns a UnmarshalTransformFunc and the typeinfo of T1.
+*/
+func MakeUnmarshalTransformFunc(fn interface{}) (UnmarshalTransformFunc, reflect.Type) {
+	fn_rv := reflect.ValueOf(fn)
+	if fn_rv.Kind() != reflect.Func {
+		panic("no")
+	}
+	fn_rt := fn_rv.Type()
+	if fn_rt.NumIn() != 1 {
+		panic("no")
+	}
+	if fn_rt.NumOut() != 2 {
+		panic("no")
+	}
+	if !fn_rt.Out(1).AssignableTo(err_rt) {
+		panic("no")
+	}
+	// nothing to do for checking `fn_rf.Out(0)` -- because we don't know what entry we're about to be used for.  TODO redesign to make less sketchy.
+	in_rt := fn_rt.In(0)
+	return func(serialForm reflect.Value) (liveForm reflect.Value, err error) {
+		results := fn_rv.Call([]reflect.Value{serialForm})
+		if results[1].IsNil() {
+			return results[0], nil
+		}
+		return results[0], results[1].Interface().(error)
+	}, in_rt
+}

vendor/github.com/polydawn/refmt/obj/atlas/unionMorphism.go

@@ -0,0 +1,45 @@
+package atlas
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+)
+
+type UnionKeyedMorphism struct {
+	// Mapping of typehint key strings to atlasEntry that should be delegated to.
+	Elements map[string]*AtlasEntry
+	// Mapping of rtid to string (roughly the dual of the Elements map).
+	Mappings map[uintptr]string
+	// Purely to have in readiness for error messaging.
+	KnownMembers []string
+}
+
+func (x *BuilderCore) KeyedUnion() *BuilderUnionKeyedMorphism {
+	if x.entry.Type.Kind() != reflect.Interface {
+		panic(fmt.Errorf("cannot use union morphisms for type %q, which is kind %s", x.entry.Type, x.entry.Type.Kind()))
+	}
+	x.entry.UnionKeyedMorphism = &UnionKeyedMorphism{
+		Elements: make(map[string]*AtlasEntry),
+		Mappings: make(map[uintptr]string),
+	}
+	return &BuilderUnionKeyedMorphism{x.entry}
+}
+
+type BuilderUnionKeyedMorphism struct {
+	entry *AtlasEntry
+}
+
+func (x *BuilderUnionKeyedMorphism) Of(elements map[string]*AtlasEntry) *AtlasEntry {
+	cfg := x.entry.UnionKeyedMorphism
+	for hint, ent := range elements {
+		// FIXME: check that all the delegates are... well struct or map machines really, but definitely blacklisting other delegating machinery.
+		// FIXME: and sanity check that they can all be assigned to the interface ffs.
+
+		cfg.Elements[hint] = ent
+		cfg.Mappings[reflect.ValueOf(ent.Type).Pointer()] = hint
+		cfg.KnownMembers = append(cfg.KnownMembers, hint)
+	}
+	sort.Strings(cfg.KnownMembers)
+	return x.entry
+}

vendor/github.com/polydawn/refmt/obj/builtins.go

@@ -0,0 +1,24 @@
+package obj
+
+import (
+	. "reflect"
+)
+
+var (
+	rtid_bool    = ValueOf(TypeOf(false)).Pointer()
+	rtid_string  = ValueOf(TypeOf("")).Pointer()
+	rtid_bytes   = ValueOf(TypeOf([]byte{})).Pointer()
+	rtid_int     = ValueOf(TypeOf(int(0))).Pointer()
+	rtid_int8    = ValueOf(TypeOf(int8(0))).Pointer()
+	rtid_int16   = ValueOf(TypeOf(int16(0))).Pointer()
+	rtid_int32   = ValueOf(TypeOf(int32(0))).Pointer()
+	rtid_int64   = ValueOf(TypeOf(int64(0))).Pointer()
+	rtid_uint    = ValueOf(TypeOf(uint(0))).Pointer()
+	rtid_uint8   = ValueOf(TypeOf(uint8(0))).Pointer()
+	rtid_uint16  = ValueOf(TypeOf(uint16(0))).Pointer()
+	rtid_uint32  = ValueOf(TypeOf(uint32(0))).Pointer()
+	rtid_uint64  = ValueOf(TypeOf(uint64(0))).Pointer()
+	rtid_uintptr = ValueOf(TypeOf(uintptr(0))).Pointer()
+	rtid_float32 = ValueOf(TypeOf(float32(0))).Pointer()
+	rtid_float64 = ValueOf(TypeOf(float64(0))).Pointer()
+)

vendor/github.com/polydawn/refmt/obj/doc.go

@@ -0,0 +1,6 @@
+/*
+	The `obj` package defines Marshaller and Unmarshaller types,
+	which can be used to convert in-memory values to token streams,
+	and token streams to unpack in-memory values.
+*/
+package obj

vendor/github.com/polydawn/refmt/obj/empty.go

@@ -0,0 +1,31 @@
+package obj
+
+import "reflect"
+
+// The missing definition of 'reflect.IsZero' you've always wanted.
+func isEmptyValue(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		return v.Len() == 0
+	case reflect.Bool:
+		return !v.Bool()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return v.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return v.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return v.Float() == 0
+	case reflect.Interface, reflect.Ptr:
+		return v.IsNil()
+	case reflect.Struct:
+		// Note: we can't rely on a *literal* "is zero" check because
+		// non-zero pointers may still point to *empty* things.
+		for i := 0; i < v.NumField(); i++ {
+			if !isEmptyValue(v.Field(i)) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}

vendor/github.com/polydawn/refmt/obj/errors.go

@@ -0,0 +1,82 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+// General note: avoid using reflect.Type here.  It doesn't do well with `go-cmp`,
+//  which in turn makes our tests for error paths a lot jankier.
+
+// ErrInvalidUnmarshalTarget describes an invalid argument passed to Unmarshaller.Bind.
+// (Unmarshalling must target a non-nil pointer so that it can address the value.)
+type ErrInvalidUnmarshalTarget struct {
+	Type reflect.Type
+}
+
+func (e ErrInvalidUnmarshalTarget) Error() string {
+	if e.Type == nil {
+		return "unmarshal error: invalid target (nil)"
+	}
+	if e.Type.Kind() != reflect.Ptr {
+		return "unmarshal error: invalid target (non-pointer " + e.Type.String() + ")"
+	}
+	return "unmarshal error: invalid target (nil " + e.Type.String() + ")"
+}
+
+// ErrUnmarshalTypeCantFit is the error returned when unmarshalling cannot
+// coerce the tokens in the stream into the kind of variables the unmarshal is targetting,
+// for example if a map open token comes when an int is expected,
+// or an int token comes when a string is expected.
+type ErrUnmarshalTypeCantFit struct {
+	Token  Token
+	Value  reflect.Value
+	LenLim int // Set only if Value.Kind == Array and Token is bytes of a mismatch length.
+}
+
+func (e ErrUnmarshalTypeCantFit) Error() string {
+	switch e.LenLim {
+	case 0:
+		return fmt.Sprintf("unmarshal error: cannot assign %s to %s field", e.Token, e.Value.Kind())
+	default:
+		return fmt.Sprintf("unmarshal error: cannot assign %s to fixed length=%d byte array", e.Token, e.Value.Len())
+	}
+}
+
+// ErrMalformedTokenStream is the error returned when unmarshalling recieves ae
+// completely invalid transition, such as when a map value is expected, but the
+// map suddenly closes, or an array close is recieved with no matching array open.
+type ErrMalformedTokenStream struct {
+	Got      TokenType // Token in the stream that triggered the error.
+	Expected string    // Freeform string describing valid token types.  Often a summary like "array close or start of value", or "map close or key".
+}
+
+func (e ErrMalformedTokenStream) Error() string {
+	return fmt.Sprintf("malformed stream: invalid appearance of %s token; expected %s", e.Got, e.Expected)
+}
+
+// ErrNoSuchField is the error returned when unmarshalling into a struct and
+// the token stream for the map contains a key which is not defined for the struct.
+type ErrNoSuchField struct {
+	Name string // Field name from the token.
+	Type string // Type name of the struct we're operating on.
+}
+
+func (e ErrNoSuchField) Error() string {
+	return fmt.Sprintf("unmarshal error: stream contains key %q, but there's no such field in structs of type %s", e.Name, e.Type)
+}
+
+// ErrNoSuchUnionMember is the error returned when unmarshalling into a union
+// interface and the token stream contains a key which does not name any of the
+// known members of the union.
+type ErrNoSuchUnionMember struct {
+	Name         string       // Key name from the token.
+	Type         reflect.Type // The interface type we're trying to fill.
+	KnownMembers []string     // Members we expected isntead.
+}
+
+func (e ErrNoSuchUnionMember) Error() string {
+	return fmt.Sprintf("unmarshal error: cannot unmarshal into union %s: %q is not one of the known members (expected one of %s)", e.Type, e.Name, e.KnownMembers)
+}

vendor/github.com/polydawn/refmt/obj/marshal.go

@@ -0,0 +1,110 @@
+package obj
+
+import (
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+/*
+	Allocates the machinery for treating an object like a `TokenSource`.
+	This machinery will walk over structures in memory,
+	emitting tokens representing values and fields as it visits them.
+
+	Initialization must be finished by calling `Bind` to set the value to visit;
+	after this, the `Step` function is ready to be pumped.
+	Subsequent calls to `Bind` do a full reset, leaving `Step` ready to call
+	again and making all of the machinery reusable without re-allocating.
+*/
+func NewMarshaller(atl atlas.Atlas) *Marshaller {
+	d := &Marshaller{
+		marshalSlab: marshalSlab{
+			atlas: atl,
+			rows:  make([]marshalSlabRow, 0, 10),
+		},
+		stack: make([]MarshalMachine, 0, 10),
+	}
+	return d
+}
+
+func (d *Marshaller) Bind(v interface{}) error {
+	d.stack = d.stack[0:0]
+	d.marshalSlab.rows = d.marshalSlab.rows[0:0]
+	rv := reflect.ValueOf(v)
+	if !rv.IsValid() {
+		// if given an untyped nil, swap in a less spikey nil thunk instead.
+		rv = nil_rv
+	}
+	rt := rv.Type()
+	d.step = d.marshalSlab.requisitionMachine(rt)
+	return d.step.Reset(&d.marshalSlab, rv, rt)
+}
+
+type Marshaller struct {
+	marshalSlab marshalSlab
+	stack       []MarshalMachine
+	step        MarshalMachine
+}
+
+type MarshalMachine interface {
+	Reset(*marshalSlab, reflect.Value, reflect.Type) error
+	Step(*Marshaller, *marshalSlab, *Token) (done bool, err error)
+}
+
+type marshalMachineStep func(*Marshaller, *marshalSlab, *Token) (done bool, err error)
+
+func (d *Marshaller) Step(tok *Token) (bool, error) {
+	tok.Tagged = false
+	//	fmt.Printf("> next step is %#v\n", d.step)
+	done, err := d.step.Step(d, &d.marshalSlab, tok)
+	//	fmt.Printf(">> yield is %#v\n", TokenToString(*tok))
+	// If the step errored: out, entirely.
+	if err != nil {
+		return true, err
+	}
+	// If the step wasn't done, return same status.
+	if !done {
+		return false, nil
+	}
+	// If it WAS done, pop next, or if stack empty, we're entirely done.
+	nSteps := len(d.stack) - 1
+	if nSteps == -1 {
+		return true, nil // that's all folks
+	}
+	//	fmt.Printf(">> popping up from %#v\n", d.stack)
+	d.step = d.stack[nSteps]
+	d.stack = d.stack[0:nSteps]
+	return false, nil
+}
+
+/*
+	Starts the process of recursing marshalling over value `rv`.
+
+	Caller provides the machine to use (this is an optimization for maps and slices,
+	which already know the machine and keep reusing it for all their entries).
+	This method pushes the first step with `tok` (the upstream tends to have peeked at
+	it in order to decide what to do, but if recursing, it belongs to the next obj),
+	then saves this new machine onto the driver's stack: future calls to step
+	the driver will then continuing stepping the new machine it returns a done status,
+	at which point we'll finally "return" by popping back to the last machine on the stack
+	(which is presumably the same one that just called this Recurse method).
+
+	In other words, your MarshalMachine calls this when it wants to deal
+	with an object, and by the time we call back to your machine again,
+	that object will be traversed and the stream ready for you to continue.
+*/
+func (d *Marshaller) Recurse(tok *Token, rv reflect.Value, rt reflect.Type, nextMach MarshalMachine) (err error) {
+	//	fmt.Printf(">>> pushing into recursion with %#v\n", nextMach)
+	// Push the current machine onto the stack (we'll resume it when the new one is done),
+	d.stack = append(d.stack, d.step)
+	// Initialize the machine for this new target value.
+	err = nextMach.Reset(&d.marshalSlab, rv, rt)
+	if err != nil {
+		return
+	}
+	d.step = nextMach
+	// Immediately make a step (we're still the delegate in charge of someone else's step).
+	_, err = d.Step(tok)
+	return
+}

vendor/github.com/polydawn/refmt/obj/marshalBuiltins.go

@@ -0,0 +1,92 @@
+package obj
+
+import (
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type ptrDerefDelegateMarshalMachine struct {
+	MarshalMachine
+	peelCount int
+
+	isNil bool
+}
+
+func (mach *ptrDerefDelegateMarshalMachine) Reset(slab *marshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.isNil = false
+	for i := 0; i < mach.peelCount; i++ {
+		if rv.IsNil() {
+			mach.isNil = true
+			return nil
+		}
+		rv = rv.Elem()
+	}
+	return mach.MarshalMachine.Reset(slab, rv, rv.Type()) // REVIEW: we could have cached the peeled rt at mach conf time; worth it?
+}
+func (mach *ptrDerefDelegateMarshalMachine) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	if mach.isNil {
+		tok.Type = TNull
+		return true, nil
+	}
+	return mach.MarshalMachine.Step(driver, slab, tok)
+}
+
+type marshalMachinePrimitive struct {
+	kind reflect.Kind
+
+	rv reflect.Value
+}
+
+func (mach *marshalMachinePrimitive) Reset(_ *marshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.rv = rv
+	return nil
+}
+func (mach *marshalMachinePrimitive) Step(_ *Marshaller, _ *marshalSlab, tok *Token) (done bool, err error) {
+	switch mach.kind {
+	case reflect.Bool:
+		tok.Type = TBool
+		tok.Bool = mach.rv.Bool()
+		return true, nil
+	case reflect.String:
+		tok.Type = TString
+		tok.Str = mach.rv.String()
+		return true, nil
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		tok.Type = TInt
+		tok.Int = mach.rv.Int()
+		return true, nil
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		tok.Type = TUint
+		tok.Uint = mach.rv.Uint()
+		return true, nil
+	case reflect.Float32, reflect.Float64:
+		tok.Type = TFloat64
+		tok.Float64 = mach.rv.Float()
+		return true, nil
+	case reflect.Slice: // implicitly bytes; no other slices are "primitive"
+		if mach.rv.IsNil() {
+			tok.Type = TNull
+			return true, nil
+		}
+		tok.Type = TBytes
+		tok.Bytes = mach.rv.Bytes()
+		return true, nil
+	case reflect.Array: // implicitly bytes; no other arrays are "primitive"
+		tok.Type = TBytes
+		// Unfortunately, there does not seem to be any efficient way to extract the contents of a byte array into a slice via reflect.
+		// Since the lengths are part of the type, it is almost understandable that the stdlib reflect package has a hard time expressing this;
+		// however, it drives me somewhat up the wall that they do not provide a case for arrays inside the `Value.Bytes` method, and panic.
+		// Attempting to `Value.Convert(Type)` from a fixed-length array to a slice of the same type is also rejected.
+		// Nor does `reflect.AppendSlice` accept an array kind as the second parameter; no, only slices there too.
+		// So... we iterate.  If anyone knows a better way to do this, PRs extremely welcome.
+		n := mach.rv.Len()
+		tok.Bytes = make([]byte, n)
+		for i := 0; i < n; i++ {
+			tok.Bytes[i] = byte(mach.rv.Index(i).Uint())
+		}
+		return true, nil
+	default:
+		panic("unhandled")
+	}
+}

vendor/github.com/polydawn/refmt/obj/marshalMapWildcard.go

@@ -0,0 +1,148 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type marshalMachineMapWildcard struct {
+	morphism *atlas.MapMorphism // set on initialization
+
+	target_rv   reflect.Value
+	value_rt    reflect.Type
+	keyStringer atlas.MarshalTransformFunc
+	valueMach   MarshalMachine
+	keys        []wildcardMapStringyKey
+	index       int
+	value       bool
+}
+
+func (mach *marshalMachineMapWildcard) Reset(slab *marshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+
+	// Pick machinery for handling the value types.
+	mach.value_rt = rt.Elem()
+	mach.valueMach = slab.requisitionMachine(mach.value_rt)
+
+	// Enumerate all the keys (must do this up front, one way or another),
+	// flip them into strings,
+	// and sort them (optional, arguably, but right now you're getting it).
+	key_rt := rt.Key()
+	switch key_rt.Kind() {
+	case reflect.String:
+		// continue.
+		// note: stdlib json.marshal supports all the int types here as well, and will
+		//  tostring them.  but this is not supported symmetrically; so we simply... don't.
+		//  we could also consider supporting anything that uses a MarshalTransformFunc
+		//  to become a string kind; that's a fair bit of code, perhaps later.
+		mach.keyStringer = nil
+	case reflect.Struct:
+		// composite keys requires some fancy footwork, but we can do it.
+		// Interestingly enough, we don't need full-on machinery here; because the
+		//  tokenized form is restricted to being a string, the transform func is enough.
+		rtid := reflect.ValueOf(key_rt).Pointer()
+		atlEnt, ok := slab.atlas.Get(rtid)
+		if !ok || atlEnt.MarshalTransformTargetType == nil || atlEnt.MarshalTransformTargetType.Kind() != reflect.String {
+			return fmt.Errorf("unsupported map key type %q (if you want to use struct keys, your atlas needs a transform to string)", key_rt.Name())
+		}
+		mach.keyStringer = atlEnt.MarshalTransformFunc
+	default:
+		return fmt.Errorf("unsupported map key type %q", key_rt.Name())
+	}
+	keys_rv := mach.target_rv.MapKeys()
+	mach.keys = make([]wildcardMapStringyKey, len(keys_rv))
+	for i, v := range keys_rv {
+		mach.keys[i].rv = v
+		if mach.keyStringer == nil {
+			mach.keys[i].s = v.String()
+		} else {
+			trans_rv, err := mach.keyStringer(v)
+			if err != nil {
+				return fmt.Errorf("unsupported map key type %q: errors in stringifying: %s", key_rt.Name(), err)
+			}
+			mach.keys[i].s = trans_rv.String()
+		}
+	}
+
+	ksm := atlas.KeySortMode_Default
+	if mach.morphism != nil {
+		ksm = mach.morphism.KeySortMode
+	}
+
+	switch ksm {
+	case atlas.KeySortMode_Default:
+		sort.Sort(wildcardMapStringyKey_byString(mach.keys))
+	case atlas.KeySortMode_Strings:
+		sort.Sort(wildcardMapStringyKey_byString(mach.keys))
+	case atlas.KeySortMode_RFC7049:
+		sort.Sort(wildcardMapStringyKey_RFC7049(mach.keys))
+	default:
+		panic(fmt.Errorf("unknown map key sort mode %q", ksm))
+	}
+
+	mach.index = -1
+	return nil
+}
+
+func (mach *marshalMachineMapWildcard) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	if mach.index < 0 {
+		if mach.target_rv.IsNil() {
+			tok.Type = TNull
+			mach.index++
+			return true, nil
+		}
+		tok.Type = TMapOpen
+		tok.Length = mach.target_rv.Len()
+		mach.index++
+		return false, nil
+	}
+	if mach.index == len(mach.keys) {
+		tok.Type = TMapClose
+		mach.index++
+		slab.release()
+		return true, nil
+	}
+	if mach.index > len(mach.keys) {
+		return true, fmt.Errorf("invalid state: value already consumed")
+	}
+	if mach.value {
+		val_rv := mach.target_rv.MapIndex(mach.keys[mach.index].rv)
+		mach.value = false
+		mach.index++
+		return false, driver.Recurse(tok, val_rv, mach.value_rt, mach.valueMach)
+	}
+	tok.Type = TString
+	tok.Str = mach.keys[mach.index].s
+	mach.value = true
+	return false, nil
+}
+
+// Holder for the reflect.Value and string form of a key.
+// We need the reflect.Value for looking up the map value;
+// and we need the string for sorting.
+type wildcardMapStringyKey struct {
+	rv reflect.Value
+	s  string
+}
+
+type wildcardMapStringyKey_byString []wildcardMapStringyKey
+
+func (x wildcardMapStringyKey_byString) Len() int           { return len(x) }
+func (x wildcardMapStringyKey_byString) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
+func (x wildcardMapStringyKey_byString) Less(i, j int) bool { return x[i].s < x[j].s }
+
+type wildcardMapStringyKey_RFC7049 []wildcardMapStringyKey
+
+func (x wildcardMapStringyKey_RFC7049) Len() int      { return len(x) }
+func (x wildcardMapStringyKey_RFC7049) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x wildcardMapStringyKey_RFC7049) Less(i, j int) bool {
+	li, lj := len(x[i].s), len(x[j].s)
+	if li == lj {
+		return x[i].s < x[j].s
+	}
+	return li < lj
+}

vendor/github.com/polydawn/refmt/obj/marshalSlab.go

@@ -0,0 +1,219 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+/*
+	A lovely mechanism to stash marshalMachine objects pre-allocated and avoid mallocs.
+	Works together with the Atlas: the Atlas says what kind of machinery is needed;
+	the marshalSlab "allocates" it and returns it upon your request.
+*/
+type marshalSlab struct {
+	atlas atlas.Atlas
+	rows  []marshalSlabRow
+}
+
+type marshalSlabRow struct {
+	ptrDerefDelegateMarshalMachine
+	marshalMachinePrimitive
+	marshalMachineWildcard
+	marshalMachineMapWildcard
+	marshalMachineSliceWildcard
+	marshalMachineStructAtlas
+	marshalMachineTransform
+	marshalMachineUnionKeyed
+
+	errThunkMarshalMachine
+}
+
+// A thunk value that can be used to trigger `isNil` paths.
+// (Substituting an 'invalid' kind reflect.Value with this is an easy way
+// to emit a null without needing any additional special cases or error handling.)
+var nil_rv reflect.Value = reflect.Zero(reflect.PtrTo(reflect.TypeOf(0)))
+
+/*
+	Return a reference to a machine from the slab.
+	*You must release() when done.*
+
+	Errors -- including "no info in Atlas for this type" -- are expressed by
+	returning a machine that is a constantly-erroring thunk.
+*/
+func (slab *marshalSlab) requisitionMachine(rt reflect.Type) MarshalMachine {
+	// Acquire a row.
+	off := len(slab.rows)
+	slab.grow()
+	row := &slab.rows[off]
+
+	// Yield machinery.
+	return _yieldMarshalMachinePtr(row, slab.atlas, rt)
+}
+
+/*
+	Like requisitionMachine, but does *not* grow the slab; assumes the current
+	tip row is usable.
+	Thus, you must grow() before using, and release correspondingly.
+*/
+func (slab *marshalSlab) yieldMachine(rt reflect.Type) MarshalMachine {
+	// Grab the last row.
+	row := &slab.rows[len(slab.rows)-1]
+
+	// Yield machinery.
+	return _yieldMarshalMachinePtr(row, slab.atlas, rt)
+}
+
+func _yieldMarshalMachinePtr(row *marshalSlabRow, atl atlas.Atlas, rt reflect.Type) MarshalMachine {
+	// Indirect pointers as necessary.
+	//  Keep count of how many times we do this; we'll use this again at the end.
+	peelCount := 0
+	for rt.Kind() == reflect.Ptr {
+		rt = rt.Elem()
+		peelCount++
+	}
+
+	// Figure out what machinery to use at heart.
+	mach := _yieldBareMarshalMachinePtr(row, atl, rt)
+	// If nil answer, we had no match: yield an error thunk.
+	if mach == nil {
+		mach := &row.errThunkMarshalMachine
+		mach.err = fmt.Errorf("no machine found")
+		return mach
+	}
+
+	// If no indirection steps, return;
+	//  otherwise wrap it in the ptrDeref machine and return that.
+	if peelCount == 0 {
+		return mach
+	}
+	row.ptrDerefDelegateMarshalMachine.MarshalMachine = mach
+	row.ptrDerefDelegateMarshalMachine.peelCount = peelCount
+	row.ptrDerefDelegateMarshalMachine.isNil = false
+	return &row.ptrDerefDelegateMarshalMachine
+}
+
+// Like _yieldMarshalMachinePtr, but assumes the ptr unwrapping has already been done.
+func _yieldBareMarshalMachinePtr(row *marshalSlabRow, atl atlas.Atlas, rt reflect.Type) MarshalMachine {
+	rtid := reflect.ValueOf(rt).Pointer()
+
+	// Check primitives first; cheapest (and unoverridable).
+	switch rtid {
+	case rtid_bool,
+		rtid_string,
+		rtid_int, rtid_int8, rtid_int16, rtid_int32, rtid_int64,
+		rtid_uint, rtid_uint8, rtid_uint16, rtid_uint32, rtid_uint64, rtid_uintptr,
+		rtid_float32, rtid_float64,
+		rtid_bytes:
+		row.marshalMachinePrimitive.kind = rt.Kind()
+		return &row.marshalMachinePrimitive
+	}
+
+	// Consult atlas second.
+	if entry, ok := atl.Get(rtid); ok {
+		return _yieldMarshalMachinePtrForAtlasEntry(row, entry, atl)
+	}
+
+	// If no specific behavior found, use default behavior based on kind.
+	switch rt.Kind() {
+	case reflect.Bool,
+		reflect.String,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+		reflect.Float32, reflect.Float64:
+		row.marshalMachinePrimitive.kind = rt.Kind()
+		return &row.marshalMachinePrimitive
+	case reflect.Slice:
+		// un-typedef'd byte slices were handled already, but a typedef'd one still gets gets treated like a special kind:
+		if rt.Elem().Kind() == reflect.Uint8 {
+			row.marshalMachinePrimitive.kind = rt.Kind()
+			return &row.marshalMachinePrimitive
+		}
+		return &row.marshalMachineSliceWildcard
+	case reflect.Array:
+		// arrays of bytes have a similar special case to slices for when they're typedefed.
+		if rt.Elem().Kind() == reflect.Uint8 {
+			row.marshalMachinePrimitive.kind = rt.Kind()
+			return &row.marshalMachinePrimitive
+		}
+		return &row.marshalMachineSliceWildcard.marshalMachineArrayWildcard
+	case reflect.Map:
+		row.marshalMachineMapWildcard.morphism = atl.GetDefaultMapMorphism()
+		return &row.marshalMachineMapWildcard
+	case reflect.Struct:
+		// TODO here we could also invoke automatic atlas autogen, if configured to be permitted
+		mach := &row.errThunkMarshalMachine
+		mach.err = fmt.Errorf("missing an atlas entry describing how to marshal type %v (and auto-atlasing for structs is not enabled)", rt)
+		return mach
+	case reflect.Interface:
+		return &row.marshalMachineWildcard
+	case reflect.Func:
+		panic(fmt.Errorf("functions cannot be marshalled!"))
+	case reflect.Ptr:
+		panic(fmt.Errorf("unreachable: ptrs must already be resolved"))
+	default:
+		panic(fmt.Errorf("excursion %s", rt.Kind()))
+	}
+}
+
+// given that we already have an atlasEntry in mind, yield a configured machine for it.
+// it seems odd that this might still require a whole atlas, but tis so;
+// some things (e.g. transform funcs) need to get additional machinery for delegation.
+func _yieldMarshalMachinePtrForAtlasEntry(row *marshalSlabRow, entry *atlas.AtlasEntry, atl atlas.Atlas) MarshalMachine {
+	// Switch across which of the union of configurations is applicable.
+	switch {
+	case entry.MarshalTransformFunc != nil:
+		// Return a machine that calls the func(s), then later a real machine.
+		// The entry.MarshalTransformTargetType is used to do a recursive lookup.
+		// We can't just call the func here because we're still working off typeinfo
+		// and don't have a real value to transform until later.
+		row.marshalMachineTransform.trFunc = entry.MarshalTransformFunc
+		// Pick delegate without growing stack.  (This currently means recursive transform won't fly.)
+		row.marshalMachineTransform.delegate = _yieldMarshalMachinePtr(row, atl, entry.MarshalTransformTargetType)
+		// If tags are in play: have the transformer machine glue that on.
+
+		row.marshalMachineTransform.tagged = entry.Tagged
+		row.marshalMachineTransform.tag = entry.Tag
+		return &row.marshalMachineTransform
+	case entry.StructMap != nil:
+		row.marshalMachineStructAtlas.cfg = entry
+		return &row.marshalMachineStructAtlas
+	case entry.UnionKeyedMorphism != nil:
+		row.marshalMachineUnionKeyed.cfg = entry
+		return &row.marshalMachineUnionKeyed
+	case entry.MapMorphism != nil:
+		row.marshalMachineMapWildcard.morphism = entry.MapMorphism
+		return &row.marshalMachineMapWildcard
+	default:
+		panic("invalid atlas entry")
+	}
+}
+
+// Returns the top row of the slab.  Useful for machines that need to delegate
+// to another type that's definitely not their own.  Be careful with that
+// caveat; if the delegation can be to another system that uses in-row delegation,
+// this is not trivially safe to compose and you should grow the slab instead.
+func (s *marshalSlab) tip() *marshalSlabRow {
+	return &s.rows[len(s.rows)-1]
+}
+
+func (s *marshalSlab) grow() {
+	s.rows = append(s.rows, marshalSlabRow{})
+}
+
+func (s *marshalSlab) release() {
+	s.rows = s.rows[0 : len(s.rows)-1]
+}
+
+type errThunkMarshalMachine struct {
+	err error
+}
+
+func (m *errThunkMarshalMachine) Reset(_ *marshalSlab, _ reflect.Value, _ reflect.Type) error {
+	return m.err
+}
+func (m *errThunkMarshalMachine) Step(d *Marshaller, s *marshalSlab, tok *Token) (done bool, err error) {
+	return true, m.err
+}

vendor/github.com/polydawn/refmt/obj/marshalSliceWildcard.go

@@ -0,0 +1,63 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+// Encodes a slice.
+// This machine just wraps the array machine, checking to make sure the value isn't nil.
+type marshalMachineSliceWildcard struct {
+	marshalMachineArrayWildcard
+}
+
+func (mach *marshalMachineSliceWildcard) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	if mach.index < 0 {
+		if mach.target_rv.IsNil() {
+			tok.Type = TNull
+			return true, nil
+		}
+	}
+	return mach.marshalMachineArrayWildcard.Step(driver, slab, tok)
+}
+
+type marshalMachineArrayWildcard struct {
+	target_rv reflect.Value
+	value_rt  reflect.Type
+	valueMach MarshalMachine
+	index     int
+	length    int
+}
+
+func (mach *marshalMachineArrayWildcard) Reset(slab *marshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.value_rt = rt.Elem()
+	mach.valueMach = slab.requisitionMachine(mach.value_rt)
+	mach.index = -1
+	mach.length = mach.target_rv.Len()
+	return nil
+}
+
+func (mach *marshalMachineArrayWildcard) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	if mach.index < 0 {
+		tok.Type = TArrOpen
+		tok.Length = mach.target_rv.Len()
+		mach.index++
+		return false, nil
+	}
+	if mach.index == mach.length {
+		tok.Type = TArrClose
+		mach.index++
+		slab.release()
+		return true, nil
+	}
+	if mach.index > mach.length {
+		return true, fmt.Errorf("invalid state: value already consumed")
+	}
+	rv := mach.target_rv.Index(mach.index)
+	driver.Recurse(tok, rv, mach.value_rt, mach.valueMach)
+	mach.index++
+	return false, nil
+}

vendor/github.com/polydawn/refmt/obj/marshalStruct.go

@@ -0,0 +1,108 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type marshalMachineStructAtlas struct {
+	cfg *atlas.AtlasEntry // set on initialization
+
+	target_rv reflect.Value
+	index     int           // Progress marker
+	value_rv  reflect.Value // Next value (or nil if next step is key).
+}
+
+func (mach *marshalMachineStructAtlas) Reset(slab *marshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.target_rv = rv
+	mach.index = -1
+	mach.value_rv = reflect.Value{}
+	slab.grow() // we'll reuse the same row for all fields
+	return nil
+}
+
+func (mach *marshalMachineStructAtlas) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	//fmt.Printf("--step on %#v: i=%d/%d v=%v\n", mach.target_rv, mach.index, len(mach.cfg.Fields), mach.value)
+
+	// Check boundaries and do the special steps or either start or end.
+	nEntries := len(mach.cfg.StructMap.Fields)
+	if mach.index < 0 {
+		tok.Type = TMapOpen
+		tok.Length = countEmittableStructFields(mach.cfg, mach.target_rv)
+		tok.Tagged = mach.cfg.Tagged
+		tok.Tag = mach.cfg.Tag
+		mach.index++
+		return false, nil
+	}
+	if mach.index == nEntries {
+		tok.Type = TMapClose
+		mach.index++
+		slab.release()
+		return true, nil
+	}
+	if mach.index > nEntries {
+		return true, fmt.Errorf("invalid state: entire struct (%d fields) already consumed", nEntries)
+	}
+
+	// If value loaded from last step, recurse into handling that.
+	fieldEntry := mach.cfg.StructMap.Fields[mach.index]
+	if mach.value_rv != (reflect.Value{}) {
+		child_rv := mach.value_rv
+		mach.index++
+		mach.value_rv = reflect.Value{}
+		return false, driver.Recurse(
+			tok,
+			child_rv,
+			fieldEntry.Type,
+			slab.yieldMachine(fieldEntry.Type),
+		)
+	}
+
+	// If value was nil, that indicates we're supposed to pick the value and yield a key.
+	//  We have to look ahead to the value because if it's zero and tagged as
+	//  omitEmpty, then we have to skip emitting the key as well.
+	for fieldEntry.Ignore {
+		mach.index++
+		if mach.index == nEntries {
+			tok.Type = TMapClose
+			mach.index++
+			slab.release()
+			return true, nil
+		}
+		fieldEntry = mach.cfg.StructMap.Fields[mach.index]
+	}
+	mach.value_rv = fieldEntry.ReflectRoute.TraverseToValue(mach.target_rv)
+	if fieldEntry.OmitEmpty && isEmptyValue(mach.value_rv) {
+		mach.value_rv = reflect.Value{}
+		mach.index++
+		return mach.Step(driver, slab, tok)
+	}
+	tok.Type = TString
+	tok.Str = fieldEntry.SerialName
+	return false, nil
+}
+
+// Count how many fields in a struct should actually be marshalled.
+// Fields that are tagged omitEmpty and are isEmptyValue are not counted, and
+// StructMapEntry used to flag ignored fields unmarshalling never count, so
+// this number may be less than the number of fields in the AtlasEntry.StructMap.
+func countEmittableStructFields(cfg *atlas.AtlasEntry, target_rv reflect.Value) int {
+	total := 0
+	for _, fieldEntry := range cfg.StructMap.Fields {
+		if fieldEntry.Ignore {
+			continue
+		}
+		if !fieldEntry.OmitEmpty {
+			total++
+			continue
+		}
+		if !isEmptyValue(fieldEntry.ReflectRoute.TraverseToValue(target_rv)) {
+			total++
+			continue
+		}
+	}
+	return total
+}

vendor/github.com/polydawn/refmt/obj/marshalTransform.go

@@ -0,0 +1,35 @@
+package obj
+
+import (
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type marshalMachineTransform struct {
+	trFunc   atlas.MarshalTransformFunc
+	delegate MarshalMachine
+	tagged   bool // Used to apply tag to first step (without forcing delegate to know).
+	tag      int
+	first    bool // This resets; 'tagged' persists (because it's type info).
+}
+
+func (mach *marshalMachineTransform) Reset(slab *marshalSlab, rv reflect.Value, _ reflect.Type) error {
+	tr_rv, err := mach.trFunc(rv)
+	if err != nil {
+		return err
+	}
+	mach.first = true
+	return mach.delegate.Reset(slab, tr_rv, tr_rv.Type())
+}
+
+func (mach *marshalMachineTransform) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	done, err = mach.delegate.Step(driver, slab, tok)
+	if mach.first && mach.tagged {
+		tok.Tagged = true
+		tok.Tag = mach.tag
+		mach.first = false
+	}
+	return
+}

vendor/github.com/polydawn/refmt/obj/marshalUnionKeyed.go

@@ -0,0 +1,71 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type marshalMachineUnionKeyed struct {
+	cfg *atlas.AtlasEntry // set on initialization
+
+	target_rv   reflect.Value // the element (interface already unwrapped).
+	elementName string        // the serial name for this union member type.
+
+	step     marshalMachineStep
+	delegate MarshalMachine // actual machine, picked based on content of the interface.
+}
+
+func (mach *marshalMachineUnionKeyed) Reset(slab *marshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv.Elem()
+	if mach.target_rv.Kind() == reflect.Invalid {
+		return fmt.Errorf("nil is not a valid member for the union for interface %q", mach.cfg.Type.Name())
+	}
+	element_rt := mach.target_rv.Type()
+	mach.elementName = mach.cfg.UnionKeyedMorphism.Mappings[reflect.ValueOf(element_rt).Pointer()]
+	if mach.elementName == "" {
+		return fmt.Errorf("type %q is not one of the known members of the union for interface %q", element_rt.Name(), mach.cfg.Type.Name())
+	}
+	delegateAtlasEnt := mach.cfg.UnionKeyedMorphism.Elements[mach.elementName]
+	mach.delegate = _yieldMarshalMachinePtrForAtlasEntry(slab.tip(), delegateAtlasEnt, slab.atlas)
+	if err := mach.delegate.Reset(slab, mach.target_rv, delegateAtlasEnt.Type); err != nil {
+		return err
+	}
+	mach.step = mach.step_emitMapOpen
+	return nil
+}
+
+func (mach *marshalMachineUnionKeyed) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	return mach.step(driver, slab, tok)
+}
+
+func (mach *marshalMachineUnionKeyed) step_emitMapOpen(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	tok.Type = TMapOpen
+	tok.Length = 1
+	mach.step = mach.step_emitKey
+	return false, nil
+}
+
+func (mach *marshalMachineUnionKeyed) step_emitKey(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	tok.Type = TString
+	tok.Str = mach.elementName
+	mach.step = mach.step_delegate
+	return false, nil
+}
+
+func (mach *marshalMachineUnionKeyed) step_delegate(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	done, err = mach.delegate.Step(driver, slab, tok)
+	if done && err == nil {
+		mach.step = mach.step_emitMapClose
+		return false, nil
+	}
+	return
+}
+
+func (mach *marshalMachineUnionKeyed) step_emitMapClose(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	tok.Type = TMapClose
+	mach.step = nil
+	return true, nil
+}

vendor/github.com/polydawn/refmt/obj/marshalWildcard.go

@@ -0,0 +1,37 @@
+package obj
+
+import (
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+/*
+	A MarshalMachine that unwraps an `interface{}` value,
+	selects the correct machinery for handling its content,
+	and delegates immediately to that machine.
+*/
+type marshalMachineWildcard struct {
+	delegate MarshalMachine
+}
+
+func (mach *marshalMachineWildcard) Reset(slab *marshalSlab, rv reflect.Value, rt reflect.Type) error {
+	// If the interface contains nil, go no further; we'll simply yield that single token.
+	if rv.IsNil() {
+		mach.delegate = nil
+		return nil
+	}
+	// Pick, reset, and retain a delegate machine for the interior type.
+	unwrap_rv := rv.Elem() // unwrap iface
+	unwrap_rt := unwrap_rv.Type()
+	mach.delegate = slab.requisitionMachine(unwrap_rt)
+	return mach.delegate.Reset(slab, unwrap_rv, unwrap_rt)
+}
+
+func (mach marshalMachineWildcard) Step(driver *Marshaller, slab *marshalSlab, tok *Token) (done bool, err error) {
+	if mach.delegate == nil {
+		tok.Type = TNull
+		return true, nil
+	}
+	return mach.delegate.Step(driver, slab, tok)
+}

vendor/github.com/polydawn/refmt/obj/unmarshal.go

@@ -0,0 +1,106 @@
+package obj
+
+import (
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+/*
+	Allocates the machinery for treating an in-memory object like a `TokenSink`.
+	This machinery will walk over values,	using received tokens to fill in
+	fields as it visits them.
+
+	Initialization must be finished by calling `Bind` to set the value to visit;
+	after this, the `Step` function is ready to be pumped.
+	Subsequent calls to `Bind` do a full reset, leaving `Step` ready to call
+	again and making all of the machinery reusable without re-allocating.
+*/
+func NewUnmarshaller(atl atlas.Atlas) *Unmarshaller {
+	d := &Unmarshaller{
+		unmarshalSlab: unmarshalSlab{
+			atlas: atl,
+			rows:  make([]unmarshalSlabRow, 0, 10),
+		},
+		stack: make([]UnmarshalMachine, 0, 10),
+	}
+	return d
+}
+
+func (d *Unmarshaller) Bind(v interface{}) error {
+	d.stack = d.stack[0:0]
+	d.unmarshalSlab.rows = d.unmarshalSlab.rows[0:0]
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr || rv.IsNil() {
+		err := ErrInvalidUnmarshalTarget{reflect.TypeOf(v)}
+		d.step = &errThunkUnmarshalMachine{err}
+		return err
+	}
+	rv = rv.Elem() // Let's just always be addressible, shall we?
+	rt := rv.Type()
+	d.step = d.unmarshalSlab.requisitionMachine(rt)
+	return d.step.Reset(&d.unmarshalSlab, rv, rt)
+}
+
+type Unmarshaller struct {
+	unmarshalSlab unmarshalSlab
+	stack         []UnmarshalMachine
+	step          UnmarshalMachine
+}
+
+type UnmarshalMachine interface {
+	Reset(*unmarshalSlab, reflect.Value, reflect.Type) error
+	Step(*Unmarshaller, *unmarshalSlab, *Token) (done bool, err error)
+}
+
+func (d *Unmarshaller) Step(tok *Token) (bool, error) {
+	done, err := d.step.Step(d, &d.unmarshalSlab, tok)
+	// If the step errored: out, entirely.
+	if err != nil {
+		return true, err
+	}
+	// If the step wasn't done, return same status.
+	if !done {
+		return false, nil
+	}
+	// If it WAS done, pop next, or if stack empty, we're entirely done.
+	nSteps := len(d.stack) - 1
+	if nSteps == -1 {
+		return true, nil // that's all folks
+	}
+	d.step = d.stack[nSteps]
+	d.stack = d.stack[0:nSteps]
+	return false, nil
+}
+
+/*
+	Starts the process of recursing unmarshalling over value `rv`.
+
+	Caller provides the machine to use (this is an optimization for maps and slices,
+	which already know the machine and keep reusing it for all their entries).
+	This method pushes the first step with `tok` (the upstream tends to have peeked at
+	it in order to decide what to do, but if recursing, it belongs to the next obj),
+	then saves this new machine onto the driver's stack: future calls to step
+	the driver will then continuing stepping the new machine it returns a done status,
+	at which point we'll finally "return" by popping back to the last machine on the stack
+	(which is presumably the same one that just called this Recurse method).
+
+	In other words, your UnmarshalMachine calls this when it wants to deal
+	with an object, and by the time we call back to your machine again,
+	that object will be traversed and the stream ready for you to continue.
+*/
+func (d *Unmarshaller) Recurse(tok *Token, rv reflect.Value, rt reflect.Type, nextMach UnmarshalMachine) (err error) {
+	//	fmt.Printf(">>> pushing into recursion with %#v\n", nextMach)
+	// Push the current machine onto the stack (we'll resume it when the new one is done),
+	d.stack = append(d.stack, d.step)
+	// Initialize the machine for this new target value.
+	err = nextMach.Reset(&d.unmarshalSlab, rv, rt)
+	if err != nil {
+		return
+	}
+	d.step = nextMach
+	// Immediately make a step (we're still the delegate in charge of someone else's step).
+	_, err = d.Step(tok)
+	return
+}

vendor/github.com/polydawn/refmt/obj/unmarshalArrayWildcard.go

@@ -0,0 +1,93 @@
+package obj
+
+import (
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineArrayWildcard struct {
+	target_rv reflect.Value
+	value_rt  reflect.Type
+	valueMach UnmarshalMachine
+	phase     unmarshalMachineArrayWildcardPhase
+	index     int
+	maxLen    int
+}
+
+type unmarshalMachineArrayWildcardPhase uint8
+
+const (
+	unmarshalMachineArrayWildcardPhase_initial unmarshalMachineArrayWildcardPhase = iota
+	unmarshalMachineArrayWildcardPhase_acceptValueOrClose
+)
+
+func (mach *unmarshalMachineArrayWildcard) Reset(slab *unmarshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.value_rt = rt.Elem()
+	mach.valueMach = slab.requisitionMachine(mach.value_rt)
+	mach.phase = unmarshalMachineArrayWildcardPhase_initial
+	mach.index = 0
+	mach.maxLen = rt.Len()
+	return nil
+}
+
+func (mach *unmarshalMachineArrayWildcard) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch mach.phase {
+	case unmarshalMachineArrayWildcardPhase_initial:
+		return mach.step_Initial(driver, slab, tok)
+	case unmarshalMachineArrayWildcardPhase_acceptValueOrClose:
+		return mach.step_AcceptValueOrClose(driver, slab, tok)
+	}
+	panic("unreachable")
+}
+
+func (mach *unmarshalMachineArrayWildcard) step_Initial(_ *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// If it's a special state, start an object.
+	//  (Or, blow up if its a special state that's silly).
+	switch tok.Type {
+	case TMapOpen:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TArrOpen:
+		// Great.  Consumed.
+		mach.phase = unmarshalMachineArrayWildcardPhase_acceptValueOrClose
+		// Initialize the array.  Its length is encoded in its type.
+		mach.target_rv.Set(reflect.Zero(mach.target_rv.Type()))
+		return false, nil
+	case TMapClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TArrClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TNull:
+		mach.target_rv.Set(reflect.Zero(mach.target_rv.Type()))
+		return true, nil
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	}
+}
+
+func (mach *unmarshalMachineArrayWildcard) step_AcceptValueOrClose(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// Either form of open token are valid, but
+	// - an arrClose is ours
+	// - and a mapClose is clearly invalid.
+	switch tok.Type {
+	case TMapClose:
+		// no special checks for ends of wildcard slice; no such thing as incomplete.
+		return true, ErrMalformedTokenStream{tok.Type, "start of value or end of array"}
+	case TArrClose:
+		// release the slab row we requisitioned for our value machine.
+		slab.release()
+		return true, nil
+	}
+
+	// Return an error if we're about to exceed our length limit.
+	if mach.index >= mach.maxLen {
+		return true, ErrMalformedTokenStream{tok.Type, "end of array (out of space)"}
+	}
+
+	// Recurse on a handle to the next index.
+	rv := mach.target_rv.Index(mach.index)
+	mach.index++
+	return false, driver.Recurse(tok, rv, mach.value_rt, mach.valueMach)
+	// Step simply remains `step_AcceptValueOrClose` -- arrays don't have much state machine.
+}

vendor/github.com/polydawn/refmt/obj/unmarshalBuiltins.go

@@ -0,0 +1,176 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type ptrDerefDelegateUnmarshalMachine struct {
+	UnmarshalMachine     // a delegate machine, already set for us by the slab
+	peelCount        int // how deep the pointers go, already discoverd by the slab
+
+	ptr_rv    reflect.Value // the top ptr, which we will use if setting to nil, or if we have to recursively make ptrs for `**X` types.
+	firstStep bool
+}
+
+func (mach *ptrDerefDelegateUnmarshalMachine) Reset(slab *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.ptr_rv = rv
+	mach.firstStep = true
+	// we defer reseting the delegate machine until later, in case we get a nil, which can save a lot of time.
+	return nil
+}
+func (mach *ptrDerefDelegateUnmarshalMachine) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// If first step: we have to do initializations.
+	if mach.firstStep {
+		mach.firstStep = false
+		// If nil: easy road.  Nil the ptr.
+		if tok.Type == TNull {
+			mach.ptr_rv.Set(reflect.Zero(mach.ptr_rv.Type()))
+			return true, nil
+		}
+		// Walk the pointers: if some already exist, we accept them unmodified;
+		//  if any are nil, make a new one, and recursively.
+		rv := mach.ptr_rv
+		for i := 0; i < mach.peelCount; i++ {
+			if rv.IsNil() {
+				rv.Set(reflect.New(rv.Type().Elem()))
+				rv = rv.Elem()
+			} else {
+				rv = rv.Elem()
+			}
+		}
+		if err := mach.UnmarshalMachine.Reset(slab, rv, rv.Type()); err != nil {
+			return true, err
+		}
+	}
+	// The remainder of the time: it's just delegation.
+	return mach.UnmarshalMachine.Step(driver, slab, tok)
+}
+
+type unmarshalMachinePrimitive struct {
+	kind reflect.Kind
+
+	rv reflect.Value
+}
+
+func (mach *unmarshalMachinePrimitive) Reset(_ *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.rv = rv
+	return nil
+}
+func (mach *unmarshalMachinePrimitive) Step(_ *Unmarshaller, _ *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch mach.kind {
+	case reflect.Bool:
+		switch tok.Type {
+		case TBool:
+			mach.rv.SetBool(tok.Bool)
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.String:
+		switch tok.Type {
+		case TString:
+			mach.rv.SetString(tok.Str)
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		switch tok.Type {
+		case TInt:
+			mach.rv.SetInt(tok.Int)
+			return true, nil
+		case TUint:
+			mach.rv.SetInt(int64(tok.Uint)) // todo: overflow check
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		switch tok.Type {
+		case TInt:
+			if tok.Int >= 0 {
+				mach.rv.SetUint(uint64(tok.Int))
+				return true, nil
+			}
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		case TUint:
+			mach.rv.SetUint(tok.Uint)
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Float32, reflect.Float64:
+		switch tok.Type {
+		case TFloat64:
+			mach.rv.SetFloat(tok.Float64)
+			return true, nil
+		case TInt:
+			mach.rv.SetFloat(float64(tok.Int))
+			return true, nil
+		case TUint:
+			mach.rv.SetFloat(float64(tok.Uint))
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Slice: // implicitly bytes; no other slices are "primitive"
+		switch tok.Type {
+		case TBytes:
+			mach.rv.SetBytes(tok.Bytes)
+			return true, nil
+		case TNull:
+			mach.rv.SetBytes(nil)
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Array: // implicitly bytes; no other arrays are "primitive"
+		switch tok.Type {
+		case TBytes:
+			// Unfortunately, there does not seem to be any efficient way to bulk set the contents of a byte array via reflect.
+			// There are similar complaints regarding slices on the marshalling side: apparently, we have no choice but to loop.
+			n := mach.rv.Len()
+			// We match aggressively on length.  If the provided input is too short, we reject that too: we assume you asked for a fixed-length array for a reason.
+			if len(tok.Bytes) != n {
+				return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, n}
+			}
+			for i := 0; i < n; i++ {
+				mach.rv.Index(i).SetUint(uint64(tok.Bytes[i]))
+			}
+			return true, nil
+		case TNull:
+			if mach.rv.Len() != 0 {
+				return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+			}
+			mach.rv.SetBytes(nil)
+			return true, nil
+		default:
+			return true, ErrUnmarshalTypeCantFit{*tok, mach.rv, 0}
+		}
+	case reflect.Interface:
+		switch tok.Type {
+		case TString:
+			mach.rv.Set(reflect.ValueOf(tok.Str))
+		case TBytes:
+			mach.rv.Set(reflect.ValueOf(tok.Bytes))
+		case TBool:
+			mach.rv.Set(reflect.ValueOf(tok.Bool))
+		case TInt:
+			mach.rv.Set(reflect.ValueOf(int(tok.Int))) // Unmarshalling with no particular type info should default to using plain 'int' whenever viable.
+		case TUint:
+			mach.rv.Set(reflect.ValueOf(int(tok.Uint))) // Unmarshalling with no particular type info should default to using plain 'int' whenever viable.
+		case TFloat64:
+			mach.rv.Set(reflect.ValueOf(tok.Float64))
+		case TNull:
+			mach.rv.Set(reflect.ValueOf(nil))
+		default: // any of the other token types should not have been routed here to begin with.
+			panic(fmt.Errorf("unhandled: %v", mach.kind))
+		}
+		return true, nil
+	default:
+		panic(fmt.Errorf("unhandled: %v", mach.kind))
+	}
+}

vendor/github.com/polydawn/refmt/obj/unmarshalMapWildcard.go

@@ -0,0 +1,151 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineMapStringWildcard struct {
+	target_rv     reflect.Value                // Handle to the map.  Can set to zero, or set k=v pairs into, etc.
+	value_rt      reflect.Type                 // Type info for map values (cached for convenience in recurse calls).
+	valueMach     UnmarshalMachine             // Machine for map values.
+	valueZero_rv  reflect.Value                // Cached instance of the zero value of the value type, for re-zeroing tmp_rv.
+	key_rv        reflect.Value                // Addressable handle to a slot for keys to unmarshal into.
+	keyDestringer atlas.UnmarshalTransformFunc // Transform str->foo, to be used if keys are not plain strings.
+	tmp_rv        reflect.Value                // Addressable handle to a slot for values to unmarshal into.
+	phase         unmarshalMachineMapStringWildcardPhase
+}
+
+type unmarshalMachineMapStringWildcardPhase uint8
+
+const (
+	unmarshalMachineMapStringWildcardPhase_initial          unmarshalMachineMapStringWildcardPhase = iota
+	unmarshalMachineMapStringWildcardPhase_acceptKeyOrClose                                        // doesn't commit prev value
+	unmarshalMachineMapStringWildcardPhase_acceptValue
+	unmarshalMachineMapStringWildcardPhase_acceptAnotherKeyOrClose
+)
+
+func (mach *unmarshalMachineMapStringWildcard) Reset(slab *unmarshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.value_rt = rt.Elem()
+	mach.valueMach = slab.requisitionMachine(mach.value_rt)
+	mach.valueZero_rv = reflect.Zero(mach.value_rt)
+	key_rt := rt.Key()
+	mach.key_rv = reflect.New(key_rt).Elem()
+	if mach.key_rv.Kind() != reflect.String {
+		rtid := reflect.ValueOf(key_rt).Pointer()
+		atlEnt, ok := slab.atlas.Get(rtid)
+		if !ok || atlEnt.UnmarshalTransformTargetType == nil || atlEnt.UnmarshalTransformTargetType.Kind() != reflect.String {
+			return fmt.Errorf("unsupported map key type %q (if you want to use struct keys, your atlas needs a transform from string)", key_rt.Name())
+		}
+		mach.keyDestringer = atlEnt.UnmarshalTransformFunc
+	}
+	mach.tmp_rv = reflect.New(mach.value_rt).Elem()
+	mach.phase = unmarshalMachineMapStringWildcardPhase_initial
+	return nil
+}
+
+func (mach *unmarshalMachineMapStringWildcard) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch mach.phase {
+	case unmarshalMachineMapStringWildcardPhase_initial:
+		return mach.step_Initial(driver, slab, tok)
+	case unmarshalMachineMapStringWildcardPhase_acceptKeyOrClose:
+		return mach.step_AcceptKeyOrClose(driver, slab, tok)
+	case unmarshalMachineMapStringWildcardPhase_acceptValue:
+		return mach.step_AcceptValue(driver, slab, tok)
+	case unmarshalMachineMapStringWildcardPhase_acceptAnotherKeyOrClose:
+		return mach.step_AcceptAnotherKeyOrClose(driver, slab, tok)
+	}
+	panic("unreachable")
+}
+
+func (mach *unmarshalMachineMapStringWildcard) step_Initial(_ *Unmarshaller, _ *unmarshalSlab, tok *Token) (done bool, err error) {
+	// If it's a special state, start an object.
+	//  (Or, blow up if its a special state that's silly).
+	switch tok.Type {
+	case TNull:
+		mach.target_rv.Set(reflect.Zero(mach.target_rv.Type()))
+		return true, nil
+	case TMapOpen:
+		// Great.  Consumed.
+		mach.phase = unmarshalMachineMapStringWildcardPhase_acceptKeyOrClose
+		// Initialize the map if it's nil.
+		if mach.target_rv.IsNil() {
+			mach.target_rv.Set(reflect.MakeMap(mach.target_rv.Type()))
+		}
+		return false, nil
+	case TMapClose:
+		return true, fmt.Errorf("unexpected mapClose; expected start of map")
+	case TArrClose:
+		return true, fmt.Errorf("unexpected arrClose; expected start of map")
+	case TArrOpen:
+		fallthrough
+	default:
+		return true, ErrUnmarshalTypeCantFit{*tok, mach.target_rv, 0}
+	}
+}
+
+func (mach *unmarshalMachineMapStringWildcard) step_AcceptKeyOrClose(_ *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// Switch on tokens.
+	switch tok.Type {
+	case TMapOpen:
+		return true, fmt.Errorf("unexpected mapOpen; expected map key")
+	case TArrOpen:
+		return true, fmt.Errorf("unexpected arrOpen; expected map key")
+	case TMapClose:
+		// no special checks for ends of wildcard map; no such thing as incomplete.
+		// release the slab row we requisitioned for our value machine.
+		slab.release()
+		return true, nil
+	case TArrClose:
+		return true, fmt.Errorf("unexpected arrClose; expected map key")
+	case TString:
+		if mach.keyDestringer != nil {
+			key_rv, err := mach.keyDestringer(reflect.ValueOf(tok.Str))
+			if err != nil {
+				return true, fmt.Errorf("unsupported map key type %q: errors in stringifying: %s", mach.key_rv.Type().Name(), err)
+			}
+			mach.key_rv.Set(key_rv)
+		} else {
+			mach.key_rv.SetString(tok.Str)
+		}
+		if err = mach.mustAcceptKey(mach.key_rv); err != nil {
+			return true, err
+		}
+		mach.phase = unmarshalMachineMapStringWildcardPhase_acceptValue
+		return false, nil
+	default:
+		return true, fmt.Errorf("unexpected token %s; expected key string or end of map", tok)
+	}
+}
+
+func (mach *unmarshalMachineMapStringWildcard) mustAcceptKey(key_rv reflect.Value) error {
+	if exists := mach.target_rv.MapIndex(key_rv).IsValid(); exists {
+		return fmt.Errorf("repeated key %q", key_rv)
+	}
+	return nil
+}
+
+func (mach *unmarshalMachineMapStringWildcard) step_AcceptValue(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	mach.phase = unmarshalMachineMapStringWildcardPhase_acceptAnotherKeyOrClose
+	mach.tmp_rv.Set(mach.valueZero_rv)
+	return false, driver.Recurse(
+		tok,
+		mach.tmp_rv,
+		mach.value_rt,
+		mach.valueMach,
+	)
+}
+
+func (mach *unmarshalMachineMapStringWildcard) step_AcceptAnotherKeyOrClose(_ *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// First, save any refs from the last value.
+	//  (This is fiddly: the delay comes mostly from the handling of slices, which may end up re-allocating
+	//   themselves during their decoding.)
+	mach.target_rv.SetMapIndex(mach.key_rv, mach.tmp_rv)
+
+	// The rest is the same as the very first acceptKeyOrClose (and has the same future state transitions).
+	return mach.step_AcceptKeyOrClose(nil, slab, tok)
+}

vendor/github.com/polydawn/refmt/obj/unmarshalSlab.go

@@ -0,0 +1,187 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+/*
+	A lovely mechanism to stash unmarshalMachine objects pre-allocated and avoid mallocs.
+	Works together with the Atlas: the Atlas says what kind of machinery is needed;
+	the unmarshalSlab "allocates" it and returns it upon your request.
+*/
+type unmarshalSlab struct {
+	atlas atlas.Atlas
+	rows  []unmarshalSlabRow
+}
+
+type unmarshalSlabRow struct {
+	ptrDerefDelegateUnmarshalMachine
+	unmarshalMachinePrimitive
+	unmarshalMachineWildcard
+	unmarshalMachineMapStringWildcard
+	unmarshalMachineSliceWildcard
+	unmarshalMachineArrayWildcard
+	unmarshalMachineStructAtlas
+	unmarshalMachineTransform
+	unmarshalMachineUnionKeyed
+
+	errThunkUnmarshalMachine
+}
+
+/*
+	Return a reference to a machine from the slab.
+	*You must release() when done.*
+
+	Errors -- including "no info in Atlas for this type" -- are expressed by
+	returning a machine that is a constantly-erroring thunk.
+*/
+func (slab *unmarshalSlab) requisitionMachine(rt reflect.Type) UnmarshalMachine {
+	// Acquire a row.
+	off := len(slab.rows)
+	slab.grow()
+	row := &slab.rows[off]
+
+	// Indirect pointers as necessary.
+	//  Keep count of how many times we do this; we'll use this again at the end.
+	peelCount := 0
+	for rt.Kind() == reflect.Ptr {
+		rt = rt.Elem()
+		peelCount++
+	}
+
+	// Figure out what machinery to use at heart.
+	mach := _yieldUnmarshalMachinePtr(row, slab.atlas, rt)
+	// If nil answer, we had no match: yield an error thunk.
+	if mach == nil {
+		mach := &row.errThunkUnmarshalMachine
+		mach.err = fmt.Errorf("no machine found")
+		return mach
+	}
+
+	// If no indirection steps, return;
+	//  otherwise wrap it in the ptrDeref machine and return that.
+	if peelCount == 0 {
+		return mach
+	}
+	row.ptrDerefDelegateUnmarshalMachine.UnmarshalMachine = mach
+	row.ptrDerefDelegateUnmarshalMachine.peelCount = peelCount
+	return &row.ptrDerefDelegateUnmarshalMachine
+}
+
+func _yieldUnmarshalMachinePtr(row *unmarshalSlabRow, atl atlas.Atlas, rt reflect.Type) UnmarshalMachine {
+	rtid := reflect.ValueOf(rt).Pointer()
+
+	// Check primitives first; cheapest (and unoverridable).
+	switch rtid {
+	case rtid_bool,
+		rtid_string,
+		rtid_int, rtid_int8, rtid_int16, rtid_int32, rtid_int64,
+		rtid_uint, rtid_uint8, rtid_uint16, rtid_uint32, rtid_uint64, rtid_uintptr,
+		rtid_float32, rtid_float64,
+		rtid_bytes:
+		row.unmarshalMachinePrimitive.kind = rt.Kind()
+		return &row.unmarshalMachinePrimitive
+	}
+
+	// Consult atlas second.
+	if entry, ok := atl.Get(rtid); ok {
+		return _yieldUnmarshalMachinePtrForAtlasEntry(row, entry, atl)
+	}
+
+	// If no specific behavior found, use default behavior based on kind.
+	switch rt.Kind() {
+
+	case reflect.Bool,
+		reflect.String,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+		reflect.Float32, reflect.Float64:
+		row.unmarshalMachinePrimitive.kind = rt.Kind()
+		return &row.unmarshalMachinePrimitive
+	case reflect.Slice:
+		// un-typedef'd byte slices were handled already, but a typedef'd one still gets gets treated like a special kind:
+		if rt.Elem().Kind() == reflect.Uint8 {
+			row.unmarshalMachinePrimitive.kind = rt.Kind()
+			return &row.unmarshalMachinePrimitive
+		}
+		return &row.unmarshalMachineSliceWildcard
+	case reflect.Array:
+		// arrays of bytes have a similar special case to slices for when they're typedefed.
+		if rt.Elem().Kind() == reflect.Uint8 {
+			row.unmarshalMachinePrimitive.kind = rt.Kind()
+			return &row.unmarshalMachinePrimitive
+		}
+		return &row.unmarshalMachineArrayWildcard
+	case reflect.Map:
+		return &row.unmarshalMachineMapStringWildcard
+	case reflect.Struct:
+		// TODO here we could also invoke automatic atlas autogen, if configured to be permitted
+		mach := &row.errThunkUnmarshalMachine
+		mach.err = fmt.Errorf("missing an atlas entry describing how to unmarshal type %v (and auto-atlasing for structs is not enabled)", rt)
+		return mach
+	case reflect.Interface:
+		return &row.unmarshalMachineWildcard
+	case reflect.Func:
+		panic(fmt.Errorf("functions cannot be unmarshalled!"))
+	case reflect.Ptr:
+		panic(fmt.Errorf("unreachable: ptrs must already be resolved"))
+	default:
+		panic(fmt.Errorf("excursion %s", rt.Kind()))
+	}
+}
+
+// given that we already have an atlasEntry in mind, yield a configured machine for it.
+// it seems odd that this might still require a whole atlas, but tis so;
+// some things (e.g. transform funcs) need to get additional machinery for delegation.
+func _yieldUnmarshalMachinePtrForAtlasEntry(row *unmarshalSlabRow, entry *atlas.AtlasEntry, atl atlas.Atlas) UnmarshalMachine {
+	// Switch across which of the union of configurations is applicable.
+	switch {
+	case entry.UnmarshalTransformFunc != nil:
+		// Return a machine that calls the func(s), then later a real machine.
+		// The entry.UnmarshalTransformTargetType is used to do a recursive lookup.
+		// We can't just call the func here because we're still working off typeinfo
+		// and don't have a real value to transform until later.
+		row.unmarshalMachineTransform.trFunc = entry.UnmarshalTransformFunc
+		row.unmarshalMachineTransform.recv_rt = entry.UnmarshalTransformTargetType
+		// Pick delegate without growing stack.  (This currently means recursive transform won't fly.)
+		row.unmarshalMachineTransform.delegate = _yieldUnmarshalMachinePtr(row, atl, entry.UnmarshalTransformTargetType)
+		return &row.unmarshalMachineTransform
+	case entry.StructMap != nil:
+		row.unmarshalMachineStructAtlas.cfg = entry
+		return &row.unmarshalMachineStructAtlas
+	case entry.UnionKeyedMorphism != nil:
+		row.unmarshalMachineUnionKeyed.cfg = entry.UnionKeyedMorphism
+		return &row.unmarshalMachineUnionKeyed
+	default:
+		panic("invalid atlas entry")
+	}
+}
+
+// Returns the top row of the slab.  Useful for machines that need to delegate
+//  to another type that's definitely not their own (comes up for the wildcard delegators).
+func (s *unmarshalSlab) tip() *unmarshalSlabRow {
+	return &s.rows[len(s.rows)-1]
+}
+
+func (s *unmarshalSlab) grow() {
+	s.rows = append(s.rows, unmarshalSlabRow{})
+}
+
+func (s *unmarshalSlab) release() {
+	s.rows = s.rows[0 : len(s.rows)-1]
+}
+
+type errThunkUnmarshalMachine struct {
+	err error
+}
+
+func (m *errThunkUnmarshalMachine) Reset(_ *unmarshalSlab, _ reflect.Value, _ reflect.Type) error {
+	return m.err
+}
+func (m *errThunkUnmarshalMachine) Step(d *Unmarshaller, s *unmarshalSlab, tok *Token) (done bool, err error) {
+	return true, m.err
+}

vendor/github.com/polydawn/refmt/obj/unmarshalSliceWildcard.go

@@ -0,0 +1,87 @@
+package obj
+
+import (
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineSliceWildcard struct {
+	target_rv    reflect.Value // target slice handle
+	working_rv   reflect.Value // working slice (target is set to this at end)
+	value_rt     reflect.Type
+	valueZero_rv reflect.Value // a zero of the slice's value type (for kicking append)
+	valueMach    UnmarshalMachine
+	phase        unmarshalMachineArrayWildcardPhase
+	index        int
+}
+
+func (mach *unmarshalMachineSliceWildcard) Reset(slab *unmarshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.working_rv = rv
+	mach.value_rt = rt.Elem()
+	mach.valueZero_rv = reflect.Zero(mach.value_rt)
+	mach.valueMach = slab.requisitionMachine(mach.value_rt)
+	mach.phase = unmarshalMachineArrayWildcardPhase_initial
+	mach.index = 0
+	return nil
+}
+
+func (mach *unmarshalMachineSliceWildcard) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch mach.phase {
+	case unmarshalMachineArrayWildcardPhase_initial:
+		return mach.step_Initial(driver, slab, tok)
+	case unmarshalMachineArrayWildcardPhase_acceptValueOrClose:
+		return mach.step_AcceptValueOrClose(driver, slab, tok)
+	}
+	panic("unreachable")
+}
+
+func (mach *unmarshalMachineSliceWildcard) step_Initial(_ *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// If it's a special state, start an object.
+	//  (Or, blow up if its a special state that's silly).
+	switch tok.Type {
+	case TMapOpen:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TArrOpen:
+		// Great.  Consumed.
+		mach.phase = unmarshalMachineArrayWildcardPhase_acceptValueOrClose
+		// Initialize the slice.
+		mach.target_rv.Set(reflect.MakeSlice(mach.target_rv.Type(), 0, 0))
+		return false, nil
+	case TMapClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TArrClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	case TNull:
+		mach.target_rv.Set(reflect.Zero(mach.target_rv.Type()))
+		return true, nil
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "start of array"}
+	}
+}
+
+func (mach *unmarshalMachineSliceWildcard) step_AcceptValueOrClose(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// Either form of open token are valid, but
+	// - an arrClose is ours
+	// - and a mapClose is clearly invalid.
+	switch tok.Type {
+	case TMapClose:
+		// no special checks for ends of wildcard slice; no such thing as incomplete.
+		return true, ErrMalformedTokenStream{tok.Type, "start of value or end of array"}
+	case TArrClose:
+		mach.target_rv.Set(mach.working_rv)
+		// release the slab row we requisitioned for our value machine.
+		slab.release()
+		return true, nil
+	}
+
+	// Grow the slice if necessary.
+	mach.working_rv = reflect.Append(mach.working_rv, mach.valueZero_rv)
+
+	// Recurse on a handle to the next index.
+	rv := mach.working_rv.Index(mach.index)
+	mach.index++
+	return false, driver.Recurse(tok, rv, mach.value_rt, mach.valueMach)
+	// Step simply remains `step_AcceptValueOrClose` -- arrays don't have much state machine.
+}

vendor/github.com/polydawn/refmt/obj/unmarshalStruct.go

@@ -0,0 +1,109 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineStructAtlas struct {
+	cfg *atlas.AtlasEntry // set on initialization
+
+	rv         reflect.Value
+	expectLen  int                  // Length header from mapOpen token.  If it was set, we validate it.
+	index      int                  // Progress marker: our distance into the stream of pairs.
+	value      bool                 // Progress marker: whether the next token is a value.
+	fieldEntry atlas.StructMapEntry // Which field we expect next: set when consuming a key.
+}
+
+func (mach *unmarshalMachineStructAtlas) Reset(_ *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.rv = rv
+	// not necessary to reset expectLen because MapOpen tokens also consistently use the -1 convention.
+	mach.index = -1
+	mach.value = false
+	return nil
+}
+
+func (mach *unmarshalMachineStructAtlas) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// Starter state.
+	if mach.index < 0 {
+		switch tok.Type {
+		case TMapOpen:
+			// Great.  Consumed.
+			mach.expectLen = tok.Length
+			mach.index++
+			return false, nil
+		case TMapClose:
+			return true, ErrMalformedTokenStream{tok.Type, "start of map"}
+		case TArrOpen:
+			return true, ErrMalformedTokenStream{tok.Type, "start of map"}
+		case TArrClose:
+			return true, ErrMalformedTokenStream{tok.Type, "start of map"}
+		case TNull:
+			mach.rv.Set(reflect.Zero(mach.rv.Type()))
+			return true, nil
+		default:
+			return true, ErrMalformedTokenStream{tok.Type, "start of map"}
+		}
+	}
+
+	// Accept value:
+	if mach.value {
+		var child_rv reflect.Value
+		var child_rt reflect.Type
+		if mach.fieldEntry.Ignore {
+			// Use a dummy slot to slurp up the value.  This could be more efficient.
+			child_rt = reflect.TypeOf((*interface{})(nil)).Elem()
+			child_rv = reflect.New(child_rt).Elem()
+		} else {
+			child_rt = mach.fieldEntry.Type
+			child_rv = mach.fieldEntry.ReflectRoute.TraverseToValue(mach.rv)
+		}
+		mach.index++
+		mach.value = false
+		return false, driver.Recurse(
+			tok,
+			child_rv,
+			child_rt,
+			slab.requisitionMachine(child_rt),
+		)
+	}
+
+	// Accept key or end:
+	if mach.index > 0 {
+		slab.release()
+	}
+	switch tok.Type {
+	case TMapClose:
+		// If we got length header, validate that; error if mismatch.
+		if mach.expectLen >= 0 {
+			if mach.expectLen != mach.index {
+				return true, fmt.Errorf("malformed map token stream: declared length %d, actually got %d entries", mach.expectLen, mach.index)
+			}
+		}
+
+		// Future: this would be a reasonable place to check that all required fields have been filled in, if we add such a feature.
+
+		return true, nil
+	case TString:
+		for n := 0; n < len(mach.cfg.StructMap.Fields); n++ {
+			fieldEntry := mach.cfg.StructMap.Fields[n]
+			if fieldEntry.SerialName != tok.Str {
+				continue
+			}
+			mach.fieldEntry = fieldEntry
+			mach.value = true
+			break
+		}
+		if mach.value == false {
+			// FUTURE: it should be configurable per atlas.StructMap whether this is considered an error or to be tolerated.
+			// Currently we're being extremely strict about it, which is a divergence from the stdlib json behavior.
+			return true, ErrNoSuchField{tok.Str, mach.cfg.Type.String()}
+		}
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "map key"}
+	}
+	return false, nil
+}

vendor/github.com/polydawn/refmt/obj/unmarshalTransform.go

@@ -0,0 +1,38 @@
+package obj
+
+import (
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineTransform struct {
+	trFunc   atlas.UnmarshalTransformFunc
+	recv_rt  reflect.Type
+	delegate UnmarshalMachine // machine for handling the recv type, stepped to completion before transform applied.
+
+	target_rv reflect.Value // given on Reset, retained until last step, and set into after using trFunc
+	recv_rv   reflect.Value // if set, handle to slot where slice is stored; content must be placed into target at end.
+}
+
+func (mach *unmarshalMachineTransform) Reset(slab *unmarshalSlab, rv reflect.Value, _ reflect.Type) error {
+	mach.target_rv = rv
+	mach.recv_rv = reflect.New(mach.recv_rt).Elem() // REVIEW: this behavior with ptr vs not for in_rt.  the star-star case is prob not what want.
+	return mach.delegate.Reset(slab, mach.recv_rv, mach.recv_rt)
+}
+
+func (mach *unmarshalMachineTransform) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	done, err = mach.delegate.Step(driver, slab, tok)
+	if err != nil {
+		return
+	}
+	if !done {
+		return
+	}
+	// on the last step, use transform, and finally set in real target.
+	tr_rv, err := mach.trFunc(mach.recv_rv)
+	// do attempt the set even if error.  user may appreciate partial progress.
+	mach.target_rv.Set(tr_rv)
+	return true, err
+}

vendor/github.com/polydawn/refmt/obj/unmarshalUnionKeyed.go

@@ -0,0 +1,109 @@
+package obj
+
+import (
+	"reflect"
+
+	"github.com/polydawn/refmt/obj/atlas"
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineUnionKeyed struct {
+	cfg *atlas.UnionKeyedMorphism // set on initialization
+
+	target_rv reflect.Value
+	target_rt reflect.Type
+
+	phase    unmarshalMachineUnionKeyedPhase
+	tmp_rv   reflect.Value
+	delegate UnmarshalMachine // actual machine, once we've demuxed with the second token (the key).
+}
+
+type unmarshalMachineUnionKeyedPhase uint8
+
+const (
+	unmarshalMachineUnionKeyedPhase_acceptMapOpen unmarshalMachineUnionKeyedPhase = iota
+	unmarshalMachineUnionKeyedPhase_acceptKey
+	unmarshalMachineUnionKeyedPhase_delegate
+	unmarshalMachineUnionKeyedPhase_acceptMapClose
+)
+
+func (mach *unmarshalMachineUnionKeyed) Reset(_ *unmarshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.target_rt = rt
+	mach.phase = unmarshalMachineUnionKeyedPhase_acceptMapOpen
+	return nil
+}
+
+func (mach *unmarshalMachineUnionKeyed) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch mach.phase {
+	case unmarshalMachineUnionKeyedPhase_acceptMapOpen:
+		return mach.step_acceptMapOpen(driver, slab, tok)
+	case unmarshalMachineUnionKeyedPhase_acceptKey:
+		return mach.step_acceptKey(driver, slab, tok)
+	case unmarshalMachineUnionKeyedPhase_delegate:
+		return mach.step_delegate(driver, slab, tok)
+	case unmarshalMachineUnionKeyedPhase_acceptMapClose:
+		return mach.step_acceptMapClose(driver, slab, tok)
+	}
+	panic("unreachable")
+}
+
+func (mach *unmarshalMachineUnionKeyed) step_acceptMapOpen(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch tok.Type {
+	case TMapOpen:
+		switch tok.Length {
+		case -1: // pass
+		case 1: // correct
+		default:
+			return true, ErrMalformedTokenStream{tok.Type, "unions in keyed format must be maps with exactly one entry"} // FIXME not malformed per se
+		}
+		mach.phase = unmarshalMachineUnionKeyedPhase_acceptKey
+		return false, nil
+	// REVIEW: is case TNull perhaps conditionally acceptable?
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "start of union value"} // FIXME not malformed per se
+	}
+}
+
+func (mach *unmarshalMachineUnionKeyed) step_acceptKey(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch tok.Type {
+	case TString:
+		// Look up the configuration for this key.
+		delegateAtlasEnt, ok := mach.cfg.Elements[tok.Str]
+		if !ok {
+			return true, ErrNoSuchUnionMember{tok.Str, mach.target_rt, mach.cfg.KnownMembers}
+		}
+		// Allocate a new concrete value, and hang on to that rv handle.
+		//  Assigning into the interface must be done at the end if it's a non-pointer.
+		mach.tmp_rv = reflect.New(delegateAtlasEnt.Type).Elem()
+		// Get and configure a machine for the delegation.
+		delegate := _yieldUnmarshalMachinePtrForAtlasEntry(slab.tip(), delegateAtlasEnt, slab.atlas)
+		if err := delegate.Reset(slab, mach.tmp_rv, delegateAtlasEnt.Type); err != nil {
+			return true, err
+		}
+		mach.delegate = delegate
+		mach.phase = unmarshalMachineUnionKeyedPhase_delegate
+		return false, nil
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "map key"}
+	}
+}
+
+func (mach *unmarshalMachineUnionKeyed) step_delegate(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	done, err = mach.delegate.Step(driver, slab, tok)
+	if done && err == nil {
+		mach.phase = unmarshalMachineUnionKeyedPhase_acceptMapClose
+		return false, nil
+	}
+	return
+}
+
+func (mach *unmarshalMachineUnionKeyed) step_acceptMapClose(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	switch tok.Type {
+	case TMapClose:
+		mach.target_rv.Set(mach.tmp_rv)
+		return true, nil
+	default:
+		return true, ErrMalformedTokenStream{tok.Type, "map close at end of union value"}
+	}
+}

vendor/github.com/polydawn/refmt/obj/unmarshalWildcard.go

@@ -0,0 +1,112 @@
+package obj
+
+import (
+	"fmt"
+	"reflect"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type unmarshalMachineWildcard struct {
+	target_rv reflect.Value
+	target_rt reflect.Type
+	delegate  UnmarshalMachine // actual machine, once we've demuxed with the first token.
+	holder_rv reflect.Value    // if set, handle to slot where slice is stored; content must be placed into target at end.
+}
+
+func (mach *unmarshalMachineWildcard) Reset(_ *unmarshalSlab, rv reflect.Value, rt reflect.Type) error {
+	mach.target_rv = rv
+	mach.target_rt = rt
+	mach.delegate = nil
+	mach.holder_rv = reflect.Value{}
+	return nil
+}
+
+func (mach *unmarshalMachineWildcard) Step(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	if mach.delegate == nil {
+		done, err = mach.prepareDemux(driver, slab, tok)
+		if done {
+			return
+		}
+	}
+	done, err = mach.delegate.Step(driver, slab, tok)
+	if !done {
+		return
+	}
+	if mach.holder_rv.IsValid() {
+		mach.target_rv.Set(mach.holder_rv)
+	}
+	return
+}
+
+func (mach *unmarshalMachineWildcard) prepareDemux(driver *Unmarshaller, slab *unmarshalSlab, tok *Token) (done bool, err error) {
+	// If a "tag" is set in the token, we try to follow that as a hint for
+	//  any specifically customized behaviors for how this should be unmarshalled.
+	if tok.Tagged == true {
+		atlasEntry, exists := slab.atlas.GetEntryByTag(tok.Tag)
+		if !exists {
+			return true, fmt.Errorf("missing an unmarshaller for tag %v", tok.Tag)
+		}
+		value_rt := atlasEntry.Type
+		mach.holder_rv = reflect.New(value_rt).Elem()
+		mach.delegate = _yieldUnmarshalMachinePtr(slab.tip(), slab.atlas, value_rt)
+		if err := mach.delegate.Reset(slab, mach.holder_rv, value_rt); err != nil {
+			return true, err
+		}
+		return false, nil
+	}
+
+	// Switch on token type: we may be able to delegate to a primitive machine,
+	//  but we may also need to initialize a container type and then hand off.
+	switch tok.Type {
+	case TMapOpen:
+		child := make(map[string]interface{})
+		child_rv := reflect.ValueOf(child)
+		mach.target_rv.Set(child_rv)
+		mach.delegate = &slab.tip().unmarshalMachineMapStringWildcard
+		if err := mach.delegate.Reset(slab, child_rv, child_rv.Type()); err != nil {
+			return true, err
+		}
+		return false, nil
+
+	case TArrOpen:
+		// Stdlib has very interesting branch here: 'case reflect.Interface: if v.NumMethod() == 0 {'
+		//  If that matches, it goes on a *totally different* branch that leaves the reflective path entirely forever.
+		//   (Which is kind of interesting, because it also means it will never reuse memory there.  If you wanted that.)
+
+		// This definitely went through a few discovery steps...
+		// - https://play.golang.org/p/Qbtpxwh68e
+		// - https://play.golang.org/p/l5RQujLnDN
+		// - https://play.golang.org/p/Z2ilpPk0vk
+		// - https://play.golang.org/p/jV9VFDht6F -- finally getting somewhere good
+
+		holder := make([]interface{}, 0)
+		mach.holder_rv = reflect.ValueOf(&holder).Elem()
+		mach.delegate = &slab.tip().unmarshalMachineSliceWildcard
+		if err := mach.delegate.Reset(slab, mach.holder_rv, mach.holder_rv.Type()); err != nil {
+			return true, err
+		}
+		return false, nil
+
+	case TMapClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of value"}
+
+	case TArrClose:
+		return true, ErrMalformedTokenStream{tok.Type, "start of value"}
+
+	case TNull:
+		mach.target_rv.Set(reflect.Zero(mach.target_rt))
+		return true, nil
+
+	default:
+		// If it wasn't the start of composite, shell out to the machine for literals.
+		// Don't bother to replace our internal step func because literal machines are never multi-call,
+		//  and this lets us avoid grabbing a pointer and it shuffling around.
+		delegateMach := slab.tip().unmarshalMachinePrimitive
+		delegateMach.kind = reflect.Interface
+		if err := delegateMach.Reset(slab, mach.target_rv, nil); err != nil {
+			return true, err
+		}
+		return delegateMach.Step(driver, slab, tok)
+	}
+}

vendor/github.com/polydawn/refmt/shared/pump.go

@@ -0,0 +1,46 @@
+/*
+	The `shared` package defines helper types and functions used
+	internally by all the other refmt packages; it is not user-facing.
+*/
+package shared
+
+import (
+	"fmt"
+
+	. "github.com/polydawn/refmt/tok"
+)
+
+type TokenSource interface {
+	Step(fillme *Token) (done bool, err error)
+}
+
+type TokenSink interface {
+	Step(consume *Token) (done bool, err error)
+}
+
+type TokenPump struct {
+	TokenSource
+	TokenSink
+}
+
+func (p TokenPump) Run() error {
+	var tok Token
+	var srcDone, sinkDone bool
+	var err error
+	for {
+		srcDone, err = p.TokenSource.Step(&tok)
+		if err != nil {
+			return err
+		}
+		sinkDone, err = p.TokenSink.Step(&tok)
+		if err != nil {
+			return err
+		}
+		if srcDone {
+			if sinkDone {
+				return nil
+			}
+			return fmt.Errorf("src at end of item but sink expects more")
+		}
+	}
+}

vendor/github.com/polydawn/refmt/shared/reader.go

@@ -0,0 +1,301 @@
+package shared
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+const (
+	scratchByteArrayLen = 32
+)
+
+var (
+	zeroByteSlice = []byte{}[:0:0]
+)
+
+var (
+	_ SlickReader = &SlickReaderStream{}
+	_ SlickReader = &SlickReaderSlice{}
+)
+
+func NewReader(r io.Reader) SlickReader {
+	return &SlickReaderStream{br: &readerToScanner{r: r}}
+}
+
+func NewBytesReader(buf *bytes.Buffer) SlickReader {
+	return &SlickReaderStream{br: buf}
+}
+
+func NewSliceReader(b []byte) SlickReader {
+	return &SlickReaderSlice{b: b}
+}
+
+// SlickReader is a hybrid of reader and buffer interfaces with methods giving
+// specific attention to the performance needs found in a decoder.
+// Implementations cover io.Reader as well as []byte directly.
+//
+// In particular, it allows:
+//
+//   - returning byte-slices with zero-copying (you were warned!) when possible
+//   - returning byte-slices for short reads which will be reused (you were warned!)
+//   - putting a 'track' point in the buffer, and later yielding all those bytes at once
+//   - counting the number of bytes read (for use in parser error messages, mainly)
+//
+// All of these shortcuts mean correct usage is essential to avoid unexpected behaviors,
+// but in return allow avoiding many, many common sources of memory allocations in a parser.
+type SlickReader interface {
+
+	// Read n bytes into a byte slice which may be shared and must not be reused
+	// After any additional calls to this reader.
+	// Readnzc will use the implementation scratch buffer if possible,
+	// i.e. n < len(scratchbuf), or may return a view of the []byte being decoded from.
+	// Requesting a zero length read will return `zeroByteSlice`, a len-zero cap-zero slice.
+	Readnzc(n int) ([]byte, error)
+
+	// Read n bytes into a new byte slice.
+	// If zero-copy views into existing buffers are acceptable (e.g. you know you
+	// won't later mutate, reference or expose this memory again), prefer `Readnzc`.
+	// If you already have an existing slice of sufficient size to reuse, prefer `Readb`.
+	// Requesting a zero length read will return `zeroByteSlice`, a len-zero cap-zero slice.
+	Readn(n int) ([]byte, error)
+
+	// Read `len(b)` bytes into the given slice, starting at its beginning,
+	// overwriting all values, and disregarding any extra capacity.
+	Readb(b []byte) error
+
+	Readn1() (uint8, error)
+	Unreadn1()
+	NumRead() int // number of bytes read
+	Track()
+	StopTrack() []byte
+}
+
+// SlickReaderStream is a SlickReader that reads off an io.Reader.
+// Initialize it by wrapping an ioDecByteScanner around your io.Reader and dumping it in.
+// While this implementation does use some internal buffers, it's still advisable
+// to use a buffered reader to avoid small reads for any external IO like disk or network.
+type SlickReaderStream struct {
+	br         readerScanner
+	scratch    [scratchByteArrayLen]byte // temp byte array re-used internally for efficiency during read.
+	n          int                       // num read
+	tracking   []byte                    // tracking bytes read
+	isTracking bool
+}
+
+func (z *SlickReaderStream) NumRead() int {
+	return z.n
+}
+
+func (z *SlickReaderStream) Readnzc(n int) (bs []byte, err error) {
+	if n == 0 {
+		return zeroByteSlice, nil
+	}
+	if n < len(z.scratch) {
+		bs = z.scratch[:n]
+	} else {
+		bs = make([]byte, n)
+	}
+	err = z.Readb(bs)
+	return
+}
+
+func (z *SlickReaderStream) Readn(n int) (bs []byte, err error) {
+	if n == 0 {
+		return zeroByteSlice, nil
+	}
+	bs = make([]byte, n)
+	err = z.Readb(bs)
+	return
+}
+
+func (z *SlickReaderStream) Readb(bs []byte) error {
+	if len(bs) == 0 {
+		return nil
+	}
+	n, err := io.ReadAtLeast(z.br, bs, len(bs))
+	z.n += n
+	if z.isTracking {
+		z.tracking = append(z.tracking, bs...)
+	}
+	return err
+}
+
+func (z *SlickReaderStream) Readn1() (b uint8, err error) {
+	b, err = z.br.ReadByte()
+	if err != nil {
+		return
+	}
+	z.n++
+	if z.isTracking {
+		z.tracking = append(z.tracking, b)
+	}
+	return
+}
+
+func (z *SlickReaderStream) Unreadn1() {
+	err := z.br.UnreadByte()
+	if err != nil {
+		panic(err)
+	}
+	z.n--
+	if z.isTracking {
+		if l := len(z.tracking) - 1; l >= 0 {
+			z.tracking = z.tracking[:l]
+		}
+	}
+}
+
+func (z *SlickReaderStream) Track() {
+	if z.tracking != nil {
+		z.tracking = z.tracking[:0]
+	}
+	z.isTracking = true
+}
+
+func (z *SlickReaderStream) StopTrack() (bs []byte) {
+	z.isTracking = false
+	return z.tracking
+}
+
+// SlickReaderSlice implements SlickReader by reading a byte slice directly.
+// Often this means the zero-copy methods can simply return subslices.
+type SlickReaderSlice struct {
+	b []byte // data
+	c int    // cursor
+	a int    // available
+	t int    // track start
+}
+
+func (z *SlickReaderSlice) reset(in []byte) {
+	z.b = in
+	z.a = len(in)
+	z.c = 0
+	z.t = 0
+}
+
+func (z *SlickReaderSlice) NumRead() int {
+	return z.c
+}
+
+func (z *SlickReaderSlice) Unreadn1() {
+	if z.c == 0 || len(z.b) == 0 {
+		panic(errors.New("cannot unread last byte read"))
+	}
+	z.c--
+	z.a++
+	return
+}
+
+func (z *SlickReaderSlice) Readnzc(n int) (bs []byte, err error) {
+	if n == 0 {
+		return zeroByteSlice, nil
+	} else if z.a == 0 {
+		return zeroByteSlice, io.EOF
+	} else if n > z.a {
+		return zeroByteSlice, io.ErrUnexpectedEOF
+	} else {
+		c0 := z.c
+		z.c = c0 + n
+		z.a = z.a - n
+		bs = z.b[c0:z.c]
+	}
+	return
+}
+
+func (z *SlickReaderSlice) Readn(n int) (bs []byte, err error) {
+	if n == 0 {
+		return zeroByteSlice, nil
+	}
+	bs = make([]byte, n)
+	err = z.Readb(bs)
+	return
+}
+
+func (z *SlickReaderSlice) Readn1() (v uint8, err error) {
+	if z.a == 0 {
+		panic(io.EOF)
+	}
+	v = z.b[z.c]
+	z.c++
+	z.a--
+	return
+}
+
+func (z *SlickReaderSlice) Readb(bs []byte) error {
+	bs2, err := z.Readnzc(len(bs))
+	copy(bs, bs2)
+	return err
+}
+
+func (z *SlickReaderSlice) Track() {
+	z.t = z.c
+}
+
+func (z *SlickReaderSlice) StopTrack() (bs []byte) {
+	return z.b[z.t:z.c]
+}
+
+// conjoin the io.Reader and io.ByteScanner interfaces.
+type readerScanner interface {
+	io.Reader
+	io.ByteScanner
+}
+
+// readerToScanner decorates an `io.Reader` with all the methods to also
+// fulfill the `io.ByteScanner` interface.
+type readerToScanner struct {
+	r  io.Reader
+	l  byte    // last byte
+	ls byte    // last byte status. 0: init-canDoNothing, 1: canRead, 2: canUnread
+	b  [1]byte // tiny buffer for reading single bytes
+}
+
+func (z *readerToScanner) Read(p []byte) (n int, err error) {
+	var firstByte bool
+	if z.ls == 1 {
+		z.ls = 2
+		p[0] = z.l
+		if len(p) == 1 {
+			n = 1
+			return
+		}
+		firstByte = true
+		p = p[1:]
+	}
+	n, err = z.r.Read(p)
+	if n > 0 {
+		if err == io.EOF && n == len(p) {
+			err = nil // read was successful, so postpone EOF (till next time)
+		}
+		z.l = p[n-1]
+		z.ls = 2
+	}
+	if firstByte {
+		n++
+	}
+	return
+}
+
+func (z *readerToScanner) ReadByte() (c byte, err error) {
+	n, err := z.Read(z.b[:])
+	if n == 1 {
+		c = z.b[0]
+		if err == io.EOF {
+			err = nil // read was successful, so postpone EOF (till next time)
+		}
+	}
+	return
+}
+
+func (z *readerToScanner) UnreadByte() (err error) {
+	x := z.ls
+	if x == 0 {
+		err = errors.New("cannot unread - nothing has been read")
+	} else if x == 1 {
+		err = errors.New("cannot unread - last byte has not been read")
+	} else if x == 2 {
+		z.ls = 1
+	}
+	return
+}

vendor/github.com/polydawn/refmt/tok/token.go

@@ -0,0 +1,192 @@
+/*
+	Package containing Token struct and TokenType info.
+	Tokens are the lingua-franca used between all the refmt packages.
+	Users typically do not refer to these types.
+*/
+package tok
+
+import (
+	"bytes"
+	"fmt"
+)
+
+type Token struct {
+	// The type of token.  Indicates which of the value fields has meaning,
+	// or has a special value to indicate beginnings and endings of maps and arrays.
+	Type   TokenType
+	Length int // If this is a TMapOpen or TArrOpen, a length may be specified.  Use -1 for unknown.
+
+	Str     string  // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+	Bytes   []byte  // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+	Bool    bool    // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+	Int     int64   // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+	Uint    uint64  // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+	Float64 float64 // Value union.  Only one of these has meaning, depending on the value of 'Type'.
+
+	Tagged bool // Extension slot for cbor.
+	Tag    int  // Extension slot for cbor.  Only applicable if tagged=true.
+}
+
+type TokenType byte
+
+const (
+	TMapOpen  TokenType = '{'
+	TMapClose TokenType = '}'
+	TArrOpen  TokenType = '['
+	TArrClose TokenType = ']'
+	TNull     TokenType = '0'
+
+	TString  TokenType = 's'
+	TBytes   TokenType = 'x'
+	TBool    TokenType = 'b'
+	TInt     TokenType = 'i'
+	TUint    TokenType = 'u'
+	TFloat64 TokenType = 'f'
+)
+
+func (tt TokenType) String() string {
+	switch tt {
+	case TMapOpen:
+		return "map open"
+	case TMapClose:
+		return "map close"
+	case TArrOpen:
+		return "array open"
+	case TArrClose:
+		return "array close"
+	case TNull:
+		return "null"
+	case TString:
+		return "string"
+	case TBytes:
+		return "bytes"
+	case TBool:
+		return "bool"
+	case TInt:
+		return "int"
+	case TUint:
+		return "uint"
+	case TFloat64:
+		return "float"
+	}
+	return "invalid"
+}
+
+func (tt TokenType) IsValid() bool {
+	switch tt {
+	case TString, TBytes, TBool, TInt, TUint, TFloat64, TNull:
+		return true
+	case TMapOpen, TMapClose, TArrOpen, TArrClose:
+		return true
+	default:
+		return false
+	}
+}
+
+func (tt TokenType) IsValue() bool {
+	switch tt {
+	case TString, TBytes, TBool, TInt, TUint, TFloat64:
+		return true
+	default:
+		return false
+	}
+}
+
+func (tt TokenType) IsSpecial() bool {
+	switch tt {
+	case TMapOpen, TMapClose, TArrOpen, TArrClose, TNull:
+		return true
+	default:
+		return false
+	}
+}
+
+/*
+	Checks if the content of two tokens is the same.
+	Tokens are considered the same if their type one of the special
+	consts (map/array open/close) and that type and the optional length field are equal;
+	or, if type indicates a value, then they are the same if those values are equal.
+	In either path, values that are *not* specified as relevant by the Token's Type
+	are disregarded in the comparison.
+
+	If the Token.Type is not valid, the result will be false.
+
+	This method is primarily useful for testing.
+*/
+func IsTokenEqual(t1, t2 Token) bool {
+	if t1.Type != t2.Type {
+		return false
+	}
+	switch t1.Type {
+	case TMapOpen, TArrOpen:
+		return t1.Length == t2.Length
+	case TMapClose, TArrClose, TNull:
+		return true
+	case TString, TBool, TInt, TUint, TFloat64:
+		return t1.Value() == t2.Value()
+	case TBytes:
+		return bytes.Equal(t1.Bytes, t2.Bytes)
+	default:
+		return false
+	}
+}
+
+// Returns the value attached to this token, or nil.
+// This boxes the value into an `interface{}`, which almost certainly
+// incurs a memory allocation via `runtime.convT2E` in the process,
+// so this this method should not be used when performance is a concern.
+func (t Token) Value() interface{} {
+	switch t.Type {
+	case TString:
+		return t.Str
+	case TBytes:
+		return t.Bytes
+	case TBool:
+		return t.Bool
+	case TInt:
+		return t.Int
+	case TUint:
+		return t.Uint
+	case TFloat64:
+		return t.Float64
+	default:
+		return nil
+	}
+}
+
+func (t Token) String() string {
+	if !t.Tagged {
+		return t.StringSansTag()
+	}
+	return fmt.Sprintf("_%d:%s", t.Tag, t.StringSansTag())
+}
+
+func (t Token) StringSansTag() string {
+	switch t.Type {
+	case TMapOpen:
+		if t.Length == -1 {
+			return "<{>"
+		}
+		return fmt.Sprintf("<{:%d>", t.Length)
+	case TMapClose:
+		return "<}>"
+	case TArrOpen:
+		if t.Length == -1 {
+			return "<[>"
+		}
+		return fmt.Sprintf("<[:%d>", t.Length)
+	case TArrClose:
+		return "<]>"
+	case TNull:
+		return "<0>"
+	case TString:
+		return fmt.Sprintf("<%c:%q>", t.Type, t.Value())
+	}
+	if t.Type.IsValue() {
+		return fmt.Sprintf("<%c:%v>", t.Type, t.Value())
+	}
+	return "<INVALID>"
+}
+
+func TokStr(x string) Token { return Token{Type: TString, Str: x} } // Util for testing.
+func TokInt(x int64) Token  { return Token{Type: TInt, Int: x} }    // Util for testing.