tendermint amino

Amino is an encoding library that can handle Interfaces. This is achieved by prefixing bytes before each “concrete type”.

This blob will focus on the implement of this speciality.

code version: go-amino@v0.15.1

example

Register interface and concrete is required in both encoder and decoder. The order of register is anonymous.

func Example() {

	defer func() {
		if e := recover(); e != nil {
			fmt.Println("Recovered:", e)
		}
	}()

	type Message interface{}

	type bcMessage struct {
		Message string
		Height  int
	}

	// encoder
	var cdc = amino.NewCodec()
	cdc.RegisterInterface((*Message)(nil), nil)
	cdc.RegisterConcrete(&bcMessage{}, "bcMessage", nil)

	var bm = &bcMessage{Message: "ABC", Height: 100}
	var msg = bm

	var bz []byte // the marshalled bytes.
	var err error
	bz, err = cdc.MarshalBinaryLengthPrefixed(msg)
	fmt.Printf("Encoded: %X (err: %v)\n", bz, err)

	// decoder
	var cdc2 = amino.NewCodec()
	cdc2.RegisterInterface((*Message)(nil), nil)
	cdc2.RegisterConcrete(&bcMessage{}, "bcMessage", nil)

	var msg2 Message
	err = cdc2.UnmarshalBinaryLengthPrefixed(bz, &msg2)
	fmt.Printf("Decoded: %v (err: %v)\n", msg2, err)
	var bm2 = msg2.(*bcMessage)
	fmt.Printf("Decoded successfully: %v\n", *bm == *bm2)

	// Output:
	// Encoded: 0B740613650A034142431064 (err: <nil>)
	// Decoded: &{ABC 100} (err: <nil>)
	// Decoded successfully: true
}

code view

codec

typeInfos store all typeInfo including interface and concrete;
interfaceInfos store all interface typeInfo;
concreteInfos store all concrete typeInfo;
disfixToTypeInfo is used to find concrete typeInfo quickly via the prefix and disambiguation bytes;
nameToTypeInfo is used to find concrete typeInfo quickly via the name of concrete.

type Codec struct {
	mtx              sync.RWMutex
	sealed           bool
	typeInfos        map[reflect.Type]*TypeInfo
	interfaceInfos   []*TypeInfo
	concreteInfos    []*TypeInfo
	disfixToTypeInfo map[DisfixBytes]*TypeInfo
	nameToTypeInfo   map[string]*TypeInfo
}

TypeInfo

StructInfo store fields of this struct/concrete.

type TypeInfo struct {
	Type      reflect.Type // Interface type.
	PtrToType reflect.Type
	ZeroValue reflect.Value
	ZeroProto interface{}
	InterfaceInfo
	ConcreteInfo
	StructInfo
}

InterfaceInfo

Implementers store all Implementer of this interface.

type InterfaceInfo struct {
	Priority     []DisfixBytes               // Disfix priority.
	Implementers map[PrefixBytes][]*TypeInfo // Mutated over time.
	InterfaceOptions
}

ConcreteInfo

Disamb and prefix is derived from name. and Disamb is usable when prefix conflict in Implementers of interface.

type ConcreteInfo struct {

	// These fields are only set when registered (as implementing an interface).
	Registered       bool // Registered with RegisterConcrete().
	PointerPreferred bool // Deserialize to pointer type if possible.
	// NilPreferred     bool        // Deserialize to nil for empty structs if PointerPreferred.
	Name            string      // Registered name.
	Disamb          DisambBytes // Disambiguation bytes derived from name.
	Prefix          PrefixBytes // Prefix bytes derived from name.
	ConcreteOptions             // Registration options.

	// These fields get set for all concrete types,
	// even those not manually registered (e.g. are never interface values).
	IsAminoMarshaler       bool         // Implements MarshalAmino() (<ReprObject>, error).
	AminoMarshalReprType   reflect.Type // <ReprType>
	IsAminoUnmarshaler     bool         // Implements UnmarshalAmino(<ReprObject>) (error).
	AminoUnmarshalReprType reflect.Type // <ReprType>
}

Implements

collect Implementers from cdc.concreteInfos when register interface.

// Find all conflicting prefixes for concrete types
// that "implement" the interface.  "Implement" in quotes because
// we only consider the pointer, for extra safety.
func (cdc *Codec) collectImplementers_nolock(info *TypeInfo) {
	for _, cinfo := range cdc.concreteInfos {
		if cinfo.PtrToType.Implements(info.Type) {
			info.Implementers[cinfo.Prefix] = append(
				info.Implementers[cinfo.Prefix], cinfo)
		}
	}
}

add implementer to cdc.interfaceInfos when register concrete.

func (cdc *Codec) addCheckConflictsWithConcrete_nolock(cinfo *TypeInfo) {

	// Iterate over registered interfaces that this "implements".
	// "Implement" in quotes because we only consider the pointer, for extra
	// safety.
	for _, iinfo := range cdc.interfaceInfos {
		if !cinfo.PtrToType.Implements(iinfo.Type) {
			continue
		}

		// Add cinfo to iinfo.Implementers.
		var origImpls = iinfo.Implementers[cinfo.Prefix]
		iinfo.Implementers[cinfo.Prefix] = append(origImpls, cinfo)

		// Finally, check that all conflicts are in `.Priority`.
		// NOTE: This could be optimized, but it's non-trivial.
		err := cdc.checkConflictsInPrio_nolock(iinfo)
		if err != nil {
			// Return to previous state.
			iinfo.Implementers[cinfo.Prefix] = origImpls
			panic(err)
		}
	}
}

Computing the prefix and disambiguation bytes

To compute the disambiguation bytes, we take hash := sha256(concreteTypeName), and drop the leading 0x00 bytes.

func nameToDisfix(name string) (db DisambBytes, pb PrefixBytes) {
	hasher := sha256.New()
	hasher.Write([]byte(name))
	bz := hasher.Sum(nil)
	for bz[0] == 0x00 {
		bz = bz[1:]
	}
	copy(db[:], bz[0:3])
	bz = bz[3:]
	for bz[0] == 0x00 {
		bz = bz[1:]
	}
	copy(pb[:], bz[0:4])
	return
}

To example above. sha256(“bcMessage”)

ce659374061365e36aa68c851dfde7ce382a57c9322bafab64edcfeaf7da0a02

The 3th to 7th bytes is 74061365e, which match with the encode result in upper example.

encode

func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv reflect.Value, fopts FieldOptions, bare bool) (err error) {
	if printLog {
		fmt.Println("(e) encodeReflectBinaryInterface")
		defer func() {
			fmt.Printf("(e) -> err: %v\n", err)
		}()
	}

	// Special case when rv is nil, write 0x00 to denote an empty byteslice.
	if rv.IsNil() {
		_, err = w.Write([]byte{0x00})
		return
	}

	// Get concrete non-pointer reflect value & type.
	var crv, isPtr, isNilPtr = derefPointers(rv.Elem())
	if isPtr && crv.Kind() == reflect.Interface {
		// See "MARKER: No interface-pointers" in codec.go
		panic("should not happen")
	}
	if isNilPtr {
		panic(fmt.Sprintf("Illegal nil-pointer of type %v for registered interface %v. "+
			"For compatibility with other languages, nil-pointer interface values are forbidden.", crv.Type(), iinfo.Type))
	}
	var crt = crv.Type()

	// Get *TypeInfo for concrete type.
	var cinfo *TypeInfo
	cinfo, err = cdc.getTypeInfo_wlock(crt)
	if err != nil {
		return
	}
	if !cinfo.Registered {
		err = fmt.Errorf("Cannot encode unregistered concrete type %v.", crt)
		return
	}

	// For Proto3 compatibility, encode interfaces as ByteLength.
	buf := bytes.NewBuffer(nil)

	// Write disambiguation bytes if needed.
	var needDisamb bool = false
	if iinfo.AlwaysDisambiguate {
		needDisamb = true
	} else if len(iinfo.Implementers[cinfo.Prefix]) > 1 {
		needDisamb = true
	}
	if needDisamb {
		_, err = buf.Write(append([]byte{0x00}, cinfo.Disamb[:]...))
		if err != nil {
			return
		}
	}

	// Write prefix bytes.
	_, err = buf.Write(cinfo.Prefix.Bytes())
	if err != nil {
		return
	}

	// Write actual concrete value.
	err = cdc.encodeReflectBinary(buf, cinfo, crv, fopts, true)
	if err != nil {
		return
	}

	if bare {
		// Write byteslice without byte-length prefixing.
		_, err = w.Write(buf.Bytes())
	} else {
		// Write byte-length prefixed byteslice.
		err = EncodeByteSlice(w, buf.Bytes())
	}
	return
}

decode to get type info

func DecodeDisambPrefixBytes(bz []byte) (db DisambBytes, hasDb bool, pb PrefixBytes, hasPb bool, n int, err error) {
	// Validate
	if len(bz) < 4 {
		err = errors.New("EOF while reading prefix bytes.")
		return // hasPb = false
	}
	if bz[0] == 0x00 { // Disfix
		// Validate
		if len(bz) < 8 {
			err = errors.New("EOF while reading disamb bytes.")
			return // hasPb = false
		}
		copy(db[0:3], bz[1:4])
		copy(pb[0:4], bz[4:8])
		hasDb = true
		hasPb = true
		n = 8
		return
	} else { // Prefix
		// General case with no disambiguation
		copy(pb[0:4], bz[0:4])
		hasDb = false
		hasPb = true
		n = 4
		return
	}
}