diff --git a/examples/app.ts b/examples/app.ts index 775eed265..1fa44de72 100644 --- a/examples/app.ts +++ b/examples/app.ts @@ -132,29 +132,44 @@ async function main() { // example: APP_READ_STATE const appInfo = await algodClient.getApplicationByID(appId).do(); - const globalState = appInfo.params.globalState[0]; - console.log(`Raw global state - ${algosdk.stringifyJSON(globalState)}`); + if (!appInfo.params.globalState || appInfo.params.globalState.length === 0) { + throw new Error('Global state not present'); + } + const { globalState } = appInfo.params; + console.log( + `Raw global state - ${globalState.map((kv) => algosdk.encodeJSON(kv))}` + ); - // decode b64 string key with Buffer - const globalKey = algosdk.base64ToString(globalState.key); + const globalKey = algosdk.base64ToBytes(globalState[0].key); // show global value - const globalValue = globalState.value.bytes; + const globalValue = algosdk.base64ToBytes(globalState[0].value.bytes); - console.log(`Decoded global state - ${globalKey}: ${globalValue}`); + console.log( + `Decoded global state - ${algosdk.bytesToBase64(globalKey)}: ${algosdk.bytesToBase64(globalValue)}` + ); const accountAppInfo = await algodClient .accountApplicationInformation(caller.addr, appId) .do(); + if ( + !accountAppInfo.appLocalState || + !accountAppInfo.appLocalState.keyValue || + accountAppInfo.appLocalState.keyValue.length === 0 + ) { + throw new Error('Local state values not present'); + } + const localState = accountAppInfo.appLocalState.keyValue; + console.log( + `Raw local state - ${localState.map((kv) => algosdk.encodeJSON(kv))}` + ); - const localState = accountAppInfo.appLocalState.keyValue[0]; - console.log(`Raw local state - ${algosdk.stringifyJSON(localState)}`); - - // decode b64 string key with Buffer - const localKey = algosdk.base64ToString(localState.key); + const localKey = algosdk.base64ToBytes(localState[0].key); // get uint value directly - const localValue = localState.value.uint; + const localValue = localState[0].value.uint; - console.log(`Decoded local state - ${localKey}: ${localValue}`); + console.log( + `Decoded local state - ${algosdk.bytesToBase64(localKey)}: ${localValue}` + ); // example: APP_READ_STATE // example: APP_CLOSEOUT diff --git a/examples/codec.ts b/examples/codec.ts index e92f7f4d9..3ba74c7ed 100644 --- a/examples/codec.ts +++ b/examples/codec.ts @@ -20,7 +20,7 @@ async function main() { // example: CODEC_BASE64 const b64Encoded = 'SGksIEknbSBkZWNvZGVkIGZyb20gYmFzZTY0'; - const b64Decoded = algosdk.base64ToString(b64Encoded); + const b64Decoded = algosdk.base64ToBytes(b64Encoded); console.log(b64Encoded, b64Decoded); // example: CODEC_BASE64 diff --git a/package-lock.json b/package-lock.json index 513d54a03..cd7733d46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "2.8.0", "license": "MIT", "dependencies": { - "algorand-msgpack": "^1.0.1", + "algorand-msgpack": "^1.1.0", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", @@ -1195,9 +1195,9 @@ } }, "node_modules/algorand-msgpack": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/algorand-msgpack/-/algorand-msgpack-1.0.1.tgz", - "integrity": "sha512-8vVEar8APmTT7cp6Ye2pJg7tHFSgXfJlXmq85v3MTlqPQp14KnxndbyKIq5xjCBT/oarx9p5ncDfRq3hzYg/NA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/algorand-msgpack/-/algorand-msgpack-1.1.0.tgz", + "integrity": "sha512-08k7pBQnkaUB5p+jL7f1TRaUIlTSDE0cesFu1mD7llLao+1cAhtvvZmGE3OnisTd0xOn118QMw74SRqddqaYvw==", "engines": { "node": ">= 14" } diff --git a/package.json b/package.json index 92f792b46..aaf31fa31 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "url": "git://github.com/algorand/js-algorand-sdk.git" }, "dependencies": { - "algorand-msgpack": "^1.0.1", + "algorand-msgpack": "^1.1.0", "hi-base32": "^0.5.1", "js-sha256": "^0.9.0", "js-sha3": "^0.8.0", diff --git a/src/encoding/binarydata.ts b/src/encoding/binarydata.ts index add1f4980..1a6e726fe 100644 --- a/src/encoding/binarydata.ts +++ b/src/encoding/binarydata.ts @@ -13,18 +13,6 @@ export function base64ToBytes(base64String: string): Uint8Array { return Uint8Array.from(binString, (m) => m.codePointAt(0)!); } -/** - * Decode a base64 string for Node.js and browser environments. - * @returns A decoded string - */ -export function base64ToString(base64String: string): string { - if (isNode()) { - return Buffer.from(base64String, 'base64').toString(); - } - const binString = base64ToBytes(base64String); - return new TextDecoder().decode(binString); -} - /** * Convert a Uint8Array to a base64 string for Node.js and browser environments. * @returns A base64 string @@ -40,6 +28,14 @@ export function bytesToBase64(byteArray: Uint8Array): string { return btoa(binString); } +/** + * Convert a byte array to a UTF-8 string. Warning: not all byte arrays are valid UTF-8. + * @returns A decoded string + */ +export function bytesToString(byteArray: Uint8Array): string { + return new TextDecoder().decode(byteArray); +} + /** * Returns a Uint8Array given an input string or Uint8Array. * @returns A base64 string diff --git a/src/encoding/encoding.ts b/src/encoding/encoding.ts index 5e6a59466..b274411ee 100644 --- a/src/encoding/encoding.ts +++ b/src/encoding/encoding.ts @@ -16,10 +16,11 @@ import { decode as msgpackDecode, DecoderOptions, IntMode, + RawBinaryString, } from 'algorand-msgpack'; -import { bytesToBase64 } from './binarydata.js'; +import { bytesToBase64, coerceToBytes } from './binarydata.js'; import IntDecoding from '../types/intDecoding.js'; -import { stringifyJSON, parseJSON } from '../utils/utils.js'; +import { stringifyJSON, parseJSON, arrayEqual } from '../utils/utils.js'; // Errors export const ERROR_CONTAINS_EMPTY_STRING = @@ -43,44 +44,115 @@ function containsEmpty(obj: Record) { } /** - * rawEncode encodes objects using msgpack, regardless of whether there are + * msgpackRawEncode encodes objects using msgpack, regardless of whether there are * empty or 0 value fields. * @param obj - a dictionary to be encoded. May or may not contain empty or 0 values. * @returns msgpack representation of the object */ -export function rawEncode(obj: unknown) { +export function msgpackRawEncode(obj: unknown) { // enable the canonical option const options: EncoderOptions = { sortKeys: true }; return msgpackEncode(obj, options); } /** - * encode encodes objects using msgpack - * @param obj - a dictionary to be encoded. Must not contain empty or 0 values. - * @returns msgpack representation of the object + * encodeObj takes a javascript object and returns its msgpack encoding + * Note that the encoding sorts the fields alphabetically + * @param o - js object to be encoded. Must not contain empty or 0 values. + * @returns Uint8Array binary representation * @throws Error containing ERROR_CONTAINS_EMPTY_STRING if the object contains empty or zero values + * + * @deprecated Use {@link msgpackRawEncode} instead. Note that function does not + * check for empty values like this one does. */ -export function encode(obj: Record) { +export function encodeObj(obj: Record) { // Check for empty values const emptyCheck = containsEmpty(obj); if (emptyCheck.containsEmpty) { throw new Error(ERROR_CONTAINS_EMPTY_STRING + emptyCheck.firstEmptyKey); } + return msgpackRawEncode(obj); +} - // enable the canonical option - return rawEncode(obj); +function intDecodingToIntMode(intDecoding: IntDecoding): IntMode { + switch (intDecoding) { + case IntDecoding.UNSAFE: + return IntMode.UNSAFE_NUMBER; + case IntDecoding.SAFE: + return IntMode.SAFE_NUMBER; + case IntDecoding.MIXED: + return IntMode.MIXED; + case IntDecoding.BIGINT: + return IntMode.BIGINT; + default: + throw new Error(`Invalid intDecoding: ${intDecoding}`); + } +} + +/** + * Decodes msgpack bytes into a plain JavaScript object. + * @param buffer - The msgpack bytes to decode + * @param options - Options for decoding, including int decoding mode. See {@link IntDecoding} for more information. + * @returns The decoded object + */ +export function msgpackRawDecode( + buffer: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, + }; + return msgpackDecode(buffer, decoderOptions); +} + +/** + * decodeObj takes a Uint8Array and returns its javascript obj + * @param o - Uint8Array to decode + * @returns object + * + * @deprecated Use {@link msgpackRawDecode} instead. Note that this function uses `IntDecoding.MIXED` + * while `msgpackRawDecode` defaults to `IntDecoding.BIGINT` for int decoding, though it is + * configurable. + */ +export function decodeObj(o: ArrayLike) { + return msgpackRawDecode(o, { intDecoding: IntDecoding.MIXED }); } -export function decode(buffer: ArrayLike) { - // TODO: consider different int mode - const options: DecoderOptions = { intMode: IntMode.MIXED }; - return msgpackDecode(buffer, options); +/** + * Decodes msgpack bytes into a Map object. This supports decoding non-string map keys. + * @param encoded - The msgpack bytes to decode + * @param options - Options for decoding, including int decoding mode. See {@link IntDecoding} for more information. + * @returns The decoded Map object + */ +export function msgpackRawDecodeAsMap( + encoded: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, + useMap: true, + }; + return msgpackDecode(encoded, decoderOptions); } -export function decodeAsMap(encoded: ArrayLike) { - // TODO: consider different int mode - const options: DecoderOptions = { intMode: IntMode.MIXED, useMap: true }; - return msgpackDecode(encoded, options); +function msgpackRawDecodeAsMapWithRawStrings( + encoded: ArrayLike, + options?: { intDecoding: IntDecoding } +) { + const decoderOptions: DecoderOptions = { + intMode: options?.intDecoding + ? intDecodingToIntMode(options?.intDecoding) + : IntMode.BIGINT, + useMap: true, + rawBinaryStringKeys: true, + rawBinaryStringValues: true, + useRawBinaryStringClass: true, + }; + return msgpackDecode(encoded, decoderOptions); } export type MsgpackEncodingData = @@ -159,6 +231,225 @@ export function jsonEncodingDataToMsgpackEncodingData( /* eslint-disable class-methods-use-this */ /* eslint-disable no-useless-constructor,no-empty-function */ +enum MsgpackObjectPathSegmentKind { + MAP_VALUE, + ARRAY_ELEMENT, +} + +interface MsgpackObjectPathSegment { + kind: MsgpackObjectPathSegmentKind; + key: string | number | bigint | Uint8Array | RawBinaryString; +} + +/** + * This class is used to index into an encoded msgpack object and extract raw strings. + */ +export class MsgpackRawStringProvider { + // eslint-disable-next-line no-use-before-define + private readonly parent?: MsgpackRawStringProvider; + + private readonly baseObjectBytes?: ArrayLike; + + private readonly segment?: MsgpackObjectPathSegment; + + private resolvedCache: MsgpackEncodingData = null; + private resolvedCachePresent = false; + + public constructor({ + parent, + segment, + baseObjectBytes, + }: + | { + parent: MsgpackRawStringProvider; + segment: MsgpackObjectPathSegment; + baseObjectBytes?: undefined; + } + | { + parent?: undefined; + segment?: undefined; + baseObjectBytes: ArrayLike; + }) { + this.parent = parent; + this.segment = segment; + this.baseObjectBytes = baseObjectBytes; + } + + /** + * Create a new provider that resolves to the current provider's map value at the given key. + */ + public withMapValue( + key: string | number | bigint | Uint8Array | RawBinaryString + ): MsgpackRawStringProvider { + return new MsgpackRawStringProvider({ + parent: this, + segment: { + kind: MsgpackObjectPathSegmentKind.MAP_VALUE, + key, + }, + }); + } + + /** + * Create a new provider that resolves to the current provider's array element at the given index. + */ + public withArrayElement(index: number): MsgpackRawStringProvider { + return new MsgpackRawStringProvider({ + parent: this, + segment: { + kind: MsgpackObjectPathSegmentKind.ARRAY_ELEMENT, + key: index, + }, + }); + } + + /** + * Get the raw string at the current location. If the current location is not a raw string, an error is thrown. + */ + public getRawStringAtCurrentLocation(): Uint8Array { + const resolved = this.resolve(); + if (resolved instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + return resolved.rawBinaryValue as Uint8Array; + } + throw new Error( + `Invalid type. Expected RawBinaryString, got ${resolved} (${typeof resolved})` + ); + } + + /** + * Get the raw string map keys and values at the current location. If the current location is not a map, an error is thrown. + */ + public getRawStringKeysAndValuesAtCurrentLocation(): Map< + Uint8Array, + MsgpackEncodingData + > { + const resolved = this.resolve(); + if (!(resolved instanceof Map)) { + throw new Error( + `Invalid type. Expected Map, got ${resolved} (${typeof resolved})` + ); + } + const keysAndValues = new Map(); + for (const [key, value] of resolved) { + if (key instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + keysAndValues.set(key.rawBinaryValue as Uint8Array, value); + } else { + throw new Error( + `Invalid type for map key. Expected RawBinaryString, got ${key} (${typeof key})` + ); + } + } + return keysAndValues; + } + + /** + * Resolve the provider by extracting the value it indicates from the base msgpack object. + */ + private resolve(): MsgpackEncodingData { + if (this.resolvedCachePresent) { + return this.resolvedCache; + } + let parentResolved: MsgpackEncodingData; + if (this.parent) { + parentResolved = this.parent.resolve(); + } else { + // Need to parse baseObjectBytes + parentResolved = msgpackRawDecodeAsMapWithRawStrings( + this.baseObjectBytes! + ) as MsgpackEncodingData; + } + if (!this.segment) { + this.resolvedCache = parentResolved; + this.resolvedCachePresent = true; + return parentResolved; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.MAP_VALUE) { + if (!(parentResolved instanceof Map)) { + throw new Error( + `Invalid type. Expected Map, got ${parentResolved} (${typeof parentResolved})` + ); + } + // All decoded map keys will be raw strings, and Map objects compare complex values by reference, + // so we must check all the values for value-equality. + if ( + typeof this.segment.key === 'string' || + this.segment.key instanceof Uint8Array || + this.segment.key instanceof RawBinaryString + ) { + const targetBytes = + this.segment.key instanceof RawBinaryString + ? // Decoded rawBinaryValue will always be a Uint8Array + (this.segment.key.rawBinaryValue as Uint8Array) + : coerceToBytes(this.segment.key); + const targetIsRawString = + typeof this.segment.key === 'string' || + this.segment.key instanceof RawBinaryString; + for (const [key, value] of parentResolved) { + let potentialKeyBytes: Uint8Array | undefined; + if (targetIsRawString) { + if (key instanceof RawBinaryString) { + // Decoded rawBinaryValue will always be a Uint8Array + potentialKeyBytes = key.rawBinaryValue as Uint8Array; + } + } else if (key instanceof Uint8Array) { + potentialKeyBytes = key; + } + if (potentialKeyBytes && arrayEqual(targetBytes, potentialKeyBytes)) { + this.resolvedCache = value; + break; + } + } + } else { + this.resolvedCache = parentResolved.get(this.segment.key); + } + this.resolvedCachePresent = true; + return this.resolvedCache; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.ARRAY_ELEMENT) { + if (!Array.isArray(parentResolved)) { + throw new Error( + `Invalid type. Expected Array, got ${parentResolved} (${typeof parentResolved})` + ); + } + this.resolvedCache = parentResolved[this.segment.key as number]; + this.resolvedCachePresent = true; + return this.resolvedCache; + } + throw new Error(`Invalid segment kind: ${this.segment.kind}`); + } + + /** + * Get the path string of the current location indicated by the provider. Useful for debugging. + */ + public getPathString(): string { + const parentPathString = this.parent ? this.parent.getPathString() : 'root'; + if (!this.segment) { + return parentPathString; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.MAP_VALUE) { + return `${parentPathString} -> map key "${this.segment.key}" (${typeof this.segment.key})`; + } + if (this.segment.kind === MsgpackObjectPathSegmentKind.ARRAY_ELEMENT) { + return `${parentPathString} -> array index ${this.segment.key} (${typeof this.segment.key})`; + } + return `${parentPathString} -> unknown segment kind ${this.segment.kind}`; + } +} + +/** + * Options for {@link Schema.prepareJSON} + */ +export interface PrepareJSONOptions { + /** + * If true, allows invalid UTF-8 binary strings to be converted to JSON strings. + * + * Otherwise, an error will be thrown if encoding a binary string to a JSON cannot be done losslessly. + */ + lossyBinaryStringConversion?: boolean; +} + /** * A Schema is used to prepare objects for encoding and decoding from msgpack and JSON. * @@ -187,16 +478,23 @@ export abstract class Schema { /** * Restores the encoding data from a msgpack encoding object. * @param encoded - The msgpack encoding object to restore. + * @param rawStringProvider - A provider for raw strings. * @returns The original encoding data. */ - public abstract fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown; + public abstract fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown; /** * Prepares the encoding data for encoding to JSON. * @param data - The JSON encoding data to be prepared. * @returns A value ready to be JSON encoded. */ - public abstract prepareJSON(data: unknown): JSONEncodingData; + public abstract prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData; /** * Restores the encoding data from a JSON encoding object. @@ -246,10 +544,12 @@ export function decodeMsgpack( encoded: ArrayLike, c: EncodableClass ): T { + const decoded = msgpackRawDecodeAsMap(encoded) as MsgpackEncodingData; + const rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: encoded, + }); return c.fromEncodingData( - c.encodingSchema.fromPreparedMsgpack( - decodeAsMap(encoded) as MsgpackEncodingData - ) + c.encodingSchema.fromPreparedMsgpack(decoded, rawStringProvider) ); } @@ -259,7 +559,9 @@ export function decodeMsgpack( * @returns A msgpack byte array encoding of the object */ export function encodeMsgpack(e: Encodable): Uint8Array { - return rawEncode(e.getEncodingSchema().prepareMsgpack(e.toEncodingData())); + return msgpackRawEncode( + e.getEncodingSchema().prepareMsgpack(e.toEncodingData()) + ); } /** @@ -273,20 +575,38 @@ export function decodeJSON( c: EncodableClass ): T { const decoded: JSONEncodingData = parseJSON(encoded, { - intDecoding: IntDecoding.MIXED, + intDecoding: IntDecoding.BIGINT, }); return c.fromEncodingData( c.encodingSchema.fromPreparedJSON(decoded) as JSONEncodingData ); } +export interface EncodeJSONOptions { + /** + * Adds indentation, white space, and line break characters to the return-value JSON text to make + * it easier to read. + */ + space?: string | number; + + /** + * If true, allows invalid UTF-8 binary strings to be converted to JSON strings. + * + * Otherwise, an error will be thrown if encoding a binary string to a JSON cannot be done losslessly. + */ + lossyBinaryStringConversion?: boolean; +} + /** * Encode an Encodable object to a JSON string. * @param e - The object to encode - * @param space - Adds indentation, white space, and line break characters to the return-value JSON text to make it easier to read. + * @param options - Optional encoding options. See {@link EncodeJSONOptions} for more information. * @returns A JSON string encoding of the object */ -export function encodeJSON(e: Encodable, space?: string | number): string { - const prepared = e.getEncodingSchema().prepareJSON(e.toEncodingData()); +export function encodeJSON(e: Encodable, options?: EncodeJSONOptions): string { + const { space, ...prepareJSONOptions } = options ?? {}; + const prepared = e + .getEncodingSchema() + .prepareJSON(e.toEncodingData(), prepareJSONOptions); return stringifyJSON(prepared, undefined, space); } diff --git a/src/encoding/schema/address.ts b/src/encoding/schema/address.ts index 73da74365..3eb7f5c7e 100644 --- a/src/encoding/schema/address.ts +++ b/src/encoding/schema/address.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; import { Address } from '../address.js'; /* eslint-disable class-methods-use-this */ @@ -20,12 +26,20 @@ export class AddressSchema extends Schema { throw new Error(`Invalid address: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Address { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Address { // The Address constructor checks that the input is a Uint8Array return new Address(encoded as Uint8Array); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (data instanceof Address) { return data.toString(); } diff --git a/src/encoding/schema/array.ts b/src/encoding/schema/array.ts index 9008b3b93..7306f7c26 100644 --- a/src/encoding/schema/array.ts +++ b/src/encoding/schema/array.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -22,16 +28,27 @@ export class ArraySchema extends Schema { throw new Error('ArraySchema data must be an array'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown[] { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown[] { if (Array.isArray(encoded)) { - return encoded.map((item) => this.itemSchema.fromPreparedMsgpack(item)); + return encoded.map((item, index) => + this.itemSchema.fromPreparedMsgpack( + item, + rawStringProvider.withArrayElement(index) + ) + ); } throw new Error('ArraySchema encoded data must be an array'); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (Array.isArray(data)) { - return data.map((item) => this.itemSchema.prepareJSON(item)); + return data.map((item) => this.itemSchema.prepareJSON(item, options)); } throw new Error('ArraySchema data must be an array'); } diff --git a/src/encoding/schema/binarystring.ts b/src/encoding/schema/binarystring.ts new file mode 100644 index 000000000..01308732b --- /dev/null +++ b/src/encoding/schema/binarystring.ts @@ -0,0 +1,73 @@ +import { RawBinaryString } from 'algorand-msgpack'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; +import { coerceToBytes, bytesToString, bytesToBase64 } from '../binarydata.js'; +import { arrayEqual } from '../../utils/utils.js'; + +/* eslint-disable class-methods-use-this */ + +/** + * SpecialCaseBinaryStringSchema is a schema for byte arrays which are encoded + * as strings in msgpack and JSON. + * + * This schema allows lossless conversion between the in memory representation + * and the msgpack encoded representation, but NOT between the in memory and + * JSON encoded representations if the byte array contains invalid UTF-8 + * sequences. + */ +export class SpecialCaseBinaryStringSchema extends Schema { + public defaultValue(): Uint8Array { + return new Uint8Array(); + } + + public isDefaultValue(data: unknown): boolean { + return data instanceof Uint8Array && data.byteLength === 0; + } + + public prepareMsgpack(data: unknown): MsgpackEncodingData { + if (data instanceof Uint8Array) { + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData + return new RawBinaryString(data) as unknown as MsgpackEncodingData; + } + throw new Error(`Invalid byte array: (${typeof data}) ${data}`); + } + + public fromPreparedMsgpack( + _encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { + return rawStringProvider.getRawStringAtCurrentLocation(); + } + + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { + if (data instanceof Uint8Array) { + // Not safe to convert to string for all binary data + const stringValue = bytesToString(data); + if ( + !options.lossyBinaryStringConversion && + !arrayEqual(coerceToBytes(stringValue), data) + ) { + throw new Error( + `Invalid UTF-8 byte array encountered. Encode with lossyBinaryStringConversion enabled to bypass this check. Base64 value: ${bytesToBase64(data)}` + ); + } + return stringValue; + } + throw new Error(`Invalid byte array: (${typeof data}) ${data}`); + } + + public fromPreparedJSON(encoded: JSONEncodingData): Uint8Array { + if (typeof encoded === 'string') { + return coerceToBytes(encoded); + } + throw new Error(`Invalid byte array: (${typeof encoded}) ${encoded}`); + } +} diff --git a/src/encoding/schema/boolean.ts b/src/encoding/schema/boolean.ts index a5ebff584..090a67545 100644 --- a/src/encoding/schema/boolean.ts +++ b/src/encoding/schema/boolean.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -18,14 +24,21 @@ export class BooleanSchema extends Schema { throw new Error('Invalid boolean'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): boolean { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): boolean { if (typeof encoded === 'boolean') { return encoded; } throw new Error('Invalid boolean'); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (typeof data === 'boolean') { return data; } diff --git a/src/encoding/schema/bytearray.ts b/src/encoding/schema/bytearray.ts index 0cffc0f75..5218049ca 100644 --- a/src/encoding/schema/bytearray.ts +++ b/src/encoding/schema/bytearray.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; import { base64ToBytes, bytesToBase64 } from '../binarydata.js'; /* eslint-disable class-methods-use-this */ @@ -19,14 +25,22 @@ export class ByteArraySchema extends Schema { throw new Error(`Invalid byte array: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Uint8Array { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { if (encoded instanceof Uint8Array) { return encoded; } throw new Error(`Invalid byte array: (${typeof encoded}) ${encoded}`); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (data instanceof Uint8Array) { return bytesToBase64(data); } @@ -70,7 +84,11 @@ export class FixedLengthByteArraySchema extends Schema { throw new Error('Invalid byte array'); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): Uint8Array { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): Uint8Array { if (encoded instanceof Uint8Array) { if (encoded.byteLength === this.length) { return encoded; diff --git a/src/encoding/schema/index.ts b/src/encoding/schema/index.ts index 01077572c..208799132 100644 --- a/src/encoding/schema/index.ts +++ b/src/encoding/schema/index.ts @@ -5,6 +5,8 @@ export { Uint64Schema } from './uint64.js'; export { AddressSchema } from './address.js'; export { ByteArraySchema, FixedLengthByteArraySchema } from './bytearray.js'; +export { SpecialCaseBinaryStringSchema } from './binarystring.js'; + export { ArraySchema } from './array.js'; export { NamedMapSchema, @@ -14,6 +16,7 @@ export { convertMap, Uint64MapSchema, StringMapSchema, + SpecialCaseBinaryStringMapSchema, } from './map.js'; export { OptionalSchema } from './optional.js'; diff --git a/src/encoding/schema/map.ts b/src/encoding/schema/map.ts index 21984f06b..f0afee3cb 100644 --- a/src/encoding/schema/map.ts +++ b/src/encoding/schema/map.ts @@ -1,5 +1,13 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; -import { ensureUint64 } from '../../utils/utils.js'; +import { RawBinaryString } from 'algorand-msgpack'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; +import { ensureUint64, arrayEqual } from '../../utils/utils.js'; +import { bytesToString, coerceToBytes, bytesToBase64 } from '../binarydata.js'; /* eslint-disable class-methods-use-this */ @@ -139,7 +147,8 @@ export class NamedMapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('NamedMapSchema data must be a Map'); @@ -149,7 +158,10 @@ export class NamedMapSchema extends Schema { if (encoded.has(entry.key)) { map.set( entry.key, - entry.valueSchema.fromPreparedMsgpack(encoded.get(entry.key)) + entry.valueSchema.fromPreparedMsgpack( + encoded.get(entry.key), + rawStringProvider.withMapValue(entry.key) + ) ); } else if (entry.omitEmpty) { map.set(entry.key, entry.valueSchema.defaultValue()); @@ -160,7 +172,10 @@ export class NamedMapSchema extends Schema { return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error('NamedMapSchema data must be a Map'); } @@ -170,7 +185,7 @@ export class NamedMapSchema extends Schema { if (entry.omitEmpty && entry.valueSchema.isDefaultValue(value)) { continue; } - obj[entry.key] = entry.valueSchema.prepareJSON(value); + obj[entry.key] = entry.valueSchema.prepareJSON(value, options); } return obj; } @@ -270,7 +285,8 @@ export class Uint64MapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('Uint64MapSchema data must be a Map'); @@ -281,12 +297,21 @@ export class Uint64MapSchema extends Schema { if (map.has(bigintKey)) { throw new Error(`Duplicate key: ${bigintKey}`); } - map.set(bigintKey, this.valueSchema.fromPreparedMsgpack(value)); + map.set( + bigintKey, + this.valueSchema.fromPreparedMsgpack( + value, + rawStringProvider.withMapValue(key) + ) + ); } return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error( `Uint64MapSchema data must be a Map. Got (${typeof data}) ${data}` @@ -298,7 +323,7 @@ export class Uint64MapSchema extends Schema { if (prepared.has(bigintKey)) { throw new Error(`Duplicate key: ${bigintKey}`); } - prepared.set(bigintKey, this.valueSchema.prepareJSON(value)); + prepared.set(bigintKey, this.valueSchema.prepareJSON(value, options)); } // Convert map to object const obj: { [key: string]: JSONEncodingData } = {}; @@ -364,7 +389,8 @@ export class StringMapSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider ): Map { if (!(encoded instanceof Map)) { throw new Error('StringMapSchema data must be a Map'); @@ -377,12 +403,21 @@ export class StringMapSchema extends Schema { if (map.has(key)) { throw new Error(`Duplicate key: ${key}`); } - map.set(key, this.valueSchema.fromPreparedMsgpack(value)); + map.set( + key, + this.valueSchema.fromPreparedMsgpack( + value, + rawStringProvider.withMapValue(key) + ) + ); } return map; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (!(data instanceof Map)) { throw new Error( `StringMapSchema data must be a Map. Got (${typeof data}) ${data}` @@ -396,7 +431,7 @@ export class StringMapSchema extends Schema { if (prepared.has(key)) { throw new Error(`Duplicate key: ${key}`); } - prepared.set(key, this.valueSchema.prepareJSON(value)); + prepared.set(key, this.valueSchema.prepareJSON(value, options)); } // Convert map to object const obj: { [key: string]: JSONEncodingData } = {}; @@ -424,3 +459,151 @@ export class StringMapSchema extends Schema { return map; } } + +/** + * Converts any RawBinaryString values to regular strings in a MsgpackEncodingData object. + * + * Note this conversion may be lossy if the binary data is not valid UTF-8. + * + * @returns A new object with RawBinaryString values converted to strings. + */ +function convertRawStringsInMsgpackValue( + value: MsgpackEncodingData +): MsgpackEncodingData { + if (value instanceof RawBinaryString) { + return bytesToString(value.rawBinaryValue as Uint8Array); + } + if (value instanceof Map) { + const newMap = new Map< + string | number | bigint | Uint8Array, + MsgpackEncodingData + >(); + for (const [key, val] of value) { + newMap.set( + convertRawStringsInMsgpackValue(key) as + | string + | number + | bigint + | Uint8Array, + convertRawStringsInMsgpackValue(val) + ); + } + return newMap; + } + if (Array.isArray(value)) { + return value.map(convertRawStringsInMsgpackValue); + } + return value; +} + +/** + * Schema for a map with a variable number of binary string keys. + * + * See SpecialCaseBinaryStringSchema for more information about the key type. + */ +export class SpecialCaseBinaryStringMapSchema extends Schema { + constructor(public readonly valueSchema: Schema) { + super(); + } + + public defaultValue(): Map { + return new Map(); + } + + public isDefaultValue(data: unknown): boolean { + return data instanceof Map && data.size === 0; + } + + public prepareMsgpack(data: unknown): MsgpackEncodingData { + if (!(data instanceof Map)) { + throw new Error( + `SpecialCaseBinaryStringMapSchema data must be a Map. Got (${typeof data}) ${data}` + ); + } + const prepared = new Map(); + for (const [key, value] of data) { + if (!(key instanceof Uint8Array)) { + throw new Error(`Invalid key: ${key} (${typeof key})`); + } + prepared.set( + new RawBinaryString(key), + this.valueSchema.prepareMsgpack(value) + ); + } + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData + return prepared as unknown as Map; + } + + public fromPreparedMsgpack( + _encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): Map { + const map = new Map(); + const keysAndValues = + rawStringProvider.getRawStringKeysAndValuesAtCurrentLocation(); + for (const [key, value] of keysAndValues) { + map.set( + key, + this.valueSchema.fromPreparedMsgpack( + convertRawStringsInMsgpackValue(value), + rawStringProvider.withMapValue(new RawBinaryString(key)) + ) + ); + } + return map; + } + + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { + if (!(data instanceof Map)) { + throw new Error( + `SpecialCaseBinaryStringMapSchema data must be a Map. Got (${typeof data}) ${data}` + ); + } + const prepared = new Map(); + for (const [key, value] of data) { + if (!(key instanceof Uint8Array)) { + throw new Error(`Invalid key: ${key}`); + } + // Not safe to convert to string for all binary data + const keyStringValue = bytesToString(key); + if ( + !options.lossyBinaryStringConversion && + !arrayEqual(coerceToBytes(keyStringValue), key) + ) { + throw new Error( + `Invalid UTF-8 byte array encountered. Encode with lossyBinaryStringConversion enabled to bypass this check. Base64 value: ${bytesToBase64(key)}` + ); + } + prepared.set( + keyStringValue, + this.valueSchema.prepareJSON(value, options) + ); + } + // Convert map to object + const obj: { [key: string]: JSONEncodingData } = {}; + for (const [key, value] of prepared) { + obj[key] = value; + } + return obj; + } + + public fromPreparedJSON(encoded: JSONEncodingData): Map { + if ( + encoded == null || + typeof encoded !== 'object' || + Array.isArray(encoded) + ) { + throw new Error( + 'SpecialCaseBinaryStringMapSchema data must be an object' + ); + } + const map = new Map(); + for (const [key, value] of Object.entries(encoded)) { + map.set(coerceToBytes(key), this.valueSchema.fromPreparedJSON(value)); + } + return map; + } +} diff --git a/src/encoding/schema/optional.ts b/src/encoding/schema/optional.ts index 138928e97..b083cebeb 100644 --- a/src/encoding/schema/optional.ts +++ b/src/encoding/schema/optional.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -33,20 +39,26 @@ export class OptionalSchema extends Schema { return this.valueSchema.prepareMsgpack(data); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): unknown { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + rawStringProvider: MsgpackRawStringProvider + ): unknown { // JS undefined is encoded as msgpack nil, which may be decoded as JS null if (encoded === undefined || encoded === null) { return undefined; } - return this.valueSchema.fromPreparedMsgpack(encoded); + return this.valueSchema.fromPreparedMsgpack(encoded, rawStringProvider); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + options: PrepareJSONOptions + ): JSONEncodingData { if (data === undefined) { // JSON representation does not have undefined, only null return null; } - return this.valueSchema.prepareJSON(data); + return this.valueSchema.prepareJSON(data, options); } public fromPreparedJSON(encoded: JSONEncodingData): unknown { diff --git a/src/encoding/schema/string.ts b/src/encoding/schema/string.ts index 2d9002209..4f0081fe4 100644 --- a/src/encoding/schema/string.ts +++ b/src/encoding/schema/string.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; /* eslint-disable class-methods-use-this */ @@ -18,14 +24,22 @@ export class StringSchema extends Schema { throw new Error(`Invalid string: (${typeof data}) ${data}`); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): string { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): string { if (typeof encoded === 'string') { return encoded; } throw new Error(`Invalid string: (${typeof encoded}) ${encoded}`); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { if (typeof data === 'string') { return data; } diff --git a/src/encoding/schema/uint64.ts b/src/encoding/schema/uint64.ts index c13fd5b2f..81f1dbbc5 100644 --- a/src/encoding/schema/uint64.ts +++ b/src/encoding/schema/uint64.ts @@ -1,4 +1,10 @@ -import { Schema, MsgpackEncodingData, JSONEncodingData } from '../encoding.js'; +import { + Schema, + MsgpackEncodingData, + MsgpackRawStringProvider, + JSONEncodingData, + PrepareJSONOptions, +} from '../encoding.js'; import { ensureUint64 } from '../../utils/utils.js'; /* eslint-disable class-methods-use-this */ @@ -18,11 +24,19 @@ export class Uint64Schema extends Schema { return ensureUint64(data); } - public fromPreparedMsgpack(encoded: MsgpackEncodingData): bigint { + public fromPreparedMsgpack( + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider + ): bigint { return ensureUint64(encoded); } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { return ensureUint64(data); } diff --git a/src/encoding/schema/untyped.ts b/src/encoding/schema/untyped.ts index 14411ce52..6551d6159 100644 --- a/src/encoding/schema/untyped.ts +++ b/src/encoding/schema/untyped.ts @@ -1,7 +1,9 @@ import { Schema, MsgpackEncodingData, + MsgpackRawStringProvider, JSONEncodingData, + PrepareJSONOptions, msgpackEncodingDataToJSONEncodingData, jsonEncodingDataToMsgpackEncodingData, } from '../encoding.js'; @@ -24,12 +26,18 @@ export class UntypedSchema extends Schema { } public fromPreparedMsgpack( - encoded: MsgpackEncodingData + encoded: MsgpackEncodingData, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _rawStringProvider: MsgpackRawStringProvider ): MsgpackEncodingData { return encoded; } - public prepareJSON(data: unknown): JSONEncodingData { + public prepareJSON( + data: unknown, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _options: PrepareJSONOptions + ): JSONEncodingData { return msgpackEncodingDataToJSONEncodingData(data as MsgpackEncodingData); } diff --git a/src/group.ts b/src/group.ts index 16c8d50c0..4b0318e2b 100644 --- a/src/group.ts +++ b/src/group.ts @@ -1,6 +1,6 @@ import { Transaction } from './transaction.js'; import * as nacl from './nacl/naclWrappers.js'; -import * as encoding from './encoding/encoding.js'; +import { msgpackRawEncode } from './encoding/encoding.js'; import * as utils from './utils/utils.js'; const ALGORAND_MAX_TX_GROUP_SIZE = 16; @@ -8,11 +8,14 @@ const TX_GROUP_TAG = new TextEncoder().encode('TG'); function txGroupPreimage(txnHashes: Uint8Array[]): Uint8Array { if (txnHashes.length > ALGORAND_MAX_TX_GROUP_SIZE) { - throw Error( + throw new Error( `${txnHashes.length} transactions grouped together but max group size is ${ALGORAND_MAX_TX_GROUP_SIZE}` ); } - const bytes = encoding.encode({ + if (txnHashes.length === 0) { + throw new Error('Cannot compute group ID of zero transactions'); + } + const bytes = msgpackRawEncode({ txlist: txnHashes, }); return utils.concatArrays(TX_GROUP_TAG, bytes); diff --git a/src/main.ts b/src/main.ts index 7143ecc75..e0d9779b9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,6 +1,5 @@ import * as nacl from './nacl/naclWrappers.js'; import { Address } from './encoding/address.js'; -import * as encoding from './encoding/encoding.js'; import { Transaction } from './transaction.js'; import * as convert from './convert.js'; import * as utils from './utils/utils.js'; @@ -64,25 +63,6 @@ export function verifyBytes( return nacl.verify(toBeVerified, signature, addrObj.publicKey); } -/** - * encodeObj takes a javascript object and returns its msgpack encoding - * Note that the encoding sorts the fields alphabetically - * @param o - js obj - * @returns Uint8Array binary representation - */ -export function encodeObj(o: Record) { - return new Uint8Array(encoding.encode(o)); -} - -/** - * decodeObj takes a Uint8Array and returns its javascript obj - * @param o - Uint8Array to decode - * @returns object - */ -export function decodeObj(o: ArrayLike) { - return encoding.decode(o); -} - export const ERROR_MULTISIG_BAD_SENDER = new Error( MULTISIG_BAD_SENDER_ERROR_MSG ); @@ -113,6 +93,11 @@ export { JSONEncodingData, Encodable, EncodableClass, + encodeObj, + decodeObj, + msgpackRawEncode, + msgpackRawDecode, + msgpackRawDecodeAsMap, encodeMsgpack, decodeMsgpack, encodeJSON, @@ -129,8 +114,9 @@ export { export { bytesToBigInt, bigIntToBytes } from './encoding/bigint.js'; export { base64ToBytes, - base64ToString, bytesToBase64, + bytesToString, + coerceToBytes, bytesToHex, hexToBytes, } from './encoding/binarydata.js'; diff --git a/src/transaction.ts b/src/transaction.ts index 06c3a6cc2..b27b63617 100644 --- a/src/transaction.ts +++ b/src/transaction.ts @@ -1088,7 +1088,7 @@ export class Transaction implements encoding.Encodable { ]) ); - return encoding.rawEncode(stxnSchema.prepareMsgpack(sTxn)); + return encoding.msgpackRawEncode(stxnSchema.prepareMsgpack(sTxn)); } rawTxID(): Uint8Array { diff --git a/src/types/block.ts b/src/types/block.ts index 08256be3b..d90e207fd 100644 --- a/src/types/block.ts +++ b/src/types/block.ts @@ -2,7 +2,8 @@ import { Encodable, Schema } from '../encoding/encoding.js'; import { NamedMapSchema, Uint64MapSchema, - StringMapSchema, + SpecialCaseBinaryStringMapSchema, + SpecialCaseBinaryStringSchema, ArraySchema, StringSchema, BooleanSchema, @@ -768,7 +769,7 @@ export class ValueDelta implements Encodable { }, { key: 'bs', // bytes - valueSchema: new StringSchema(), + valueSchema: new SpecialCaseBinaryStringSchema(), }, { key: 'ui', // uint @@ -778,10 +779,14 @@ export class ValueDelta implements Encodable { ); public action: number; - public bytes: string; + public bytes: Uint8Array; public uint: bigint; - public constructor(params: { action: number; bytes: string; uint: bigint }) { + public constructor(params: { + action: number; + bytes: Uint8Array; + uint: bigint; + }) { this.action = params.action; this.bytes = params.bytes; this.uint = params.uint; @@ -825,14 +830,14 @@ export class EvalDelta implements Encodable { { key: 'gd', // globalDelta valueSchema: new OptionalSchema( - new StringMapSchema(ValueDelta.encodingSchema) + new SpecialCaseBinaryStringMapSchema(ValueDelta.encodingSchema) ), }, { key: 'ld', // localDeltas valueSchema: new OptionalSchema( new Uint64MapSchema( - new StringMapSchema(ValueDelta.encodingSchema) + new SpecialCaseBinaryStringMapSchema(ValueDelta.encodingSchema) ) ), }, @@ -845,7 +850,7 @@ export class EvalDelta implements Encodable { { key: 'lg', // logs valueSchema: new OptionalSchema( - new ArraySchema(new StringSchema()) + new ArraySchema(new SpecialCaseBinaryStringSchema()) ), }, { @@ -861,13 +866,13 @@ export class EvalDelta implements Encodable { return this.encodingSchemaValue; } - public globalDelta: Map; + public globalDelta: Map; /** * When decoding EvalDeltas, the integer key represents an offset into * [txn.Sender, txn.Accounts[0], txn.Accounts[1], ...] */ - public localDeltas: Map>; + public localDeltas: Map>; /** * If a program modifies the local of an account that is not the Sender, or @@ -876,22 +881,22 @@ export class EvalDelta implements Encodable { */ public sharedAccts: Address[]; - public logs: string[]; + public logs: Uint8Array[]; // eslint-disable-next-line no-use-before-define public innerTxns: SignedTxnWithAD[]; public constructor(params: { - globalDelta?: Map; - localDeltas?: Map>; + globalDelta?: Map; + localDeltas?: Map>; sharedAccts?: Address[]; - logs?: string[]; + logs?: Uint8Array[]; // eslint-disable-next-line no-use-before-define innerTxns?: SignedTxnWithAD[]; }) { - this.globalDelta = params.globalDelta ?? new Map(); + this.globalDelta = params.globalDelta ?? new Map(); this.localDeltas = - params.localDeltas ?? new Map>(); + params.localDeltas ?? new Map>(); this.sharedAccts = params.sharedAccts ?? []; this.logs = params.logs ?? []; this.innerTxns = params.innerTxns ?? []; @@ -930,14 +935,14 @@ export class EvalDelta implements Encodable { } return new EvalDelta({ globalDelta: data.get('gd') - ? convertMap(data.get('gd') as Map, (key, value) => [ - key, - ValueDelta.fromEncodingData(value), - ]) + ? convertMap( + data.get('gd') as Map, + (key, value) => [key, ValueDelta.fromEncodingData(value)] + ) : undefined, localDeltas: data.get('ld') ? convertMap( - data.get('ld') as Map>, + data.get('ld') as Map>, (key, value) => [ Number(key), convertMap(value, (k, v) => [k, ValueDelta.fromEncodingData(v)]), diff --git a/tests/2.Encoding.ts b/tests/2.Encoding.ts index 05d7c7afc..c3bdbfc91 100644 --- a/tests/2.Encoding.ts +++ b/tests/2.Encoding.ts @@ -1,12 +1,9 @@ /* eslint-env mocha */ import assert from 'assert'; -import algosdk from '../src/index.js'; +import { RawBinaryString } from 'algorand-msgpack'; +import algosdk, { bytesToString, coerceToBytes } from '../src/index.js'; import * as utils from '../src/utils/utils.js'; -import { - Schema, - MsgpackEncodingData, - JSONEncodingData, -} from '../src/encoding/encoding.js'; +import { Schema, MsgpackRawStringProvider } from '../src/encoding/encoding.js'; import { BooleanSchema, StringSchema, @@ -14,11 +11,13 @@ import { AddressSchema, ByteArraySchema, FixedLengthByteArraySchema, + SpecialCaseBinaryStringSchema, ArraySchema, NamedMapSchema, NamedMapEntry, Uint64MapSchema, StringMapSchema, + SpecialCaseBinaryStringMapSchema, UntypedSchema, OptionalSchema, allOmitEmpty, @@ -897,7 +896,9 @@ describe('encoding', () => { ['\uFFFD\uFFFD', '/v8='], // Non UTF-8 bytes should still decode to same (invalid) output ]; for (const [testCase, expectedEncoding] of testCases) { - const actualB64Decoding = algosdk.base64ToString(expectedEncoding); + const actualB64Decoding = algosdk.bytesToString( + algosdk.base64ToBytes(expectedEncoding) + ); assert.deepStrictEqual( actualB64Decoding, testCase, @@ -944,10 +945,10 @@ describe('encoding', () => { name: string; schema: Schema; values: unknown[]; - preparedMsgpackValues: MsgpackEncodingData[]; + preparedMsgpackValues: algosdk.MsgpackEncodingData[]; // The expected output from calling `fromPreparedMsgpack`. If not provided, `values` will be used. expectedValuesFromPreparedMsgpack?: unknown[]; - preparedJsonValues: JSONEncodingData[]; + preparedJsonValues: algosdk.JSONEncodingData[]; // The expected output from calling `fromPreparedJSON`. If not provided, `values` will be used. expectedValuesFromPreparedJson?: unknown[]; } @@ -967,6 +968,21 @@ describe('encoding', () => { preparedMsgpackValues: ['', 'abc'], preparedJsonValues: ['', 'abc'], }, + { + name: 'SpecialCaseBinaryStringSchema', + schema: new SpecialCaseBinaryStringSchema(), + values: [Uint8Array.from([]), Uint8Array.from([97, 98, 99])], + preparedMsgpackValues: [ + // Cast is needed because RawBinaryString is not part of the standard MsgpackEncodingData + new RawBinaryString( + Uint8Array.from([]) + ) as unknown as algosdk.MsgpackEncodingData, + new RawBinaryString( + Uint8Array.from([97, 98, 99]) + ) as unknown as algosdk.MsgpackEncodingData, + ], + preparedJsonValues: ['', 'abc'], + }, { name: 'Uint64Schema', schema: new Uint64Schema(), @@ -1086,6 +1102,40 @@ describe('encoding', () => { }, ], }, + { + name: 'Uint64MapSchema of SpecialCaseBinaryStringSchema', + schema: new Uint64MapSchema(new SpecialCaseBinaryStringSchema()), + values: [ + new Map(), + new Map([ + [0n, Uint8Array.from([])], + [1n, Uint8Array.from([97])], + [2n, Uint8Array.from([98])], + [BigInt('18446744073709551615'), Uint8Array.from([99])], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [0n, new RawBinaryString(Uint8Array.from([]))], + [1n, new RawBinaryString(Uint8Array.from([97]))], + [2n, new RawBinaryString(Uint8Array.from([98]))], + [ + BigInt('18446744073709551615'), + new RawBinaryString(Uint8Array.from([99])), + ], + ]), + ], + preparedJsonValues: [ + {}, + { + 0: '', + 1: 'a', + 2: 'b', + '18446744073709551615': 'c', + }, + ], + }, { name: 'StringMapSchema of BooleanSchema', schema: new StringMapSchema(new BooleanSchema()), @@ -1117,6 +1167,82 @@ describe('encoding', () => { }, ], }, + { + name: 'SpecialCaseBinaryStringMapSchema of BooleanSchema', + schema: new SpecialCaseBinaryStringMapSchema(new BooleanSchema()), + values: [ + new Map(), + new Map([ + [Uint8Array.from([97]), true], + [Uint8Array.from([98]), false], + [Uint8Array.from([99]), true], + [Uint8Array.from([]), true], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [new RawBinaryString(Uint8Array.from([97])), true], + [new RawBinaryString(Uint8Array.from([98])), false], + [new RawBinaryString(Uint8Array.from([99])), true], + [new RawBinaryString(Uint8Array.from([])), true], + ]), + ], + preparedJsonValues: [ + {}, + { + a: true, + b: false, + c: true, + '': true, + }, + ], + }, + { + name: 'SpecialCaseBinaryStringMapSchema of SpecialCaseBinaryStringSchema', + schema: new SpecialCaseBinaryStringMapSchema( + new SpecialCaseBinaryStringSchema() + ), + values: [ + new Map(), + new Map([ + [Uint8Array.from([97]), Uint8Array.from([120])], + [Uint8Array.from([98]), Uint8Array.from([121])], + [Uint8Array.from([99]), Uint8Array.from([122])], + [Uint8Array.from([]), Uint8Array.from([])], + ]), + ], + preparedMsgpackValues: [ + new Map(), + new Map([ + [ + new RawBinaryString(Uint8Array.from([97])), + new RawBinaryString(Uint8Array.from([120])), + ], + [ + new RawBinaryString(Uint8Array.from([98])), + new RawBinaryString(Uint8Array.from([121])), + ], + [ + new RawBinaryString(Uint8Array.from([99])), + new RawBinaryString(Uint8Array.from([122])), + ], + [ + new RawBinaryString(Uint8Array.from([])), + new RawBinaryString(Uint8Array.from([])), + ], + ]), + ], + preparedJsonValues: [ + {}, + { + a: 'x', + b: 'y', + c: 'z', + '': '', + }, + ], + }, ]; const primitiveTestcases = testcases.slice(); @@ -1210,8 +1336,15 @@ describe('encoding', () => { const actualMsgpack = testcase.schema.prepareMsgpack(value); assert.deepStrictEqual(actualMsgpack, preparedMsgpackValue); - const roundtripMsgpackValue = - testcase.schema.fromPreparedMsgpack(actualMsgpack); + const msgpackBytes = algosdk.msgpackRawEncode(actualMsgpack); + const rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + + const roundtripMsgpackValue = testcase.schema.fromPreparedMsgpack( + actualMsgpack, + rawStringProvider + ); const roundtripMsgpackExpectedValue = testcase.expectedValuesFromPreparedMsgpack ? testcase.expectedValuesFromPreparedMsgpack[i] @@ -1221,7 +1354,7 @@ describe('encoding', () => { roundtripMsgpackExpectedValue ); - const actualJson = testcase.schema.prepareJSON(value); + const actualJson = testcase.schema.prepareJSON(value, {}); assert.deepStrictEqual(actualJson, preparedJsonValue); const roundtripJsonValue = @@ -1261,6 +1394,11 @@ describe('encoding', () => { emptyValue: '', nonemptyValue: 'abc', }, + { + schema: new SpecialCaseBinaryStringSchema(), + emptyValue: Uint8Array.from([]), + nonemptyValue: Uint8Array.from([97, 98, 99]), + }, { schema: new AddressSchema(), emptyValue: algosdk.Address.zeroAddress(), @@ -1344,6 +1482,16 @@ describe('encoding', () => { ['', true], ]), }, + { + schema: new SpecialCaseBinaryStringMapSchema(new BooleanSchema()), + emptyValue: new Map(), + nonemptyValue: new Map([ + [Uint8Array.from([97]), true], + [Uint8Array.from([98]), false], + [Uint8Array.from([99]), true], + [Uint8Array.from([]), true], + ]), + }, ]; for (const testValue of testValues.slice()) { @@ -1396,15 +1544,21 @@ describe('encoding', () => { let prepareMsgpackResult = schema.prepareMsgpack(allEmptyValues); // All empty values should be omitted assert.deepStrictEqual(prepareMsgpackResult, new Map()); - let fromPreparedMsgpackResult = - schema.fromPreparedMsgpack(prepareMsgpackResult); + let msgpackBytes = algosdk.msgpackRawEncode(prepareMsgpackResult); + let rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + let fromPreparedMsgpackResult = schema.fromPreparedMsgpack( + prepareMsgpackResult, + rawStringProvider + ); // Omitted values should be restored with their default/empty values assert.deepStrictEqual( fromPreparedMsgpackResult, allEmptyValuesRestored ); - let prepareJsonResult = schema.prepareJSON(allEmptyValues); + let prepareJsonResult = schema.prepareJSON(allEmptyValues, {}); // All empty values should be omitted assert.deepStrictEqual(prepareJsonResult, {}); let fromPreparedJsonResult = schema.fromPreparedJSON(prepareJsonResult); @@ -1422,12 +1576,18 @@ describe('encoding', () => { assert.ok(prepareMsgpackResult instanceof Map); // All values are present assert.strictEqual(prepareMsgpackResult.size, testValues.length); - fromPreparedMsgpackResult = - schema.fromPreparedMsgpack(prepareMsgpackResult); + msgpackBytes = algosdk.msgpackRawEncode(prepareMsgpackResult); + rawStringProvider = new MsgpackRawStringProvider({ + baseObjectBytes: msgpackBytes, + }); + fromPreparedMsgpackResult = schema.fromPreparedMsgpack( + prepareMsgpackResult, + rawStringProvider + ); // Values are restored properly assert.deepStrictEqual(fromPreparedMsgpackResult, allNonemptyValues); - prepareJsonResult = schema.prepareJSON(allNonemptyValues); + prepareJsonResult = schema.prepareJSON(allNonemptyValues, {}); // All values are present assert.strictEqual( Object.keys(prepareJsonResult as object).length, @@ -1468,7 +1628,8 @@ describe('encoding', () => { ['a', ''], ['b', ''], ['c', ''], - ]) + ]), + {} ), {} ); @@ -1492,7 +1653,8 @@ describe('encoding', () => { ['a', '1'], ['b', '2'], ['c', '3'], - ]) + ]), + {} ), { a: '1', @@ -1534,7 +1696,8 @@ describe('encoding', () => { ['c', ''], ]), ], - ]) + ]), + {} ), {} ); @@ -1573,7 +1736,8 @@ describe('encoding', () => { ['c', '3'], ]), ], - ]) + ]), + {} ), { map: { a: '1', b: '2' }, @@ -1766,6 +1930,241 @@ describe('encoding', () => { ); }); }); + describe('lossyBinaryStringConversion', () => { + const invalidUtf8String = Uint8Array.from([ + 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, + 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, + 135, 121, + ]); + + const invalidUtf8StringEncoded = bytesToString(invalidUtf8String); + const invalidUtf8StringDecoded = coerceToBytes(invalidUtf8StringEncoded); + + it('should have lossy string conversion for invalid UTF-8 string', () => { + assert.notStrictEqual(invalidUtf8String, invalidUtf8StringDecoded); + }); + + it('should lossily prepare invalid UTF-8 strings by default and when enabled', () => { + const options = { + lossyBinaryStringConversion: true, + }; + + const schema = new SpecialCaseBinaryStringSchema(); + const prepared = schema.prepareJSON(invalidUtf8String, options); + assert.strictEqual(prepared, invalidUtf8StringEncoded); + assert.deepStrictEqual( + schema.fromPreparedJSON(prepared), + invalidUtf8StringDecoded + ); + + const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); + const preparedMap = mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + options + ); + const expectedPreparedMap: Record = {}; + expectedPreparedMap[invalidUtf8StringEncoded] = + invalidUtf8StringEncoded; + assert.deepStrictEqual(preparedMap, expectedPreparedMap); + assert.deepStrictEqual( + mapSchema.fromPreparedJSON(preparedMap), + new Map([[invalidUtf8StringDecoded, invalidUtf8StringDecoded]]) + ); + }); + it('should error when preparing invalid UTF-8 strings when disabled and by default', () => { + for (const options of [{}, { lossyBinaryStringConversion: false }]) { + const schema = new SpecialCaseBinaryStringSchema(); + assert.throws( + () => schema.prepareJSON(invalidUtf8String, options), + /Invalid UTF-8 byte array encountered/ + ); + + const mapSchema = new SpecialCaseBinaryStringMapSchema(schema); + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[Uint8Array.from([97]), invalidUtf8String]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, Uint8Array.from([97])]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + + assert.throws( + () => + mapSchema.prepareJSON( + new Map([[invalidUtf8String, invalidUtf8String]]), + options + ), + /Invalid UTF-8 byte array encountered/ + ); + } + }); + }); + describe('MsgpackRawStringProvider', () => { + it('correctly records paths and provides raw strings', () => { + const baseObject = new Map([ + ['a', new Map([['a1', 'abc']])], + ['b', [new Map(), new Map([[BigInt(17), 'def']])]], + ]); + const baseProvider = new MsgpackRawStringProvider({ + baseObjectBytes: algosdk.msgpackRawEncode(baseObject), + }); + assert.strictEqual(baseProvider.getPathString(), 'root'); + assert.throws( + () => baseProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + baseProvider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map([ + [ + Uint8Array.from([97]), + new Map([ + [ + new RawBinaryString(Uint8Array.from([97, 49])), + new RawBinaryString(Uint8Array.from([97, 98, 99])), + ], + ]), + ], + [ + Uint8Array.from([98]), + [ + new Map(), + new Map([ + [ + BigInt(17), + new RawBinaryString(Uint8Array.from([100, 101, 102])), + ], + ]), + ], + ], + ]) + ); + + // Test with both string and raw string form + for (const firstKey of [ + 'a', + new RawBinaryString(Uint8Array.from([97])), + ]) { + const firstValueProvider = baseProvider.withMapValue(firstKey); + assert.strictEqual( + firstValueProvider.getPathString(), + `root -> map key "${firstKey}" (${typeof firstKey})` + ); + assert.throws( + () => firstValueProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + firstValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map([ + [ + Uint8Array.from([97, 49]), + new RawBinaryString(Uint8Array.from([97, 98, 99])), + ], + ]) + ); + + // Test with both string and raw string form + for (const firstFirstKey of [ + 'a1', + new RawBinaryString(Uint8Array.from([97, 49])), + ]) { + const firstFirstValueProvider = + firstValueProvider.withMapValue(firstFirstKey); + assert.strictEqual( + firstFirstValueProvider.getPathString(), + `root -> map key "${firstKey}" (${typeof firstKey}) -> map key "${firstFirstKey}" (${typeof firstFirstKey})` + ); + assert.deepStrictEqual( + firstFirstValueProvider.getRawStringAtCurrentLocation(), + Uint8Array.from([97, 98, 99]) + ); + assert.throws( + () => + firstFirstValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + } + } + + // Test with both string and raw string form + for (const secondKey of [ + 'b', + new RawBinaryString(Uint8Array.from([98])), + ]) { + const secondValueProvider = baseProvider.withMapValue(secondKey); + assert.strictEqual( + secondValueProvider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey})` + ); + assert.throws( + () => secondValueProvider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.throws( + () => + secondValueProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + + const secondIndex0Provider = secondValueProvider.withArrayElement(0); + assert.strictEqual( + secondIndex0Provider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 0 (number)` + ); + assert.throws( + () => secondIndex0Provider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.deepStrictEqual( + secondIndex0Provider.getRawStringKeysAndValuesAtCurrentLocation(), + new Map() + ); + + const secondIndex1Provider = secondValueProvider.withArrayElement(1); + assert.strictEqual( + secondIndex1Provider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 1 (number)` + ); + assert.throws( + () => secondIndex1Provider.getRawStringAtCurrentLocation(), + /Invalid type\. Expected RawBinaryString, got/ + ); + assert.throws( + () => + secondIndex1Provider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type for map key\. Expected RawBinaryString, got 17 \(bigint\)/ + ); + + const secondIndex1FirstProvider = secondIndex1Provider.withMapValue( + BigInt(17) + ); + assert.strictEqual( + secondIndex1FirstProvider.getPathString(), + `root -> map key "${secondKey}" (${typeof secondKey}) -> array index 1 (number) -> map key "17" (bigint)` + ); + assert.deepStrictEqual( + secondIndex1FirstProvider.getRawStringAtCurrentLocation(), + Uint8Array.from([100, 101, 102]) + ); + assert.throws( + () => + secondIndex1FirstProvider.getRawStringKeysAndValuesAtCurrentLocation(), + /Invalid type\. Expected Map, got/ + ); + } + }); + }); }); describe('BlockResponse', () => { it('should decode block response correctly', () => { @@ -1923,8 +2322,8 @@ describe('encoding', () => { ], ]), ], - ['rnd', 94], - ['step', 2], + ['rnd', BigInt(94)], + ['step', BigInt(2)], [ 'vote', [ @@ -2014,47 +2413,47 @@ describe('encoding', () => { configAsset: BigInt(7777), applicationID: BigInt(8888), evalDelta: new algosdk.EvalDelta({ - globalDelta: new Map([ + globalDelta: new Map([ [ - 'globalKey1', + algosdk.coerceToBytes('globalKey1'), new algosdk.ValueDelta({ action: 1, uint: BigInt(0), - bytes: 'abc', + bytes: algosdk.coerceToBytes('abc'), }), ], [ - 'globalKey2', + algosdk.coerceToBytes('globalKey2'), new algosdk.ValueDelta({ action: 2, uint: BigInt(50), - bytes: '', + bytes: new Uint8Array(), }), ], ]), - localDeltas: new Map>([ + localDeltas: new Map>([ [ 0, - new Map([ + new Map([ [ - 'localKey1', + algosdk.coerceToBytes('localKey1'), new algosdk.ValueDelta({ action: 1, uint: BigInt(0), - bytes: 'def', + bytes: algosdk.coerceToBytes('def'), }), ], ]), ], [ 2, - new Map([ + new Map([ [ - 'localKey2', + algosdk.coerceToBytes('localKey2'), new algosdk.ValueDelta({ action: 2, uint: BigInt(51), - bytes: '', + bytes: new Uint8Array(), }), ], ]), @@ -2066,7 +2465,7 @@ describe('encoding', () => { ), algosdk.Address.zeroAddress(), ], - logs: ['log1', 'log2'], + logs: [algosdk.coerceToBytes('log1'), algosdk.coerceToBytes('log2')], innerTxns: [ new algosdk.SignedTxnWithAD({ signedTxn: new algosdk.SignedTransaction({ @@ -2092,7 +2491,10 @@ describe('encoding', () => { }), applyData: new algosdk.ApplyData({ evalDelta: new algosdk.EvalDelta({ - logs: ['log3', 'log4'], + logs: [ + algosdk.coerceToBytes('log3'), + algosdk.coerceToBytes('log4'), + ], }), }), }), @@ -2103,5 +2505,121 @@ describe('encoding', () => { const reencoded = algosdk.encodeMsgpack(applyData); assert.deepStrictEqual(reencoded, encodedApplyData); }); + it('should decode EvalDelta with invalid UTF-8 strings correctly', () => { + const encodedEvalDelta = algosdk.base64ToBytes( + 'haJnZIKkZzH+/4KiYXQBomJzpHYx/v+kZzL+/4KiYXQConVpMqNpdHiRgqJkdIGibGeSpmxvZzP+/6b+/2xvZzSjdHhuhqRhcGlkzR5ho2ZlZc0D6KJmdlyibHbNBESjc25kxCB/k/5DqCp+P1WW/80hkucvP2neluTb4OL7yjQQnAxn6aR0eXBlpGFwcGyibGSCAIGkbDH+/4KiYXQBomJzpHYy/v8CgaRsMv7/gqJhdAKidWkzomxnkqZsb2cx/v+m/v9sb2cyonNhksQgCbEzlTT2uNiZwobypXnCOg5IqgxtO92MuwR8vJwv3ePEIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + ); + const evalDelta = algosdk.decodeMsgpack( + encodedEvalDelta, + algosdk.EvalDelta + ); + const invalidUtf8 = algosdk.base64ToBytes('/v8='); + const expectedEvalDelta = new algosdk.EvalDelta({ + globalDelta: new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('g1'), invalidUtf8), + new algosdk.ValueDelta({ + action: 1, + uint: BigInt(0), + bytes: utils.concatArrays( + algosdk.coerceToBytes('v1'), + invalidUtf8 + ), + }), + ], + [ + utils.concatArrays(algosdk.coerceToBytes('g2'), invalidUtf8), + new algosdk.ValueDelta({ + action: 2, + uint: BigInt(50), + bytes: new Uint8Array(), + }), + ], + ]), + localDeltas: new Map>([ + [ + 0, + new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('l1'), invalidUtf8), + new algosdk.ValueDelta({ + action: 1, + uint: BigInt(0), + bytes: utils.concatArrays( + algosdk.coerceToBytes('v2'), + invalidUtf8 + ), + }), + ], + ]), + ], + [ + 2, + new Map([ + [ + utils.concatArrays(algosdk.coerceToBytes('l2'), invalidUtf8), + new algosdk.ValueDelta({ + action: 2, + uint: BigInt(51), + bytes: new Uint8Array(), + }), + ], + ]), + ], + ]), + sharedAccts: [ + algosdk.Address.fromString( + 'BGYTHFJU624NRGOCQ3ZKK6OCHIHERKQMNU553DF3AR6LZHBP3XR5JLNCUI' + ), + algosdk.Address.zeroAddress(), + ], + logs: [ + utils.concatArrays(algosdk.coerceToBytes('log1'), invalidUtf8), + utils.concatArrays(invalidUtf8, algosdk.coerceToBytes('log2')), + ], + innerTxns: [ + new algosdk.SignedTxnWithAD({ + signedTxn: new algosdk.SignedTransaction({ + txn: new algosdk.Transaction({ + sender: new algosdk.Address( + algosdk.base64ToBytes( + 'f5P+Q6gqfj9Vlv/NIZLnLz9p3pbk2+Di+8o0EJwMZ+k=' + ) + ), + type: algosdk.TransactionType.appl, + suggestedParams: { + flatFee: true, + fee: BigInt(1000), + firstValid: BigInt(92), + lastValid: BigInt(1092), + minFee: BigInt(1000), + }, + appCallParams: { + appIndex: BigInt(7777), + onComplete: algosdk.OnApplicationComplete.NoOpOC, + }, + }), + }), + applyData: new algosdk.ApplyData({ + evalDelta: new algosdk.EvalDelta({ + logs: [ + utils.concatArrays( + algosdk.coerceToBytes('log3'), + invalidUtf8 + ), + utils.concatArrays( + invalidUtf8, + algosdk.coerceToBytes('log4') + ), + ], + }), + }), + }), + ], + }); + assert.deepStrictEqual(evalDelta, expectedEvalDelta); + const reencoded = algosdk.encodeMsgpack(evalDelta); + assert.deepStrictEqual(reencoded, encodedEvalDelta); + }); }); }); diff --git a/tests/5.Transaction.ts b/tests/5.Transaction.ts index f329a50ac..a516a1466 100644 --- a/tests/5.Transaction.ts +++ b/tests/5.Transaction.ts @@ -1064,7 +1064,7 @@ describe('Sign', () => { ); assert.ok(encRep instanceof Map && !encRep.has('fadd')); - const encTxn = algosdk.encodeObj(encRep); + const encTxn = algosdk.msgpackRawEncode(encRep); const golden = algosdk.base64ToBytes( 'iqRhZnJ6w6RmYWlkzScPo2ZlZc0IeqJmdgGjZ2VurG1vY2stbmV0d29ya6JnaMQgSGO1GKSzyE7IEPItTxCByw9x8FmnrCDexi9/cOUJOiKibHbNA+mkbm90ZcQDewzIo3NuZMQgoImqaSLjuZj63/bNSAjd+eAh5JROOJ6j1cY4eGaJGX6kdHlwZaRhZnJ6' ); diff --git a/tests/7.AlgoSDK.ts b/tests/7.AlgoSDK.ts index 863d9ccee..aec20ad69 100644 --- a/tests/7.AlgoSDK.ts +++ b/tests/7.AlgoSDK.ts @@ -20,12 +20,20 @@ describe('Algosdk (AKA end to end)', () => { describe('#encoding', () => { it('should encode and decode', () => { const o = { a: [1, 2, 3, 4, 5], b: 3486, c: 'skfg' }; - assert.deepStrictEqual(o, algosdk.decodeObj(algosdk.encodeObj(o))); + assert.deepStrictEqual( + o, + algosdk.msgpackRawDecode(algosdk.msgpackRawEncode(o), { + intDecoding: algosdk.IntDecoding.MIXED, + }) + ); }); it('should encode and decode strings', () => { const o = 'Hi there'; - assert.deepStrictEqual(o, algosdk.decodeObj(algosdk.encodeObj(o as any))); + assert.deepStrictEqual( + o, + algosdk.msgpackRawDecode(algosdk.msgpackRawEncode(o)) + ); }); it('should not mutate unsigned transaction when going to or from encoded buffer', () => { @@ -1067,7 +1075,8 @@ describe('Algosdk (AKA end to end)', () => { it('should be properly serialized to JSON', () => { const forEncoding = algosdk.modelsv2.DryrunRequest.encodingSchema.prepareJSON( - req.toEncodingData() + req.toEncodingData(), + {} ); const actual = algosdk.stringifyJSON(forEncoding, undefined, 2); diff --git a/tsdoc.json b/tsdoc.json index a21a07602..46e8bac58 100644 --- a/tsdoc.json +++ b/tsdoc.json @@ -16,6 +16,8 @@ "@throws": true, "@returns": true, "@param": true, - "@category": true + "@category": true, + "@deprecated": true, + "@link": true } } diff --git a/v2_TO_v3_MIGRATION_GUIDE.md b/v2_TO_v3_MIGRATION_GUIDE.md index 49658121b..259a6c09f 100644 --- a/v2_TO_v3_MIGRATION_GUIDE.md +++ b/v2_TO_v3_MIGRATION_GUIDE.md @@ -301,6 +301,15 @@ In order to facilitate `bigint` as a first-class type in this SDK, additional JS If your v2 code uses `JSON.parse` or `JSON.stringify` on types which can now contain `bigint`s in v3, you may receive an error such as `TypeError: Do not know how to serialize a BigInt`. Consider using these new functions instead. Or, if the types are `Encodable`, use the new `encodeJSON` and `decodeJSON` functions described in the [Object Encoding and Decoding](#object-encoding-and-decoding) section. +### Msgpack Operations + +The functions `encodeObj` and `decodeObj`, used to encode and decode msgpack objects, have been deprecated in v3 in favor of new functions, `msgpackRawEncode` and `msgpackRawDecode`. These functions have clearer names and differ slightly from the old functions. Specifically: + +- `msgpackRawEncode` will encode an object to msgpack, but will not check for empty values and throw errors if any are found. This additional check has become unnecessary due to the new encoding and decoding system in v3. +- `msgpackRawDecode` will decode a msgpack object, but unlike `decodeObj` which always uses `IntDecoding.MIXED` to decode integers, `msgpackRawDecode` can use any provided `IntDecoding` option. If none are provided, it will default to `IntDecoding.BIGINT`. Generally speaking, `IntDecoding.BIGINT` is preferred because it can handle all possible integer values, and the type of an integer will not change depending on the value (like it can with `IntDecoding.MIXED`), meaning code which query integer values from the decoded object will be more predictable. + +Though in the vast majority of cases, you will not need to use these functions directly. Instead, the `encodeMsgpack` and `decodeMsgpack` functions are preferred, which are discussed in the [Object Encoding and Decoding](#object-encoding-and-decoding) section. + ### IntDecoding The `IntDecoding.DEFAULT` option has been renamed to `IntDecoding.UNSAFE` in v3. It behaves identically to the v2 `IntDecoding.DEFAULT` option, but the name has been changed to better reflect the fact that other options should be preferred. @@ -336,3 +345,11 @@ const encoded = algosdk.encodeMsgpack(txn); // Uint8Array of msgpack-encoded tra const decoded = algosdk.decodeMsgpack(encoded, algosdk.Transaction); // Decoded Transaction instance assert.deepStrictEqual(txn, decoded); ``` + +### Base64 Encoding + +The `base64ToString` function has been removed in v3. Instead, you may combine the `base64ToBytes` and `bytesToString` to achieve the same thing, like so: + +```typescript +algosdk.bytesToString(algosdk.base64ToBytes('SGVsbG8gV29ybGQ=')); +```